def pre_process(self, *args, **kwargs): wrf_config = self.get_config(**kwargs) wrf_home = wrf_config.get('wrf_home') nfs_metgrid_dir = os.path.join(wrf_config.get('nfs_dir'), 'metgrid') logging.info('Running em_real...') em_real_dir = utils.get_em_real_dir(wrf_home) logging.info('Cleaning up files') utils.delete_files_with_prefix(em_real_dir, 'met_em*') utils.delete_files_with_prefix(em_real_dir, 'rsl*') # Linking met_em.* # logging.info('Creating met_em.d* symlinks') logging.info('Copying met_em.d.zip file') utils.copy_files_with_prefix(nfs_metgrid_dir, 'metgrid.zip', em_real_dir) logging.info('Unzipping met_em.d.zip') ZipFile(os.path.join(em_real_dir, 'metgrid.zip'), 'r', compression=zipfile.ZIP_DEFLATED).extractall(path=em_real_dir) logging.info('Cleaning up met_em.d.zip') os.remove(os.path.join(em_real_dir, 'metgrid.zip'))
def create_daily_gif(start, output_dir, output_filename, output_prefix): tmp_dir = tempfile.mkdtemp(prefix='tmp_jaxa_daily') utils.copy_files_with_prefix( output_dir, output_prefix + '_' + start.strftime('%Y-%m-%d') + '*.png', tmp_dir) logging.info('Writing gif ' + output_filename) gif_list = [os.path.join(tmp_dir, i) for i in sorted(os.listdir(tmp_dir))] if len(gif_list) > 0: ext_utils.create_gif(gif_list, os.path.join(output_dir, output_filename)) else: logging.info('No images found to create the gif') logging.info('Cleaning up ' + tmp_dir) shutil.rmtree(tmp_dir)
def create_rf_plots_wrf(nc_f, plots_output_dir, plots_output_base_dir, lon_min=None, lat_min=None, lon_max=None, lat_max=None, filter_threshold=0.05, run_prefix='WRF'): if not all([lon_min, lat_min, lon_max, lat_max]): lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_EXTENT variables = ext_utils.extract_variables(nc_f, 'RAINC, RAINNC', lat_min, lat_max, lon_min, lon_max) lats = variables['XLAT'] lons = variables['XLONG'] # cell size is calc based on the mean between the lat and lon points cz = np.round(np.mean(np.append(lons[1:len(lons)] - lons[0: len(lons) - 1], lats[1:len(lats)] - lats[0: len(lats) - 1])), 3) clevs = [0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 70, 100, 150, 200, 250, 300, 400, 500, 600, 750] cmap = cm.s3pcpn basemap = Basemap(projection='merc', llcrnrlon=lon_min, llcrnrlat=lat_min, urcrnrlon=lon_max, urcrnrlat=lat_max, resolution='h') data = variables['RAINC'] + variables['RAINNC'] logging.info('Filtering with the threshold %f' % filter_threshold) data[data < filter_threshold] = 0.0 variables['PRECIP'] = data prefix = 'wrf_plots' with TemporaryDirectory(prefix=prefix) as temp_dir: t0 = dt.datetime.strptime(variables['Times'][0], '%Y-%m-%d_%H:%M:%S') t1 = dt.datetime.strptime(variables['Times'][1], '%Y-%m-%d_%H:%M:%S') step = (t1 - t0).total_seconds() / 3600.0 inst_precip = ext_utils.get_two_element_average(variables['PRECIP']) cum_precip = ext_utils.get_two_element_average(variables['PRECIP'], return_diff=False) for i in range(1, len(variables['Times'])): time = variables['Times'][i] ts = dt.datetime.strptime(time, '%Y-%m-%d_%H:%M:%S') lk_ts = utils.datetime_utc_to_lk(ts, shift_mins=30) logging.info('processing %s', time) # instantaneous precipitation (hourly) inst_file = os.path.join(temp_dir, 'wrf_inst_' + lk_ts.strftime('%Y-%m-%d_%H:%M:%S')) ext_utils.create_asc_file(np.flip(inst_precip[i - 1], 0), lats, lons, inst_file + '.asc', cell_size=cz) title = { 'label': 'Hourly rf for %s LK' % lk_ts.strftime('%Y-%m-%d_%H:%M:%S'), 'fontsize': 30 } ext_utils.create_contour_plot(inst_precip[i - 1], inst_file + '.png', lat_min, lon_min, lat_max, lon_max, title, clevs=clevs, cmap=cmap, basemap=basemap) if (i * step) % 24 == 0: t = 'Daily rf from %s LK to %s LK' % ( (lk_ts - dt.timedelta(hours=24)).strftime('%Y-%m-%d_%H:%M:%S'), lk_ts.strftime('%Y-%m-%d_%H:%M:%S')) d = int(i * step / 24) - 1 logging.info('Creating images for D%d' % d) cum_file = os.path.join(temp_dir, 'wrf_cum_%dd' % d) if i * step / 24 > 1: cum_precip_24h = cum_precip[i - 1] - cum_precip[i - 1 - int(24 / step)] else: cum_precip_24h = cum_precip[i - 1] ext_utils.create_asc_file(np.flip(cum_precip_24h, 0), lats, lons, cum_file + '.asc', cell_size=cz) ext_utils.create_contour_plot(cum_precip_24h, cum_file + '.png', lat_min, lon_min, lat_max, lon_max, t, clevs=clevs, cmap=cmap, basemap=basemap) gif_file = os.path.join(temp_dir, 'wrf_inst_%dd' % d) images = [os.path.join(temp_dir, 'wrf_inst_' + j.strftime('%Y-%m-%d_%H:%M:%S') + '.png') for j in np.arange(lk_ts - dt.timedelta(hours=24 - step), lk_ts + dt.timedelta(hours=step), dt.timedelta(hours=step)).astype(dt.datetime)] ext_utils.create_gif(images, gif_file + '.gif') logging.info('Creating the zips') utils.create_zip_with_prefix(temp_dir, '*.png', os.path.join(temp_dir, 'pngs.zip')) utils.create_zip_with_prefix(temp_dir, '*.asc', os.path.join(temp_dir, 'ascs.zip')) logging.info('Cleaning up instantaneous pngs and ascs - wrf_inst_*') utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.png') utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.asc') logging.info('Copying pngs to ' + plots_output_dir) utils.move_files_with_prefix(temp_dir, '*.png', plots_output_dir) logging.info('Copying ascs to ' + plots_output_dir) utils.move_files_with_prefix(temp_dir, '*.asc', plots_output_dir) logging.info('Copying gifs to ' + plots_output_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', plots_output_dir) logging.info('Copying zips to ' + plots_output_dir) utils.copy_files_with_prefix(temp_dir, '*.zip', plots_output_dir) plots_latest_dir = os.path.join(plots_output_base_dir, 'latest', run_prefix, os.path.basename(plots_output_dir)) # <nfs>/latest/wrf0 .. 3 utils.create_dir_if_not_exists(plots_latest_dir) # todo: this needs to be adjusted to handle the multiple runs logging.info('Copying gifs to ' + plots_latest_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', plots_latest_dir)
def run_em_real(wrf_config): logging.info('Running em_real...') wrf_home = wrf_config.get('wrf_home') em_real_dir = utils.get_em_real_dir(wrf_home) procs = wrf_config.get('procs') run_id = wrf_config.get('run_id') output_dir = utils.create_dir_if_not_exists( os.path.join(wrf_config.get('nfs_dir'), 'results', run_id, 'wrf')) archive_dir = utils.create_dir_if_not_exists( os.path.join(wrf_config.get('archive_dir'), 'results', run_id, 'wrf')) logging.info('Backup the output dir') utils.backup_dir(output_dir) logs_dir = utils.create_dir_if_not_exists(os.path.join(output_dir, 'logs')) logging.info('Copying metgrid.zip') metgrid_dir = os.path.join(wrf_config.get('nfs_dir'), 'metgrid') if wrf_config.is_set('wps_run_id'): logging.info('wps_run_id is set. Copying metgrid from ' + wrf_config.get('wps_run_id')) utils.copy_files_with_prefix( metgrid_dir, wrf_config.get('wps_run_id') + '_metgrid.zip', em_real_dir) metgrid_zip = os.path.join( em_real_dir, wrf_config.get('wps_run_id') + '_metgrid.zip') else: utils.copy_files_with_prefix(metgrid_dir, wrf_config.get('run_id') + '_metgrid.zip', em_real_dir) metgrid_zip = os.path.join(em_real_dir, wrf_config.get('run_id') + '_metgrid.zip') logging.info('Extracting metgrid.zip') ZipFile(metgrid_zip, 'r', compression=ZIP_DEFLATED).extractall(path=em_real_dir) # logs destination: nfs/logs/xxxx/rsl* try: try: logging.info('Starting real.exe') utils.run_subprocess( 'mpirun --allow-run-as-root -np %d ./real.exe' % procs, cwd=em_real_dir) finally: logging.info('Moving Real log files...') utils.create_zip_with_prefix(em_real_dir, 'rsl*', os.path.join(em_real_dir, 'real_rsl.zip'), clean_up=True) utils.move_files_with_prefix(em_real_dir, 'real_rsl.zip', logs_dir) try: logging.info('Starting wrf.exe') utils.run_subprocess( 'mpirun --allow-run-as-root -np %d ./wrf.exe' % procs, cwd=em_real_dir) finally: logging.info('Moving WRF log files...') utils.create_zip_with_prefix(em_real_dir, 'rsl*', os.path.join(em_real_dir, 'wrf_rsl.zip'), clean_up=True) utils.move_files_with_prefix(em_real_dir, 'wrf_rsl.zip', logs_dir) finally: logging.info('Moving namelist input file') utils.move_files_with_prefix(em_real_dir, 'namelist.input', output_dir) logging.info('WRF em_real: DONE! Moving data to the output dir') logging.info('Extracting rf from domain3') d03_nc = glob.glob(os.path.join(em_real_dir, 'wrfout_d03_*'))[0] ncks_query = 'ncks -v %s %s %s' % ('RAINC,RAINNC,XLAT,XLONG,Times', d03_nc, d03_nc + '_rf') utils.run_subprocess(ncks_query) logging.info('Extracting rf from domain1') d01_nc = glob.glob(os.path.join(em_real_dir, 'wrfout_d01_*'))[0] ncks_query = 'ncks -v %s %s %s' % ('RAINC,RAINNC,XLAT,XLONG,Times', d01_nc, d01_nc + '_rf') utils.run_subprocess(ncks_query) logging.info('Moving data to the output dir') utils.move_files_with_prefix(em_real_dir, 'wrfout_d03*_rf', output_dir) utils.move_files_with_prefix(em_real_dir, 'wrfout_d01*_rf', output_dir) logging.info('Moving data to the archive dir') utils.move_files_with_prefix(em_real_dir, 'wrfout_*', archive_dir) logging.info('Cleaning up files') utils.delete_files_with_prefix(em_real_dir, 'met_em*') utils.delete_files_with_prefix(em_real_dir, 'rsl*') os.remove(metgrid_zip)
def process(self, *args, **kwargs): config = self.get_config(**kwargs) logging.info('wrf conifg: ' + config.to_json_string()) start_date = config.get('start_date') d03_dir = config.get('wrf_output_dir') d03_sl = os.path.join(d03_dir, 'wrfout_d01_' + start_date + ':00_SL') # create a temp work dir & get a local copy of the d03.._SL temp_dir = utils.create_dir_if_not_exists( os.path.join(config.get('wrf_home'), 'temp_d01')) shutil.copy2(d03_sl, temp_dir) d03_sl = os.path.join(temp_dir, os.path.basename(d03_sl)) lat_min = -3.06107 lon_min = 71.2166 lat_max = 18.1895 lon_max = 90.3315 variables = ext_utils.extract_variables(d03_sl, 'RAINC, RAINNC', lat_min, lat_max, lon_min, lon_max) lats = variables['XLAT'] lons = variables['XLONG'] # cell size is calc based on the mean between the lat and lon points cz = np.round( np.mean( np.append(lons[1:len(lons)] - lons[0:len(lons) - 1], lats[1:len(lats)] - lats[0:len(lats) - 1])), 3) # clevs = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30]) # clevs_cum = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100]) # norm = colors.BoundaryNorm(boundaries=clevs, ncolors=256) # norm_cum = colors.BoundaryNorm(boundaries=clevs_cum, ncolors=256) # cmap = plt.get_cmap('jet') clevs = [ 0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 70, 100, 150, 200, 250, 300, 400, 500, 600, 750 ] clevs_cum = clevs norm = None cmap = cm.s3pcpn basemap = Basemap(projection='merc', llcrnrlon=lon_min, llcrnrlat=lat_min, urcrnrlon=lon_max, urcrnrlat=lat_max, resolution='h') filter_threshold = 0.05 data = variables['RAINC'] + variables['RAINNC'] logging.info('Filtering with the threshold %f' % filter_threshold) data[data < filter_threshold] = 0.0 variables['PRECIP'] = data for i in range(1, len(variables['Times'])): time = variables['Times'][i] ts = dt.datetime.strptime(time, '%Y-%m-%d_%H:%M:%S') lk_ts = utils.datetime_utc_to_lk(ts) logging.info('processing %s', time) # instantaneous precipitation (hourly) inst_precip = variables['PRECIP'][i] - variables['PRECIP'][i - 1] inst_file = os.path.join(temp_dir, 'wrf_inst_' + time) title = { 'label': '3Hourly rf for %s LK\n%s UTC' % (lk_ts.strftime('%Y-%m-%d_%H:%M:%S'), time), 'fontsize': 30 } ext_utils.create_contour_plot(inst_precip, inst_file + '.png', lat_min, lon_min, lat_max, lon_max, title, clevs=clevs, cmap=cmap, basemap=basemap, norm=norm) if i % 8 == 0: d = int(i / 8) - 1 logging.info('Creating gif for D%d' % d) gif_file = os.path.join(temp_dir, 'wrf_inst_D01_%dd' % d) images = [ os.path.join( temp_dir, 'wrf_inst_' + i.strftime('%Y-%m-%d_%H:%M:%S') + '.png') for i in np.arange(ts - dt.timedelta(hours=24 - 3), ts + dt.timedelta( hours=3), dt.timedelta( hours=3)).astype(dt.datetime) ] ext_utils.create_gif(images, gif_file + '.gif') # move all the data in the tmp dir to the nfs logging.info('Copying gifs to ' + d03_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', d03_dir) d03_latest_dir = os.path.join(config.get('nfs_dir'), 'latest', os.path.basename(config.get('wrf_home'))) # <nfs>/latest/wrf0 .. 3 utils.create_dir_if_not_exists(d03_latest_dir) # todo: this needs to be adjusted to handle the multiple runs logging.info('Copying gifs to ' + d03_latest_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', d03_latest_dir) logging.info('Cleaning up the dir ' + temp_dir) shutil.rmtree(temp_dir)
def process(self, *args, **kwargs): config = self.get_config(**kwargs) logging.info('wrf conifg: ' + config.to_json_string()) start_date = config.get('start_date') d03_dir = config.get('wrf_output_dir') d03_sl = os.path.join(d03_dir, 'wrfout_d03_' + start_date + ':00_SL') # create a temp work dir & get a local copy of the d03.._SL temp_dir = utils.create_dir_if_not_exists( os.path.join(config.get('wrf_home'), 'temp')) shutil.copy2(d03_sl, temp_dir) d03_sl = os.path.join(temp_dir, os.path.basename(d03_sl)) lat_min = 5.722969 lon_min = 79.52146 lat_max = 10.06425 lon_max = 82.18992 variables = ext_utils.extract_variables(d03_sl, 'RAINC, RAINNC', lat_min, lat_max, lon_min, lon_max) lats = variables['XLAT'] lons = variables['XLONG'] # cell size is calc based on the mean between the lat and lon points cz = np.round( np.mean( np.append(lons[1:len(lons)] - lons[0:len(lons) - 1], lats[1:len(lats)] - lats[0:len(lats) - 1])), 3) # clevs = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30]) # clevs_cum = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100]) # norm = colors.BoundaryNorm(boundaries=clevs, ncolors=256) # norm_cum = colors.BoundaryNorm(boundaries=clevs_cum, ncolors=256) # cmap = plt.get_cmap('jet') clevs = [ 0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 70, 100, 150, 200, 250, 300, 400, 500, 600, 750 ] clevs_cum = clevs norm = None norm_cum = None cmap = cm.s3pcpn basemap = Basemap(projection='merc', llcrnrlon=lon_min, llcrnrlat=lat_min, urcrnrlon=lon_max, urcrnrlat=lat_max, resolution='h') filter_threshold = 0.05 data = variables['RAINC'] + variables['RAINNC'] logging.info('Filtering with the threshold %f' % filter_threshold) data[data < filter_threshold] = 0.0 variables['PRECIP'] = data pngs = [] ascs = [] for i in range(1, len(variables['Times'])): time = variables['Times'][i] ts = dt.datetime.strptime(time, '%Y-%m-%d_%H:%M:%S') lk_ts = utils.datetime_utc_to_lk(ts) logging.info('processing %s', time) # instantaneous precipitation (hourly) inst_precip = variables['PRECIP'][i] - variables['PRECIP'][i - 1] inst_file = os.path.join(temp_dir, 'wrf_inst_' + time) title = { 'label': 'Hourly rf for %s LK\n%s UTC' % (lk_ts.strftime('%Y-%m-%d_%H:%M:%S'), time), 'fontsize': 30 } ext_utils.create_asc_file(np.flip(inst_precip, 0), lats, lons, inst_file + '.asc', cell_size=cz) ascs.append(inst_file + '.asc') ext_utils.create_contour_plot(inst_precip, inst_file + '.png', lat_min, lon_min, lat_max, lon_max, title, clevs=clevs, cmap=cmap, basemap=basemap, norm=norm) pngs.append(inst_file + '.png') if i % 24 == 0: t = 'Daily rf from %s LK to %s LK' % ( (lk_ts - dt.timedelta(hours=24)).strftime('%Y-%m-%d_%H:%M:%S'), lk_ts.strftime('%Y-%m-%d_%H:%M:%S')) d = int(i / 24) - 1 logging.info('Creating images for D%d' % d) cum_file = os.path.join(temp_dir, 'wrf_cum_%dd' % d) ext_utils.create_asc_file(np.flip(variables['PRECIP'][i], 0), lats, lons, cum_file + '.asc', cell_size=cz) ascs.append(cum_file + '.asc') ext_utils.create_contour_plot(variables['PRECIP'][i] - variables['PRECIP'][i - 24], cum_file + '.png', lat_min, lon_min, lat_max, lon_max, t, clevs=clevs, cmap=cmap, basemap=basemap, norm=norm_cum) pngs.append(inst_file + '.png') gif_file = os.path.join(temp_dir, 'wrf_inst_%dd' % d) images = [ os.path.join( temp_dir, 'wrf_inst_' + i.strftime('%Y-%m-%d_%H:%M:%S') + '.png') for i in np.arange(ts - dt.timedelta(hours=23), ts + dt.timedelta( hours=1), dt.timedelta( hours=1)).astype(dt.datetime) ] ext_utils.create_gif(images, gif_file + '.gif') logging.info('Creating the zips') utils.create_zip_with_prefix(temp_dir, '*.png', os.path.join(temp_dir, 'pngs.zip')) utils.create_zip_with_prefix(temp_dir, '*.asc', os.path.join(temp_dir, 'ascs.zip')) # utils.create_zipfile(pngs, os.path.join(temp_dir, 'pngs.zip')) # utils.create_zipfile(ascs, os.path.join(temp_dir, 'ascs.zip')) logging.info('Cleaning up instantaneous pngs and ascs - wrf_inst_*') utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.png') utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.asc') logging.info('Copying pngs to ' + d03_dir) utils.move_files_with_prefix(temp_dir, '*.png', d03_dir) logging.info('Copying ascs to ' + d03_dir) utils.move_files_with_prefix(temp_dir, '*.asc', d03_dir) logging.info('Copying gifs to ' + d03_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', d03_dir) logging.info('Copying zips to ' + d03_dir) utils.copy_files_with_prefix(temp_dir, '*.zip', d03_dir) d03_latest_dir = os.path.join(config.get('nfs_dir'), 'latest', os.path.basename(config.get('wrf_home'))) # <nfs>/latest/wrf0 .. 3 utils.create_dir_if_not_exists(d03_latest_dir) # todo: this needs to be adjusted to handle the multiple runs logging.info('Copying gifs to ' + d03_latest_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', d03_latest_dir) logging.info('Cleaning up temp dir') shutil.rmtree(temp_dir)