def test_all(self): logging.basicConfig( level=logging.DEBUG, format= '%(asctime)s %(threadName)s %(module)s %(levelname)s %(message)s') wrf_home = tempfile.mkdtemp(prefix='wrf_test_') nfs_dir = os.path.join(wrf_home, 'nfs') run_id = 'WRF_test0' output_dir = os.path.join(nfs_dir, 'results', run_id, 'wrf') utils.create_dir_if_not_exists(output_dir) shutil.copy( res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00'), output_dir) wrf_conf_dict = { "wrf_home": wrf_home, "nfs_dir": nfs_dir, "period": 0.25, "start_date": "2017-10-02_12:00" } db_conf_dict = { "host": "localhost", "user": "******", "password": "******", "db": "testdb" } extract_data_wrf.run(run_id, wrf_conf_dict, db_conf_dict, upsert=True)
def make_plots(v, out_dir, basemap, start=0, end=-1): start_rf = v['PRECIP'][0] wrfutils.create_dir_if_not_exists(out_dir) for j in range(start, end if end > 0 else len(v['PRECIP'])): out = out_dir + '/' + v['Times'][j] + 'cum.png' if j != 0: utils.create_contour_plot(v['PRECIP'][j] - start_rf, out, lat_min, lon_min, lat_max, lon_max, out, basemap=basemap, clevs=clevs, cmap=plt.get_cmap('jet'), overwrite=True, norm=norm) else: utils.create_contour_plot(v['PRECIP'][j], out, lat_min, lon_min, lat_max, lon_max, out, basemap=basemap, clevs=clevs, cmap=plt.get_cmap('jet'), overwrite=True, norm=norm)
def extract_jaxa_weather_stations(nc_f, weather_stations_file, output_dir): nc_fid = Dataset(nc_f, 'r') stations = pd.read_csv(weather_stations_file, header=0, sep=',') output_file_dir = os.path.join(output_dir, 'jaxa-stations-wrf-forecast') utils.create_dir_if_not_exists(output_file_dir) for idx, station in stations.iterrows(): logging.info('Extracting station ' + str(station)) rf, times = extract_point_rf_series(nc_f, station[2], station[1]) output_file_path = os.path.join(output_file_dir, station[3] + '-' + str(station[0]) + '-' + times[0].split('_')[0] + '.txt') output_file = open(output_file_path, 'w') output_file.write('jaxa-stations-wrf-forecast\n') output_file.write(', '.join(stations.columns.values) + '\n') output_file.write(', '.join(str(x) for x in station) + '\n') output_file.write('timestamp, rainfall\n') for i in range(len(times)): output_file.write('%s, %f\n' % (times[i], rf[i])) output_file.close() nc_fid.close()
def post_process(self, *args, **kwargs): # make a sym link in the nfs dir wrf_config = self.get_config(**kwargs) wps_dir = utils.get_wps_dir(wrf_config.get('wrf_home')) nfs_metgrid_dir = os.path.join(wrf_config.get('nfs_dir'), 'metgrid') utils.create_dir_if_not_exists(nfs_metgrid_dir) # utils.delete_files_with_prefix(nfs_metgrid_dir, 'met_em.d*') # utils.create_symlink_with_prefix(wps_dir, 'met_em.d*', nfs_metgrid_dir) utils.create_zip_with_prefix(wps_dir, 'met_em.d*', os.path.join(wps_dir, 'metgrid.zip')) utils.delete_files_with_prefix(nfs_metgrid_dir, 'met_em.d*') utils.move_files_with_prefix(wps_dir, 'metgrid.zip', nfs_metgrid_dir)
def test_extract_kelani_basin_rainfall_flo2d_obs_150m(self): adapter = ext_utils.get_curw_adapter(mysql_config_path='/home/curw/Desktop/2018-05/mysql.json') wrf_output_dir = tempfile.mkdtemp(prefix='flo2d_obs_') files = ['wrfout_d03_2018-05-23_18:00:00_rf'] run_prefix = 'wrf0' for f in files: out_dir = utils.create_dir_if_not_exists( os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000'), 'wrf')) shutil.copy2('/home/curw/Desktop/2018-05/2018-05-23_18:00/wrf0/%s' % f, out_dir) run_date = dt.datetime.strptime('2018-05-23_18:00', '%Y-%m-%d_%H:%M') now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*']) d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'wrf', 'wrfout_d03_*'))[0] obs_stations = {'Kottawa North Dharmapala School': [79.95818, 6.865576, 'A&T Labs', 'wrf_79.957123_6.859688'], 'IBATTARA2': [79.919, 6.908, 'CUrW IoT', 'wrf_79.902664_6.913757'], 'Malabe': [79.95738, 6.90396, 'A&T Labs', 'wrf_79.957123_6.913757'], # 'Mutwal': [79.8609, 6.95871, 'A&T Labs', 'wrf_79.875435_6.967812'], 'Glencourse': [80.20305, 6.97805, 'Irrigation Department', 'wrf_80.202187_6.967812'], # 'Waga': [80.11828, 6.90678, 'A&T Labs', 'wrf_80.120499_6.913757'], } start_ts = '2018-05-26_00:00' kelani_lower_basin_points = res_mgr.get_resource_path('extraction/local/klb_glecourse_points_150m.txt') kelani_lower_basin_shp = res_mgr.get_resource_path('extraction/shp/klb_glencourse/klb_glencourse.shp') duration_days = (8, 0) extract_kelani_basin_rainfall_flo2d_with_obs(d03_nc_f, adapter, obs_stations, os.path.join(wrf_output_dir, now, 'klb_flo2d'), start_ts, duration_days=duration_days, kelani_lower_basin_shp=kelani_lower_basin_shp, kelani_lower_basin_points=kelani_lower_basin_points, output_prefix='RAINCELL_150m')
def create_rainfall_for_mike21(d0_rf_file, prev_rf_files, output_dir): d0 = np.genfromtxt(d0_rf_file, dtype=str) t0 = dt.datetime.strptime(' '.join(d0[0][:-1]), '%Y-%m-%d %H:%M:%S') t1 = dt.datetime.strptime(' '.join(d0[1][:-1]), '%Y-%m-%d %H:%M:%S') res_min = int((t1 - t0).total_seconds() / 60) lines_per_day = int(24 * 60 / res_min) prev_days = len(prev_rf_files) output = None for i in range(prev_days): if prev_rf_files[prev_days - 1 - i] is not None: if output is not None: output = np.append(output, np.genfromtxt(prev_rf_files[prev_days - 1 - i], dtype=str, max_rows=lines_per_day), axis=0) else: output = np.genfromtxt(prev_rf_files[prev_days - 1 - i], dtype=str, max_rows=lines_per_day) else: output = None # if any of the previous files are missing, skip prepending past data to the forecast break if output is not None: output = np.append(output, d0, axis=0) else: output = d0 out_file = os.path.join(utils.create_dir_if_not_exists(output_dir), 'rf_mike21.txt') with open(out_file, 'w') as out_f: for line in output: out_f.write('%s %s\t%s\n' % (line[0], line[1], line[2]))
def test_create_rainfall_for_mike21(self): wrf_output_dir = tempfile.mkdtemp(prefix='mike21_') run_date = dt.datetime.strptime('2017-12-11_18:00', '%Y-%m-%d_%H:%M') basin_shp_file = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp') files = ['wrfout_d03_2017-12-09_18:00:00_rf', 'wrfout_d03_2017-12-10_18:00:00_rf', 'wrfout_d03_2017-12-11_18:00:00_rf'] run_prefix = 'wrf0' for f in files: d03_nc_f = res_mgr.get_resource_path('test/%s' % f) out_dir = utils.create_dir_if_not_exists( os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000'))) extract_mean_rainfall_from_shp_file(d03_nc_f, out_dir, 'klb_mean_rf', 'klb_mean', basin_shp_file, constants.KELANI_LOWER_BASIN_EXTENT) now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*']) prev_1 = '_'.join([run_prefix, (run_date - dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*']) prev_2 = '_'.join([run_prefix, (run_date - dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*']) d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'klb_mean_rf', 'klb_mean_rf.txt'))[0] d03_nc_f_prev_1 = glob.glob(os.path.join(wrf_output_dir, prev_1, 'klb_mean_rf', 'klb_mean_rf.txt'))[0] d03_nc_f_prev_2 = glob.glob(os.path.join(wrf_output_dir, prev_2, 'klb_mean_rf', 'klb_mean_rf.txt'))[0] create_rainfall_for_mike21(d03_nc_f, [d03_nc_f_prev_1, d03_nc_f_prev_2], os.path.join(wrf_output_dir, now, 'mike_21'))
def test_download_gfs_data(): wrf_home = '/tmp/wrf' gfs_dir = wrf_home + '/gfs' utils.create_dir_if_not_exists(wrf_home) utils.create_dir_if_not_exists(gfs_dir) conf = get_wrf_config(wrf_home, start_date='2017-08-27_00:00', gfs_dir=gfs_dir, period=0.25) gfs_date, start_inv = download_gfs_data(conf) logging.info('gfs date %s and start inventory %s' % (gfs_date, start_inv)) files = os.listdir(gfs_dir) assert len(files) == int( 24 * conf.get('period') / conf.get('gfs_step')) + 1
def extract_metro_col_rf_for_mike21(nc_f, output_dir, prev_rf_files=None, points_file=None): if not prev_rf_files: prev_rf_files = [] if not points_file: points_file = res_mgr.get_resource_path('extraction/local/metro_col_sub_catch_centroids.txt') points = np.genfromtxt(points_file, delimiter=',', names=True, dtype=None) point_prcp = ext_utils.extract_points_array_rf_series(nc_f, points) t0 = dt.datetime.strptime(point_prcp['Times'][0], '%Y-%m-%d %H:%M:%S') t1 = dt.datetime.strptime(point_prcp['Times'][1], '%Y-%m-%d %H:%M:%S') res_min = int((t1 - t0).total_seconds() / 60) lines_per_day = int(24 * 60 / res_min) prev_days = len(prev_rf_files) output = None for i in range(prev_days): if prev_rf_files[prev_days - 1 - i] is not None: if output is not None: output = np.append(output, ext_utils.extract_points_array_rf_series(prev_rf_files[prev_days - 1 - i], points)[ :lines_per_day], axis=0) else: output = ext_utils.extract_points_array_rf_series(prev_rf_files[prev_days - 1 - i], points)[ :lines_per_day] else: output = None # if any of the previous files are missing, skip prepending past data to the forecast break if output is not None: output = np.append(output, point_prcp, axis=0) else: output = point_prcp fmt = '%s' for _ in range(len(output[0]) - 1): fmt = fmt + ',%g' header = ','.join(output.dtype.names) utils.create_dir_if_not_exists(output_dir) np.savetxt(os.path.join(output_dir, 'met_col_rf_mike21.txt'), output, fmt=fmt, delimiter=',', header=header, comments='', encoding='utf-8')
def post_process(self, *args, **kwargs): config = self.get_config(**kwargs) wrf_home = config.get('wrf_home') em_real_dir = utils.get_em_real_dir(wrf_home) start_date = config.get('start_date') logging.info('Moving the WRF logs') utils.move_files_with_prefix( em_real_dir, 'rsl*', os.path.join(utils.get_logs_dir(wrf_home), 'rsl-wrf-%s' % start_date)) logging.info('Moving the WRF files to output directory') # move the d03 to nfs # ex: /mnt/disks/wrf-mod/nfs/output/wrf0/2017-08-13_00:00/0 .. n d03_dir = utils.get_incremented_dir_path( os.path.join(config.get('nfs_dir'), 'output', os.path.basename(wrf_home), start_date, '0')) self.add_config_item('wrf_output_dir', d03_dir) d03_file = os.path.join(em_real_dir, 'wrfout_d03_' + start_date + ':00') ext_utils.ncks_extract_variables( d03_file, ['RAINC', 'RAINNC', 'XLAT', 'XLONG', 'Times'], d03_file + '_SL') d01_file = os.path.join(em_real_dir, 'wrfout_d01_' + start_date + ':00') ext_utils.ncks_extract_variables( d01_file, ['RAINC', 'RAINNC', 'XLAT', 'XLONG', 'Times'], d01_file + '_SL') # move the wrfout_SL and the namelist files to the nfs utils.create_dir_if_not_exists(d03_dir) shutil.move(d03_file + '_SL', d03_dir) shutil.move(d01_file + '_SL', d03_dir) shutil.copy2(os.path.join(em_real_dir, 'namelist.input'), d03_dir) # move the rest to the OUTPUT dir of each run # todo: in the docker impl - FIND A BETTER WAY archive_dir = utils.get_incremented_dir_path( os.path.join(utils.get_output_dir(wrf_home), start_date)) utils.move_files_with_prefix(em_real_dir, 'wrfout_d*', archive_dir)
def extract_mean_rainfall_from_shp_file(nc_f, wrf_output, output_prefix, output_name, basin_shp_file, basin_extent, curw_db_adapter=None, curw_db_upsert=False, run_prefix='WRF', run_name='Cloud-1'): lon_min, lat_min, lon_max, lat_max = basin_extent nc_vars = ext_utils.extract_variables(nc_f, ['RAINC', 'RAINNC'], lat_min, lat_max, lon_min, lon_max) lats = nc_vars['XLAT'] lons = nc_vars['XLONG'] prcp = nc_vars['RAINC'] + nc_vars['RAINNC'] times = nc_vars['Times'] diff = ext_utils.get_two_element_average(prcp) polys = shapefile.Reader(basin_shp_file) output_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, output_prefix)) with TemporaryDirectory(prefix=output_prefix) as temp_dir: output_file_path = os.path.join(temp_dir, output_prefix + '.txt') kub_rf = {} with open(output_file_path, 'w') as output_file: kub_rf[output_name] = [] for t in range(0, len(times) - 1): cnt = 0 rf_sum = 0.0 for y in range(0, len(lats)): for x in range(0, len(lons)): if utils.is_inside_polygon(polys, lats[y], lons[x]): cnt = cnt + 1 rf_sum = rf_sum + diff[t, y, x] mean_rf = rf_sum / cnt t_str = ( utils.datetime_utc_to_lk(dt.datetime.strptime(times[t], '%Y-%m-%d_%H:%M:%S'), shift_mins=30)).strftime('%Y-%m-%d %H:%M:%S') output_file.write('%s\t%.4f\n' % (t_str, mean_rf)) kub_rf[output_name].append([t_str, mean_rf]) utils.move_files_with_prefix(temp_dir, '*.txt', output_dir) if curw_db_adapter is not None: station = [Station.CUrW, output_name, output_name, -999, -999, 0, 'Kelani upper basin mean rainfall'] if ext_utils.create_station_if_not_exists(curw_db_adapter, station): logging.info('%s station created' % output_name) logging.info('Pushing data to the db...') ext_utils.push_rainfall_to_db(curw_db_adapter, kub_rf, upsert=curw_db_upsert, name=run_name, source=run_prefix) else: logging.info('curw_db_adapter not available. Unable to push data!')
def test_create_rainfall_for_mike21_obs(self): adapter = ext_utils.get_curw_adapter() wrf_output_dir = tempfile.mkdtemp(prefix='mike21_obs_') out_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output_dir, 'klb_mean_rf')) shutil.copy2('/home/curw/Desktop/2018-05/klb_mean_rf/klb_mean_rf.txt', out_dir) d0_mean_rf = os.path.join(out_dir, 'klb_mean_rf.txt') obs_stations = {'Kottawa North Dharmapala School': [79.95818, 6.865576, 'A&T Labs'], 'IBATTARA2': [79.919, 6.908, 'CUrW IoT'], 'Malabe': [79.95738, 6.90396, 'A&T Labs'], 'Mutwal': [79.8609, 6.95871, 'A&T Labs']} start_ts = '2018-05-21_00:00' create_rainfall_for_mike21_obs(d0_mean_rf, adapter, obs_stations, out_dir, start_ts, )
def create_rainfall_for_mike21_obs(d0_rf_file, adapter, obs_stations, output_dir, start_ts, duration_days=None, kelani_lower_basin_shp=None): if kelani_lower_basin_shp is None: kelani_lower_basin_shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp') if duration_days is None: duration_days = (2, 3) obs_start = dt.datetime.strptime(start_ts, '%Y-%m-%d_%H:%M') - dt.timedelta(days=duration_days[0]) obs_end = dt.datetime.strptime(start_ts, '%Y-%m-%d_%H:%M') # forecast_end = dt.datetime.strptime(start_ts, '%Y-%m-%d_%H:%M') + dt.timedelta(days=duration_days[1]) obs = _get_observed_precip(obs_stations, obs_start, obs_end, duration_days, adapter) thess_poly = spatial_utils.get_voronoi_polygons(obs_stations, kelani_lower_basin_shp, add_total_area=False) observed = None for i, _id in enumerate(thess_poly['id']): if observed is not None: observed = observed + obs[_id].astype(float) * thess_poly['area'][i] else: observed = obs[_id].astype(float) * thess_poly['area'][i] observed = observed / sum(thess_poly['area']) d0 = np.genfromtxt(d0_rf_file, dtype=str) t0 = dt.datetime.strptime(' '.join(d0[0][:-1]), '%Y-%m-%d %H:%M:%S') t1 = dt.datetime.strptime(' '.join(d0[1][:-1]), '%Y-%m-%d %H:%M:%S') res_min = int((t1 - t0).total_seconds() / 60) # prev_output = np.append(prev_output, d0, axis=0) out_file = os.path.join(utils.create_dir_if_not_exists(output_dir), 'rf_mike21_obs.txt') with open(out_file, 'w') as out_f: for index in observed.index: out_f.write('%s:00\t%.4f\n' % (index, observed.precip[index])) forecast_start_idx = int( np.where((d0[:, 0] == obs_end.strftime('%Y-%m-%d')) & (d0[:, 1] == obs_end.strftime('%H:%M:%S')))[0]) # note: no need to convert to utc as rf_mike21.txt has times in LK for i in range(forecast_start_idx + 1, int(24 * 60 * duration_days[1] / res_min)): if i < len(d0): out_f.write('%s %s\t%s\n' % (d0[i][0], d0[i][1], d0[i][2])) else: out_f.write('%s\t0.0\n' % (obs_end + dt.timedelta(hours=i - forecast_start_idx - 1)).strftime( '%Y-%m-%d %H:%M:%S'))
def extract_weather_stations(nc_f, wrf_output, weather_stations=None, curw_db_adapter=None, curw_db_upsert=False, run_prefix='WRF', run_name='Cloud-1'): if weather_stations is None: weather_stations = res_mgr.get_resource_path('extraction/local/kelani_basin_stations.txt') nc_fid = Dataset(nc_f, 'r') times_len, times = ext_utils.extract_time_data(nc_f) prefix = 'stations_rf' stations_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix)) stations_rf = {} with TemporaryDirectory(prefix=prefix) as temp_dir: with open(weather_stations, 'r') as csvfile: stations = csv.reader(csvfile, delimiter=' ') for row in stations: logging.info(' '.join(row)) lon = row[1] lat = row[2] station_prcp = nc_fid.variables['RAINC'][:, lat, lon] + nc_fid.variables['RAINNC'][:, lat, lon] station_diff = ext_utils.get_two_element_average(station_prcp) stations_rf[row[0]] = [] station_file_path = os.path.join(temp_dir, row[0] + '_%s.txt' % prefix) with open(station_file_path, 'w') as station_file: for t in range(0, len(times) - 1): t_str = ( utils.datetime_utc_to_lk(dt.datetime.strptime(times[t], '%Y-%m-%d_%H:%M:%S'), shift_mins=30)).strftime('%Y-%m-%d %H:%M:%S') station_file.write('%s\t%.4f\n' % (t_str, station_diff[t])) stations_rf[row[0]].append([t_str, station_diff[t]]) utils.move_files_with_prefix(temp_dir, '*.txt', stations_dir) if curw_db_adapter is not None: logging.info('Pushing data to the db...') ext_utils.push_rainfall_to_db(curw_db_adapter, stations_rf, upsert=curw_db_upsert, name=run_name, source=run_prefix) else: logging.info('curw_db_adapter not available. Unable to push data!') nc_fid.close()
def extract_weather_stations2(nc_f, wrf_output, weather_stations=None, curw_db_adapter=None, curw_db_upsert=False, run_prefix='WRF', run_name='Cloud-1'): if weather_stations is None: weather_stations = res_mgr.get_resource_path('extraction/local/wrf_stations.txt') points = np.genfromtxt(weather_stations, delimiter=',', names=True, dtype=None) point_prcp = ext_utils.extract_points_array_rf_series(nc_f, points) t0 = dt.datetime.strptime(point_prcp['Times'][0], '%Y-%m-%d %H:%M:%S') t1 = dt.datetime.strptime(point_prcp['Times'][1], '%Y-%m-%d %H:%M:%S') res_min = int((t1 - t0).total_seconds() / 60) prefix = 'stations_rf' stations_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix)) stations_rf = {} with TemporaryDirectory(prefix=prefix) as temp_dir: for point in points: logging.info(str(point)) station_name = point[0].decode() stations_rf[station_name] = [] station_file_path = os.path.join(temp_dir, station_name + '_%s.txt' % prefix) with open(station_file_path, 'w') as station_file: for t in range(0, len(point_prcp)): station_file.write('%s\t%.4f\n' % (point_prcp['Times'][t], point_prcp[station_name][t])) stations_rf[station_name].append([point_prcp['Times'][t], point_prcp[station_name][t]]) utils.move_files_with_prefix(temp_dir, '*.txt', stations_dir) if curw_db_adapter is not None: logging.info('Pushing data to the db...') ext_utils.push_rainfall_to_db(curw_db_adapter, stations_rf, upsert=curw_db_upsert, name=run_name, source=run_prefix) else: logging.info('curw_db_adapter not available. Unable to push data!')
def test_extract_kelani_basin_rainfall_flo2d(self): wrf_output_dir = tempfile.mkdtemp(prefix='flo2d_') files = ['wrfout_d03_2017-12-09_18:00:00_rf', 'wrfout_d03_2017-12-10_18:00:00_rf', 'wrfout_d03_2017-12-11_18:00:00_rf'] run_prefix = 'wrf0' for f in files: out_dir = utils.create_dir_if_not_exists( os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000'), 'wrf')) shutil.copy2(res_mgr.get_resource_path('test/%s' % f), out_dir) run_date = dt.datetime.strptime('2017-12-11_18:00', '%Y-%m-%d_%H:%M') now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*']) prev_1 = '_'.join([run_prefix, (run_date - dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*']) prev_2 = '_'.join([run_prefix, (run_date - dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*']) d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'wrf', 'wrfout_d03_*'))[0] d03_nc_f_prev_1 = glob.glob(os.path.join(wrf_output_dir, prev_1, 'wrf', 'wrfout_d03_*'))[0] d03_nc_f_prev_2 = glob.glob(os.path.join(wrf_output_dir, prev_2, 'wrf', 'wrfout_d03_*'))[0] kelani_basin_flo2d_file = res_mgr.get_resource_path('extraction/local/kelani_basin_points_250m.txt') extract_kelani_basin_rainfall_flo2d(d03_nc_f, [d03_nc_f_prev_1, d03_nc_f_prev_2], os.path.join(wrf_output_dir, now, 'klb_flo2d'), kelani_basin_file=kelani_basin_flo2d_file, )
def test_extract_kelani_basin_rainfall_flo2d_obs(self): mysql_conf_path = '/home/curw/Desktop/2018-05/mysql.json' adapter = ext_utils.get_curw_adapter(mysql_config_path=mysql_conf_path) wrf_output_dir = tempfile.mkdtemp(prefix='flo2d_obs_') files = ['wrfout_d03_2018-05-23_18:00:00_rf'] run_prefix = 'wrf0' for f in files: out_dir = utils.create_dir_if_not_exists( os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000'), 'wrf')) shutil.copy2('/home/curw/Desktop/2018-05/2018-05-23_18:00/wrf0/%s' % f, out_dir) run_date = dt.datetime.strptime('2018-05-23_18:00', '%Y-%m-%d_%H:%M') start_ts_lk = '2018-05-26_00:00' now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*']) d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'wrf', 'wrfout_d03_*'))[0] obs_stations = { 'Kottawa North Dharmapala School': [79.95818, 6.865576, 'A&T Labs', 'wrf_79.957123_6.859688'], 'IBATTARA2': [79.919, 6.908, 'CUrW IoT', 'wrf_79.902664_6.913757'], 'Malabe': [79.95738, 6.90396, 'A&T Labs', 'wrf_79.957123_6.913757'], # 'Mutwal': [79.8609, 6.95871, 'A&T Labs', 'wrf_79.875435_6.967812'], # 'Mulleriyawa': [79.941176, 6.923571, 'A&T Labs', 'wrf_79.929893_6.913757'], 'Orugodawatta': [79.87887, 6.943741, 'CUrW IoT', 'wrf_79.875435_6.940788'], } duration_days = (8, 0) # kelani_lower_basin_points = res_mgr.get_resource_path('extraction/local/kelani_basin_points_30m.txt') kelani_lower_basin_points = None extract_kelani_basin_rainfall_flo2d_with_obs(d03_nc_f, adapter, obs_stations, os.path.join(wrf_output_dir, now, 'klb_flo2d'), start_ts_lk, kelani_lower_basin_points=kelani_lower_basin_points, duration_days=duration_days)
def run_wps(wrf_config): logging.info('Running WPS: START') wrf_home = wrf_config.get('wrf_home') wps_dir = utils.get_wps_dir(wrf_home) output_dir = utils.create_dir_if_not_exists( os.path.join(wrf_config.get('nfs_dir'), 'results', wrf_config.get('run_id'), 'wps')) logging.info('Backup the output dir') utils.backup_dir(output_dir) logs_dir = utils.create_dir_if_not_exists(os.path.join(output_dir, 'logs')) logging.info('Cleaning up files') utils.delete_files_with_prefix(wps_dir, 'FILE:*') utils.delete_files_with_prefix(wps_dir, 'PFILE:*') utils.delete_files_with_prefix(wps_dir, 'met_em*') # Linking VTable if not os.path.exists(os.path.join(wps_dir, 'Vtable')): logging.info('Creating Vtable symlink') os.symlink(os.path.join(wps_dir, 'ungrib/Variable_Tables/Vtable.NAM'), os.path.join(wps_dir, 'Vtable')) # Running link_grib.csh gfs_date, gfs_cycle, start = utils.get_appropriate_gfs_inventory( wrf_config) dest = utils.get_gfs_data_url_dest_tuple(wrf_config.get('gfs_url'), wrf_config.get('gfs_inv'), gfs_date, gfs_cycle, '', wrf_config.get('gfs_res'), '')[1].replace('.grb2', '') utils.run_subprocess('csh link_grib.csh %s/%s' % (wrf_config.get('gfs_dir'), dest), cwd=wps_dir) try: # Starting ungrib.exe try: utils.run_subprocess('./ungrib.exe', cwd=wps_dir) finally: utils.move_files_with_prefix(wps_dir, 'ungrib.log', logs_dir) # Starting geogrid.exe' if not check_geogrid_output(wps_dir): logging.info('Geogrid output not available') try: utils.run_subprocess('./geogrid.exe', cwd=wps_dir) finally: utils.move_files_with_prefix(wps_dir, 'geogrid.log', logs_dir) # Starting metgrid.exe' try: utils.run_subprocess('./metgrid.exe', cwd=wps_dir) finally: utils.move_files_with_prefix(wps_dir, 'metgrid.log', logs_dir) finally: logging.info('Moving namelist wps file') utils.move_files_with_prefix(wps_dir, 'namelist.wps', output_dir) logging.info('Running WPS: DONE') logging.info('Zipping metgrid data') metgrid_zip = os.path.join(wps_dir, wrf_config.get('run_id') + '_metgrid.zip') utils.create_zip_with_prefix(wps_dir, 'met_em.d*', metgrid_zip) logging.info('Moving metgrid data') dest_dir = os.path.join(wrf_config.get('nfs_dir'), 'metgrid') utils.move_files_with_prefix(wps_dir, metgrid_zip, dest_dir)
def create_rf_plots_wrf(nc_f, plots_output_dir, plots_output_base_dir, lon_min=None, lat_min=None, lon_max=None, lat_max=None, filter_threshold=0.05, run_prefix='WRF'): if not all([lon_min, lat_min, lon_max, lat_max]): lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_EXTENT variables = ext_utils.extract_variables(nc_f, 'RAINC, RAINNC', lat_min, lat_max, lon_min, lon_max) lats = variables['XLAT'] lons = variables['XLONG'] # cell size is calc based on the mean between the lat and lon points cz = np.round(np.mean(np.append(lons[1:len(lons)] - lons[0: len(lons) - 1], lats[1:len(lats)] - lats[0: len(lats) - 1])), 3) clevs = [0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 70, 100, 150, 200, 250, 300, 400, 500, 600, 750] cmap = cm.s3pcpn basemap = Basemap(projection='merc', llcrnrlon=lon_min, llcrnrlat=lat_min, urcrnrlon=lon_max, urcrnrlat=lat_max, resolution='h') data = variables['RAINC'] + variables['RAINNC'] logging.info('Filtering with the threshold %f' % filter_threshold) data[data < filter_threshold] = 0.0 variables['PRECIP'] = data prefix = 'wrf_plots' with TemporaryDirectory(prefix=prefix) as temp_dir: t0 = dt.datetime.strptime(variables['Times'][0], '%Y-%m-%d_%H:%M:%S') t1 = dt.datetime.strptime(variables['Times'][1], '%Y-%m-%d_%H:%M:%S') step = (t1 - t0).total_seconds() / 3600.0 inst_precip = ext_utils.get_two_element_average(variables['PRECIP']) cum_precip = ext_utils.get_two_element_average(variables['PRECIP'], return_diff=False) for i in range(1, len(variables['Times'])): time = variables['Times'][i] ts = dt.datetime.strptime(time, '%Y-%m-%d_%H:%M:%S') lk_ts = utils.datetime_utc_to_lk(ts, shift_mins=30) logging.info('processing %s', time) # instantaneous precipitation (hourly) inst_file = os.path.join(temp_dir, 'wrf_inst_' + lk_ts.strftime('%Y-%m-%d_%H:%M:%S')) ext_utils.create_asc_file(np.flip(inst_precip[i - 1], 0), lats, lons, inst_file + '.asc', cell_size=cz) title = { 'label': 'Hourly rf for %s LK' % lk_ts.strftime('%Y-%m-%d_%H:%M:%S'), 'fontsize': 30 } ext_utils.create_contour_plot(inst_precip[i - 1], inst_file + '.png', lat_min, lon_min, lat_max, lon_max, title, clevs=clevs, cmap=cmap, basemap=basemap) if (i * step) % 24 == 0: t = 'Daily rf from %s LK to %s LK' % ( (lk_ts - dt.timedelta(hours=24)).strftime('%Y-%m-%d_%H:%M:%S'), lk_ts.strftime('%Y-%m-%d_%H:%M:%S')) d = int(i * step / 24) - 1 logging.info('Creating images for D%d' % d) cum_file = os.path.join(temp_dir, 'wrf_cum_%dd' % d) if i * step / 24 > 1: cum_precip_24h = cum_precip[i - 1] - cum_precip[i - 1 - int(24 / step)] else: cum_precip_24h = cum_precip[i - 1] ext_utils.create_asc_file(np.flip(cum_precip_24h, 0), lats, lons, cum_file + '.asc', cell_size=cz) ext_utils.create_contour_plot(cum_precip_24h, cum_file + '.png', lat_min, lon_min, lat_max, lon_max, t, clevs=clevs, cmap=cmap, basemap=basemap) gif_file = os.path.join(temp_dir, 'wrf_inst_%dd' % d) images = [os.path.join(temp_dir, 'wrf_inst_' + j.strftime('%Y-%m-%d_%H:%M:%S') + '.png') for j in np.arange(lk_ts - dt.timedelta(hours=24 - step), lk_ts + dt.timedelta(hours=step), dt.timedelta(hours=step)).astype(dt.datetime)] ext_utils.create_gif(images, gif_file + '.gif') logging.info('Creating the zips') utils.create_zip_with_prefix(temp_dir, '*.png', os.path.join(temp_dir, 'pngs.zip')) utils.create_zip_with_prefix(temp_dir, '*.asc', os.path.join(temp_dir, 'ascs.zip')) logging.info('Cleaning up instantaneous pngs and ascs - wrf_inst_*') utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.png') utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.asc') logging.info('Copying pngs to ' + plots_output_dir) utils.move_files_with_prefix(temp_dir, '*.png', plots_output_dir) logging.info('Copying ascs to ' + plots_output_dir) utils.move_files_with_prefix(temp_dir, '*.asc', plots_output_dir) logging.info('Copying gifs to ' + plots_output_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', plots_output_dir) logging.info('Copying zips to ' + plots_output_dir) utils.copy_files_with_prefix(temp_dir, '*.zip', plots_output_dir) plots_latest_dir = os.path.join(plots_output_base_dir, 'latest', run_prefix, os.path.basename(plots_output_dir)) # <nfs>/latest/wrf0 .. 3 utils.create_dir_if_not_exists(plots_latest_dir) # todo: this needs to be adjusted to handle the multiple runs logging.info('Copying gifs to ' + plots_latest_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', plots_latest_dir)
def process(self, *args, **kwargs): config = self.get_config(**kwargs) logging.info('wrf conifg: ' + config.to_json_string()) start_date = config.get('start_date') d03_dir = config.get('wrf_output_dir') d03_sl = os.path.join(d03_dir, 'wrfout_d03_' + start_date + ':00_SL') # create a temp work dir & get a local copy of the d03.._SL temp_dir = utils.create_dir_if_not_exists( os.path.join(config.get('wrf_home'), 'temp')) shutil.copy2(d03_sl, temp_dir) d03_sl = os.path.join(temp_dir, os.path.basename(d03_sl)) lat_min = 5.722969 lon_min = 79.52146 lat_max = 10.06425 lon_max = 82.18992 variables = ext_utils.extract_variables(d03_sl, 'RAINC, RAINNC', lat_min, lat_max, lon_min, lon_max) lats = variables['XLAT'] lons = variables['XLONG'] # cell size is calc based on the mean between the lat and lon points cz = np.round( np.mean( np.append(lons[1:len(lons)] - lons[0:len(lons) - 1], lats[1:len(lats)] - lats[0:len(lats) - 1])), 3) # clevs = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30]) # clevs_cum = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100]) # norm = colors.BoundaryNorm(boundaries=clevs, ncolors=256) # norm_cum = colors.BoundaryNorm(boundaries=clevs_cum, ncolors=256) # cmap = plt.get_cmap('jet') clevs = [ 0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 70, 100, 150, 200, 250, 300, 400, 500, 600, 750 ] clevs_cum = clevs norm = None norm_cum = None cmap = cm.s3pcpn basemap = Basemap(projection='merc', llcrnrlon=lon_min, llcrnrlat=lat_min, urcrnrlon=lon_max, urcrnrlat=lat_max, resolution='h') filter_threshold = 0.05 data = variables['RAINC'] + variables['RAINNC'] logging.info('Filtering with the threshold %f' % filter_threshold) data[data < filter_threshold] = 0.0 variables['PRECIP'] = data pngs = [] ascs = [] for i in range(1, len(variables['Times'])): time = variables['Times'][i] ts = dt.datetime.strptime(time, '%Y-%m-%d_%H:%M:%S') lk_ts = utils.datetime_utc_to_lk(ts) logging.info('processing %s', time) # instantaneous precipitation (hourly) inst_precip = variables['PRECIP'][i] - variables['PRECIP'][i - 1] inst_file = os.path.join(temp_dir, 'wrf_inst_' + time) title = { 'label': 'Hourly rf for %s LK\n%s UTC' % (lk_ts.strftime('%Y-%m-%d_%H:%M:%S'), time), 'fontsize': 30 } ext_utils.create_asc_file(np.flip(inst_precip, 0), lats, lons, inst_file + '.asc', cell_size=cz) ascs.append(inst_file + '.asc') ext_utils.create_contour_plot(inst_precip, inst_file + '.png', lat_min, lon_min, lat_max, lon_max, title, clevs=clevs, cmap=cmap, basemap=basemap, norm=norm) pngs.append(inst_file + '.png') if i % 24 == 0: t = 'Daily rf from %s LK to %s LK' % ( (lk_ts - dt.timedelta(hours=24)).strftime('%Y-%m-%d_%H:%M:%S'), lk_ts.strftime('%Y-%m-%d_%H:%M:%S')) d = int(i / 24) - 1 logging.info('Creating images for D%d' % d) cum_file = os.path.join(temp_dir, 'wrf_cum_%dd' % d) ext_utils.create_asc_file(np.flip(variables['PRECIP'][i], 0), lats, lons, cum_file + '.asc', cell_size=cz) ascs.append(cum_file + '.asc') ext_utils.create_contour_plot(variables['PRECIP'][i] - variables['PRECIP'][i - 24], cum_file + '.png', lat_min, lon_min, lat_max, lon_max, t, clevs=clevs, cmap=cmap, basemap=basemap, norm=norm_cum) pngs.append(inst_file + '.png') gif_file = os.path.join(temp_dir, 'wrf_inst_%dd' % d) images = [ os.path.join( temp_dir, 'wrf_inst_' + i.strftime('%Y-%m-%d_%H:%M:%S') + '.png') for i in np.arange(ts - dt.timedelta(hours=23), ts + dt.timedelta( hours=1), dt.timedelta( hours=1)).astype(dt.datetime) ] ext_utils.create_gif(images, gif_file + '.gif') logging.info('Creating the zips') utils.create_zip_with_prefix(temp_dir, '*.png', os.path.join(temp_dir, 'pngs.zip')) utils.create_zip_with_prefix(temp_dir, '*.asc', os.path.join(temp_dir, 'ascs.zip')) # utils.create_zipfile(pngs, os.path.join(temp_dir, 'pngs.zip')) # utils.create_zipfile(ascs, os.path.join(temp_dir, 'ascs.zip')) logging.info('Cleaning up instantaneous pngs and ascs - wrf_inst_*') utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.png') utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.asc') logging.info('Copying pngs to ' + d03_dir) utils.move_files_with_prefix(temp_dir, '*.png', d03_dir) logging.info('Copying ascs to ' + d03_dir) utils.move_files_with_prefix(temp_dir, '*.asc', d03_dir) logging.info('Copying gifs to ' + d03_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', d03_dir) logging.info('Copying zips to ' + d03_dir) utils.copy_files_with_prefix(temp_dir, '*.zip', d03_dir) d03_latest_dir = os.path.join(config.get('nfs_dir'), 'latest', os.path.basename(config.get('wrf_home'))) # <nfs>/latest/wrf0 .. 3 utils.create_dir_if_not_exists(d03_latest_dir) # todo: this needs to be adjusted to handle the multiple runs logging.info('Copying gifs to ' + d03_latest_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', d03_latest_dir) logging.info('Cleaning up temp dir') shutil.rmtree(temp_dir)
def process(self, *args, **kwargs): config = self.get_config(**kwargs) logging.info('wrf conifg: ' + config.to_json_string()) start_date = config.get('start_date') d03_dir = config.get('wrf_output_dir') d03_sl = os.path.join(d03_dir, 'wrfout_d01_' + start_date + ':00_SL') # create a temp work dir & get a local copy of the d03.._SL temp_dir = utils.create_dir_if_not_exists( os.path.join(config.get('wrf_home'), 'temp_d01')) shutil.copy2(d03_sl, temp_dir) d03_sl = os.path.join(temp_dir, os.path.basename(d03_sl)) lat_min = -3.06107 lon_min = 71.2166 lat_max = 18.1895 lon_max = 90.3315 variables = ext_utils.extract_variables(d03_sl, 'RAINC, RAINNC', lat_min, lat_max, lon_min, lon_max) lats = variables['XLAT'] lons = variables['XLONG'] # cell size is calc based on the mean between the lat and lon points cz = np.round( np.mean( np.append(lons[1:len(lons)] - lons[0:len(lons) - 1], lats[1:len(lats)] - lats[0:len(lats) - 1])), 3) # clevs = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30]) # clevs_cum = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100]) # norm = colors.BoundaryNorm(boundaries=clevs, ncolors=256) # norm_cum = colors.BoundaryNorm(boundaries=clevs_cum, ncolors=256) # cmap = plt.get_cmap('jet') clevs = [ 0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 70, 100, 150, 200, 250, 300, 400, 500, 600, 750 ] clevs_cum = clevs norm = None cmap = cm.s3pcpn basemap = Basemap(projection='merc', llcrnrlon=lon_min, llcrnrlat=lat_min, urcrnrlon=lon_max, urcrnrlat=lat_max, resolution='h') filter_threshold = 0.05 data = variables['RAINC'] + variables['RAINNC'] logging.info('Filtering with the threshold %f' % filter_threshold) data[data < filter_threshold] = 0.0 variables['PRECIP'] = data for i in range(1, len(variables['Times'])): time = variables['Times'][i] ts = dt.datetime.strptime(time, '%Y-%m-%d_%H:%M:%S') lk_ts = utils.datetime_utc_to_lk(ts) logging.info('processing %s', time) # instantaneous precipitation (hourly) inst_precip = variables['PRECIP'][i] - variables['PRECIP'][i - 1] inst_file = os.path.join(temp_dir, 'wrf_inst_' + time) title = { 'label': '3Hourly rf for %s LK\n%s UTC' % (lk_ts.strftime('%Y-%m-%d_%H:%M:%S'), time), 'fontsize': 30 } ext_utils.create_contour_plot(inst_precip, inst_file + '.png', lat_min, lon_min, lat_max, lon_max, title, clevs=clevs, cmap=cmap, basemap=basemap, norm=norm) if i % 8 == 0: d = int(i / 8) - 1 logging.info('Creating gif for D%d' % d) gif_file = os.path.join(temp_dir, 'wrf_inst_D01_%dd' % d) images = [ os.path.join( temp_dir, 'wrf_inst_' + i.strftime('%Y-%m-%d_%H:%M:%S') + '.png') for i in np.arange(ts - dt.timedelta(hours=24 - 3), ts + dt.timedelta( hours=3), dt.timedelta( hours=3)).astype(dt.datetime) ] ext_utils.create_gif(images, gif_file + '.gif') # move all the data in the tmp dir to the nfs logging.info('Copying gifs to ' + d03_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', d03_dir) d03_latest_dir = os.path.join(config.get('nfs_dir'), 'latest', os.path.basename(config.get('wrf_home'))) # <nfs>/latest/wrf0 .. 3 utils.create_dir_if_not_exists(d03_latest_dir) # todo: this needs to be adjusted to handle the multiple runs logging.info('Copying gifs to ' + d03_latest_dir) utils.copy_files_with_prefix(temp_dir, '*.gif', d03_latest_dir) logging.info('Cleaning up the dir ' + temp_dir) shutil.rmtree(temp_dir)
def extract_kelani_basin_rainfall_flo2d_with_obs(nc_f, adapter, obs_stations, output_dir, start_ts_lk, duration_days=None, output_prefix='RAINCELL', kelani_lower_basin_points=None, kelani_lower_basin_shp=None): """ check test_extract_kelani_basin_rainfall_flo2d_obs test case :param nc_f: file path of the wrf output :param adapter: :param obs_stations: dict of stations. {station_name: [lon, lat, name variable, nearest wrf point station name]} :param output_dir: :param start_ts_lk: start time of the forecast/ end time of the observations :param duration_days: (optional) a tuple (observation days, forecast days) default (2,3) :param output_prefix: (optional) output file name of the RAINCELL file. ex: output_prefix=RAINCELL-150m --> RAINCELL-150m.DAT :param kelani_lower_basin_points: (optional) :param kelani_lower_basin_shp: (optional) :return: """ if duration_days is None: duration_days = (2, 3) if kelani_lower_basin_points is None: kelani_lower_basin_points = res_mgr.get_resource_path('extraction/local/kelani_basin_points_250m.txt') if kelani_lower_basin_shp is None: kelani_lower_basin_shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp') points = np.genfromtxt(kelani_lower_basin_points, delimiter=',') kel_lon_min = np.min(points, 0)[1] kel_lat_min = np.min(points, 0)[2] kel_lon_max = np.max(points, 0)[1] kel_lat_max = np.max(points, 0)[2] diff, kel_lats, kel_lons, times = ext_utils.extract_area_rf_series(nc_f, kel_lat_min, kel_lat_max, kel_lon_min, kel_lon_max) def get_bins(arr): sz = len(arr) return (arr[1:sz - 1] + arr[0:sz - 2]) / 2 lat_bins = get_bins(kel_lats) lon_bins = get_bins(kel_lons) t0 = dt.datetime.strptime(times[0], '%Y-%m-%d_%H:%M:%S') t1 = dt.datetime.strptime(times[1], '%Y-%m-%d_%H:%M:%S') utils.create_dir_if_not_exists(output_dir) obs_start = dt.datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') - dt.timedelta(days=duration_days[0]) obs_end = dt.datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') forecast_end = dt.datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') + dt.timedelta(days=duration_days[1]) obs = _get_observed_precip(obs_stations, obs_start, obs_end, duration_days, adapter) thess_poly = spatial_utils.get_voronoi_polygons(obs_stations, kelani_lower_basin_shp, add_total_area=False) output_file_path = os.path.join(output_dir, output_prefix + '.DAT') # update points array with the thessian polygon idx point_thess_idx = [] for point in points: point_thess_idx.append(spatial_utils.is_inside_geo_df(thess_poly, lon=point[1], lat=point[2])) pass with open(output_file_path, 'w') as output_file: res_mins = int((t1 - t0).total_seconds() / 60) data_hours = int(sum(duration_days) * 24 * 60 / res_mins) start_ts_lk = obs_start.strftime('%Y-%m-%d %H:%M:%S') end_ts = forecast_end.strftime('%Y-%m-%d %H:%M:%S') output_file.write("%d %d %s %s\n" % (res_mins, data_hours, start_ts_lk, end_ts)) for t in range(int(24 * 60 * duration_days[0] / res_mins) + 1): for i, point in enumerate(points): rf = float(obs[point_thess_idx[i]].values[t]) if point_thess_idx[i] is not None else 0 output_file.write('%d %.1f\n' % (point[0], rf)) forecast_start_idx = int( np.where(times == utils.datetime_lk_to_utc(obs_end, shift_mins=30).strftime('%Y-%m-%d_%H:%M:%S'))[0]) for t in range(int(24 * 60 * duration_days[1] / res_mins) - 1): for point in points: rf_x = np.digitize(point[1], lon_bins) rf_y = np.digitize(point[2], lat_bins) if t + forecast_start_idx + 1 < len(times): output_file.write('%d %.1f\n' % (point[0], diff[t + forecast_start_idx + 1, rf_y, rf_x])) else: output_file.write('%d %.1f\n' % (point[0], 0))
def extract_metro_colombo(nc_f, wrf_output, wrf_output_base, curw_db_adapter=None, curw_db_upsert=False, run_prefix='WRF', run_name='Cloud-1'): """ extract Metro-Colombo rf and divide area into to 4 quadrants :param wrf_output_base: :param run_name: :param nc_f: :param wrf_output: :param curw_db_adapter: If not none, data will be pushed to the db :param run_prefix: :param curw_db_upsert: :return: """ prefix = 'met_col' lon_min, lat_min, lon_max, lat_max = constants.COLOMBO_EXTENT nc_vars = ext_utils.extract_variables(nc_f, ['RAINC', 'RAINNC'], lat_min, lat_max, lon_min, lon_max) lats = nc_vars['XLAT'] lons = nc_vars['XLONG'] prcp = nc_vars['RAINC'] + nc_vars['RAINNC'] times = nc_vars['Times'] diff = ext_utils.get_two_element_average(prcp) width = len(lons) height = len(lats) output_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix)) basin_rf = np.mean(diff[0:(len(times) - 1 if len(times) < 24 else 24), :, :]) alpha_file_path = os.path.join(wrf_output_base, prefix + '_alphas.txt') utils.create_dir_if_not_exists(os.path.dirname(alpha_file_path)) with open(alpha_file_path, 'a+') as alpha_file: t = utils.datetime_utc_to_lk(dt.datetime.strptime(times[0], '%Y-%m-%d_%H:%M:%S'), shift_mins=30) alpha_file.write('%s\t%f\n' % (t.strftime('%Y-%m-%d_%H:%M:%S'), basin_rf)) cz = ext_utils.get_mean_cell_size(lats, lons) no_data = -99 divs = (2, 2) div_rf = {} for i in range(divs[0] * divs[1]): div_rf[prefix + str(i)] = [] with TemporaryDirectory(prefix=prefix) as temp_dir: subsection_file_path = os.path.join(temp_dir, 'sub_means.txt') with open(subsection_file_path, 'w') as subsection_file: for tm in range(0, len(times) - 1): t_str = ( utils.datetime_utc_to_lk(dt.datetime.strptime(times[tm], '%Y-%m-%d_%H:%M:%S'), shift_mins=30)).strftime('%Y-%m-%d %H:%M:%S') output_file_path = os.path.join(temp_dir, 'rf_' + t_str.replace(' ', '_') + '.asc') ext_utils.create_asc_file(np.flip(diff[tm, :, :], 0), lats, lons, output_file_path, cell_size=cz, no_data_val=no_data) # writing subsection file x_idx = [round(i * width / divs[0]) for i in range(0, divs[0] + 1)] y_idx = [round(i * height / divs[1]) for i in range(0, divs[1] + 1)] subsection_file.write(t_str) for j in range(len(y_idx) - 1): for i in range(len(x_idx) - 1): quad = j * divs[1] + i sub_sec_mean = np.mean(diff[tm, y_idx[j]:y_idx[j + 1], x_idx[i]: x_idx[i + 1]]) subsection_file.write('\t%.4f' % sub_sec_mean) div_rf[prefix + str(quad)].append([t_str, sub_sec_mean]) subsection_file.write('\n') utils.create_zip_with_prefix(temp_dir, 'rf_*.asc', os.path.join(temp_dir, 'ascs.zip'), clean_up=True) utils.move_files_with_prefix(temp_dir, '*', output_dir) # writing to the database if curw_db_adapter is not None: for i in range(divs[0] * divs[1]): name = prefix + str(i) station = [Station.CUrW, name, name, -999, -999, 0, "met col quadrant %d" % i] if ext_utils.create_station_if_not_exists(curw_db_adapter, station): logging.info('%s station created' % name) logging.info('Pushing data to the db...') ext_utils.push_rainfall_to_db(curw_db_adapter, div_rf, upsert=curw_db_upsert, source=run_prefix, name=run_name) else: logging.info('curw_db_adapter not available. Unable to push data!') return basin_rf
def extract_kelani_basin_rainfall_flo2d(nc_f, nc_f_prev_days, output_dir, avg_basin_rf=1.0, kelani_basin_file=None, target_rfs=None, output_prefix='RAINCELL'): """ :param output_prefix: :param nc_f: :param nc_f_prev_days: :param output_dir: :param avg_basin_rf: :param kelani_basin_file: :param target_rfs: :return: """ if target_rfs is None: target_rfs = [100, 150, 200, 250, 300] if kelani_basin_file is None: kelani_basin_file = res_mgr.get_resource_path('extraction/local/kelani_basin_points_250m.txt') points = np.genfromtxt(kelani_basin_file, delimiter=',') kel_lon_min = np.min(points, 0)[1] kel_lat_min = np.min(points, 0)[2] kel_lon_max = np.max(points, 0)[1] kel_lat_max = np.max(points, 0)[2] diff, kel_lats, kel_lons, times = ext_utils.extract_area_rf_series(nc_f, kel_lat_min, kel_lat_max, kel_lon_min, kel_lon_max) def get_bins(arr): sz = len(arr) return (arr[1:sz - 1] + arr[0:sz - 2]) / 2 lat_bins = get_bins(kel_lats) lon_bins = get_bins(kel_lons) t0 = dt.datetime.strptime(times[0], '%Y-%m-%d_%H:%M:%S') t1 = dt.datetime.strptime(times[1], '%Y-%m-%d_%H:%M:%S') t_end = dt.datetime.strptime(times[-1], '%Y-%m-%d_%H:%M:%S') utils.create_dir_if_not_exists(output_dir) prev_diff = [] prev_days = len(nc_f_prev_days) for i in range(prev_days): if nc_f_prev_days[i]: p_diff, _, _, _ = ext_utils.extract_area_rf_series(nc_f_prev_days[i], kel_lat_min, kel_lat_max, kel_lon_min, kel_lon_max) prev_diff.append(p_diff) else: prev_diff.append(None) def write_forecast_to_raincell_file(output_file_path, alpha): with open(output_file_path, 'w') as output_file: res_mins = int((t1 - t0).total_seconds() / 60) data_hours = int(len(times) + prev_days * 24 * 60 / res_mins) start_ts = utils.datetime_utc_to_lk(t0 - dt.timedelta(days=prev_days), shift_mins=30).strftime( '%Y-%m-%d %H:%M:%S') end_ts = utils.datetime_utc_to_lk(t_end, shift_mins=30).strftime('%Y-%m-%d %H:%M:%S') output_file.write("%d %d %s %s\n" % (res_mins, data_hours, start_ts, end_ts)) for d in range(prev_days): for t in range(int(24 * 60 / res_mins)): for point in points: rf_x = np.digitize(point[1], lon_bins) rf_y = np.digitize(point[2], lat_bins) if prev_diff[prev_days - 1 - d] is not None: output_file.write('%d %.1f\n' % (point[0], prev_diff[prev_days - 1 - d][t, rf_y, rf_x])) else: output_file.write('%d %.1f\n' % (point[0], 0)) for t in range(len(times)): for point in points: rf_x = np.digitize(point[1], lon_bins) rf_y = np.digitize(point[2], lat_bins) if t < int(24 * 60 / res_mins): output_file.write('%d %.1f\n' % (point[0], diff[t, rf_y, rf_x] * alpha)) else: output_file.write('%d %.1f\n' % (point[0], diff[t, rf_y, rf_x])) with TemporaryDirectory(prefix='curw_raincell') as temp_dir: raincell_temp = os.path.join(temp_dir, output_prefix + '.DAT') write_forecast_to_raincell_file(raincell_temp, 1) for target_rf in target_rfs: write_forecast_to_raincell_file('%s.%d' % (raincell_temp, target_rf), target_rf / avg_basin_rf) utils.create_zip_with_prefix(temp_dir, output_prefix + '.DAT*', os.path.join(temp_dir, output_prefix + '.zip'), clean_up=True) utils.move_files_with_prefix(temp_dir, output_prefix + '.zip', utils.create_dir_if_not_exists(output_dir))
def run_em_real(wrf_config): logging.info('Running em_real...') wrf_home = wrf_config.get('wrf_home') em_real_dir = utils.get_em_real_dir(wrf_home) procs = wrf_config.get('procs') run_id = wrf_config.get('run_id') output_dir = utils.create_dir_if_not_exists( os.path.join(wrf_config.get('nfs_dir'), 'results', run_id, 'wrf')) archive_dir = utils.create_dir_if_not_exists( os.path.join(wrf_config.get('archive_dir'), 'results', run_id, 'wrf')) logging.info('Backup the output dir') utils.backup_dir(output_dir) logs_dir = utils.create_dir_if_not_exists(os.path.join(output_dir, 'logs')) logging.info('Copying metgrid.zip') metgrid_dir = os.path.join(wrf_config.get('nfs_dir'), 'metgrid') if wrf_config.is_set('wps_run_id'): logging.info('wps_run_id is set. Copying metgrid from ' + wrf_config.get('wps_run_id')) utils.copy_files_with_prefix( metgrid_dir, wrf_config.get('wps_run_id') + '_metgrid.zip', em_real_dir) metgrid_zip = os.path.join( em_real_dir, wrf_config.get('wps_run_id') + '_metgrid.zip') else: utils.copy_files_with_prefix(metgrid_dir, wrf_config.get('run_id') + '_metgrid.zip', em_real_dir) metgrid_zip = os.path.join(em_real_dir, wrf_config.get('run_id') + '_metgrid.zip') logging.info('Extracting metgrid.zip') ZipFile(metgrid_zip, 'r', compression=ZIP_DEFLATED).extractall(path=em_real_dir) # logs destination: nfs/logs/xxxx/rsl* try: try: logging.info('Starting real.exe') utils.run_subprocess( 'mpirun --allow-run-as-root -np %d ./real.exe' % procs, cwd=em_real_dir) finally: logging.info('Moving Real log files...') utils.create_zip_with_prefix(em_real_dir, 'rsl*', os.path.join(em_real_dir, 'real_rsl.zip'), clean_up=True) utils.move_files_with_prefix(em_real_dir, 'real_rsl.zip', logs_dir) try: logging.info('Starting wrf.exe') utils.run_subprocess( 'mpirun --allow-run-as-root -np %d ./wrf.exe' % procs, cwd=em_real_dir) finally: logging.info('Moving WRF log files...') utils.create_zip_with_prefix(em_real_dir, 'rsl*', os.path.join(em_real_dir, 'wrf_rsl.zip'), clean_up=True) utils.move_files_with_prefix(em_real_dir, 'wrf_rsl.zip', logs_dir) finally: logging.info('Moving namelist input file') utils.move_files_with_prefix(em_real_dir, 'namelist.input', output_dir) logging.info('WRF em_real: DONE! Moving data to the output dir') logging.info('Extracting rf from domain3') d03_nc = glob.glob(os.path.join(em_real_dir, 'wrfout_d03_*'))[0] ncks_query = 'ncks -v %s %s %s' % ('RAINC,RAINNC,XLAT,XLONG,Times', d03_nc, d03_nc + '_rf') utils.run_subprocess(ncks_query) logging.info('Extracting rf from domain1') d01_nc = glob.glob(os.path.join(em_real_dir, 'wrfout_d01_*'))[0] ncks_query = 'ncks -v %s %s %s' % ('RAINC,RAINNC,XLAT,XLONG,Times', d01_nc, d01_nc + '_rf') utils.run_subprocess(ncks_query) logging.info('Moving data to the output dir') utils.move_files_with_prefix(em_real_dir, 'wrfout_d03*_rf', output_dir) utils.move_files_with_prefix(em_real_dir, 'wrfout_d01*_rf', output_dir) logging.info('Moving data to the archive dir') utils.move_files_with_prefix(em_real_dir, 'wrfout_*', archive_dir) logging.info('Cleaning up files') utils.delete_files_with_prefix(em_real_dir, 'met_em*') utils.delete_files_with_prefix(em_real_dir, 'rsl*') os.remove(metgrid_zip)