def test_create_rainfall_for_mike21(self): wrf_output_dir = tempfile.mkdtemp(prefix='mike21_') run_date = dt.datetime.strptime('2017-12-11_18:00', '%Y-%m-%d_%H:%M') basin_shp_file = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp') files = ['wrfout_d03_2017-12-09_18:00:00_rf', 'wrfout_d03_2017-12-10_18:00:00_rf', 'wrfout_d03_2017-12-11_18:00:00_rf'] run_prefix = 'wrf0' for f in files: d03_nc_f = res_mgr.get_resource_path('test/%s' % f) out_dir = utils.create_dir_if_not_exists( os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000'))) extract_mean_rainfall_from_shp_file(d03_nc_f, out_dir, 'klb_mean_rf', 'klb_mean', basin_shp_file, constants.KELANI_LOWER_BASIN_EXTENT) now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*']) prev_1 = '_'.join([run_prefix, (run_date - dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*']) prev_2 = '_'.join([run_prefix, (run_date - dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*']) d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'klb_mean_rf', 'klb_mean_rf.txt'))[0] d03_nc_f_prev_1 = glob.glob(os.path.join(wrf_output_dir, prev_1, 'klb_mean_rf', 'klb_mean_rf.txt'))[0] d03_nc_f_prev_2 = glob.glob(os.path.join(wrf_output_dir, prev_2, 'klb_mean_rf', 'klb_mean_rf.txt'))[0] create_rainfall_for_mike21(d03_nc_f, [d03_nc_f_prev_1, d03_nc_f_prev_2], os.path.join(wrf_output_dir, now, 'mike_21'))
def test_extract_mean_rainfall_from_shp_file(self): adapter = ext_utils.get_curw_adapter() basin_shp_file = res_mgr.get_resource_path('extraction/shp/kelani-upper-basin.shp') basin_extent = constants.KELANI_UPPER_BASIN_EXTENT extract_mean_rainfall_from_shp_file(res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00'), tempfile.mkdtemp(prefix='temp_'), 'kub_mean_rf', 'kub_mean', basin_shp_file, basin_extent, curw_db_adapter=adapter, curw_db_upsert=True)
def test_extract_kelani_basin_rainfall_flo2d_obs_150m(self): adapter = ext_utils.get_curw_adapter(mysql_config_path='/home/curw/Desktop/2018-05/mysql.json') wrf_output_dir = tempfile.mkdtemp(prefix='flo2d_obs_') files = ['wrfout_d03_2018-05-23_18:00:00_rf'] run_prefix = 'wrf0' for f in files: out_dir = utils.create_dir_if_not_exists( os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000'), 'wrf')) shutil.copy2('/home/curw/Desktop/2018-05/2018-05-23_18:00/wrf0/%s' % f, out_dir) run_date = dt.datetime.strptime('2018-05-23_18:00', '%Y-%m-%d_%H:%M') now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*']) d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'wrf', 'wrfout_d03_*'))[0] obs_stations = {'Kottawa North Dharmapala School': [79.95818, 6.865576, 'A&T Labs', 'wrf_79.957123_6.859688'], 'IBATTARA2': [79.919, 6.908, 'CUrW IoT', 'wrf_79.902664_6.913757'], 'Malabe': [79.95738, 6.90396, 'A&T Labs', 'wrf_79.957123_6.913757'], # 'Mutwal': [79.8609, 6.95871, 'A&T Labs', 'wrf_79.875435_6.967812'], 'Glencourse': [80.20305, 6.97805, 'Irrigation Department', 'wrf_80.202187_6.967812'], # 'Waga': [80.11828, 6.90678, 'A&T Labs', 'wrf_80.120499_6.913757'], } start_ts = '2018-05-26_00:00' kelani_lower_basin_points = res_mgr.get_resource_path('extraction/local/klb_glecourse_points_150m.txt') kelani_lower_basin_shp = res_mgr.get_resource_path('extraction/shp/klb_glencourse/klb_glencourse.shp') duration_days = (8, 0) extract_kelani_basin_rainfall_flo2d_with_obs(d03_nc_f, adapter, obs_stations, os.path.join(wrf_output_dir, now, 'klb_flo2d'), start_ts, duration_days=duration_days, kelani_lower_basin_shp=kelani_lower_basin_shp, kelani_lower_basin_points=kelani_lower_basin_points, output_prefix='RAINCELL_150m')
def test_create_rf_plots_wrf(self): out_base_dir = tempfile.mkdtemp(prefix='rf_plots_') out_dir = os.path.join(out_base_dir, 'plots_D03') create_rf_plots_wrf(res_mgr.get_resource_path('test/wrfout_d03_2017-12-09_18:00:00_rf'), out_dir, out_base_dir) out_dir = os.path.join(out_base_dir, 'plots_D01') lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_D01_EXTENT create_rf_plots_wrf(res_mgr.get_resource_path('test/wrfout_d01_2017-12-09_18:00:00_rf'), out_dir, out_base_dir, lat_min=lat_min, lon_min=lon_min, lat_max=lat_max, lon_max=lon_max)
def test_extract_point_rf_series(self): points = np.genfromtxt(res_mgr.get_resource_path( 'extraction/local/metro_col_sub_catch_centroids.txt'), delimiter=',', names=True, dtype=None) extract_points_array_rf_series( res_mgr.get_resource_path( '/test/wrfout_d03_2017-12-09_18:00:00_rf'), points)
def test_all(self): logging.basicConfig( level=logging.DEBUG, format= '%(asctime)s %(threadName)s %(module)s %(levelname)s %(message)s') wrf_home = tempfile.mkdtemp(prefix='wrf_test_') nfs_dir = os.path.join(wrf_home, 'nfs') run_id = 'WRF_test0' output_dir = os.path.join(nfs_dir, 'results', run_id, 'wrf') utils.create_dir_if_not_exists(output_dir) shutil.copy( res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00'), output_dir) wrf_conf_dict = { "wrf_home": wrf_home, "nfs_dir": nfs_dir, "period": 0.25, "start_date": "2017-10-02_12:00" } db_conf_dict = { "host": "localhost", "user": "******", "password": "******", "db": "testdb" } extract_data_wrf.run(run_id, wrf_conf_dict, db_conf_dict, upsert=True)
def test_compare_voronoi_polygons(self): files = ['up_in_3_thie_join.shp', 'up_in_11_thie_join.shp'] out = tempfile.mkdtemp(prefix='voronoi_') for f in files: voronoi_shp_file = res_mgr.get_resource_path('test/shp/%s' % f) area_shp_file = res_mgr.get_resource_path('extraction/shp/kub-wgs84/kub-wgs84.shp') shape_df = gpd.GeoDataFrame.from_file(voronoi_shp_file) points = {} for i in range(len(shape_df)): points[shape_df['OBJECTID'][i]] = [shape_df['x'][i], shape_df['y'][i]] result = get_voronoi_polygons(points, area_shp_file, ['OBJECTID', 1], output_shape_file=os.path.join(out, '%s_out.shp' % f)) for i in range(len(shape_df)): self.assertAlmostEqual(result['area'][i], shape_df['Shape_Area'][i], places=4)
def replace_namelist_wps(wrf_config, start_date=None, end_date=None): logging.info('Replacing namelist.wps...') if os.path.exists(wrf_config.get('namelist_wps')): f = wrf_config.get('namelist_wps') else: f = res_mgr.get_resource_path( os.path.join('execution', constants.DEFAULT_NAMELIST_WPS_TEMPLATE)) dest = os.path.join(utils.get_wps_dir(wrf_config.get('wrf_home')), 'namelist.wps') replace_file_with_values(wrf_config, f, dest, 'namelist_wps_dict', start_date, end_date)
def get_curw_adapter(mysql_config_path=None): if mysql_config_path is None: mysql_config_path = res_mgr.get_resource_path( 'config/mysql_config.json') with open(mysql_config_path) as data_file: config = json.load(data_file) return mysqladapter(host=config['host'], user=config['user'], password=config['password'], db=config['db'])
def get_curw_adapter(mysql_config=None, mysql_config_path=None): if mysql_config_path is None: mysql_config_path = res_mgr.get_resource_path( 'config/mysql_config.json') with open(mysql_config_path) as data_file: config = json.load(data_file) if mysql_config is not None and isinstance(mysql_config, dict): config.update(mysql_config) return MySQLAdapter(**config)
def test_push_wrf_rainfall_to_db(self): config = { "host": "localhost", "user": "******", "password": "******", "db": "testdb" } adapter = ext_utils.get_curw_adapter(mysql_config=config) nc_f = res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00') lon_min, lat_min, lon_max, lat_max = constants.KELANI_KALU_BASIN_EXTENT push_wrf_rainfall_to_db(nc_f, curw_db_adapter=adapter, lat_min=lat_min, lon_min=lon_min, lat_max=lat_max, lon_max=lon_max, upsert=True)
def test_get_voronoi_polygons(self): points = { 'Colombo': [79.8653, 6.898158], 'IBATTARA3': [79.86, 6.89], 'Isurupaya': [79.92, 6.89], 'Borella': [79.86, 6.93, ], 'Kompannaveediya': [79.85, 6.92], } shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp') out = tempfile.mkdtemp(prefix='voronoi_') result = get_voronoi_polygons(points, shp, ['OBJECTID', 1], output_shape_file=os.path.join(out, 'out.shp')) print(result)
def test_extract_kelani_basin_rainfall_flo2d(self): wrf_output_dir = tempfile.mkdtemp(prefix='flo2d_') files = ['wrfout_d03_2017-12-09_18:00:00_rf', 'wrfout_d03_2017-12-10_18:00:00_rf', 'wrfout_d03_2017-12-11_18:00:00_rf'] run_prefix = 'wrf0' for f in files: out_dir = utils.create_dir_if_not_exists( os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000'), 'wrf')) shutil.copy2(res_mgr.get_resource_path('test/%s' % f), out_dir) run_date = dt.datetime.strptime('2017-12-11_18:00', '%Y-%m-%d_%H:%M') now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*']) prev_1 = '_'.join([run_prefix, (run_date - dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*']) prev_2 = '_'.join([run_prefix, (run_date - dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*']) d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'wrf', 'wrfout_d03_*'))[0] d03_nc_f_prev_1 = glob.glob(os.path.join(wrf_output_dir, prev_1, 'wrf', 'wrfout_d03_*'))[0] d03_nc_f_prev_2 = glob.glob(os.path.join(wrf_output_dir, prev_2, 'wrf', 'wrfout_d03_*'))[0] kelani_basin_flo2d_file = res_mgr.get_resource_path('extraction/local/kelani_basin_points_250m.txt') extract_kelani_basin_rainfall_flo2d(d03_nc_f, [d03_nc_f_prev_1, d03_nc_f_prev_2], os.path.join(wrf_output_dir, now, 'klb_flo2d'), kelani_basin_file=kelani_basin_flo2d_file, )
def test_is_inside_polygon(self): points = { 'Colombo': [79.8653, 6.898158], 'IBATTARA3': [79.86, 6.89], 'Isurupaya': [79.92, 6.89], 'Borella': [79.86, 6.93, ], 'Kompannaveediya': [79.85, 6.92], } shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp') result = get_voronoi_polygons(points, shp, ['OBJECTID', 1]) print(result) for k in points.keys(): pp = is_inside_geo_df(result, points[k][0], points[k][1]) print(points[k], pp) self.assertEqual(pp, k)
def create_rainfall_for_mike21_obs(d0_rf_file, adapter, obs_stations, output_dir, start_ts, duration_days=None, kelani_lower_basin_shp=None): if kelani_lower_basin_shp is None: kelani_lower_basin_shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp') if duration_days is None: duration_days = (2, 3) obs_start = dt.datetime.strptime(start_ts, '%Y-%m-%d_%H:%M') - dt.timedelta(days=duration_days[0]) obs_end = dt.datetime.strptime(start_ts, '%Y-%m-%d_%H:%M') # forecast_end = dt.datetime.strptime(start_ts, '%Y-%m-%d_%H:%M') + dt.timedelta(days=duration_days[1]) obs = _get_observed_precip(obs_stations, obs_start, obs_end, duration_days, adapter) thess_poly = spatial_utils.get_voronoi_polygons(obs_stations, kelani_lower_basin_shp, add_total_area=False) observed = None for i, _id in enumerate(thess_poly['id']): if observed is not None: observed = observed + obs[_id].astype(float) * thess_poly['area'][i] else: observed = obs[_id].astype(float) * thess_poly['area'][i] observed = observed / sum(thess_poly['area']) d0 = np.genfromtxt(d0_rf_file, dtype=str) t0 = dt.datetime.strptime(' '.join(d0[0][:-1]), '%Y-%m-%d %H:%M:%S') t1 = dt.datetime.strptime(' '.join(d0[1][:-1]), '%Y-%m-%d %H:%M:%S') res_min = int((t1 - t0).total_seconds() / 60) # prev_output = np.append(prev_output, d0, axis=0) out_file = os.path.join(utils.create_dir_if_not_exists(output_dir), 'rf_mike21_obs.txt') with open(out_file, 'w') as out_f: for index in observed.index: out_f.write('%s:00\t%.4f\n' % (index, observed.precip[index])) forecast_start_idx = int( np.where((d0[:, 0] == obs_end.strftime('%Y-%m-%d')) & (d0[:, 1] == obs_end.strftime('%H:%M:%S')))[0]) # note: no need to convert to utc as rf_mike21.txt has times in LK for i in range(forecast_start_idx + 1, int(24 * 60 * duration_days[1] / res_min)): if i < len(d0): out_f.write('%s %s\t%s\n' % (d0[i][0], d0[i][1], d0[i][2])) else: out_f.write('%s\t0.0\n' % (obs_end + dt.timedelta(hours=i - forecast_start_idx - 1)).strftime( '%Y-%m-%d %H:%M:%S'))
def extract_weather_stations(nc_f, wrf_output, weather_stations=None, curw_db_adapter=None, curw_db_upsert=False, run_prefix='WRF', run_name='Cloud-1'): if weather_stations is None: weather_stations = res_mgr.get_resource_path('extraction/local/kelani_basin_stations.txt') nc_fid = Dataset(nc_f, 'r') times_len, times = ext_utils.extract_time_data(nc_f) prefix = 'stations_rf' stations_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix)) stations_rf = {} with TemporaryDirectory(prefix=prefix) as temp_dir: with open(weather_stations, 'r') as csvfile: stations = csv.reader(csvfile, delimiter=' ') for row in stations: logging.info(' '.join(row)) lon = row[1] lat = row[2] station_prcp = nc_fid.variables['RAINC'][:, lat, lon] + nc_fid.variables['RAINNC'][:, lat, lon] station_diff = ext_utils.get_two_element_average(station_prcp) stations_rf[row[0]] = [] station_file_path = os.path.join(temp_dir, row[0] + '_%s.txt' % prefix) with open(station_file_path, 'w') as station_file: for t in range(0, len(times) - 1): t_str = ( utils.datetime_utc_to_lk(dt.datetime.strptime(times[t], '%Y-%m-%d_%H:%M:%S'), shift_mins=30)).strftime('%Y-%m-%d %H:%M:%S') station_file.write('%s\t%.4f\n' % (t_str, station_diff[t])) stations_rf[row[0]].append([t_str, station_diff[t]]) utils.move_files_with_prefix(temp_dir, '*.txt', stations_dir) if curw_db_adapter is not None: logging.info('Pushing data to the db...') ext_utils.push_rainfall_to_db(curw_db_adapter, stations_rf, upsert=curw_db_upsert, name=run_name, source=run_prefix) else: logging.info('curw_db_adapter not available. Unable to push data!') nc_fid.close()
def extract_metro_col_rf_for_mike21(nc_f, output_dir, prev_rf_files=None, points_file=None): if not prev_rf_files: prev_rf_files = [] if not points_file: points_file = res_mgr.get_resource_path('extraction/local/metro_col_sub_catch_centroids.txt') points = np.genfromtxt(points_file, delimiter=',', names=True, dtype=None) point_prcp = ext_utils.extract_points_array_rf_series(nc_f, points) t0 = dt.datetime.strptime(point_prcp['Times'][0], '%Y-%m-%d %H:%M:%S') t1 = dt.datetime.strptime(point_prcp['Times'][1], '%Y-%m-%d %H:%M:%S') res_min = int((t1 - t0).total_seconds() / 60) lines_per_day = int(24 * 60 / res_min) prev_days = len(prev_rf_files) output = None for i in range(prev_days): if prev_rf_files[prev_days - 1 - i] is not None: if output is not None: output = np.append(output, ext_utils.extract_points_array_rf_series(prev_rf_files[prev_days - 1 - i], points)[ :lines_per_day], axis=0) else: output = ext_utils.extract_points_array_rf_series(prev_rf_files[prev_days - 1 - i], points)[ :lines_per_day] else: output = None # if any of the previous files are missing, skip prepending past data to the forecast break if output is not None: output = np.append(output, point_prcp, axis=0) else: output = point_prcp fmt = '%s' for _ in range(len(output[0]) - 1): fmt = fmt + ',%g' header = ','.join(output.dtype.names) utils.create_dir_if_not_exists(output_dir) np.savetxt(os.path.join(output_dir, 'met_col_rf_mike21.txt'), output, fmt=fmt, delimiter=',', header=header, comments='', encoding='utf-8')
def test_get_voronoi_polygons_kub(self): points = { 'Colombo': [79.8653, 6.898158], 'IBATTARA3': [79.86, 6.89], 'Isurupaya': [79.92, 6.89], 'Daraniyagala': [80.33805556, 6.924444444], 'Glencourse': [80.20305556, 6.978055556], 'Hanwella': [80.08166667, 6.909722222], 'Holombuwa': [80.26480556, 7.185166667], 'Kitulgala': [80.41777778, 6.989166667], 'Borella': [79.86, 6.93, ], 'Kompannaveediya': [79.85, 6.92], } shp = res_mgr.get_resource_path('extraction/shp/kelani-upper-basin.shp') out = tempfile.mkdtemp(prefix='voronoi_') print(out) result = get_voronoi_polygons(points, shp, ['OBJECTID', 1], output_shape_file=os.path.join(out, 'out.shp')) print(result)
def extract_weather_stations2(nc_f, wrf_output, weather_stations=None, curw_db_adapter=None, curw_db_upsert=False, run_prefix='WRF', run_name='Cloud-1'): if weather_stations is None: weather_stations = res_mgr.get_resource_path('extraction/local/wrf_stations.txt') points = np.genfromtxt(weather_stations, delimiter=',', names=True, dtype=None) point_prcp = ext_utils.extract_points_array_rf_series(nc_f, points) t0 = dt.datetime.strptime(point_prcp['Times'][0], '%Y-%m-%d %H:%M:%S') t1 = dt.datetime.strptime(point_prcp['Times'][1], '%Y-%m-%d %H:%M:%S') res_min = int((t1 - t0).total_seconds() / 60) prefix = 'stations_rf' stations_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix)) stations_rf = {} with TemporaryDirectory(prefix=prefix) as temp_dir: for point in points: logging.info(str(point)) station_name = point[0].decode() stations_rf[station_name] = [] station_file_path = os.path.join(temp_dir, station_name + '_%s.txt' % prefix) with open(station_file_path, 'w') as station_file: for t in range(0, len(point_prcp)): station_file.write('%s\t%.4f\n' % (point_prcp['Times'][t], point_prcp[station_name][t])) stations_rf[station_name].append([point_prcp['Times'][t], point_prcp[station_name][t]]) utils.move_files_with_prefix(temp_dir, '*.txt', stations_dir) if curw_db_adapter is not None: logging.info('Pushing data to the db...') ext_utils.push_rainfall_to_db(curw_db_adapter, stations_rf, upsert=curw_db_upsert, name=run_name, source=run_prefix) else: logging.info('curw_db_adapter not available. Unable to push data!')
def test_extract_metro_col_rf_for_mike21(self): out_dir = tempfile.mkdtemp(prefix='met_col_mike21') prev = [res_mgr.get_resource_path('test/wrfout_d03_2017-12-10_18:00:00_rf'), res_mgr.get_resource_path('test/wrfout_d03_2017-12-09_18:00:00_rf')] extract_metro_col_rf_for_mike21(res_mgr.get_resource_path('test/wrfout_d03_2017-12-11_18:00:00_rf'), out_dir, prev_rf_files=prev)
def extract_kelani_basin_rainfall_flo2d_with_obs(nc_f, adapter, obs_stations, output_dir, start_ts_lk, duration_days=None, output_prefix='RAINCELL', kelani_lower_basin_points=None, kelani_lower_basin_shp=None): """ check test_extract_kelani_basin_rainfall_flo2d_obs test case :param nc_f: file path of the wrf output :param adapter: :param obs_stations: dict of stations. {station_name: [lon, lat, name variable, nearest wrf point station name]} :param output_dir: :param start_ts_lk: start time of the forecast/ end time of the observations :param duration_days: (optional) a tuple (observation days, forecast days) default (2,3) :param output_prefix: (optional) output file name of the RAINCELL file. ex: output_prefix=RAINCELL-150m --> RAINCELL-150m.DAT :param kelani_lower_basin_points: (optional) :param kelani_lower_basin_shp: (optional) :return: """ if duration_days is None: duration_days = (2, 3) if kelani_lower_basin_points is None: kelani_lower_basin_points = res_mgr.get_resource_path('extraction/local/kelani_basin_points_250m.txt') if kelani_lower_basin_shp is None: kelani_lower_basin_shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp') points = np.genfromtxt(kelani_lower_basin_points, delimiter=',') kel_lon_min = np.min(points, 0)[1] kel_lat_min = np.min(points, 0)[2] kel_lon_max = np.max(points, 0)[1] kel_lat_max = np.max(points, 0)[2] diff, kel_lats, kel_lons, times = ext_utils.extract_area_rf_series(nc_f, kel_lat_min, kel_lat_max, kel_lon_min, kel_lon_max) def get_bins(arr): sz = len(arr) return (arr[1:sz - 1] + arr[0:sz - 2]) / 2 lat_bins = get_bins(kel_lats) lon_bins = get_bins(kel_lons) t0 = dt.datetime.strptime(times[0], '%Y-%m-%d_%H:%M:%S') t1 = dt.datetime.strptime(times[1], '%Y-%m-%d_%H:%M:%S') utils.create_dir_if_not_exists(output_dir) obs_start = dt.datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') - dt.timedelta(days=duration_days[0]) obs_end = dt.datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') forecast_end = dt.datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') + dt.timedelta(days=duration_days[1]) obs = _get_observed_precip(obs_stations, obs_start, obs_end, duration_days, adapter) thess_poly = spatial_utils.get_voronoi_polygons(obs_stations, kelani_lower_basin_shp, add_total_area=False) output_file_path = os.path.join(output_dir, output_prefix + '.DAT') # update points array with the thessian polygon idx point_thess_idx = [] for point in points: point_thess_idx.append(spatial_utils.is_inside_geo_df(thess_poly, lon=point[1], lat=point[2])) pass with open(output_file_path, 'w') as output_file: res_mins = int((t1 - t0).total_seconds() / 60) data_hours = int(sum(duration_days) * 24 * 60 / res_mins) start_ts_lk = obs_start.strftime('%Y-%m-%d %H:%M:%S') end_ts = forecast_end.strftime('%Y-%m-%d %H:%M:%S') output_file.write("%d %d %s %s\n" % (res_mins, data_hours, start_ts_lk, end_ts)) for t in range(int(24 * 60 * duration_days[0] / res_mins) + 1): for i, point in enumerate(points): rf = float(obs[point_thess_idx[i]].values[t]) if point_thess_idx[i] is not None else 0 output_file.write('%d %.1f\n' % (point[0], rf)) forecast_start_idx = int( np.where(times == utils.datetime_lk_to_utc(obs_end, shift_mins=30).strftime('%Y-%m-%d_%H:%M:%S'))[0]) for t in range(int(24 * 60 * duration_days[1] / res_mins) - 1): for point in points: rf_x = np.digitize(point[1], lon_bins) rf_y = np.digitize(point[2], lat_bins) if t + forecast_start_idx + 1 < len(times): output_file.write('%d %.1f\n' % (point[0], diff[t + forecast_start_idx + 1, rf_y, rf_x])) else: output_file.write('%d %.1f\n' % (point[0], 0))
def __init__(self): self.shape_file = res_mgr.get_resource_path( 'extraction/shp/kub-wgs84/kub-wgs84.shp') self.output_dir = 'kub_observation_mean/output_shape/kub' self.percentage_factor = 100
def test_extract_metro_colombo(self): # adapter = ext_utils.get_curw_adapter() adapter = None out_dir = tempfile.mkdtemp(prefix='metro_col_') extract_metro_colombo(res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00'), out_dir, out_dir, curw_db_adapter=adapter, curw_db_upsert=True)
def test_extract_weather_stations(self): adapter = ext_utils.get_curw_adapter() out_dir = tempfile.mkdtemp(prefix='stations_') extract_weather_stations(res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00'), out_dir, curw_db_adapter=adapter, curw_db_upsert=True)
def run(run_id, wrf_config_dict, db_config_dict, upsert=False, run_name='Cloud-1', data_type=Data.data, procedures=sys.maxsize): """ Procedure #1: Extracting data from D03 Procedure #2: Extract rainfall data for the metro colombo area Procedure #3: Extract weather station rainfall Procedure #4: Extract Kelani upper Basin mean rainfall Procedure #5: Extract Kelani lower Basin mean rainfall Procedure #6: Create plots for D03 Procedure #7: Extract Kelani lower Basin rainfall for FLO2D Procedure #8: Extract Kelani lower Basin rainfall for MIKE21 Procedure #9: Create plots for D01 Procedure #10: Extract rf data from metro col for MIKE21 """ def _nth_bit(a, n): return (a >> n) & 1 logging.info('**** Extracting data from WRF **** Run ID: ' + run_id) run_prefix = run_id.split('_')[0] config = get_wrf_config(**wrf_config_dict) config.set('run_id', run_id) output_dir_base = os.path.join(config.get('nfs_dir'), 'results') run_output_dir = os.path.join(output_dir_base, run_id) wrf_output_dir = os.path.join(run_output_dir, 'wrf') logging.info('WRF output dir: ' + wrf_output_dir) db_adapter = ext_utils.get_curw_adapter( mysql_config=db_config_dict) if db_config_dict else None logging.info('Creating temp file space') with TemporaryDirectory(prefix='wrfout_') as temp_dir: try: logging.info('Copying wrfout_D03* to temp_dir ' + temp_dir) d03_nc_f = shutil.copy2( glob.glob(os.path.join(wrf_output_dir, 'wrfout_d03_*'))[0], temp_dir) if _nth_bit(procedures, 1): logging.info('Procedure #1: Extracting data from ' + d03_nc_f) try: logging.info( 'Extract WRF data points in the Kelani and Kalu basins' ) lon_min, lat_min, lon_max, lat_max = constants.KELANI_KALU_BASIN_EXTENT extractor.push_wrf_rainfall_to_db( d03_nc_f, curw_db_adapter=db_adapter, lat_min=lat_min, lon_min=lon_min, lat_max=lat_max, lon_max=lon_max, run_prefix=run_prefix, upsert=upsert) except Exception as e: logging.error( 'Extract WRF data points in the Kelani and Kalu basins FAILED: ' + str(e) + '\n' + traceback.format_exc()) if _nth_bit(procedures, 2): try: logging.info( 'Procedure #2: Extract rainfall data for the metro colombo area' ) basin_rf = extractor.extract_metro_colombo( d03_nc_f, run_output_dir, output_dir_base, curw_db_adapter=db_adapter, run_prefix=run_prefix, run_name=run_name, curw_db_upsert=upsert) logging.info('Basin rainfall' + str(basin_rf)) except Exception as e: logging.error( 'Extract rainfall data for the metro colombo area FAILED: ' + str(e) + '\n' + traceback.format_exc()) if _nth_bit(procedures, 3): try: logging.info( 'Procedure #3: Extract weather station rainfall') extractor.extract_weather_stations( d03_nc_f, run_output_dir, curw_db_adapter=db_adapter, curw_db_upsert=upsert, run_prefix=run_prefix, run_name=run_name) except Exception as e: logging.error('Extract weather station rainfall FAILED: ' + str(e) + '\n' + traceback.format_exc()) if _nth_bit(procedures, 4): try: logging.info( 'Procedure #4: Extract Kelani upper Basin mean rainfall' ) basin_shp_file = res_mgr.get_resource_path( 'extraction/shp/kelani-upper-basin.shp') extractor.extract_mean_rainfall_from_shp_file( d03_nc_f, run_output_dir, 'kub_mean_rf', 'kub_mean', basin_shp_file, constants.KELANI_UPPER_BASIN_EXTENT, curw_db_adapter=db_adapter, run_prefix=run_prefix, run_name=run_name, curw_db_upsert=upsert) except Exception as e: logging.error( 'Extract Kelani upper Basin mean rainfall FAILED: ' + str(e) + '\n' + traceback.format_exc()) if _nth_bit(procedures, 5): try: logging.info( 'Procedure #5: Extract Kelani lower Basin mean rainfall' ) basin_shp_file = res_mgr.get_resource_path( 'extraction/shp/klb-wgs84/klb-wgs84.shp') extractor.extract_mean_rainfall_from_shp_file( d03_nc_f, run_output_dir, 'klb_mean_rf', 'klb_mean', basin_shp_file, constants.KELANI_LOWER_BASIN_EXTENT, curw_db_adapter=db_adapter, run_prefix=run_prefix, run_name=run_name, curw_db_upsert=upsert) except Exception as e: logging.error( 'Extract Kelani lower Basin mean rainfall FAILED: ' + str(e) + '\n' + traceback.format_exc()) if _nth_bit(procedures, 6): try: logging.info('Procedure #6: Create plots for D03') lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_EXTENT extractor.create_rf_plots_wrf(d03_nc_f, os.path.join( run_output_dir, 'plots_D03'), output_dir_base, lat_min=lat_min, lon_min=lon_min, lat_max=lat_max, lon_max=lon_max, run_prefix=run_prefix) except Exception as e: logging.error('Create plots for D03 FAILED: ' + str(e) + '\n' + traceback.format_exc()) if _nth_bit(procedures, 7): try: logging.info( 'Procedure #7: Extract Kelani lower Basin rainfall for FLO2D' ) run_date = dt.datetime.strptime(config.get('start_date'), '%Y-%m-%d_%H:%M') prev_days = 5 d03_nc_f_prev = [] for i in range(prev_days): prev = '_'.join([ run_prefix, (run_date - dt.timedelta(days=i + 1) ).strftime('%Y-%m-%d_%H:%M'), '*' ]) try: d03_nc_f_prev.append( shutil.copy2( glob.glob( os.path.join(output_dir_base, prev, 'wrf', 'wrfout_d03_*'))[0], temp_dir)) except IndexError as e: logging.warning( 'File for %s not found. Filling with 0s: %s' % (prev, str(e))) d03_nc_f_prev.append(None) logging.info('250m model') kelani_basin_flo2d_file = res_mgr.get_resource_path( 'extraction/local/kelani_basin_points_250m.txt') extractor.extract_kelani_basin_rainfall_flo2d( d03_nc_f, d03_nc_f_prev[:2], os.path.join(run_output_dir, 'klb_flo2d'), kelani_basin_file=kelani_basin_flo2d_file) logging.info('150m model') kelani_basin_flo2d_file = res_mgr.get_resource_path( 'extraction/local/klb_glecourse_points_150m.txt') extractor.extract_kelani_basin_rainfall_flo2d( d03_nc_f, d03_nc_f_prev, os.path.join(run_output_dir, 'klb_flo2d'), kelani_basin_file=kelani_basin_flo2d_file, output_prefix='RAINCELL_150m', target_rfs=[]) except Exception as e: logging.error( 'Extract Kelani lower Basin mean rainfall for FLO2D FAILED: ' + str(e) + '\n' + traceback.format_exc()) if _nth_bit(procedures, 8): try: logging.info( 'Procedure #8: Extract Kelani lower Basin rainfall for MIKE21' ) run_date = dt.datetime.strptime(config.get('start_date'), '%Y-%m-%d_%H:%M') now = shutil.copy2( os.path.join(output_dir_base, config.get('run_id'), 'klb_mean_rf', 'klb_mean_rf.txt'), os.path.join(temp_dir, 'klb.txt')) prev_1 = '_'.join([ run_prefix, (run_date - dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*' ]) prev_2 = '_'.join([ run_prefix, (run_date - dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*' ]) try: klb_prev_1 = utils.copy_if_not_exists( glob.glob( os.path.join(output_dir_base, prev_1, 'klb_mean_rf', 'klb_mean_rf.txt'))[0], os.path.join(temp_dir, 'klb1.txt')) except IndexError as e: logging.warning( 'File for %s not found. Filling with 0s: %s' % (prev_1, str(e))) klb_prev_1 = None try: klb_prev_2 = utils.copy_if_not_exists( glob.glob( os.path.join(output_dir_base, prev_2, 'klb_mean_rf', 'klb_mean_rf.txt'))[0], os.path.join(temp_dir, 'klb2.txt')) except IndexError as e: logging.warning( 'File for %s not found. Filling with 0s: %s' % (prev_2, str(e))) klb_prev_2 = None extractor.create_rainfall_for_mike21( now, [klb_prev_1, klb_prev_2], os.path.join(run_output_dir, 'klb_mike21')) except Exception as e: logging.error( 'Extract Kelani lower Basin mean rainfall for MIKE21 FAILED: ' + str(e) + '\n' + traceback.format_exc()) if _nth_bit(procedures, 10): try: logging.info( 'Procedure #10: Extract rf data from metro col for MIKE21' ) run_date = dt.datetime.strptime(config.get('start_date'), '%Y-%m-%d_%H:%M') prev_1 = '_'.join([ run_prefix, (run_date - dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*' ]) prev_2 = '_'.join([ run_prefix, (run_date - dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*' ]) try: d03_nc_f_prev_1 = utils.copy_if_not_exists( glob.glob( os.path.join(output_dir_base, prev_1, 'wrf', 'wrfout_d03_*'))[0], temp_dir) except IndexError as e: logging.warning( 'File for %s not found. Filling with 0s: %s' % (prev_1, str(e))) d03_nc_f_prev_1 = None try: d03_nc_f_prev_2 = utils.copy_if_not_exists( glob.glob( os.path.join(output_dir_base, prev_2, 'wrf', 'wrfout_d03_*'))[0], temp_dir) except IndexError as e: logging.warning( 'File for %s not found. Filling with 0s: %s' % (prev_2, str(e))) d03_nc_f_prev_2 = None prev = [d03_nc_f_prev_1, d03_nc_f_prev_2] extractor.extract_metro_col_rf_for_mike21( d03_nc_f, os.path.join(run_output_dir, 'met_col_mike21'), prev_rf_files=prev) except Exception as e: logging.error( 'Extract rf data from metro col for MIKE21: ' + str(e) + '\n' + traceback.format_exc()) except Exception as e: logging.error( 'Copying wrfout_d03_* to temp_dir %s FAILED: %s\n%s' % (temp_dir, str(e), traceback.format_exc())) try: d01_nc_f = shutil.copy2( glob.glob(os.path.join(wrf_output_dir, 'wrfout_d01_*'))[0], temp_dir) logging.info('Extracting data from ' + d01_nc_f) if _nth_bit(procedures, 9): try: logging.info('Procedure #9: Create plots for D01') lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_D01_EXTENT extractor.create_rf_plots_wrf(d01_nc_f, os.path.join( run_output_dir, 'plots_D01'), output_dir_base, lat_min=lat_min, lon_min=lon_min, lat_max=lat_max, lon_max=lon_max, run_prefix=run_prefix) except Exception as e: logging.error('Create plots for D01 FAILED: ' + str(e) + '\n' + traceback.format_exc()) except Exception as e: logging.error( 'Copying wrfout_d01_* to temp_dir %s FAILED: %s\n%s' % (temp_dir, str(e), traceback.format_exc())) logging.info('**** Extracting data from WRF **** Run ID: ' + run_id + ' COMPLETED!')
def extract_kelani_basin_rainfall_flo2d(nc_f, nc_f_prev_days, output_dir, avg_basin_rf=1.0, kelani_basin_file=None, target_rfs=None, output_prefix='RAINCELL'): """ :param output_prefix: :param nc_f: :param nc_f_prev_days: :param output_dir: :param avg_basin_rf: :param kelani_basin_file: :param target_rfs: :return: """ if target_rfs is None: target_rfs = [100, 150, 200, 250, 300] if kelani_basin_file is None: kelani_basin_file = res_mgr.get_resource_path('extraction/local/kelani_basin_points_250m.txt') points = np.genfromtxt(kelani_basin_file, delimiter=',') kel_lon_min = np.min(points, 0)[1] kel_lat_min = np.min(points, 0)[2] kel_lon_max = np.max(points, 0)[1] kel_lat_max = np.max(points, 0)[2] diff, kel_lats, kel_lons, times = ext_utils.extract_area_rf_series(nc_f, kel_lat_min, kel_lat_max, kel_lon_min, kel_lon_max) def get_bins(arr): sz = len(arr) return (arr[1:sz - 1] + arr[0:sz - 2]) / 2 lat_bins = get_bins(kel_lats) lon_bins = get_bins(kel_lons) t0 = dt.datetime.strptime(times[0], '%Y-%m-%d_%H:%M:%S') t1 = dt.datetime.strptime(times[1], '%Y-%m-%d_%H:%M:%S') t_end = dt.datetime.strptime(times[-1], '%Y-%m-%d_%H:%M:%S') utils.create_dir_if_not_exists(output_dir) prev_diff = [] prev_days = len(nc_f_prev_days) for i in range(prev_days): if nc_f_prev_days[i]: p_diff, _, _, _ = ext_utils.extract_area_rf_series(nc_f_prev_days[i], kel_lat_min, kel_lat_max, kel_lon_min, kel_lon_max) prev_diff.append(p_diff) else: prev_diff.append(None) def write_forecast_to_raincell_file(output_file_path, alpha): with open(output_file_path, 'w') as output_file: res_mins = int((t1 - t0).total_seconds() / 60) data_hours = int(len(times) + prev_days * 24 * 60 / res_mins) start_ts = utils.datetime_utc_to_lk(t0 - dt.timedelta(days=prev_days), shift_mins=30).strftime( '%Y-%m-%d %H:%M:%S') end_ts = utils.datetime_utc_to_lk(t_end, shift_mins=30).strftime('%Y-%m-%d %H:%M:%S') output_file.write("%d %d %s %s\n" % (res_mins, data_hours, start_ts, end_ts)) for d in range(prev_days): for t in range(int(24 * 60 / res_mins)): for point in points: rf_x = np.digitize(point[1], lon_bins) rf_y = np.digitize(point[2], lat_bins) if prev_diff[prev_days - 1 - d] is not None: output_file.write('%d %.1f\n' % (point[0], prev_diff[prev_days - 1 - d][t, rf_y, rf_x])) else: output_file.write('%d %.1f\n' % (point[0], 0)) for t in range(len(times)): for point in points: rf_x = np.digitize(point[1], lon_bins) rf_y = np.digitize(point[2], lat_bins) if t < int(24 * 60 / res_mins): output_file.write('%d %.1f\n' % (point[0], diff[t, rf_y, rf_x] * alpha)) else: output_file.write('%d %.1f\n' % (point[0], diff[t, rf_y, rf_x])) with TemporaryDirectory(prefix='curw_raincell') as temp_dir: raincell_temp = os.path.join(temp_dir, output_prefix + '.DAT') write_forecast_to_raincell_file(raincell_temp, 1) for target_rf in target_rfs: write_forecast_to_raincell_file('%s.%d' % (raincell_temp, target_rf), target_rf / avg_basin_rf) utils.create_zip_with_prefix(temp_dir, output_prefix + '.DAT*', os.path.join(temp_dir, output_prefix + '.zip'), clean_up=True) utils.move_files_with_prefix(temp_dir, output_prefix + '.zip', utils.create_dir_if_not_exists(output_dir))