def test_create_rainfall_for_mike21(self):
wrf_output_dir = tempfile.mkdtemp(prefix='mike21_')
run_date = dt.datetime.strptime('2017-12-11_18:00', '%Y-%m-%d_%H:%M')
basin_shp_file = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp')
files = ['wrfout_d03_2017-12-09_18:00:00_rf', 'wrfout_d03_2017-12-10_18:00:00_rf',
'wrfout_d03_2017-12-11_18:00:00_rf']
run_prefix = 'wrf0'
for f in files:
d03_nc_f = res_mgr.get_resource_path('test/%s' % f)
out_dir = utils.create_dir_if_not_exists(
os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000')))
extract_mean_rainfall_from_shp_file(d03_nc_f, out_dir, 'klb_mean_rf', 'klb_mean',
basin_shp_file, constants.KELANI_LOWER_BASIN_EXTENT)
now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*'])
prev_1 = '_'.join([run_prefix, (run_date - dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*'])
prev_2 = '_'.join([run_prefix, (run_date - dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*'])
d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'klb_mean_rf', 'klb_mean_rf.txt'))[0]
d03_nc_f_prev_1 = glob.glob(os.path.join(wrf_output_dir, prev_1, 'klb_mean_rf', 'klb_mean_rf.txt'))[0]
d03_nc_f_prev_2 = glob.glob(os.path.join(wrf_output_dir, prev_2, 'klb_mean_rf', 'klb_mean_rf.txt'))[0]
create_rainfall_for_mike21(d03_nc_f, [d03_nc_f_prev_1, d03_nc_f_prev_2],
os.path.join(wrf_output_dir, now, 'mike_21'))
def test_extract_mean_rainfall_from_shp_file(self):
adapter = ext_utils.get_curw_adapter()
basin_shp_file = res_mgr.get_resource_path('extraction/shp/kelani-upper-basin.shp')
basin_extent = constants.KELANI_UPPER_BASIN_EXTENT
extract_mean_rainfall_from_shp_file(res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00'),
tempfile.mkdtemp(prefix='temp_'), 'kub_mean_rf', 'kub_mean', basin_shp_file,
basin_extent, curw_db_adapter=adapter, curw_db_upsert=True)
def test_extract_kelani_basin_rainfall_flo2d_obs_150m(self):
adapter = ext_utils.get_curw_adapter(mysql_config_path='/home/curw/Desktop/2018-05/mysql.json')
wrf_output_dir = tempfile.mkdtemp(prefix='flo2d_obs_')
files = ['wrfout_d03_2018-05-23_18:00:00_rf']
run_prefix = 'wrf0'
for f in files:
out_dir = utils.create_dir_if_not_exists(
os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000'), 'wrf'))
shutil.copy2('/home/curw/Desktop/2018-05/2018-05-23_18:00/wrf0/%s' % f, out_dir)
run_date = dt.datetime.strptime('2018-05-23_18:00', '%Y-%m-%d_%H:%M')
now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*'])
d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'wrf', 'wrfout_d03_*'))[0]
obs_stations = {'Kottawa North Dharmapala School': [79.95818, 6.865576, 'A&T Labs', 'wrf_79.957123_6.859688'],
'IBATTARA2': [79.919, 6.908, 'CUrW IoT', 'wrf_79.902664_6.913757'],
'Malabe': [79.95738, 6.90396, 'A&T Labs', 'wrf_79.957123_6.913757'],
# 'Mutwal': [79.8609, 6.95871, 'A&T Labs', 'wrf_79.875435_6.967812'],
'Glencourse': [80.20305, 6.97805, 'Irrigation Department', 'wrf_80.202187_6.967812'],
# 'Waga': [80.11828, 6.90678, 'A&T Labs', 'wrf_80.120499_6.913757'],
}
start_ts = '2018-05-26_00:00'
kelani_lower_basin_points = res_mgr.get_resource_path('extraction/local/klb_glecourse_points_150m.txt')
kelani_lower_basin_shp = res_mgr.get_resource_path('extraction/shp/klb_glencourse/klb_glencourse.shp')
duration_days = (8, 0)
extract_kelani_basin_rainfall_flo2d_with_obs(d03_nc_f, adapter, obs_stations,
os.path.join(wrf_output_dir, now, 'klb_flo2d'), start_ts,
duration_days=duration_days,
kelani_lower_basin_shp=kelani_lower_basin_shp,
kelani_lower_basin_points=kelani_lower_basin_points,
output_prefix='RAINCELL_150m')
def test_create_rf_plots_wrf(self):
out_base_dir = tempfile.mkdtemp(prefix='rf_plots_')
out_dir = os.path.join(out_base_dir, 'plots_D03')
create_rf_plots_wrf(res_mgr.get_resource_path('test/wrfout_d03_2017-12-09_18:00:00_rf'), out_dir, out_base_dir)
out_dir = os.path.join(out_base_dir, 'plots_D01')
lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_D01_EXTENT
create_rf_plots_wrf(res_mgr.get_resource_path('test/wrfout_d01_2017-12-09_18:00:00_rf'), out_dir, out_base_dir,
lat_min=lat_min, lon_min=lon_min, lat_max=lat_max, lon_max=lon_max)
def test_extract_point_rf_series(self):
points = np.genfromtxt(res_mgr.get_resource_path(
'extraction/local/metro_col_sub_catch_centroids.txt'),
delimiter=',',
names=True,
dtype=None)
extract_points_array_rf_series(
res_mgr.get_resource_path(
'/test/wrfout_d03_2017-12-09_18:00:00_rf'), points)
def test_all(self):
logging.basicConfig(
level=logging.DEBUG,
format=
'%(asctime)s %(threadName)s %(module)s %(levelname)s %(message)s')
wrf_home = tempfile.mkdtemp(prefix='wrf_test_')
nfs_dir = os.path.join(wrf_home, 'nfs')
run_id = 'WRF_test0'
output_dir = os.path.join(nfs_dir, 'results', run_id, 'wrf')
utils.create_dir_if_not_exists(output_dir)
shutil.copy(
res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00'),
output_dir)
wrf_conf_dict = {
"wrf_home": wrf_home,
"nfs_dir": nfs_dir,
"period": 0.25,
"start_date": "2017-10-02_12:00"
}
db_conf_dict = {
"host": "localhost",
"user": "******",
"password": "******",
"db": "testdb"
}
extract_data_wrf.run(run_id, wrf_conf_dict, db_conf_dict, upsert=True)
def test_compare_voronoi_polygons(self):
files = ['up_in_3_thie_join.shp', 'up_in_11_thie_join.shp']
out = tempfile.mkdtemp(prefix='voronoi_')
for f in files:
voronoi_shp_file = res_mgr.get_resource_path('test/shp/%s' % f)
area_shp_file = res_mgr.get_resource_path('extraction/shp/kub-wgs84/kub-wgs84.shp')
shape_df = gpd.GeoDataFrame.from_file(voronoi_shp_file)
points = {}
for i in range(len(shape_df)):
points[shape_df['OBJECTID'][i]] = [shape_df['x'][i], shape_df['y'][i]]
result = get_voronoi_polygons(points, area_shp_file, ['OBJECTID', 1],
output_shape_file=os.path.join(out, '%s_out.shp' % f))
for i in range(len(shape_df)):
self.assertAlmostEqual(result['area'][i], shape_df['Shape_Area'][i], places=4)
def replace_namelist_wps(wrf_config, start_date=None, end_date=None):
logging.info('Replacing namelist.wps...')
if os.path.exists(wrf_config.get('namelist_wps')):
f = wrf_config.get('namelist_wps')
else:
f = res_mgr.get_resource_path(
os.path.join('execution', constants.DEFAULT_NAMELIST_WPS_TEMPLATE))
dest = os.path.join(utils.get_wps_dir(wrf_config.get('wrf_home')),
'namelist.wps')
replace_file_with_values(wrf_config, f, dest, 'namelist_wps_dict',
start_date, end_date)
def get_curw_adapter(mysql_config_path=None):
if mysql_config_path is None:
mysql_config_path = res_mgr.get_resource_path(
'config/mysql_config.json')
with open(mysql_config_path) as data_file:
config = json.load(data_file)
return mysqladapter(host=config['host'],
user=config['user'],
password=config['password'],
db=config['db'])
def get_curw_adapter(mysql_config=None, mysql_config_path=None):
if mysql_config_path is None:
mysql_config_path = res_mgr.get_resource_path(
'config/mysql_config.json')
with open(mysql_config_path) as data_file:
config = json.load(data_file)
if mysql_config is not None and isinstance(mysql_config, dict):
config.update(mysql_config)
return MySQLAdapter(**config)
def test_push_wrf_rainfall_to_db(self):
config = {
"host": "localhost",
"user": "******",
"password": "******",
"db": "testdb"
}
adapter = ext_utils.get_curw_adapter(mysql_config=config)
nc_f = res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00')
lon_min, lat_min, lon_max, lat_max = constants.KELANI_KALU_BASIN_EXTENT
push_wrf_rainfall_to_db(nc_f, curw_db_adapter=adapter, lat_min=lat_min, lon_min=lon_min,
lat_max=lat_max, lon_max=lon_max, upsert=True)
def test_get_voronoi_polygons(self):
points = {
'Colombo': [79.8653, 6.898158],
'IBATTARA3': [79.86, 6.89],
'Isurupaya': [79.92, 6.89],
'Borella': [79.86, 6.93, ],
'Kompannaveediya': [79.85, 6.92],
}
shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp')
out = tempfile.mkdtemp(prefix='voronoi_')
result = get_voronoi_polygons(points, shp, ['OBJECTID', 1], output_shape_file=os.path.join(out, 'out.shp'))
print(result)
def test_extract_kelani_basin_rainfall_flo2d(self):
wrf_output_dir = tempfile.mkdtemp(prefix='flo2d_')
files = ['wrfout_d03_2017-12-09_18:00:00_rf', 'wrfout_d03_2017-12-10_18:00:00_rf',
'wrfout_d03_2017-12-11_18:00:00_rf']
run_prefix = 'wrf0'
for f in files:
out_dir = utils.create_dir_if_not_exists(
os.path.join(wrf_output_dir, f.replace('wrfout_d03', run_prefix).replace(':00_rf', '_0000'), 'wrf'))
shutil.copy2(res_mgr.get_resource_path('test/%s' % f), out_dir)
run_date = dt.datetime.strptime('2017-12-11_18:00', '%Y-%m-%d_%H:%M')
now = '_'.join([run_prefix, run_date.strftime('%Y-%m-%d_%H:%M'), '*'])
prev_1 = '_'.join([run_prefix, (run_date - dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*'])
prev_2 = '_'.join([run_prefix, (run_date - dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*'])
d03_nc_f = glob.glob(os.path.join(wrf_output_dir, now, 'wrf', 'wrfout_d03_*'))[0]
d03_nc_f_prev_1 = glob.glob(os.path.join(wrf_output_dir, prev_1, 'wrf', 'wrfout_d03_*'))[0]
d03_nc_f_prev_2 = glob.glob(os.path.join(wrf_output_dir, prev_2, 'wrf', 'wrfout_d03_*'))[0]
kelani_basin_flo2d_file = res_mgr.get_resource_path('extraction/local/kelani_basin_points_250m.txt')
extract_kelani_basin_rainfall_flo2d(d03_nc_f, [d03_nc_f_prev_1, d03_nc_f_prev_2],
os.path.join(wrf_output_dir, now, 'klb_flo2d'),
kelani_basin_file=kelani_basin_flo2d_file, )
def test_is_inside_polygon(self):
points = {
'Colombo': [79.8653, 6.898158],
'IBATTARA3': [79.86, 6.89],
'Isurupaya': [79.92, 6.89],
'Borella': [79.86, 6.93, ],
'Kompannaveediya': [79.85, 6.92],
}
shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp')
result = get_voronoi_polygons(points, shp, ['OBJECTID', 1])
print(result)
for k in points.keys():
pp = is_inside_geo_df(result, points[k][0], points[k][1])
print(points[k], pp)
self.assertEqual(pp, k)
def create_rainfall_for_mike21_obs(d0_rf_file, adapter, obs_stations, output_dir, start_ts, duration_days=None,
kelani_lower_basin_shp=None):
if kelani_lower_basin_shp is None:
kelani_lower_basin_shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp')
if duration_days is None:
duration_days = (2, 3)
obs_start = dt.datetime.strptime(start_ts, '%Y-%m-%d_%H:%M') - dt.timedelta(days=duration_days[0])
obs_end = dt.datetime.strptime(start_ts, '%Y-%m-%d_%H:%M')
# forecast_end = dt.datetime.strptime(start_ts, '%Y-%m-%d_%H:%M') + dt.timedelta(days=duration_days[1])
obs = _get_observed_precip(obs_stations, obs_start, obs_end, duration_days, adapter)
thess_poly = spatial_utils.get_voronoi_polygons(obs_stations, kelani_lower_basin_shp, add_total_area=False)
observed = None
for i, _id in enumerate(thess_poly['id']):
if observed is not None:
observed = observed + obs[_id].astype(float) * thess_poly['area'][i]
else:
observed = obs[_id].astype(float) * thess_poly['area'][i]
observed = observed / sum(thess_poly['area'])
d0 = np.genfromtxt(d0_rf_file, dtype=str)
t0 = dt.datetime.strptime(' '.join(d0[0][:-1]), '%Y-%m-%d %H:%M:%S')
t1 = dt.datetime.strptime(' '.join(d0[1][:-1]), '%Y-%m-%d %H:%M:%S')
res_min = int((t1 - t0).total_seconds() / 60)
# prev_output = np.append(prev_output, d0, axis=0)
out_file = os.path.join(utils.create_dir_if_not_exists(output_dir), 'rf_mike21_obs.txt')
with open(out_file, 'w') as out_f:
for index in observed.index:
out_f.write('%s:00\t%.4f\n' % (index, observed.precip[index]))
forecast_start_idx = int(
np.where((d0[:, 0] == obs_end.strftime('%Y-%m-%d')) & (d0[:, 1] == obs_end.strftime('%H:%M:%S')))[0])
# note: no need to convert to utc as rf_mike21.txt has times in LK
for i in range(forecast_start_idx + 1, int(24 * 60 * duration_days[1] / res_min)):
if i < len(d0):
out_f.write('%s %s\t%s\n' % (d0[i][0], d0[i][1], d0[i][2]))
else:
out_f.write('%s\t0.0\n' % (obs_end + dt.timedelta(hours=i - forecast_start_idx - 1)).strftime(
'%Y-%m-%d %H:%M:%S'))
def extract_weather_stations(nc_f, wrf_output, weather_stations=None, curw_db_adapter=None, curw_db_upsert=False,
run_prefix='WRF', run_name='Cloud-1'):
if weather_stations is None:
weather_stations = res_mgr.get_resource_path('extraction/local/kelani_basin_stations.txt')
nc_fid = Dataset(nc_f, 'r')
times_len, times = ext_utils.extract_time_data(nc_f)
prefix = 'stations_rf'
stations_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix))
stations_rf = {}
with TemporaryDirectory(prefix=prefix) as temp_dir:
with open(weather_stations, 'r') as csvfile:
stations = csv.reader(csvfile, delimiter=' ')
for row in stations:
logging.info(' '.join(row))
lon = row[1]
lat = row[2]
station_prcp = nc_fid.variables['RAINC'][:, lat, lon] + nc_fid.variables['RAINNC'][:, lat, lon]
station_diff = ext_utils.get_two_element_average(station_prcp)
stations_rf[row[0]] = []
station_file_path = os.path.join(temp_dir, row[0] + '_%s.txt' % prefix)
with open(station_file_path, 'w') as station_file:
for t in range(0, len(times) - 1):
t_str = (
utils.datetime_utc_to_lk(dt.datetime.strptime(times[t], '%Y-%m-%d_%H:%M:%S'),
shift_mins=30)).strftime('%Y-%m-%d %H:%M:%S')
station_file.write('%s\t%.4f\n' % (t_str, station_diff[t]))
stations_rf[row[0]].append([t_str, station_diff[t]])
utils.move_files_with_prefix(temp_dir, '*.txt', stations_dir)
if curw_db_adapter is not None:
logging.info('Pushing data to the db...')
ext_utils.push_rainfall_to_db(curw_db_adapter, stations_rf, upsert=curw_db_upsert, name=run_name,
source=run_prefix)
else:
logging.info('curw_db_adapter not available. Unable to push data!')
nc_fid.close()
def extract_metro_col_rf_for_mike21(nc_f, output_dir, prev_rf_files=None, points_file=None):
if not prev_rf_files:
prev_rf_files = []
if not points_file:
points_file = res_mgr.get_resource_path('extraction/local/metro_col_sub_catch_centroids.txt')
points = np.genfromtxt(points_file, delimiter=',', names=True, dtype=None)
point_prcp = ext_utils.extract_points_array_rf_series(nc_f, points)
t0 = dt.datetime.strptime(point_prcp['Times'][0], '%Y-%m-%d %H:%M:%S')
t1 = dt.datetime.strptime(point_prcp['Times'][1], '%Y-%m-%d %H:%M:%S')
res_min = int((t1 - t0).total_seconds() / 60)
lines_per_day = int(24 * 60 / res_min)
prev_days = len(prev_rf_files)
output = None
for i in range(prev_days):
if prev_rf_files[prev_days - 1 - i] is not None:
if output is not None:
output = np.append(output,
ext_utils.extract_points_array_rf_series(prev_rf_files[prev_days - 1 - i], points)[
:lines_per_day], axis=0)
else:
output = ext_utils.extract_points_array_rf_series(prev_rf_files[prev_days - 1 - i], points)[
:lines_per_day]
else:
output = None # if any of the previous files are missing, skip prepending past data to the forecast
break
if output is not None:
output = np.append(output, point_prcp, axis=0)
else:
output = point_prcp
fmt = '%s'
for _ in range(len(output[0]) - 1):
fmt = fmt + ',%g'
header = ','.join(output.dtype.names)
utils.create_dir_if_not_exists(output_dir)
np.savetxt(os.path.join(output_dir, 'met_col_rf_mike21.txt'), output, fmt=fmt, delimiter=',', header=header,
comments='', encoding='utf-8')
def test_get_voronoi_polygons_kub(self):
points = {
'Colombo': [79.8653, 6.898158],
'IBATTARA3': [79.86, 6.89],
'Isurupaya': [79.92, 6.89],
'Daraniyagala': [80.33805556, 6.924444444],
'Glencourse': [80.20305556, 6.978055556],
'Hanwella': [80.08166667, 6.909722222],
'Holombuwa': [80.26480556, 7.185166667],
'Kitulgala': [80.41777778, 6.989166667],
'Borella': [79.86, 6.93, ],
'Kompannaveediya': [79.85, 6.92],
}
shp = res_mgr.get_resource_path('extraction/shp/kelani-upper-basin.shp')
out = tempfile.mkdtemp(prefix='voronoi_')
print(out)
result = get_voronoi_polygons(points, shp, ['OBJECTID', 1], output_shape_file=os.path.join(out, 'out.shp'))
print(result)
def extract_weather_stations2(nc_f, wrf_output, weather_stations=None, curw_db_adapter=None, curw_db_upsert=False,
run_prefix='WRF', run_name='Cloud-1'):
if weather_stations is None:
weather_stations = res_mgr.get_resource_path('extraction/local/wrf_stations.txt')
points = np.genfromtxt(weather_stations, delimiter=',', names=True, dtype=None)
point_prcp = ext_utils.extract_points_array_rf_series(nc_f, points)
t0 = dt.datetime.strptime(point_prcp['Times'][0], '%Y-%m-%d %H:%M:%S')
t1 = dt.datetime.strptime(point_prcp['Times'][1], '%Y-%m-%d %H:%M:%S')
res_min = int((t1 - t0).total_seconds() / 60)
prefix = 'stations_rf'
stations_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix))
stations_rf = {}
with TemporaryDirectory(prefix=prefix) as temp_dir:
for point in points:
logging.info(str(point))
station_name = point[0].decode()
stations_rf[station_name] = []
station_file_path = os.path.join(temp_dir, station_name + '_%s.txt' % prefix)
with open(station_file_path, 'w') as station_file:
for t in range(0, len(point_prcp)):
station_file.write('%s\t%.4f\n' % (point_prcp['Times'][t], point_prcp[station_name][t]))
stations_rf[station_name].append([point_prcp['Times'][t], point_prcp[station_name][t]])
utils.move_files_with_prefix(temp_dir, '*.txt', stations_dir)
if curw_db_adapter is not None:
logging.info('Pushing data to the db...')
ext_utils.push_rainfall_to_db(curw_db_adapter, stations_rf, upsert=curw_db_upsert, name=run_name,
source=run_prefix)
else:
logging.info('curw_db_adapter not available. Unable to push data!')
def test_extract_metro_col_rf_for_mike21(self):
out_dir = tempfile.mkdtemp(prefix='met_col_mike21')
prev = [res_mgr.get_resource_path('test/wrfout_d03_2017-12-10_18:00:00_rf'),
res_mgr.get_resource_path('test/wrfout_d03_2017-12-09_18:00:00_rf')]
extract_metro_col_rf_for_mike21(res_mgr.get_resource_path('test/wrfout_d03_2017-12-11_18:00:00_rf'), out_dir,
prev_rf_files=prev)
def extract_kelani_basin_rainfall_flo2d_with_obs(nc_f, adapter, obs_stations, output_dir, start_ts_lk,
duration_days=None, output_prefix='RAINCELL',
kelani_lower_basin_points=None, kelani_lower_basin_shp=None):
"""
check test_extract_kelani_basin_rainfall_flo2d_obs test case
:param nc_f: file path of the wrf output
:param adapter:
:param obs_stations: dict of stations. {station_name: [lon, lat, name variable, nearest wrf point station name]}
:param output_dir:
:param start_ts_lk: start time of the forecast/ end time of the observations
:param duration_days: (optional) a tuple (observation days, forecast days) default (2,3)
:param output_prefix: (optional) output file name of the RAINCELL file. ex: output_prefix=RAINCELL-150m --> RAINCELL-150m.DAT
:param kelani_lower_basin_points: (optional)
:param kelani_lower_basin_shp: (optional)
:return:
"""
if duration_days is None:
duration_days = (2, 3)
if kelani_lower_basin_points is None:
kelani_lower_basin_points = res_mgr.get_resource_path('extraction/local/kelani_basin_points_250m.txt')
if kelani_lower_basin_shp is None:
kelani_lower_basin_shp = res_mgr.get_resource_path('extraction/shp/klb-wgs84/klb-wgs84.shp')
points = np.genfromtxt(kelani_lower_basin_points, delimiter=',')
kel_lon_min = np.min(points, 0)[1]
kel_lat_min = np.min(points, 0)[2]
kel_lon_max = np.max(points, 0)[1]
kel_lat_max = np.max(points, 0)[2]
diff, kel_lats, kel_lons, times = ext_utils.extract_area_rf_series(nc_f, kel_lat_min, kel_lat_max, kel_lon_min,
kel_lon_max)
def get_bins(arr):
sz = len(arr)
return (arr[1:sz - 1] + arr[0:sz - 2]) / 2
lat_bins = get_bins(kel_lats)
lon_bins = get_bins(kel_lons)
t0 = dt.datetime.strptime(times[0], '%Y-%m-%d_%H:%M:%S')
t1 = dt.datetime.strptime(times[1], '%Y-%m-%d_%H:%M:%S')
utils.create_dir_if_not_exists(output_dir)
obs_start = dt.datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') - dt.timedelta(days=duration_days[0])
obs_end = dt.datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M')
forecast_end = dt.datetime.strptime(start_ts_lk, '%Y-%m-%d_%H:%M') + dt.timedelta(days=duration_days[1])
obs = _get_observed_precip(obs_stations, obs_start, obs_end, duration_days, adapter)
thess_poly = spatial_utils.get_voronoi_polygons(obs_stations, kelani_lower_basin_shp, add_total_area=False)
output_file_path = os.path.join(output_dir, output_prefix + '.DAT')
# update points array with the thessian polygon idx
point_thess_idx = []
for point in points:
point_thess_idx.append(spatial_utils.is_inside_geo_df(thess_poly, lon=point[1], lat=point[2]))
pass
with open(output_file_path, 'w') as output_file:
res_mins = int((t1 - t0).total_seconds() / 60)
data_hours = int(sum(duration_days) * 24 * 60 / res_mins)
start_ts_lk = obs_start.strftime('%Y-%m-%d %H:%M:%S')
end_ts = forecast_end.strftime('%Y-%m-%d %H:%M:%S')
output_file.write("%d %d %s %s\n" % (res_mins, data_hours, start_ts_lk, end_ts))
for t in range(int(24 * 60 * duration_days[0] / res_mins) + 1):
for i, point in enumerate(points):
rf = float(obs[point_thess_idx[i]].values[t]) if point_thess_idx[i] is not None else 0
output_file.write('%d %.1f\n' % (point[0], rf))
forecast_start_idx = int(
np.where(times == utils.datetime_lk_to_utc(obs_end, shift_mins=30).strftime('%Y-%m-%d_%H:%M:%S'))[0])
for t in range(int(24 * 60 * duration_days[1] / res_mins) - 1):
for point in points:
rf_x = np.digitize(point[1], lon_bins)
rf_y = np.digitize(point[2], lat_bins)
if t + forecast_start_idx + 1 < len(times):
output_file.write('%d %.1f\n' % (point[0], diff[t + forecast_start_idx + 1, rf_y, rf_x]))
else:
output_file.write('%d %.1f\n' % (point[0], 0))
def __init__(self):
self.shape_file = res_mgr.get_resource_path(
'extraction/shp/kub-wgs84/kub-wgs84.shp')
self.output_dir = 'kub_observation_mean/output_shape/kub'
self.percentage_factor = 100
def test_extract_metro_colombo(self):
# adapter = ext_utils.get_curw_adapter()
adapter = None
out_dir = tempfile.mkdtemp(prefix='metro_col_')
extract_metro_colombo(res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00'), out_dir, out_dir,
curw_db_adapter=adapter, curw_db_upsert=True)
def test_extract_weather_stations(self):
adapter = ext_utils.get_curw_adapter()
out_dir = tempfile.mkdtemp(prefix='stations_')
extract_weather_stations(res_mgr.get_resource_path('test/wrfout_d03_2017-10-02_12:00:00'),
out_dir, curw_db_adapter=adapter, curw_db_upsert=True)
def run(run_id,
wrf_config_dict,
db_config_dict,
upsert=False,
run_name='Cloud-1',
data_type=Data.data,
procedures=sys.maxsize):
"""
Procedure #1: Extracting data from D03
Procedure #2: Extract rainfall data for the metro colombo area
Procedure #3: Extract weather station rainfall
Procedure #4: Extract Kelani upper Basin mean rainfall
Procedure #5: Extract Kelani lower Basin mean rainfall
Procedure #6: Create plots for D03
Procedure #7: Extract Kelani lower Basin rainfall for FLO2D
Procedure #8: Extract Kelani lower Basin rainfall for MIKE21
Procedure #9: Create plots for D01
Procedure #10: Extract rf data from metro col for MIKE21
"""
def _nth_bit(a, n):
return (a >> n) & 1
logging.info('**** Extracting data from WRF **** Run ID: ' + run_id)
run_prefix = run_id.split('_')[0]
config = get_wrf_config(**wrf_config_dict)
config.set('run_id', run_id)
output_dir_base = os.path.join(config.get('nfs_dir'), 'results')
run_output_dir = os.path.join(output_dir_base, run_id)
wrf_output_dir = os.path.join(run_output_dir, 'wrf')
logging.info('WRF output dir: ' + wrf_output_dir)
db_adapter = ext_utils.get_curw_adapter(
mysql_config=db_config_dict) if db_config_dict else None
logging.info('Creating temp file space')
with TemporaryDirectory(prefix='wrfout_') as temp_dir:
try:
logging.info('Copying wrfout_D03* to temp_dir ' + temp_dir)
d03_nc_f = shutil.copy2(
glob.glob(os.path.join(wrf_output_dir, 'wrfout_d03_*'))[0],
temp_dir)
if _nth_bit(procedures, 1):
logging.info('Procedure #1: Extracting data from ' + d03_nc_f)
try:
logging.info(
'Extract WRF data points in the Kelani and Kalu basins'
)
lon_min, lat_min, lon_max, lat_max = constants.KELANI_KALU_BASIN_EXTENT
extractor.push_wrf_rainfall_to_db(
d03_nc_f,
curw_db_adapter=db_adapter,
lat_min=lat_min,
lon_min=lon_min,
lat_max=lat_max,
lon_max=lon_max,
run_prefix=run_prefix,
upsert=upsert)
except Exception as e:
logging.error(
'Extract WRF data points in the Kelani and Kalu basins FAILED: '
+ str(e) + '\n' + traceback.format_exc())
if _nth_bit(procedures, 2):
try:
logging.info(
'Procedure #2: Extract rainfall data for the metro colombo area'
)
basin_rf = extractor.extract_metro_colombo(
d03_nc_f,
run_output_dir,
output_dir_base,
curw_db_adapter=db_adapter,
run_prefix=run_prefix,
run_name=run_name,
curw_db_upsert=upsert)
logging.info('Basin rainfall' + str(basin_rf))
except Exception as e:
logging.error(
'Extract rainfall data for the metro colombo area FAILED: '
+ str(e) + '\n' + traceback.format_exc())
if _nth_bit(procedures, 3):
try:
logging.info(
'Procedure #3: Extract weather station rainfall')
extractor.extract_weather_stations(
d03_nc_f,
run_output_dir,
curw_db_adapter=db_adapter,
curw_db_upsert=upsert,
run_prefix=run_prefix,
run_name=run_name)
except Exception as e:
logging.error('Extract weather station rainfall FAILED: ' +
str(e) + '\n' + traceback.format_exc())
if _nth_bit(procedures, 4):
try:
logging.info(
'Procedure #4: Extract Kelani upper Basin mean rainfall'
)
basin_shp_file = res_mgr.get_resource_path(
'extraction/shp/kelani-upper-basin.shp')
extractor.extract_mean_rainfall_from_shp_file(
d03_nc_f,
run_output_dir,
'kub_mean_rf',
'kub_mean',
basin_shp_file,
constants.KELANI_UPPER_BASIN_EXTENT,
curw_db_adapter=db_adapter,
run_prefix=run_prefix,
run_name=run_name,
curw_db_upsert=upsert)
except Exception as e:
logging.error(
'Extract Kelani upper Basin mean rainfall FAILED: ' +
str(e) + '\n' + traceback.format_exc())
if _nth_bit(procedures, 5):
try:
logging.info(
'Procedure #5: Extract Kelani lower Basin mean rainfall'
)
basin_shp_file = res_mgr.get_resource_path(
'extraction/shp/klb-wgs84/klb-wgs84.shp')
extractor.extract_mean_rainfall_from_shp_file(
d03_nc_f,
run_output_dir,
'klb_mean_rf',
'klb_mean',
basin_shp_file,
constants.KELANI_LOWER_BASIN_EXTENT,
curw_db_adapter=db_adapter,
run_prefix=run_prefix,
run_name=run_name,
curw_db_upsert=upsert)
except Exception as e:
logging.error(
'Extract Kelani lower Basin mean rainfall FAILED: ' +
str(e) + '\n' + traceback.format_exc())
if _nth_bit(procedures, 6):
try:
logging.info('Procedure #6: Create plots for D03')
lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_EXTENT
extractor.create_rf_plots_wrf(d03_nc_f,
os.path.join(
run_output_dir,
'plots_D03'),
output_dir_base,
lat_min=lat_min,
lon_min=lon_min,
lat_max=lat_max,
lon_max=lon_max,
run_prefix=run_prefix)
except Exception as e:
logging.error('Create plots for D03 FAILED: ' + str(e) +
'\n' + traceback.format_exc())
if _nth_bit(procedures, 7):
try:
logging.info(
'Procedure #7: Extract Kelani lower Basin rainfall for FLO2D'
)
run_date = dt.datetime.strptime(config.get('start_date'),
'%Y-%m-%d_%H:%M')
prev_days = 5
d03_nc_f_prev = []
for i in range(prev_days):
prev = '_'.join([
run_prefix, (run_date - dt.timedelta(days=i + 1)
).strftime('%Y-%m-%d_%H:%M'), '*'
])
try:
d03_nc_f_prev.append(
shutil.copy2(
glob.glob(
os.path.join(output_dir_base, prev,
'wrf',
'wrfout_d03_*'))[0],
temp_dir))
except IndexError as e:
logging.warning(
'File for %s not found. Filling with 0s: %s' %
(prev, str(e)))
d03_nc_f_prev.append(None)
logging.info('250m model')
kelani_basin_flo2d_file = res_mgr.get_resource_path(
'extraction/local/kelani_basin_points_250m.txt')
extractor.extract_kelani_basin_rainfall_flo2d(
d03_nc_f,
d03_nc_f_prev[:2],
os.path.join(run_output_dir, 'klb_flo2d'),
kelani_basin_file=kelani_basin_flo2d_file)
logging.info('150m model')
kelani_basin_flo2d_file = res_mgr.get_resource_path(
'extraction/local/klb_glecourse_points_150m.txt')
extractor.extract_kelani_basin_rainfall_flo2d(
d03_nc_f,
d03_nc_f_prev,
os.path.join(run_output_dir, 'klb_flo2d'),
kelani_basin_file=kelani_basin_flo2d_file,
output_prefix='RAINCELL_150m',
target_rfs=[])
except Exception as e:
logging.error(
'Extract Kelani lower Basin mean rainfall for FLO2D FAILED: '
+ str(e) + '\n' + traceback.format_exc())
if _nth_bit(procedures, 8):
try:
logging.info(
'Procedure #8: Extract Kelani lower Basin rainfall for MIKE21'
)
run_date = dt.datetime.strptime(config.get('start_date'),
'%Y-%m-%d_%H:%M')
now = shutil.copy2(
os.path.join(output_dir_base, config.get('run_id'),
'klb_mean_rf', 'klb_mean_rf.txt'),
os.path.join(temp_dir, 'klb.txt'))
prev_1 = '_'.join([
run_prefix,
(run_date -
dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*'
])
prev_2 = '_'.join([
run_prefix,
(run_date -
dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*'
])
try:
klb_prev_1 = utils.copy_if_not_exists(
glob.glob(
os.path.join(output_dir_base, prev_1,
'klb_mean_rf',
'klb_mean_rf.txt'))[0],
os.path.join(temp_dir, 'klb1.txt'))
except IndexError as e:
logging.warning(
'File for %s not found. Filling with 0s: %s' %
(prev_1, str(e)))
klb_prev_1 = None
try:
klb_prev_2 = utils.copy_if_not_exists(
glob.glob(
os.path.join(output_dir_base, prev_2,
'klb_mean_rf',
'klb_mean_rf.txt'))[0],
os.path.join(temp_dir, 'klb2.txt'))
except IndexError as e:
logging.warning(
'File for %s not found. Filling with 0s: %s' %
(prev_2, str(e)))
klb_prev_2 = None
extractor.create_rainfall_for_mike21(
now, [klb_prev_1, klb_prev_2],
os.path.join(run_output_dir, 'klb_mike21'))
except Exception as e:
logging.error(
'Extract Kelani lower Basin mean rainfall for MIKE21 FAILED: '
+ str(e) + '\n' + traceback.format_exc())
if _nth_bit(procedures, 10):
try:
logging.info(
'Procedure #10: Extract rf data from metro col for MIKE21'
)
run_date = dt.datetime.strptime(config.get('start_date'),
'%Y-%m-%d_%H:%M')
prev_1 = '_'.join([
run_prefix,
(run_date -
dt.timedelta(days=1)).strftime('%Y-%m-%d_%H:%M'), '*'
])
prev_2 = '_'.join([
run_prefix,
(run_date -
dt.timedelta(days=2)).strftime('%Y-%m-%d_%H:%M'), '*'
])
try:
d03_nc_f_prev_1 = utils.copy_if_not_exists(
glob.glob(
os.path.join(output_dir_base, prev_1, 'wrf',
'wrfout_d03_*'))[0], temp_dir)
except IndexError as e:
logging.warning(
'File for %s not found. Filling with 0s: %s' %
(prev_1, str(e)))
d03_nc_f_prev_1 = None
try:
d03_nc_f_prev_2 = utils.copy_if_not_exists(
glob.glob(
os.path.join(output_dir_base, prev_2, 'wrf',
'wrfout_d03_*'))[0], temp_dir)
except IndexError as e:
logging.warning(
'File for %s not found. Filling with 0s: %s' %
(prev_2, str(e)))
d03_nc_f_prev_2 = None
prev = [d03_nc_f_prev_1, d03_nc_f_prev_2]
extractor.extract_metro_col_rf_for_mike21(
d03_nc_f,
os.path.join(run_output_dir, 'met_col_mike21'),
prev_rf_files=prev)
except Exception as e:
logging.error(
'Extract rf data from metro col for MIKE21: ' +
str(e) + '\n' + traceback.format_exc())
except Exception as e:
logging.error(
'Copying wrfout_d03_* to temp_dir %s FAILED: %s\n%s' %
(temp_dir, str(e), traceback.format_exc()))
try:
d01_nc_f = shutil.copy2(
glob.glob(os.path.join(wrf_output_dir, 'wrfout_d01_*'))[0],
temp_dir)
logging.info('Extracting data from ' + d01_nc_f)
if _nth_bit(procedures, 9):
try:
logging.info('Procedure #9: Create plots for D01')
lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_D01_EXTENT
extractor.create_rf_plots_wrf(d01_nc_f,
os.path.join(
run_output_dir,
'plots_D01'),
output_dir_base,
lat_min=lat_min,
lon_min=lon_min,
lat_max=lat_max,
lon_max=lon_max,
run_prefix=run_prefix)
except Exception as e:
logging.error('Create plots for D01 FAILED: ' + str(e) +
'\n' + traceback.format_exc())
except Exception as e:
logging.error(
'Copying wrfout_d01_* to temp_dir %s FAILED: %s\n%s' %
(temp_dir, str(e), traceback.format_exc()))
logging.info('**** Extracting data from WRF **** Run ID: ' + run_id +
' COMPLETED!')
def extract_kelani_basin_rainfall_flo2d(nc_f, nc_f_prev_days, output_dir, avg_basin_rf=1.0, kelani_basin_file=None,
target_rfs=None, output_prefix='RAINCELL'):
"""
:param output_prefix:
:param nc_f:
:param nc_f_prev_days:
:param output_dir:
:param avg_basin_rf:
:param kelani_basin_file:
:param target_rfs:
:return:
"""
if target_rfs is None:
target_rfs = [100, 150, 200, 250, 300]
if kelani_basin_file is None:
kelani_basin_file = res_mgr.get_resource_path('extraction/local/kelani_basin_points_250m.txt')
points = np.genfromtxt(kelani_basin_file, delimiter=',')
kel_lon_min = np.min(points, 0)[1]
kel_lat_min = np.min(points, 0)[2]
kel_lon_max = np.max(points, 0)[1]
kel_lat_max = np.max(points, 0)[2]
diff, kel_lats, kel_lons, times = ext_utils.extract_area_rf_series(nc_f, kel_lat_min, kel_lat_max, kel_lon_min,
kel_lon_max)
def get_bins(arr):
sz = len(arr)
return (arr[1:sz - 1] + arr[0:sz - 2]) / 2
lat_bins = get_bins(kel_lats)
lon_bins = get_bins(kel_lons)
t0 = dt.datetime.strptime(times[0], '%Y-%m-%d_%H:%M:%S')
t1 = dt.datetime.strptime(times[1], '%Y-%m-%d_%H:%M:%S')
t_end = dt.datetime.strptime(times[-1], '%Y-%m-%d_%H:%M:%S')
utils.create_dir_if_not_exists(output_dir)
prev_diff = []
prev_days = len(nc_f_prev_days)
for i in range(prev_days):
if nc_f_prev_days[i]:
p_diff, _, _, _ = ext_utils.extract_area_rf_series(nc_f_prev_days[i], kel_lat_min, kel_lat_max, kel_lon_min,
kel_lon_max)
prev_diff.append(p_diff)
else:
prev_diff.append(None)
def write_forecast_to_raincell_file(output_file_path, alpha):
with open(output_file_path, 'w') as output_file:
res_mins = int((t1 - t0).total_seconds() / 60)
data_hours = int(len(times) + prev_days * 24 * 60 / res_mins)
start_ts = utils.datetime_utc_to_lk(t0 - dt.timedelta(days=prev_days), shift_mins=30).strftime(
'%Y-%m-%d %H:%M:%S')
end_ts = utils.datetime_utc_to_lk(t_end, shift_mins=30).strftime('%Y-%m-%d %H:%M:%S')
output_file.write("%d %d %s %s\n" % (res_mins, data_hours, start_ts, end_ts))
for d in range(prev_days):
for t in range(int(24 * 60 / res_mins)):
for point in points:
rf_x = np.digitize(point[1], lon_bins)
rf_y = np.digitize(point[2], lat_bins)
if prev_diff[prev_days - 1 - d] is not None:
output_file.write('%d %.1f\n' % (point[0], prev_diff[prev_days - 1 - d][t, rf_y, rf_x]))
else:
output_file.write('%d %.1f\n' % (point[0], 0))
for t in range(len(times)):
for point in points:
rf_x = np.digitize(point[1], lon_bins)
rf_y = np.digitize(point[2], lat_bins)
if t < int(24 * 60 / res_mins):
output_file.write('%d %.1f\n' % (point[0], diff[t, rf_y, rf_x] * alpha))
else:
output_file.write('%d %.1f\n' % (point[0], diff[t, rf_y, rf_x]))
with TemporaryDirectory(prefix='curw_raincell') as temp_dir:
raincell_temp = os.path.join(temp_dir, output_prefix + '.DAT')
write_forecast_to_raincell_file(raincell_temp, 1)
for target_rf in target_rfs:
write_forecast_to_raincell_file('%s.%d' % (raincell_temp, target_rf), target_rf / avg_basin_rf)
utils.create_zip_with_prefix(temp_dir, output_prefix + '.DAT*', os.path.join(temp_dir, output_prefix + '.zip'),
clean_up=True)
utils.move_files_with_prefix(temp_dir, output_prefix + '.zip', utils.create_dir_if_not_exists(output_dir))
