def process_fcst_ts_from_hechms_outputs(curw_fcst_pool,
extract_stations,
i,
start,
end,
sim_tag=None):
FCST_TS = Fcst_Timeseries(curw_fcst_pool)
try:
# [station_name,latitude,longitude,target,model,version,sim_tag,station]
source_model = extract_stations[i][4]
version = extract_stations[i][5]
station_id = extract_stations[i][7]
if sim_tag is None:
sim_tag = extract_stations[i][6]
variable_id = 3 # Discharge
unit_id = 3 # m3/s | Instantaneous
source_id = get_source_id(pool=curw_fcst_pool,
model=source_model,
version=version)
fcst_series = FCST_TS.get_latest_timeseries(sim_tag,
station_id,
source_id,
variable_id,
unit_id,
start=None)
if (fcst_series is None) or (len(fcst_series) < 1):
return None
fcst_series.insert(0, ['time', 'value'])
fcst_df = list_of_lists_to_df_first_row_as_columns(fcst_series)
if start is None:
start = (fcst_df['time'].min()).strftime(DATE_TIME_FORMAT)
if end is None:
end = (fcst_df['time'].max()).strftime(DATE_TIME_FORMAT)
df = (pd.date_range(start=start, end=end,
freq='60min')).to_frame(name='time')
processed_df = pd.merge(df, fcst_df, on='time', how='left')
processed_df.interpolate(method='linear',
limit_direction='both',
limit=100)
processed_df.fillna(inplace=True, value=0)
processed_df['time'] = processed_df['time'].dt.strftime(
DATE_TIME_FORMAT)
return processed_df.values.tolist()
except Exception as e:
traceback.print_exc()
def process_fcst_ts_from_flo2d_outputs(curw_fcst_pool, fcst_start):
global latest_fgt
FCST_TS = Fcst_Timeseries(curw_fcst_pool)
try:
# [station_name,latitude,longitude,target,model,version,sim_tag,station]
source_model = extract_stations[i][4]
version = extract_stations[i][5]
sim_tag = extract_stations[i][6]
station_id = extract_stations[i][7]
variable_id = 3 # Discharge
unit_id = 3 # m3/s | Instantaneous
source_id = get_source_id(pool=curw_fcst_pool,
model=source_model,
version=version)
fcst_series = FCST_TS.get_latest_timeseries(sim_tag,
station_id,
source_id,
variable_id,
unit_id,
start=None)
if (fcst_series is None) or (len(fcst_series) < 1):
return None
latest_fgt = (FCST_TS.get_end_date(
sim_tag, station_id, source_id, variable_id,
unit_id)).strftime(COMMON_DATE_TIME_FORMAT)
fcst_series.insert(0, ['time', 'value'])
fcst_df = list_of_lists_to_df_first_row_as_columns(fcst_series)
fcst_end = (fcst_df['time'].max()).strftime(COMMON_DATE_TIME_FORMAT)
if fcst_start is None:
fcst_start = (
fcst_df['time'].min()).strftime(COMMON_DATE_TIME_FORMAT)
df = (pd.date_range(start=fcst_start, end=fcst_end,
freq='15min')).to_frame(name='time')
processed_df = pd.merge(df, fcst_df, on='time', how='left')
# processed_df.interpolate(method='linear', limit_direction='both')
processed_df = processed_df.dropna()
processed_df['time'] = processed_df['time'].dt.strftime(
COMMON_DATE_TIME_FORMAT)
return processed_df.values.tolist()
except Exception as e:
traceback.print_exc()
logger.error("Exception occurred")
def update_rainfall_fcsts(flo2d_model, method, grid_interpolation, model_list,
timestep):
"""
Update rainfall forecasts for flo2d models
:param flo2d_model: flo2d model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param model_list: list of forecast model and their versions used to calculate the rainfall
e.g.: [["WRF_E", "v4"],["WRF_SE", "v4"]]
:param timestep: output timeseries timestep
:return:
"""
try:
# Connect to the database
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
Sim_TS = Sim_Timeseries(pool=curw_sim_pool)
Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool)
flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format(
flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)]
flo2d_wrf_mapping = get_flo2d_cells_to_wrf_grid_mappings(
pool=curw_sim_pool,
grid_interpolation=grid_interpolation,
flo2d_model=flo2d_model)
for flo2d_index in range(len(flo2d_grids)): # len(flo2d_grids)
lat = flo2d_grids[flo2d_index][2]
lon = flo2d_grids[flo2d_index][1]
cell_id = flo2d_grids[flo2d_index][0]
meta_data = {
'latitude':
float('%.6f' % float(lat)),
'longitude':
float('%.6f' % float(lon)),
'model':
flo2d_model,
'method':
method,
'grid_id':
'{}_{}_{}'.format(flo2d_model, grid_interpolation,
(str(cell_id)).zfill(10))
}
tms_id = Sim_TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
method=meta_data.get('method'))
if tms_id is None:
tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
Sim_TS.insert_run(meta_data=meta_data)
obs_end = Sim_TS.get_obs_end(id_=tms_id)
fcst_timeseries = []
for i in range(len(model_list)):
source_id = get_source_id(pool=curw_fcst_pool,
model=model_list[i][0],
version=model_list[i][1])
sim_tag = model_list[i][2]
coefficient = model_list[i][3]
temp_timeseries = []
if timestep == 5:
if obs_end is not None:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.
get(meta_data['grid_id']),
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1))
else:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.
get(meta_data['grid_id']),
source_id=source_id,
variable_id=1,
unit_id=1))
elif timestep == 15:
if obs_end is not None:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.get(
meta_data['grid_id']),
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1)
else:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.get(
meta_data['grid_id']),
source_id=source_id,
variable_id=1,
unit_id=1)
if coefficient != 1:
for j in range(len(temp_timeseries)):
temp_timeseries[j][1] = float(
temp_timeseries[j][1]) * coefficient
if i == 0:
fcst_timeseries = temp_timeseries
else:
fcst_timeseries = append_value_for_timestamp(
existing_ts=fcst_timeseries, new_ts=temp_timeseries)
sum_timeseries = summed_timeseries(fcst_timeseries)
for i in range(len(sum_timeseries)):
if float(sum_timeseries[i][1]) < 0:
sum_timeseries[i][1] = 0
if sum_timeseries is not None and len(sum_timeseries) > 0:
Sim_TS.insert_data(timeseries=sum_timeseries,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating fcst rainfalls in curw_sim.")
finally:
destroy_Pool(curw_sim_pool)
destroy_Pool(curw_fcst_pool)
def update_rainfall_fcsts(target_model, method, grid_interpolation, model_list,
timestep):
"""
Update rainfall forecasts for flo2d models
:param target_model: target model for which input ins prepared
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param model_list: list of forecast model and their versions used to calculate the rainfall
e.g.: [["WRF_E", "4.0", "evening_18hrs"],["WRF_SE", "v4", ,"evening_18hrs"],["WRF_Ensemble", "4.0", ,"MME"]]
:param timestep: output timeseries timestep
:return:
"""
try:
# Connect to the database
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
Sim_TS = Sim_Timeseries(pool=curw_sim_pool)
Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool)
# [hash_id, station_id, station_name, latitude, longitude]
active_obs_stations = read_csv(
'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'
)
obs_stations_dict = {
} # keys: obs station id , value: [name, latitude, longitude]
for obs_index in range(len(active_obs_stations)):
obs_stations_dict[active_obs_stations[obs_index][1]] = [
active_obs_stations[obs_index][2],
active_obs_stations[obs_index][3],
active_obs_stations[obs_index][4]
]
obs_d03_mapping = get_obs_to_d03_grid_mappings_for_rainfall(
pool=curw_sim_pool, grid_interpolation=grid_interpolation)
for obs_id in obs_stations_dict.keys():
meta_data = {
'latitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[1])),
'longitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[2])),
'model':
target_model,
'method':
method,
'grid_id':
'rainfall_{}_{}_{}'.format(obs_id,
obs_stations_dict.get(obs_id)[0],
grid_interpolation)
}
tms_id = Sim_TS.get_timeseries_id_if_exists(meta_data=meta_data)
if tms_id is None:
tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
Sim_TS.insert_run(meta_data=meta_data)
obs_end = Sim_TS.get_obs_end(id_=tms_id)
fcst_timeseries = []
for i in range(len(model_list)):
source_id = get_source_id(pool=curw_fcst_pool,
model=model_list[i][0],
version=model_list[i][1])
sim_tag = model_list[i][2]
coefficient = model_list[i][3]
temp_timeseries = []
if timestep == 5:
if obs_end is not None:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=obs_d03_mapping.
get(meta_data['grid_id'])[0],
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1))
else:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=obs_d03_mapping.
get(meta_data['grid_id'])[0],
source_id=source_id,
variable_id=1,
unit_id=1))
elif timestep == 15:
if obs_end is not None:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=obs_d03_mapping.get(
meta_data['grid_id'])[0],
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1)
else:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=obs_d03_mapping.get(
meta_data['grid_id'])[0],
source_id=source_id,
variable_id=1,
unit_id=1)
if coefficient != 1:
for j in range(len(temp_timeseries)):
temp_timeseries[j][1] = float(
temp_timeseries[j][1]) * coefficient
if i == 0:
fcst_timeseries = temp_timeseries
else:
fcst_timeseries = append_value_for_timestamp(
existing_ts=fcst_timeseries, new_ts=temp_timeseries)
sum_timeseries = summed_timeseries(fcst_timeseries)
for i in range(len(sum_timeseries)):
if float(sum_timeseries[i][1]) < 0:
sum_timeseries[i][1] = 0
if sum_timeseries is not None and len(sum_timeseries) > 0:
Sim_TS.insert_data(timeseries=sum_timeseries,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating fcst rainfalls in curw_sim.")
finally:
destroy_Pool(curw_sim_pool)
destroy_Pool(curw_fcst_pool)