def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep,
start_time, end_time):
"""
Update rainfall observations for flo2d models
:param flo2d_model: flo2d model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param timestep: output timeseries timestep
:return:
"""
obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
try:
# Connect to the database
curw_obs_pool = get_Pool(host=con_params.CURW_OBS_HOST,
user=con_params.CURW_OBS_USERNAME,
password=con_params.CURW_OBS_PASSWORD,
port=con_params.CURW_OBS_PORT,
db=con_params.CURW_OBS_DATABASE)
curw_obs_connection = curw_obs_pool.connection()
curw_sim_pool = get_Pool(host=con_params.CURW_SIM_HOST,
user=con_params.CURW_SIM_USERNAME,
password=con_params.CURW_SIM_PASSWORD,
port=con_params.CURW_SIM_PORT,
db=con_params.CURW_SIM_DATABASE)
TS = Sim_Timeseries(pool=curw_sim_pool)
# [hash_id, station_id, station_name, latitude, longitude]
# active_obs_stations = read_csv(os.path.join(ROOT_DIR,'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'))
active_obs_stations = extract_active_curw_obs_rainfall_stations(
start_time=start_time, end_time=end_time)[1:]
flo2d_grids = read_csv(
os.path.join(ROOT_DIR, 'grids/flo2d/{}m.csv'.format(
flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)]
stations_dict_for_obs = {} # keys: obs station id , value: hash id
for obs_index in range(len(active_obs_stations)):
stations_dict_for_obs[active_obs_stations[obs_index]
[1]] = active_obs_stations[obs_index][0]
# flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model)
flo2d_obs_mapping = find_nearest_obs_stations_for_flo2d_stations(
flo2d_stations_csv=os.path.join(
ROOT_DIR, 'grids/flo2d/{}m.csv'.format(flo2d_model)),
obs_stations=active_obs_stations,
flo2d_model=flo2d_model)
for flo2d_index in range(len(flo2d_grids)):
lat = flo2d_grids[flo2d_index][2]
lon = flo2d_grids[flo2d_index][1]
cell_id = flo2d_grids[flo2d_index][0]
meta_data = {
'latitude':
float('%.6f' % float(lat)),
'longitude':
float('%.6f' % float(lon)),
'model':
flo2d_model,
'method':
method,
'grid_id':
'{}_{}_{}'.format(flo2d_model, grid_interpolation,
(str(cell_id)).zfill(10))
}
tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
method=meta_data.get('method'))
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
print("grid_id:", cell_id)
print("grid map:", flo2d_obs_mapping.get(cell_id))
obs1_station_id = flo2d_obs_mapping.get(cell_id)[0]
obs2_station_id = flo2d_obs_mapping.get(cell_id)[1]
obs3_station_id = flo2d_obs_mapping.get(cell_id)[2]
obs_timeseries = []
if timestep == 5:
if obs1_station_id != str(-1):
obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)
ts = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs1_hash_id,
end_time=end_time)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_5_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
if obs2_station_id != str(-1):
obs2_hash_id = stations_dict_for_obs.get(
obs2_station_id)
ts2 = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs2_hash_id,
end_time=end_time)
if ts2 is not None and len(ts2) > 1:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts2,
OBS_TS=obs_timeseries)
if obs_timeseries is not None and len(
obs_timeseries) > 0:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_5_min_ts(
newly_extracted_timeseries=ts2,
expected_start=expected_start)[1:])
# obs_start = ts2[-1][0]
if obs3_station_id != str(-1):
obs3_hash_id = stations_dict_for_obs.get(
obs3_station_id)
ts3 = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs3_hash_id,
end_time=end_time)
if ts3 is not None and len(ts3) > 1 and len(
obs_timeseries) > 0:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts3,
OBS_TS=obs_timeseries)
if obs_timeseries is not None:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_5_min_ts(
newly_extracted_timeseries=ts3,
expected_start=expected_start)[1:])
elif timestep == 15:
if obs1_station_id != str(-1):
obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)
ts = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs1_hash_id,
end_time=end_time)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_15_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
if obs2_station_id != str(-1):
obs2_hash_id = stations_dict_for_obs.get(
obs2_station_id)
ts2 = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs2_hash_id,
end_time=end_time)
if ts2 is not None and len(ts2) > 1:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts2,
OBS_TS=obs_timeseries)
if obs_timeseries is not None and len(
obs_timeseries) > 0:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_15_min_ts(
newly_extracted_timeseries=ts2,
expected_start=expected_start)[1:])
# obs_start = ts2[-1][0]
if obs3_station_id != str(-1):
obs3_hash_id = stations_dict_for_obs.get(
obs3_station_id)
ts3 = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs3_hash_id,
end_time=end_time)
if ts3 is not None and len(ts3) > 1 and len(
obs_timeseries) > 0:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts3,
OBS_TS=obs_timeseries)
if obs_timeseries is not None:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_15_min_ts(
newly_extracted_timeseries=ts3,
expected_start=expected_start)[1:])
for i in range(len(obs_timeseries)):
if obs_timeseries[i][1] == -99999:
obs_timeseries[i][1] = 0
print("### obs timeseries length ###", len(obs_timeseries))
if obs_timeseries is not None and len(
obs_timeseries) > 0 and obs_timeseries[-1][0] != end_time:
obs_timeseries.append(
[datetime.strptime(end_time, DATE_TIME_FORMAT), 0])
final_ts = process_continuous_ts(original_ts=obs_timeseries,
expected_start=datetime.strptime(
start_time, DATE_TIME_FORMAT),
filling_value=0,
timestep=timestep)
if final_ts is not None and len(final_ts) > 0:
TS.insert_data(timeseries=final_ts, tms_id=tms_id, upsert=True)
TS.update_latest_obs(id_=tms_id, obs_end=(final_ts[-1][1]))
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating obs rainfalls in curw_sim.")
finally:
curw_obs_connection.close()
destroy_Pool(pool=curw_sim_pool)
destroy_Pool(pool=curw_obs_pool)
logger.info("Process finished")
def update_rainfall_obs(target_model, method, timestep, start_time, end_time):
"""
Update rainfall observations for flo2d models
:param model: target model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param timestep:
:return:
"""
obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
try:
# Connect to the database
curw_obs_pool = get_Pool(host=con_params.CURW_OBS_HOST,
user=con_params.CURW_OBS_USERNAME,
password=con_params.CURW_OBS_PASSWORD,
port=con_params.CURW_OBS_PORT,
db=con_params.CURW_OBS_DATABASE)
curw_obs_connection = curw_obs_pool.connection()
curw_sim_pool = get_Pool(host=con_params.CURW_SIM_HOST,
user=con_params.CURW_SIM_USERNAME,
password=con_params.CURW_SIM_PASSWORD,
port=con_params.CURW_SIM_PORT,
db=con_params.CURW_SIM_DATABASE)
TS = Timeseries(pool=curw_sim_pool)
# [hash_id, station_id, station_name, latitude, longitude]
active_obs_stations = extract_active_curw_obs_rainfall_stations(
start_time=start_time, end_time=end_time)[1:]
obs_stations_dict = {
} # keys: obs station id , value: [hash id, name, latitude, longitude]
for obs_index in range(len(active_obs_stations)):
obs_stations_dict[active_obs_stations[obs_index][1]] = [
active_obs_stations[obs_index][0],
active_obs_stations[obs_index][2],
active_obs_stations[obs_index][3],
active_obs_stations[obs_index][4]
]
for obs_id in obs_stations_dict.keys():
meta_data = {
'latitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[2])),
'longitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[3])),
'model':
target_model,
'method':
method,
'grid_id':
'rainfall_{}_{}'.format(obs_id,
obs_stations_dict.get(obs_id)[1])
}
tms_id = TS.get_timeseries_id_if_exists(meta_data=meta_data)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
TS.update_grid_id(id_=tms_id, grid_id=meta_data['grid_id'])
obs_hash_id = obs_stations_dict.get(obs_id)[0]
obs_timeseries = []
if timestep == 5:
ts = extract_obs_rain_5_min_ts(connection=curw_obs_connection,
start_time=obs_start,
end_time=end_time,
id=obs_hash_id)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_5_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
elif timestep == 15:
ts = extract_obs_rain_15_min_ts(connection=curw_obs_connection,
start_time=obs_start,
end_time=end_time,
id=obs_hash_id)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_15_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
# for i in range(len(obs_timeseries)):
# if obs_timeseries[i][1] == -99999:
# obs_timeseries[i][1] = 0
if obs_timeseries is not None and len(obs_timeseries) > 0:
TS.replace_data(timeseries=obs_timeseries, tms_id=tms_id)
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating obs rainfalls in curw_sim.")
finally:
curw_obs_connection.close()
destroy_Pool(pool=curw_sim_pool)
destroy_Pool(pool=curw_obs_pool)
def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep):
"""
Update rainfall observations for flo2d models
:param flo2d_model: flo2d model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param timestep: output timeseries timestep
:return:
"""
now = datetime.now()
OBS_START_STRING = (now - timedelta(days=10)).strftime('%Y-%m-%d %H:00:00')
OBS_START = datetime.strptime(OBS_START_STRING, '%Y-%m-%d %H:%M:%S')
try:
# Connect to the database
curw_obs_pool = get_Pool(host=CURW_OBS_HOST,
user=CURW_OBS_USERNAME,
password=CURW_OBS_PASSWORD,
port=CURW_OBS_PORT,
db=CURW_OBS_DATABASE)
curw_obs_connection = curw_obs_pool.connection()
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
# test ######
# pool = get_Pool(host=HOST, user=USERNAME, password=PASSWORD, port=PORT, db=DATABASE)
TS = Sim_Timeseries(pool=curw_sim_pool)
# [hash_id, station_id, station_name, latitude, longitude]
active_obs_stations = read_csv(
'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'
)
flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format(
flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)]
stations_dict_for_obs = {} # keys: obs station id , value: hash id
for obs_index in range(len(active_obs_stations)):
stations_dict_for_obs[active_obs_stations[obs_index]
[1]] = active_obs_stations[obs_index][0]
flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(
pool=curw_sim_pool,
grid_interpolation=grid_interpolation,
flo2d_model=flo2d_model)
for flo2d_index in range(len(flo2d_grids)):
obs_start = OBS_START
lat = flo2d_grids[flo2d_index][2]
lon = flo2d_grids[flo2d_index][1]
cell_id = flo2d_grids[flo2d_index][0]
meta_data = {
'latitude':
float('%.6f' % float(lat)),
'longitude':
float('%.6f' % float(lon)),
'model':
flo2d_model,
'method':
method,
'grid_id':
'{}_{}_{}'.format(flo2d_model, grid_interpolation,
(str(cell_id)).zfill(10))
}
tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
method=meta_data.get('method'))
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
obs_end = TS.get_obs_end(id_=tms_id)
if obs_end is not None:
obs_start = obs_end - timedelta(hours=1)
obs1_station_id = str(
flo2d_obs_mapping.get(meta_data['grid_id'])[0])
obs2_station_id = str(
flo2d_obs_mapping.get(meta_data['grid_id'])[1])
obs3_station_id = str(
flo2d_obs_mapping.get(meta_data['grid_id'])[2])
obs_timeseries = []
if timestep == 5:
if obs1_station_id != str(-1):
obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)
ts = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs1_hash_id)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_5_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
if obs2_station_id != str(-1):
obs2_hash_id = stations_dict_for_obs.get(
obs2_station_id)
ts2 = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs2_hash_id)
if ts2 is not None and len(ts2) > 1:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts2,
OBS_TS=obs_timeseries)
if obs_timeseries is not None and len(
obs_timeseries) > 0:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_5_min_ts(
newly_extracted_timeseries=ts2,
expected_start=expected_start)[1:])
# obs_start = ts2[-1][0]
if obs3_station_id != str(-1):
obs3_hash_id = stations_dict_for_obs.get(
obs3_station_id)
ts3 = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs3_hash_id)
if ts3 is not None and len(ts3) > 1 and len(
obs_timeseries) > 0:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts3,
OBS_TS=obs_timeseries)
if obs_timeseries is not None:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_5_min_ts(
newly_extracted_timeseries=ts3,
expected_start=expected_start)[1:])
elif timestep == 15:
if obs1_station_id != str(-1):
obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)
ts = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs1_hash_id)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_15_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
if obs2_station_id != str(-1):
obs2_hash_id = stations_dict_for_obs.get(
obs2_station_id)
ts2 = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs2_hash_id)
if ts2 is not None and len(ts2) > 1:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts2,
OBS_TS=obs_timeseries)
if obs_timeseries is not None and len(
obs_timeseries) > 0:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_15_min_ts(
newly_extracted_timeseries=ts2,
expected_start=expected_start)[1:])
# obs_start = ts2[-1][0]
if obs3_station_id != str(-1):
obs3_hash_id = stations_dict_for_obs.get(
obs3_station_id)
ts3 = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs3_hash_id)
if ts3 is not None and len(ts3) > 1 and len(
obs_timeseries) > 0:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts3,
OBS_TS=obs_timeseries)
if obs_timeseries is not None:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_15_min_ts(
newly_extracted_timeseries=ts3,
expected_start=expected_start)[1:])
for i in range(len(obs_timeseries)):
if obs_timeseries[i][1] == -99999:
obs_timeseries[i][1] = 0
if obs_timeseries is not None and len(obs_timeseries) > 0:
TS.insert_data(timeseries=obs_timeseries,
tms_id=tms_id,
upsert=True)
TS.update_latest_obs(id_=tms_id,
obs_end=(obs_timeseries[-1][1]))
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating obs rainfalls in curw_sim.")
finally:
curw_obs_connection.close()
destroy_Pool(pool=curw_sim_pool)
destroy_Pool(pool=curw_obs_pool)
logger.info("Process finished")