def import_old_data():
# Connect to the database
pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
connection = pool.connection()
curw_fcst_new_to_old_hash_id_mapping = read_csv(
"curw_fcst_new_to_old_hash_id_mapping.csv")
TS = Timeseries(pool=pool)
try:
for hash_index in range(len(curw_fcst_new_to_old_hash_id_mapping)):
print("##### Hash index: ", hash_index, " #####")
fgt_list = []
# Extract fgts
with connection.cursor() as cursor1:
sql_statement = "select distinct `fgt` from `data_v3` where `id`=%s order by `fgt` desc;"
cursor1.execute(
sql_statement,
curw_fcst_new_to_old_hash_id_mapping[hash_index][1])
fgts = cursor1.fetchall()
for fgt in fgts:
fgt_list.append(fgt.get('fgt'))
for fgt in fgt_list:
timeseries = []
with connection.cursor() as cursor2:
sql_statement = "select * from `data_v3` where `id`=%s and `fgt`=%s;"
cursor2.execute(
sql_statement,
(curw_fcst_new_to_old_hash_id_mapping[hash_index][1],
fgt))
results = cursor2.fetchall()
for result in results:
timeseries.append([
curw_fcst_new_to_old_hash_id_mapping[hash_index]
[0],
result.get('time'),
result.get('fgt'),
result.get('value')
])
TS.insert_data(timeseries=timeseries, upsert=True)
TS.update_start_date(
id_=curw_fcst_new_to_old_hash_id_mapping[hash_index][0],
start_date=fgt)
except Exception as ex:
traceback.print_exc()
finally:
connection.close()
destroy_Pool(pool=pool)
print()
def find_nearest_stations_for_flo2d_grids(flo2d_grids_csv, stations_dict):
"""
:param flo2d_stations_csv:
:param stations_dict: key = point_name, value = [latitude,longitude]
:param flo2d_model:
:return:
"""
flo2d_station = read_csv(flo2d_grids_csv) # [Grid_ID,X,Y]
# key: flo2d_grid_id value: [lat, lon]
flo2d_stations_mapping_dict = {}
for flo2d_index in range(len(flo2d_station)):
flo2d_lat = float(flo2d_station[flo2d_index][2])
flo2d_lng = float(flo2d_station[flo2d_index][1])
distances = {}
for key in stations_dict.keys():
lat = float(stations_dict.get(key)[1])
lng = float(stations_dict.get(key)[0])
intermediate_value = cos(radians(flo2d_lat)) * cos(
radians(lat)) * cos(radians(lng) - radians(flo2d_lng)) + sin(
radians(flo2d_lat)) * sin(radians(lat))
if intermediate_value < 1:
distance = 6371 * acos(intermediate_value)
else:
distance = 6371 * acos(1)
distances[key] = distance
sorted_distances = collections.OrderedDict(
sorted(distances.items(), key=operator.itemgetter(1))[:10])
flo2d_stations_mapping = []
count = 0
for key in sorted_distances.keys():
if count < 1 and sorted_distances.get(key) <= 25:
flo2d_stations_mapping = stations_dict.get(key)
count += 1
elif count < 1:
flo2d_stations_mapping = [-1, -1]
count += 1
else:
continue
# print(flo2d_obs_mapping)
flo2d_stations_mapping_dict[flo2d_station[flo2d_index]
[0]] = flo2d_stations_mapping
print(json.dumps(flo2d_stations_mapping_dict))
return flo2d_stations_mapping_dict
def update_discharge_from_hechms(curw_sim_pool, curw_fcst_pool, flo2d_model,
method, start_time, end_time, sim_tag):
try:
TS = DTimeseries(pool=curw_sim_pool)
# [station_name,latitude,longitude,target,model,version,sim_tag,station]
extract_stations = read_csv(
os.path.join(ROOT_DIR,
'grids/discharge_stations/flo2d_stations.csv'))
for i in range(len(extract_stations)):
station_name = extract_stations[i][0]
latitude = extract_stations[i][1]
longitude = extract_stations[i][2]
target_model = extract_stations[i][3]
if target_model == flo2d_model:
if station_name in ("glencourse"):
meta_data = {
'latitude': float('%.6f' % float(latitude)),
'longitude': float('%.6f' % float(longitude)),
'model': target_model,
'method': method,
'grid_id': 'discharge_{}'.format(station_name)
}
tms_id = TS.get_timeseries_id_if_exists(
meta_data=meta_data)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
processed_discharge_ts = process_fcst_ts_from_hechms_outputs(
curw_fcst_pool=curw_fcst_pool,
extract_stations=extract_stations,
i=i,
start=start_time,
end=end_time,
sim_tag=sim_tag)
if processed_discharge_ts is not None and len(
processed_discharge_ts) > 0:
TS.insert_data(timeseries=processed_discharge_ts,
tms_id=tms_id,
upsert=True)
else:
continue # skip the current iteration
except Exception as e:
traceback.print_exc()
def find_nearest_obs_stations_for_flo2d_stations(flo2d_stations_csv,
obs_stations, flo2d_model):
# obs_stations : [hash_id,station_id,station_name,latitude,longitude]
flo2d_station = read_csv(flo2d_stations_csv) # [Grid_ID,X,Y]
flo2d_obs_mapping_dict = {}
for flo2d_index in range(len(flo2d_station)):
grid_id = flo2d_station[flo2d_index][0]
flo2d_obs_mapping = []
flo2d_lat = float(flo2d_station[flo2d_index][2])
flo2d_lng = float(flo2d_station[flo2d_index][1])
distances = {}
for obs_index in range(len(obs_stations)):
lat = float(obs_stations[obs_index][3])
lng = float(obs_stations[obs_index][4])
intermediate_value = cos(radians(flo2d_lat)) * cos(
radians(lat)) * cos(radians(lng) - radians(flo2d_lng)) + sin(
radians(flo2d_lat)) * sin(radians(lat))
if intermediate_value < 1:
distance = 6371 * acos(intermediate_value)
else:
distance = 6371 * acos(1)
distances[obs_stations[obs_index][1]] = distance
sorted_distances = collections.OrderedDict(
sorted(distances.items(), key=operator.itemgetter(1))[:10])
count = 0
for key in sorted_distances.keys():
if count < 3 and sorted_distances.get(key) <= 25:
flo2d_obs_mapping.append(key)
count += 1
elif count < 3:
flo2d_obs_mapping.append("-1")
count += 1
# print(flo2d_obs_mapping)
flo2d_obs_mapping_dict[grid_id] = flo2d_obs_mapping
# flo2d_grid_mappings[dict.get("grid_id")] = [dict.get("obs1"), dict.get("obs2"), dict.get("obs3")]
flo2d_grid_mappings_dict = {}
return flo2d_obs_mapping_dict
def process_tide_fcsts_from_Mobile_Geographics(existing_ts_end, obs_end):
data = read_csv('{}/{}.csv'.format(INPUT_DIR, 'colombo_MGF'))
raw_timeseries = []
expected_fcst_end = datetime.strptime(
(datetime.now() + timedelta(days=60)).strftime("%Y-%m-01 00:00:00"),
COMMON_DATE_TIME_FORMAT)
if existing_ts_end is None or existing_ts_end < expected_fcst_end:
for i in range(len(data)):
time = datetime.strptime(
"{} {} {}".format(data[i][0], data[i][1], data[i][2]),
"%m/%d/%Y %I:%M %p")
formatted_time = time.strftime(COMMON_DATE_TIME_FORMAT)
raw_timeseries.append([formatted_time, data[i][3]])
if existing_ts_end is not None:
fcst_start = (existing_ts_end).strftime(COMMON_DATE_TIME_FORMAT)
elif obs_end is not None:
fcst_start = (obs_end).strftime(COMMON_DATE_TIME_FORMAT)
else:
fcst_start = ((datetime.now() -
timedelta(days=30))).strftime("%Y-%m-%d 00:00:00")
raw_timeseries = extract_ts_from(fcst_start, raw_timeseries)
raw_timeseries.insert(0, ['time', 'value'])
timeseries_df = list_of_lists_to_df_first_row_as_columns(
raw_timeseries)
timeseries_df['time'] = pd.to_datetime(timeseries_df['time'],
format=COMMON_DATE_TIME_FORMAT)
timeseries_df['time'] = timeseries_df['time'].dt.round('h')
timeseries_df.set_index('time', inplace=True)
timeseries_df['value'] = pd.to_numeric(timeseries_df['value'])
hourly_ts_df = timeseries_df.resample('H').asfreq()
hourly_ts_df = hourly_ts_df.interpolate(method='linear',
limit_direction='both',
limit=100) ####temp###
hourly_ts_df = hourly_ts_df.fillna(-99999.000)
hourly_ts_df.index = hourly_ts_df.index.map(str)
pd.set_option('display.max_rows', hourly_ts_df.shape[0] + 2)
pd.set_option('display.max_columns', hourly_ts_df.shape[1] + 2)
print(hourly_ts_df)
processed_timeseries = hourly_ts_df.reset_index().values.tolist()
return processed_timeseries
else:
return None
def divide_flo2d_grids_to_polygons(flo2d_model, polygons):
flo2d_grids = read_csv(
os.path.join(ROOT_DIR, 'grids/flo2d/{}m.csv'.format(
flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)]
for grid in flo2d_grids:
point = Point(float(grid[1]), float(grid[2]))
for index, row in polygons.iterrows():
polygon = polygons.iloc[index]['geometry']
if point.within(polygon):
grid.append(polygons.iloc[index]['id'])
continue
return flo2d_grids
def update_tide_obs(curw_sim_pool, obs_connection, method, timestep, start_time, end_time, flo2d_model="flo2d"):
# [station_name,latitude,longitude,target]
extract_stations = read_csv('grids/tide_stations/extract_stations.csv')
extract_stations_dict = {} # keys: station_name , value: [latitude, longitude, target_model]
for obs_index in range(len(extract_stations)):
extract_stations_dict[extract_stations[obs_index][0]] = [extract_stations[obs_index][1],
extract_stations[obs_index][2],
extract_stations[obs_index][3]]
for station_name in extract_stations_dict.keys():
meta_data = {
'latitude': float('%.6f' % float(extract_stations_dict.get(station_name)[0])),
'longitude': float('%.6f' % float(extract_stations_dict.get(station_name)[1])),
'model': flo2d_model, 'method': method,
'grid_id': 'tide_{}'.format(station_name)
}
TS = Timeseries(pool=curw_sim_pool)
tms_id = TS.get_timeseries_id_if_exists(meta_data=meta_data)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
processed_tide_ts = prepare_obs_tide_ts(connection=obs_connection, start_date=start_time, end_date=end_time,
extract_station=station_name)
for i in range(len(processed_tide_ts)):
if len(processed_tide_ts[i]) < 2:
processed_tide_ts.remove(processed_tide_ts[i])
final_tide_ts = fill_ts_missing_entries(start=start_time, end=end_time, timeseries=processed_tide_ts,
interpolation_method='linear', timestep=timestep)
if final_tide_ts is not None and len(final_tide_ts) > 0:
TS.insert_data(timeseries=final_tide_ts, tms_id=tms_id, upsert=True)
def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep):
"""
Update rainfall observations for flo2d models
:param flo2d_model: flo2d model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param timestep: output timeseries timestep
:return:
"""
now = datetime.now()
OBS_START_STRING = (now - timedelta(days=10)).strftime('%Y-%m-%d %H:00:00')
OBS_START = datetime.strptime(OBS_START_STRING, '%Y-%m-%d %H:%M:%S')
try:
# Connect to the database
curw_obs_pool = get_Pool(host=CURW_OBS_HOST,
user=CURW_OBS_USERNAME,
password=CURW_OBS_PASSWORD,
port=CURW_OBS_PORT,
db=CURW_OBS_DATABASE)
curw_obs_connection = curw_obs_pool.connection()
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
# test ######
# pool = get_Pool(host=HOST, user=USERNAME, password=PASSWORD, port=PORT, db=DATABASE)
TS = Sim_Timeseries(pool=curw_sim_pool)
# [hash_id, station_id, station_name, latitude, longitude]
active_obs_stations = read_csv(
'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'
)
flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format(
flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)]
stations_dict_for_obs = {} # keys: obs station id , value: hash id
for obs_index in range(len(active_obs_stations)):
stations_dict_for_obs[active_obs_stations[obs_index]
[1]] = active_obs_stations[obs_index][0]
flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(
pool=curw_sim_pool,
grid_interpolation=grid_interpolation,
flo2d_model=flo2d_model)
for flo2d_index in range(len(flo2d_grids)):
obs_start = OBS_START
lat = flo2d_grids[flo2d_index][2]
lon = flo2d_grids[flo2d_index][1]
cell_id = flo2d_grids[flo2d_index][0]
meta_data = {
'latitude':
float('%.6f' % float(lat)),
'longitude':
float('%.6f' % float(lon)),
'model':
flo2d_model,
'method':
method,
'grid_id':
'{}_{}_{}'.format(flo2d_model, grid_interpolation,
(str(cell_id)).zfill(10))
}
tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
method=meta_data.get('method'))
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
obs_end = TS.get_obs_end(id_=tms_id)
if obs_end is not None:
obs_start = obs_end - timedelta(hours=1)
obs1_station_id = str(
flo2d_obs_mapping.get(meta_data['grid_id'])[0])
obs2_station_id = str(
flo2d_obs_mapping.get(meta_data['grid_id'])[1])
obs3_station_id = str(
flo2d_obs_mapping.get(meta_data['grid_id'])[2])
obs_timeseries = []
if timestep == 5:
if obs1_station_id != str(-1):
obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)
ts = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs1_hash_id)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_5_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
if obs2_station_id != str(-1):
obs2_hash_id = stations_dict_for_obs.get(
obs2_station_id)
ts2 = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs2_hash_id)
if ts2 is not None and len(ts2) > 1:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts2,
OBS_TS=obs_timeseries)
if obs_timeseries is not None and len(
obs_timeseries) > 0:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_5_min_ts(
newly_extracted_timeseries=ts2,
expected_start=expected_start)[1:])
# obs_start = ts2[-1][0]
if obs3_station_id != str(-1):
obs3_hash_id = stations_dict_for_obs.get(
obs3_station_id)
ts3 = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs3_hash_id)
if ts3 is not None and len(ts3) > 1 and len(
obs_timeseries) > 0:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts3,
OBS_TS=obs_timeseries)
if obs_timeseries is not None:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_5_min_ts(
newly_extracted_timeseries=ts3,
expected_start=expected_start)[1:])
elif timestep == 15:
if obs1_station_id != str(-1):
obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)
ts = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs1_hash_id)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_15_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
if obs2_station_id != str(-1):
obs2_hash_id = stations_dict_for_obs.get(
obs2_station_id)
ts2 = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs2_hash_id)
if ts2 is not None and len(ts2) > 1:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts2,
OBS_TS=obs_timeseries)
if obs_timeseries is not None and len(
obs_timeseries) > 0:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_15_min_ts(
newly_extracted_timeseries=ts2,
expected_start=expected_start)[1:])
# obs_start = ts2[-1][0]
if obs3_station_id != str(-1):
obs3_hash_id = stations_dict_for_obs.get(
obs3_station_id)
ts3 = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs3_hash_id)
if ts3 is not None and len(ts3) > 1 and len(
obs_timeseries) > 0:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts3,
OBS_TS=obs_timeseries)
if obs_timeseries is not None:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_15_min_ts(
newly_extracted_timeseries=ts3,
expected_start=expected_start)[1:])
for i in range(len(obs_timeseries)):
if obs_timeseries[i][1] == -99999:
obs_timeseries[i][1] = 0
if obs_timeseries is not None and len(obs_timeseries) > 0:
TS.insert_data(timeseries=obs_timeseries,
tms_id=tms_id,
upsert=True)
TS.update_latest_obs(id_=tms_id,
obs_end=(obs_timeseries[-1][1]))
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating obs rainfalls in curw_sim.")
finally:
curw_obs_connection.close()
destroy_Pool(pool=curw_sim_pool)
destroy_Pool(pool=curw_obs_pool)
logger.info("Process finished")
def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep,
start_time, end_time):
"""
Update rainfall observations for flo2d models
:param flo2d_model: flo2d model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param timestep: output timeseries timestep
:return:
"""
obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
try:
# Connect to the database
curw_obs_pool = get_Pool(host=con_params.CURW_OBS_HOST,
user=con_params.CURW_OBS_USERNAME,
password=con_params.CURW_OBS_PASSWORD,
port=con_params.CURW_OBS_PORT,
db=con_params.CURW_OBS_DATABASE)
curw_obs_connection = curw_obs_pool.connection()
curw_sim_pool = get_Pool(host=con_params.CURW_SIM_HOST,
user=con_params.CURW_SIM_USERNAME,
password=con_params.CURW_SIM_PASSWORD,
port=con_params.CURW_SIM_PORT,
db=con_params.CURW_SIM_DATABASE)
TS = Sim_Timeseries(pool=curw_sim_pool)
# [hash_id, station_id, station_name, latitude, longitude]
# active_obs_stations = read_csv(os.path.join(ROOT_DIR,'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'))
active_obs_stations = extract_active_curw_obs_rainfall_stations(
start_time=start_time, end_time=end_time)[1:]
flo2d_grids = read_csv(
os.path.join(ROOT_DIR, 'grids/flo2d/{}m.csv'.format(
flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)]
stations_dict_for_obs = {} # keys: obs station id , value: hash id
for obs_index in range(len(active_obs_stations)):
stations_dict_for_obs[active_obs_stations[obs_index]
[1]] = active_obs_stations[obs_index][0]
# flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model)
flo2d_obs_mapping = find_nearest_obs_stations_for_flo2d_stations(
flo2d_stations_csv=os.path.join(
ROOT_DIR, 'grids/flo2d/{}m.csv'.format(flo2d_model)),
obs_stations=active_obs_stations,
flo2d_model=flo2d_model)
for flo2d_index in range(len(flo2d_grids)):
lat = flo2d_grids[flo2d_index][2]
lon = flo2d_grids[flo2d_index][1]
cell_id = flo2d_grids[flo2d_index][0]
meta_data = {
'latitude':
float('%.6f' % float(lat)),
'longitude':
float('%.6f' % float(lon)),
'model':
flo2d_model,
'method':
method,
'grid_id':
'{}_{}_{}'.format(flo2d_model, grid_interpolation,
(str(cell_id)).zfill(10))
}
tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
method=meta_data.get('method'))
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
print("grid_id:", cell_id)
print("grid map:", flo2d_obs_mapping.get(cell_id))
obs1_station_id = flo2d_obs_mapping.get(cell_id)[0]
obs2_station_id = flo2d_obs_mapping.get(cell_id)[1]
obs3_station_id = flo2d_obs_mapping.get(cell_id)[2]
obs_timeseries = []
if timestep == 5:
if obs1_station_id != str(-1):
obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)
ts = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs1_hash_id,
end_time=end_time)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_5_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
if obs2_station_id != str(-1):
obs2_hash_id = stations_dict_for_obs.get(
obs2_station_id)
ts2 = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs2_hash_id,
end_time=end_time)
if ts2 is not None and len(ts2) > 1:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts2,
OBS_TS=obs_timeseries)
if obs_timeseries is not None and len(
obs_timeseries) > 0:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_5_min_ts(
newly_extracted_timeseries=ts2,
expected_start=expected_start)[1:])
# obs_start = ts2[-1][0]
if obs3_station_id != str(-1):
obs3_hash_id = stations_dict_for_obs.get(
obs3_station_id)
ts3 = extract_obs_rain_5_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs3_hash_id,
end_time=end_time)
if ts3 is not None and len(ts3) > 1 and len(
obs_timeseries) > 0:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts3,
OBS_TS=obs_timeseries)
if obs_timeseries is not None:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_5_min_ts(
newly_extracted_timeseries=ts3,
expected_start=expected_start)[1:])
elif timestep == 15:
if obs1_station_id != str(-1):
obs1_hash_id = stations_dict_for_obs.get(obs1_station_id)
ts = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs1_hash_id,
end_time=end_time)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_15_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
if obs2_station_id != str(-1):
obs2_hash_id = stations_dict_for_obs.get(
obs2_station_id)
ts2 = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs2_hash_id,
end_time=end_time)
if ts2 is not None and len(ts2) > 1:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts2,
OBS_TS=obs_timeseries)
if obs_timeseries is not None and len(
obs_timeseries) > 0:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_15_min_ts(
newly_extracted_timeseries=ts2,
expected_start=expected_start)[1:])
# obs_start = ts2[-1][0]
if obs3_station_id != str(-1):
obs3_hash_id = stations_dict_for_obs.get(
obs3_station_id)
ts3 = extract_obs_rain_15_min_ts(
connection=curw_obs_connection,
start_time=obs_start,
id=obs3_hash_id,
end_time=end_time)
if ts3 is not None and len(ts3) > 1 and len(
obs_timeseries) > 0:
obs_timeseries = fill_missing_values(
newly_extracted_timeseries=ts3,
OBS_TS=obs_timeseries)
if obs_timeseries is not None:
expected_start = obs_timeseries[-1][0]
else:
expected_start = obs_start
obs_timeseries.extend(
process_15_min_ts(
newly_extracted_timeseries=ts3,
expected_start=expected_start)[1:])
for i in range(len(obs_timeseries)):
if obs_timeseries[i][1] == -99999:
obs_timeseries[i][1] = 0
print("### obs timeseries length ###", len(obs_timeseries))
if obs_timeseries is not None and len(
obs_timeseries) > 0 and obs_timeseries[-1][0] != end_time:
obs_timeseries.append(
[datetime.strptime(end_time, DATE_TIME_FORMAT), 0])
final_ts = process_continuous_ts(original_ts=obs_timeseries,
expected_start=datetime.strptime(
start_time, DATE_TIME_FORMAT),
filling_value=0,
timestep=timestep)
if final_ts is not None and len(final_ts) > 0:
TS.insert_data(timeseries=final_ts, tms_id=tms_id, upsert=True)
TS.update_latest_obs(id_=tms_id, obs_end=(final_ts[-1][1]))
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating obs rainfalls in curw_sim.")
finally:
curw_obs_connection.close()
destroy_Pool(pool=curw_sim_pool)
destroy_Pool(pool=curw_obs_pool)
logger.info("Process finished")
# source details
FLO2D_250_params = json.loads(open('flo2d_250.json').read())
FLO2D_150_params = json.loads(open('flo2d_150.json').read())
FLO2D_model = 'FLO2D'
FLO2D_250_version = '250'
FLO2D_150_version = '150'
# unit details
unit = 'm'
unit_type = UnitType.getType('Instantaneous')
# variable details
variable = 'WaterLevel'
# station details
flo2d_250_grids = read_csv('flo2d_250m.csv')
flo2d_150_grids = read_csv('flo2d_150m.csv')
pool = get_Pool(host=CURW_FCST_HOST,
port=CURW_FCST_PORT,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
db=CURW_FCST_DATABASE)
# ########
# # test
# ########
#
# USERNAME = "******"
# PASSWORD = "******"
# HOST = "127.0.0.1"
ROOT_DIRECTORY = '/home/uwcc-admin/curw_mike_data_handler'
if __name__ == "__main__":
set_db_config_file_path(
os.path.join(ROOT_DIRECTORY, 'db_adapter_config.json'))
try:
##################################
# Initialize parameters for MIKE #
##################################
# station details
mike_stations = read_csv(
os.path.join(ROOT_DIRECTORY, 'resources/mike_stations.csv'))
pool = get_Pool(host=con_params.CURW_FCST_HOST,
port=con_params.CURW_FCST_PORT,
user=con_params.CURW_FCST_USERNAME,
password=con_params.CURW_FCST_PASSWORD,
db=con_params.CURW_FCST_DATABASE)
for station in mike_stations:
id = station[0]
station_name = station[1]
lat = station[2]
lon = station[3]
add_station(pool=pool,
name=station_name,
latitude="%.6f" % float(lat),
def update_rainfall_obs(curw_obs_pool, curw_sim_pool, flo2d_model, method, grid_interpolation, timestep, start_time, end_time):
"""
Update rainfall observations for flo2d models
:param flo2d_model: flo2d model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param timestep: output timeseries timestep
:return:
"""
# obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S')
try:
curw_obs_connection = curw_obs_pool.connection()
# [hash_id, station_id, station_name, latitude, longitude]
# active_obs_stations = read_csv(os.path.join(ROOT_DIR,'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'))
active_obs_stations = extract_active_curw_obs_rainfall_stations(curw_obs_pool=curw_obs_pool, start_time=start_time, end_time=end_time)[1:]
flo2d_grids = read_csv(os.path.join(ROOT_DIR,'grids/flo2d/{}m.csv'.format(flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)]
stations_dict_for_obs = { } # keys: obs station id , value: hash id
for obs_index in range(len(active_obs_stations)):
stations_dict_for_obs[active_obs_stations[obs_index][1]] = active_obs_stations[obs_index][0]
# flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model)
flo2d_obs_mapping = find_nearest_obs_stations_for_flo2d_stations(
flo2d_stations_csv=os.path.join(ROOT_DIR,'grids/flo2d/{}m.csv'.format(flo2d_model)),
obs_stations=active_obs_stations, flo2d_model=flo2d_model)
# retrieve observed timeseries
obs_df = pd.DataFrame()
obs_df['time'] = pd.date_range(start=start_time, end=end_time, freq='5min')
for obs_id in stations_dict_for_obs.keys():
ts = extract_obs_rain_5_min_ts(connection=curw_obs_connection, start_time=start_time,
id=stations_dict_for_obs.get(obs_id), end_time=end_time)
ts.insert(0, ['time', obs_id])
ts_df = list_of_lists_to_df_first_row_as_columns(ts)
ts_df[obs_id] = ts_df[obs_id].astype('float64')
obs_df = pd.merge(obs_df, ts_df, how="left", on='time')
obs_df.set_index('time', inplace=True)
obs_df['0'] = 0
if timestep == 15:
obs_df = obs_df.resample('15min', label='right', closed='right').sum()
TS = Sim_Timeseries(pool=curw_sim_pool)
for flo2d_index in range(len(flo2d_grids)):
lat = flo2d_grids[flo2d_index][2]
lon = flo2d_grids[flo2d_index][1]
cell_id = flo2d_grids[flo2d_index][0]
meta_data = {
'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)),
'model': flo2d_model, 'method': method,
'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10))
}
tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method'))
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
print(datetime.now().strftime(DATE_TIME_FORMAT))
print("grid_id:", cell_id)
obs_station_ids = flo2d_obs_mapping.get(cell_id)
if len(obs_station_ids) == 1:
obs_ts_df = obs_df[obs_station_ids].to_frame(name='final')
elif len(obs_station_ids) == 2:
obs_ts_df = obs_df[obs_station_ids]
obs_ts_df[obs_station_ids[0]] = obs_ts_df[obs_station_ids[0]].fillna(obs_ts_df[obs_station_ids[1]])
obs_ts_df['final'] = obs_ts_df[obs_station_ids[0]]
elif len(obs_station_ids) == 3:
obs_ts_df = obs_df[obs_station_ids]
obs_ts_df[obs_station_ids[1]] = obs_ts_df[obs_station_ids[1]].fillna(obs_ts_df[obs_station_ids[2]])
obs_ts_df[obs_station_ids[0]] = obs_ts_df[obs_station_ids[0]].fillna(obs_ts_df[obs_station_ids[1]])
obs_ts_df['final'] = obs_ts_df[obs_station_ids[0]]
else:
obs_ts_df = obs_df['0'].to_frame(name='final')
final_ts_df = obs_ts_df['final'].reset_index()
final_ts_df['time'] = final_ts_df['time'].dt.strftime(DATE_TIME_FORMAT)
final_ts = final_ts_df.values.tolist()
if final_ts is not None and len(final_ts) > 0:
TS.replace_data(timeseries=final_ts, tms_id=tms_id)
TS.update_latest_obs(id_=tms_id, obs_end=(final_ts[-1][1]))
except Exception as e:
traceback.print_exc()
logger.error("Exception occurred while updating obs rainfalls in curw_sim.")
finally:
curw_obs_connection.close()
destroy_Pool(pool=curw_sim_pool)
destroy_Pool(pool=curw_obs_pool)
logger.info("Process finished")
def update_rainfall_obs(target_model, method, grid_interpolation, timestep):
"""
Update rainfall observations for flo2d models
:param model: target model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param timestep:
:return:
"""
now = datetime.now()
OBS_START_STRING = (now + timedelta(hours=3)).strftime('%Y-%m-%d %H:00:00')
OBS_START = datetime.strptime(OBS_START_STRING, '%Y-%m-%d %H:%M:%S')
try:
# Connect to the database
curw_obs_pool = get_Pool(host=CURW_OBS_HOST,
user=CURW_OBS_USERNAME,
password=CURW_OBS_PASSWORD,
port=CURW_OBS_PORT,
db=CURW_OBS_DATABASE)
curw_obs_connection = curw_obs_pool.connection()
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
TS = Timeseries(pool=curw_sim_pool)
# [hash_id, station_id, station_name, latitude, longitude]
active_obs_stations = read_csv(
'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'
)
obs_stations_dict = {
} # keys: obs station id , value: [hash id, name, latitude, longitude]
for obs_index in range(len(active_obs_stations)):
obs_stations_dict[active_obs_stations[obs_index][1]] = [
active_obs_stations[obs_index][0],
active_obs_stations[obs_index][2],
active_obs_stations[obs_index][3],
active_obs_stations[obs_index][4]
]
for obs_id in obs_stations_dict.keys():
obs_start = OBS_START
meta_data = {
'latitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[2])),
'longitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[3])),
'model':
target_model,
'method':
method,
'grid_id':
'rainfall_{}_{}_{}'.format(obs_id,
obs_stations_dict.get(obs_id)[1],
grid_interpolation)
}
tms_id = TS.get_timeseries_id_if_exists(meta_data=meta_data)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
TS.update_grid_id(id_=tms_id, grid_id=meta_data['grid_id'])
obs_end = TS.get_obs_end(id_=tms_id)
if obs_end is not None:
obs_start = obs_end
obs_hash_id = obs_stations_dict.get(obs_id)[0]
obs_timeseries = []
if timestep == 5:
ts = extract_obs_rain_5_min_ts(connection=curw_obs_connection,
start_time=obs_start,
id=obs_hash_id)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_5_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
elif timestep == 15:
ts = extract_obs_rain_15_min_ts(connection=curw_obs_connection,
start_time=obs_start,
id=obs_hash_id)
if ts is not None and len(ts) > 1:
obs_timeseries.extend(
process_15_min_ts(newly_extracted_timeseries=ts,
expected_start=obs_start)[1:])
# obs_start = ts[-1][0]
# for i in range(len(obs_timeseries)):
# if obs_timeseries[i][1] == -99999:
# obs_timeseries[i][1] = 0
if obs_timeseries is not None and len(obs_timeseries) > 0:
TS.insert_data(timeseries=obs_timeseries,
tms_id=tms_id,
upsert=True)
TS.update_latest_obs(id_=tms_id,
obs_end=(obs_timeseries[-1][1]))
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating obs rainfalls in curw_sim.")
finally:
curw_obs_connection.close()
destroy_Pool(pool=curw_sim_pool)
destroy_Pool(pool=curw_obs_pool)
open(
os.path.join(
ROOT_DIRECTORY,
'res/flo2d_extract_stations/flo2d_150_v2.json')).read())
FLO2D_model = 'FLO2D'
FLO2D_150_v2_version = '150_v2'
# unit details
unit = 'm'
unit_type = UnitType.getType('Instantaneous')
# variable details
variable = 'WaterLevel'
# station details
flo2d_150_v2_grids = read_csv(
os.path.join(ROOT_DIRECTORY, 'res/grids/flo2d_150_v2m.csv'))
pool = get_Pool(host=con_params.CURW_FCST_HOST,
port=con_params.CURW_FCST_PORT,
user=con_params.CURW_FCST_USERNAME,
password=con_params.CURW_FCST_PASSWORD,
db=con_params.CURW_FCST_DATABASE)
add_source(pool=pool,
model=FLO2D_model,
version=FLO2D_150_v2_version,
parameters=FLO2D_150_v2_params)
# add_variable(pool=pool, variable=variable)
# add_unit(pool=pool, unit=unit, unit_type=unit_type)
# add flo2d 150 v2 output stations
curw_obs_pool = get_Pool(host=CURW_OBS_HOST,
user=CURW_OBS_USERNAME,
password=CURW_OBS_PASSWORD,
port=CURW_OBS_PORT,
db=CURW_OBS_DATABASE)
connection = curw_obs_pool.connection()
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
# [station_name,latitude,longitude,target]
extract_stations = read_csv('grids/tide_stations/extract_stations.csv')
extract_stations_dict = {
} # keys: station_name , value: [latitude, longitude, target_model]
for obs_index in range(len(extract_stations)):
extract_stations_dict[extract_stations[obs_index][0]] = [
extract_stations[obs_index][1], extract_stations[obs_index][2],
extract_stations[obs_index][3]
]
methods = []
methods.append(MethodEnum.getAbbreviation(MethodEnum.TSF))
methods.append(MethodEnum.getAbbreviation(MethodEnum.MGF))
for station_name in extract_stations_dict.keys():
return processed_timeseries
else:
return None
if __name__ == "__main__":
try:
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
# [station_name,latitude,longitude,target]
extract_stations = read_csv('grids/tide_stations/extract_stations.csv')
extract_stations_dict = {
} # keys: station_name , value: [latitude, longitude, target_method]
for obs_index in range(len(extract_stations)):
extract_stations_dict[extract_stations[obs_index][0]] = [
extract_stations[obs_index][1], extract_stations[obs_index][2],
extract_stations[obs_index][3]
]
for station_name in extract_stations_dict.keys():
fcst_station_name = None
methods = []
if station_name in ('colombo', 'mattakkuliya'):
fcst_station_name = 'colombo' #temporary#
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
TS = Timeseries(pool=curw_sim_pool)
# [station_name,latitude,longitude,target,model,version,sim_tag,station]
extract_stations = read_csv(
'grids/discharge_stations/mike_stations.csv')
for i in range(len(extract_stations)):
station_name = extract_stations[i][0]
latitude = extract_stations[i][1]
longitude = extract_stations[i][2]
target_model = extract_stations[i][3]
if station_name in ('ambatale'):
method = MethodEnum.getAbbreviation(MethodEnum.MME)
else:
continue ## skip the current station and move to next iteration
meta_data = {
'latitude': float('%.6f' % float(latitude)),
'longitude': float('%.6f' % float(longitude)),
from db_adapter.curw_sim.timeseries.waterlevel import Timeseries
from db_adapter.curw_sim.timeseries import MethodEnum
from db_adapter.curw_sim.common import fill_ts_missing_entries
from db_adapter.logger import logger
if __name__ == "__main__":
try:
print("Fill missing values in hanwella waterlevel series")
curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT, db=CURW_SIM_DATABASE)
method = MethodEnum.getAbbreviation(MethodEnum.SF)
# [station_name,latitude,longitude,target]
extract_stations = read_csv('grids/waterlevel_stations/extract_stations.csv')
extract_stations_dict = {} # keys: station_name , value: [latitude, longitude, target_method]
for obs_index in range(len(extract_stations)):
extract_stations_dict[extract_stations[obs_index][0]] = [extract_stations[obs_index][1],
extract_stations[obs_index][2],
extract_stations[obs_index][3]]
station_name = 'hanwella'
meta_data = {
'latitude': float('%.6f' % float(extract_stations_dict.get(station_name)[0])),
'longitude': float('%.6f' % float(extract_stations_dict.get(station_name)[1])),
'model': extract_stations_dict.get(station_name)[2], 'method': method,
'grid_id': 'waterlevel_{}'.format(station_name)
}
def update_rainfall_fcsts(target_model, method, grid_interpolation, model_list,
timestep):
"""
Update rainfall forecasts for flo2d models
:param target_model: target model for which input ins prepared
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param model_list: list of forecast model and their versions used to calculate the rainfall
e.g.: [["WRF_E", "4.0", "evening_18hrs"],["WRF_SE", "v4", ,"evening_18hrs"],["WRF_Ensemble", "4.0", ,"MME"]]
:param timestep: output timeseries timestep
:return:
"""
try:
# Connect to the database
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
Sim_TS = Sim_Timeseries(pool=curw_sim_pool)
Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool)
# [hash_id, station_id, station_name, latitude, longitude]
active_obs_stations = read_csv(
'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'
)
obs_stations_dict = {
} # keys: obs station id , value: [name, latitude, longitude]
for obs_index in range(len(active_obs_stations)):
obs_stations_dict[active_obs_stations[obs_index][1]] = [
active_obs_stations[obs_index][2],
active_obs_stations[obs_index][3],
active_obs_stations[obs_index][4]
]
obs_d03_mapping = get_obs_to_d03_grid_mappings_for_rainfall(
pool=curw_sim_pool, grid_interpolation=grid_interpolation)
for obs_id in obs_stations_dict.keys():
meta_data = {
'latitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[1])),
'longitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[2])),
'model':
target_model,
'method':
method,
'grid_id':
'rainfall_{}_{}_{}'.format(obs_id,
obs_stations_dict.get(obs_id)[0],
grid_interpolation)
}
tms_id = Sim_TS.get_timeseries_id_if_exists(meta_data=meta_data)
if tms_id is None:
tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
Sim_TS.insert_run(meta_data=meta_data)
obs_end = Sim_TS.get_obs_end(id_=tms_id)
fcst_timeseries = []
for i in range(len(model_list)):
source_id = get_source_id(pool=curw_fcst_pool,
model=model_list[i][0],
version=model_list[i][1])
sim_tag = model_list[i][2]
coefficient = model_list[i][3]
temp_timeseries = []
if timestep == 5:
if obs_end is not None:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=obs_d03_mapping.
get(meta_data['grid_id'])[0],
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1))
else:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=obs_d03_mapping.
get(meta_data['grid_id'])[0],
source_id=source_id,
variable_id=1,
unit_id=1))
elif timestep == 15:
if obs_end is not None:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=obs_d03_mapping.get(
meta_data['grid_id'])[0],
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1)
else:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=obs_d03_mapping.get(
meta_data['grid_id'])[0],
source_id=source_id,
variable_id=1,
unit_id=1)
if coefficient != 1:
for j in range(len(temp_timeseries)):
temp_timeseries[j][1] = float(
temp_timeseries[j][1]) * coefficient
if i == 0:
fcst_timeseries = temp_timeseries
else:
fcst_timeseries = append_value_for_timestamp(
existing_ts=fcst_timeseries, new_ts=temp_timeseries)
sum_timeseries = summed_timeseries(fcst_timeseries)
for i in range(len(sum_timeseries)):
if float(sum_timeseries[i][1]) < 0:
sum_timeseries[i][1] = 0
if sum_timeseries is not None and len(sum_timeseries) > 0:
Sim_TS.insert_data(timeseries=sum_timeseries,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating fcst rainfalls in curw_sim.")
finally:
destroy_Pool(curw_sim_pool)
destroy_Pool(curw_fcst_pool)
if __name__=="__main__":
try:
curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT, db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT, db=CURW_FCST_DATABASE)
TS = Timeseries(pool=curw_sim_pool)
# [station_name,latitude,longitude,target,model,version,sim_tag,station]
extract_stations = read_csv('grids/discharge_stations/flo2d_stations.csv')
for i in range(len(extract_stations)):
station_name = extract_stations[i][0]
latitude = extract_stations[i][1]
longitude = extract_stations[i][2]
target_model = extract_stations[i][3]
method = extract_stations[i][4]
meta_data = {
'latitude': float('%.6f' % float(latitude)),
'longitude': float('%.6f' % float(longitude)),
'model': target_model, 'method': method,
'grid_id': 'discharge_{}'.format(station_name)
}
def update_rainfall_fcsts(flo2d_model, method, grid_interpolation, model_list,
timestep):
"""
Update rainfall forecasts for flo2d models
:param flo2d_model: flo2d model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param model_list: list of forecast model and their versions used to calculate the rainfall
e.g.: [["WRF_E", "v4"],["WRF_SE", "v4"]]
:param timestep: output timeseries timestep
:return:
"""
try:
# Connect to the database
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
Sim_TS = Sim_Timeseries(pool=curw_sim_pool)
Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool)
flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format(
flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)]
flo2d_wrf_mapping = get_flo2d_cells_to_wrf_grid_mappings(
pool=curw_sim_pool,
grid_interpolation=grid_interpolation,
flo2d_model=flo2d_model)
for flo2d_index in range(len(flo2d_grids)): # len(flo2d_grids)
lat = flo2d_grids[flo2d_index][2]
lon = flo2d_grids[flo2d_index][1]
cell_id = flo2d_grids[flo2d_index][0]
meta_data = {
'latitude':
float('%.6f' % float(lat)),
'longitude':
float('%.6f' % float(lon)),
'model':
flo2d_model,
'method':
method,
'grid_id':
'{}_{}_{}'.format(flo2d_model, grid_interpolation,
(str(cell_id)).zfill(10))
}
tms_id = Sim_TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
method=meta_data.get('method'))
if tms_id is None:
tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
Sim_TS.insert_run(meta_data=meta_data)
obs_end = Sim_TS.get_obs_end(id_=tms_id)
fcst_timeseries = []
for i in range(len(model_list)):
source_id = get_source_id(pool=curw_fcst_pool,
model=model_list[i][0],
version=model_list[i][1])
sim_tag = model_list[i][2]
coefficient = model_list[i][3]
temp_timeseries = []
if timestep == 5:
if obs_end is not None:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.
get(meta_data['grid_id']),
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1))
else:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.
get(meta_data['grid_id']),
source_id=source_id,
variable_id=1,
unit_id=1))
elif timestep == 15:
if obs_end is not None:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.get(
meta_data['grid_id']),
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1)
else:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.get(
meta_data['grid_id']),
source_id=source_id,
variable_id=1,
unit_id=1)
if coefficient != 1:
for j in range(len(temp_timeseries)):
temp_timeseries[j][1] = float(
temp_timeseries[j][1]) * coefficient
if i == 0:
fcst_timeseries = temp_timeseries
else:
fcst_timeseries = append_value_for_timestamp(
existing_ts=fcst_timeseries, new_ts=temp_timeseries)
sum_timeseries = summed_timeseries(fcst_timeseries)
for i in range(len(sum_timeseries)):
if float(sum_timeseries[i][1]) < 0:
sum_timeseries[i][1] = 0
if sum_timeseries is not None and len(sum_timeseries) > 0:
Sim_TS.insert_data(timeseries=sum_timeseries,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating fcst rainfalls in curw_sim.")
finally:
destroy_Pool(curw_sim_pool)
destroy_Pool(curw_fcst_pool)
# Initialize parameters for DISTRIBUTED HECHMS #
################################################
# source details
model = 'HECHMS'
version = 'distributed'
# unit details
unit = 'm3/s'
unit_type = UnitType.getType('Instantaneous')
# variable details
variable = 'Discharge'
# station details
hechms_stations = read_csv('hechms_stations.csv')
pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD,
db=CURW_FCST_DATABASE)
# ########
# # test
# ########
# USERNAME = "******"
# PASSWORD = "******"
# HOST = "127.0.0.1"
# PORT = 3306
# DATABASE = "curw_fcst"
# pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE)
def update_waterlevel_obs(obs_connection, curw_sim_pool, flo2d_model, method,
timestep, start_time, end_time):
try:
# [station_name,latitude,longitude,target]
extract_stations = read_csv(
'grids/waterlevel_stations/extract_stations.csv')
extract_stations_dict = {
} # keys: target_model , value: [latitude, longitude, station_name]
# older version ::: keys: station_name , value: [latitude, longitude, target_model]
for obs_index in range(len(extract_stations)):
extract_stations_dict[extract_stations[obs_index][3]] = [
extract_stations[obs_index][1], extract_stations[obs_index][2],
extract_stations[obs_index][0]
]
station_name = extract_stations_dict.get(flo2d_model)[2]
meta_data = {
'latitude':
float('%.6f' % float(extract_stations_dict.get(flo2d_model)[0])),
'longitude':
float('%.6f' % float(extract_stations_dict.get(flo2d_model)[1])),
'model':
flo2d_model,
'method':
method,
'grid_id':
'waterlevel_{}'.format(station_name)
}
TS = Timeseries(pool=curw_sim_pool)
tms_id = TS.get_timeseries_id_if_exists(meta_data=meta_data)
ranwala_ts = []
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
TS.insert_run(meta_data=meta_data)
with obs_connection.cursor() as cursor1:
cursor1.callproc('getWL', (RANWALA_WL_ID, start_time, end_time))
results = cursor1.fetchall()
for result in results:
ranwala_ts.append([result.get('time'), result.get('value')])
interpolated_ranwala_ts = fill_ts_missing_entries(
start=start_time,
end=end_time,
timeseries=ranwala_ts,
interpolation_method='linear',
timestep=60)
estimated_wl_ts = []
if station_name == 'hanwella':
estimated_wl_ts = calculate_hanwella_wl_from_ranwala(
interpolated_ranwala_ts)
elif station_name == 'glencourse':
estimated_wl_ts = calculate_glencourse_wl_from_ranwala(
interpolated_ranwala_ts)
if estimated_wl_ts is not None and len(estimated_wl_ts) > 0:
TS.insert_data(timeseries=estimated_wl_ts,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
shape_file_path = os.path.join(ROOT_DIR,
'shape_files/250m_model/250m_model.shp')
output_shape_file_path = os.path.join(
ROOT_DIR, 'shape_files/output_temp', "{}_out_shp.shp".format(
(datetime.now()).strftime("%Y-%m-%d_%H-%M-%S")))
polygons = get_voronoi_polygons(points_dict=points_dict,
shape_file=shape_file_path,
shape_attribute=['Id', 0],
output_shape_file=output_shape_file_path,
add_total_area=True)
flo2d_model = "flo2d_250"
flo2d_grids = read_csv(
os.path.join(ROOT_DIR, 'grids/flo2d/{}m.csv'.format(
flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)]
for grid in flo2d_grids:
point = Point(float(grid[1]), float(grid[2]))
for index, row in polygons.iterrows():
polygon = polygons.iloc[index]['geometry']
if point.within(polygon):
grid.append(polygons.iloc[index]['id'])
continue
print(flo2d_grids)
count = 0
for grid in flo2d_grids:
def update_discharge_obs(curw_sim_pool, flo2d_model, method, timestep,
start_time, end_time):
try:
discharge_TS = DTimeseries(pool=curw_sim_pool)
waterlevel_TS = WLTimeseries(pool=curw_sim_pool)
# [station_name,latitude,longitude,target]
extract_stations = read_csv(
'grids/discharge_stations/flo2d_stations.csv')
extract_stations_dict = {
} # keys: target_model , value: [latitude, longitude, station_name]
# keys: station_name , value: [latitude, longitude, target_model]
for obs_index in range(len(extract_stations)):
extract_stations_dict[extract_stations[obs_index][3]] = [
extract_stations[obs_index][1], extract_stations[obs_index][2],
extract_stations[obs_index][0]
]
station_name = extract_stations_dict.get(flo2d_model)[2]
meta_data = {
'latitude':
float('%.6f' % float(extract_stations_dict.get(flo2d_model)[0])),
'longitude':
float('%.6f' % float(extract_stations_dict.get(flo2d_model)[1])),
'model':
flo2d_model,
'method':
method,
'grid_id':
'discharge_{}'.format(station_name)
}
wl_meta_data = {
'latitude':
float('%.6f' % float(extract_stations_dict.get(flo2d_model)[0])),
'longitude':
float('%.6f' % float(extract_stations_dict.get(flo2d_model)[1])),
'model':
flo2d_model,
'method':
method,
'grid_id':
'waterlevel_{}'.format(station_name)
}
tms_id = discharge_TS.get_timeseries_id_if_exists(meta_data=meta_data)
wl_tms_id = waterlevel_TS.get_timeseries_id_if_exists(
meta_data=wl_meta_data)
if wl_tms_id is None:
print("Warning!!! {} waterlevel timeseries doesn't exist.".format(
station_name))
exit(1)
timeseries = []
if tms_id is None:
tms_id = discharge_TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
discharge_TS.insert_run(meta_data=meta_data)
wl_timeseries = waterlevel_TS.get_timeseries(id_=wl_tms_id,
start_date=start_time,
end_date=end_time)
estimated_discharge_ts = []
if station_name == 'hanwella':
estimated_discharge_ts = calculate_hanwella_discharge(
wl_timeseries)
elif station_name == 'glencourse':
estimated_discharge_ts = calculate_glencourse_discharge(
wl_timeseries)
if estimated_discharge_ts is not None and len(
estimated_discharge_ts) > 0:
discharge_TS.insert_data(timeseries=estimated_discharge_ts,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
