def process_fcst_ts_from_hechms_outputs(curw_fcst_pool,
extract_stations,
i,
start,
end,
sim_tag=None):
FCST_TS = Fcst_Timeseries(curw_fcst_pool)
try:
# [station_name,latitude,longitude,target,model,version,sim_tag,station]
source_model = extract_stations[i][4]
version = extract_stations[i][5]
station_id = extract_stations[i][7]
if sim_tag is None:
sim_tag = extract_stations[i][6]
variable_id = 3 # Discharge
unit_id = 3 # m3/s | Instantaneous
source_id = get_source_id(pool=curw_fcst_pool,
model=source_model,
version=version)
fcst_series = FCST_TS.get_latest_timeseries(sim_tag,
station_id,
source_id,
variable_id,
unit_id,
start=None)
if (fcst_series is None) or (len(fcst_series) < 1):
return None
fcst_series.insert(0, ['time', 'value'])
fcst_df = list_of_lists_to_df_first_row_as_columns(fcst_series)
if start is None:
start = (fcst_df['time'].min()).strftime(DATE_TIME_FORMAT)
if end is None:
end = (fcst_df['time'].max()).strftime(DATE_TIME_FORMAT)
df = (pd.date_range(start=start, end=end,
freq='60min')).to_frame(name='time')
processed_df = pd.merge(df, fcst_df, on='time', how='left')
processed_df.interpolate(method='linear',
limit_direction='both',
limit=100)
processed_df.fillna(inplace=True, value=0)
processed_df['time'] = processed_df['time'].dt.strftime(
DATE_TIME_FORMAT)
return processed_df.values.tolist()
except Exception as e:
traceback.print_exc()
def process_fcst_ts_from_flo2d_outputs(curw_fcst_pool, fcst_start):
global latest_fgt
FCST_TS = Fcst_Timeseries(curw_fcst_pool)
try:
# [station_name,latitude,longitude,target,model,version,sim_tag,station]
source_model = extract_stations[i][4]
version = extract_stations[i][5]
sim_tag = extract_stations[i][6]
station_id = extract_stations[i][7]
variable_id = 3 # Discharge
unit_id = 3 # m3/s | Instantaneous
source_id = get_source_id(pool=curw_fcst_pool,
model=source_model,
version=version)
fcst_series = FCST_TS.get_latest_timeseries(sim_tag,
station_id,
source_id,
variable_id,
unit_id,
start=None)
if (fcst_series is None) or (len(fcst_series) < 1):
return None
latest_fgt = (FCST_TS.get_end_date(
sim_tag, station_id, source_id, variable_id,
unit_id)).strftime(COMMON_DATE_TIME_FORMAT)
fcst_series.insert(0, ['time', 'value'])
fcst_df = list_of_lists_to_df_first_row_as_columns(fcst_series)
fcst_end = (fcst_df['time'].max()).strftime(COMMON_DATE_TIME_FORMAT)
if fcst_start is None:
fcst_start = (
fcst_df['time'].min()).strftime(COMMON_DATE_TIME_FORMAT)
df = (pd.date_range(start=fcst_start, end=fcst_end,
freq='15min')).to_frame(name='time')
processed_df = pd.merge(df, fcst_df, on='time', how='left')
# processed_df.interpolate(method='linear', limit_direction='both')
processed_df = processed_df.dropna()
processed_df['time'] = processed_df['time'].dt.strftime(
COMMON_DATE_TIME_FORMAT)
return processed_df.values.tolist()
except Exception as e:
traceback.print_exc()
logger.error("Exception occurred")
def import_old_data():
# Connect to the database
pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
connection = pool.connection()
curw_fcst_new_to_old_hash_id_mapping = read_csv(
"curw_fcst_new_to_old_hash_id_mapping.csv")
TS = Timeseries(pool=pool)
try:
for hash_index in range(len(curw_fcst_new_to_old_hash_id_mapping)):
print("##### Hash index: ", hash_index, " #####")
fgt_list = []
# Extract fgts
with connection.cursor() as cursor1:
sql_statement = "select distinct `fgt` from `data_v3` where `id`=%s order by `fgt` desc;"
cursor1.execute(
sql_statement,
curw_fcst_new_to_old_hash_id_mapping[hash_index][1])
fgts = cursor1.fetchall()
for fgt in fgts:
fgt_list.append(fgt.get('fgt'))
for fgt in fgt_list:
timeseries = []
with connection.cursor() as cursor2:
sql_statement = "select * from `data_v3` where `id`=%s and `fgt`=%s;"
cursor2.execute(
sql_statement,
(curw_fcst_new_to_old_hash_id_mapping[hash_index][1],
fgt))
results = cursor2.fetchall()
for result in results:
timeseries.append([
curw_fcst_new_to_old_hash_id_mapping[hash_index]
[0],
result.get('time'),
result.get('fgt'),
result.get('value')
])
TS.insert_data(timeseries=timeseries, upsert=True)
TS.update_start_date(
id_=curw_fcst_new_to_old_hash_id_mapping[hash_index][0],
start_date=fgt)
except Exception as ex:
traceback.print_exc()
finally:
connection.close()
destroy_Pool(pool=pool)
print()
def save_forecast_timeseries_to_db(pool, timeseries, run_date, run_time, opts,
flo2d_stations, fgt):
print('EXTRACTFLO2DWATERLEVEL:: save_forecast_timeseries >>', opts)
# {
# 'tms_id' : '',
# 'sim_tag' : '',
# 'station_id' : '',
# 'source_id' : '',
# 'unit_id' : '',
# 'variable_id': ''
# }
# Convert date time with offset
date_time = datetime.strptime('%s %s' % (run_date, run_time),
COMMON_DATE_TIME_FORMAT)
if 'utcOffset' in opts:
date_time = date_time + opts['utcOffset']
run_date = date_time.strftime('%Y-%m-%d')
run_time = date_time.strftime('%H:%M:%S')
# If there is an offset, shift by offset before proceed
forecast_timeseries = []
if 'utcOffset' in opts:
print('Shift by utcOffset:', opts['utcOffset'].resolution)
for item in timeseries:
forecast_timeseries.append([
datetime.strptime(item[0], COMMON_DATE_TIME_FORMAT) +
opts['utcOffset'], item[1]
])
forecast_timeseries = extractForecastTimeseries(
timeseries=forecast_timeseries,
extract_date=run_date,
extract_time=run_time)
else:
forecast_timeseries = extractForecastTimeseries(timeseries=timeseries,
extract_date=run_date,
extract_time=run_time)
elementNo = opts.get('elementNo')
tms_meta = opts.get('tms_meta')
tms_meta['latitude'] = str(flo2d_stations.get(elementNo)[1])
tms_meta['longitude'] = str(flo2d_stations.get(elementNo)[2])
tms_meta['station_id'] = flo2d_stations.get(elementNo)[0]
try:
TS = Timeseries(pool=pool)
tms_id = TS.get_timeseries_id_if_exists(meta_data=tms_meta)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=tms_meta)
tms_meta['tms_id'] = tms_id
TS.insert_run(run_meta=tms_meta)
TS.update_start_date(id_=tms_id, start_date=fgt)
TS.insert_data(timeseries=forecast_timeseries,
tms_id=tms_id,
fgt=fgt,
upsert=True)
TS.update_latest_fgt(id_=tms_id, fgt=fgt)
except Exception:
logger.error(
"Exception occurred while pushing data to the curw_fcst database")
traceback.print_exc()
def read_netcdf_file(pool, rainc_net_cdf_file_path, rainnc_net_cdf_file_path,
source_id, variable_id, unit_id, tms_meta, fgt):
"""
:param pool: database connection pool
:param rainc_net_cdf_file_path:
:param rainnc_net_cdf_file_path:
:param source_id:
:param variable_id:
:param unit_id:
:param tms_meta:
:return:
rainc_unit_info: mm
lat_unit_info: degree_north
time_unit_info: minutes since 2019-04-02T18:00:00
"""
if not os.path.exists(rainc_net_cdf_file_path):
logger.warning('no rainc netcdf')
print('no rainc netcdf')
elif not os.path.exists(rainnc_net_cdf_file_path):
logger.warning('no rainnc netcdf')
print('no rainnc netcdf')
else:
"""
RAINC netcdf data extraction
"""
nc_fid = Dataset(rainc_net_cdf_file_path, mode='r')
time_unit_info = nc_fid.variables['XTIME'].units
time_unit_info_list = time_unit_info.split(' ')
lats = nc_fid.variables['XLAT'][0, :, 0]
lons = nc_fid.variables['XLONG'][0, 0, :]
lon_min = lons[0].item()
lat_min = lats[0].item()
lon_max = lons[-1].item()
lat_max = lats[-1].item()
print('[lon_min, lat_min, lon_max, lat_max] :',
[lon_min, lat_min, lon_max, lat_max])
lat_inds = np.where((lats >= lat_min) & (lats <= lat_max))
lon_inds = np.where((lons >= lon_min) & (lons <= lon_max))
rainc = nc_fid.variables['RAINC'][:, lat_inds[0], lon_inds[0]]
"""
RAINNC netcdf data extraction
"""
nnc_fid = Dataset(rainnc_net_cdf_file_path, mode='r')
rainnc = nnc_fid.variables['RAINNC'][:, lat_inds[0], lon_inds[0]]
# times = list(set(nc_fid.variables['XTIME'][:])) # set is used to remove duplicates
times = nc_fid.variables['XTIME'][:]
# ts_start_date = datetime.strptime(time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S')
# ts_end_date = datetime.strptime(time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S') + timedelta(
# minutes=float(sorted(set(times))[-2]))
# start_date = datetime_utc_to_lk(ts_start_date, shift_mins=0).strftime('%Y-%m-%d %H:%M:%S')
# end_date = datetime_utc_to_lk(ts_end_date, shift_mins=0).strftime('%Y-%m-%d %H:%M:%S')
start_date = fgt
end_date = fgt
prcp = rainc + rainnc
nc_fid.close()
nnc_fid.close()
diff = get_per_time_slot_values(prcp)
width = len(lons)
height = len(lats)
ts = Timeseries(pool)
for y in range(height):
for x in range(width):
lat = float('%.6f' % lats[y])
lon = float('%.6f' % lons[x])
tms_meta['latitude'] = str(lat)
tms_meta['longitude'] = str(lon)
station_prefix = '{}_{}'.format(lat, lon)
station_id = wrf_v3_stations.get(station_prefix)
if station_id is None:
add_station(pool=pool,
name=station_prefix,
latitude=lat,
longitude=lon,
description="WRF point",
station_type=StationEnum.WRF)
tms_id = ts.get_timeseries_id_if_exists(tms_meta)
logger.info("Existing timeseries id: {}".format(tms_id))
if tms_id is None:
tms_id = ts.generate_timeseries_id(tms_meta)
logger.info('HASH SHA256 created: {}'.format(tms_id))
run = (tms_id, tms_meta['sim_tag'], start_date, end_date,
station_id, source_id, variable_id, unit_id)
try:
ts.insert_run(run)
except Exception:
logger.error(
"Exception occurred while inserting run entry {}".
format(run))
traceback.print_exc()
else:
# ts.update_latest_fgt(id_=tms_id, fgt=fgt)
ts.update_start_date(id_=tms_id,
start_date=fgt) # to run backward
data_list = []
# generate timeseries for each station
for i in range(len(diff)):
ts_time = datetime.strptime(
time_unit_info_list[2],
'%Y-%m-%dT%H:%M:%S') + timedelta(
minutes=times[i].item())
t = datetime_utc_to_lk(ts_time, shift_mins=0)
data_list.append([
tms_id,
t.strftime('%Y-%m-%d %H:%M:%S'), fgt,
float(diff[i, y, x])
])
push_rainfall_to_db(ts=ts, ts_data=data_list)
def save_forecast_timeseries_to_db(pool, timeseries, run_date, run_time,
tms_meta, fgt):
print('EXTRACTFLO2DWATERLEVEL:: save_forecast_timeseries >>', tms_meta)
# {
# 'tms_id' : '',
# 'sim_tag' : '',
# 'station_id' : '',
# 'source_id' : '',
# 'unit_id' : '',
# 'variable_id': ''
# }
date_time = datetime.strptime('%s %s' % (run_date, run_time),
COMMON_DATE_TIME_FORMAT)
forecast_timeseries = []
if 'utcOffset' in tms_meta: # If there is an offset, shift by offset before proceed
print('Shift by utcOffset:', tms_meta['utcOffset'].resolution)
# Convert date time with offset
date_time = date_time + tms_meta['utcOffset']
for item in timeseries:
forecast_timeseries.append([
datetime.strptime(item[0], COMMON_DATE_TIME_FORMAT) +
tms_meta['utcOffset'], item[1]
])
else:
forecast_timeseries = timeseries
try:
TS = Timeseries(pool=pool)
tms_id = TS.get_timeseries_id_if_exists(meta_data=tms_meta)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=tms_meta)
tms_meta['tms_id'] = tms_id
TS.insert_run(run_meta=tms_meta)
TS.update_start_date(id_=tms_id, start_date=fgt)
TS.insert_data(timeseries=forecast_timeseries,
tms_id=tms_id,
fgt=fgt,
upsert=True)
TS.update_latest_fgt(id_=tms_id, fgt=fgt)
except Exception:
logger.error(
"Exception occurred while pushing data to the curw_fcst database")
traceback.print_exc()
def save_forecast_timeseries_to_db(pool, output, mike_stations, fgt, tms_meta):
print('EXTRACT_MIKE_DISCHARGE:: save_forecast_timeseries >>', tms_meta)
# {
# 'tms_id' : '',
# 'sim_tag' : '',
# 'station_id' : '',
# 'source_id' : '',
# 'unit_id' : '',
# 'variable_id': ''
# }
# iterating the stations
for station in output.columns:
if station in mike_stations.keys():
ts = output[station].reset_index().values.tolist(
) # including index
tms_meta['latitude'] = str(mike_stations.get(station)[1])
tms_meta['longitude'] = str(mike_stations.get(station)[2])
tms_meta['station_id'] = mike_stations.get(station)[0]
try:
TS = Timeseries(pool=pool)
tms_id = TS.get_timeseries_id_if_exists(meta_data=tms_meta)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=tms_meta)
tms_meta['tms_id'] = tms_id
TS.insert_run(run_meta=tms_meta)
TS.update_start_date(id_=tms_id, start_date=fgt)
TS.insert_data(timeseries=ts,
tms_id=tms_id,
fgt=fgt,
upsert=True)
TS.update_latest_fgt(id_=tms_id, fgt=fgt)
except Exception:
logger.error(
"Exception occurred while pushing data to the curw_fcst database"
)
traceback.print_exc()
else:
print("### {} not included in the database. ###".format(station))
def read_netcdf_file(pool, rainnc_net_cdf_file_path, tms_meta):
"""
:param pool: database connection pool
:param rainnc_net_cdf_file_path:
:param source_id:
:param variable_id:
:param unit_id:
:param tms_meta:
:return:
rainc_unit_info: mm
lat_unit_info: degree_north
time_unit_info: minutes since 2019-04-02T18:00:00
"""
if not os.path.exists(rainnc_net_cdf_file_path):
logger.warning(
'no rainnc netcdf :: {}'.format(rainnc_net_cdf_file_path))
return
else:
"""
RAINNC netcdf data extraction
"""
fgt = get_file_last_modified_time(rainnc_net_cdf_file_path)
nnc_fid = Dataset(rainnc_net_cdf_file_path, mode='r')
time_unit_info = nnc_fid.variables['XTIME'].units
time_unit_info_list = time_unit_info.split(' ')
lats = nnc_fid.variables['XLAT'][0, :, 0]
lons = nnc_fid.variables['XLONG'][0, 0, :]
lon_min = lons[0].item()
lat_min = lats[0].item()
lon_max = lons[-1].item()
lat_max = lats[-1].item()
lat_inds = np.where((lats >= lat_min) & (lats <= lat_max))
lon_inds = np.where((lons >= lon_min) & (lons <= lon_max))
rainnc = nnc_fid.variables['RAINNC'][:, lat_inds[0], lon_inds[0]]
times = nnc_fid.variables['XTIME'][:]
start_date = fgt
end_date = fgt
nnc_fid.close()
diff = get_per_time_slot_values(rainnc)
width = len(lons)
height = len(lats)
ts = Timeseries(pool)
for y in range(height):
for x in range(width):
lat = float('%.6f' % lats[y])
lon = float('%.6f' % lons[x])
tms_meta['latitude'] = str(lat)
tms_meta['longitude'] = str(lon)
station_prefix = 'wrf_{}_{}'.format(lat, lon)
station_id = wrf_v3_stations.get(station_prefix)
if station_id is None:
add_station(pool=pool,
name=station_prefix,
latitude=lat,
longitude=lon,
description="WRF point",
station_type=StationEnum.WRF)
station_id = get_station_id(pool=pool,
latitude=lat,
longitude=lon,
station_type=StationEnum.WRF)
tms_id = ts.get_timeseries_id_if_exists(tms_meta)
if tms_id is None:
tms_id = ts.generate_timeseries_id(tms_meta)
run_meta = {
'tms_id': tms_id,
'sim_tag': tms_meta['sim_tag'],
'start_date': start_date,
'end_date': end_date,
'station_id': station_id,
'source_id': tms_meta['source_id'],
'unit_id': tms_meta['unit_id'],
'variable_id': tms_meta['variable_id']
}
try:
ts.insert_run(run_meta)
except Exception:
logger.error(
"Exception occurred while inserting run entry {}".
format(run_meta))
traceback.print_exc()
data_list = []
# generate timeseries for each station
for i in range(len(diff)):
ts_time = datetime.strptime(
time_unit_info_list[2],
'%Y-%m-%dT%H:%M:%S') + timedelta(
minutes=times[i + 1].item())
t = datetime_utc_to_lk(ts_time, shift_mins=0)
data_list.append([
tms_id,
t.strftime('%Y-%m-%d %H:%M:%S'), fgt,
float(diff[i, y, x])
])
push_rainfall_to_db(ts=ts,
ts_data=data_list,
tms_id=tms_id,
fgt=fgt)
def update_MME_tagged_series(pool, start, end, variables, coefficients,
sub_region, tms_meta, fgt):
for index, row in sub_region.iterrows():
lat = float('%.6f' % row['latitude'])
lon = float('%.6f' % row['longitude'])
tms_meta['latitude'] = str(lat)
tms_meta['longitude'] = str(lon)
station_prefix = 'wrf_{}_{}'.format(lat, lon)
station_id = wrf_v3_stations.get(station_prefix)
TS = Timeseries(pool=pool)
tms_id = TS.get_timeseries_id_if_exists(tms_meta)
if tms_id is None:
tms_id = TS.generate_timeseries_id(tms_meta)
run_meta = {
'tms_id': tms_id,
'sim_tag': tms_meta['sim_tag'],
'start_date': fgt,
'end_date': fgt,
'station_id': station_id,
'source_id': tms_meta['source_id'],
'unit_id': tms_meta['unit_id'],
'variable_id': tms_meta['variable_id']
}
try:
TS.insert_run(run_meta)
except Exception:
time.sleep(5)
try:
TS.insert_run(run_meta)
except Exception:
logger.error(
"Exception occurred while inserting run entry {}".
format(run_meta))
traceback.print_exc()
timeseries = calculate_MME_series(TS=TS,
start=start,
end=end,
variables=variables,
coefficients=coefficients,
station_id=station_id,
variable_id=tms_meta['variable_id'],
unit_id=tms_meta['unit_id'])
formatted_timeseries = []
for i in range(len(timeseries)):
formatted_timeseries.append([
tms_id, timeseries[i][0].strftime('%Y-%m-%d %H:%M:00'), fgt,
float('%.3f' % timeseries[i][1])
])
push_rainfall_to_db(ts=TS,
ts_data=formatted_timeseries,
tms_id=tms_id,
fgt=fgt)
import traceback
from db_adapter.base import get_Pool, destroy_Pool
from db_adapter.curw_fcst.timeseries import Timeseries
try:
USERNAME = "******"
PASSWORD = "******"
HOST = "127.0.0.1"
PORT = 3306
DATABASE = "curw_fcst"
pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE)
ts = Timeseries(pool=pool)
ts.update_latest_fgt(id_="02fef97c984cf709f98b57c7278f6b8ccdd6ae165c68e204c2695d8c7fb8e32e", fgt="2019-07-20 23:00:00")
except Exception as e:
traceback.print_exc()
finally:
destroy_Pool(pool=pool)
print("Process Finished.")
def update_rainfall_fcsts(flo2d_model, method, grid_interpolation, model_list,
timestep):
"""
Update rainfall forecasts for flo2d models
:param flo2d_model: flo2d model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param model_list: list of forecast model and their versions used to calculate the rainfall
e.g.: [["WRF_E", "v4"],["WRF_SE", "v4"]]
:param timestep: output timeseries timestep
:return:
"""
try:
# Connect to the database
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
Sim_TS = Sim_Timeseries(pool=curw_sim_pool)
Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool)
flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format(
flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)]
flo2d_wrf_mapping = get_flo2d_cells_to_wrf_grid_mappings(
pool=curw_sim_pool,
grid_interpolation=grid_interpolation,
flo2d_model=flo2d_model)
for flo2d_index in range(len(flo2d_grids)): # len(flo2d_grids)
lat = flo2d_grids[flo2d_index][2]
lon = flo2d_grids[flo2d_index][1]
cell_id = flo2d_grids[flo2d_index][0]
meta_data = {
'latitude':
float('%.6f' % float(lat)),
'longitude':
float('%.6f' % float(lon)),
'model':
flo2d_model,
'method':
method,
'grid_id':
'{}_{}_{}'.format(flo2d_model, grid_interpolation,
(str(cell_id)).zfill(10))
}
tms_id = Sim_TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
method=meta_data.get('method'))
if tms_id is None:
tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
Sim_TS.insert_run(meta_data=meta_data)
obs_end = Sim_TS.get_obs_end(id_=tms_id)
fcst_timeseries = []
for i in range(len(model_list)):
source_id = get_source_id(pool=curw_fcst_pool,
model=model_list[i][0],
version=model_list[i][1])
sim_tag = model_list[i][2]
coefficient = model_list[i][3]
temp_timeseries = []
if timestep == 5:
if obs_end is not None:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.
get(meta_data['grid_id']),
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1))
else:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.
get(meta_data['grid_id']),
source_id=source_id,
variable_id=1,
unit_id=1))
elif timestep == 15:
if obs_end is not None:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.get(
meta_data['grid_id']),
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1)
else:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.get(
meta_data['grid_id']),
source_id=source_id,
variable_id=1,
unit_id=1)
if coefficient != 1:
for j in range(len(temp_timeseries)):
temp_timeseries[j][1] = float(
temp_timeseries[j][1]) * coefficient
if i == 0:
fcst_timeseries = temp_timeseries
else:
fcst_timeseries = append_value_for_timestamp(
existing_ts=fcst_timeseries, new_ts=temp_timeseries)
sum_timeseries = summed_timeseries(fcst_timeseries)
for i in range(len(sum_timeseries)):
if float(sum_timeseries[i][1]) < 0:
sum_timeseries[i][1] = 0
if sum_timeseries is not None and len(sum_timeseries) > 0:
Sim_TS.insert_data(timeseries=sum_timeseries,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating fcst rainfalls in curw_sim.")
finally:
destroy_Pool(curw_sim_pool)
destroy_Pool(curw_fcst_pool)
def update_rainfall_fcsts(target_model, method, grid_interpolation, model_list,
timestep):
"""
Update rainfall forecasts for flo2d models
:param target_model: target model for which input ins prepared
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param model_list: list of forecast model and their versions used to calculate the rainfall
e.g.: [["WRF_E", "4.0", "evening_18hrs"],["WRF_SE", "v4", ,"evening_18hrs"],["WRF_Ensemble", "4.0", ,"MME"]]
:param timestep: output timeseries timestep
:return:
"""
try:
# Connect to the database
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
Sim_TS = Sim_Timeseries(pool=curw_sim_pool)
Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool)
# [hash_id, station_id, station_name, latitude, longitude]
active_obs_stations = read_csv(
'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'
)
obs_stations_dict = {
} # keys: obs station id , value: [name, latitude, longitude]
for obs_index in range(len(active_obs_stations)):
obs_stations_dict[active_obs_stations[obs_index][1]] = [
active_obs_stations[obs_index][2],
active_obs_stations[obs_index][3],
active_obs_stations[obs_index][4]
]
obs_d03_mapping = get_obs_to_d03_grid_mappings_for_rainfall(
pool=curw_sim_pool, grid_interpolation=grid_interpolation)
for obs_id in obs_stations_dict.keys():
meta_data = {
'latitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[1])),
'longitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[2])),
'model':
target_model,
'method':
method,
'grid_id':
'rainfall_{}_{}_{}'.format(obs_id,
obs_stations_dict.get(obs_id)[0],
grid_interpolation)
}
tms_id = Sim_TS.get_timeseries_id_if_exists(meta_data=meta_data)
if tms_id is None:
tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
Sim_TS.insert_run(meta_data=meta_data)
obs_end = Sim_TS.get_obs_end(id_=tms_id)
fcst_timeseries = []
for i in range(len(model_list)):
source_id = get_source_id(pool=curw_fcst_pool,
model=model_list[i][0],
version=model_list[i][1])
sim_tag = model_list[i][2]
coefficient = model_list[i][3]
temp_timeseries = []
if timestep == 5:
if obs_end is not None:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=obs_d03_mapping.
get(meta_data['grid_id'])[0],
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1))
else:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=obs_d03_mapping.
get(meta_data['grid_id'])[0],
source_id=source_id,
variable_id=1,
unit_id=1))
elif timestep == 15:
if obs_end is not None:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=obs_d03_mapping.get(
meta_data['grid_id'])[0],
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1)
else:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=obs_d03_mapping.get(
meta_data['grid_id'])[0],
source_id=source_id,
variable_id=1,
unit_id=1)
if coefficient != 1:
for j in range(len(temp_timeseries)):
temp_timeseries[j][1] = float(
temp_timeseries[j][1]) * coefficient
if i == 0:
fcst_timeseries = temp_timeseries
else:
fcst_timeseries = append_value_for_timestamp(
existing_ts=fcst_timeseries, new_ts=temp_timeseries)
sum_timeseries = summed_timeseries(fcst_timeseries)
for i in range(len(sum_timeseries)):
if float(sum_timeseries[i][1]) < 0:
sum_timeseries[i][1] = 0
if sum_timeseries is not None and len(sum_timeseries) > 0:
Sim_TS.insert_data(timeseries=sum_timeseries,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating fcst rainfalls in curw_sim.")
finally:
destroy_Pool(curw_sim_pool)
destroy_Pool(curw_fcst_pool)
