def extract_wrf_data(wrf_model, config_data, tms_meta):
logger.info(
"######################################## {} #######################################"
.format(wrf_model))
for date in config_data['dates']:
run_date_str = date
daily_dir = 'STATIONS_{}'.format(run_date_str)
output_dir = os.path.join(config_data['wrf_dir'], daily_dir)
rainnc_net_cdf_file = 'd03_RAINNC_{}_{}.nc'.format(
run_date_str, wrf_model)
rainnc_net_cdf_file_path = os.path.join(output_dir,
rainnc_net_cdf_file)
source_name = "{}_{}".format(config_data['model'], wrf_model)
source_id = get_source_id(pool=pool,
model=source_name,
version=config_data['version'])
tms_meta['model'] = source_name
tms_meta['source_id'] = source_id
try:
read_netcdf_file(pool=pool,
rainnc_net_cdf_file_path=rainnc_net_cdf_file_path,
tms_meta=tms_meta)
except Exception as e:
logger.error("WRF_{} netcdf file reading error.".format(wrf_model))
traceback.print_exc()
def process_fcst_ts_from_hechms_outputs(curw_fcst_pool,
extract_stations,
i,
start,
end,
sim_tag=None):
FCST_TS = Fcst_Timeseries(curw_fcst_pool)
try:
# [station_name,latitude,longitude,target,model,version,sim_tag,station]
source_model = extract_stations[i][4]
version = extract_stations[i][5]
station_id = extract_stations[i][7]
if sim_tag is None:
sim_tag = extract_stations[i][6]
variable_id = 3 # Discharge
unit_id = 3 # m3/s | Instantaneous
source_id = get_source_id(pool=curw_fcst_pool,
model=source_model,
version=version)
fcst_series = FCST_TS.get_latest_timeseries(sim_tag,
station_id,
source_id,
variable_id,
unit_id,
start=None)
if (fcst_series is None) or (len(fcst_series) < 1):
return None
fcst_series.insert(0, ['time', 'value'])
fcst_df = list_of_lists_to_df_first_row_as_columns(fcst_series)
if start is None:
start = (fcst_df['time'].min()).strftime(DATE_TIME_FORMAT)
if end is None:
end = (fcst_df['time'].max()).strftime(DATE_TIME_FORMAT)
df = (pd.date_range(start=start, end=end,
freq='60min')).to_frame(name='time')
processed_df = pd.merge(df, fcst_df, on='time', how='left')
processed_df.interpolate(method='linear',
limit_direction='both',
limit=100)
processed_df.fillna(inplace=True, value=0)
processed_df['time'] = processed_df['time'].dt.strftime(
DATE_TIME_FORMAT)
return processed_df.values.tolist()
except Exception as e:
traceback.print_exc()
def extract_wrf_data(wrf_system, config_data, tms_meta):
logger.info(
"######################################## {} #######################################"
.format(wrf_system))
for date in config_data['dates']:
# /wrf_nfs/wrf/4.0/18/A/2019-07-30/d03_RAINNC.nc
output_dir = os.path.join(config_data['wrf_dir'],
config_data['version'],
config_data['gfs_data_hour'], wrf_system,
date)
rainnc_net_cdf_file = 'd03_RAINNC.nc'
rainnc_net_cdf_file_path = os.path.join(output_dir,
rainnc_net_cdf_file)
try:
source_name = "{}_{}".format(config_data['model'], wrf_system)
source_id = get_source_id(pool=pool,
model=source_name,
version=tms_meta['version'])
if source_id is None:
add_source(pool=pool,
model=source_name,
version=tms_meta['version'])
source_id = get_source_id(pool=pool,
model=source_name,
version=tms_meta['version'])
except Exception:
msg = "Exception occurred while loading source meta data for WRF_{} from database.".format(
wrf_system)
logger.error(msg)
email_content[datetime.now().strftime(
COMMON_DATE_TIME_FORMAT)] = msg
return False
tms_meta['model'] = source_name
tms_meta['source_id'] = source_id
return read_netcdf_file(
pool=pool,
rainnc_net_cdf_file_path=rainnc_net_cdf_file_path,
tms_meta=tms_meta)
def process_fcst_ts_from_flo2d_outputs(curw_fcst_pool, fcst_start):
global latest_fgt
FCST_TS = Fcst_Timeseries(curw_fcst_pool)
try:
# [station_name,latitude,longitude,target,model,version,sim_tag,station]
source_model = extract_stations[i][4]
version = extract_stations[i][5]
sim_tag = extract_stations[i][6]
station_id = extract_stations[i][7]
variable_id = 3 # Discharge
unit_id = 3 # m3/s | Instantaneous
source_id = get_source_id(pool=curw_fcst_pool,
model=source_model,
version=version)
fcst_series = FCST_TS.get_latest_timeseries(sim_tag,
station_id,
source_id,
variable_id,
unit_id,
start=None)
if (fcst_series is None) or (len(fcst_series) < 1):
return None
latest_fgt = (FCST_TS.get_end_date(
sim_tag, station_id, source_id, variable_id,
unit_id)).strftime(COMMON_DATE_TIME_FORMAT)
fcst_series.insert(0, ['time', 'value'])
fcst_df = list_of_lists_to_df_first_row_as_columns(fcst_series)
fcst_end = (fcst_df['time'].max()).strftime(COMMON_DATE_TIME_FORMAT)
if fcst_start is None:
fcst_start = (
fcst_df['time'].min()).strftime(COMMON_DATE_TIME_FORMAT)
df = (pd.date_range(start=fcst_start, end=fcst_end,
freq='15min')).to_frame(name='time')
processed_df = pd.merge(df, fcst_df, on='time', how='left')
# processed_df.interpolate(method='linear', limit_direction='both')
processed_df = processed_df.dropna()
processed_df['time'] = processed_df['time'].dt.strftime(
COMMON_DATE_TIME_FORMAT)
return processed_df.values.tolist()
except Exception as e:
traceback.print_exc()
logger.error("Exception occurred")
def calculate_MME_series(TS, start, end, variables, station_id, variable_id,
unit_id):
index = pd.date_range(start=start, end=end, freq='15min')
df = pd.DataFrame(index=index)
for variable in variables:
model = variable[0]
version = variable[1]
sim_tag = variable[2]
coefficient = variable[3]
try:
source_id = get_source_id(pool=pool, model=model, version=version)
except Exception:
try:
time.sleep(3)
source_id = get_source_id(pool=pool,
model=model,
version=version)
except Exception:
msg = "Exception occurred while loading source id from database."
logger.error(msg)
exit(1)
fcst_ts = TS.get_latest_timeseries(sim_tag=sim_tag,
source_id=source_id,
station_id=station_id,
variable_id=variable_id,
unit_id=unit_id)
timeseries = list_of_lists_to_df_first_column_as_index(fcst_ts)
timeseries[[0]] = timeseries[[0]].astype('float64') * coefficient
df = df.join(timeseries, lsuffix='_left', rsuffix='_right')
df.fillna(0)
df['sum'] = df.sum(axis=1)
timeseries_df = df['sum']
return timeseries_df.reset_index().values.tolist()
timdep_file_path = os.path.join(output_dir, TIMDEP_FILE)
pool = get_Pool(host=con_params.CURW_FCST_HOST,
port=con_params.CURW_FCST_PORT,
db=con_params.CURW_FCST_DATABASE,
user=con_params.CURW_FCST_USERNAME,
password=con_params.CURW_FCST_PASSWORD)
flo2d_model_name = '{}_{}'.format(model, version)
flo2d_source = json.loads(
get_source_parameters(pool=pool, model=model, version=version))
flo2d_stations = get_flo2d_output_stations(
pool=pool, flo2d_model=StationEnum.getType(flo2d_model_name))
source_id = get_source_id(pool=pool, model=model, version=version)
variable_id = get_variable_id(pool=pool, variable=variable)
unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
tms_meta = {
'sim_tag': sim_tag,
'model': model,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value,
'source_id': source_id,
'variable_id': variable_id,
'unit_id': unit_id
def extract_distrubuted_hechms_outputs(output_file_name, output_dir, run_date,
run_time):
"""
Config.json
{
"output_file_name": "DailyDischarge.csv",
"output_dir": "",
"run_date": "2019-05-24",
"run_time": "00:00:00",
"utc_offset": "",
"sim_tag": "hourly_run",
"model": "HECHMS",
"version": "single",
"unit": "m3/s",
"unit_type": "Instantaneous",
"variable": "Discharge",
"station_name": "Hanwella"
}
"""
try:
config = json.loads(open('config.json').read())
# output related details
output_file_name = output_file_name
output_dir = output_dir
run_date = run_date
run_time = run_time
utc_offset = read_attribute_from_config_file('utc_offset', config,
False)
if utc_offset is None:
utc_offset = ''
# sim tag
sim_tag = read_attribute_from_config_file('sim_tag', config, True)
# source details
model = read_attribute_from_config_file('model', config, True)
version = read_attribute_from_config_file('version', config, True)
# unit details
unit = read_attribute_from_config_file('unit', config, True)
unit_type = UnitType.getType(
read_attribute_from_config_file('unit_type', config, True))
# variable details
variable = read_attribute_from_config_file('variable', config, True)
# station details
station_name = read_attribute_from_config_file('station_name', config,
True)
out_file_path = os.path.join(output_dir, output_file_name)
if not os.path.exists(out_file_path):
msg = 'no file :: {}'.format(out_file_path)
logger.warning(msg)
print(msg)
exit(1)
fgt = get_file_last_modified_time(out_file_path)
print("fgt, ", fgt)
timeseries = read_csv(out_file_path)
pool = get_Pool(host=CURW_FCST_HOST,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD)
hechms_stations = get_hechms_stations(pool=pool)
station_id = hechms_stations.get(station_name)[0]
lat = str(hechms_stations.get(station_name)[1])
lon = str(hechms_stations.get(station_name)[2])
source_id = get_source_id(pool=pool, model=model, version=version)
variable_id = get_variable_id(pool=pool, variable=variable)
unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
tms_meta = {
'sim_tag': sim_tag,
'model': model,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value,
'latitude': lat,
'longitude': lon,
'station_id': station_id,
'source_id': source_id,
'variable_id': variable_id,
'unit_id': unit_id
}
utcOffset = getUTCOffset(utc_offset, default=True)
if utcOffset != timedelta():
tms_meta['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool,
timeseries=timeseries,
run_date=run_date,
run_time=run_time,
tms_meta=tms_meta,
fgt=fgt)
except Exception as e:
logger.error('JSON config data loading error.')
print('JSON config data loading error.')
traceback.print_exc()
finally:
logger.info("Process finished.")
print("Process finished.")
rainc_net_cdf_file_path = os.path.join(output_dir,
rainc_net_cdf_file)
logger.info(
"rainc_net_cdf_file_path : {}".format(rainc_net_cdf_file_path))
rainnc_net_cdf_file_path = os.path.join(output_dir,
rainnc_net_cdf_file)
logger.info("rainnc_net_cdf_file_path : {}".format(
rainnc_net_cdf_file_path))
fgt = get_file_last_modified_time(rainnc_net_cdf_file_path)
sim_tag = 'evening_18hrs'
source_name = "{}_{}".format(model, wrf_model)
source_id = get_source_id(pool=pool,
model=source_name,
version=version)
tms_meta = {
'sim_tag': sim_tag,
'model': source_name,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value
}
try:
read_netcdf_file(
pool=pool,
rainc_net_cdf_file_path=rainc_net_cdf_file_path,
def extract_distrubuted_hechms_outputs(target_model, db_user, db_pwd, db_host,
db_name, out_file_path, run_date,
run_time):
"""
Config.json
{
"output_file_name": "DailyDischarge.csv",
"output_dir": "",
"run_date": "2019-05-24",
"run_time": "00:00:00",
"utc_offset": "",
"sim_tag": "hourly_run",
"model": "HECHMS",
"version": "single",
"unit": "m3/s",
"unit_type": "Instantaneous",
"variable": "Discharge",
"station_name": "Hanwella"
}
"""
try:
config = json.loads(
open('/home/curw/git/distributed_hechms/uploads/config.json').read(
))
# output related details
run_date = run_date
run_time = run_time
utc_offset = read_attribute_from_config_file('utc_offset', config,
False)
if utc_offset is None:
utc_offset = ''
# sim tag
sim_tag = read_attribute_from_config_file('sim_tag', config, True)
print("extract_distrubuted_hechms_outputs|sim_tag : ", sim_tag)
# source details
model = read_attribute_from_config_file('model', config, True)
print("extract_distrubuted_hechms_outputs|model : ", model)
version_config = read_attribute_from_config_file(
'version_config', config, True)
print("extract_distrubuted_hechms_outputs|version_config : ",
version_config)
version = version_config[target_model]
print("extract_distrubuted_hechms_outputs|version : ", version)
# unit details
unit = read_attribute_from_config_file('unit', config, True)
print("extract_distrubuted_hechms_outputs|unit : ", unit)
unit_type = UnitType.getType(
read_attribute_from_config_file('unit_type', config, True))
print("extract_distrubuted_hechms_outputs|unit_type : ", unit_type)
# variable details
variable = read_attribute_from_config_file('variable', config, True)
print("extract_distrubuted_hechms_outputs|variable : ", variable)
# station details
station_name = read_attribute_from_config_file('station_name', config,
True)
print("extract_distrubuted_hechms_outputs|station_name : ",
station_name)
if not os.path.exists(out_file_path):
msg = 'no file :: {}'.format(out_file_path)
logger.warning(msg)
print(msg)
exit(1)
fgt = get_file_last_modified_time(out_file_path)
print("extract_distrubuted_hechms_outputs|fgt : ", fgt)
timeseries = read_csv(out_file_path)
pool = get_Pool(host=db_host,
port=3306,
db=db_name,
user=db_user,
password=db_pwd)
hechms_stations = get_hechms_stations(pool=pool)
print("extract_distrubuted_hechms_outputs|hechms_stations : ",
hechms_stations)
station_id = hechms_stations.get(station_name)[0]
lat = str(hechms_stations.get(station_name)[1])
lon = str(hechms_stations.get(station_name)[2])
print("extract_distrubuted_hechms_outputs|[station_id, lat, lon] : ",
[station_id, lat, lon])
source_id = get_source_id(pool=pool, model=model, version=version)
print("extract_distrubuted_hechms_outputs|source_id : ", source_id)
variable_id = get_variable_id(pool=pool, variable=variable)
print("extract_distrubuted_hechms_outputs|variable_id : ", variable_id)
unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
print("extract_distrubuted_hechms_outputs|unit_id : ", unit_id)
tms_meta = {
'sim_tag': sim_tag,
'model': model,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value,
'latitude': lat,
'longitude': lon,
'station_id': station_id,
'source_id': source_id,
'variable_id': variable_id,
'unit_id': unit_id
}
print("extract_distrubuted_hechms_outputs|tms_meta : ", tms_meta)
utcOffset = getUTCOffset(utc_offset, default=True)
if utcOffset != timedelta():
tms_meta['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool,
timeseries=timeseries,
run_date=run_date,
run_time=run_time,
tms_meta=tms_meta,
fgt=fgt)
return {'Result': 'Success'}
except Exception as e:
logger.error('JSON config data loading error.')
print('JSON config data loading error.')
traceback.print_exc()
return {'Result': 'Fail'}
def calculate_MME_series(TS, start, end, variables, coefficients, station_id,
variable_id, unit_id):
index = pd.date_range(start=start, end=end, freq='15min')
df = pd.DataFrame(index=index)
boundary = pd.DataFrame(index=index)
for index, variable in enumerate(variables):
model = variable[0]
version = variable[1]
sim_tag = variable[2]
coefficient = float(coefficients[index])
try:
source_id = get_source_id(pool=pool, model=model, version=version)
except Exception:
try:
time.sleep(3)
source_id = get_source_id(pool=pool,
model=model,
version=version)
except Exception:
msg = "Exception occurred while loading source id from database."
logger.error(msg)
exit(1)
fcst_ts = TS.get_latest_timeseries(sim_tag=sim_tag,
source_id=source_id,
station_id=station_id,
variable_id=variable_id,
unit_id=unit_id)
timeseries = list_of_lists_to_df_first_column_as_index(fcst_ts)
# start -- boundary dataframe creation #
boundary_temp = timeseries.copy()
boundary_temp[[0]] = boundary_temp[[0]].astype('float64')
boundary.join(boundary_temp, lsuffix='_left', rsuffix='_right')
# end -- boundary dataframe creation #
timeseries[[0]] = timeseries[[0]].astype('float64') * coefficient
df = df.join(timeseries, lsuffix='_left', rsuffix='_right')
df.fillna(0)
df['sum'] = df.sum(axis=1) + float(
coefficients[index +
1]) # + coefficients[index+1] ==> adding the constant
# start - boundary condition check #
boundary['min'] = boundary.min(axis=1)
boundary['max'] = boundary.max(axis=1)
for index, row in boundary.iterrows():
value = df.loc[index, 'sum']
min = boundary.loc[index, 'min']
max = boundary.loc[index, 'max']
if value > max:
df.loc[index, 'sum'] = max
elif value < min:
df.loc[index, 'sum'] = max
print(value, max, min, df.loc[index, 'sum'])
# end - boundary condition check #
timeseries_df = df['sum']
return timeseries_df.reset_index().values.tolist()
def upload_waterlevels(dir_path, ts_start_date, ts_start_time, run_date, run_time):
"""
Config.json
{
"HYCHAN_OUT_FILE": "HYCHAN.OUT",
"TIMDEP_FILE": "TIMDEP.OUT",
"output_dir": "",
"run_date": "2019-05-24",
"run_time": "",
"ts_start_date": "",
"ts_start_time": "",
"utc_offset": "",
"sim_tag": "",
"model": "WRF",
"version": "v3",
"unit": "mm",
"unit_type": "Accumulative",
"variable": "Precipitation"
}
"""
try:
config_path = os.path.join(os.getcwd(), 'extract', 'config_curw_fcst.json')
config = json.loads(open(config_path).read())
# flo2D related details
HYCHAN_OUT_FILE = read_attribute_from_config_file('HYCHAN_OUT_FILE', config, True)
TIMDEP_FILE = read_attribute_from_config_file('TIMDEP_FILE', config, True)
output_dir = dir_path
run_date = run_date
run_time = run_time
ts_start_date = ts_start_date
ts_start_time = ts_start_time
utc_offset = read_attribute_from_config_file('utc_offset', config, False)
if utc_offset is None:
utc_offset = ''
# sim tag
sim_tag = read_attribute_from_config_file('sim_tag', config, True)
# source details
model = read_attribute_from_config_file('model', config, True)
version = read_attribute_from_config_file('version', config, True)
# unit details
unit = read_attribute_from_config_file('unit', config, True)
unit_type = UnitType.getType(read_attribute_from_config_file('unit_type', config, True))
# variable details
variable = read_attribute_from_config_file('variable', config, True)
hychan_out_file_path = os.path.join(output_dir, HYCHAN_OUT_FILE)
timdep_file_path = os.path.join(output_dir, TIMDEP_FILE)
pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD)
# pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE)
flo2d_model_name = '{}_{}'.format(model, version)
flo2d_source = json.loads(get_source_parameters(pool=pool, model=model, version=version))
print("############### source ############", flo2d_source)
flo2d_stations = get_flo2d_output_stations(pool=pool, flo2d_model=StationEnum.getType(flo2d_model_name))
print("############### 1st occurrence ############", flo2d_stations)
source_id = get_source_id(pool=pool, model=model, version=version)
variable_id = get_variable_id(pool=pool, variable=variable)
unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
tms_meta = {
'sim_tag': sim_tag,
'model': model,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value,
'source_id': source_id,
'variable_id': variable_id,
'unit_id': unit_id
}
CHANNEL_CELL_MAP = flo2d_source["CHANNEL_CELL_MAP"]
FLOOD_PLAIN_CELL_MAP = flo2d_source["FLOOD_PLAIN_CELL_MAP"]
ELEMENT_NUMBERS = CHANNEL_CELL_MAP.keys()
FLOOD_ELEMENT_NUMBERS = FLOOD_PLAIN_CELL_MAP.keys()
SERIES_LENGTH = 0
MISSING_VALUE = -999
utcOffset = getUTCOffset(utc_offset, default=True)
print('Extract Water Level Result of FLO2D on', run_date, '@', run_time, 'with Base time of', ts_start_date,
'@', ts_start_time)
# Check HYCHAN.OUT file exists
if not os.path.exists(hychan_out_file_path):
print('Unable to find file : ', hychan_out_file_path)
traceback.print_exc()
#####################################
# Calculate the size of time series #
#####################################
bufsize = 65536
with open(hychan_out_file_path) as infile:
isWaterLevelLines = False
isCounting = False
countSeriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series
while True:
lines = infile.readlines(bufsize)
if not lines or SERIES_LENGTH:
break
for line in lines:
if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
isWaterLevelLines = True
elif isWaterLevelLines:
cols = line.split()
if len(cols) > 0 and cols[0].replace('.', '', 1).isdigit():
countSeriesSize += 1
isCounting = True
elif isWaterLevelLines and isCounting:
SERIES_LENGTH = countSeriesSize
break
print('Series Length is :', SERIES_LENGTH)
bufsize = 65536
#################################################################
# Extract Channel Water Level elevations from HYCHAN.OUT file #
#################################################################
print('Extract Channel Water Level Result of FLO2D (HYCHAN.OUT) on', run_date, '@', run_time,
'with Base time of',
ts_start_date, '@', ts_start_time)
with open(hychan_out_file_path) as infile:
isWaterLevelLines = False
isSeriesComplete = False
waterLevelLines = []
seriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series
while True:
lines = infile.readlines(bufsize)
if not lines:
break
for line in lines:
if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
seriesSize = 0
elementNo = line.split()[5]
if elementNo in ELEMENT_NUMBERS:
isWaterLevelLines = True
waterLevelLines.append(line)
else:
isWaterLevelLines = False
elif isWaterLevelLines:
cols = line.split()
if len(cols) > 0 and isfloat(cols[0]):
seriesSize += 1
waterLevelLines.append(line)
if seriesSize == SERIES_LENGTH:
isSeriesComplete = True
if isSeriesComplete:
baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S')
timeseries = []
elementNo = waterLevelLines[0].split()[5]
# print('Extracted Cell No', elementNo, CHANNEL_CELL_MAP[elementNo])
for ts in waterLevelLines[1:]:
v = ts.split()
if len(v) < 1:
continue
# Get flood level (Elevation)
value = v[1]
# Get flood depth (Depth)
# value = v[2]
if not isfloat(value):
value = MISSING_VALUE
continue # If value is not present, skip
if value == 'NaN':
continue # If value is NaN, skip
timeStep = float(v[0])
currentStepTime = baseTime + timedelta(hours=timeStep)
dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S")
timeseries.append([dateAndTime, value])
# Save Forecast values into Database
opts = {
'elementNo': elementNo,
'tms_meta': tms_meta
}
# print('>>>>>', opts)
if utcOffset != timedelta():
opts['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries,
run_date=run_date, run_time=run_time, opts=opts,
flo2d_stations=flo2d_stations)
isWaterLevelLines = False
isSeriesComplete = False
waterLevelLines = []
# -- END for loop
# -- END while loop
#################################################################
# Extract Flood Plain water elevations from TIMEDEP.OUT file #
#################################################################
if not os.path.exists(timdep_file_path):
print('Unable to find file : ', timdep_file_path)
traceback.print_exc()
print('Extract Flood Plain Water Level Result of FLO2D (TIMEDEP.OUT) on', run_date, '@', run_time,
'with Base time of', ts_start_date,
'@', ts_start_time)
with open(timdep_file_path) as infile:
waterLevelLines = []
waterLevelSeriesDict = dict.fromkeys(FLOOD_ELEMENT_NUMBERS, [])
while True:
lines = infile.readlines(bufsize)
if not lines:
break
for line in lines:
if len(line.split()) == 1:
# continue
if len(waterLevelLines) > 0:
waterLevels = get_water_level_of_channels(waterLevelLines, FLOOD_ELEMENT_NUMBERS)
# Get Time stamp Ref:http://stackoverflow.com/a/13685221/1461060
# print('waterLevelLines[0].split() : ', waterLevelLines[0].split())
ModelTime = float(waterLevelLines[0].split()[0])
baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S')
currentStepTime = baseTime + timedelta(hours=ModelTime)
dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S")
for elementNo in FLOOD_ELEMENT_NUMBERS:
tmpTS = waterLevelSeriesDict[elementNo][:]
if elementNo in waterLevels:
tmpTS.append([dateAndTime, waterLevels[elementNo]])
else:
tmpTS.append([dateAndTime, MISSING_VALUE])
waterLevelSeriesDict[elementNo] = tmpTS
isWaterLevelLines = False
# for l in waterLevelLines :
# print(l)
waterLevelLines = []
waterLevelLines.append(line)
# print('len(FLOOD_ELEMENT_NUMBERS) : ', len(FLOOD_ELEMENT_NUMBERS))
for elementNo in FLOOD_ELEMENT_NUMBERS:
# Save Forecast values into Database
opts = {
'elementNo': elementNo,
'tms_meta': tms_meta
}
if utcOffset != timedelta():
opts['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool, timeseries=waterLevelSeriesDict[elementNo],
run_date=run_date, run_time=run_time, opts=opts,
flo2d_stations=flo2d_stations)
except Exception as e:
logger.error('JSON config data loading error.')
print('JSON config data loading error.')
traceback.print_exc()
finally:
logger.info("Process finished.")
print("Process finished.")
def update_rainfall_fcsts(flo2d_model, method, grid_interpolation, model_list,
timestep):
"""
Update rainfall forecasts for flo2d models
:param flo2d_model: flo2d model
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param model_list: list of forecast model and their versions used to calculate the rainfall
e.g.: [["WRF_E", "v4"],["WRF_SE", "v4"]]
:param timestep: output timeseries timestep
:return:
"""
try:
# Connect to the database
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
Sim_TS = Sim_Timeseries(pool=curw_sim_pool)
Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool)
flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format(
flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)]
flo2d_wrf_mapping = get_flo2d_cells_to_wrf_grid_mappings(
pool=curw_sim_pool,
grid_interpolation=grid_interpolation,
flo2d_model=flo2d_model)
for flo2d_index in range(len(flo2d_grids)): # len(flo2d_grids)
lat = flo2d_grids[flo2d_index][2]
lon = flo2d_grids[flo2d_index][1]
cell_id = flo2d_grids[flo2d_index][0]
meta_data = {
'latitude':
float('%.6f' % float(lat)),
'longitude':
float('%.6f' % float(lon)),
'model':
flo2d_model,
'method':
method,
'grid_id':
'{}_{}_{}'.format(flo2d_model, grid_interpolation,
(str(cell_id)).zfill(10))
}
tms_id = Sim_TS.get_timeseries_id(grid_id=meta_data.get('grid_id'),
method=meta_data.get('method'))
if tms_id is None:
tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
Sim_TS.insert_run(meta_data=meta_data)
obs_end = Sim_TS.get_obs_end(id_=tms_id)
fcst_timeseries = []
for i in range(len(model_list)):
source_id = get_source_id(pool=curw_fcst_pool,
model=model_list[i][0],
version=model_list[i][1])
sim_tag = model_list[i][2]
coefficient = model_list[i][3]
temp_timeseries = []
if timestep == 5:
if obs_end is not None:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.
get(meta_data['grid_id']),
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1))
else:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.
get(meta_data['grid_id']),
source_id=source_id,
variable_id=1,
unit_id=1))
elif timestep == 15:
if obs_end is not None:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.get(
meta_data['grid_id']),
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1)
else:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=flo2d_wrf_mapping.get(
meta_data['grid_id']),
source_id=source_id,
variable_id=1,
unit_id=1)
if coefficient != 1:
for j in range(len(temp_timeseries)):
temp_timeseries[j][1] = float(
temp_timeseries[j][1]) * coefficient
if i == 0:
fcst_timeseries = temp_timeseries
else:
fcst_timeseries = append_value_for_timestamp(
existing_ts=fcst_timeseries, new_ts=temp_timeseries)
sum_timeseries = summed_timeseries(fcst_timeseries)
for i in range(len(sum_timeseries)):
if float(sum_timeseries[i][1]) < 0:
sum_timeseries[i][1] = 0
if sum_timeseries is not None and len(sum_timeseries) > 0:
Sim_TS.insert_data(timeseries=sum_timeseries,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating fcst rainfalls in curw_sim.")
finally:
destroy_Pool(curw_sim_pool)
destroy_Pool(curw_fcst_pool)
def update_rainfall_fcsts(target_model, method, grid_interpolation, model_list,
timestep):
"""
Update rainfall forecasts for flo2d models
:param target_model: target model for which input ins prepared
:param method: value interpolation method
:param grid_interpolation: grid interpolation method
:param model_list: list of forecast model and their versions used to calculate the rainfall
e.g.: [["WRF_E", "4.0", "evening_18hrs"],["WRF_SE", "v4", ,"evening_18hrs"],["WRF_Ensemble", "4.0", ,"MME"]]
:param timestep: output timeseries timestep
:return:
"""
try:
# Connect to the database
curw_sim_pool = get_Pool(host=CURW_SIM_HOST,
user=CURW_SIM_USERNAME,
password=CURW_SIM_PASSWORD,
port=CURW_SIM_PORT,
db=CURW_SIM_DATABASE)
curw_fcst_pool = get_Pool(host=CURW_FCST_HOST,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE)
Sim_TS = Sim_Timeseries(pool=curw_sim_pool)
Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool)
# [hash_id, station_id, station_name, latitude, longitude]
active_obs_stations = read_csv(
'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv'
)
obs_stations_dict = {
} # keys: obs station id , value: [name, latitude, longitude]
for obs_index in range(len(active_obs_stations)):
obs_stations_dict[active_obs_stations[obs_index][1]] = [
active_obs_stations[obs_index][2],
active_obs_stations[obs_index][3],
active_obs_stations[obs_index][4]
]
obs_d03_mapping = get_obs_to_d03_grid_mappings_for_rainfall(
pool=curw_sim_pool, grid_interpolation=grid_interpolation)
for obs_id in obs_stations_dict.keys():
meta_data = {
'latitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[1])),
'longitude':
float('%.6f' % float(obs_stations_dict.get(obs_id)[2])),
'model':
target_model,
'method':
method,
'grid_id':
'rainfall_{}_{}_{}'.format(obs_id,
obs_stations_dict.get(obs_id)[0],
grid_interpolation)
}
tms_id = Sim_TS.get_timeseries_id_if_exists(meta_data=meta_data)
if tms_id is None:
tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data)
meta_data['id'] = tms_id
Sim_TS.insert_run(meta_data=meta_data)
obs_end = Sim_TS.get_obs_end(id_=tms_id)
fcst_timeseries = []
for i in range(len(model_list)):
source_id = get_source_id(pool=curw_fcst_pool,
model=model_list[i][0],
version=model_list[i][1])
sim_tag = model_list[i][2]
coefficient = model_list[i][3]
temp_timeseries = []
if timestep == 5:
if obs_end is not None:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=obs_d03_mapping.
get(meta_data['grid_id'])[0],
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1))
else:
temp_timeseries = convert_15_min_ts_to_5_mins_ts(
newly_extracted_timeseries=Fcst_TS.
get_latest_timeseries(sim_tag=sim_tag,
station_id=obs_d03_mapping.
get(meta_data['grid_id'])[0],
source_id=source_id,
variable_id=1,
unit_id=1))
elif timestep == 15:
if obs_end is not None:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=obs_d03_mapping.get(
meta_data['grid_id'])[0],
start=obs_end,
source_id=source_id,
variable_id=1,
unit_id=1)
else:
temp_timeseries = Fcst_TS.get_latest_timeseries(
sim_tag=sim_tag,
station_id=obs_d03_mapping.get(
meta_data['grid_id'])[0],
source_id=source_id,
variable_id=1,
unit_id=1)
if coefficient != 1:
for j in range(len(temp_timeseries)):
temp_timeseries[j][1] = float(
temp_timeseries[j][1]) * coefficient
if i == 0:
fcst_timeseries = temp_timeseries
else:
fcst_timeseries = append_value_for_timestamp(
existing_ts=fcst_timeseries, new_ts=temp_timeseries)
sum_timeseries = summed_timeseries(fcst_timeseries)
for i in range(len(sum_timeseries)):
if float(sum_timeseries[i][1]) < 0:
sum_timeseries[i][1] = 0
if sum_timeseries is not None and len(sum_timeseries) > 0:
Sim_TS.insert_data(timeseries=sum_timeseries,
tms_id=tms_id,
upsert=True)
except Exception as e:
traceback.print_exc()
logger.error(
"Exception occurred while updating fcst rainfalls in curw_sim.")
finally:
destroy_Pool(curw_sim_pool)
destroy_Pool(curw_fcst_pool)
'version': 'v3',
'parameters': {}
}]
pool = get_Pool(host=HOST,
port=PORT,
user=USERNAME,
password=PASSWORD,
db=DATABASE)
print("########### Add Sources #################################")
print(add_sources(sources=sources, pool=pool))
print("########### Get Sources by id ###########################")
print("Id 3:", get_source_by_id(pool=pool, id_="3"))
print("########## Retrieve source id ###########################")
print("'model': 'OBS_WATER_LEVEL', 'version': ''",
get_source_id(pool=pool, model="OBS_WATER_LEVEL", version=""))
print("######### Delete source by id ###########################")
print("Id 3 deleted status: ", delete_source_by_id(pool=pool, id_=3))
print("######### Delete source with given model, version #######")
print("model': 'wrfSE', 'version': 'v3' delete status :",
delete_source(pool=pool, model="wrfSE", version="v3"))
destroy_Pool(pool)
print("bdjhsvdhsvdjh")
exit(0)
