utc_offset = read_attribute_from_config_file('utc_offset', config,
False)
if utc_offset is None:
utc_offset = ''
# sim tag
if sim_tag is None:
sim_tag = read_attribute_from_config_file('sim_tag', config, True)
# source details
model = read_attribute_from_config_file('model', config, True)
version = "_".join(flo2d_model.split("_")[1:])
# unit details
unit = read_attribute_from_config_file('unit', config, True)
unit_type = UnitType.getType(
read_attribute_from_config_file('unit_type', config, True))
# variable details
variable = read_attribute_from_config_file('variable', config, True)
hychan_out_file_path = os.path.join(output_dir, HYCHAN_OUT_FILE)
timdep_file_path = os.path.join(output_dir, TIMDEP_FILE)
pool = get_Pool(host=con_params.CURW_FCST_HOST,
port=con_params.CURW_FCST_PORT,
db=con_params.CURW_FCST_DATABASE,
user=con_params.CURW_FCST_USERNAME,
password=con_params.CURW_FCST_PASSWORD)
flo2d_model_name = '{}_{}'.format(model, version)
def extract_distrubuted_hechms_outputs(output_file_name, output_dir, run_date,
run_time):
"""
Config.json
{
"output_file_name": "DailyDischarge.csv",
"output_dir": "",
"run_date": "2019-05-24",
"run_time": "00:00:00",
"utc_offset": "",
"sim_tag": "hourly_run",
"model": "HECHMS",
"version": "single",
"unit": "m3/s",
"unit_type": "Instantaneous",
"variable": "Discharge",
"station_name": "Hanwella"
}
"""
try:
config = json.loads(open('config.json').read())
# output related details
output_file_name = output_file_name
output_dir = output_dir
run_date = run_date
run_time = run_time
utc_offset = read_attribute_from_config_file('utc_offset', config,
False)
if utc_offset is None:
utc_offset = ''
# sim tag
sim_tag = read_attribute_from_config_file('sim_tag', config, True)
# source details
model = read_attribute_from_config_file('model', config, True)
version = read_attribute_from_config_file('version', config, True)
# unit details
unit = read_attribute_from_config_file('unit', config, True)
unit_type = UnitType.getType(
read_attribute_from_config_file('unit_type', config, True))
# variable details
variable = read_attribute_from_config_file('variable', config, True)
# station details
station_name = read_attribute_from_config_file('station_name', config,
True)
out_file_path = os.path.join(output_dir, output_file_name)
if not os.path.exists(out_file_path):
msg = 'no file :: {}'.format(out_file_path)
logger.warning(msg)
print(msg)
exit(1)
fgt = get_file_last_modified_time(out_file_path)
print("fgt, ", fgt)
timeseries = read_csv(out_file_path)
pool = get_Pool(host=CURW_FCST_HOST,
port=CURW_FCST_PORT,
db=CURW_FCST_DATABASE,
user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD)
hechms_stations = get_hechms_stations(pool=pool)
station_id = hechms_stations.get(station_name)[0]
lat = str(hechms_stations.get(station_name)[1])
lon = str(hechms_stations.get(station_name)[2])
source_id = get_source_id(pool=pool, model=model, version=version)
variable_id = get_variable_id(pool=pool, variable=variable)
unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
tms_meta = {
'sim_tag': sim_tag,
'model': model,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value,
'latitude': lat,
'longitude': lon,
'station_id': station_id,
'source_id': source_id,
'variable_id': variable_id,
'unit_id': unit_id
}
utcOffset = getUTCOffset(utc_offset, default=True)
if utcOffset != timedelta():
tms_meta['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool,
timeseries=timeseries,
run_date=run_date,
run_time=run_time,
tms_meta=tms_meta,
fgt=fgt)
except Exception as e:
logger.error('JSON config data loading error.')
print('JSON config data loading error.')
traceback.print_exc()
finally:
logger.info("Process finished.")
print("Process finished.")
if 'wrf_model_list' in config and (config['wrf_model_list'] != ""):
wrf_model_list = config['wrf_model_list']
wrf_model_list = wrf_model_list.split(',')
else:
logger.error("wrf_model_list not specified in config file.")
exit(1)
# unit details
if 'unit' in config and (config['unit'] != ""):
unit = config['unit']
else:
logger.error("unit not specified in config file.")
exit(1)
if 'unit_type' in config and (config['unit_type'] != ""):
unit_type = UnitType.getType(config['unit_type'])
else:
logger.error("unit_type not specified in config file.")
exit(1)
# variable details
if 'variable' in config and (config['variable'] != ""):
variable = config['variable']
else:
logger.error("variable not specified in config file.")
exit(1)
# connection params
if 'host' in config and (config['host'] != ""):
host = config['host']
else:
def extract_distrubuted_hechms_outputs(target_model, db_user, db_pwd, db_host,
db_name, out_file_path, run_date,
run_time):
"""
Config.json
{
"output_file_name": "DailyDischarge.csv",
"output_dir": "",
"run_date": "2019-05-24",
"run_time": "00:00:00",
"utc_offset": "",
"sim_tag": "hourly_run",
"model": "HECHMS",
"version": "single",
"unit": "m3/s",
"unit_type": "Instantaneous",
"variable": "Discharge",
"station_name": "Hanwella"
}
"""
try:
config = json.loads(
open('/home/curw/git/distributed_hechms/uploads/config.json').read(
))
# output related details
run_date = run_date
run_time = run_time
utc_offset = read_attribute_from_config_file('utc_offset', config,
False)
if utc_offset is None:
utc_offset = ''
# sim tag
sim_tag = read_attribute_from_config_file('sim_tag', config, True)
print("extract_distrubuted_hechms_outputs|sim_tag : ", sim_tag)
# source details
model = read_attribute_from_config_file('model', config, True)
print("extract_distrubuted_hechms_outputs|model : ", model)
version_config = read_attribute_from_config_file(
'version_config', config, True)
print("extract_distrubuted_hechms_outputs|version_config : ",
version_config)
version = version_config[target_model]
print("extract_distrubuted_hechms_outputs|version : ", version)
# unit details
unit = read_attribute_from_config_file('unit', config, True)
print("extract_distrubuted_hechms_outputs|unit : ", unit)
unit_type = UnitType.getType(
read_attribute_from_config_file('unit_type', config, True))
print("extract_distrubuted_hechms_outputs|unit_type : ", unit_type)
# variable details
variable = read_attribute_from_config_file('variable', config, True)
print("extract_distrubuted_hechms_outputs|variable : ", variable)
# station details
station_name = read_attribute_from_config_file('station_name', config,
True)
print("extract_distrubuted_hechms_outputs|station_name : ",
station_name)
if not os.path.exists(out_file_path):
msg = 'no file :: {}'.format(out_file_path)
logger.warning(msg)
print(msg)
exit(1)
fgt = get_file_last_modified_time(out_file_path)
print("extract_distrubuted_hechms_outputs|fgt : ", fgt)
timeseries = read_csv(out_file_path)
pool = get_Pool(host=db_host,
port=3306,
db=db_name,
user=db_user,
password=db_pwd)
hechms_stations = get_hechms_stations(pool=pool)
print("extract_distrubuted_hechms_outputs|hechms_stations : ",
hechms_stations)
station_id = hechms_stations.get(station_name)[0]
lat = str(hechms_stations.get(station_name)[1])
lon = str(hechms_stations.get(station_name)[2])
print("extract_distrubuted_hechms_outputs|[station_id, lat, lon] : ",
[station_id, lat, lon])
source_id = get_source_id(pool=pool, model=model, version=version)
print("extract_distrubuted_hechms_outputs|source_id : ", source_id)
variable_id = get_variable_id(pool=pool, variable=variable)
print("extract_distrubuted_hechms_outputs|variable_id : ", variable_id)
unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
print("extract_distrubuted_hechms_outputs|unit_id : ", unit_id)
tms_meta = {
'sim_tag': sim_tag,
'model': model,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value,
'latitude': lat,
'longitude': lon,
'station_id': station_id,
'source_id': source_id,
'variable_id': variable_id,
'unit_id': unit_id
}
print("extract_distrubuted_hechms_outputs|tms_meta : ", tms_meta)
utcOffset = getUTCOffset(utc_offset, default=True)
if utcOffset != timedelta():
tms_meta['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool,
timeseries=timeseries,
run_date=run_date,
run_time=run_time,
tms_meta=tms_meta,
fgt=fgt)
return {'Result': 'Success'}
except Exception as e:
logger.error('JSON config data loading error.')
print('JSON config data loading error.')
traceback.print_exc()
return {'Result': 'Fail'}
if __name__=="__main__":
try:
################################################
# Initialize parameters for DISTRIBUTED HECHMS #
################################################
# source details
model = 'HECHMS'
version = 'distributed'
# unit details
unit = 'm3/s'
unit_type = UnitType.getType('Instantaneous')
# variable details
variable = 'Discharge'
# station details
hechms_stations = read_csv('hechms_stations.csv')
pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD,
db=CURW_FCST_DATABASE)
# ########
# # test
# ########
# USERNAME = "******"
def upload_waterlevels(dir_path, ts_start_date, ts_start_time, run_date, run_time):
"""
Config.json
{
"HYCHAN_OUT_FILE": "HYCHAN.OUT",
"TIMDEP_FILE": "TIMDEP.OUT",
"output_dir": "",
"run_date": "2019-05-24",
"run_time": "",
"ts_start_date": "",
"ts_start_time": "",
"utc_offset": "",
"sim_tag": "",
"model": "WRF",
"version": "v3",
"unit": "mm",
"unit_type": "Accumulative",
"variable": "Precipitation"
}
"""
try:
config_path = os.path.join(os.getcwd(), 'extract', 'config_curw_fcst.json')
config = json.loads(open(config_path).read())
# flo2D related details
HYCHAN_OUT_FILE = read_attribute_from_config_file('HYCHAN_OUT_FILE', config, True)
TIMDEP_FILE = read_attribute_from_config_file('TIMDEP_FILE', config, True)
output_dir = dir_path
run_date = run_date
run_time = run_time
ts_start_date = ts_start_date
ts_start_time = ts_start_time
utc_offset = read_attribute_from_config_file('utc_offset', config, False)
if utc_offset is None:
utc_offset = ''
# sim tag
sim_tag = read_attribute_from_config_file('sim_tag', config, True)
# source details
model = read_attribute_from_config_file('model', config, True)
version = read_attribute_from_config_file('version', config, True)
# unit details
unit = read_attribute_from_config_file('unit', config, True)
unit_type = UnitType.getType(read_attribute_from_config_file('unit_type', config, True))
# variable details
variable = read_attribute_from_config_file('variable', config, True)
hychan_out_file_path = os.path.join(output_dir, HYCHAN_OUT_FILE)
timdep_file_path = os.path.join(output_dir, TIMDEP_FILE)
pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD)
# pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE)
flo2d_model_name = '{}_{}'.format(model, version)
flo2d_source = json.loads(get_source_parameters(pool=pool, model=model, version=version))
print("############### source ############", flo2d_source)
flo2d_stations = get_flo2d_output_stations(pool=pool, flo2d_model=StationEnum.getType(flo2d_model_name))
print("############### 1st occurrence ############", flo2d_stations)
source_id = get_source_id(pool=pool, model=model, version=version)
variable_id = get_variable_id(pool=pool, variable=variable)
unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
tms_meta = {
'sim_tag': sim_tag,
'model': model,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value,
'source_id': source_id,
'variable_id': variable_id,
'unit_id': unit_id
}
CHANNEL_CELL_MAP = flo2d_source["CHANNEL_CELL_MAP"]
FLOOD_PLAIN_CELL_MAP = flo2d_source["FLOOD_PLAIN_CELL_MAP"]
ELEMENT_NUMBERS = CHANNEL_CELL_MAP.keys()
FLOOD_ELEMENT_NUMBERS = FLOOD_PLAIN_CELL_MAP.keys()
SERIES_LENGTH = 0
MISSING_VALUE = -999
utcOffset = getUTCOffset(utc_offset, default=True)
print('Extract Water Level Result of FLO2D on', run_date, '@', run_time, 'with Base time of', ts_start_date,
'@', ts_start_time)
# Check HYCHAN.OUT file exists
if not os.path.exists(hychan_out_file_path):
print('Unable to find file : ', hychan_out_file_path)
traceback.print_exc()
#####################################
# Calculate the size of time series #
#####################################
bufsize = 65536
with open(hychan_out_file_path) as infile:
isWaterLevelLines = False
isCounting = False
countSeriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series
while True:
lines = infile.readlines(bufsize)
if not lines or SERIES_LENGTH:
break
for line in lines:
if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
isWaterLevelLines = True
elif isWaterLevelLines:
cols = line.split()
if len(cols) > 0 and cols[0].replace('.', '', 1).isdigit():
countSeriesSize += 1
isCounting = True
elif isWaterLevelLines and isCounting:
SERIES_LENGTH = countSeriesSize
break
print('Series Length is :', SERIES_LENGTH)
bufsize = 65536
#################################################################
# Extract Channel Water Level elevations from HYCHAN.OUT file #
#################################################################
print('Extract Channel Water Level Result of FLO2D (HYCHAN.OUT) on', run_date, '@', run_time,
'with Base time of',
ts_start_date, '@', ts_start_time)
with open(hychan_out_file_path) as infile:
isWaterLevelLines = False
isSeriesComplete = False
waterLevelLines = []
seriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series
while True:
lines = infile.readlines(bufsize)
if not lines:
break
for line in lines:
if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
seriesSize = 0
elementNo = line.split()[5]
if elementNo in ELEMENT_NUMBERS:
isWaterLevelLines = True
waterLevelLines.append(line)
else:
isWaterLevelLines = False
elif isWaterLevelLines:
cols = line.split()
if len(cols) > 0 and isfloat(cols[0]):
seriesSize += 1
waterLevelLines.append(line)
if seriesSize == SERIES_LENGTH:
isSeriesComplete = True
if isSeriesComplete:
baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S')
timeseries = []
elementNo = waterLevelLines[0].split()[5]
# print('Extracted Cell No', elementNo, CHANNEL_CELL_MAP[elementNo])
for ts in waterLevelLines[1:]:
v = ts.split()
if len(v) < 1:
continue
# Get flood level (Elevation)
value = v[1]
# Get flood depth (Depth)
# value = v[2]
if not isfloat(value):
value = MISSING_VALUE
continue # If value is not present, skip
if value == 'NaN':
continue # If value is NaN, skip
timeStep = float(v[0])
currentStepTime = baseTime + timedelta(hours=timeStep)
dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S")
timeseries.append([dateAndTime, value])
# Save Forecast values into Database
opts = {
'elementNo': elementNo,
'tms_meta': tms_meta
}
# print('>>>>>', opts)
if utcOffset != timedelta():
opts['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries,
run_date=run_date, run_time=run_time, opts=opts,
flo2d_stations=flo2d_stations)
isWaterLevelLines = False
isSeriesComplete = False
waterLevelLines = []
# -- END for loop
# -- END while loop
#################################################################
# Extract Flood Plain water elevations from TIMEDEP.OUT file #
#################################################################
if not os.path.exists(timdep_file_path):
print('Unable to find file : ', timdep_file_path)
traceback.print_exc()
print('Extract Flood Plain Water Level Result of FLO2D (TIMEDEP.OUT) on', run_date, '@', run_time,
'with Base time of', ts_start_date,
'@', ts_start_time)
with open(timdep_file_path) as infile:
waterLevelLines = []
waterLevelSeriesDict = dict.fromkeys(FLOOD_ELEMENT_NUMBERS, [])
while True:
lines = infile.readlines(bufsize)
if not lines:
break
for line in lines:
if len(line.split()) == 1:
# continue
if len(waterLevelLines) > 0:
waterLevels = get_water_level_of_channels(waterLevelLines, FLOOD_ELEMENT_NUMBERS)
# Get Time stamp Ref:http://stackoverflow.com/a/13685221/1461060
# print('waterLevelLines[0].split() : ', waterLevelLines[0].split())
ModelTime = float(waterLevelLines[0].split()[0])
baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S')
currentStepTime = baseTime + timedelta(hours=ModelTime)
dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S")
for elementNo in FLOOD_ELEMENT_NUMBERS:
tmpTS = waterLevelSeriesDict[elementNo][:]
if elementNo in waterLevels:
tmpTS.append([dateAndTime, waterLevels[elementNo]])
else:
tmpTS.append([dateAndTime, MISSING_VALUE])
waterLevelSeriesDict[elementNo] = tmpTS
isWaterLevelLines = False
# for l in waterLevelLines :
# print(l)
waterLevelLines = []
waterLevelLines.append(line)
# print('len(FLOOD_ELEMENT_NUMBERS) : ', len(FLOOD_ELEMENT_NUMBERS))
for elementNo in FLOOD_ELEMENT_NUMBERS:
# Save Forecast values into Database
opts = {
'elementNo': elementNo,
'tms_meta': tms_meta
}
if utcOffset != timedelta():
opts['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool, timeseries=waterLevelSeriesDict[elementNo],
run_date=run_date, run_time=run_time, opts=opts,
flo2d_stations=flo2d_stations)
except Exception as e:
logger.error('JSON config data loading error.')
print('JSON config data loading error.')
traceback.print_exc()
finally:
logger.info("Process finished.")
print("Process finished.")