variable = read_attribute_from_config_file('variable', config, True)
hychan_out_file_path = os.path.join(output_dir, HYCHAN_OUT_FILE)
timdep_file_path = os.path.join(output_dir, TIMDEP_FILE)
pool = get_Pool(host=con_params.CURW_FCST_HOST,
port=con_params.CURW_FCST_PORT,
db=con_params.CURW_FCST_DATABASE,
user=con_params.CURW_FCST_USERNAME,
password=con_params.CURW_FCST_PASSWORD)
flo2d_model_name = '{}_{}'.format(model, version)
flo2d_source = json.loads(
get_source_parameters(pool=pool, model=model, version=version))
flo2d_stations = get_flo2d_output_stations(
pool=pool, flo2d_model=StationEnum.getType(flo2d_model_name))
source_id = get_source_id(pool=pool, model=model, version=version)
variable_id = get_variable_id(pool=pool, variable=variable)
unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
tms_meta = {
'sim_tag': sim_tag,
'model': model,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value,
'source_id': source_id,
def upload_waterlevels(dir_path, ts_start_date, ts_start_time, run_date, run_time):
"""
Config.json
{
"HYCHAN_OUT_FILE": "HYCHAN.OUT",
"TIMDEP_FILE": "TIMDEP.OUT",
"output_dir": "",
"run_date": "2019-05-24",
"run_time": "",
"ts_start_date": "",
"ts_start_time": "",
"utc_offset": "",
"sim_tag": "",
"model": "WRF",
"version": "v3",
"unit": "mm",
"unit_type": "Accumulative",
"variable": "Precipitation"
}
"""
try:
config_path = os.path.join(os.getcwd(), 'extract', 'config_curw_fcst.json')
config = json.loads(open(config_path).read())
# flo2D related details
HYCHAN_OUT_FILE = read_attribute_from_config_file('HYCHAN_OUT_FILE', config, True)
TIMDEP_FILE = read_attribute_from_config_file('TIMDEP_FILE', config, True)
output_dir = dir_path
run_date = run_date
run_time = run_time
ts_start_date = ts_start_date
ts_start_time = ts_start_time
utc_offset = read_attribute_from_config_file('utc_offset', config, False)
if utc_offset is None:
utc_offset = ''
# sim tag
sim_tag = read_attribute_from_config_file('sim_tag', config, True)
# source details
model = read_attribute_from_config_file('model', config, True)
version = read_attribute_from_config_file('version', config, True)
# unit details
unit = read_attribute_from_config_file('unit', config, True)
unit_type = UnitType.getType(read_attribute_from_config_file('unit_type', config, True))
# variable details
variable = read_attribute_from_config_file('variable', config, True)
hychan_out_file_path = os.path.join(output_dir, HYCHAN_OUT_FILE)
timdep_file_path = os.path.join(output_dir, TIMDEP_FILE)
pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME,
password=CURW_FCST_PASSWORD)
# pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE)
flo2d_model_name = '{}_{}'.format(model, version)
flo2d_source = json.loads(get_source_parameters(pool=pool, model=model, version=version))
print("############### source ############", flo2d_source)
flo2d_stations = get_flo2d_output_stations(pool=pool, flo2d_model=StationEnum.getType(flo2d_model_name))
print("############### 1st occurrence ############", flo2d_stations)
source_id = get_source_id(pool=pool, model=model, version=version)
variable_id = get_variable_id(pool=pool, variable=variable)
unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type)
tms_meta = {
'sim_tag': sim_tag,
'model': model,
'version': version,
'variable': variable,
'unit': unit,
'unit_type': unit_type.value,
'source_id': source_id,
'variable_id': variable_id,
'unit_id': unit_id
}
CHANNEL_CELL_MAP = flo2d_source["CHANNEL_CELL_MAP"]
FLOOD_PLAIN_CELL_MAP = flo2d_source["FLOOD_PLAIN_CELL_MAP"]
ELEMENT_NUMBERS = CHANNEL_CELL_MAP.keys()
FLOOD_ELEMENT_NUMBERS = FLOOD_PLAIN_CELL_MAP.keys()
SERIES_LENGTH = 0
MISSING_VALUE = -999
utcOffset = getUTCOffset(utc_offset, default=True)
print('Extract Water Level Result of FLO2D on', run_date, '@', run_time, 'with Base time of', ts_start_date,
'@', ts_start_time)
# Check HYCHAN.OUT file exists
if not os.path.exists(hychan_out_file_path):
print('Unable to find file : ', hychan_out_file_path)
traceback.print_exc()
#####################################
# Calculate the size of time series #
#####################################
bufsize = 65536
with open(hychan_out_file_path) as infile:
isWaterLevelLines = False
isCounting = False
countSeriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series
while True:
lines = infile.readlines(bufsize)
if not lines or SERIES_LENGTH:
break
for line in lines:
if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
isWaterLevelLines = True
elif isWaterLevelLines:
cols = line.split()
if len(cols) > 0 and cols[0].replace('.', '', 1).isdigit():
countSeriesSize += 1
isCounting = True
elif isWaterLevelLines and isCounting:
SERIES_LENGTH = countSeriesSize
break
print('Series Length is :', SERIES_LENGTH)
bufsize = 65536
#################################################################
# Extract Channel Water Level elevations from HYCHAN.OUT file #
#################################################################
print('Extract Channel Water Level Result of FLO2D (HYCHAN.OUT) on', run_date, '@', run_time,
'with Base time of',
ts_start_date, '@', ts_start_time)
with open(hychan_out_file_path) as infile:
isWaterLevelLines = False
isSeriesComplete = False
waterLevelLines = []
seriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series
while True:
lines = infile.readlines(bufsize)
if not lines:
break
for line in lines:
if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
seriesSize = 0
elementNo = line.split()[5]
if elementNo in ELEMENT_NUMBERS:
isWaterLevelLines = True
waterLevelLines.append(line)
else:
isWaterLevelLines = False
elif isWaterLevelLines:
cols = line.split()
if len(cols) > 0 and isfloat(cols[0]):
seriesSize += 1
waterLevelLines.append(line)
if seriesSize == SERIES_LENGTH:
isSeriesComplete = True
if isSeriesComplete:
baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S')
timeseries = []
elementNo = waterLevelLines[0].split()[5]
# print('Extracted Cell No', elementNo, CHANNEL_CELL_MAP[elementNo])
for ts in waterLevelLines[1:]:
v = ts.split()
if len(v) < 1:
continue
# Get flood level (Elevation)
value = v[1]
# Get flood depth (Depth)
# value = v[2]
if not isfloat(value):
value = MISSING_VALUE
continue # If value is not present, skip
if value == 'NaN':
continue # If value is NaN, skip
timeStep = float(v[0])
currentStepTime = baseTime + timedelta(hours=timeStep)
dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S")
timeseries.append([dateAndTime, value])
# Save Forecast values into Database
opts = {
'elementNo': elementNo,
'tms_meta': tms_meta
}
# print('>>>>>', opts)
if utcOffset != timedelta():
opts['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries,
run_date=run_date, run_time=run_time, opts=opts,
flo2d_stations=flo2d_stations)
isWaterLevelLines = False
isSeriesComplete = False
waterLevelLines = []
# -- END for loop
# -- END while loop
#################################################################
# Extract Flood Plain water elevations from TIMEDEP.OUT file #
#################################################################
if not os.path.exists(timdep_file_path):
print('Unable to find file : ', timdep_file_path)
traceback.print_exc()
print('Extract Flood Plain Water Level Result of FLO2D (TIMEDEP.OUT) on', run_date, '@', run_time,
'with Base time of', ts_start_date,
'@', ts_start_time)
with open(timdep_file_path) as infile:
waterLevelLines = []
waterLevelSeriesDict = dict.fromkeys(FLOOD_ELEMENT_NUMBERS, [])
while True:
lines = infile.readlines(bufsize)
if not lines:
break
for line in lines:
if len(line.split()) == 1:
# continue
if len(waterLevelLines) > 0:
waterLevels = get_water_level_of_channels(waterLevelLines, FLOOD_ELEMENT_NUMBERS)
# Get Time stamp Ref:http://stackoverflow.com/a/13685221/1461060
# print('waterLevelLines[0].split() : ', waterLevelLines[0].split())
ModelTime = float(waterLevelLines[0].split()[0])
baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S')
currentStepTime = baseTime + timedelta(hours=ModelTime)
dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S")
for elementNo in FLOOD_ELEMENT_NUMBERS:
tmpTS = waterLevelSeriesDict[elementNo][:]
if elementNo in waterLevels:
tmpTS.append([dateAndTime, waterLevels[elementNo]])
else:
tmpTS.append([dateAndTime, MISSING_VALUE])
waterLevelSeriesDict[elementNo] = tmpTS
isWaterLevelLines = False
# for l in waterLevelLines :
# print(l)
waterLevelLines = []
waterLevelLines.append(line)
# print('len(FLOOD_ELEMENT_NUMBERS) : ', len(FLOOD_ELEMENT_NUMBERS))
for elementNo in FLOOD_ELEMENT_NUMBERS:
# Save Forecast values into Database
opts = {
'elementNo': elementNo,
'tms_meta': tms_meta
}
if utcOffset != timedelta():
opts['utcOffset'] = utcOffset
# Push timeseries to database
save_forecast_timeseries_to_db(pool=pool, timeseries=waterLevelSeriesDict[elementNo],
run_date=run_date, run_time=run_time, opts=opts,
flo2d_stations=flo2d_stations)
except Exception as e:
logger.error('JSON config data loading error.')
print('JSON config data loading error.')
traceback.print_exc()
finally:
logger.info("Process finished.")
print("Process finished.")