if 'MYSQL_HOST' in CONFIG:
MYSQL_HOST = CONFIG['MYSQL_HOST']
if 'MYSQL_USER' in CONFIG:
MYSQL_USER = CONFIG['MYSQL_USER']
if 'MYSQL_DB' in CONFIG:
MYSQL_DB = CONFIG['MYSQL_DB']
if 'MYSQL_PASSWORD' in CONFIG:
MYSQL_PASSWORD = CONFIG['MYSQL_PASSWORD']
adapter = MySQLAdapter(host=MYSQL_HOST,
user=MYSQL_USER,
password=MYSQL_PASSWORD,
db=MYSQL_DB)
# TODO: Pass source name as a paramter to script
flo2d_source = adapter.get_source(name='FLO2D')
try:
flo2d_source = json.loads(flo2d_source.get('parameters', "{}"))
except Exception as e:
print(e)
traceback.print_exc()
CHANNEL_CELL_MAP = {}
if 'CHANNEL_CELL_MAP' in flo2d_source:
CHANNEL_CELL_MAP = flo2d_source['CHANNEL_CELL_MAP']
FLOOD_PLAIN_CELL_MAP = {}
if 'FLOOD_PLAIN_CELL_MAP' in flo2d_source:
FLOOD_PLAIN_CELL_MAP = flo2d_source['FLOOD_PLAIN_CELL_MAP']
"""
{
"CHANNEL_CELL_MAP": {
"179": "Wellawatta",
def upload_waterlevels_curw(dir_path, ts_start_date, ts_start_time, run_date,
run_time):
print(
'upload_waterlevels_curw|[ts_start_date, ts_start_time, run_date, run_time] : ',
[ts_start_date, ts_start_time, run_date, run_time])
SERIES_LENGTH = 0
MISSING_VALUE = -999
try:
config_path = os.path.join(os.getcwd(), 'extract', 'config.json')
# print('config_path : ', config_path)
utc_offset = ''
with open(config_path) as json_file:
config_data = json.load(json_file)
output_dir = dir_path
HYCHAN_OUT_FILE = config_data['hychan_out_file']
TIMDEP_FILE = config_data['timdep_out_file']
FLO2D_MODEL = config_data['flo2d_model']
RUN_NAME = config_data['run_name']
hychan_out_file_path = os.path.join(dir_path, HYCHAN_OUT_FILE)
timdep_file_path = os.path.join(dir_path, TIMDEP_FILE)
# print('hychan_out_file_path : ', hychan_out_file_path)
# print('timdep_file_path : ', timdep_file_path)
forceInsert = True
MYSQL_HOST = config_data['db_host']
MYSQL_USER = config_data['db_user']
MYSQL_DB = config_data['db_name']
MYSQL_PASSWORD = config_data['db_password']
# if 'UTC_OFFSET' in config_data and len(
# config_data['UTC_OFFSET']): # Use FLO2D Config file data, if available
# UTC_OFFSET = config_data['UTC_OFFSET']
# if utc_offset:
# UTC_OFFSET = config_data
utcOffset = getUTCOffset('', default=True)
adapter = MySQLAdapter(host=MYSQL_HOST,
user=MYSQL_USER,
password=MYSQL_PASSWORD,
db=MYSQL_DB)
flo2d_source = adapter.get_source(name=FLO2D_MODEL)
try:
flo2d_source = json.loads(flo2d_source.get('parameters', "{}"))
CHANNEL_CELL_MAP = {}
if 'CHANNEL_CELL_MAP' in flo2d_source:
CHANNEL_CELL_MAP = flo2d_source['CHANNEL_CELL_MAP']
FLOOD_PLAIN_CELL_MAP = {}
if 'FLOOD_PLAIN_CELL_MAP' in flo2d_source:
FLOOD_PLAIN_CELL_MAP = flo2d_source['FLOOD_PLAIN_CELL_MAP']
ELEMENT_NUMBERS = CHANNEL_CELL_MAP.keys()
FLOOD_ELEMENT_NUMBERS = FLOOD_PLAIN_CELL_MAP.keys()
# Calculate the size of time series
bufsize = 65536
with open(hychan_out_file_path) as infile:
isWaterLevelLines = False
isCounting = False
countSeriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series
while True:
lines = infile.readlines(bufsize)
if not lines or SERIES_LENGTH:
break
for line in lines:
if line.startswith(
'CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
isWaterLevelLines = True
elif isWaterLevelLines:
cols = line.split()
if len(cols) > 0 and cols[0].replace(
'.', '', 1).isdigit():
countSeriesSize += 1
isCounting = True
elif isWaterLevelLines and isCounting:
SERIES_LENGTH = countSeriesSize
break
# print('Series Length is :', SERIES_LENGTH)
bufsize = 65536
#################################################################
# Extract Channel Water Level elevations from HYCHAN.OUT file #
#################################################################
with open(hychan_out_file_path) as infile:
isWaterLevelLines = False
isSeriesComplete = False
waterLevelLines = []
seriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series
while True:
lines = infile.readlines(bufsize)
if not lines:
break
for line in lines:
if line.startswith(
'CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5):
seriesSize = 0
elementNo = line.split()[5]
if elementNo in ELEMENT_NUMBERS:
isWaterLevelLines = True
waterLevelLines.append(line)
else:
isWaterLevelLines = False
elif isWaterLevelLines:
cols = line.split()
if len(cols) > 0 and isfloat(cols[0]):
seriesSize += 1
waterLevelLines.append(line)
if seriesSize == SERIES_LENGTH:
isSeriesComplete = True
if isSeriesComplete:
baseTime = datetime.strptime(
'%s %s' % (ts_start_date, ts_start_time),
'%Y-%m-%d %H:%M:%S')
timeseries = []
elementNo = waterLevelLines[0].split()[5]
# print('Extracted Cell No', elementNo, CHANNEL_CELL_MAP[elementNo])
for ts in waterLevelLines[1:]:
v = ts.split()
if len(v) < 1:
continue
# Get flood level (Elevation)
value = v[1]
# Get flood depth (Depth)
# value = v[2]
if not isfloat(value):
value = MISSING_VALUE
continue # If value is not present, skip
if value == 'NaN':
continue # If value is NaN, skip
timeStep = float(v[0])
currentStepTime = baseTime + timedelta(
hours=timeStep)
dateAndTime = currentStepTime.strftime(
"%Y-%m-%d %H:%M:%S")
timeseries.append([dateAndTime, value])
# Save Forecast values into Database
opts = {
'forceInsert': forceInsert,
'station': CHANNEL_CELL_MAP[elementNo],
'run_name': RUN_NAME
}
# print('>>>>>', opts)
if utcOffset != timedelta():
opts['utcOffset'] = utcOffset
# folder_path_ts = os.path.join(dir_path, 'time_series')
# if not os.path.exists(folder_path_ts):
# try:
# os.makedirs(folder_path_ts)
# except OSError as e:
# print(str(e))
# file_path = os.path.join(folder_path_ts, 'time_series_' + elementNo + '.txt')
# with open(file_path, 'w') as f:
# for item in timeseries:
# f.write("%s\n" % item)
save_forecast_timeseries(
adapter, timeseries, run_date, run_time,
opts)
isWaterLevelLines = False
isSeriesComplete = False
waterLevelLines = []
# -- END for loop
# -- END while loop
#################################################################
# Extract Flood Plain water elevations from BASE.OUT file #
#################################################################
with open(timdep_file_path) as infile:
waterLevelLines = []
waterLevelSeriesDict = dict.fromkeys(
FLOOD_ELEMENT_NUMBERS, [])
while True:
lines = infile.readlines(bufsize)
if not lines:
break
for line in lines:
if len(line.split()) == 1:
if len(waterLevelLines) > 0:
waterLevels = get_water_level_of_channels(
waterLevelLines, FLOOD_ELEMENT_NUMBERS)
# Get Time stamp Ref:http://stackoverflow.com/a/13685221/1461060
# print(waterLevelLines[0].split())
ModelTime = float(
waterLevelLines[0].split()[0])
baseTime = datetime.strptime(
'%s %s' %
(ts_start_date, ts_start_time),
'%Y-%m-%d %H:%M:%S')
currentStepTime = baseTime + timedelta(
hours=ModelTime)
dateAndTime = currentStepTime.strftime(
"%Y-%m-%d %H:%M:%S")
for elementNo in FLOOD_ELEMENT_NUMBERS:
tmpTS = waterLevelSeriesDict[
elementNo][:]
if elementNo in waterLevels:
tmpTS.append([
dateAndTime,
waterLevels[elementNo]
])
else:
tmpTS.append(
[dateAndTime, MISSING_VALUE])
waterLevelSeriesDict[elementNo] = tmpTS
isWaterLevelLines = False
# for l in waterLevelLines :
# #print(l)
waterLevelLines = []
waterLevelLines.append(line)
for elementNo in FLOOD_ELEMENT_NUMBERS:
opts = {
'forceInsert': forceInsert,
'station': FLOOD_PLAIN_CELL_MAP[elementNo],
'run_name': RUN_NAME,
'source': FLO2D_MODEL
}
if utcOffset != timedelta():
opts['utcOffset'] = utcOffset
# folder_path_ts = os.path.join(dir_path, 'time_series')
# if not os.path.exists(folder_path_ts):
# try:
# os.makedirs(folder_path_ts)
# except OSError as e:
# print(str(e))
# file_path = os.path.join(folder_path_ts, 'time_series_'+elementNo+'.txt')
# with open(file_path, 'w') as f:
# for item in waterLevelSeriesDict[elementNo]:
# f.write("%s\n" % item)
save_forecast_timeseries(
adapter, waterLevelSeriesDict[elementNo], run_date,
run_time, opts)
# print('Extracted Cell No', elementNo, FLOOD_PLAIN_CELL_MAP[elementNo])
except Exception as ex:
print('source data loading exception:', str(ex))
except Exception as e:
print('config reading exception:', str(e))
if 'FLO2D_MODEL' in CONFIG:
FLO2D_MODEL = CONFIG['FLO2D_MODEL']
if 'MYSQL_HOST' in CONFIG:
MYSQL_HOST = CONFIG['MYSQL_HOST']
if 'MYSQL_USER' in CONFIG:
MYSQL_USER = CONFIG['MYSQL_USER']
if 'MYSQL_DB' in CONFIG:
MYSQL_DB = CONFIG['MYSQL_DB']
if 'MYSQL_PASSWORD' in CONFIG:
MYSQL_PASSWORD = CONFIG['MYSQL_PASSWORD']
adapter = MySQLAdapter(host=MYSQL_HOST, user=MYSQL_USER, password=MYSQL_PASSWORD, db=MYSQL_DB)
# TODO: Pass source name as a paramter to script
flo2d_source = adapter.get_source(name=FLO2D_MODEL)
try:
flo2d_source = json.loads(flo2d_source.get('parameters', "{}"))
except Exception as e:
print(e)
traceback.print_exc()
CHANNEL_CELL_MAP = {}
if 'CHANNEL_CELL_MAP' in flo2d_source:
CHANNEL_CELL_MAP = flo2d_source['CHANNEL_CELL_MAP']
FLOOD_PLAIN_CELL_MAP = {}
if 'FLOOD_PLAIN_CELL_MAP' in flo2d_source:
FLOOD_PLAIN_CELL_MAP = flo2d_source['FLOOD_PLAIN_CELL_MAP']
"""
{