def run_hechms_workflow(run_datetime=datetime.now().
strftime('%Y-%m-%d_%H:%M:%S'),
back_days=2,
forward_days=3,
initial_wl=0,
pop_method='MME'):
print('prepare_input_files.')
print('run_datetime : ', run_datetime)
print('back_days : ', back_days)
print('forward_days : ', forward_days)
print('initial_wl : ', initial_wl)
print('pop_method : ', pop_method)
exec_datetime = datetime.strptime(run_datetime, '%Y-%m-%d_%H:%M:%S')
exec_datetime = exec_datetime.strftime('%Y-%m-%d %H:%M:%S')
file_date = (datetime.strptime(run_datetime,
'%Y-%m-%d_%H:%M:%S')).strftime('%Y-%m-%d')
print('file_date : ', file_date)
file_time = (datetime.strptime(run_datetime,
'%Y-%m-%d_%H:%M:%S')).strftime('%H:%M:%S')
print('file_time : ', file_time)
run_datetime = datetime.strptime(run_datetime, '%Y-%m-%d_%H:%M:%S')
run_datetime = datetime.strptime(
run_datetime.strftime('%Y-%m-%d 00:00:00'), '%Y-%m-%d %H:%M:%S')
print('run_datetime : ', run_datetime)
to_date = run_datetime + timedelta(days=forward_days)
from_date = run_datetime - timedelta(days=back_days)
from_date = from_date.strftime('%Y-%m-%d %H:%M:%S')
to_date = to_date.strftime('%Y-%m-%d %H:%M:%S')
print('{from_date, to_date} : ', {from_date, to_date})
output_dir = os.path.join(OUTPUT_DIR, file_date, file_time)
print('output_dir : ', output_dir)
output_file = os.path.join(output_dir, 'DailyRain.csv')
try:
create_dir_if_not_exists(output_dir)
get_mean_rain(from_date, to_date, output_dir, 'hechms', pop_method)
rain_fall_file = Path(output_file)
if rain_fall_file.is_file():
create_dir_if_not_exists(
os.path.join(OUTPUT_DIR, 'distributed_model'))
subprocess.call(COPY_MODEL_TEMPLATE_CMD, shell=True)
create_gage_file_by_rain_file('distributed_model', output_file)
create_control_file_by_rain_file('distributed_model', output_file)
create_run_file('distributed_model', initial_wl,
run_datetime.strftime('%Y-%m-%d %H:%M:%S'),
from_date)
hechms_input = os.path.join(
HEC_HMS_MODEL_DIR,
HEC_INPUT_DSS.replace('{MODEL_NAME}', 'distributed_model'))
hechms_output = os.path.join(
HEC_HMS_MODEL_DIR,
HEC_OUTPUT_DSS.replace('{MODEL_NAME}', 'distributed_model'))
try:
print('hechms_input : ', hechms_input)
subprocess.call(FILE_REMOVE_CMD.replace(
'{FILE_NAME}', hechms_input),
shell=True)
print('hechms_output : ', hechms_output)
subprocess.call(FILE_REMOVE_CMD.replace(
'{FILE_NAME}', hechms_output),
shell=True)
ts_start_date = from_date.strftime('%Y-%m-%d')
ts_start_time = '00:00:00'
print('[ts_start_date, ts_start_time] : ',
[ts_start_date, ts_start_time])
ret_code = execute_pre_dssvue(exec_datetime, ts_start_date,
ts_start_time)
print('execute_pre_dssvue|ret_code : ', ret_code)
if ret_code == 0:
ret_code = execute_hechms('distributed_model',
HEC_HMS_MODEL_DIR)
print('execute_hechms|ret_code : ', ret_code)
if ret_code == 0:
ret_code = execute_post_dssvue(exec_datetime,
ts_start_date,
ts_start_time)
print('execute_post_dssvue|ret_code : ', ret_code)
if ret_code == 0:
output_dir = os.path.join(OUTPUT_DIR, file_date,
file_time)
print('output_dir : ', output_dir)
output_file = os.path.join(output_dir,
'DailyDischarge.csv')
try:
print(
'extract_distrubuted_hechms_outputs|[output_file, file_date] : ',
[output_file, file_date])
response = extract_distrubuted_hechms_outputs(
output_file, file_date, '00:00:00')
return jsonify(response)
except Exception as e:
return jsonify({'Result': 'Fail'})
else:
return jsonify({'Result': 'Fail'})
else:
return jsonify({'Result': 'Fail'})
else:
print('pre-processing has failed')
return jsonify({'Result': 'Fail'})
except Exception as e:
print('Remove hechms input/output files|Exception: ', e)
logging.debug(
"Remove hechms input/output files|Exception|{}".format(e))
return jsonify({'Result': 'Fail'})
else:
print('input mean rain file creation has failed')
return jsonify({'Result': 'Fail'})
except Exception as e:
print('prepare_input_files|Exception: ', e)
logging.debug("prepare_input_files|Exception|{}".format(e))
return jsonify({'Result': 'Fail'})
def get_run_file(run_datetime=datetime.now().strftime('%Y-%m-%d %H:%M:%S')):
print('create_run_file.')
run_datetime = datetime.strptime(run_datetime, '%Y-%m-%d %H:%M:%S')
create_run_file('distributed_model',
run_datetime.strftime('%Y-%m-%d %H:%M:%S'))
return jsonify({'timeseries': {}})
def prepare_input_files(
run_datetime=datetime.now().strftime('%Y-%m-%d_%H:%M:%S'),
back_days=2,
forward_days=3,
initial_wl=0):
print('prepare_input_files.')
print('run_datetime : ', run_datetime)
print('back_days : ', back_days)
print('forward_days : ', forward_days)
print('initial_wl : ', initial_wl)
file_date = (datetime.strptime(run_datetime,
'%Y-%m-%d_%H:%M:%S')).strftime('%Y-%m-%d')
print('file_date : ', file_date)
file_time = (datetime.strptime(run_datetime,
'%Y-%m-%d_%H:%M:%S')).strftime('%H:%M:%S')
print('file_time : ', file_time)
run_datetime = datetime.strptime(run_datetime, '%Y-%m-%d_%H:%M:%S')
run_datetime = datetime.strptime(
run_datetime.strftime('%Y-%m-%d 00:00:00'), '%Y-%m-%d %H:%M:%S')
print('run_datetime : ', run_datetime)
to_date = run_datetime + timedelta(days=forward_days)
from_date = run_datetime - timedelta(days=back_days)
from_date = from_date.strftime('%Y-%m-%d %H:%M:%S')
to_date = to_date.strftime('%Y-%m-%d %H:%M:%S')
print('{from_date, to_date} : ', {from_date, to_date})
output_dir = os.path.join(OUTPUT_DIR, file_date, file_time)
print('output_dir : ', output_dir)
output_file = os.path.join(output_dir, 'DailyRain.csv')
try:
create_dir_if_not_exists(output_dir)
get_mean_rain(from_date, to_date, output_dir)
rain_fall_file = Path(output_file)
if rain_fall_file.is_file():
subprocess.call(COPY_MODEL_TEMPLATE_CMD, shell=True)
create_gage_file_by_rain_file('distributed_model', output_file)
create_control_file_by_rain_file('distributed_model', output_file)
create_run_file('distributed_model', initial_wl,
run_datetime.strftime('%Y-%m-%d %H:%M:%S'),
from_date)
hechms_input = os.path.join(
HEC_HMS_MODEL_DIR,
HEC_INPUT_DSS.replace('{MODEL_NAME}', 'distributed_model'))
hechms_output = os.path.join(
HEC_HMS_MODEL_DIR,
HEC_OUTPUT_DSS.replace('{MODEL_NAME}', 'distributed_model'))
try:
print('hechms_input : ', hechms_input)
subprocess.call(FILE_REMOVE_CMD.replace(
'{FILE_NAME}', hechms_input),
shell=True)
print('hechms_output : ', hechms_output)
subprocess.call(FILE_REMOVE_CMD.replace(
'{FILE_NAME}', hechms_output),
shell=True)
# subprocess.call(COPY_BASIN_CMD, shell=True)
# subprocess.call(COPY_MET_CMD, shell=True)
except Exception as e:
print('Remove hechms input/output files|Exception: ', e)
logging.debug(
"Remove hechms input/output files|Exception|{}".format(e))
return jsonify({'Result': 'Success'})
else:
return jsonify({'Result': 'Fail'})
except Exception as e:
print('prepare_input_files|Exception: ', e)
logging.debug("prepare_input_files|Exception|{}".format(e))
return jsonify({'Result': 'Fail'})
def prepare_input_files(run_datetime=datetime.now().
strftime('%Y-%m-%d_%H:%M:%S'),
back_days=2,
forward_days=3,
initial_wl=0,
pop_method='MME'):
print('prepare_input_files.')
print('run_datetime : ', run_datetime)
print('back_days : ', back_days)
print('forward_days : ', forward_days)
print('initial_wl : ', initial_wl)
print('pop_method : ', pop_method)
exec_datetime = datetime.strptime(run_datetime, '%Y-%m-%d_%H:%M:%S')
exec_datetime = exec_datetime.strftime('%Y-%m-%d %H:%M:%S')
file_date = (datetime.strptime(run_datetime,
'%Y-%m-%d_%H:%M:%S')).strftime('%Y-%m-%d')
print('file_date : ', file_date)
file_time = (datetime.strptime(run_datetime,
'%Y-%m-%d_%H:%M:%S')).strftime('%H:%M:%S')
print('file_time : ', file_time)
run_datetime = datetime.strptime(run_datetime, '%Y-%m-%d_%H:%M:%S')
run_datetime = datetime.strptime(
run_datetime.strftime('%Y-%m-%d 00:00:00'), '%Y-%m-%d %H:%M:%S')
print('run_datetime : ', run_datetime)
to_date = run_datetime + timedelta(days=forward_days)
from_date = run_datetime - timedelta(days=back_days)
from_date = from_date.strftime('%Y-%m-%d %H:%M:%S')
to_date = to_date.strftime('%Y-%m-%d %H:%M:%S')
print('{from_date, to_date} : ', {from_date, to_date})
output_dir = os.path.join(OUTPUT_DIR, file_date, file_time)
print('output_dir : ', output_dir)
output_file = os.path.join(output_dir, 'DailyRain.csv')
try:
create_dir_if_not_exists(output_dir)
if pop_method.isupper():
print('Capital tag has used|pop_method : ', pop_method)
get_mean_rain(from_date, to_date, output_dir, 'hechms', pop_method)
else:
print('Event tag has used|pop_method : ', pop_method)
sim_tag = pop_method[:len(pop_method) - 3]
wrf_model = int(pop_method[len(pop_method) - 2:])
print('prepare_input_files|[sim_tag, wrf_model, exec_datetime] : ',
[sim_tag, wrf_model, exec_datetime])
get_event_mean_rain(exec_datetime, forward_days, back_days,
output_dir, wrf_model, sim_tag)
rain_fall_file = Path(output_file)
if rain_fall_file.is_file():
create_dir_if_not_exists(
os.path.join(OUTPUT_DIR, 'distributed_model'))
subprocess.call(COPY_MODEL_TEMPLATE_CMD, shell=True)
create_gage_file_by_rain_file('distributed_model', output_file)
create_control_file_by_rain_file('distributed_model', output_file)
create_run_file('distributed_model', initial_wl,
run_datetime.strftime('%Y-%m-%d %H:%M:%S'),
from_date)
hechms_input = os.path.join(
HEC_HMS_MODEL_DIR,
HEC_INPUT_DSS.replace('{MODEL_NAME}', 'distributed_model'))
hechms_output = os.path.join(
HEC_HMS_MODEL_DIR,
HEC_OUTPUT_DSS.replace('{MODEL_NAME}', 'distributed_model'))
try:
print('hechms_input : ', hechms_input)
subprocess.call(FILE_REMOVE_CMD.replace(
'{FILE_NAME}', hechms_input),
shell=True)
print('hechms_output : ', hechms_output)
subprocess.call(FILE_REMOVE_CMD.replace(
'{FILE_NAME}', hechms_output),
shell=True)
except Exception as e:
print('Remove hechms input/output files|Exception: ', e)
logging.debug(
"Remove hechms input/output files|Exception|{}".format(e))
return jsonify({'Result': 'Success'})
else:
return jsonify({'Result': 'Fail'})
except Exception as e:
print('prepare_input_files|Exception: ', e)
logging.debug("prepare_input_files|Exception|{}".format(e))
return jsonify({'Result': 'Fail'})
def run_hechms_workflow(
db_user,
db_pwd,
db_host,
db_name,
run_datetime=datetime.now().strftime('%Y-%m-%d_%H:%M:%S'),
back_days=2,
forward_days=3,
initial_wl=0,
pop_method='MME',
target_model='HDC'):
print('run_datetime : ', run_datetime)
print('back_days : ', back_days)
print('forward_days : ', forward_days)
print('initial_wl : ', initial_wl)
print('pop_method : ', pop_method)
exec_datetime = datetime.strptime(run_datetime, '%Y-%m-%d_%H:%M:%S')
# exec_datetime = exec_datetime.strftime('%Y-%m-%d %H:%M:%S')
file_date = (datetime.strptime(run_datetime,
'%Y-%m-%d_%H:%M:%S')).strftime('%Y-%m-%d')
print('file_date : ', file_date)
file_time = (datetime.strptime(run_datetime,
'%Y-%m-%d_%H:%M:%S')).strftime('%H:%M:%S')
print('file_time : ', file_time)
run_datetime = datetime.strptime(run_datetime, '%Y-%m-%d_%H:%M:%S')
run_datetime = datetime.strptime(
run_datetime.strftime('%Y-%m-%d 00:00:00'), '%Y-%m-%d %H:%M:%S')
print('run_datetime : ', run_datetime)
to_date = run_datetime + timedelta(days=forward_days)
from_date = run_datetime - timedelta(days=back_days)
from_date = from_date.strftime('%Y-%m-%d %H:%M:%S')
to_date = to_date.strftime('%Y-%m-%d %H:%M:%S')
print('{from_date, to_date} : ', {from_date, to_date})
# output_dir = os.path.join(OUTPUT_DIR, file_date, file_time)
output_dir = OUTPUT_DIR
print('output_dir : ', output_dir)
output_file = os.path.join(output_dir, 'DailyRain.csv')
try:
create_dir_if_not_exists(output_dir)
print('run_hechms_workflow|get_basin_rain|start')
get_basin_rain(from_date, to_date, output_dir, 'hechms', pop_method,
ALLOWED_RAIN_ERROR,
exec_datetime.strftime('%Y-%m-%d %H:00:00'), db_user,
db_pwd, db_host, db_name, target_model)
print('run_hechms_workflow|get_basin_rain|end')
rain_fall_file = Path(output_file)
if rain_fall_file.is_file():
create_dir_if_not_exists(os.path.join(OUTPUT_DIR, 'hechms_model'))
subprocess.call(COPY_TEMPLATE_CMD.format(target_model), shell=True)
create_gage_file_by_rain_file('hechms_model', output_file)
create_control_file_by_rain_file('hechms_model', output_file)
create_run_file('hechms_model', initial_wl,
run_datetime.strftime('%Y-%m-%d %H:%M:%S'),
from_date)
state_file = get_state_file_name(from_date)
hechms_input = os.path.join(
HEC_HMS_MODEL_DIR,
HEC_INPUT_DSS.replace('{MODEL_NAME}', 'hechms_model'))
hechms_output = os.path.join(
HEC_HMS_MODEL_DIR,
HEC_OUTPUT_DSS.replace('{MODEL_NAME}', 'hechms_model'))
try:
print('hechms_input : ', hechms_input)
subprocess.call(FILE_REMOVE_CMD.replace(
'{FILE_NAME}', hechms_input),
shell=True)
print('hechms_output : ', hechms_output)
subprocess.call(FILE_REMOVE_CMD.replace(
'{FILE_NAME}', hechms_output),
shell=True)
ts_start_date = (datetime.strptime(
from_date, '%Y-%m-%d %H:%M:%S')).strftime('%Y-%m-%d')
ts_start_time = '00:00:00'
print('[ts_start_date, ts_start_time] : ',
[ts_start_date, ts_start_time])
sub_catchment_shape_file = os.path.join(
RESOURCE_PATH, 'sub_catchments/sub_subcatchments.shp')
update_basin_init_values(
'{} {}'.format(ts_start_date, ts_start_time), db_user,
db_pwd, db_host, sub_catchment_shape_file, target_model)
ret_code = execute_pre_dssvue(exec_datetime, ts_start_date,
ts_start_time)
print('execute_pre_dssvue|ret_code : ', ret_code)
if ret_code == 0:
ret_code = execute_hechms('hechms_model',
HEC_HMS_MODEL_DIR)
print('execute_hechms|ret_code : ', ret_code)
if ret_code == 0:
ret_code = execute_post_dssvue(exec_datetime,
ts_start_date,
ts_start_time,
target_model)
print('execute_post_dssvue|ret_code : ', ret_code)
if ret_code == 0:
output_file = os.path.join(OUTPUT_DIR,
'DailyDischarge.csv')
print('output_file : ', output_file)
# state_file_copy_cmd = FILE_COPY_CMD_TEMPLATE.format(state_file, STATE_BACKUP_DIR)
# print('state_file_copy_cmd : ', state_file_copy_cmd)
# subprocess.call(state_file_copy_cmd, shell=True)
try:
print(
'extract_distrubuted_hechms_outputs|[output_file, file_date] : ',
[output_file, file_date])
print(
'extract_distrubuted_hechms_outputs|cloud|started'
)
extract_distrubuted_hechms_outputs(
target_model, db_user, db_pwd, db_host,
'curw_fcst', output_file, file_date,
'00:00:00')
print(
'extract_distrubuted_hechms_outputs|cloud|completed'
)
upload_rain_file = os.path.join(
OUTPUT_DIR, 'DailyRain.csv')
upload_discharge_file = os.path.join(
OUTPUT_DIR, 'DailyDischarge.csv')
bucket_rain_file = 'hechms/{}/{}/{}/DailyRain.csv'.format(
target_model, file_date, file_time)
bucket_discharge_file = 'hechms/{}/{}/{}/DailyDischarge.csv'.format(
target_model, file_date, file_time)
print(
'extract_distrubuted_hechms_outputs|upload_rain_file : ',
upload_rain_file)
print(
'extract_distrubuted_hechms_outputs|bucket_rain_file : ',
bucket_rain_file)
print(
'extract_distrubuted_hechms_outputs|upload_discharge_file : ',
upload_discharge_file)
print(
'extract_distrubuted_hechms_outputs|bucket_discharge_file : ',
bucket_discharge_file)
print(
'extract_distrubuted_hechms_outputs|file upload started'
)
try:
upload_file_to_bucket(
GOOGLE_BUCKET_KEY_PATH, BUCKET_NAME,
upload_rain_file, bucket_rain_file)
upload_file_to_bucket(
GOOGLE_BUCKET_KEY_PATH, BUCKET_NAME,
upload_discharge_file,
bucket_discharge_file)
print(
'extract_distrubuted_hechms_outputs|file upload completed'
)
except Exception as e:
print(
'extract_distrubuted_hechms_outputs|Exception : ',
str(e))
return True
except Exception as e:
return False
else:
return False
else:
return False
else:
print('pre-processing has failed')
return False
except Exception as e:
print('Remove hechms input/output files|Exception: ', e)
return False
else:
print('input mean rain file creation has failed')
return False
except Exception as e:
print('prepare_input_files|Exception: ', e)
return False