def create_inflow(dir_path, run_date, run_time): try: config_path = os.path.join(os.getcwd(), 'inflowdat', 'config.json') config = json.loads(open(config_path).read()) start_date_time = datetime.strptime('%s %s' % (run_date, run_time), '%Y-%m-%d %H:%M:%S') start = (datetime.strptime(start_date_time.strftime('%Y-%m-%d 00:00:00'), '%Y-%m-%d %H:%M:%S') - timedelta(days=2))\ .strftime('%Y-%m-%d %H:%M:%S') end = (datetime.strptime(start, '%Y-%m-%d %H:%M:%S') + timedelta(days=5)).strftime('%Y-%m-%d %H:%M:%S') target_stations = read_attribute_from_config_file('station_names', config, True) output_dir = dir_path file_name = 'INFLOW.DAT' pool = get_Pool(host=FCST_HOST, port=FCST_PORT, user=FCST_USER, password=FCST_PASSWORD, db=FCST_DB) hechms_stations = get_hechms_stations(pool=pool) target_station_ids = [] for i in range(len(target_stations)): target_station_ids.append(hechms_stations.get(target_stations[i])[0]) print("target stations: ", target_station_ids) obs_wl = get_obs_waterlevel(station_id=OBS_WL_ID, start=start) if obs_wl is None: obs_wl = 0.5 fcst_discharges = extract_fcst_discharge_ts(pool=pool, start=start, end=end, station_ids=target_station_ids) for id in target_station_ids: # file = '{}_{}'.format(id, file_name) file_path = os.path.join(output_dir, file_name) prepare_inflow(inflow_file_path=file_path, fcst_discharge_ts=fcst_discharges.get(id), obs_wl=obs_wl) except Exception as e: traceback.print_exc() finally: destroy_Pool(pool)
def extract_distrubuted_hechms_outputs(output_file_name, output_dir, run_date, run_time): """ Config.json { "output_file_name": "DailyDischarge.csv", "output_dir": "", "run_date": "2019-05-24", "run_time": "00:00:00", "utc_offset": "", "sim_tag": "hourly_run", "model": "HECHMS", "version": "single", "unit": "m3/s", "unit_type": "Instantaneous", "variable": "Discharge", "station_name": "Hanwella" } """ try: config = json.loads(open('config.json').read()) # output related details output_file_name = output_file_name output_dir = output_dir run_date = run_date run_time = run_time utc_offset = read_attribute_from_config_file('utc_offset', config, False) if utc_offset is None: utc_offset = '' # sim tag sim_tag = read_attribute_from_config_file('sim_tag', config, True) # source details model = read_attribute_from_config_file('model', config, True) version = read_attribute_from_config_file('version', config, True) # unit details unit = read_attribute_from_config_file('unit', config, True) unit_type = UnitType.getType( read_attribute_from_config_file('unit_type', config, True)) # variable details variable = read_attribute_from_config_file('variable', config, True) # station details station_name = read_attribute_from_config_file('station_name', config, True) out_file_path = os.path.join(output_dir, output_file_name) if not os.path.exists(out_file_path): msg = 'no file :: {}'.format(out_file_path) logger.warning(msg) print(msg) exit(1) fgt = get_file_last_modified_time(out_file_path) print("fgt, ", fgt) timeseries = read_csv(out_file_path) pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD) hechms_stations = get_hechms_stations(pool=pool) station_id = hechms_stations.get(station_name)[0] lat = str(hechms_stations.get(station_name)[1]) lon = str(hechms_stations.get(station_name)[2]) source_id = get_source_id(pool=pool, model=model, version=version) variable_id = get_variable_id(pool=pool, variable=variable) unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type) tms_meta = { 'sim_tag': sim_tag, 'model': model, 'version': version, 'variable': variable, 'unit': unit, 'unit_type': unit_type.value, 'latitude': lat, 'longitude': lon, 'station_id': station_id, 'source_id': source_id, 'variable_id': variable_id, 'unit_id': unit_id } utcOffset = getUTCOffset(utc_offset, default=True) if utcOffset != timedelta(): tms_meta['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries, run_date=run_date, run_time=run_time, tms_meta=tms_meta, fgt=fgt) except Exception as e: logger.error('JSON config data loading error.') print('JSON config data loading error.') traceback.print_exc() finally: logger.info("Process finished.") print("Process finished.")
logger.warning(msg) print(msg) exit(1) fgt = get_file_last_modified_time(out_file_path) print("fgt, ", fgt) timeseries = read_csv(out_file_path) pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD) hechms_stations = get_hechms_stations(pool=pool) station_id = hechms_stations.get(station_name)[0] lat = str(hechms_stations.get(station_name)[1]) lon = str(hechms_stations.get(station_name)[2]) source_id = get_source_id(pool=pool, model=model, version=version) variable_id = get_variable_id(pool=pool, variable=variable) unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type) tms_meta = { 'sim_tag': sim_tag, 'model': model, 'version': version,
def extract_distrubuted_hechms_outputs(target_model, db_user, db_pwd, db_host, db_name, out_file_path, run_date, run_time): """ Config.json { "output_file_name": "DailyDischarge.csv", "output_dir": "", "run_date": "2019-05-24", "run_time": "00:00:00", "utc_offset": "", "sim_tag": "hourly_run", "model": "HECHMS", "version": "single", "unit": "m3/s", "unit_type": "Instantaneous", "variable": "Discharge", "station_name": "Hanwella" } """ try: config = json.loads( open('/home/curw/git/distributed_hechms/uploads/config.json').read( )) # output related details run_date = run_date run_time = run_time utc_offset = read_attribute_from_config_file('utc_offset', config, False) if utc_offset is None: utc_offset = '' # sim tag sim_tag = read_attribute_from_config_file('sim_tag', config, True) print("extract_distrubuted_hechms_outputs|sim_tag : ", sim_tag) # source details model = read_attribute_from_config_file('model', config, True) print("extract_distrubuted_hechms_outputs|model : ", model) version_config = read_attribute_from_config_file( 'version_config', config, True) print("extract_distrubuted_hechms_outputs|version_config : ", version_config) version = version_config[target_model] print("extract_distrubuted_hechms_outputs|version : ", version) # unit details unit = read_attribute_from_config_file('unit', config, True) print("extract_distrubuted_hechms_outputs|unit : ", unit) unit_type = UnitType.getType( read_attribute_from_config_file('unit_type', config, True)) print("extract_distrubuted_hechms_outputs|unit_type : ", unit_type) # variable details variable = read_attribute_from_config_file('variable', config, True) print("extract_distrubuted_hechms_outputs|variable : ", variable) # station details station_name = read_attribute_from_config_file('station_name', config, True) print("extract_distrubuted_hechms_outputs|station_name : ", station_name) if not os.path.exists(out_file_path): msg = 'no file :: {}'.format(out_file_path) logger.warning(msg) print(msg) exit(1) fgt = get_file_last_modified_time(out_file_path) print("extract_distrubuted_hechms_outputs|fgt : ", fgt) timeseries = read_csv(out_file_path) pool = get_Pool(host=db_host, port=3306, db=db_name, user=db_user, password=db_pwd) hechms_stations = get_hechms_stations(pool=pool) print("extract_distrubuted_hechms_outputs|hechms_stations : ", hechms_stations) station_id = hechms_stations.get(station_name)[0] lat = str(hechms_stations.get(station_name)[1]) lon = str(hechms_stations.get(station_name)[2]) print("extract_distrubuted_hechms_outputs|[station_id, lat, lon] : ", [station_id, lat, lon]) source_id = get_source_id(pool=pool, model=model, version=version) print("extract_distrubuted_hechms_outputs|source_id : ", source_id) variable_id = get_variable_id(pool=pool, variable=variable) print("extract_distrubuted_hechms_outputs|variable_id : ", variable_id) unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type) print("extract_distrubuted_hechms_outputs|unit_id : ", unit_id) tms_meta = { 'sim_tag': sim_tag, 'model': model, 'version': version, 'variable': variable, 'unit': unit, 'unit_type': unit_type.value, 'latitude': lat, 'longitude': lon, 'station_id': station_id, 'source_id': source_id, 'variable_id': variable_id, 'unit_id': unit_id } print("extract_distrubuted_hechms_outputs|tms_meta : ", tms_meta) utcOffset = getUTCOffset(utc_offset, default=True) if utcOffset != timedelta(): tms_meta['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries, run_date=run_date, run_time=run_time, tms_meta=tms_meta, fgt=fgt) return {'Result': 'Success'} except Exception as e: logger.error('JSON config data loading error.') print('JSON config data loading error.') traceback.print_exc() return {'Result': 'Fail'}