def extract_wrf_data(wrf_model, config_data, tms_meta): logger.info( "######################################## {} #######################################" .format(wrf_model)) for date in config_data['dates']: run_date_str = date daily_dir = 'STATIONS_{}'.format(run_date_str) output_dir = os.path.join(config_data['wrf_dir'], daily_dir) rainnc_net_cdf_file = 'd03_RAINNC_{}_{}.nc'.format( run_date_str, wrf_model) rainnc_net_cdf_file_path = os.path.join(output_dir, rainnc_net_cdf_file) source_name = "{}_{}".format(config_data['model'], wrf_model) source_id = get_source_id(pool=pool, model=source_name, version=config_data['version']) tms_meta['model'] = source_name tms_meta['source_id'] = source_id try: read_netcdf_file(pool=pool, rainnc_net_cdf_file_path=rainnc_net_cdf_file_path, tms_meta=tms_meta) except Exception as e: logger.error("WRF_{} netcdf file reading error.".format(wrf_model)) traceback.print_exc()
def process_fcst_ts_from_hechms_outputs(curw_fcst_pool, extract_stations, i, start, end, sim_tag=None): FCST_TS = Fcst_Timeseries(curw_fcst_pool) try: # [station_name,latitude,longitude,target,model,version,sim_tag,station] source_model = extract_stations[i][4] version = extract_stations[i][5] station_id = extract_stations[i][7] if sim_tag is None: sim_tag = extract_stations[i][6] variable_id = 3 # Discharge unit_id = 3 # m3/s | Instantaneous source_id = get_source_id(pool=curw_fcst_pool, model=source_model, version=version) fcst_series = FCST_TS.get_latest_timeseries(sim_tag, station_id, source_id, variable_id, unit_id, start=None) if (fcst_series is None) or (len(fcst_series) < 1): return None fcst_series.insert(0, ['time', 'value']) fcst_df = list_of_lists_to_df_first_row_as_columns(fcst_series) if start is None: start = (fcst_df['time'].min()).strftime(DATE_TIME_FORMAT) if end is None: end = (fcst_df['time'].max()).strftime(DATE_TIME_FORMAT) df = (pd.date_range(start=start, end=end, freq='60min')).to_frame(name='time') processed_df = pd.merge(df, fcst_df, on='time', how='left') processed_df.interpolate(method='linear', limit_direction='both', limit=100) processed_df.fillna(inplace=True, value=0) processed_df['time'] = processed_df['time'].dt.strftime( DATE_TIME_FORMAT) return processed_df.values.tolist() except Exception as e: traceback.print_exc()
def extract_wrf_data(wrf_system, config_data, tms_meta): logger.info( "######################################## {} #######################################" .format(wrf_system)) for date in config_data['dates']: # /wrf_nfs/wrf/4.0/18/A/2019-07-30/d03_RAINNC.nc output_dir = os.path.join(config_data['wrf_dir'], config_data['version'], config_data['gfs_data_hour'], wrf_system, date) rainnc_net_cdf_file = 'd03_RAINNC.nc' rainnc_net_cdf_file_path = os.path.join(output_dir, rainnc_net_cdf_file) try: source_name = "{}_{}".format(config_data['model'], wrf_system) source_id = get_source_id(pool=pool, model=source_name, version=tms_meta['version']) if source_id is None: add_source(pool=pool, model=source_name, version=tms_meta['version']) source_id = get_source_id(pool=pool, model=source_name, version=tms_meta['version']) except Exception: msg = "Exception occurred while loading source meta data for WRF_{} from database.".format( wrf_system) logger.error(msg) email_content[datetime.now().strftime( COMMON_DATE_TIME_FORMAT)] = msg return False tms_meta['model'] = source_name tms_meta['source_id'] = source_id return read_netcdf_file( pool=pool, rainnc_net_cdf_file_path=rainnc_net_cdf_file_path, tms_meta=tms_meta)
def process_fcst_ts_from_flo2d_outputs(curw_fcst_pool, fcst_start): global latest_fgt FCST_TS = Fcst_Timeseries(curw_fcst_pool) try: # [station_name,latitude,longitude,target,model,version,sim_tag,station] source_model = extract_stations[i][4] version = extract_stations[i][5] sim_tag = extract_stations[i][6] station_id = extract_stations[i][7] variable_id = 3 # Discharge unit_id = 3 # m3/s | Instantaneous source_id = get_source_id(pool=curw_fcst_pool, model=source_model, version=version) fcst_series = FCST_TS.get_latest_timeseries(sim_tag, station_id, source_id, variable_id, unit_id, start=None) if (fcst_series is None) or (len(fcst_series) < 1): return None latest_fgt = (FCST_TS.get_end_date( sim_tag, station_id, source_id, variable_id, unit_id)).strftime(COMMON_DATE_TIME_FORMAT) fcst_series.insert(0, ['time', 'value']) fcst_df = list_of_lists_to_df_first_row_as_columns(fcst_series) fcst_end = (fcst_df['time'].max()).strftime(COMMON_DATE_TIME_FORMAT) if fcst_start is None: fcst_start = ( fcst_df['time'].min()).strftime(COMMON_DATE_TIME_FORMAT) df = (pd.date_range(start=fcst_start, end=fcst_end, freq='15min')).to_frame(name='time') processed_df = pd.merge(df, fcst_df, on='time', how='left') # processed_df.interpolate(method='linear', limit_direction='both') processed_df = processed_df.dropna() processed_df['time'] = processed_df['time'].dt.strftime( COMMON_DATE_TIME_FORMAT) return processed_df.values.tolist() except Exception as e: traceback.print_exc() logger.error("Exception occurred")
def calculate_MME_series(TS, start, end, variables, station_id, variable_id, unit_id): index = pd.date_range(start=start, end=end, freq='15min') df = pd.DataFrame(index=index) for variable in variables: model = variable[0] version = variable[1] sim_tag = variable[2] coefficient = variable[3] try: source_id = get_source_id(pool=pool, model=model, version=version) except Exception: try: time.sleep(3) source_id = get_source_id(pool=pool, model=model, version=version) except Exception: msg = "Exception occurred while loading source id from database." logger.error(msg) exit(1) fcst_ts = TS.get_latest_timeseries(sim_tag=sim_tag, source_id=source_id, station_id=station_id, variable_id=variable_id, unit_id=unit_id) timeseries = list_of_lists_to_df_first_column_as_index(fcst_ts) timeseries[[0]] = timeseries[[0]].astype('float64') * coefficient df = df.join(timeseries, lsuffix='_left', rsuffix='_right') df.fillna(0) df['sum'] = df.sum(axis=1) timeseries_df = df['sum'] return timeseries_df.reset_index().values.tolist()
timdep_file_path = os.path.join(output_dir, TIMDEP_FILE) pool = get_Pool(host=con_params.CURW_FCST_HOST, port=con_params.CURW_FCST_PORT, db=con_params.CURW_FCST_DATABASE, user=con_params.CURW_FCST_USERNAME, password=con_params.CURW_FCST_PASSWORD) flo2d_model_name = '{}_{}'.format(model, version) flo2d_source = json.loads( get_source_parameters(pool=pool, model=model, version=version)) flo2d_stations = get_flo2d_output_stations( pool=pool, flo2d_model=StationEnum.getType(flo2d_model_name)) source_id = get_source_id(pool=pool, model=model, version=version) variable_id = get_variable_id(pool=pool, variable=variable) unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type) tms_meta = { 'sim_tag': sim_tag, 'model': model, 'version': version, 'variable': variable, 'unit': unit, 'unit_type': unit_type.value, 'source_id': source_id, 'variable_id': variable_id, 'unit_id': unit_id
def extract_distrubuted_hechms_outputs(output_file_name, output_dir, run_date, run_time): """ Config.json { "output_file_name": "DailyDischarge.csv", "output_dir": "", "run_date": "2019-05-24", "run_time": "00:00:00", "utc_offset": "", "sim_tag": "hourly_run", "model": "HECHMS", "version": "single", "unit": "m3/s", "unit_type": "Instantaneous", "variable": "Discharge", "station_name": "Hanwella" } """ try: config = json.loads(open('config.json').read()) # output related details output_file_name = output_file_name output_dir = output_dir run_date = run_date run_time = run_time utc_offset = read_attribute_from_config_file('utc_offset', config, False) if utc_offset is None: utc_offset = '' # sim tag sim_tag = read_attribute_from_config_file('sim_tag', config, True) # source details model = read_attribute_from_config_file('model', config, True) version = read_attribute_from_config_file('version', config, True) # unit details unit = read_attribute_from_config_file('unit', config, True) unit_type = UnitType.getType( read_attribute_from_config_file('unit_type', config, True)) # variable details variable = read_attribute_from_config_file('variable', config, True) # station details station_name = read_attribute_from_config_file('station_name', config, True) out_file_path = os.path.join(output_dir, output_file_name) if not os.path.exists(out_file_path): msg = 'no file :: {}'.format(out_file_path) logger.warning(msg) print(msg) exit(1) fgt = get_file_last_modified_time(out_file_path) print("fgt, ", fgt) timeseries = read_csv(out_file_path) pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD) hechms_stations = get_hechms_stations(pool=pool) station_id = hechms_stations.get(station_name)[0] lat = str(hechms_stations.get(station_name)[1]) lon = str(hechms_stations.get(station_name)[2]) source_id = get_source_id(pool=pool, model=model, version=version) variable_id = get_variable_id(pool=pool, variable=variable) unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type) tms_meta = { 'sim_tag': sim_tag, 'model': model, 'version': version, 'variable': variable, 'unit': unit, 'unit_type': unit_type.value, 'latitude': lat, 'longitude': lon, 'station_id': station_id, 'source_id': source_id, 'variable_id': variable_id, 'unit_id': unit_id } utcOffset = getUTCOffset(utc_offset, default=True) if utcOffset != timedelta(): tms_meta['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries, run_date=run_date, run_time=run_time, tms_meta=tms_meta, fgt=fgt) except Exception as e: logger.error('JSON config data loading error.') print('JSON config data loading error.') traceback.print_exc() finally: logger.info("Process finished.") print("Process finished.")
rainc_net_cdf_file_path = os.path.join(output_dir, rainc_net_cdf_file) logger.info( "rainc_net_cdf_file_path : {}".format(rainc_net_cdf_file_path)) rainnc_net_cdf_file_path = os.path.join(output_dir, rainnc_net_cdf_file) logger.info("rainnc_net_cdf_file_path : {}".format( rainnc_net_cdf_file_path)) fgt = get_file_last_modified_time(rainnc_net_cdf_file_path) sim_tag = 'evening_18hrs' source_name = "{}_{}".format(model, wrf_model) source_id = get_source_id(pool=pool, model=source_name, version=version) tms_meta = { 'sim_tag': sim_tag, 'model': source_name, 'version': version, 'variable': variable, 'unit': unit, 'unit_type': unit_type.value } try: read_netcdf_file( pool=pool, rainc_net_cdf_file_path=rainc_net_cdf_file_path,
def extract_distrubuted_hechms_outputs(target_model, db_user, db_pwd, db_host, db_name, out_file_path, run_date, run_time): """ Config.json { "output_file_name": "DailyDischarge.csv", "output_dir": "", "run_date": "2019-05-24", "run_time": "00:00:00", "utc_offset": "", "sim_tag": "hourly_run", "model": "HECHMS", "version": "single", "unit": "m3/s", "unit_type": "Instantaneous", "variable": "Discharge", "station_name": "Hanwella" } """ try: config = json.loads( open('/home/curw/git/distributed_hechms/uploads/config.json').read( )) # output related details run_date = run_date run_time = run_time utc_offset = read_attribute_from_config_file('utc_offset', config, False) if utc_offset is None: utc_offset = '' # sim tag sim_tag = read_attribute_from_config_file('sim_tag', config, True) print("extract_distrubuted_hechms_outputs|sim_tag : ", sim_tag) # source details model = read_attribute_from_config_file('model', config, True) print("extract_distrubuted_hechms_outputs|model : ", model) version_config = read_attribute_from_config_file( 'version_config', config, True) print("extract_distrubuted_hechms_outputs|version_config : ", version_config) version = version_config[target_model] print("extract_distrubuted_hechms_outputs|version : ", version) # unit details unit = read_attribute_from_config_file('unit', config, True) print("extract_distrubuted_hechms_outputs|unit : ", unit) unit_type = UnitType.getType( read_attribute_from_config_file('unit_type', config, True)) print("extract_distrubuted_hechms_outputs|unit_type : ", unit_type) # variable details variable = read_attribute_from_config_file('variable', config, True) print("extract_distrubuted_hechms_outputs|variable : ", variable) # station details station_name = read_attribute_from_config_file('station_name', config, True) print("extract_distrubuted_hechms_outputs|station_name : ", station_name) if not os.path.exists(out_file_path): msg = 'no file :: {}'.format(out_file_path) logger.warning(msg) print(msg) exit(1) fgt = get_file_last_modified_time(out_file_path) print("extract_distrubuted_hechms_outputs|fgt : ", fgt) timeseries = read_csv(out_file_path) pool = get_Pool(host=db_host, port=3306, db=db_name, user=db_user, password=db_pwd) hechms_stations = get_hechms_stations(pool=pool) print("extract_distrubuted_hechms_outputs|hechms_stations : ", hechms_stations) station_id = hechms_stations.get(station_name)[0] lat = str(hechms_stations.get(station_name)[1]) lon = str(hechms_stations.get(station_name)[2]) print("extract_distrubuted_hechms_outputs|[station_id, lat, lon] : ", [station_id, lat, lon]) source_id = get_source_id(pool=pool, model=model, version=version) print("extract_distrubuted_hechms_outputs|source_id : ", source_id) variable_id = get_variable_id(pool=pool, variable=variable) print("extract_distrubuted_hechms_outputs|variable_id : ", variable_id) unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type) print("extract_distrubuted_hechms_outputs|unit_id : ", unit_id) tms_meta = { 'sim_tag': sim_tag, 'model': model, 'version': version, 'variable': variable, 'unit': unit, 'unit_type': unit_type.value, 'latitude': lat, 'longitude': lon, 'station_id': station_id, 'source_id': source_id, 'variable_id': variable_id, 'unit_id': unit_id } print("extract_distrubuted_hechms_outputs|tms_meta : ", tms_meta) utcOffset = getUTCOffset(utc_offset, default=True) if utcOffset != timedelta(): tms_meta['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries, run_date=run_date, run_time=run_time, tms_meta=tms_meta, fgt=fgt) return {'Result': 'Success'} except Exception as e: logger.error('JSON config data loading error.') print('JSON config data loading error.') traceback.print_exc() return {'Result': 'Fail'}
def calculate_MME_series(TS, start, end, variables, coefficients, station_id, variable_id, unit_id): index = pd.date_range(start=start, end=end, freq='15min') df = pd.DataFrame(index=index) boundary = pd.DataFrame(index=index) for index, variable in enumerate(variables): model = variable[0] version = variable[1] sim_tag = variable[2] coefficient = float(coefficients[index]) try: source_id = get_source_id(pool=pool, model=model, version=version) except Exception: try: time.sleep(3) source_id = get_source_id(pool=pool, model=model, version=version) except Exception: msg = "Exception occurred while loading source id from database." logger.error(msg) exit(1) fcst_ts = TS.get_latest_timeseries(sim_tag=sim_tag, source_id=source_id, station_id=station_id, variable_id=variable_id, unit_id=unit_id) timeseries = list_of_lists_to_df_first_column_as_index(fcst_ts) # start -- boundary dataframe creation # boundary_temp = timeseries.copy() boundary_temp[[0]] = boundary_temp[[0]].astype('float64') boundary.join(boundary_temp, lsuffix='_left', rsuffix='_right') # end -- boundary dataframe creation # timeseries[[0]] = timeseries[[0]].astype('float64') * coefficient df = df.join(timeseries, lsuffix='_left', rsuffix='_right') df.fillna(0) df['sum'] = df.sum(axis=1) + float( coefficients[index + 1]) # + coefficients[index+1] ==> adding the constant # start - boundary condition check # boundary['min'] = boundary.min(axis=1) boundary['max'] = boundary.max(axis=1) for index, row in boundary.iterrows(): value = df.loc[index, 'sum'] min = boundary.loc[index, 'min'] max = boundary.loc[index, 'max'] if value > max: df.loc[index, 'sum'] = max elif value < min: df.loc[index, 'sum'] = max print(value, max, min, df.loc[index, 'sum']) # end - boundary condition check # timeseries_df = df['sum'] return timeseries_df.reset_index().values.tolist()
def upload_waterlevels(dir_path, ts_start_date, ts_start_time, run_date, run_time): """ Config.json { "HYCHAN_OUT_FILE": "HYCHAN.OUT", "TIMDEP_FILE": "TIMDEP.OUT", "output_dir": "", "run_date": "2019-05-24", "run_time": "", "ts_start_date": "", "ts_start_time": "", "utc_offset": "", "sim_tag": "", "model": "WRF", "version": "v3", "unit": "mm", "unit_type": "Accumulative", "variable": "Precipitation" } """ try: config_path = os.path.join(os.getcwd(), 'extract', 'config_curw_fcst.json') config = json.loads(open(config_path).read()) # flo2D related details HYCHAN_OUT_FILE = read_attribute_from_config_file('HYCHAN_OUT_FILE', config, True) TIMDEP_FILE = read_attribute_from_config_file('TIMDEP_FILE', config, True) output_dir = dir_path run_date = run_date run_time = run_time ts_start_date = ts_start_date ts_start_time = ts_start_time utc_offset = read_attribute_from_config_file('utc_offset', config, False) if utc_offset is None: utc_offset = '' # sim tag sim_tag = read_attribute_from_config_file('sim_tag', config, True) # source details model = read_attribute_from_config_file('model', config, True) version = read_attribute_from_config_file('version', config, True) # unit details unit = read_attribute_from_config_file('unit', config, True) unit_type = UnitType.getType(read_attribute_from_config_file('unit_type', config, True)) # variable details variable = read_attribute_from_config_file('variable', config, True) hychan_out_file_path = os.path.join(output_dir, HYCHAN_OUT_FILE) timdep_file_path = os.path.join(output_dir, TIMDEP_FILE) pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD) # pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE) flo2d_model_name = '{}_{}'.format(model, version) flo2d_source = json.loads(get_source_parameters(pool=pool, model=model, version=version)) print("############### source ############", flo2d_source) flo2d_stations = get_flo2d_output_stations(pool=pool, flo2d_model=StationEnum.getType(flo2d_model_name)) print("############### 1st occurrence ############", flo2d_stations) source_id = get_source_id(pool=pool, model=model, version=version) variable_id = get_variable_id(pool=pool, variable=variable) unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type) tms_meta = { 'sim_tag': sim_tag, 'model': model, 'version': version, 'variable': variable, 'unit': unit, 'unit_type': unit_type.value, 'source_id': source_id, 'variable_id': variable_id, 'unit_id': unit_id } CHANNEL_CELL_MAP = flo2d_source["CHANNEL_CELL_MAP"] FLOOD_PLAIN_CELL_MAP = flo2d_source["FLOOD_PLAIN_CELL_MAP"] ELEMENT_NUMBERS = CHANNEL_CELL_MAP.keys() FLOOD_ELEMENT_NUMBERS = FLOOD_PLAIN_CELL_MAP.keys() SERIES_LENGTH = 0 MISSING_VALUE = -999 utcOffset = getUTCOffset(utc_offset, default=True) print('Extract Water Level Result of FLO2D on', run_date, '@', run_time, 'with Base time of', ts_start_date, '@', ts_start_time) # Check HYCHAN.OUT file exists if not os.path.exists(hychan_out_file_path): print('Unable to find file : ', hychan_out_file_path) traceback.print_exc() ##################################### # Calculate the size of time series # ##################################### bufsize = 65536 with open(hychan_out_file_path) as infile: isWaterLevelLines = False isCounting = False countSeriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series while True: lines = infile.readlines(bufsize) if not lines or SERIES_LENGTH: break for line in lines: if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5): isWaterLevelLines = True elif isWaterLevelLines: cols = line.split() if len(cols) > 0 and cols[0].replace('.', '', 1).isdigit(): countSeriesSize += 1 isCounting = True elif isWaterLevelLines and isCounting: SERIES_LENGTH = countSeriesSize break print('Series Length is :', SERIES_LENGTH) bufsize = 65536 ################################################################# # Extract Channel Water Level elevations from HYCHAN.OUT file # ################################################################# print('Extract Channel Water Level Result of FLO2D (HYCHAN.OUT) on', run_date, '@', run_time, 'with Base time of', ts_start_date, '@', ts_start_time) with open(hychan_out_file_path) as infile: isWaterLevelLines = False isSeriesComplete = False waterLevelLines = [] seriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series while True: lines = infile.readlines(bufsize) if not lines: break for line in lines: if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5): seriesSize = 0 elementNo = line.split()[5] if elementNo in ELEMENT_NUMBERS: isWaterLevelLines = True waterLevelLines.append(line) else: isWaterLevelLines = False elif isWaterLevelLines: cols = line.split() if len(cols) > 0 and isfloat(cols[0]): seriesSize += 1 waterLevelLines.append(line) if seriesSize == SERIES_LENGTH: isSeriesComplete = True if isSeriesComplete: baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S') timeseries = [] elementNo = waterLevelLines[0].split()[5] # print('Extracted Cell No', elementNo, CHANNEL_CELL_MAP[elementNo]) for ts in waterLevelLines[1:]: v = ts.split() if len(v) < 1: continue # Get flood level (Elevation) value = v[1] # Get flood depth (Depth) # value = v[2] if not isfloat(value): value = MISSING_VALUE continue # If value is not present, skip if value == 'NaN': continue # If value is NaN, skip timeStep = float(v[0]) currentStepTime = baseTime + timedelta(hours=timeStep) dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S") timeseries.append([dateAndTime, value]) # Save Forecast values into Database opts = { 'elementNo': elementNo, 'tms_meta': tms_meta } # print('>>>>>', opts) if utcOffset != timedelta(): opts['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries, run_date=run_date, run_time=run_time, opts=opts, flo2d_stations=flo2d_stations) isWaterLevelLines = False isSeriesComplete = False waterLevelLines = [] # -- END for loop # -- END while loop ################################################################# # Extract Flood Plain water elevations from TIMEDEP.OUT file # ################################################################# if not os.path.exists(timdep_file_path): print('Unable to find file : ', timdep_file_path) traceback.print_exc() print('Extract Flood Plain Water Level Result of FLO2D (TIMEDEP.OUT) on', run_date, '@', run_time, 'with Base time of', ts_start_date, '@', ts_start_time) with open(timdep_file_path) as infile: waterLevelLines = [] waterLevelSeriesDict = dict.fromkeys(FLOOD_ELEMENT_NUMBERS, []) while True: lines = infile.readlines(bufsize) if not lines: break for line in lines: if len(line.split()) == 1: # continue if len(waterLevelLines) > 0: waterLevels = get_water_level_of_channels(waterLevelLines, FLOOD_ELEMENT_NUMBERS) # Get Time stamp Ref:http://stackoverflow.com/a/13685221/1461060 # print('waterLevelLines[0].split() : ', waterLevelLines[0].split()) ModelTime = float(waterLevelLines[0].split()[0]) baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S') currentStepTime = baseTime + timedelta(hours=ModelTime) dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S") for elementNo in FLOOD_ELEMENT_NUMBERS: tmpTS = waterLevelSeriesDict[elementNo][:] if elementNo in waterLevels: tmpTS.append([dateAndTime, waterLevels[elementNo]]) else: tmpTS.append([dateAndTime, MISSING_VALUE]) waterLevelSeriesDict[elementNo] = tmpTS isWaterLevelLines = False # for l in waterLevelLines : # print(l) waterLevelLines = [] waterLevelLines.append(line) # print('len(FLOOD_ELEMENT_NUMBERS) : ', len(FLOOD_ELEMENT_NUMBERS)) for elementNo in FLOOD_ELEMENT_NUMBERS: # Save Forecast values into Database opts = { 'elementNo': elementNo, 'tms_meta': tms_meta } if utcOffset != timedelta(): opts['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db(pool=pool, timeseries=waterLevelSeriesDict[elementNo], run_date=run_date, run_time=run_time, opts=opts, flo2d_stations=flo2d_stations) except Exception as e: logger.error('JSON config data loading error.') print('JSON config data loading error.') traceback.print_exc() finally: logger.info("Process finished.") print("Process finished.")
def update_rainfall_fcsts(flo2d_model, method, grid_interpolation, model_list, timestep): """ Update rainfall forecasts for flo2d models :param flo2d_model: flo2d model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param model_list: list of forecast model and their versions used to calculate the rainfall e.g.: [["WRF_E", "v4"],["WRF_SE", "v4"]] :param timestep: output timeseries timestep :return: """ try: # Connect to the database curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) curw_fcst_pool = get_Pool(host=CURW_FCST_HOST, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE) Sim_TS = Sim_Timeseries(pool=curw_sim_pool) Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool) flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format( flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)] flo2d_wrf_mapping = get_flo2d_cells_to_wrf_grid_mappings( pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model) for flo2d_index in range(len(flo2d_grids)): # len(flo2d_grids) lat = flo2d_grids[flo2d_index][2] lon = flo2d_grids[flo2d_index][1] cell_id = flo2d_grids[flo2d_index][0] meta_data = { 'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)), 'model': flo2d_model, 'method': method, 'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10)) } tms_id = Sim_TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method')) if tms_id is None: tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id Sim_TS.insert_run(meta_data=meta_data) obs_end = Sim_TS.get_obs_end(id_=tms_id) fcst_timeseries = [] for i in range(len(model_list)): source_id = get_source_id(pool=curw_fcst_pool, model=model_list[i][0], version=model_list[i][1]) sim_tag = model_list[i][2] coefficient = model_list[i][3] temp_timeseries = [] if timestep == 5: if obs_end is not None: temp_timeseries = convert_15_min_ts_to_5_mins_ts( newly_extracted_timeseries=Fcst_TS. get_latest_timeseries(sim_tag=sim_tag, station_id=flo2d_wrf_mapping. get(meta_data['grid_id']), start=obs_end, source_id=source_id, variable_id=1, unit_id=1)) else: temp_timeseries = convert_15_min_ts_to_5_mins_ts( newly_extracted_timeseries=Fcst_TS. get_latest_timeseries(sim_tag=sim_tag, station_id=flo2d_wrf_mapping. get(meta_data['grid_id']), source_id=source_id, variable_id=1, unit_id=1)) elif timestep == 15: if obs_end is not None: temp_timeseries = Fcst_TS.get_latest_timeseries( sim_tag=sim_tag, station_id=flo2d_wrf_mapping.get( meta_data['grid_id']), start=obs_end, source_id=source_id, variable_id=1, unit_id=1) else: temp_timeseries = Fcst_TS.get_latest_timeseries( sim_tag=sim_tag, station_id=flo2d_wrf_mapping.get( meta_data['grid_id']), source_id=source_id, variable_id=1, unit_id=1) if coefficient != 1: for j in range(len(temp_timeseries)): temp_timeseries[j][1] = float( temp_timeseries[j][1]) * coefficient if i == 0: fcst_timeseries = temp_timeseries else: fcst_timeseries = append_value_for_timestamp( existing_ts=fcst_timeseries, new_ts=temp_timeseries) sum_timeseries = summed_timeseries(fcst_timeseries) for i in range(len(sum_timeseries)): if float(sum_timeseries[i][1]) < 0: sum_timeseries[i][1] = 0 if sum_timeseries is not None and len(sum_timeseries) > 0: Sim_TS.insert_data(timeseries=sum_timeseries, tms_id=tms_id, upsert=True) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating fcst rainfalls in curw_sim.") finally: destroy_Pool(curw_sim_pool) destroy_Pool(curw_fcst_pool)
def update_rainfall_fcsts(target_model, method, grid_interpolation, model_list, timestep): """ Update rainfall forecasts for flo2d models :param target_model: target model for which input ins prepared :param method: value interpolation method :param grid_interpolation: grid interpolation method :param model_list: list of forecast model and their versions used to calculate the rainfall e.g.: [["WRF_E", "4.0", "evening_18hrs"],["WRF_SE", "v4", ,"evening_18hrs"],["WRF_Ensemble", "4.0", ,"MME"]] :param timestep: output timeseries timestep :return: """ try: # Connect to the database curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) curw_fcst_pool = get_Pool(host=CURW_FCST_HOST, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE) Sim_TS = Sim_Timeseries(pool=curw_sim_pool) Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool) # [hash_id, station_id, station_name, latitude, longitude] active_obs_stations = read_csv( 'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv' ) obs_stations_dict = { } # keys: obs station id , value: [name, latitude, longitude] for obs_index in range(len(active_obs_stations)): obs_stations_dict[active_obs_stations[obs_index][1]] = [ active_obs_stations[obs_index][2], active_obs_stations[obs_index][3], active_obs_stations[obs_index][4] ] obs_d03_mapping = get_obs_to_d03_grid_mappings_for_rainfall( pool=curw_sim_pool, grid_interpolation=grid_interpolation) for obs_id in obs_stations_dict.keys(): meta_data = { 'latitude': float('%.6f' % float(obs_stations_dict.get(obs_id)[1])), 'longitude': float('%.6f' % float(obs_stations_dict.get(obs_id)[2])), 'model': target_model, 'method': method, 'grid_id': 'rainfall_{}_{}_{}'.format(obs_id, obs_stations_dict.get(obs_id)[0], grid_interpolation) } tms_id = Sim_TS.get_timeseries_id_if_exists(meta_data=meta_data) if tms_id is None: tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id Sim_TS.insert_run(meta_data=meta_data) obs_end = Sim_TS.get_obs_end(id_=tms_id) fcst_timeseries = [] for i in range(len(model_list)): source_id = get_source_id(pool=curw_fcst_pool, model=model_list[i][0], version=model_list[i][1]) sim_tag = model_list[i][2] coefficient = model_list[i][3] temp_timeseries = [] if timestep == 5: if obs_end is not None: temp_timeseries = convert_15_min_ts_to_5_mins_ts( newly_extracted_timeseries=Fcst_TS. get_latest_timeseries(sim_tag=sim_tag, station_id=obs_d03_mapping. get(meta_data['grid_id'])[0], start=obs_end, source_id=source_id, variable_id=1, unit_id=1)) else: temp_timeseries = convert_15_min_ts_to_5_mins_ts( newly_extracted_timeseries=Fcst_TS. get_latest_timeseries(sim_tag=sim_tag, station_id=obs_d03_mapping. get(meta_data['grid_id'])[0], source_id=source_id, variable_id=1, unit_id=1)) elif timestep == 15: if obs_end is not None: temp_timeseries = Fcst_TS.get_latest_timeseries( sim_tag=sim_tag, station_id=obs_d03_mapping.get( meta_data['grid_id'])[0], start=obs_end, source_id=source_id, variable_id=1, unit_id=1) else: temp_timeseries = Fcst_TS.get_latest_timeseries( sim_tag=sim_tag, station_id=obs_d03_mapping.get( meta_data['grid_id'])[0], source_id=source_id, variable_id=1, unit_id=1) if coefficient != 1: for j in range(len(temp_timeseries)): temp_timeseries[j][1] = float( temp_timeseries[j][1]) * coefficient if i == 0: fcst_timeseries = temp_timeseries else: fcst_timeseries = append_value_for_timestamp( existing_ts=fcst_timeseries, new_ts=temp_timeseries) sum_timeseries = summed_timeseries(fcst_timeseries) for i in range(len(sum_timeseries)): if float(sum_timeseries[i][1]) < 0: sum_timeseries[i][1] = 0 if sum_timeseries is not None and len(sum_timeseries) > 0: Sim_TS.insert_data(timeseries=sum_timeseries, tms_id=tms_id, upsert=True) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating fcst rainfalls in curw_sim.") finally: destroy_Pool(curw_sim_pool) destroy_Pool(curw_fcst_pool)
'version': 'v3', 'parameters': {} }] pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE) print("########### Add Sources #################################") print(add_sources(sources=sources, pool=pool)) print("########### Get Sources by id ###########################") print("Id 3:", get_source_by_id(pool=pool, id_="3")) print("########## Retrieve source id ###########################") print("'model': 'OBS_WATER_LEVEL', 'version': ''", get_source_id(pool=pool, model="OBS_WATER_LEVEL", version="")) print("######### Delete source by id ###########################") print("Id 3 deleted status: ", delete_source_by_id(pool=pool, id_=3)) print("######### Delete source with given model, version #######") print("model': 'wrfSE', 'version': 'v3' delete status :", delete_source(pool=pool, model="wrfSE", version="v3")) destroy_Pool(pool) print("bdjhsvdhsvdjh") exit(0)