def add_unit(pool, unit, unit_type): """ Insert units into the database :param pool: database connection pool :param unit: string :param unit_type: UnitType enum value. This value can be any of {Accumulative, Instantaneous, Mean} set :return: True if the unit has been added to the "Unit" table of the database, else False """ connection = pool.connection() try: if get_unit_id(pool=pool, unit=unit, unit_type=unit_type) is None: with connection.cursor() as cursor: sql_statement = "INSERT INTO `unit` (`unit`, `type`) VALUES ( %s, %s)" row_count = cursor.execute(sql_statement, (unit, unit_type.value)) connection.commit() return True if row_count > 0 else False else: logger.info( "Unit with unit={}, unit_type={} already exists in the database" .format(unit, unit_type)) return False except Exception as exception: connection.rollback() error_message = "Insertion of unit: unit={}, unit_type={} failed".format( unit, unit_type) logger.error(error_message) traceback.print_exc() raise exception finally: if connection is not None: connection.close()
def add_source(pool, model, version, parameters=None): """ Insert sources into the database :param pool: database connection pool :param model: string :param version: string :param parameters: JSON :return: True if the source has been added to the "Source' table of the database, else False """ connection = pool.connection() try: if get_source_id(pool=pool, model=model, version=version) is None: with connection.cursor() as cursor: sql_statement = "INSERT INTO `source` (`model`, `version`, `parameters`) VALUES ( %s, %s, %s)" row_count = cursor.execute( sql_statement, (model, version, json.dumps(parameters))) connection.commit() return True if row_count > 0 else False else: logger.info( "Source with model={} and version={} already exists in the database" .format(model, version)) return False except Exception as exception: connection.rollback() error_message = "Insertion of source: model={}, version={} and parameters={} failed".format( model, version, parameters) logger.error(error_message) traceback.print_exc() raise exception finally: if connection is not None: connection.close()
def delete_variable(pool, variable): """ Delete variable from Variable table, given variable name :param pool: database connection pool :param variable: string :return: True if the deletion was successful, else False """ connection = pool.connection() try: with connection.cursor() as cursor: sql_statement = "DELETE FROM `variable` WHERE `variable`=%s" row_count = cursor.execute(sql_statement, variable) connection.commit() if row_count > 0: return True else: logger.info( "There's no record of variable in the database with variable={}" .format(variable)) return False except Exception as exception: connection.rollback() error_message = "Deleting variable with variable={} failed.".format( variable) logger.error(error_message) traceback.print_exc() raise exception finally: if connection is not None: connection.close()
def delete_unit(pool, unit, unit_type): """ Delete unit from Unit table, given unit and unit_type :param pool: database connection pool :param unit: string :param unit_type: UnitType enum value. This value can be any of {Accumulative, Instantaneous, Mean} set :return: True if the deletion was successful, else False """ connection = pool.connection() try: with connection.cursor() as cursor: sql_statement = "DELETE FROM `unit` WHERE `unit`=%s and `type`=%s" row_count = cursor.execute(sql_statement, (unit, unit_type.value)) connection.commit() if row_count > 0: return True else: logger.info( "There's no record of unit in the database with unit={} and unit_type={}" .format(unit, unit_type)) return False except Exception as exception: connection.rollback() error_message = "Deleting unit with unit={} and unit_type={} failed.".format( unit, unit_type) logger.error(error_message) traceback.print_exc() raise exception finally: if connection is not None: connection.close()
def delete_source(pool, model, version): """ Delete source from Source table, given model and version :param pool: database connection pool :param model: str :param version: str :return: True if the deletion was successful """ connection = pool.connection() try: with connection.cursor() as cursor: sql_statement = "DELETE FROM `source` WHERE `model`=%s and `version`=%s" row_count = cursor.execute(sql_statement, (model, version)) connection.commit() if row_count > 0: return True else: logger.info( "There's no record of source in the database with model={} and version={}" .format(model, version)) return False except Exception as exception: connection.rollback() error_message = "Deleting source with model={} and version={} failed.".format( model, version) logger.error(error_message) traceback.print_exc() raise exception finally: if connection is not None: connection.close()
def add_variable(pool, variable): """ Insert variables into the database :param pool: database connection pool :param variable: string :return: True if the variable has been added to the "Variable" table of the database, else False """ connection = pool.connection() try: if get_variable_id(pool=pool, variable=variable) is None: with connection.cursor() as cursor: sql_statement = "INSERT INTO `variable` (`variable`) VALUES ( %s)" row_count = cursor.execute(sql_statement, variable) connection.commit() return True if row_count > 0 else False else: logger.info( "Variable with variable={} already exists in the database". format(variable)) return False except Exception as exception: connection.rollback() error_message = "Insertion of variable: variable={} failed".format( variable) logger.error(error_message) traceback.print_exc() raise exception finally: if connection is not None: connection.close()
def delete_source_by_id(pool, id_): """ Delete source from Source table by id :param pool: database connection pool :param id_: :return: True if the deletion was successful, else False """ connection = pool.connection() try: with connection.cursor() as cursor: sql_statement = "DELETE FROM `source` WHERE `id`=%s" row_count = cursor.execute(sql_statement, id_) connection.commit() if row_count > 0: return True else: logger.info( "There's no record of source in the database with the source id {}" .format(id_)) return False except Exception as exception: connection.rollback() error_message = "Deleting source with id {} failed.".format(id_) logger.error(error_message) traceback.print_exc() raise exception finally: if connection is not None: connection.close()
def add_station(pool, name, latitude, longitude, description, station_type): """ Insert sources into the database Station ids ranged as below; - 1 xx xxx - CUrW (stationId: curw_<SOMETHING>) - 2 xx xxx - Megapolis (stationId: megapolis_<SOMETHING>) - 3 xx xxx - Government (stationId: gov_<SOMETHING>. May follow as gov_irr_<SOMETHING>) - 4 xx xxx - Public (stationId: pub_<SOMETHING>) - 8 xx xxx - Satellite (stationId: sat_<SOMETHING>) Simulation models station ids ranged over 1’000’000 as below; - 1 1xx xxx - WRF (stationId: [;<prefix>_]wrf_<SOMETHING>) - 1 2xx xxx - FLO2D (stationId: [;<prefix>_]flo2d_<SOMETHING>)model - 1 3xx xxx - MIKE (stationId: [;<prefix>_]mike_<SOMETHING>) :param pool: database connection pool :param name: string :param latitude: double :param longitude: double :param description: string :param station_type: StationEnum: which defines the station type such as 'CUrW' :return: True if the station is added into the 'Station' table, else False """ initial_value = station_type.value range_ = StationEnum.getRange(station_type) connection = pool.connection() try: if get_station_id(pool=pool, latitude=latitude, longitude=longitude, station_type=station_type) is None: with connection.cursor() as cursor1: sql_statement = "SELECT `id` FROM `station` WHERE `id` BETWEEN %s and %s ORDER BY `id` DESC" row_count = cursor1.execute(sql_statement, (initial_value, initial_value + range_ - 1)) if row_count > 0: station_id = cursor1.fetchone()['id'] + 1 else: station_id = initial_value with connection.cursor() as cursor2: sql_statement = "INSERT INTO `station` (`id`, `name`, `latitude`, `longitude`, `description`) " \ "VALUES ( %s, %s, %s, %s, %s)" row_count = cursor2.execute(sql_statement, (station_id, name, latitude, longitude, description)) connection.commit() return True if row_count > 0 else False else: logger.info("Station with latitude={} longitude={} and station_type={} already exists in the database" .format(latitude, longitude, station_type)) return False except Exception as exception: connection.rollback() error_message = "Insertion of station: name={}, latitude={}, longitude={}, description={}, " \ "and station_type={} failed.".format(name, latitude, longitude, description, station_type) logger.error(error_message) traceback.print_exc() raise exception finally: if connection is not None: connection.close()
def create_test_schema_db(): # connect to the MySQL engine engine = get_engine(DIALECT_MYSQL, DRIVER_PYMYSQL, HOST, PORT, DATABASE, USERNAME, PASSWORD) # create the schema using classes defined CurwFcstBase.metadata.create_all(engine) logger.info("test_schema schema generated.")
def gen_all_d03_rfields_locally(source_names, version, sim_tag): """ Generate d03 rfields for SL extent :param source_names: e.g.: WRF_A,WRF_C :param version: e.g.: v4.0 :param sim_tag: e.g.: "evening_18hrs" :return: True if successful, False otherwise """ rfield_command_d03 = "sudo nice -n -15 nohup /home/uwcc-admin/curw_rfield_extractor/gen_SL_d03_rfield.py -m {} -v {} -s {} 2>&1 " \ "/home/uwcc-admin/curw_rfield_extractor/rfield.log".format(source_names, version, sim_tag) logger.info("Generate {} d03 rfield files.".format(source_names)) output = os.system(rfield_command_d03) if output is not 0: return False return True
def extract_wrf_data(wrf_system, config_data, tms_meta): logger.info( "######################################## {} #######################################" .format(wrf_system)) for date in config_data['dates']: # /wrf_nfs/wrf/4.0/18/A/2019-07-30/d03_RAINNC.nc output_dir = os.path.join(config_data['wrf_dir'], config_data['version'], config_data['gfs_data_hour'], wrf_system, date) rainnc_net_cdf_file = 'd03_RAINNC.nc' rainnc_net_cdf_file_path = os.path.join(output_dir, rainnc_net_cdf_file) try: source_name = "{}_{}".format(config_data['model'], wrf_system) source_id = get_source_id(pool=pool, model=source_name, version=tms_meta['version']) if source_id is None: add_source(pool=pool, model=source_name, version=tms_meta['version']) source_id = get_source_id(pool=pool, model=source_name, version=tms_meta['version']) except Exception: msg = "Exception occurred while loading source meta data for WRF_{} from database.".format( wrf_system) logger.error(msg) email_content[datetime.now().strftime( COMMON_DATE_TIME_FORMAT)] = msg return False tms_meta['model'] = source_name tms_meta['source_id'] = source_id return read_netcdf_file( pool=pool, rainnc_net_cdf_file_path=rainnc_net_cdf_file_path, tms_meta=tms_meta)
def gen_all_d03_rfields(source_names, version, sim_tag, rfield_host, rfield_key, rfield_user): """ Generate d03 rfields for SL extent :param source_names: e.g.: WRF_A,WRF_C :param version: e.g.: v4.0 :param sim_tag: e.g.: "evening_18hrs" :param rfield_host: :param rfield_key: :param rfield_user: :return: True if successful, False otherwise """ rfield_command_d03 = "sudo nice -n -15 nohup ./curw_rfield_extractor/gen_SL_d03_rfield.py -m {} -v {} -s {} 2>&1 " \ "./curw_rfield_extractor/rfield.log".format(source_names, version, sim_tag) logger.info("Generate {} d03 rfield files.".format(source_names)) return run_remote_command(host=rfield_host, key=rfield_key, user=rfield_user, command=rfield_command_d03)
def delete_station(pool, latitude, longitude, station_type): """ Delete station from Station table :param pool: database connection pool :param latitude: :param longitude: :param station_type: StationEnum: which defines the station type such as 'CUrW' :return: True if the deletion was successful, else False """ connection = pool.connection() try: initial_value = str(station_type.value) if len(initial_value)==6: pattern = "{}_____".format(initial_value[0]) elif len(initial_value)==7: pattern = "{}{}_____".format(initial_value[0], initial_value[1]) with connection.cursor() as cursor: sql_statement = "DELETE FROM `station` WHERE `id` like %s and `latitude`=%s and `longitude`=%s and `station_type`=%s;" row_count = cursor.execute(sql_statement, (pattern, latitude, longitude, StationEnum.getTypeString(station_type))) connection.commit() if row_count > 0: return True else: logger.info("There's no record of station in the database with latitude={}, " "longitude={}, and station_type{}".format(latitude, longitude, station_type)) return False except Exception as exception: connection.rollback() error_message = "Deleting station with latitude={}, longitude={}, and station_type{} failed."\ .format(latitude, longitude, station_type) logger.error(error_message) traceback.print_exc() raise exception finally: if connection is not None: connection.close()
kelani_basin_rfield_status = gen_kelani_basin_rfields_locally( source_names=source_list, version=version, sim_tag=sim_tag) if not kelani_basin_rfield_status: email_content[datetime.now().strftime( COMMON_DATE_TIME_FORMAT )] = "Kelani basin rfiled generation for {} failed".format( source_list) # d03_rfield_status = gen_all_d03_rfields(source_names=source_list, version=version, sim_tag=sim_tag, # rfield_host=rfield_host, rfield_key=rfield_key, rfield_user=rfield_user) d03_rfield_status = gen_all_d03_rfields_locally( source_names=source_list, version=version, sim_tag=sim_tag) if not d03_rfield_status: email_content[datetime.now().strftime( COMMON_DATE_TIME_FORMAT )] = "SL d03 rfiled generation for {} failed".format(source_list) except Exception as e: msg = 'Multiprocessing error.' logger.error(msg) email_content[datetime.now().strftime(COMMON_DATE_TIME_FORMAT)] = msg traceback.print_exc() finally: mp_pool.close() destroy_Pool(pool) logger.info("Process finished.") logger.info("Email Content {}".format(json.dumps(email_content)))
from db_adapter.logger import logger def get_engine(dialect, driver, host, port, db, user, password): """ Connecting to database """ db_url = "%s+%s://%s:%s@%s:%d/%s" % (dialect, driver, user, password, host, port, db) return create_engine(db_url, echo=False) def get_sessionmaker(engine): return sessionmaker(bind=engine) # ---- declarative_base ---- # allows us to create classes that include directives to describe the actual # database table they will be mapped to. # CurwFcstBase class for all the schema model classes of "curw-fcst" database logger.info("Declaring an orm mapping for curw_fcst database.") CurwFcstBase = declarative_base() # CurwObsBase class for all the schema model classes of "curw-obs" database logger.info("Declaring an orm mapping for curw_obs database.") CurwObsBase = declarative_base() # CurwSimBase class for all the schema model classes of "curw-sim" database logger.info("Declaring an orm mapping for curw_sim database.") CurwSimBase = declarative_base()
tms_id = TS.get_timeseries_id_if_exists(meta_data=meta_data) start_date = (datetime.now() - timedelta(days=1)).strftime("%Y-%m-%d %H:00:00") filled_ts = [] if tms_id is None: exit(0) else: end_date = TS.get_obs_end(id_=tms_id) original_ts = TS.get_timeseries(id_=tms_id, start_date=start_date, end_date=end_date) filled_ts = fill_ts_missing_entries(start=start_date, end=end_date, timeseries=original_ts, interpolation_method='linear', timestep=60) for i in range(len(filled_ts)): if filled_ts[i][1] < 0.2: filled_ts[i][1] = 0.2 if filled_ts is not None and len(filled_ts) > 0: TS.insert_data(timeseries=filled_ts, tms_id=tms_id, upsert=True) TS.update_latest_obs(id_=tms_id, obs_end=filled_ts[-1][1]) except Exception as e: traceback.print_exc() logger.error("Exception occurred.") finally: logger.info("Filled missing values") destroy_Pool(pool=curw_sim_pool)
# ######## # USERNAME = "******" # PASSWORD = "******" # HOST = "127.0.0.1" # PORT = 3306 # DATABASE = "curw_fcst" # pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE) add_source(pool=pool, model=model, version=version, parameters=None) # add_variable(pool=pool, variable=variable) # add_unit(pool=pool, unit=unit, unit_type=unit_type) # add hechms output stations for i in range(len(hechms_stations)): station_name = hechms_stations[i][0] lat = hechms_stations[i][1] lon = hechms_stations[i][2] add_station(pool=pool, name=station_name, latitude="%.6f" % float(lat), longitude="%.6f" % float(lon), station_type=StationEnum.HECHMS, description="hecHMS output station") except Exception: logger.info("Initialization process failed.") traceback.print_exc() finally: logger.info("Initialization process finished.") destroy_Pool(pool=pool)
def update_rainfall_from_file(curw_sim_pool, flo2d_grid_polygon_map, stations_dict, rainfall_df, flo2d_model, method, grid_interpolation, timestep, start_time=None, end_time=None): """ Update rainfall observations for flo2d models :param flo2d_model: flo2d model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param timestep: output timeseries timestep :return: """ # start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') try: TS = Sim_Timeseries(pool=curw_sim_pool) # # [hash_id, station_id, station_name, latitude, longitude] # flo2d_grid_polygon_map :: [Grid_ ID, X(longitude), Y(latitude), matching_point] # stations_dict_for_obs = { } # keys: obs station id , value: hash id for grid in flo2d_grid_polygon_map: lat = grid[2] lon = grid[1] cell_id = grid[0] meta_data = { 'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)), 'model': flo2d_model, 'method': method, 'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10)) } if len(grid) > 3: polygon = grid[3] poly_lat = stations_dict.get(polygon)[1] poly_lon = stations_dict.get(polygon)[0] processed_ts = rainfall_df.loc[ (rainfall_df['latitude'] == poly_lat) & (rainfall_df['longitude'] == poly_lon)][[ 'time', 'value' ]].values.tolist() else: processed_ts = rainfall_df.groupby('time').mean().round( 3)['value'].reset_index().values.tolist() tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method')) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) print("grid_id:", meta_data['grid_id']) print(processed_ts) # for i in range(len(obs_timeseries)): # if obs_timeseries[i][1] == -99999: # obs_timeseries[i][1] = 0 if processed_ts is not None and len(processed_ts) > 0: TS.insert_data(timeseries=processed_ts, tms_id=tms_id, upsert=True) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating obs rainfalls in curw_sim.") finally: destroy_Pool(pool=curw_sim_pool) logger.info("Process finished")
def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep): """ Update rainfall observations for flo2d models :param flo2d_model: flo2d model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param timestep: output timeseries timestep :return: """ now = datetime.now() OBS_START_STRING = (now - timedelta(days=10)).strftime('%Y-%m-%d %H:00:00') OBS_START = datetime.strptime(OBS_START_STRING, '%Y-%m-%d %H:%M:%S') try: # Connect to the database curw_obs_pool = get_Pool(host=CURW_OBS_HOST, user=CURW_OBS_USERNAME, password=CURW_OBS_PASSWORD, port=CURW_OBS_PORT, db=CURW_OBS_DATABASE) curw_obs_connection = curw_obs_pool.connection() curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) # test ###### # pool = get_Pool(host=HOST, user=USERNAME, password=PASSWORD, port=PORT, db=DATABASE) TS = Sim_Timeseries(pool=curw_sim_pool) # [hash_id, station_id, station_name, latitude, longitude] active_obs_stations = read_csv( 'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv' ) flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format( flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)] stations_dict_for_obs = {} # keys: obs station id , value: hash id for obs_index in range(len(active_obs_stations)): stations_dict_for_obs[active_obs_stations[obs_index] [1]] = active_obs_stations[obs_index][0] flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings( pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model) for flo2d_index in range(len(flo2d_grids)): obs_start = OBS_START lat = flo2d_grids[flo2d_index][2] lon = flo2d_grids[flo2d_index][1] cell_id = flo2d_grids[flo2d_index][0] meta_data = { 'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)), 'model': flo2d_model, 'method': method, 'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10)) } tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method')) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) obs_end = TS.get_obs_end(id_=tms_id) if obs_end is not None: obs_start = obs_end - timedelta(hours=1) obs1_station_id = str( flo2d_obs_mapping.get(meta_data['grid_id'])[0]) obs2_station_id = str( flo2d_obs_mapping.get(meta_data['grid_id'])[1]) obs3_station_id = str( flo2d_obs_mapping.get(meta_data['grid_id'])[2]) obs_timeseries = [] if timestep == 5: if obs1_station_id != str(-1): obs1_hash_id = stations_dict_for_obs.get(obs1_station_id) ts = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs1_hash_id) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_5_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] if obs2_station_id != str(-1): obs2_hash_id = stations_dict_for_obs.get( obs2_station_id) ts2 = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs2_hash_id) if ts2 is not None and len(ts2) > 1: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts2, OBS_TS=obs_timeseries) if obs_timeseries is not None and len( obs_timeseries) > 0: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_5_min_ts( newly_extracted_timeseries=ts2, expected_start=expected_start)[1:]) # obs_start = ts2[-1][0] if obs3_station_id != str(-1): obs3_hash_id = stations_dict_for_obs.get( obs3_station_id) ts3 = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs3_hash_id) if ts3 is not None and len(ts3) > 1 and len( obs_timeseries) > 0: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts3, OBS_TS=obs_timeseries) if obs_timeseries is not None: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_5_min_ts( newly_extracted_timeseries=ts3, expected_start=expected_start)[1:]) elif timestep == 15: if obs1_station_id != str(-1): obs1_hash_id = stations_dict_for_obs.get(obs1_station_id) ts = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs1_hash_id) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_15_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] if obs2_station_id != str(-1): obs2_hash_id = stations_dict_for_obs.get( obs2_station_id) ts2 = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs2_hash_id) if ts2 is not None and len(ts2) > 1: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts2, OBS_TS=obs_timeseries) if obs_timeseries is not None and len( obs_timeseries) > 0: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_15_min_ts( newly_extracted_timeseries=ts2, expected_start=expected_start)[1:]) # obs_start = ts2[-1][0] if obs3_station_id != str(-1): obs3_hash_id = stations_dict_for_obs.get( obs3_station_id) ts3 = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs3_hash_id) if ts3 is not None and len(ts3) > 1 and len( obs_timeseries) > 0: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts3, OBS_TS=obs_timeseries) if obs_timeseries is not None: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_15_min_ts( newly_extracted_timeseries=ts3, expected_start=expected_start)[1:]) for i in range(len(obs_timeseries)): if obs_timeseries[i][1] == -99999: obs_timeseries[i][1] = 0 if obs_timeseries is not None and len(obs_timeseries) > 0: TS.insert_data(timeseries=obs_timeseries, tms_id=tms_id, upsert=True) TS.update_latest_obs(id_=tms_id, obs_end=(obs_timeseries[-1][1])) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating obs rainfalls in curw_sim.") finally: curw_obs_connection.close() destroy_Pool(pool=curw_sim_pool) destroy_Pool(pool=curw_obs_pool) logger.info("Process finished")
def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep, start_time, end_time): """ Update rainfall observations for flo2d models :param flo2d_model: flo2d model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param timestep: output timeseries timestep :return: """ obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') try: # Connect to the database curw_obs_pool = get_Pool(host=con_params.CURW_OBS_HOST, user=con_params.CURW_OBS_USERNAME, password=con_params.CURW_OBS_PASSWORD, port=con_params.CURW_OBS_PORT, db=con_params.CURW_OBS_DATABASE) curw_obs_connection = curw_obs_pool.connection() curw_sim_pool = get_Pool(host=con_params.CURW_SIM_HOST, user=con_params.CURW_SIM_USERNAME, password=con_params.CURW_SIM_PASSWORD, port=con_params.CURW_SIM_PORT, db=con_params.CURW_SIM_DATABASE) TS = Sim_Timeseries(pool=curw_sim_pool) # [hash_id, station_id, station_name, latitude, longitude] # active_obs_stations = read_csv(os.path.join(ROOT_DIR,'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv')) active_obs_stations = extract_active_curw_obs_rainfall_stations( start_time=start_time, end_time=end_time)[1:] flo2d_grids = read_csv( os.path.join(ROOT_DIR, 'grids/flo2d/{}m.csv'.format( flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)] stations_dict_for_obs = {} # keys: obs station id , value: hash id for obs_index in range(len(active_obs_stations)): stations_dict_for_obs[active_obs_stations[obs_index] [1]] = active_obs_stations[obs_index][0] # flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model) flo2d_obs_mapping = find_nearest_obs_stations_for_flo2d_stations( flo2d_stations_csv=os.path.join( ROOT_DIR, 'grids/flo2d/{}m.csv'.format(flo2d_model)), obs_stations=active_obs_stations, flo2d_model=flo2d_model) for flo2d_index in range(len(flo2d_grids)): lat = flo2d_grids[flo2d_index][2] lon = flo2d_grids[flo2d_index][1] cell_id = flo2d_grids[flo2d_index][0] meta_data = { 'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)), 'model': flo2d_model, 'method': method, 'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10)) } tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method')) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) print("grid_id:", cell_id) print("grid map:", flo2d_obs_mapping.get(cell_id)) obs1_station_id = flo2d_obs_mapping.get(cell_id)[0] obs2_station_id = flo2d_obs_mapping.get(cell_id)[1] obs3_station_id = flo2d_obs_mapping.get(cell_id)[2] obs_timeseries = [] if timestep == 5: if obs1_station_id != str(-1): obs1_hash_id = stations_dict_for_obs.get(obs1_station_id) ts = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs1_hash_id, end_time=end_time) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_5_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] if obs2_station_id != str(-1): obs2_hash_id = stations_dict_for_obs.get( obs2_station_id) ts2 = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs2_hash_id, end_time=end_time) if ts2 is not None and len(ts2) > 1: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts2, OBS_TS=obs_timeseries) if obs_timeseries is not None and len( obs_timeseries) > 0: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_5_min_ts( newly_extracted_timeseries=ts2, expected_start=expected_start)[1:]) # obs_start = ts2[-1][0] if obs3_station_id != str(-1): obs3_hash_id = stations_dict_for_obs.get( obs3_station_id) ts3 = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs3_hash_id, end_time=end_time) if ts3 is not None and len(ts3) > 1 and len( obs_timeseries) > 0: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts3, OBS_TS=obs_timeseries) if obs_timeseries is not None: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_5_min_ts( newly_extracted_timeseries=ts3, expected_start=expected_start)[1:]) elif timestep == 15: if obs1_station_id != str(-1): obs1_hash_id = stations_dict_for_obs.get(obs1_station_id) ts = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs1_hash_id, end_time=end_time) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_15_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] if obs2_station_id != str(-1): obs2_hash_id = stations_dict_for_obs.get( obs2_station_id) ts2 = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs2_hash_id, end_time=end_time) if ts2 is not None and len(ts2) > 1: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts2, OBS_TS=obs_timeseries) if obs_timeseries is not None and len( obs_timeseries) > 0: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_15_min_ts( newly_extracted_timeseries=ts2, expected_start=expected_start)[1:]) # obs_start = ts2[-1][0] if obs3_station_id != str(-1): obs3_hash_id = stations_dict_for_obs.get( obs3_station_id) ts3 = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs3_hash_id, end_time=end_time) if ts3 is not None and len(ts3) > 1 and len( obs_timeseries) > 0: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts3, OBS_TS=obs_timeseries) if obs_timeseries is not None: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_15_min_ts( newly_extracted_timeseries=ts3, expected_start=expected_start)[1:]) for i in range(len(obs_timeseries)): if obs_timeseries[i][1] == -99999: obs_timeseries[i][1] = 0 print("### obs timeseries length ###", len(obs_timeseries)) if obs_timeseries is not None and len( obs_timeseries) > 0 and obs_timeseries[-1][0] != end_time: obs_timeseries.append( [datetime.strptime(end_time, DATE_TIME_FORMAT), 0]) final_ts = process_continuous_ts(original_ts=obs_timeseries, expected_start=datetime.strptime( start_time, DATE_TIME_FORMAT), filling_value=0, timestep=timestep) if final_ts is not None and len(final_ts) > 0: TS.insert_data(timeseries=final_ts, tms_id=tms_id, upsert=True) TS.update_latest_obs(id_=tms_id, obs_end=(final_ts[-1][1])) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating obs rainfalls in curw_sim.") finally: curw_obs_connection.close() destroy_Pool(pool=curw_sim_pool) destroy_Pool(pool=curw_obs_pool) logger.info("Process finished")
def update_rainfall_obs(curw_obs_pool, curw_sim_pool, flo2d_model, method, grid_interpolation, timestep, start_time, end_time): """ Update rainfall observations for flo2d models :param flo2d_model: flo2d model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param timestep: output timeseries timestep :return: """ # obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') try: curw_obs_connection = curw_obs_pool.connection() # [hash_id, station_id, station_name, latitude, longitude] # active_obs_stations = read_csv(os.path.join(ROOT_DIR,'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv')) active_obs_stations = extract_active_curw_obs_rainfall_stations(curw_obs_pool=curw_obs_pool, start_time=start_time, end_time=end_time)[1:] flo2d_grids = read_csv(os.path.join(ROOT_DIR,'grids/flo2d/{}m.csv'.format(flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)] stations_dict_for_obs = { } # keys: obs station id , value: hash id for obs_index in range(len(active_obs_stations)): stations_dict_for_obs[active_obs_stations[obs_index][1]] = active_obs_stations[obs_index][0] # flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model) flo2d_obs_mapping = find_nearest_obs_stations_for_flo2d_stations( flo2d_stations_csv=os.path.join(ROOT_DIR,'grids/flo2d/{}m.csv'.format(flo2d_model)), obs_stations=active_obs_stations, flo2d_model=flo2d_model) # retrieve observed timeseries obs_df = pd.DataFrame() obs_df['time'] = pd.date_range(start=start_time, end=end_time, freq='5min') for obs_id in stations_dict_for_obs.keys(): ts = extract_obs_rain_5_min_ts(connection=curw_obs_connection, start_time=start_time, id=stations_dict_for_obs.get(obs_id), end_time=end_time) ts.insert(0, ['time', obs_id]) ts_df = list_of_lists_to_df_first_row_as_columns(ts) ts_df[obs_id] = ts_df[obs_id].astype('float64') obs_df = pd.merge(obs_df, ts_df, how="left", on='time') obs_df.set_index('time', inplace=True) obs_df['0'] = 0 if timestep == 15: obs_df = obs_df.resample('15min', label='right', closed='right').sum() TS = Sim_Timeseries(pool=curw_sim_pool) for flo2d_index in range(len(flo2d_grids)): lat = flo2d_grids[flo2d_index][2] lon = flo2d_grids[flo2d_index][1] cell_id = flo2d_grids[flo2d_index][0] meta_data = { 'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)), 'model': flo2d_model, 'method': method, 'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10)) } tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method')) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) print(datetime.now().strftime(DATE_TIME_FORMAT)) print("grid_id:", cell_id) obs_station_ids = flo2d_obs_mapping.get(cell_id) if len(obs_station_ids) == 1: obs_ts_df = obs_df[obs_station_ids].to_frame(name='final') elif len(obs_station_ids) == 2: obs_ts_df = obs_df[obs_station_ids] obs_ts_df[obs_station_ids[0]] = obs_ts_df[obs_station_ids[0]].fillna(obs_ts_df[obs_station_ids[1]]) obs_ts_df['final'] = obs_ts_df[obs_station_ids[0]] elif len(obs_station_ids) == 3: obs_ts_df = obs_df[obs_station_ids] obs_ts_df[obs_station_ids[1]] = obs_ts_df[obs_station_ids[1]].fillna(obs_ts_df[obs_station_ids[2]]) obs_ts_df[obs_station_ids[0]] = obs_ts_df[obs_station_ids[0]].fillna(obs_ts_df[obs_station_ids[1]]) obs_ts_df['final'] = obs_ts_df[obs_station_ids[0]] else: obs_ts_df = obs_df['0'].to_frame(name='final') final_ts_df = obs_ts_df['final'].reset_index() final_ts_df['time'] = final_ts_df['time'].dt.strftime(DATE_TIME_FORMAT) final_ts = final_ts_df.values.tolist() if final_ts is not None and len(final_ts) > 0: TS.replace_data(timeseries=final_ts, tms_id=tms_id) TS.update_latest_obs(id_=tms_id, obs_end=(final_ts[-1][1])) except Exception as e: traceback.print_exc() logger.error("Exception occurred while updating obs rainfalls in curw_sim.") finally: curw_obs_connection.close() destroy_Pool(pool=curw_sim_pool) destroy_Pool(pool=curw_obs_pool) logger.info("Process finished")
def extract_distrubuted_hechms_outputs(output_file_name, output_dir, run_date, run_time): """ Config.json { "output_file_name": "DailyDischarge.csv", "output_dir": "", "run_date": "2019-05-24", "run_time": "00:00:00", "utc_offset": "", "sim_tag": "hourly_run", "model": "HECHMS", "version": "single", "unit": "m3/s", "unit_type": "Instantaneous", "variable": "Discharge", "station_name": "Hanwella" } """ try: config = json.loads(open('config.json').read()) # output related details output_file_name = output_file_name output_dir = output_dir run_date = run_date run_time = run_time utc_offset = read_attribute_from_config_file('utc_offset', config, False) if utc_offset is None: utc_offset = '' # sim tag sim_tag = read_attribute_from_config_file('sim_tag', config, True) # source details model = read_attribute_from_config_file('model', config, True) version = read_attribute_from_config_file('version', config, True) # unit details unit = read_attribute_from_config_file('unit', config, True) unit_type = UnitType.getType( read_attribute_from_config_file('unit_type', config, True)) # variable details variable = read_attribute_from_config_file('variable', config, True) # station details station_name = read_attribute_from_config_file('station_name', config, True) out_file_path = os.path.join(output_dir, output_file_name) if not os.path.exists(out_file_path): msg = 'no file :: {}'.format(out_file_path) logger.warning(msg) print(msg) exit(1) fgt = get_file_last_modified_time(out_file_path) print("fgt, ", fgt) timeseries = read_csv(out_file_path) pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD) hechms_stations = get_hechms_stations(pool=pool) station_id = hechms_stations.get(station_name)[0] lat = str(hechms_stations.get(station_name)[1]) lon = str(hechms_stations.get(station_name)[2]) source_id = get_source_id(pool=pool, model=model, version=version) variable_id = get_variable_id(pool=pool, variable=variable) unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type) tms_meta = { 'sim_tag': sim_tag, 'model': model, 'version': version, 'variable': variable, 'unit': unit, 'unit_type': unit_type.value, 'latitude': lat, 'longitude': lon, 'station_id': station_id, 'source_id': source_id, 'variable_id': variable_id, 'unit_id': unit_id } utcOffset = getUTCOffset(utc_offset, default=True) if utcOffset != timedelta(): tms_meta['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries, run_date=run_date, run_time=run_time, tms_meta=tms_meta, fgt=fgt) except Exception as e: logger.error('JSON config data loading error.') print('JSON config data loading error.') traceback.print_exc() finally: logger.info("Process finished.") print("Process finished.")
# Save Forecast values into Database opts = {'elementNo': elementNo, 'tms_meta': tms_meta} if utcOffset != timedelta(): opts['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db( pool=pool, timeseries=waterLevelSeriesDict[elementNo], run_date=run_date, run_time=run_time, opts=opts, flo2d_stations=flo2d_stations, fgt=fgt) run_info = json.loads( open( os.path.join(os.path.dirname(hychan_out_file_path), "run_meta.json")).read()) insert_run_metadata(pool=pool, source_id=source_id, variable_id=variable_id, sim_tag=sim_tag, fgt=fgt, metadata=run_info, template_path=template_path) except Exception as e: traceback.print_exc() finally: logger.info("Process finished.") print("Process finished.")
def upload_waterlevels(dir_path, ts_start_date, ts_start_time, run_date, run_time): """ Config.json { "HYCHAN_OUT_FILE": "HYCHAN.OUT", "TIMDEP_FILE": "TIMDEP.OUT", "output_dir": "", "run_date": "2019-05-24", "run_time": "", "ts_start_date": "", "ts_start_time": "", "utc_offset": "", "sim_tag": "", "model": "WRF", "version": "v3", "unit": "mm", "unit_type": "Accumulative", "variable": "Precipitation" } """ try: config_path = os.path.join(os.getcwd(), 'extract', 'config_curw_fcst.json') config = json.loads(open(config_path).read()) # flo2D related details HYCHAN_OUT_FILE = read_attribute_from_config_file('HYCHAN_OUT_FILE', config, True) TIMDEP_FILE = read_attribute_from_config_file('TIMDEP_FILE', config, True) output_dir = dir_path run_date = run_date run_time = run_time ts_start_date = ts_start_date ts_start_time = ts_start_time utc_offset = read_attribute_from_config_file('utc_offset', config, False) if utc_offset is None: utc_offset = '' # sim tag sim_tag = read_attribute_from_config_file('sim_tag', config, True) # source details model = read_attribute_from_config_file('model', config, True) version = read_attribute_from_config_file('version', config, True) # unit details unit = read_attribute_from_config_file('unit', config, True) unit_type = UnitType.getType(read_attribute_from_config_file('unit_type', config, True)) # variable details variable = read_attribute_from_config_file('variable', config, True) hychan_out_file_path = os.path.join(output_dir, HYCHAN_OUT_FILE) timdep_file_path = os.path.join(output_dir, TIMDEP_FILE) pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD) # pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE) flo2d_model_name = '{}_{}'.format(model, version) flo2d_source = json.loads(get_source_parameters(pool=pool, model=model, version=version)) print("############### source ############", flo2d_source) flo2d_stations = get_flo2d_output_stations(pool=pool, flo2d_model=StationEnum.getType(flo2d_model_name)) print("############### 1st occurrence ############", flo2d_stations) source_id = get_source_id(pool=pool, model=model, version=version) variable_id = get_variable_id(pool=pool, variable=variable) unit_id = get_unit_id(pool=pool, unit=unit, unit_type=unit_type) tms_meta = { 'sim_tag': sim_tag, 'model': model, 'version': version, 'variable': variable, 'unit': unit, 'unit_type': unit_type.value, 'source_id': source_id, 'variable_id': variable_id, 'unit_id': unit_id } CHANNEL_CELL_MAP = flo2d_source["CHANNEL_CELL_MAP"] FLOOD_PLAIN_CELL_MAP = flo2d_source["FLOOD_PLAIN_CELL_MAP"] ELEMENT_NUMBERS = CHANNEL_CELL_MAP.keys() FLOOD_ELEMENT_NUMBERS = FLOOD_PLAIN_CELL_MAP.keys() SERIES_LENGTH = 0 MISSING_VALUE = -999 utcOffset = getUTCOffset(utc_offset, default=True) print('Extract Water Level Result of FLO2D on', run_date, '@', run_time, 'with Base time of', ts_start_date, '@', ts_start_time) # Check HYCHAN.OUT file exists if not os.path.exists(hychan_out_file_path): print('Unable to find file : ', hychan_out_file_path) traceback.print_exc() ##################################### # Calculate the size of time series # ##################################### bufsize = 65536 with open(hychan_out_file_path) as infile: isWaterLevelLines = False isCounting = False countSeriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series while True: lines = infile.readlines(bufsize) if not lines or SERIES_LENGTH: break for line in lines: if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5): isWaterLevelLines = True elif isWaterLevelLines: cols = line.split() if len(cols) > 0 and cols[0].replace('.', '', 1).isdigit(): countSeriesSize += 1 isCounting = True elif isWaterLevelLines and isCounting: SERIES_LENGTH = countSeriesSize break print('Series Length is :', SERIES_LENGTH) bufsize = 65536 ################################################################# # Extract Channel Water Level elevations from HYCHAN.OUT file # ################################################################# print('Extract Channel Water Level Result of FLO2D (HYCHAN.OUT) on', run_date, '@', run_time, 'with Base time of', ts_start_date, '@', ts_start_time) with open(hychan_out_file_path) as infile: isWaterLevelLines = False isSeriesComplete = False waterLevelLines = [] seriesSize = 0 # HACK: When it comes to the end of file, unable to detect end of time series while True: lines = infile.readlines(bufsize) if not lines: break for line in lines: if line.startswith('CHANNEL HYDROGRAPH FOR ELEMENT NO:', 5): seriesSize = 0 elementNo = line.split()[5] if elementNo in ELEMENT_NUMBERS: isWaterLevelLines = True waterLevelLines.append(line) else: isWaterLevelLines = False elif isWaterLevelLines: cols = line.split() if len(cols) > 0 and isfloat(cols[0]): seriesSize += 1 waterLevelLines.append(line) if seriesSize == SERIES_LENGTH: isSeriesComplete = True if isSeriesComplete: baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S') timeseries = [] elementNo = waterLevelLines[0].split()[5] # print('Extracted Cell No', elementNo, CHANNEL_CELL_MAP[elementNo]) for ts in waterLevelLines[1:]: v = ts.split() if len(v) < 1: continue # Get flood level (Elevation) value = v[1] # Get flood depth (Depth) # value = v[2] if not isfloat(value): value = MISSING_VALUE continue # If value is not present, skip if value == 'NaN': continue # If value is NaN, skip timeStep = float(v[0]) currentStepTime = baseTime + timedelta(hours=timeStep) dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S") timeseries.append([dateAndTime, value]) # Save Forecast values into Database opts = { 'elementNo': elementNo, 'tms_meta': tms_meta } # print('>>>>>', opts) if utcOffset != timedelta(): opts['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db(pool=pool, timeseries=timeseries, run_date=run_date, run_time=run_time, opts=opts, flo2d_stations=flo2d_stations) isWaterLevelLines = False isSeriesComplete = False waterLevelLines = [] # -- END for loop # -- END while loop ################################################################# # Extract Flood Plain water elevations from TIMEDEP.OUT file # ################################################################# if not os.path.exists(timdep_file_path): print('Unable to find file : ', timdep_file_path) traceback.print_exc() print('Extract Flood Plain Water Level Result of FLO2D (TIMEDEP.OUT) on', run_date, '@', run_time, 'with Base time of', ts_start_date, '@', ts_start_time) with open(timdep_file_path) as infile: waterLevelLines = [] waterLevelSeriesDict = dict.fromkeys(FLOOD_ELEMENT_NUMBERS, []) while True: lines = infile.readlines(bufsize) if not lines: break for line in lines: if len(line.split()) == 1: # continue if len(waterLevelLines) > 0: waterLevels = get_water_level_of_channels(waterLevelLines, FLOOD_ELEMENT_NUMBERS) # Get Time stamp Ref:http://stackoverflow.com/a/13685221/1461060 # print('waterLevelLines[0].split() : ', waterLevelLines[0].split()) ModelTime = float(waterLevelLines[0].split()[0]) baseTime = datetime.strptime('%s %s' % (ts_start_date, ts_start_time), '%Y-%m-%d %H:%M:%S') currentStepTime = baseTime + timedelta(hours=ModelTime) dateAndTime = currentStepTime.strftime("%Y-%m-%d %H:%M:%S") for elementNo in FLOOD_ELEMENT_NUMBERS: tmpTS = waterLevelSeriesDict[elementNo][:] if elementNo in waterLevels: tmpTS.append([dateAndTime, waterLevels[elementNo]]) else: tmpTS.append([dateAndTime, MISSING_VALUE]) waterLevelSeriesDict[elementNo] = tmpTS isWaterLevelLines = False # for l in waterLevelLines : # print(l) waterLevelLines = [] waterLevelLines.append(line) # print('len(FLOOD_ELEMENT_NUMBERS) : ', len(FLOOD_ELEMENT_NUMBERS)) for elementNo in FLOOD_ELEMENT_NUMBERS: # Save Forecast values into Database opts = { 'elementNo': elementNo, 'tms_meta': tms_meta } if utcOffset != timedelta(): opts['utcOffset'] = utcOffset # Push timeseries to database save_forecast_timeseries_to_db(pool=pool, timeseries=waterLevelSeriesDict[elementNo], run_date=run_date, run_time=run_time, opts=opts, flo2d_stations=flo2d_stations) except Exception as e: logger.error('JSON config data loading error.') print('JSON config data loading error.') traceback.print_exc() finally: logger.info("Process finished.") print("Process finished.")