def import_old_data(): # Connect to the database pool = get_Pool(host=CURW_FCST_HOST, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE) connection = pool.connection() curw_fcst_new_to_old_hash_id_mapping = read_csv( "curw_fcst_new_to_old_hash_id_mapping.csv") TS = Timeseries(pool=pool) try: for hash_index in range(len(curw_fcst_new_to_old_hash_id_mapping)): print("##### Hash index: ", hash_index, " #####") fgt_list = [] # Extract fgts with connection.cursor() as cursor1: sql_statement = "select distinct `fgt` from `data_v3` where `id`=%s order by `fgt` desc;" cursor1.execute( sql_statement, curw_fcst_new_to_old_hash_id_mapping[hash_index][1]) fgts = cursor1.fetchall() for fgt in fgts: fgt_list.append(fgt.get('fgt')) for fgt in fgt_list: timeseries = [] with connection.cursor() as cursor2: sql_statement = "select * from `data_v3` where `id`=%s and `fgt`=%s;" cursor2.execute( sql_statement, (curw_fcst_new_to_old_hash_id_mapping[hash_index][1], fgt)) results = cursor2.fetchall() for result in results: timeseries.append([ curw_fcst_new_to_old_hash_id_mapping[hash_index] [0], result.get('time'), result.get('fgt'), result.get('value') ]) TS.insert_data(timeseries=timeseries, upsert=True) TS.update_start_date( id_=curw_fcst_new_to_old_hash_id_mapping[hash_index][0], start_date=fgt) except Exception as ex: traceback.print_exc() finally: connection.close() destroy_Pool(pool=pool) print()
def find_nearest_stations_for_flo2d_grids(flo2d_grids_csv, stations_dict): """ :param flo2d_stations_csv: :param stations_dict: key = point_name, value = [latitude,longitude] :param flo2d_model: :return: """ flo2d_station = read_csv(flo2d_grids_csv) # [Grid_ID,X,Y] # key: flo2d_grid_id value: [lat, lon] flo2d_stations_mapping_dict = {} for flo2d_index in range(len(flo2d_station)): flo2d_lat = float(flo2d_station[flo2d_index][2]) flo2d_lng = float(flo2d_station[flo2d_index][1]) distances = {} for key in stations_dict.keys(): lat = float(stations_dict.get(key)[1]) lng = float(stations_dict.get(key)[0]) intermediate_value = cos(radians(flo2d_lat)) * cos( radians(lat)) * cos(radians(lng) - radians(flo2d_lng)) + sin( radians(flo2d_lat)) * sin(radians(lat)) if intermediate_value < 1: distance = 6371 * acos(intermediate_value) else: distance = 6371 * acos(1) distances[key] = distance sorted_distances = collections.OrderedDict( sorted(distances.items(), key=operator.itemgetter(1))[:10]) flo2d_stations_mapping = [] count = 0 for key in sorted_distances.keys(): if count < 1 and sorted_distances.get(key) <= 25: flo2d_stations_mapping = stations_dict.get(key) count += 1 elif count < 1: flo2d_stations_mapping = [-1, -1] count += 1 else: continue # print(flo2d_obs_mapping) flo2d_stations_mapping_dict[flo2d_station[flo2d_index] [0]] = flo2d_stations_mapping print(json.dumps(flo2d_stations_mapping_dict)) return flo2d_stations_mapping_dict
def update_discharge_from_hechms(curw_sim_pool, curw_fcst_pool, flo2d_model, method, start_time, end_time, sim_tag): try: TS = DTimeseries(pool=curw_sim_pool) # [station_name,latitude,longitude,target,model,version,sim_tag,station] extract_stations = read_csv( os.path.join(ROOT_DIR, 'grids/discharge_stations/flo2d_stations.csv')) for i in range(len(extract_stations)): station_name = extract_stations[i][0] latitude = extract_stations[i][1] longitude = extract_stations[i][2] target_model = extract_stations[i][3] if target_model == flo2d_model: if station_name in ("glencourse"): meta_data = { 'latitude': float('%.6f' % float(latitude)), 'longitude': float('%.6f' % float(longitude)), 'model': target_model, 'method': method, 'grid_id': 'discharge_{}'.format(station_name) } tms_id = TS.get_timeseries_id_if_exists( meta_data=meta_data) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) processed_discharge_ts = process_fcst_ts_from_hechms_outputs( curw_fcst_pool=curw_fcst_pool, extract_stations=extract_stations, i=i, start=start_time, end=end_time, sim_tag=sim_tag) if processed_discharge_ts is not None and len( processed_discharge_ts) > 0: TS.insert_data(timeseries=processed_discharge_ts, tms_id=tms_id, upsert=True) else: continue # skip the current iteration except Exception as e: traceback.print_exc()
def find_nearest_obs_stations_for_flo2d_stations(flo2d_stations_csv, obs_stations, flo2d_model): # obs_stations : [hash_id,station_id,station_name,latitude,longitude] flo2d_station = read_csv(flo2d_stations_csv) # [Grid_ID,X,Y] flo2d_obs_mapping_dict = {} for flo2d_index in range(len(flo2d_station)): grid_id = flo2d_station[flo2d_index][0] flo2d_obs_mapping = [] flo2d_lat = float(flo2d_station[flo2d_index][2]) flo2d_lng = float(flo2d_station[flo2d_index][1]) distances = {} for obs_index in range(len(obs_stations)): lat = float(obs_stations[obs_index][3]) lng = float(obs_stations[obs_index][4]) intermediate_value = cos(radians(flo2d_lat)) * cos( radians(lat)) * cos(radians(lng) - radians(flo2d_lng)) + sin( radians(flo2d_lat)) * sin(radians(lat)) if intermediate_value < 1: distance = 6371 * acos(intermediate_value) else: distance = 6371 * acos(1) distances[obs_stations[obs_index][1]] = distance sorted_distances = collections.OrderedDict( sorted(distances.items(), key=operator.itemgetter(1))[:10]) count = 0 for key in sorted_distances.keys(): if count < 3 and sorted_distances.get(key) <= 25: flo2d_obs_mapping.append(key) count += 1 elif count < 3: flo2d_obs_mapping.append("-1") count += 1 # print(flo2d_obs_mapping) flo2d_obs_mapping_dict[grid_id] = flo2d_obs_mapping # flo2d_grid_mappings[dict.get("grid_id")] = [dict.get("obs1"), dict.get("obs2"), dict.get("obs3")] flo2d_grid_mappings_dict = {} return flo2d_obs_mapping_dict
def process_tide_fcsts_from_Mobile_Geographics(existing_ts_end, obs_end): data = read_csv('{}/{}.csv'.format(INPUT_DIR, 'colombo_MGF')) raw_timeseries = [] expected_fcst_end = datetime.strptime( (datetime.now() + timedelta(days=60)).strftime("%Y-%m-01 00:00:00"), COMMON_DATE_TIME_FORMAT) if existing_ts_end is None or existing_ts_end < expected_fcst_end: for i in range(len(data)): time = datetime.strptime( "{} {} {}".format(data[i][0], data[i][1], data[i][2]), "%m/%d/%Y %I:%M %p") formatted_time = time.strftime(COMMON_DATE_TIME_FORMAT) raw_timeseries.append([formatted_time, data[i][3]]) if existing_ts_end is not None: fcst_start = (existing_ts_end).strftime(COMMON_DATE_TIME_FORMAT) elif obs_end is not None: fcst_start = (obs_end).strftime(COMMON_DATE_TIME_FORMAT) else: fcst_start = ((datetime.now() - timedelta(days=30))).strftime("%Y-%m-%d 00:00:00") raw_timeseries = extract_ts_from(fcst_start, raw_timeseries) raw_timeseries.insert(0, ['time', 'value']) timeseries_df = list_of_lists_to_df_first_row_as_columns( raw_timeseries) timeseries_df['time'] = pd.to_datetime(timeseries_df['time'], format=COMMON_DATE_TIME_FORMAT) timeseries_df['time'] = timeseries_df['time'].dt.round('h') timeseries_df.set_index('time', inplace=True) timeseries_df['value'] = pd.to_numeric(timeseries_df['value']) hourly_ts_df = timeseries_df.resample('H').asfreq() hourly_ts_df = hourly_ts_df.interpolate(method='linear', limit_direction='both', limit=100) ####temp### hourly_ts_df = hourly_ts_df.fillna(-99999.000) hourly_ts_df.index = hourly_ts_df.index.map(str) pd.set_option('display.max_rows', hourly_ts_df.shape[0] + 2) pd.set_option('display.max_columns', hourly_ts_df.shape[1] + 2) print(hourly_ts_df) processed_timeseries = hourly_ts_df.reset_index().values.tolist() return processed_timeseries else: return None
def divide_flo2d_grids_to_polygons(flo2d_model, polygons): flo2d_grids = read_csv( os.path.join(ROOT_DIR, 'grids/flo2d/{}m.csv'.format( flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)] for grid in flo2d_grids: point = Point(float(grid[1]), float(grid[2])) for index, row in polygons.iterrows(): polygon = polygons.iloc[index]['geometry'] if point.within(polygon): grid.append(polygons.iloc[index]['id']) continue return flo2d_grids
def update_tide_obs(curw_sim_pool, obs_connection, method, timestep, start_time, end_time, flo2d_model="flo2d"): # [station_name,latitude,longitude,target] extract_stations = read_csv('grids/tide_stations/extract_stations.csv') extract_stations_dict = {} # keys: station_name , value: [latitude, longitude, target_model] for obs_index in range(len(extract_stations)): extract_stations_dict[extract_stations[obs_index][0]] = [extract_stations[obs_index][1], extract_stations[obs_index][2], extract_stations[obs_index][3]] for station_name in extract_stations_dict.keys(): meta_data = { 'latitude': float('%.6f' % float(extract_stations_dict.get(station_name)[0])), 'longitude': float('%.6f' % float(extract_stations_dict.get(station_name)[1])), 'model': flo2d_model, 'method': method, 'grid_id': 'tide_{}'.format(station_name) } TS = Timeseries(pool=curw_sim_pool) tms_id = TS.get_timeseries_id_if_exists(meta_data=meta_data) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) processed_tide_ts = prepare_obs_tide_ts(connection=obs_connection, start_date=start_time, end_date=end_time, extract_station=station_name) for i in range(len(processed_tide_ts)): if len(processed_tide_ts[i]) < 2: processed_tide_ts.remove(processed_tide_ts[i]) final_tide_ts = fill_ts_missing_entries(start=start_time, end=end_time, timeseries=processed_tide_ts, interpolation_method='linear', timestep=timestep) if final_tide_ts is not None and len(final_tide_ts) > 0: TS.insert_data(timeseries=final_tide_ts, tms_id=tms_id, upsert=True)
def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep): """ Update rainfall observations for flo2d models :param flo2d_model: flo2d model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param timestep: output timeseries timestep :return: """ now = datetime.now() OBS_START_STRING = (now - timedelta(days=10)).strftime('%Y-%m-%d %H:00:00') OBS_START = datetime.strptime(OBS_START_STRING, '%Y-%m-%d %H:%M:%S') try: # Connect to the database curw_obs_pool = get_Pool(host=CURW_OBS_HOST, user=CURW_OBS_USERNAME, password=CURW_OBS_PASSWORD, port=CURW_OBS_PORT, db=CURW_OBS_DATABASE) curw_obs_connection = curw_obs_pool.connection() curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) # test ###### # pool = get_Pool(host=HOST, user=USERNAME, password=PASSWORD, port=PORT, db=DATABASE) TS = Sim_Timeseries(pool=curw_sim_pool) # [hash_id, station_id, station_name, latitude, longitude] active_obs_stations = read_csv( 'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv' ) flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format( flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)] stations_dict_for_obs = {} # keys: obs station id , value: hash id for obs_index in range(len(active_obs_stations)): stations_dict_for_obs[active_obs_stations[obs_index] [1]] = active_obs_stations[obs_index][0] flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings( pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model) for flo2d_index in range(len(flo2d_grids)): obs_start = OBS_START lat = flo2d_grids[flo2d_index][2] lon = flo2d_grids[flo2d_index][1] cell_id = flo2d_grids[flo2d_index][0] meta_data = { 'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)), 'model': flo2d_model, 'method': method, 'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10)) } tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method')) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) obs_end = TS.get_obs_end(id_=tms_id) if obs_end is not None: obs_start = obs_end - timedelta(hours=1) obs1_station_id = str( flo2d_obs_mapping.get(meta_data['grid_id'])[0]) obs2_station_id = str( flo2d_obs_mapping.get(meta_data['grid_id'])[1]) obs3_station_id = str( flo2d_obs_mapping.get(meta_data['grid_id'])[2]) obs_timeseries = [] if timestep == 5: if obs1_station_id != str(-1): obs1_hash_id = stations_dict_for_obs.get(obs1_station_id) ts = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs1_hash_id) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_5_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] if obs2_station_id != str(-1): obs2_hash_id = stations_dict_for_obs.get( obs2_station_id) ts2 = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs2_hash_id) if ts2 is not None and len(ts2) > 1: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts2, OBS_TS=obs_timeseries) if obs_timeseries is not None and len( obs_timeseries) > 0: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_5_min_ts( newly_extracted_timeseries=ts2, expected_start=expected_start)[1:]) # obs_start = ts2[-1][0] if obs3_station_id != str(-1): obs3_hash_id = stations_dict_for_obs.get( obs3_station_id) ts3 = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs3_hash_id) if ts3 is not None and len(ts3) > 1 and len( obs_timeseries) > 0: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts3, OBS_TS=obs_timeseries) if obs_timeseries is not None: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_5_min_ts( newly_extracted_timeseries=ts3, expected_start=expected_start)[1:]) elif timestep == 15: if obs1_station_id != str(-1): obs1_hash_id = stations_dict_for_obs.get(obs1_station_id) ts = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs1_hash_id) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_15_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] if obs2_station_id != str(-1): obs2_hash_id = stations_dict_for_obs.get( obs2_station_id) ts2 = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs2_hash_id) if ts2 is not None and len(ts2) > 1: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts2, OBS_TS=obs_timeseries) if obs_timeseries is not None and len( obs_timeseries) > 0: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_15_min_ts( newly_extracted_timeseries=ts2, expected_start=expected_start)[1:]) # obs_start = ts2[-1][0] if obs3_station_id != str(-1): obs3_hash_id = stations_dict_for_obs.get( obs3_station_id) ts3 = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs3_hash_id) if ts3 is not None and len(ts3) > 1 and len( obs_timeseries) > 0: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts3, OBS_TS=obs_timeseries) if obs_timeseries is not None: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_15_min_ts( newly_extracted_timeseries=ts3, expected_start=expected_start)[1:]) for i in range(len(obs_timeseries)): if obs_timeseries[i][1] == -99999: obs_timeseries[i][1] = 0 if obs_timeseries is not None and len(obs_timeseries) > 0: TS.insert_data(timeseries=obs_timeseries, tms_id=tms_id, upsert=True) TS.update_latest_obs(id_=tms_id, obs_end=(obs_timeseries[-1][1])) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating obs rainfalls in curw_sim.") finally: curw_obs_connection.close() destroy_Pool(pool=curw_sim_pool) destroy_Pool(pool=curw_obs_pool) logger.info("Process finished")
def update_rainfall_obs(flo2d_model, method, grid_interpolation, timestep, start_time, end_time): """ Update rainfall observations for flo2d models :param flo2d_model: flo2d model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param timestep: output timeseries timestep :return: """ obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') try: # Connect to the database curw_obs_pool = get_Pool(host=con_params.CURW_OBS_HOST, user=con_params.CURW_OBS_USERNAME, password=con_params.CURW_OBS_PASSWORD, port=con_params.CURW_OBS_PORT, db=con_params.CURW_OBS_DATABASE) curw_obs_connection = curw_obs_pool.connection() curw_sim_pool = get_Pool(host=con_params.CURW_SIM_HOST, user=con_params.CURW_SIM_USERNAME, password=con_params.CURW_SIM_PASSWORD, port=con_params.CURW_SIM_PORT, db=con_params.CURW_SIM_DATABASE) TS = Sim_Timeseries(pool=curw_sim_pool) # [hash_id, station_id, station_name, latitude, longitude] # active_obs_stations = read_csv(os.path.join(ROOT_DIR,'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv')) active_obs_stations = extract_active_curw_obs_rainfall_stations( start_time=start_time, end_time=end_time)[1:] flo2d_grids = read_csv( os.path.join(ROOT_DIR, 'grids/flo2d/{}m.csv'.format( flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)] stations_dict_for_obs = {} # keys: obs station id , value: hash id for obs_index in range(len(active_obs_stations)): stations_dict_for_obs[active_obs_stations[obs_index] [1]] = active_obs_stations[obs_index][0] # flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model) flo2d_obs_mapping = find_nearest_obs_stations_for_flo2d_stations( flo2d_stations_csv=os.path.join( ROOT_DIR, 'grids/flo2d/{}m.csv'.format(flo2d_model)), obs_stations=active_obs_stations, flo2d_model=flo2d_model) for flo2d_index in range(len(flo2d_grids)): lat = flo2d_grids[flo2d_index][2] lon = flo2d_grids[flo2d_index][1] cell_id = flo2d_grids[flo2d_index][0] meta_data = { 'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)), 'model': flo2d_model, 'method': method, 'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10)) } tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method')) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) print("grid_id:", cell_id) print("grid map:", flo2d_obs_mapping.get(cell_id)) obs1_station_id = flo2d_obs_mapping.get(cell_id)[0] obs2_station_id = flo2d_obs_mapping.get(cell_id)[1] obs3_station_id = flo2d_obs_mapping.get(cell_id)[2] obs_timeseries = [] if timestep == 5: if obs1_station_id != str(-1): obs1_hash_id = stations_dict_for_obs.get(obs1_station_id) ts = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs1_hash_id, end_time=end_time) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_5_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] if obs2_station_id != str(-1): obs2_hash_id = stations_dict_for_obs.get( obs2_station_id) ts2 = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs2_hash_id, end_time=end_time) if ts2 is not None and len(ts2) > 1: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts2, OBS_TS=obs_timeseries) if obs_timeseries is not None and len( obs_timeseries) > 0: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_5_min_ts( newly_extracted_timeseries=ts2, expected_start=expected_start)[1:]) # obs_start = ts2[-1][0] if obs3_station_id != str(-1): obs3_hash_id = stations_dict_for_obs.get( obs3_station_id) ts3 = extract_obs_rain_5_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs3_hash_id, end_time=end_time) if ts3 is not None and len(ts3) > 1 and len( obs_timeseries) > 0: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts3, OBS_TS=obs_timeseries) if obs_timeseries is not None: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_5_min_ts( newly_extracted_timeseries=ts3, expected_start=expected_start)[1:]) elif timestep == 15: if obs1_station_id != str(-1): obs1_hash_id = stations_dict_for_obs.get(obs1_station_id) ts = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs1_hash_id, end_time=end_time) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_15_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] if obs2_station_id != str(-1): obs2_hash_id = stations_dict_for_obs.get( obs2_station_id) ts2 = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs2_hash_id, end_time=end_time) if ts2 is not None and len(ts2) > 1: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts2, OBS_TS=obs_timeseries) if obs_timeseries is not None and len( obs_timeseries) > 0: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_15_min_ts( newly_extracted_timeseries=ts2, expected_start=expected_start)[1:]) # obs_start = ts2[-1][0] if obs3_station_id != str(-1): obs3_hash_id = stations_dict_for_obs.get( obs3_station_id) ts3 = extract_obs_rain_15_min_ts( connection=curw_obs_connection, start_time=obs_start, id=obs3_hash_id, end_time=end_time) if ts3 is not None and len(ts3) > 1 and len( obs_timeseries) > 0: obs_timeseries = fill_missing_values( newly_extracted_timeseries=ts3, OBS_TS=obs_timeseries) if obs_timeseries is not None: expected_start = obs_timeseries[-1][0] else: expected_start = obs_start obs_timeseries.extend( process_15_min_ts( newly_extracted_timeseries=ts3, expected_start=expected_start)[1:]) for i in range(len(obs_timeseries)): if obs_timeseries[i][1] == -99999: obs_timeseries[i][1] = 0 print("### obs timeseries length ###", len(obs_timeseries)) if obs_timeseries is not None and len( obs_timeseries) > 0 and obs_timeseries[-1][0] != end_time: obs_timeseries.append( [datetime.strptime(end_time, DATE_TIME_FORMAT), 0]) final_ts = process_continuous_ts(original_ts=obs_timeseries, expected_start=datetime.strptime( start_time, DATE_TIME_FORMAT), filling_value=0, timestep=timestep) if final_ts is not None and len(final_ts) > 0: TS.insert_data(timeseries=final_ts, tms_id=tms_id, upsert=True) TS.update_latest_obs(id_=tms_id, obs_end=(final_ts[-1][1])) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating obs rainfalls in curw_sim.") finally: curw_obs_connection.close() destroy_Pool(pool=curw_sim_pool) destroy_Pool(pool=curw_obs_pool) logger.info("Process finished")
# source details FLO2D_250_params = json.loads(open('flo2d_250.json').read()) FLO2D_150_params = json.loads(open('flo2d_150.json').read()) FLO2D_model = 'FLO2D' FLO2D_250_version = '250' FLO2D_150_version = '150' # unit details unit = 'm' unit_type = UnitType.getType('Instantaneous') # variable details variable = 'WaterLevel' # station details flo2d_250_grids = read_csv('flo2d_250m.csv') flo2d_150_grids = read_csv('flo2d_150m.csv') pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD, db=CURW_FCST_DATABASE) # ######## # # test # ######## # # USERNAME = "******" # PASSWORD = "******" # HOST = "127.0.0.1"
ROOT_DIRECTORY = '/home/uwcc-admin/curw_mike_data_handler' if __name__ == "__main__": set_db_config_file_path( os.path.join(ROOT_DIRECTORY, 'db_adapter_config.json')) try: ################################## # Initialize parameters for MIKE # ################################## # station details mike_stations = read_csv( os.path.join(ROOT_DIRECTORY, 'resources/mike_stations.csv')) pool = get_Pool(host=con_params.CURW_FCST_HOST, port=con_params.CURW_FCST_PORT, user=con_params.CURW_FCST_USERNAME, password=con_params.CURW_FCST_PASSWORD, db=con_params.CURW_FCST_DATABASE) for station in mike_stations: id = station[0] station_name = station[1] lat = station[2] lon = station[3] add_station(pool=pool, name=station_name, latitude="%.6f" % float(lat),
def update_rainfall_obs(curw_obs_pool, curw_sim_pool, flo2d_model, method, grid_interpolation, timestep, start_time, end_time): """ Update rainfall observations for flo2d models :param flo2d_model: flo2d model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param timestep: output timeseries timestep :return: """ # obs_start = datetime.strptime(start_time, '%Y-%m-%d %H:%M:%S') try: curw_obs_connection = curw_obs_pool.connection() # [hash_id, station_id, station_name, latitude, longitude] # active_obs_stations = read_csv(os.path.join(ROOT_DIR,'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv')) active_obs_stations = extract_active_curw_obs_rainfall_stations(curw_obs_pool=curw_obs_pool, start_time=start_time, end_time=end_time)[1:] flo2d_grids = read_csv(os.path.join(ROOT_DIR,'grids/flo2d/{}m.csv'.format(flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)] stations_dict_for_obs = { } # keys: obs station id , value: hash id for obs_index in range(len(active_obs_stations)): stations_dict_for_obs[active_obs_stations[obs_index][1]] = active_obs_stations[obs_index][0] # flo2d_obs_mapping = get_flo2d_cells_to_obs_grid_mappings(pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model) flo2d_obs_mapping = find_nearest_obs_stations_for_flo2d_stations( flo2d_stations_csv=os.path.join(ROOT_DIR,'grids/flo2d/{}m.csv'.format(flo2d_model)), obs_stations=active_obs_stations, flo2d_model=flo2d_model) # retrieve observed timeseries obs_df = pd.DataFrame() obs_df['time'] = pd.date_range(start=start_time, end=end_time, freq='5min') for obs_id in stations_dict_for_obs.keys(): ts = extract_obs_rain_5_min_ts(connection=curw_obs_connection, start_time=start_time, id=stations_dict_for_obs.get(obs_id), end_time=end_time) ts.insert(0, ['time', obs_id]) ts_df = list_of_lists_to_df_first_row_as_columns(ts) ts_df[obs_id] = ts_df[obs_id].astype('float64') obs_df = pd.merge(obs_df, ts_df, how="left", on='time') obs_df.set_index('time', inplace=True) obs_df['0'] = 0 if timestep == 15: obs_df = obs_df.resample('15min', label='right', closed='right').sum() TS = Sim_Timeseries(pool=curw_sim_pool) for flo2d_index in range(len(flo2d_grids)): lat = flo2d_grids[flo2d_index][2] lon = flo2d_grids[flo2d_index][1] cell_id = flo2d_grids[flo2d_index][0] meta_data = { 'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)), 'model': flo2d_model, 'method': method, 'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10)) } tms_id = TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method')) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) print(datetime.now().strftime(DATE_TIME_FORMAT)) print("grid_id:", cell_id) obs_station_ids = flo2d_obs_mapping.get(cell_id) if len(obs_station_ids) == 1: obs_ts_df = obs_df[obs_station_ids].to_frame(name='final') elif len(obs_station_ids) == 2: obs_ts_df = obs_df[obs_station_ids] obs_ts_df[obs_station_ids[0]] = obs_ts_df[obs_station_ids[0]].fillna(obs_ts_df[obs_station_ids[1]]) obs_ts_df['final'] = obs_ts_df[obs_station_ids[0]] elif len(obs_station_ids) == 3: obs_ts_df = obs_df[obs_station_ids] obs_ts_df[obs_station_ids[1]] = obs_ts_df[obs_station_ids[1]].fillna(obs_ts_df[obs_station_ids[2]]) obs_ts_df[obs_station_ids[0]] = obs_ts_df[obs_station_ids[0]].fillna(obs_ts_df[obs_station_ids[1]]) obs_ts_df['final'] = obs_ts_df[obs_station_ids[0]] else: obs_ts_df = obs_df['0'].to_frame(name='final') final_ts_df = obs_ts_df['final'].reset_index() final_ts_df['time'] = final_ts_df['time'].dt.strftime(DATE_TIME_FORMAT) final_ts = final_ts_df.values.tolist() if final_ts is not None and len(final_ts) > 0: TS.replace_data(timeseries=final_ts, tms_id=tms_id) TS.update_latest_obs(id_=tms_id, obs_end=(final_ts[-1][1])) except Exception as e: traceback.print_exc() logger.error("Exception occurred while updating obs rainfalls in curw_sim.") finally: curw_obs_connection.close() destroy_Pool(pool=curw_sim_pool) destroy_Pool(pool=curw_obs_pool) logger.info("Process finished")
def update_rainfall_obs(target_model, method, grid_interpolation, timestep): """ Update rainfall observations for flo2d models :param model: target model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param timestep: :return: """ now = datetime.now() OBS_START_STRING = (now + timedelta(hours=3)).strftime('%Y-%m-%d %H:00:00') OBS_START = datetime.strptime(OBS_START_STRING, '%Y-%m-%d %H:%M:%S') try: # Connect to the database curw_obs_pool = get_Pool(host=CURW_OBS_HOST, user=CURW_OBS_USERNAME, password=CURW_OBS_PASSWORD, port=CURW_OBS_PORT, db=CURW_OBS_DATABASE) curw_obs_connection = curw_obs_pool.connection() curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) TS = Timeseries(pool=curw_sim_pool) # [hash_id, station_id, station_name, latitude, longitude] active_obs_stations = read_csv( 'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv' ) obs_stations_dict = { } # keys: obs station id , value: [hash id, name, latitude, longitude] for obs_index in range(len(active_obs_stations)): obs_stations_dict[active_obs_stations[obs_index][1]] = [ active_obs_stations[obs_index][0], active_obs_stations[obs_index][2], active_obs_stations[obs_index][3], active_obs_stations[obs_index][4] ] for obs_id in obs_stations_dict.keys(): obs_start = OBS_START meta_data = { 'latitude': float('%.6f' % float(obs_stations_dict.get(obs_id)[2])), 'longitude': float('%.6f' % float(obs_stations_dict.get(obs_id)[3])), 'model': target_model, 'method': method, 'grid_id': 'rainfall_{}_{}_{}'.format(obs_id, obs_stations_dict.get(obs_id)[1], grid_interpolation) } tms_id = TS.get_timeseries_id_if_exists(meta_data=meta_data) if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) TS.update_grid_id(id_=tms_id, grid_id=meta_data['grid_id']) obs_end = TS.get_obs_end(id_=tms_id) if obs_end is not None: obs_start = obs_end obs_hash_id = obs_stations_dict.get(obs_id)[0] obs_timeseries = [] if timestep == 5: ts = extract_obs_rain_5_min_ts(connection=curw_obs_connection, start_time=obs_start, id=obs_hash_id) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_5_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] elif timestep == 15: ts = extract_obs_rain_15_min_ts(connection=curw_obs_connection, start_time=obs_start, id=obs_hash_id) if ts is not None and len(ts) > 1: obs_timeseries.extend( process_15_min_ts(newly_extracted_timeseries=ts, expected_start=obs_start)[1:]) # obs_start = ts[-1][0] # for i in range(len(obs_timeseries)): # if obs_timeseries[i][1] == -99999: # obs_timeseries[i][1] = 0 if obs_timeseries is not None and len(obs_timeseries) > 0: TS.insert_data(timeseries=obs_timeseries, tms_id=tms_id, upsert=True) TS.update_latest_obs(id_=tms_id, obs_end=(obs_timeseries[-1][1])) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating obs rainfalls in curw_sim.") finally: curw_obs_connection.close() destroy_Pool(pool=curw_sim_pool) destroy_Pool(pool=curw_obs_pool)
open( os.path.join( ROOT_DIRECTORY, 'res/flo2d_extract_stations/flo2d_150_v2.json')).read()) FLO2D_model = 'FLO2D' FLO2D_150_v2_version = '150_v2' # unit details unit = 'm' unit_type = UnitType.getType('Instantaneous') # variable details variable = 'WaterLevel' # station details flo2d_150_v2_grids = read_csv( os.path.join(ROOT_DIRECTORY, 'res/grids/flo2d_150_v2m.csv')) pool = get_Pool(host=con_params.CURW_FCST_HOST, port=con_params.CURW_FCST_PORT, user=con_params.CURW_FCST_USERNAME, password=con_params.CURW_FCST_PASSWORD, db=con_params.CURW_FCST_DATABASE) add_source(pool=pool, model=FLO2D_model, version=FLO2D_150_v2_version, parameters=FLO2D_150_v2_params) # add_variable(pool=pool, variable=variable) # add_unit(pool=pool, unit=unit, unit_type=unit_type) # add flo2d 150 v2 output stations
curw_obs_pool = get_Pool(host=CURW_OBS_HOST, user=CURW_OBS_USERNAME, password=CURW_OBS_PASSWORD, port=CURW_OBS_PORT, db=CURW_OBS_DATABASE) connection = curw_obs_pool.connection() curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) # [station_name,latitude,longitude,target] extract_stations = read_csv('grids/tide_stations/extract_stations.csv') extract_stations_dict = { } # keys: station_name , value: [latitude, longitude, target_model] for obs_index in range(len(extract_stations)): extract_stations_dict[extract_stations[obs_index][0]] = [ extract_stations[obs_index][1], extract_stations[obs_index][2], extract_stations[obs_index][3] ] methods = [] methods.append(MethodEnum.getAbbreviation(MethodEnum.TSF)) methods.append(MethodEnum.getAbbreviation(MethodEnum.MGF)) for station_name in extract_stations_dict.keys():
return processed_timeseries else: return None if __name__ == "__main__": try: curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) # [station_name,latitude,longitude,target] extract_stations = read_csv('grids/tide_stations/extract_stations.csv') extract_stations_dict = { } # keys: station_name , value: [latitude, longitude, target_method] for obs_index in range(len(extract_stations)): extract_stations_dict[extract_stations[obs_index][0]] = [ extract_stations[obs_index][1], extract_stations[obs_index][2], extract_stations[obs_index][3] ] for station_name in extract_stations_dict.keys(): fcst_station_name = None methods = [] if station_name in ('colombo', 'mattakkuliya'): fcst_station_name = 'colombo' #temporary#
curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) curw_fcst_pool = get_Pool(host=CURW_FCST_HOST, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE) TS = Timeseries(pool=curw_sim_pool) # [station_name,latitude,longitude,target,model,version,sim_tag,station] extract_stations = read_csv( 'grids/discharge_stations/mike_stations.csv') for i in range(len(extract_stations)): station_name = extract_stations[i][0] latitude = extract_stations[i][1] longitude = extract_stations[i][2] target_model = extract_stations[i][3] if station_name in ('ambatale'): method = MethodEnum.getAbbreviation(MethodEnum.MME) else: continue ## skip the current station and move to next iteration meta_data = { 'latitude': float('%.6f' % float(latitude)), 'longitude': float('%.6f' % float(longitude)),
from db_adapter.curw_sim.timeseries.waterlevel import Timeseries from db_adapter.curw_sim.timeseries import MethodEnum from db_adapter.curw_sim.common import fill_ts_missing_entries from db_adapter.logger import logger if __name__ == "__main__": try: print("Fill missing values in hanwella waterlevel series") curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) method = MethodEnum.getAbbreviation(MethodEnum.SF) # [station_name,latitude,longitude,target] extract_stations = read_csv('grids/waterlevel_stations/extract_stations.csv') extract_stations_dict = {} # keys: station_name , value: [latitude, longitude, target_method] for obs_index in range(len(extract_stations)): extract_stations_dict[extract_stations[obs_index][0]] = [extract_stations[obs_index][1], extract_stations[obs_index][2], extract_stations[obs_index][3]] station_name = 'hanwella' meta_data = { 'latitude': float('%.6f' % float(extract_stations_dict.get(station_name)[0])), 'longitude': float('%.6f' % float(extract_stations_dict.get(station_name)[1])), 'model': extract_stations_dict.get(station_name)[2], 'method': method, 'grid_id': 'waterlevel_{}'.format(station_name) }
def update_rainfall_fcsts(target_model, method, grid_interpolation, model_list, timestep): """ Update rainfall forecasts for flo2d models :param target_model: target model for which input ins prepared :param method: value interpolation method :param grid_interpolation: grid interpolation method :param model_list: list of forecast model and their versions used to calculate the rainfall e.g.: [["WRF_E", "4.0", "evening_18hrs"],["WRF_SE", "v4", ,"evening_18hrs"],["WRF_Ensemble", "4.0", ,"MME"]] :param timestep: output timeseries timestep :return: """ try: # Connect to the database curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) curw_fcst_pool = get_Pool(host=CURW_FCST_HOST, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE) Sim_TS = Sim_Timeseries(pool=curw_sim_pool) Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool) # [hash_id, station_id, station_name, latitude, longitude] active_obs_stations = read_csv( 'grids/obs_stations/rainfall/curw_active_rainfall_obs_stations.csv' ) obs_stations_dict = { } # keys: obs station id , value: [name, latitude, longitude] for obs_index in range(len(active_obs_stations)): obs_stations_dict[active_obs_stations[obs_index][1]] = [ active_obs_stations[obs_index][2], active_obs_stations[obs_index][3], active_obs_stations[obs_index][4] ] obs_d03_mapping = get_obs_to_d03_grid_mappings_for_rainfall( pool=curw_sim_pool, grid_interpolation=grid_interpolation) for obs_id in obs_stations_dict.keys(): meta_data = { 'latitude': float('%.6f' % float(obs_stations_dict.get(obs_id)[1])), 'longitude': float('%.6f' % float(obs_stations_dict.get(obs_id)[2])), 'model': target_model, 'method': method, 'grid_id': 'rainfall_{}_{}_{}'.format(obs_id, obs_stations_dict.get(obs_id)[0], grid_interpolation) } tms_id = Sim_TS.get_timeseries_id_if_exists(meta_data=meta_data) if tms_id is None: tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id Sim_TS.insert_run(meta_data=meta_data) obs_end = Sim_TS.get_obs_end(id_=tms_id) fcst_timeseries = [] for i in range(len(model_list)): source_id = get_source_id(pool=curw_fcst_pool, model=model_list[i][0], version=model_list[i][1]) sim_tag = model_list[i][2] coefficient = model_list[i][3] temp_timeseries = [] if timestep == 5: if obs_end is not None: temp_timeseries = convert_15_min_ts_to_5_mins_ts( newly_extracted_timeseries=Fcst_TS. get_latest_timeseries(sim_tag=sim_tag, station_id=obs_d03_mapping. get(meta_data['grid_id'])[0], start=obs_end, source_id=source_id, variable_id=1, unit_id=1)) else: temp_timeseries = convert_15_min_ts_to_5_mins_ts( newly_extracted_timeseries=Fcst_TS. get_latest_timeseries(sim_tag=sim_tag, station_id=obs_d03_mapping. get(meta_data['grid_id'])[0], source_id=source_id, variable_id=1, unit_id=1)) elif timestep == 15: if obs_end is not None: temp_timeseries = Fcst_TS.get_latest_timeseries( sim_tag=sim_tag, station_id=obs_d03_mapping.get( meta_data['grid_id'])[0], start=obs_end, source_id=source_id, variable_id=1, unit_id=1) else: temp_timeseries = Fcst_TS.get_latest_timeseries( sim_tag=sim_tag, station_id=obs_d03_mapping.get( meta_data['grid_id'])[0], source_id=source_id, variable_id=1, unit_id=1) if coefficient != 1: for j in range(len(temp_timeseries)): temp_timeseries[j][1] = float( temp_timeseries[j][1]) * coefficient if i == 0: fcst_timeseries = temp_timeseries else: fcst_timeseries = append_value_for_timestamp( existing_ts=fcst_timeseries, new_ts=temp_timeseries) sum_timeseries = summed_timeseries(fcst_timeseries) for i in range(len(sum_timeseries)): if float(sum_timeseries[i][1]) < 0: sum_timeseries[i][1] = 0 if sum_timeseries is not None and len(sum_timeseries) > 0: Sim_TS.insert_data(timeseries=sum_timeseries, tms_id=tms_id, upsert=True) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating fcst rainfalls in curw_sim.") finally: destroy_Pool(curw_sim_pool) destroy_Pool(curw_fcst_pool)
if __name__=="__main__": try: curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) curw_fcst_pool = get_Pool(host=CURW_FCST_HOST, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE) TS = Timeseries(pool=curw_sim_pool) # [station_name,latitude,longitude,target,model,version,sim_tag,station] extract_stations = read_csv('grids/discharge_stations/flo2d_stations.csv') for i in range(len(extract_stations)): station_name = extract_stations[i][0] latitude = extract_stations[i][1] longitude = extract_stations[i][2] target_model = extract_stations[i][3] method = extract_stations[i][4] meta_data = { 'latitude': float('%.6f' % float(latitude)), 'longitude': float('%.6f' % float(longitude)), 'model': target_model, 'method': method, 'grid_id': 'discharge_{}'.format(station_name) }
def update_rainfall_fcsts(flo2d_model, method, grid_interpolation, model_list, timestep): """ Update rainfall forecasts for flo2d models :param flo2d_model: flo2d model :param method: value interpolation method :param grid_interpolation: grid interpolation method :param model_list: list of forecast model and their versions used to calculate the rainfall e.g.: [["WRF_E", "v4"],["WRF_SE", "v4"]] :param timestep: output timeseries timestep :return: """ try: # Connect to the database curw_sim_pool = get_Pool(host=CURW_SIM_HOST, user=CURW_SIM_USERNAME, password=CURW_SIM_PASSWORD, port=CURW_SIM_PORT, db=CURW_SIM_DATABASE) curw_fcst_pool = get_Pool(host=CURW_FCST_HOST, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD, port=CURW_FCST_PORT, db=CURW_FCST_DATABASE) Sim_TS = Sim_Timeseries(pool=curw_sim_pool) Fcst_TS = Fcst_Timeseries(pool=curw_fcst_pool) flo2d_grids = read_csv('grids/flo2d/{}m.csv'.format( flo2d_model)) # [Grid_ ID, X(longitude), Y(latitude)] flo2d_wrf_mapping = get_flo2d_cells_to_wrf_grid_mappings( pool=curw_sim_pool, grid_interpolation=grid_interpolation, flo2d_model=flo2d_model) for flo2d_index in range(len(flo2d_grids)): # len(flo2d_grids) lat = flo2d_grids[flo2d_index][2] lon = flo2d_grids[flo2d_index][1] cell_id = flo2d_grids[flo2d_index][0] meta_data = { 'latitude': float('%.6f' % float(lat)), 'longitude': float('%.6f' % float(lon)), 'model': flo2d_model, 'method': method, 'grid_id': '{}_{}_{}'.format(flo2d_model, grid_interpolation, (str(cell_id)).zfill(10)) } tms_id = Sim_TS.get_timeseries_id(grid_id=meta_data.get('grid_id'), method=meta_data.get('method')) if tms_id is None: tms_id = Sim_TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id Sim_TS.insert_run(meta_data=meta_data) obs_end = Sim_TS.get_obs_end(id_=tms_id) fcst_timeseries = [] for i in range(len(model_list)): source_id = get_source_id(pool=curw_fcst_pool, model=model_list[i][0], version=model_list[i][1]) sim_tag = model_list[i][2] coefficient = model_list[i][3] temp_timeseries = [] if timestep == 5: if obs_end is not None: temp_timeseries = convert_15_min_ts_to_5_mins_ts( newly_extracted_timeseries=Fcst_TS. get_latest_timeseries(sim_tag=sim_tag, station_id=flo2d_wrf_mapping. get(meta_data['grid_id']), start=obs_end, source_id=source_id, variable_id=1, unit_id=1)) else: temp_timeseries = convert_15_min_ts_to_5_mins_ts( newly_extracted_timeseries=Fcst_TS. get_latest_timeseries(sim_tag=sim_tag, station_id=flo2d_wrf_mapping. get(meta_data['grid_id']), source_id=source_id, variable_id=1, unit_id=1)) elif timestep == 15: if obs_end is not None: temp_timeseries = Fcst_TS.get_latest_timeseries( sim_tag=sim_tag, station_id=flo2d_wrf_mapping.get( meta_data['grid_id']), start=obs_end, source_id=source_id, variable_id=1, unit_id=1) else: temp_timeseries = Fcst_TS.get_latest_timeseries( sim_tag=sim_tag, station_id=flo2d_wrf_mapping.get( meta_data['grid_id']), source_id=source_id, variable_id=1, unit_id=1) if coefficient != 1: for j in range(len(temp_timeseries)): temp_timeseries[j][1] = float( temp_timeseries[j][1]) * coefficient if i == 0: fcst_timeseries = temp_timeseries else: fcst_timeseries = append_value_for_timestamp( existing_ts=fcst_timeseries, new_ts=temp_timeseries) sum_timeseries = summed_timeseries(fcst_timeseries) for i in range(len(sum_timeseries)): if float(sum_timeseries[i][1]) < 0: sum_timeseries[i][1] = 0 if sum_timeseries is not None and len(sum_timeseries) > 0: Sim_TS.insert_data(timeseries=sum_timeseries, tms_id=tms_id, upsert=True) except Exception as e: traceback.print_exc() logger.error( "Exception occurred while updating fcst rainfalls in curw_sim.") finally: destroy_Pool(curw_sim_pool) destroy_Pool(curw_fcst_pool)
# Initialize parameters for DISTRIBUTED HECHMS # ################################################ # source details model = 'HECHMS' version = 'distributed' # unit details unit = 'm3/s' unit_type = UnitType.getType('Instantaneous') # variable details variable = 'Discharge' # station details hechms_stations = read_csv('hechms_stations.csv') pool = get_Pool(host=CURW_FCST_HOST, port=CURW_FCST_PORT, user=CURW_FCST_USERNAME, password=CURW_FCST_PASSWORD, db=CURW_FCST_DATABASE) # ######## # # test # ######## # USERNAME = "******" # PASSWORD = "******" # HOST = "127.0.0.1" # PORT = 3306 # DATABASE = "curw_fcst" # pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE)
def update_waterlevel_obs(obs_connection, curw_sim_pool, flo2d_model, method, timestep, start_time, end_time): try: # [station_name,latitude,longitude,target] extract_stations = read_csv( 'grids/waterlevel_stations/extract_stations.csv') extract_stations_dict = { } # keys: target_model , value: [latitude, longitude, station_name] # older version ::: keys: station_name , value: [latitude, longitude, target_model] for obs_index in range(len(extract_stations)): extract_stations_dict[extract_stations[obs_index][3]] = [ extract_stations[obs_index][1], extract_stations[obs_index][2], extract_stations[obs_index][0] ] station_name = extract_stations_dict.get(flo2d_model)[2] meta_data = { 'latitude': float('%.6f' % float(extract_stations_dict.get(flo2d_model)[0])), 'longitude': float('%.6f' % float(extract_stations_dict.get(flo2d_model)[1])), 'model': flo2d_model, 'method': method, 'grid_id': 'waterlevel_{}'.format(station_name) } TS = Timeseries(pool=curw_sim_pool) tms_id = TS.get_timeseries_id_if_exists(meta_data=meta_data) ranwala_ts = [] if tms_id is None: tms_id = TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id TS.insert_run(meta_data=meta_data) with obs_connection.cursor() as cursor1: cursor1.callproc('getWL', (RANWALA_WL_ID, start_time, end_time)) results = cursor1.fetchall() for result in results: ranwala_ts.append([result.get('time'), result.get('value')]) interpolated_ranwala_ts = fill_ts_missing_entries( start=start_time, end=end_time, timeseries=ranwala_ts, interpolation_method='linear', timestep=60) estimated_wl_ts = [] if station_name == 'hanwella': estimated_wl_ts = calculate_hanwella_wl_from_ranwala( interpolated_ranwala_ts) elif station_name == 'glencourse': estimated_wl_ts = calculate_glencourse_wl_from_ranwala( interpolated_ranwala_ts) if estimated_wl_ts is not None and len(estimated_wl_ts) > 0: TS.insert_data(timeseries=estimated_wl_ts, tms_id=tms_id, upsert=True) except Exception as e: traceback.print_exc()
shape_file_path = os.path.join(ROOT_DIR, 'shape_files/250m_model/250m_model.shp') output_shape_file_path = os.path.join( ROOT_DIR, 'shape_files/output_temp', "{}_out_shp.shp".format( (datetime.now()).strftime("%Y-%m-%d_%H-%M-%S"))) polygons = get_voronoi_polygons(points_dict=points_dict, shape_file=shape_file_path, shape_attribute=['Id', 0], output_shape_file=output_shape_file_path, add_total_area=True) flo2d_model = "flo2d_250" flo2d_grids = read_csv( os.path.join(ROOT_DIR, 'grids/flo2d/{}m.csv'.format( flo2d_model))) # [Grid_ ID, X(longitude), Y(latitude)] for grid in flo2d_grids: point = Point(float(grid[1]), float(grid[2])) for index, row in polygons.iterrows(): polygon = polygons.iloc[index]['geometry'] if point.within(polygon): grid.append(polygons.iloc[index]['id']) continue print(flo2d_grids) count = 0 for grid in flo2d_grids:
def update_discharge_obs(curw_sim_pool, flo2d_model, method, timestep, start_time, end_time): try: discharge_TS = DTimeseries(pool=curw_sim_pool) waterlevel_TS = WLTimeseries(pool=curw_sim_pool) # [station_name,latitude,longitude,target] extract_stations = read_csv( 'grids/discharge_stations/flo2d_stations.csv') extract_stations_dict = { } # keys: target_model , value: [latitude, longitude, station_name] # keys: station_name , value: [latitude, longitude, target_model] for obs_index in range(len(extract_stations)): extract_stations_dict[extract_stations[obs_index][3]] = [ extract_stations[obs_index][1], extract_stations[obs_index][2], extract_stations[obs_index][0] ] station_name = extract_stations_dict.get(flo2d_model)[2] meta_data = { 'latitude': float('%.6f' % float(extract_stations_dict.get(flo2d_model)[0])), 'longitude': float('%.6f' % float(extract_stations_dict.get(flo2d_model)[1])), 'model': flo2d_model, 'method': method, 'grid_id': 'discharge_{}'.format(station_name) } wl_meta_data = { 'latitude': float('%.6f' % float(extract_stations_dict.get(flo2d_model)[0])), 'longitude': float('%.6f' % float(extract_stations_dict.get(flo2d_model)[1])), 'model': flo2d_model, 'method': method, 'grid_id': 'waterlevel_{}'.format(station_name) } tms_id = discharge_TS.get_timeseries_id_if_exists(meta_data=meta_data) wl_tms_id = waterlevel_TS.get_timeseries_id_if_exists( meta_data=wl_meta_data) if wl_tms_id is None: print("Warning!!! {} waterlevel timeseries doesn't exist.".format( station_name)) exit(1) timeseries = [] if tms_id is None: tms_id = discharge_TS.generate_timeseries_id(meta_data=meta_data) meta_data['id'] = tms_id discharge_TS.insert_run(meta_data=meta_data) wl_timeseries = waterlevel_TS.get_timeseries(id_=wl_tms_id, start_date=start_time, end_date=end_time) estimated_discharge_ts = [] if station_name == 'hanwella': estimated_discharge_ts = calculate_hanwella_discharge( wl_timeseries) elif station_name == 'glencourse': estimated_discharge_ts = calculate_glencourse_discharge( wl_timeseries) if estimated_discharge_ts is not None and len( estimated_discharge_ts) > 0: discharge_TS.insert_data(timeseries=estimated_discharge_ts, tms_id=tms_id, upsert=True) except Exception as e: traceback.print_exc()