def read_netcdf_file(pool, rainc_net_cdf_file_path, rainnc_net_cdf_file_path, source_id, variable_id, unit_id, tms_meta, fgt): """ :param pool: database connection pool :param rainc_net_cdf_file_path: :param rainnc_net_cdf_file_path: :param source_id: :param variable_id: :param unit_id: :param tms_meta: :return: rainc_unit_info: mm lat_unit_info: degree_north time_unit_info: minutes since 2019-04-02T18:00:00 """ if not os.path.exists(rainc_net_cdf_file_path): logger.warning('no rainc netcdf') print('no rainc netcdf') elif not os.path.exists(rainnc_net_cdf_file_path): logger.warning('no rainnc netcdf') print('no rainnc netcdf') else: """ RAINC netcdf data extraction """ nc_fid = Dataset(rainc_net_cdf_file_path, mode='r') time_unit_info = nc_fid.variables['XTIME'].units time_unit_info_list = time_unit_info.split(' ') lats = nc_fid.variables['XLAT'][0, :, 0] lons = nc_fid.variables['XLONG'][0, 0, :] lon_min = lons[0].item() lat_min = lats[0].item() lon_max = lons[-1].item() lat_max = lats[-1].item() print('[lon_min, lat_min, lon_max, lat_max] :', [lon_min, lat_min, lon_max, lat_max]) lat_inds = np.where((lats >= lat_min) & (lats <= lat_max)) lon_inds = np.where((lons >= lon_min) & (lons <= lon_max)) rainc = nc_fid.variables['RAINC'][:, lat_inds[0], lon_inds[0]] """ RAINNC netcdf data extraction """ nnc_fid = Dataset(rainnc_net_cdf_file_path, mode='r') rainnc = nnc_fid.variables['RAINNC'][:, lat_inds[0], lon_inds[0]] # times = list(set(nc_fid.variables['XTIME'][:])) # set is used to remove duplicates times = nc_fid.variables['XTIME'][:] # ts_start_date = datetime.strptime(time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S') # ts_end_date = datetime.strptime(time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S') + timedelta( # minutes=float(sorted(set(times))[-2])) # start_date = datetime_utc_to_lk(ts_start_date, shift_mins=0).strftime('%Y-%m-%d %H:%M:%S') # end_date = datetime_utc_to_lk(ts_end_date, shift_mins=0).strftime('%Y-%m-%d %H:%M:%S') start_date = fgt end_date = fgt prcp = rainc + rainnc nc_fid.close() nnc_fid.close() diff = get_per_time_slot_values(prcp) width = len(lons) height = len(lats) ts = Timeseries(pool) for y in range(height): for x in range(width): lat = float('%.6f' % lats[y]) lon = float('%.6f' % lons[x]) tms_meta['latitude'] = str(lat) tms_meta['longitude'] = str(lon) station_prefix = '{}_{}'.format(lat, lon) station_id = wrf_v3_stations.get(station_prefix) if station_id is None: add_station(pool=pool, name=station_prefix, latitude=lat, longitude=lon, description="WRF point", station_type=StationEnum.WRF) tms_id = ts.get_timeseries_id_if_exists(tms_meta) logger.info("Existing timeseries id: {}".format(tms_id)) if tms_id is None: tms_id = ts.generate_timeseries_id(tms_meta) logger.info('HASH SHA256 created: {}'.format(tms_id)) run = (tms_id, tms_meta['sim_tag'], start_date, end_date, station_id, source_id, variable_id, unit_id) try: ts.insert_run(run) except Exception: logger.error( "Exception occurred while inserting run entry {}". format(run)) traceback.print_exc() else: # ts.update_latest_fgt(id_=tms_id, fgt=fgt) ts.update_start_date(id_=tms_id, start_date=fgt) # to run backward data_list = [] # generate timeseries for each station for i in range(len(diff)): ts_time = datetime.strptime( time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S') + timedelta( minutes=times[i].item()) t = datetime_utc_to_lk(ts_time, shift_mins=0) data_list.append([ tms_id, t.strftime('%Y-%m-%d %H:%M:%S'), fgt, float(diff[i, y, x]) ]) push_rainfall_to_db(ts=ts, ts_data=data_list)
# ######## # USERNAME = "******" # PASSWORD = "******" # HOST = "127.0.0.1" # PORT = 3306 # DATABASE = "curw_fcst" # pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE) add_source(pool=pool, model=model, version=version, parameters=None) # add_variable(pool=pool, variable=variable) # add_unit(pool=pool, unit=unit, unit_type=unit_type) # add hechms output stations for i in range(len(hechms_stations)): station_name = hechms_stations[i][0] lat = hechms_stations[i][1] lon = hechms_stations[i][2] add_station(pool=pool, name=station_name, latitude="%.6f" % float(lat), longitude="%.6f" % float(lon), station_type=StationEnum.HECHMS, description="hecHMS output station") except Exception: logger.info("Initialization process failed.") traceback.print_exc() finally: logger.info("Initialization process finished.") destroy_Pool(pool=pool)
# station details mike_stations = read_csv( os.path.join(ROOT_DIRECTORY, 'resources/mike_stations.csv')) pool = get_Pool(host=con_params.CURW_FCST_HOST, port=con_params.CURW_FCST_PORT, user=con_params.CURW_FCST_USERNAME, password=con_params.CURW_FCST_PASSWORD, db=con_params.CURW_FCST_DATABASE) for station in mike_stations: id = station[0] station_name = station[1] lat = station[2] lon = station[3] add_station(pool=pool, name=station_name, latitude="%.6f" % float(lat), longitude="%.6f" % float(lon), station_type=StationEnum.MIKE11, description="mike_station") destroy_Pool(pool=pool) except Exception: print("Initialization process failed.") traceback.print_exc() finally: print("Initialization process finished.")
def read_netcdf_file(pool, rainnc_net_cdf_file_path, tms_meta): """ :param pool: database connection pool :param rainnc_net_cdf_file_path: :param source_id: :param variable_id: :param unit_id: :param tms_meta: :return: rainc_unit_info: mm lat_unit_info: degree_north time_unit_info: minutes since 2019-04-02T18:00:00 """ if not os.path.exists(rainnc_net_cdf_file_path): logger.warning( 'no rainnc netcdf :: {}'.format(rainnc_net_cdf_file_path)) return else: """ RAINNC netcdf data extraction """ fgt = get_file_last_modified_time(rainnc_net_cdf_file_path) nnc_fid = Dataset(rainnc_net_cdf_file_path, mode='r') time_unit_info = nnc_fid.variables['XTIME'].units time_unit_info_list = time_unit_info.split(' ') lats = nnc_fid.variables['XLAT'][0, :, 0] lons = nnc_fid.variables['XLONG'][0, 0, :] lon_min = lons[0].item() lat_min = lats[0].item() lon_max = lons[-1].item() lat_max = lats[-1].item() lat_inds = np.where((lats >= lat_min) & (lats <= lat_max)) lon_inds = np.where((lons >= lon_min) & (lons <= lon_max)) rainnc = nnc_fid.variables['RAINNC'][:, lat_inds[0], lon_inds[0]] times = nnc_fid.variables['XTIME'][:] start_date = fgt end_date = fgt nnc_fid.close() diff = get_per_time_slot_values(rainnc) width = len(lons) height = len(lats) ts = Timeseries(pool) for y in range(height): for x in range(width): lat = float('%.6f' % lats[y]) lon = float('%.6f' % lons[x]) tms_meta['latitude'] = str(lat) tms_meta['longitude'] = str(lon) station_prefix = 'wrf_{}_{}'.format(lat, lon) station_id = wrf_v3_stations.get(station_prefix) if station_id is None: add_station(pool=pool, name=station_prefix, latitude=lat, longitude=lon, description="WRF point", station_type=StationEnum.WRF) station_id = get_station_id(pool=pool, latitude=lat, longitude=lon, station_type=StationEnum.WRF) tms_id = ts.get_timeseries_id_if_exists(tms_meta) if tms_id is None: tms_id = ts.generate_timeseries_id(tms_meta) run_meta = { 'tms_id': tms_id, 'sim_tag': tms_meta['sim_tag'], 'start_date': start_date, 'end_date': end_date, 'station_id': station_id, 'source_id': tms_meta['source_id'], 'unit_id': tms_meta['unit_id'], 'variable_id': tms_meta['variable_id'] } try: ts.insert_run(run_meta) except Exception: logger.error( "Exception occurred while inserting run entry {}". format(run_meta)) traceback.print_exc() data_list = [] # generate timeseries for each station for i in range(len(diff)): ts_time = datetime.strptime( time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S') + timedelta( minutes=times[i + 1].item()) t = datetime_utc_to_lk(ts_time, shift_mins=0) data_list.append([ tms_id, t.strftime('%Y-%m-%d %H:%M:%S'), fgt, float(diff[i, y, x]) ]) push_rainfall_to_db(ts=ts, ts_data=data_list, tms_id=tms_id, fgt=fgt)
version=FLO2D_150_version, parameters=FLO2D_150_params) add_variable(pool=pool, variable=variable) add_unit(pool=pool, unit=unit, unit_type=unit_type) # add flo2d 250 output stations channel_cell_map_250 = FLO2D_250_params.get('CHANNEL_CELL_MAP') for channel_cell_map_250_key in channel_cell_map_250.keys(): add_station( pool=pool, name="{}_{}".format( channel_cell_map_250_key, channel_cell_map_250.get(channel_cell_map_250_key)), latitude="%.6f" % float(flo2d_250_grids[int(channel_cell_map_250_key) - 1][2]), longitude="%.6f" % float(flo2d_250_grids[int(channel_cell_map_250_key) - 1][1]), station_type=StationEnum.FLO2D_250, description="{}_channel_cell_map_element".format(FLO2D_250)) flood_plain_cell_map_250 = FLO2D_250_params.get('FLOOD_PLAIN_CELL_MAP') for flood_plain_cell_map_250_key in flood_plain_cell_map_250.keys(): add_station( pool=pool, name="{}_{}".format( flood_plain_cell_map_250_key, flood_plain_cell_map_250.get( flood_plain_cell_map_250_key)),