def read_netcdf_file(pool, rainc_net_cdf_file_path, rainnc_net_cdf_file_path,
source_id, variable_id, unit_id, tms_meta, fgt):
"""
:param pool: database connection pool
:param rainc_net_cdf_file_path:
:param rainnc_net_cdf_file_path:
:param source_id:
:param variable_id:
:param unit_id:
:param tms_meta:
:return:
rainc_unit_info: mm
lat_unit_info: degree_north
time_unit_info: minutes since 2019-04-02T18:00:00
"""
if not os.path.exists(rainc_net_cdf_file_path):
logger.warning('no rainc netcdf')
print('no rainc netcdf')
elif not os.path.exists(rainnc_net_cdf_file_path):
logger.warning('no rainnc netcdf')
print('no rainnc netcdf')
else:
"""
RAINC netcdf data extraction
"""
nc_fid = Dataset(rainc_net_cdf_file_path, mode='r')
time_unit_info = nc_fid.variables['XTIME'].units
time_unit_info_list = time_unit_info.split(' ')
lats = nc_fid.variables['XLAT'][0, :, 0]
lons = nc_fid.variables['XLONG'][0, 0, :]
lon_min = lons[0].item()
lat_min = lats[0].item()
lon_max = lons[-1].item()
lat_max = lats[-1].item()
print('[lon_min, lat_min, lon_max, lat_max] :',
[lon_min, lat_min, lon_max, lat_max])
lat_inds = np.where((lats >= lat_min) & (lats <= lat_max))
lon_inds = np.where((lons >= lon_min) & (lons <= lon_max))
rainc = nc_fid.variables['RAINC'][:, lat_inds[0], lon_inds[0]]
"""
RAINNC netcdf data extraction
"""
nnc_fid = Dataset(rainnc_net_cdf_file_path, mode='r')
rainnc = nnc_fid.variables['RAINNC'][:, lat_inds[0], lon_inds[0]]
# times = list(set(nc_fid.variables['XTIME'][:])) # set is used to remove duplicates
times = nc_fid.variables['XTIME'][:]
# ts_start_date = datetime.strptime(time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S')
# ts_end_date = datetime.strptime(time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S') + timedelta(
# minutes=float(sorted(set(times))[-2]))
# start_date = datetime_utc_to_lk(ts_start_date, shift_mins=0).strftime('%Y-%m-%d %H:%M:%S')
# end_date = datetime_utc_to_lk(ts_end_date, shift_mins=0).strftime('%Y-%m-%d %H:%M:%S')
start_date = fgt
end_date = fgt
prcp = rainc + rainnc
nc_fid.close()
nnc_fid.close()
diff = get_per_time_slot_values(prcp)
width = len(lons)
height = len(lats)
ts = Timeseries(pool)
for y in range(height):
for x in range(width):
lat = float('%.6f' % lats[y])
lon = float('%.6f' % lons[x])
tms_meta['latitude'] = str(lat)
tms_meta['longitude'] = str(lon)
station_prefix = '{}_{}'.format(lat, lon)
station_id = wrf_v3_stations.get(station_prefix)
if station_id is None:
add_station(pool=pool,
name=station_prefix,
latitude=lat,
longitude=lon,
description="WRF point",
station_type=StationEnum.WRF)
tms_id = ts.get_timeseries_id_if_exists(tms_meta)
logger.info("Existing timeseries id: {}".format(tms_id))
if tms_id is None:
tms_id = ts.generate_timeseries_id(tms_meta)
logger.info('HASH SHA256 created: {}'.format(tms_id))
run = (tms_id, tms_meta['sim_tag'], start_date, end_date,
station_id, source_id, variable_id, unit_id)
try:
ts.insert_run(run)
except Exception:
logger.error(
"Exception occurred while inserting run entry {}".
format(run))
traceback.print_exc()
else:
# ts.update_latest_fgt(id_=tms_id, fgt=fgt)
ts.update_start_date(id_=tms_id,
start_date=fgt) # to run backward
data_list = []
# generate timeseries for each station
for i in range(len(diff)):
ts_time = datetime.strptime(
time_unit_info_list[2],
'%Y-%m-%dT%H:%M:%S') + timedelta(
minutes=times[i].item())
t = datetime_utc_to_lk(ts_time, shift_mins=0)
data_list.append([
tms_id,
t.strftime('%Y-%m-%d %H:%M:%S'), fgt,
float(diff[i, y, x])
])
push_rainfall_to_db(ts=ts, ts_data=data_list)
# ########
# USERNAME = "******"
# PASSWORD = "******"
# HOST = "127.0.0.1"
# PORT = 3306
# DATABASE = "curw_fcst"
# pool = get_Pool(host=HOST, port=PORT, user=USERNAME, password=PASSWORD, db=DATABASE)
add_source(pool=pool, model=model, version=version, parameters=None)
# add_variable(pool=pool, variable=variable)
# add_unit(pool=pool, unit=unit, unit_type=unit_type)
# add hechms output stations
for i in range(len(hechms_stations)):
station_name = hechms_stations[i][0]
lat = hechms_stations[i][1]
lon = hechms_stations[i][2]
add_station(pool=pool, name=station_name, latitude="%.6f" % float(lat), longitude="%.6f" % float(lon),
station_type=StationEnum.HECHMS, description="hecHMS output station")
except Exception:
logger.info("Initialization process failed.")
traceback.print_exc()
finally:
logger.info("Initialization process finished.")
destroy_Pool(pool=pool)
# station details
mike_stations = read_csv(
os.path.join(ROOT_DIRECTORY, 'resources/mike_stations.csv'))
pool = get_Pool(host=con_params.CURW_FCST_HOST,
port=con_params.CURW_FCST_PORT,
user=con_params.CURW_FCST_USERNAME,
password=con_params.CURW_FCST_PASSWORD,
db=con_params.CURW_FCST_DATABASE)
for station in mike_stations:
id = station[0]
station_name = station[1]
lat = station[2]
lon = station[3]
add_station(pool=pool,
name=station_name,
latitude="%.6f" % float(lat),
longitude="%.6f" % float(lon),
station_type=StationEnum.MIKE11,
description="mike_station")
destroy_Pool(pool=pool)
except Exception:
print("Initialization process failed.")
traceback.print_exc()
finally:
print("Initialization process finished.")
def read_netcdf_file(pool, rainnc_net_cdf_file_path, tms_meta):
"""
:param pool: database connection pool
:param rainnc_net_cdf_file_path:
:param source_id:
:param variable_id:
:param unit_id:
:param tms_meta:
:return:
rainc_unit_info: mm
lat_unit_info: degree_north
time_unit_info: minutes since 2019-04-02T18:00:00
"""
if not os.path.exists(rainnc_net_cdf_file_path):
logger.warning(
'no rainnc netcdf :: {}'.format(rainnc_net_cdf_file_path))
return
else:
"""
RAINNC netcdf data extraction
"""
fgt = get_file_last_modified_time(rainnc_net_cdf_file_path)
nnc_fid = Dataset(rainnc_net_cdf_file_path, mode='r')
time_unit_info = nnc_fid.variables['XTIME'].units
time_unit_info_list = time_unit_info.split(' ')
lats = nnc_fid.variables['XLAT'][0, :, 0]
lons = nnc_fid.variables['XLONG'][0, 0, :]
lon_min = lons[0].item()
lat_min = lats[0].item()
lon_max = lons[-1].item()
lat_max = lats[-1].item()
lat_inds = np.where((lats >= lat_min) & (lats <= lat_max))
lon_inds = np.where((lons >= lon_min) & (lons <= lon_max))
rainnc = nnc_fid.variables['RAINNC'][:, lat_inds[0], lon_inds[0]]
times = nnc_fid.variables['XTIME'][:]
start_date = fgt
end_date = fgt
nnc_fid.close()
diff = get_per_time_slot_values(rainnc)
width = len(lons)
height = len(lats)
ts = Timeseries(pool)
for y in range(height):
for x in range(width):
lat = float('%.6f' % lats[y])
lon = float('%.6f' % lons[x])
tms_meta['latitude'] = str(lat)
tms_meta['longitude'] = str(lon)
station_prefix = 'wrf_{}_{}'.format(lat, lon)
station_id = wrf_v3_stations.get(station_prefix)
if station_id is None:
add_station(pool=pool,
name=station_prefix,
latitude=lat,
longitude=lon,
description="WRF point",
station_type=StationEnum.WRF)
station_id = get_station_id(pool=pool,
latitude=lat,
longitude=lon,
station_type=StationEnum.WRF)
tms_id = ts.get_timeseries_id_if_exists(tms_meta)
if tms_id is None:
tms_id = ts.generate_timeseries_id(tms_meta)
run_meta = {
'tms_id': tms_id,
'sim_tag': tms_meta['sim_tag'],
'start_date': start_date,
'end_date': end_date,
'station_id': station_id,
'source_id': tms_meta['source_id'],
'unit_id': tms_meta['unit_id'],
'variable_id': tms_meta['variable_id']
}
try:
ts.insert_run(run_meta)
except Exception:
logger.error(
"Exception occurred while inserting run entry {}".
format(run_meta))
traceback.print_exc()
data_list = []
# generate timeseries for each station
for i in range(len(diff)):
ts_time = datetime.strptime(
time_unit_info_list[2],
'%Y-%m-%dT%H:%M:%S') + timedelta(
minutes=times[i + 1].item())
t = datetime_utc_to_lk(ts_time, shift_mins=0)
data_list.append([
tms_id,
t.strftime('%Y-%m-%d %H:%M:%S'), fgt,
float(diff[i, y, x])
])
push_rainfall_to_db(ts=ts,
ts_data=data_list,
tms_id=tms_id,
fgt=fgt)
version=FLO2D_150_version,
parameters=FLO2D_150_params)
add_variable(pool=pool, variable=variable)
add_unit(pool=pool, unit=unit, unit_type=unit_type)
# add flo2d 250 output stations
channel_cell_map_250 = FLO2D_250_params.get('CHANNEL_CELL_MAP')
for channel_cell_map_250_key in channel_cell_map_250.keys():
add_station(
pool=pool,
name="{}_{}".format(
channel_cell_map_250_key,
channel_cell_map_250.get(channel_cell_map_250_key)),
latitude="%.6f" %
float(flo2d_250_grids[int(channel_cell_map_250_key) - 1][2]),
longitude="%.6f" %
float(flo2d_250_grids[int(channel_cell_map_250_key) - 1][1]),
station_type=StationEnum.FLO2D_250,
description="{}_channel_cell_map_element".format(FLO2D_250))
flood_plain_cell_map_250 = FLO2D_250_params.get('FLOOD_PLAIN_CELL_MAP')
for flood_plain_cell_map_250_key in flood_plain_cell_map_250.keys():
add_station(
pool=pool,
name="{}_{}".format(
flood_plain_cell_map_250_key,
flood_plain_cell_map_250.get(
flood_plain_cell_map_250_key)),