def save_forecast_timeseries_to_db(pool, timeseries, run_date, run_time,
tms_meta):
print('EXTRACTFLO2DWATERLEVEL:: save_forecast_timeseries >>', tms_meta)
# {
# 'tms_id' : '',
# 'sim_tag' : '',
# 'station_id' : '',
# 'source_id' : '',
# 'unit_id' : '',
# 'variable_id': ''
# }
date_time = datetime.strptime('%s %s' % (run_date, run_time),
COMMON_DATE_TIME_FORMAT)
forecast_timeseries = []
if 'utcOffset' in tms_meta: # If there is an offset, shift by offset before proceed
print('Shift by utcOffset:', tms_meta['utcOffset'].resolution)
# Convert date time with offset
date_time = date_time + tms_meta['utcOffset']
run_date = date_time.strftime('%Y-%m-%d')
run_time = date_time.strftime('%H:%M:%S')
for item in timeseries:
forecast_timeseries.append([
datetime.strptime(item[0], COMMON_DATE_TIME_FORMAT) +
tms_meta['utcOffset'], item[1]
])
else:
forecast_timeseries = timeseries
try:
TS = Timeseries(pool=pool)
tms_id = TS.get_timeseries_id_if_exists(meta_data=tms_meta)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=tms_meta)
tms_meta['tms_id'] = tms_id
TS.insert_run(run_meta=tms_meta)
TS.update_start_date(id_=tms_id,
start_date=('%s %s' % (run_date, run_time)))
TS.insert_data(timeseries=forecast_timeseries,
tms_id=tms_id,
fgt=('%s %s' % (run_date, run_time)),
upsert=True)
TS.update_latest_fgt(id_=tms_id, fgt=('%s %s' % (run_date, run_time)))
except Exception:
logger.error(
"Exception occurred while pushing data to the curw_fcst database")
traceback.print_exc()
def save_forecast_timeseries_to_db(pool, output, mike_stations, fgt, tms_meta):
print('EXTRACT_MIKE_DISCHARGE:: save_forecast_timeseries >>', tms_meta)
# {
# 'tms_id' : '',
# 'sim_tag' : '',
# 'station_id' : '',
# 'source_id' : '',
# 'unit_id' : '',
# 'variable_id': ''
# }
# iterating the stations
for station in output.columns:
if station in mike_stations.keys():
ts = output[station].reset_index().values.tolist(
) # including index
tms_meta['latitude'] = str(mike_stations.get(station)[1])
tms_meta['longitude'] = str(mike_stations.get(station)[2])
tms_meta['station_id'] = mike_stations.get(station)[0]
try:
TS = Timeseries(pool=pool)
tms_id = TS.get_timeseries_id_if_exists(meta_data=tms_meta)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=tms_meta)
tms_meta['tms_id'] = tms_id
TS.insert_run(run_meta=tms_meta)
TS.update_start_date(id_=tms_id, start_date=fgt)
TS.insert_data(timeseries=ts,
tms_id=tms_id,
fgt=fgt,
upsert=True)
TS.update_latest_fgt(id_=tms_id, fgt=fgt)
except Exception:
logger.error(
"Exception occurred while pushing data to the curw_fcst database"
)
traceback.print_exc()
else:
print("### {} not included in the database. ###".format(station))
def save_forecast_timeseries_to_db(pool, timeseries, run_date, run_time, opts,
flo2d_stations, fgt):
print('EXTRACTFLO2DWATERLEVEL:: save_forecast_timeseries >>', opts)
# {
# 'tms_id' : '',
# 'sim_tag' : '',
# 'station_id' : '',
# 'source_id' : '',
# 'unit_id' : '',
# 'variable_id': ''
# }
# Convert date time with offset
date_time = datetime.strptime('%s %s' % (run_date, run_time),
COMMON_DATE_TIME_FORMAT)
if 'utcOffset' in opts:
date_time = date_time + opts['utcOffset']
run_date = date_time.strftime('%Y-%m-%d')
run_time = date_time.strftime('%H:%M:%S')
# If there is an offset, shift by offset before proceed
forecast_timeseries = []
if 'utcOffset' in opts:
print('Shift by utcOffset:', opts['utcOffset'].resolution)
for item in timeseries:
forecast_timeseries.append([
datetime.strptime(item[0], COMMON_DATE_TIME_FORMAT) +
opts['utcOffset'], item[1]
])
forecast_timeseries = extractForecastTimeseries(
timeseries=forecast_timeseries,
extract_date=run_date,
extract_time=run_time)
else:
forecast_timeseries = extractForecastTimeseries(timeseries=timeseries,
extract_date=run_date,
extract_time=run_time)
elementNo = opts.get('elementNo')
tms_meta = opts.get('tms_meta')
tms_meta['latitude'] = str(flo2d_stations.get(elementNo)[1])
tms_meta['longitude'] = str(flo2d_stations.get(elementNo)[2])
tms_meta['station_id'] = flo2d_stations.get(elementNo)[0]
try:
TS = Timeseries(pool=pool)
tms_id = TS.get_timeseries_id_if_exists(meta_data=tms_meta)
if tms_id is None:
tms_id = TS.generate_timeseries_id(meta_data=tms_meta)
tms_meta['tms_id'] = tms_id
TS.insert_run(run_meta=tms_meta)
TS.update_start_date(id_=tms_id, start_date=fgt)
TS.insert_data(timeseries=forecast_timeseries,
tms_id=tms_id,
fgt=fgt,
upsert=True)
TS.update_latest_fgt(id_=tms_id, fgt=fgt)
except Exception:
logger.error(
"Exception occurred while pushing data to the curw_fcst database")
traceback.print_exc()
def read_netcdf_file(pool, rainc_net_cdf_file_path, rainnc_net_cdf_file_path,
source_id, variable_id, unit_id, tms_meta, fgt):
"""
:param pool: database connection pool
:param rainc_net_cdf_file_path:
:param rainnc_net_cdf_file_path:
:param source_id:
:param variable_id:
:param unit_id:
:param tms_meta:
:return:
rainc_unit_info: mm
lat_unit_info: degree_north
time_unit_info: minutes since 2019-04-02T18:00:00
"""
if not os.path.exists(rainc_net_cdf_file_path):
logger.warning('no rainc netcdf')
print('no rainc netcdf')
elif not os.path.exists(rainnc_net_cdf_file_path):
logger.warning('no rainnc netcdf')
print('no rainnc netcdf')
else:
"""
RAINC netcdf data extraction
"""
nc_fid = Dataset(rainc_net_cdf_file_path, mode='r')
time_unit_info = nc_fid.variables['XTIME'].units
time_unit_info_list = time_unit_info.split(' ')
lats = nc_fid.variables['XLAT'][0, :, 0]
lons = nc_fid.variables['XLONG'][0, 0, :]
lon_min = lons[0].item()
lat_min = lats[0].item()
lon_max = lons[-1].item()
lat_max = lats[-1].item()
print('[lon_min, lat_min, lon_max, lat_max] :',
[lon_min, lat_min, lon_max, lat_max])
lat_inds = np.where((lats >= lat_min) & (lats <= lat_max))
lon_inds = np.where((lons >= lon_min) & (lons <= lon_max))
rainc = nc_fid.variables['RAINC'][:, lat_inds[0], lon_inds[0]]
"""
RAINNC netcdf data extraction
"""
nnc_fid = Dataset(rainnc_net_cdf_file_path, mode='r')
rainnc = nnc_fid.variables['RAINNC'][:, lat_inds[0], lon_inds[0]]
# times = list(set(nc_fid.variables['XTIME'][:])) # set is used to remove duplicates
times = nc_fid.variables['XTIME'][:]
# ts_start_date = datetime.strptime(time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S')
# ts_end_date = datetime.strptime(time_unit_info_list[2], '%Y-%m-%dT%H:%M:%S') + timedelta(
# minutes=float(sorted(set(times))[-2]))
# start_date = datetime_utc_to_lk(ts_start_date, shift_mins=0).strftime('%Y-%m-%d %H:%M:%S')
# end_date = datetime_utc_to_lk(ts_end_date, shift_mins=0).strftime('%Y-%m-%d %H:%M:%S')
start_date = fgt
end_date = fgt
prcp = rainc + rainnc
nc_fid.close()
nnc_fid.close()
diff = get_per_time_slot_values(prcp)
width = len(lons)
height = len(lats)
ts = Timeseries(pool)
for y in range(height):
for x in range(width):
lat = float('%.6f' % lats[y])
lon = float('%.6f' % lons[x])
tms_meta['latitude'] = str(lat)
tms_meta['longitude'] = str(lon)
station_prefix = '{}_{}'.format(lat, lon)
station_id = wrf_v3_stations.get(station_prefix)
if station_id is None:
add_station(pool=pool,
name=station_prefix,
latitude=lat,
longitude=lon,
description="WRF point",
station_type=StationEnum.WRF)
tms_id = ts.get_timeseries_id_if_exists(tms_meta)
logger.info("Existing timeseries id: {}".format(tms_id))
if tms_id is None:
tms_id = ts.generate_timeseries_id(tms_meta)
logger.info('HASH SHA256 created: {}'.format(tms_id))
run = (tms_id, tms_meta['sim_tag'], start_date, end_date,
station_id, source_id, variable_id, unit_id)
try:
ts.insert_run(run)
except Exception:
logger.error(
"Exception occurred while inserting run entry {}".
format(run))
traceback.print_exc()
else:
# ts.update_latest_fgt(id_=tms_id, fgt=fgt)
ts.update_start_date(id_=tms_id,
start_date=fgt) # to run backward
data_list = []
# generate timeseries for each station
for i in range(len(diff)):
ts_time = datetime.strptime(
time_unit_info_list[2],
'%Y-%m-%dT%H:%M:%S') + timedelta(
minutes=times[i].item())
t = datetime_utc_to_lk(ts_time, shift_mins=0)
data_list.append([
tms_id,
t.strftime('%Y-%m-%d %H:%M:%S'), fgt,
float(diff[i, y, x])
])
push_rainfall_to_db(ts=ts, ts_data=data_list)
def read_netcdf_file(pool, rainnc_net_cdf_file_path, tms_meta):
"""
:param pool: database connection pool
:param rainnc_net_cdf_file_path:
:param source_id:
:param variable_id:
:param unit_id:
:param tms_meta:
:return:
rainc_unit_info: mm
lat_unit_info: degree_north
time_unit_info: minutes since 2019-04-02T18:00:00
"""
if not os.path.exists(rainnc_net_cdf_file_path):
logger.warning(
'no rainnc netcdf :: {}'.format(rainnc_net_cdf_file_path))
return
else:
"""
RAINNC netcdf data extraction
"""
fgt = get_file_last_modified_time(rainnc_net_cdf_file_path)
nnc_fid = Dataset(rainnc_net_cdf_file_path, mode='r')
time_unit_info = nnc_fid.variables['XTIME'].units
time_unit_info_list = time_unit_info.split(' ')
lats = nnc_fid.variables['XLAT'][0, :, 0]
lons = nnc_fid.variables['XLONG'][0, 0, :]
lon_min = lons[0].item()
lat_min = lats[0].item()
lon_max = lons[-1].item()
lat_max = lats[-1].item()
lat_inds = np.where((lats >= lat_min) & (lats <= lat_max))
lon_inds = np.where((lons >= lon_min) & (lons <= lon_max))
rainnc = nnc_fid.variables['RAINNC'][:, lat_inds[0], lon_inds[0]]
times = nnc_fid.variables['XTIME'][:]
start_date = fgt
end_date = fgt
nnc_fid.close()
diff = get_per_time_slot_values(rainnc)
width = len(lons)
height = len(lats)
ts = Timeseries(pool)
for y in range(height):
for x in range(width):
lat = float('%.6f' % lats[y])
lon = float('%.6f' % lons[x])
tms_meta['latitude'] = str(lat)
tms_meta['longitude'] = str(lon)
station_prefix = 'wrf_{}_{}'.format(lat, lon)
station_id = wrf_v3_stations.get(station_prefix)
if station_id is None:
add_station(pool=pool,
name=station_prefix,
latitude=lat,
longitude=lon,
description="WRF point",
station_type=StationEnum.WRF)
station_id = get_station_id(pool=pool,
latitude=lat,
longitude=lon,
station_type=StationEnum.WRF)
tms_id = ts.get_timeseries_id_if_exists(tms_meta)
if tms_id is None:
tms_id = ts.generate_timeseries_id(tms_meta)
run_meta = {
'tms_id': tms_id,
'sim_tag': tms_meta['sim_tag'],
'start_date': start_date,
'end_date': end_date,
'station_id': station_id,
'source_id': tms_meta['source_id'],
'unit_id': tms_meta['unit_id'],
'variable_id': tms_meta['variable_id']
}
try:
ts.insert_run(run_meta)
except Exception:
logger.error(
"Exception occurred while inserting run entry {}".
format(run_meta))
traceback.print_exc()
data_list = []
# generate timeseries for each station
for i in range(len(diff)):
ts_time = datetime.strptime(
time_unit_info_list[2],
'%Y-%m-%dT%H:%M:%S') + timedelta(
minutes=times[i + 1].item())
t = datetime_utc_to_lk(ts_time, shift_mins=0)
data_list.append([
tms_id,
t.strftime('%Y-%m-%d %H:%M:%S'), fgt,
float(diff[i, y, x])
])
push_rainfall_to_db(ts=ts,
ts_data=data_list,
tms_id=tms_id,
fgt=fgt)
def update_MME_tagged_series(pool, start, end, variables, coefficients,
sub_region, tms_meta, fgt):
for index, row in sub_region.iterrows():
lat = float('%.6f' % row['latitude'])
lon = float('%.6f' % row['longitude'])
tms_meta['latitude'] = str(lat)
tms_meta['longitude'] = str(lon)
station_prefix = 'wrf_{}_{}'.format(lat, lon)
station_id = wrf_v3_stations.get(station_prefix)
TS = Timeseries(pool=pool)
tms_id = TS.get_timeseries_id_if_exists(tms_meta)
if tms_id is None:
tms_id = TS.generate_timeseries_id(tms_meta)
run_meta = {
'tms_id': tms_id,
'sim_tag': tms_meta['sim_tag'],
'start_date': fgt,
'end_date': fgt,
'station_id': station_id,
'source_id': tms_meta['source_id'],
'unit_id': tms_meta['unit_id'],
'variable_id': tms_meta['variable_id']
}
try:
TS.insert_run(run_meta)
except Exception:
time.sleep(5)
try:
TS.insert_run(run_meta)
except Exception:
logger.error(
"Exception occurred while inserting run entry {}".
format(run_meta))
traceback.print_exc()
timeseries = calculate_MME_series(TS=TS,
start=start,
end=end,
variables=variables,
coefficients=coefficients,
station_id=station_id,
variable_id=tms_meta['variable_id'],
unit_id=tms_meta['unit_id'])
formatted_timeseries = []
for i in range(len(timeseries)):
formatted_timeseries.append([
tms_id, timeseries[i][0].strftime('%Y-%m-%d %H:%M:00'), fgt,
float('%.3f' % timeseries[i][1])
])
push_rainfall_to_db(ts=TS,
ts_data=formatted_timeseries,
tms_id=tms_id,
fgt=fgt)