def extract_mean_rainfall_from_shp_file(nc_f, wrf_output, output_prefix, output_name, basin_shp_file, basin_extent,
curw_db_adapter=None, curw_db_upsert=False, run_prefix='WRF',
run_name='Cloud-1'):
lon_min, lat_min, lon_max, lat_max = basin_extent
nc_vars = ext_utils.extract_variables(nc_f, ['RAINC', 'RAINNC'], lat_min, lat_max, lon_min, lon_max)
lats = nc_vars['XLAT']
lons = nc_vars['XLONG']
prcp = nc_vars['RAINC'] + nc_vars['RAINNC']
times = nc_vars['Times']
diff = ext_utils.get_two_element_average(prcp)
polys = shapefile.Reader(basin_shp_file)
output_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, output_prefix))
with TemporaryDirectory(prefix=output_prefix) as temp_dir:
output_file_path = os.path.join(temp_dir, output_prefix + '.txt')
kub_rf = {}
with open(output_file_path, 'w') as output_file:
kub_rf[output_name] = []
for t in range(0, len(times) - 1):
cnt = 0
rf_sum = 0.0
for y in range(0, len(lats)):
for x in range(0, len(lons)):
if utils.is_inside_polygon(polys, lats[y], lons[x]):
cnt = cnt + 1
rf_sum = rf_sum + diff[t, y, x]
mean_rf = rf_sum / cnt
t_str = (
utils.datetime_utc_to_lk(dt.datetime.strptime(times[t], '%Y-%m-%d_%H:%M:%S'),
shift_mins=30)).strftime('%Y-%m-%d %H:%M:%S')
output_file.write('%s\t%.4f\n' % (t_str, mean_rf))
kub_rf[output_name].append([t_str, mean_rf])
utils.move_files_with_prefix(temp_dir, '*.txt', output_dir)
if curw_db_adapter is not None:
station = [Station.CUrW, output_name, output_name, -999, -999, 0, 'Kelani upper basin mean rainfall']
if ext_utils.create_station_if_not_exists(curw_db_adapter, station):
logging.info('%s station created' % output_name)
logging.info('Pushing data to the db...')
ext_utils.push_rainfall_to_db(curw_db_adapter, kub_rf, upsert=curw_db_upsert, name=run_name,
source=run_prefix)
else:
logging.info('curw_db_adapter not available. Unable to push data!')
def extract_weather_stations(nc_f, wrf_output, weather_stations=None, curw_db_adapter=None, curw_db_upsert=False,
run_prefix='WRF', run_name='Cloud-1'):
if weather_stations is None:
weather_stations = res_mgr.get_resource_path('extraction/local/kelani_basin_stations.txt')
nc_fid = Dataset(nc_f, 'r')
times_len, times = ext_utils.extract_time_data(nc_f)
prefix = 'stations_rf'
stations_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix))
stations_rf = {}
with TemporaryDirectory(prefix=prefix) as temp_dir:
with open(weather_stations, 'r') as csvfile:
stations = csv.reader(csvfile, delimiter=' ')
for row in stations:
logging.info(' '.join(row))
lon = row[1]
lat = row[2]
station_prcp = nc_fid.variables['RAINC'][:, lat, lon] + nc_fid.variables['RAINNC'][:, lat, lon]
station_diff = ext_utils.get_two_element_average(station_prcp)
stations_rf[row[0]] = []
station_file_path = os.path.join(temp_dir, row[0] + '_%s.txt' % prefix)
with open(station_file_path, 'w') as station_file:
for t in range(0, len(times) - 1):
t_str = (
utils.datetime_utc_to_lk(dt.datetime.strptime(times[t], '%Y-%m-%d_%H:%M:%S'),
shift_mins=30)).strftime('%Y-%m-%d %H:%M:%S')
station_file.write('%s\t%.4f\n' % (t_str, station_diff[t]))
stations_rf[row[0]].append([t_str, station_diff[t]])
utils.move_files_with_prefix(temp_dir, '*.txt', stations_dir)
if curw_db_adapter is not None:
logging.info('Pushing data to the db...')
ext_utils.push_rainfall_to_db(curw_db_adapter, stations_rf, upsert=curw_db_upsert, name=run_name,
source=run_prefix)
else:
logging.info('curw_db_adapter not available. Unable to push data!')
nc_fid.close()
def extract_weather_stations2(nc_f, wrf_output, weather_stations=None, curw_db_adapter=None, curw_db_upsert=False,
run_prefix='WRF', run_name='Cloud-1'):
if weather_stations is None:
weather_stations = res_mgr.get_resource_path('extraction/local/wrf_stations.txt')
points = np.genfromtxt(weather_stations, delimiter=',', names=True, dtype=None)
point_prcp = ext_utils.extract_points_array_rf_series(nc_f, points)
t0 = dt.datetime.strptime(point_prcp['Times'][0], '%Y-%m-%d %H:%M:%S')
t1 = dt.datetime.strptime(point_prcp['Times'][1], '%Y-%m-%d %H:%M:%S')
res_min = int((t1 - t0).total_seconds() / 60)
prefix = 'stations_rf'
stations_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix))
stations_rf = {}
with TemporaryDirectory(prefix=prefix) as temp_dir:
for point in points:
logging.info(str(point))
station_name = point[0].decode()
stations_rf[station_name] = []
station_file_path = os.path.join(temp_dir, station_name + '_%s.txt' % prefix)
with open(station_file_path, 'w') as station_file:
for t in range(0, len(point_prcp)):
station_file.write('%s\t%.4f\n' % (point_prcp['Times'][t], point_prcp[station_name][t]))
stations_rf[station_name].append([point_prcp['Times'][t], point_prcp[station_name][t]])
utils.move_files_with_prefix(temp_dir, '*.txt', stations_dir)
if curw_db_adapter is not None:
logging.info('Pushing data to the db...')
ext_utils.push_rainfall_to_db(curw_db_adapter, stations_rf, upsert=curw_db_upsert, name=run_name,
source=run_prefix)
else:
logging.info('curw_db_adapter not available. Unable to push data!')
def push_wrf_rainfall_to_db(nc_f, curw_db_adapter=None, lon_min=None, lat_min=None, lon_max=None,
lat_max=None, run_prefix='WRF', upsert=False, run_name='Cloud-1', station_prefix='wrf'):
"""
:param run_name:
:param nc_f:
:param curw_db_adapter: If not none, data will be pushed to the db
:param run_prefix:
:param lon_min:
:param lat_min:
:param lon_max:
:param lat_max:
:param upsert:
:return:
"""
if curw_db_adapter is None:
logging.info('curw_db_adapter not available. Unable to push data!')
return
if not all([lon_min, lat_min, lon_max, lat_max]):
lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_EXTENT
nc_vars = ext_utils.extract_variables(nc_f, ['RAINC', 'RAINNC'], lat_min, lat_max, lon_min, lon_max)
lats = nc_vars['XLAT']
lons = nc_vars['XLONG']
prcp = nc_vars['RAINC'] + nc_vars['RAINNC']
times = nc_vars['Times']
diff = ext_utils.get_two_element_average(prcp)
width = len(lons)
height = len(lats)
def random_check_stations_exist():
for _ in range(10):
_x = lons[int(random() * width)]
_y = lats[int(random() * height)]
_name = '%s_%.6f_%.6f' % (station_prefix, _x, _y)
_query = {'name': _name}
if curw_db_adapter.get_station(_query) is None:
logging.debug('Random stations check fail')
return False
logging.debug('Random stations check success')
return True
stations_exists = random_check_stations_exist()
rf_ts = {}
for y in range(height):
for x in range(width):
lat = lats[y]
lon = lons[x]
station_id = '%s_%.6f_%.6f' % (station_prefix, lon, lat)
name = station_id
if not stations_exists:
logging.info('Creating station %s ...' % name)
station = [Station.WRF, station_id, name, str(lon), str(lat), str(0), "WRF point"]
curw_db_adapter.create_station(station)
# add rf series to the dict
ts = []
for i in range(len(diff)):
t = utils.datetime_utc_to_lk(dt.datetime.strptime(times[i], '%Y-%m-%d_%H:%M:%S'), shift_mins=30)
ts.append([t.strftime('%Y-%m-%d %H:%M:%S'), diff[i, y, x]])
rf_ts[name] = ts
ext_utils.push_rainfall_to_db(curw_db_adapter, rf_ts, source=run_prefix, upsert=upsert, name=run_name)
def extract_metro_colombo(nc_f, wrf_output, wrf_output_base, curw_db_adapter=None, curw_db_upsert=False,
run_prefix='WRF', run_name='Cloud-1'):
"""
extract Metro-Colombo rf and divide area into to 4 quadrants
:param wrf_output_base:
:param run_name:
:param nc_f:
:param wrf_output:
:param curw_db_adapter: If not none, data will be pushed to the db
:param run_prefix:
:param curw_db_upsert:
:return:
"""
prefix = 'met_col'
lon_min, lat_min, lon_max, lat_max = constants.COLOMBO_EXTENT
nc_vars = ext_utils.extract_variables(nc_f, ['RAINC', 'RAINNC'], lat_min, lat_max, lon_min, lon_max)
lats = nc_vars['XLAT']
lons = nc_vars['XLONG']
prcp = nc_vars['RAINC'] + nc_vars['RAINNC']
times = nc_vars['Times']
diff = ext_utils.get_two_element_average(prcp)
width = len(lons)
height = len(lats)
output_dir = utils.create_dir_if_not_exists(os.path.join(wrf_output, prefix))
basin_rf = np.mean(diff[0:(len(times) - 1 if len(times) < 24 else 24), :, :])
alpha_file_path = os.path.join(wrf_output_base, prefix + '_alphas.txt')
utils.create_dir_if_not_exists(os.path.dirname(alpha_file_path))
with open(alpha_file_path, 'a+') as alpha_file:
t = utils.datetime_utc_to_lk(dt.datetime.strptime(times[0], '%Y-%m-%d_%H:%M:%S'), shift_mins=30)
alpha_file.write('%s\t%f\n' % (t.strftime('%Y-%m-%d_%H:%M:%S'), basin_rf))
cz = ext_utils.get_mean_cell_size(lats, lons)
no_data = -99
divs = (2, 2)
div_rf = {}
for i in range(divs[0] * divs[1]):
div_rf[prefix + str(i)] = []
with TemporaryDirectory(prefix=prefix) as temp_dir:
subsection_file_path = os.path.join(temp_dir, 'sub_means.txt')
with open(subsection_file_path, 'w') as subsection_file:
for tm in range(0, len(times) - 1):
t_str = (
utils.datetime_utc_to_lk(dt.datetime.strptime(times[tm], '%Y-%m-%d_%H:%M:%S'),
shift_mins=30)).strftime('%Y-%m-%d %H:%M:%S')
output_file_path = os.path.join(temp_dir, 'rf_' + t_str.replace(' ', '_') + '.asc')
ext_utils.create_asc_file(np.flip(diff[tm, :, :], 0), lats, lons, output_file_path, cell_size=cz,
no_data_val=no_data)
# writing subsection file
x_idx = [round(i * width / divs[0]) for i in range(0, divs[0] + 1)]
y_idx = [round(i * height / divs[1]) for i in range(0, divs[1] + 1)]
subsection_file.write(t_str)
for j in range(len(y_idx) - 1):
for i in range(len(x_idx) - 1):
quad = j * divs[1] + i
sub_sec_mean = np.mean(diff[tm, y_idx[j]:y_idx[j + 1], x_idx[i]: x_idx[i + 1]])
subsection_file.write('\t%.4f' % sub_sec_mean)
div_rf[prefix + str(quad)].append([t_str, sub_sec_mean])
subsection_file.write('\n')
utils.create_zip_with_prefix(temp_dir, 'rf_*.asc', os.path.join(temp_dir, 'ascs.zip'), clean_up=True)
utils.move_files_with_prefix(temp_dir, '*', output_dir)
# writing to the database
if curw_db_adapter is not None:
for i in range(divs[0] * divs[1]):
name = prefix + str(i)
station = [Station.CUrW, name, name, -999, -999, 0, "met col quadrant %d" % i]
if ext_utils.create_station_if_not_exists(curw_db_adapter, station):
logging.info('%s station created' % name)
logging.info('Pushing data to the db...')
ext_utils.push_rainfall_to_db(curw_db_adapter, div_rf, upsert=curw_db_upsert, source=run_prefix, name=run_name)
else:
logging.info('curw_db_adapter not available. Unable to push data!')
return basin_rf
def process_jaxa_zip_file(zip_file_path,
out_file_path,
lat_min,
lon_min,
lat_max,
lon_max,
archive_data=False,
output_prefix='jaxa_sat',
db_adapter_config=None):
sat_zip = zipfile.ZipFile(zip_file_path)
sat = np.genfromtxt(sat_zip.open(
os.path.basename(zip_file_path).replace('.zip', '')),
delimiter=',',
names=True)
sat_filt = np.sort(sat[(sat['Lat'] <= lat_max) & (sat['Lat'] >= lat_min) &
(sat['Lon'] <= lon_max) & (sat['Lon'] >= lon_min)],
order=['Lat', 'Lon'])
lats = np.sort(np.unique(sat_filt['Lat']))
lons = np.sort(np.unique(sat_filt['Lon']))
data = sat_filt['RainRate'].reshape(len(lats), len(lons))
ext_utils.create_asc_file(np.flip(data, 0), lats, lons, out_file_path)
# clevs = np.concatenate(([-1, 0], np.array([pow(2, i) for i in range(0, 9)])))
# clevs = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30])
# norm = colors.BoundaryNorm(boundaries=clevs, ncolors=256)
# cmap = plt.get_cmap('jet')
clevs = [
0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 75, 100, 150, 200, 250, 300
]
# clevs = [0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100]
norm = None
cmap = cm.s3pcpn
ts = dt.datetime.strptime(
os.path.basename(out_file_path).replace(output_prefix + '_',
'').replace('.asc', ''),
'%Y-%m-%d_%H:%M')
lk_ts = utils.datetime_utc_to_lk(ts)
title_opts = {
'label':
output_prefix + ' ' + lk_ts.strftime('%Y-%m-%d %H:%M') + ' LK\n' +
ts.strftime('%Y-%m-%d %H:%M') + ' UTC',
'fontsize':
30
}
ext_utils.create_contour_plot(data,
out_file_path + '.png',
np.min(lats),
np.min(lons),
np.max(lats),
np.max(lons),
title_opts,
clevs=clevs,
cmap=cmap,
norm=norm)
if archive_data and not utils.file_exists_nonempty(out_file_path +
'.archive'):
np.savetxt(out_file_path + '.archive', data, fmt='%g')
else:
logging.info('%s already exits' % (out_file_path + '.archive'))
if not db_adapter_config:
logging.info('db_adapter not available. Unable to push data!')
return
db_adapter = ext_utils.get_curw_adapter(mysql_config=db_adapter_config)
width = len(lons)
height = len(lats)
station_prefix = 'sat'
run_name = 'Cloud-1'
upsert = True
def random_check_stations_exist():
for _ in range(10):
_x = lons[int(random() * width)]
_y = lats[int(random() * height)]
_name = '%s_%.6f_%.6f' % (station_prefix, _x, _y)
_query = {'name': _name}
if db_adapter.get_station(_query) is None:
logging.debug('Random stations check fail')
return False
logging.debug('Random stations check success')
return True
stations_exists = random_check_stations_exist()
rf_ts = {}
for y in range(height):
for x in range(width):
lat = lats[y]
lon = lons[x]
station_id = '%s_%.6f_%.6f' % (station_prefix, lon, lat)
name = station_id
if not stations_exists:
logging.info('Creating station %s ...' % name)
station = [
Station.Sat, station_id, name,
str(lon),
str(lat),
str(0), "WRF point"
]
db_adapter.create_station(station)
# add rf series to the dict
rf_ts[name] = [[lk_ts.strftime('%Y-%m-%d %H:%M:%S'), data[y, x]]]
ext_utils.push_rainfall_to_db(db_adapter,
rf_ts,
source=station_prefix,
name=run_name,
types=['Observed'],
upsert=upsert)