def process_cumulative_plot(url_dest_list, start_ts_utc, end_ts_utc,
output_dir, lat_min, lon_min, lat_max, lon_max):
from_to = '%s-%s' % (start_ts_utc.strftime('%Y-%m-%d_%H:%M'),
end_ts_utc.strftime('%Y-%m-%d_%H:%M'))
cum_filename = os.path.join(output_dir,
'jaxa_sat_cum_rf_' + from_to + '.png')
if not utils.file_exists_nonempty(cum_filename):
total = None
for url_dest in url_dest_list:
if total is None:
total = np.genfromtxt(url_dest[2] + '.archive', dtype=float)
else:
total += np.genfromtxt(url_dest[2] + '.archive', dtype=float)
title = 'Cumulative rainfall ' + from_to
# clevs = np.concatenate(([-1, 0], np.array([pow(2, i) for i in range(0, 9)])))
clevs_cum = 10 * np.array(
[0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100])
norm_cum = colors.BoundaryNorm(boundaries=clevs_cum, ncolors=256)
cmap = plt.get_cmap('jet')
ext_utils.create_contour_plot(total,
cum_filename,
lat_min,
lon_min,
lat_max,
lon_max,
title,
clevs=clevs_cum,
cmap=cmap,
norm=norm_cum)
else:
logging.info('%s already exits' % cum_filename)
def make_plots(v, out_dir, basemap, start=0, end=-1):
start_rf = v['PRECIP'][0]
wrfutils.create_dir_if_not_exists(out_dir)
for j in range(start, end if end > 0 else len(v['PRECIP'])):
out = out_dir + '/' + v['Times'][j] + 'cum.png'
if j != 0:
utils.create_contour_plot(v['PRECIP'][j] - start_rf,
out,
lat_min,
lon_min,
lat_max,
lon_max,
out,
basemap=basemap,
clevs=clevs,
cmap=plt.get_cmap('jet'),
overwrite=True,
norm=norm)
else:
utils.create_contour_plot(v['PRECIP'][j],
out,
lat_min,
lon_min,
lat_max,
lon_max,
out,
basemap=basemap,
clevs=clevs,
cmap=plt.get_cmap('jet'),
overwrite=True,
norm=norm)
def create_rf_plots_wrf(nc_f, plots_output_dir, plots_output_base_dir, lon_min=None, lat_min=None, lon_max=None,
lat_max=None, filter_threshold=0.05, run_prefix='WRF'):
if not all([lon_min, lat_min, lon_max, lat_max]):
lon_min, lat_min, lon_max, lat_max = constants.SRI_LANKA_EXTENT
variables = ext_utils.extract_variables(nc_f, 'RAINC, RAINNC', lat_min, lat_max, lon_min, lon_max)
lats = variables['XLAT']
lons = variables['XLONG']
# cell size is calc based on the mean between the lat and lon points
cz = np.round(np.mean(np.append(lons[1:len(lons)] - lons[0: len(lons) - 1], lats[1:len(lats)]
- lats[0: len(lats) - 1])), 3)
clevs = [0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 70, 100, 150, 200, 250, 300, 400, 500, 600, 750]
cmap = cm.s3pcpn
basemap = Basemap(projection='merc', llcrnrlon=lon_min, llcrnrlat=lat_min, urcrnrlon=lon_max,
urcrnrlat=lat_max, resolution='h')
data = variables['RAINC'] + variables['RAINNC']
logging.info('Filtering with the threshold %f' % filter_threshold)
data[data < filter_threshold] = 0.0
variables['PRECIP'] = data
prefix = 'wrf_plots'
with TemporaryDirectory(prefix=prefix) as temp_dir:
t0 = dt.datetime.strptime(variables['Times'][0], '%Y-%m-%d_%H:%M:%S')
t1 = dt.datetime.strptime(variables['Times'][1], '%Y-%m-%d_%H:%M:%S')
step = (t1 - t0).total_seconds() / 3600.0
inst_precip = ext_utils.get_two_element_average(variables['PRECIP'])
cum_precip = ext_utils.get_two_element_average(variables['PRECIP'], return_diff=False)
for i in range(1, len(variables['Times'])):
time = variables['Times'][i]
ts = dt.datetime.strptime(time, '%Y-%m-%d_%H:%M:%S')
lk_ts = utils.datetime_utc_to_lk(ts, shift_mins=30)
logging.info('processing %s', time)
# instantaneous precipitation (hourly)
inst_file = os.path.join(temp_dir, 'wrf_inst_' + lk_ts.strftime('%Y-%m-%d_%H:%M:%S'))
ext_utils.create_asc_file(np.flip(inst_precip[i - 1], 0), lats, lons, inst_file + '.asc', cell_size=cz)
title = {
'label': 'Hourly rf for %s LK' % lk_ts.strftime('%Y-%m-%d_%H:%M:%S'),
'fontsize': 30
}
ext_utils.create_contour_plot(inst_precip[i - 1], inst_file + '.png', lat_min, lon_min, lat_max, lon_max,
title, clevs=clevs, cmap=cmap, basemap=basemap)
if (i * step) % 24 == 0:
t = 'Daily rf from %s LK to %s LK' % (
(lk_ts - dt.timedelta(hours=24)).strftime('%Y-%m-%d_%H:%M:%S'), lk_ts.strftime('%Y-%m-%d_%H:%M:%S'))
d = int(i * step / 24) - 1
logging.info('Creating images for D%d' % d)
cum_file = os.path.join(temp_dir, 'wrf_cum_%dd' % d)
if i * step / 24 > 1:
cum_precip_24h = cum_precip[i - 1] - cum_precip[i - 1 - int(24 / step)]
else:
cum_precip_24h = cum_precip[i - 1]
ext_utils.create_asc_file(np.flip(cum_precip_24h, 0), lats, lons, cum_file + '.asc', cell_size=cz)
ext_utils.create_contour_plot(cum_precip_24h, cum_file + '.png', lat_min, lon_min, lat_max, lon_max, t,
clevs=clevs, cmap=cmap, basemap=basemap)
gif_file = os.path.join(temp_dir, 'wrf_inst_%dd' % d)
images = [os.path.join(temp_dir, 'wrf_inst_' + j.strftime('%Y-%m-%d_%H:%M:%S') + '.png') for j in
np.arange(lk_ts - dt.timedelta(hours=24 - step), lk_ts + dt.timedelta(hours=step),
dt.timedelta(hours=step)).astype(dt.datetime)]
ext_utils.create_gif(images, gif_file + '.gif')
logging.info('Creating the zips')
utils.create_zip_with_prefix(temp_dir, '*.png', os.path.join(temp_dir, 'pngs.zip'))
utils.create_zip_with_prefix(temp_dir, '*.asc', os.path.join(temp_dir, 'ascs.zip'))
logging.info('Cleaning up instantaneous pngs and ascs - wrf_inst_*')
utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.png')
utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.asc')
logging.info('Copying pngs to ' + plots_output_dir)
utils.move_files_with_prefix(temp_dir, '*.png', plots_output_dir)
logging.info('Copying ascs to ' + plots_output_dir)
utils.move_files_with_prefix(temp_dir, '*.asc', plots_output_dir)
logging.info('Copying gifs to ' + plots_output_dir)
utils.copy_files_with_prefix(temp_dir, '*.gif', plots_output_dir)
logging.info('Copying zips to ' + plots_output_dir)
utils.copy_files_with_prefix(temp_dir, '*.zip', plots_output_dir)
plots_latest_dir = os.path.join(plots_output_base_dir, 'latest', run_prefix, os.path.basename(plots_output_dir))
# <nfs>/latest/wrf0 .. 3
utils.create_dir_if_not_exists(plots_latest_dir)
# todo: this needs to be adjusted to handle the multiple runs
logging.info('Copying gifs to ' + plots_latest_dir)
utils.copy_files_with_prefix(temp_dir, '*.gif', plots_latest_dir)
def process_jaxa_zip_file(zip_file_path,
out_file_path,
lat_min,
lon_min,
lat_max,
lon_max,
archive_data=False,
output_prefix='jaxa_sat',
db_adapter_config=None):
sat_zip = zipfile.ZipFile(zip_file_path)
sat = np.genfromtxt(sat_zip.open(
os.path.basename(zip_file_path).replace('.zip', '')),
delimiter=',',
names=True)
sat_filt = np.sort(sat[(sat['Lat'] <= lat_max) & (sat['Lat'] >= lat_min) &
(sat['Lon'] <= lon_max) & (sat['Lon'] >= lon_min)],
order=['Lat', 'Lon'])
lats = np.sort(np.unique(sat_filt['Lat']))
lons = np.sort(np.unique(sat_filt['Lon']))
data = sat_filt['RainRate'].reshape(len(lats), len(lons))
ext_utils.create_asc_file(np.flip(data, 0), lats, lons, out_file_path)
# clevs = np.concatenate(([-1, 0], np.array([pow(2, i) for i in range(0, 9)])))
# clevs = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30])
# norm = colors.BoundaryNorm(boundaries=clevs, ncolors=256)
# cmap = plt.get_cmap('jet')
clevs = [
0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 75, 100, 150, 200, 250, 300
]
# clevs = [0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100]
norm = None
cmap = cm.s3pcpn
ts = dt.datetime.strptime(
os.path.basename(out_file_path).replace(output_prefix + '_',
'').replace('.asc', ''),
'%Y-%m-%d_%H:%M')
lk_ts = utils.datetime_utc_to_lk(ts)
title_opts = {
'label':
output_prefix + ' ' + lk_ts.strftime('%Y-%m-%d %H:%M') + ' LK\n' +
ts.strftime('%Y-%m-%d %H:%M') + ' UTC',
'fontsize':
30
}
ext_utils.create_contour_plot(data,
out_file_path + '.png',
np.min(lats),
np.min(lons),
np.max(lats),
np.max(lons),
title_opts,
clevs=clevs,
cmap=cmap,
norm=norm)
if archive_data and not utils.file_exists_nonempty(out_file_path +
'.archive'):
np.savetxt(out_file_path + '.archive', data, fmt='%g')
else:
logging.info('%s already exits' % (out_file_path + '.archive'))
if not db_adapter_config:
logging.info('db_adapter not available. Unable to push data!')
return
db_adapter = ext_utils.get_curw_adapter(mysql_config=db_adapter_config)
width = len(lons)
height = len(lats)
station_prefix = 'sat'
run_name = 'Cloud-1'
upsert = True
def random_check_stations_exist():
for _ in range(10):
_x = lons[int(random() * width)]
_y = lats[int(random() * height)]
_name = '%s_%.6f_%.6f' % (station_prefix, _x, _y)
_query = {'name': _name}
if db_adapter.get_station(_query) is None:
logging.debug('Random stations check fail')
return False
logging.debug('Random stations check success')
return True
stations_exists = random_check_stations_exist()
rf_ts = {}
for y in range(height):
for x in range(width):
lat = lats[y]
lon = lons[x]
station_id = '%s_%.6f_%.6f' % (station_prefix, lon, lat)
name = station_id
if not stations_exists:
logging.info('Creating station %s ...' % name)
station = [
Station.Sat, station_id, name,
str(lon),
str(lat),
str(0), "WRF point"
]
db_adapter.create_station(station)
# add rf series to the dict
rf_ts[name] = [[lk_ts.strftime('%Y-%m-%d %H:%M:%S'), data[y, x]]]
ext_utils.push_rainfall_to_db(db_adapter,
rf_ts,
source=station_prefix,
name=run_name,
types=['Observed'],
upsert=upsert)
0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 75, 100, 150, 200, 250, 300
]
# clevs = [0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100]
norm = None
cmap = cm.s3pcpn
ts = dt.datetime.strptime(
os.path.basename(out_file_path).replace(output_prefix + '_',
'').replace('.asc', ''),
'%Y-%m-%d_%H:%M')
lk_ts = utils.datetime_utc_to_lk(ts)
title_opts = {
'label':
output_prefix + ' ' + lk_ts.strftime('%Y-%m-%d %H:%M') + ' LK\n' +
ts.strftime('%Y-%m-%d %H:%M') + ' UTC',
'fontsize':
30
}
ext_utils.create_contour_plot(data,
out_file_path + '.png',
np.min(lats),
np.min(lons),
np.max(lats),
np.max(lons),
title_opts,
clevs=clevs,
cmap=cmap,
norm=norm)
except Exception as e:
print('Exception|e:', e)
def process(self, *args, **kwargs):
config = self.get_config(**kwargs)
logging.info('wrf conifg: ' + config.to_json_string())
start_date = config.get('start_date')
d03_dir = config.get('wrf_output_dir')
d03_sl = os.path.join(d03_dir, 'wrfout_d01_' + start_date + ':00_SL')
# create a temp work dir & get a local copy of the d03.._SL
temp_dir = utils.create_dir_if_not_exists(
os.path.join(config.get('wrf_home'), 'temp_d01'))
shutil.copy2(d03_sl, temp_dir)
d03_sl = os.path.join(temp_dir, os.path.basename(d03_sl))
lat_min = -3.06107
lon_min = 71.2166
lat_max = 18.1895
lon_max = 90.3315
variables = ext_utils.extract_variables(d03_sl, 'RAINC, RAINNC',
lat_min, lat_max, lon_min,
lon_max)
lats = variables['XLAT']
lons = variables['XLONG']
# cell size is calc based on the mean between the lat and lon points
cz = np.round(
np.mean(
np.append(lons[1:len(lons)] - lons[0:len(lons) - 1],
lats[1:len(lats)] - lats[0:len(lats) - 1])), 3)
# clevs = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30])
# clevs_cum = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100])
# norm = colors.BoundaryNorm(boundaries=clevs, ncolors=256)
# norm_cum = colors.BoundaryNorm(boundaries=clevs_cum, ncolors=256)
# cmap = plt.get_cmap('jet')
clevs = [
0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 70, 100, 150, 200, 250,
300, 400, 500, 600, 750
]
clevs_cum = clevs
norm = None
cmap = cm.s3pcpn
basemap = Basemap(projection='merc',
llcrnrlon=lon_min,
llcrnrlat=lat_min,
urcrnrlon=lon_max,
urcrnrlat=lat_max,
resolution='h')
filter_threshold = 0.05
data = variables['RAINC'] + variables['RAINNC']
logging.info('Filtering with the threshold %f' % filter_threshold)
data[data < filter_threshold] = 0.0
variables['PRECIP'] = data
for i in range(1, len(variables['Times'])):
time = variables['Times'][i]
ts = dt.datetime.strptime(time, '%Y-%m-%d_%H:%M:%S')
lk_ts = utils.datetime_utc_to_lk(ts)
logging.info('processing %s', time)
# instantaneous precipitation (hourly)
inst_precip = variables['PRECIP'][i] - variables['PRECIP'][i - 1]
inst_file = os.path.join(temp_dir, 'wrf_inst_' + time)
title = {
'label':
'3Hourly rf for %s LK\n%s UTC' %
(lk_ts.strftime('%Y-%m-%d_%H:%M:%S'), time),
'fontsize':
30
}
ext_utils.create_contour_plot(inst_precip,
inst_file + '.png',
lat_min,
lon_min,
lat_max,
lon_max,
title,
clevs=clevs,
cmap=cmap,
basemap=basemap,
norm=norm)
if i % 8 == 0:
d = int(i / 8) - 1
logging.info('Creating gif for D%d' % d)
gif_file = os.path.join(temp_dir, 'wrf_inst_D01_%dd' % d)
images = [
os.path.join(
temp_dir,
'wrf_inst_' + i.strftime('%Y-%m-%d_%H:%M:%S') + '.png')
for i in np.arange(ts - dt.timedelta(hours=24 - 3), ts +
dt.timedelta(
hours=3), dt.timedelta(
hours=3)).astype(dt.datetime)
]
ext_utils.create_gif(images, gif_file + '.gif')
# move all the data in the tmp dir to the nfs
logging.info('Copying gifs to ' + d03_dir)
utils.copy_files_with_prefix(temp_dir, '*.gif', d03_dir)
d03_latest_dir = os.path.join(config.get('nfs_dir'), 'latest',
os.path.basename(config.get('wrf_home')))
# <nfs>/latest/wrf0 .. 3
utils.create_dir_if_not_exists(d03_latest_dir)
# todo: this needs to be adjusted to handle the multiple runs
logging.info('Copying gifs to ' + d03_latest_dir)
utils.copy_files_with_prefix(temp_dir, '*.gif', d03_latest_dir)
logging.info('Cleaning up the dir ' + temp_dir)
shutil.rmtree(temp_dir)
def process(self, *args, **kwargs):
config = self.get_config(**kwargs)
logging.info('wrf conifg: ' + config.to_json_string())
start_date = config.get('start_date')
d03_dir = config.get('wrf_output_dir')
d03_sl = os.path.join(d03_dir, 'wrfout_d03_' + start_date + ':00_SL')
# create a temp work dir & get a local copy of the d03.._SL
temp_dir = utils.create_dir_if_not_exists(
os.path.join(config.get('wrf_home'), 'temp'))
shutil.copy2(d03_sl, temp_dir)
d03_sl = os.path.join(temp_dir, os.path.basename(d03_sl))
lat_min = 5.722969
lon_min = 79.52146
lat_max = 10.06425
lon_max = 82.18992
variables = ext_utils.extract_variables(d03_sl, 'RAINC, RAINNC',
lat_min, lat_max, lon_min,
lon_max)
lats = variables['XLAT']
lons = variables['XLONG']
# cell size is calc based on the mean between the lat and lon points
cz = np.round(
np.mean(
np.append(lons[1:len(lons)] - lons[0:len(lons) - 1],
lats[1:len(lats)] - lats[0:len(lats) - 1])), 3)
# clevs = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30])
# clevs_cum = 10 * np.array([0.1, 0.5, 1, 2, 3, 5, 10, 15, 20, 25, 30, 50, 75, 100])
# norm = colors.BoundaryNorm(boundaries=clevs, ncolors=256)
# norm_cum = colors.BoundaryNorm(boundaries=clevs_cum, ncolors=256)
# cmap = plt.get_cmap('jet')
clevs = [
0, 1, 2.5, 5, 7.5, 10, 15, 20, 30, 40, 50, 70, 100, 150, 200, 250,
300, 400, 500, 600, 750
]
clevs_cum = clevs
norm = None
norm_cum = None
cmap = cm.s3pcpn
basemap = Basemap(projection='merc',
llcrnrlon=lon_min,
llcrnrlat=lat_min,
urcrnrlon=lon_max,
urcrnrlat=lat_max,
resolution='h')
filter_threshold = 0.05
data = variables['RAINC'] + variables['RAINNC']
logging.info('Filtering with the threshold %f' % filter_threshold)
data[data < filter_threshold] = 0.0
variables['PRECIP'] = data
pngs = []
ascs = []
for i in range(1, len(variables['Times'])):
time = variables['Times'][i]
ts = dt.datetime.strptime(time, '%Y-%m-%d_%H:%M:%S')
lk_ts = utils.datetime_utc_to_lk(ts)
logging.info('processing %s', time)
# instantaneous precipitation (hourly)
inst_precip = variables['PRECIP'][i] - variables['PRECIP'][i - 1]
inst_file = os.path.join(temp_dir, 'wrf_inst_' + time)
title = {
'label':
'Hourly rf for %s LK\n%s UTC' %
(lk_ts.strftime('%Y-%m-%d_%H:%M:%S'), time),
'fontsize':
30
}
ext_utils.create_asc_file(np.flip(inst_precip, 0),
lats,
lons,
inst_file + '.asc',
cell_size=cz)
ascs.append(inst_file + '.asc')
ext_utils.create_contour_plot(inst_precip,
inst_file + '.png',
lat_min,
lon_min,
lat_max,
lon_max,
title,
clevs=clevs,
cmap=cmap,
basemap=basemap,
norm=norm)
pngs.append(inst_file + '.png')
if i % 24 == 0:
t = 'Daily rf from %s LK to %s LK' % (
(lk_ts -
dt.timedelta(hours=24)).strftime('%Y-%m-%d_%H:%M:%S'),
lk_ts.strftime('%Y-%m-%d_%H:%M:%S'))
d = int(i / 24) - 1
logging.info('Creating images for D%d' % d)
cum_file = os.path.join(temp_dir, 'wrf_cum_%dd' % d)
ext_utils.create_asc_file(np.flip(variables['PRECIP'][i], 0),
lats,
lons,
cum_file + '.asc',
cell_size=cz)
ascs.append(cum_file + '.asc')
ext_utils.create_contour_plot(variables['PRECIP'][i] -
variables['PRECIP'][i - 24],
cum_file + '.png',
lat_min,
lon_min,
lat_max,
lon_max,
t,
clevs=clevs,
cmap=cmap,
basemap=basemap,
norm=norm_cum)
pngs.append(inst_file + '.png')
gif_file = os.path.join(temp_dir, 'wrf_inst_%dd' % d)
images = [
os.path.join(
temp_dir,
'wrf_inst_' + i.strftime('%Y-%m-%d_%H:%M:%S') + '.png')
for i in np.arange(ts - dt.timedelta(hours=23), ts +
dt.timedelta(
hours=1), dt.timedelta(
hours=1)).astype(dt.datetime)
]
ext_utils.create_gif(images, gif_file + '.gif')
logging.info('Creating the zips')
utils.create_zip_with_prefix(temp_dir, '*.png',
os.path.join(temp_dir, 'pngs.zip'))
utils.create_zip_with_prefix(temp_dir, '*.asc',
os.path.join(temp_dir, 'ascs.zip'))
# utils.create_zipfile(pngs, os.path.join(temp_dir, 'pngs.zip'))
# utils.create_zipfile(ascs, os.path.join(temp_dir, 'ascs.zip'))
logging.info('Cleaning up instantaneous pngs and ascs - wrf_inst_*')
utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.png')
utils.delete_files_with_prefix(temp_dir, 'wrf_inst_*.asc')
logging.info('Copying pngs to ' + d03_dir)
utils.move_files_with_prefix(temp_dir, '*.png', d03_dir)
logging.info('Copying ascs to ' + d03_dir)
utils.move_files_with_prefix(temp_dir, '*.asc', d03_dir)
logging.info('Copying gifs to ' + d03_dir)
utils.copy_files_with_prefix(temp_dir, '*.gif', d03_dir)
logging.info('Copying zips to ' + d03_dir)
utils.copy_files_with_prefix(temp_dir, '*.zip', d03_dir)
d03_latest_dir = os.path.join(config.get('nfs_dir'), 'latest',
os.path.basename(config.get('wrf_home')))
# <nfs>/latest/wrf0 .. 3
utils.create_dir_if_not_exists(d03_latest_dir)
# todo: this needs to be adjusted to handle the multiple runs
logging.info('Copying gifs to ' + d03_latest_dir)
utils.copy_files_with_prefix(temp_dir, '*.gif', d03_latest_dir)
logging.info('Cleaning up temp dir')
shutil.rmtree(temp_dir)