def get_lake_fraction_for_date(self, the_date=None):
"""
Get the lake ice cover fraction for the specified date
:param the_date:
:return:
"""
if the_date not in self.cached_data:
month_date = datetime(the_date.year, the_date.month, 1)
self.cached_data = {}
mr = MultiRPN(self.year_month_date_to_file_list[month_date])
self.cached_data = mr.get_all_time_records_for_name_and_level(varname=self.varname, level=self.level,
level_kind=self.level_kind)
mr.close()
# if still it is not there try looking in the previous month
if the_date not in self.cached_data:
month = the_date.month - 1
year = the_date.year
if the_date.month == 0:
month = 12
year -= 1
month_date = datetime(year, month, 1)
mr = MultiRPN(self.year_month_date_to_file_list[month_date])
self.cached_data.update(mr.get_all_time_records_for_name_and_level(varname=self.varname, level=self.level,
level_kind=self.level_kind))
mr.close()
return self.cached_data[the_date]
def test_getting_coordinates_for_the_last_read_record_should_not_fail():
r = None
try:
r = MultiRPN(in_path)
rec = r.get_first_record_for_name("I5")
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
tools.assert_equals(lons.shape, lats.shape, "Shapes of lon and lat arrays should be the same.")
finally:
if r is not None:
r.close()
delete_files()
def test_get_number_of_records():
r = None
try:
create_files()
r = MultiRPN("test_?.rpn")
msg = "Number of records should be equal to the total number in all files"
tools.assert_equals(r.get_number_of_records(), len(FILE_NAMES), msg)
finally:
if r is not None:
r.close()
delete_files()
def test_can_link_many_files(nfiles=200):
many_fnames = ["test_{}.rpn".format(i) for i in range(nfiles)]
r = None
try:
create_files(fnames=many_fnames)
r = MultiRPN("test_*.rpn")
tools.assert_greater(r.get_number_of_records(), 0, msg="There should be more than 0 records")
finally:
if r is not None:
r.close()
delete_files(fnames=many_fnames)
def test_should_find_all_records():
r = None
try:
create_files()
r = MultiRPN("test_?.rpn")
for fn in range(len(FILE_NAMES)):
rec = r.get_first_record_for_name("T{}".format(fn))
print("file {} - OK".format(fn))
print(rec.mean(), rec.shape)
finally:
if r is not None:
r.close()
delete_files()
def extract_runoff_to_nc_process(args):
in_path, out_path = args
if os.path.exists(out_path):
print("Nothing to do for: {}".format(out_path))
return # skip files that already exist
traf_name = "TRAF"
tdra_name = "TDRA"
r = MultiRPN(in_path)
traf_data = r.get_all_time_records_for_name_and_level(varname=traf_name, level=5, level_kind=level_kinds.ARBITRARY)
tdra_data = r.get_all_time_records_for_name_and_level(varname=tdra_name, level=5, level_kind=level_kinds.ARBITRARY)
r.close()
nx, ny = list(traf_data.items())[0][1].shape
with nc.Dataset(out_path, "w", format="NETCDF3_CLASSIC") as ds:
ds.createDimension("lon", nx)
ds.createDimension("lat", ny)
ds.createDimension("time", None)
varTraf = ds.createVariable(traf_name, "f4", dimensions=("time", "lon", "lat"))
varTraf.units = "kg/( m**2 * s )"
varTdra = ds.createVariable(tdra_name, "f4", dimensions=("time", "lon", "lat"))
varTdra.units = "kg/( m**2 * s )"
timeVar = ds.createVariable("time", "f4", dimensions=("time",))
sorted_dates = list(sorted(traf_data.keys()))
timeVar.units = "hours since {0}".format(sorted_dates[0])
timeVar[:] = nc.date2num(sorted_dates, timeVar.units)
varTraf[:] = np.array(
[traf_data[d] for d in sorted_dates]
)
varTdra[:] = np.array(
[tdra_data[d] for d in sorted_dates]
)
def test_get_4d_field():
r = None
vname = "T"
try:
create_files_with_same_var_for_different_times(vname=vname)
r = MultiRPN("test_?.rpn")
recs = r.get_4d_field(vname)
msg = "Not all records for {} were found".format(vname)
print(recs.keys())
tools.assert_equals(len(FILE_NAMES), len(recs), msg)
finally:
if r is not None:
r.close()
delete_files()
def extract_runoff_to_nc_process(args):
in_path, out_path = args
if os.path.exists(out_path):
print("Nothing to do for: {}".format(out_path))
return # skip files that already exist
traf_name = "TRAF"
tdra_name = "TDRA"
r = MultiRPN(in_path)
traf_data = r.get_all_time_records_for_name_and_level(varname=traf_name, level=5, level_kind=level_kinds.ARBITRARY)
tdra_data = r.get_all_time_records_for_name_and_level(varname=tdra_name, level=5, level_kind=level_kinds.ARBITRARY)
r.close()
nx, ny = list(traf_data.items())[0][1].shape
with nc.Dataset(out_path, "w", format="NETCDF3_CLASSIC") as ds:
ds.createDimension("lon", nx)
ds.createDimension("lat", ny)
ds.createDimension("time", None)
varTraf = ds.createVariable(traf_name, "f4", dimensions=("time", "lon", "lat"))
varTraf.units = "kg/( m**2 * s )"
varTdra = ds.createVariable(tdra_name, "f4", dimensions=("time", "lon", "lat"))
varTdra.units = "kg/( m**2 * s )"
timeVar = ds.createVariable("time", "f4", dimensions=("time",))
sorted_dates = list(sorted(traf_data.keys()))
timeVar.units = "hours since {0}".format(sorted_dates[0])
timeVar[:] = nc.date2num(sorted_dates, timeVar.units)
varTraf[:] = np.array(
[traf_data[d] for d in sorted_dates]
)
varTdra[:] = np.array(
[tdra_data[d] for d in sorted_dates]
)
def test_get_list_of_varnames():
r = None
vname = "T"
try:
create_files_with_same_var_for_different_times(vname=vname)
r = MultiRPN("test_?.rpn")
msg = "the list of varnames should contain {}".format(vname)
vnames = r.get_list_of_varnames()
tools.assert_in(vname, vnames, msg)
tools.assert_equal(len(vnames), 1, "There is only one unique field name in the files")
finally:
if r is not None:
r.close()
delete_files()
def get_lake_fraction_for_date(self, the_date=None):
"""
Get the lake ice cover fraction for the specified date
:param the_date:
:return:
"""
if the_date not in self.cached_data:
month_date = datetime(the_date.year, the_date.month, 1)
self.cached_data = {}
mr = MultiRPN(self.year_month_date_to_file_list[month_date])
self.cached_data = mr.get_all_time_records_for_name_and_level(
varname=self.varname, level=self.level, level_kind=self.level_kind
)
mr.close()
# if still it is not there try looking in the previous month
if the_date not in self.cached_data:
month = the_date.month - 1
year = the_date.year
if the_date.month == 0:
month = 12
year -= 1
month_date = datetime(year, month, 1)
mr = MultiRPN(self.year_month_date_to_file_list[month_date])
self.cached_data.update(
mr.get_all_time_records_for_name_and_level(
varname=self.varname, level=self.level, level_kind=self.level_kind
)
)
mr.close()
return self.cached_data[the_date]
def get_model_data(station_to_model_point, output_path=None, grid_config=None, basins_of_interest_shp="",
cell_manager=None, vname=None):
lons, lats, bmp = None, None, None
data_mask = None
monthly_diagnostics_case = False
if output_path.name.lower().endswith("diagnostics"):
fname_pattern = "pm*_moyenne"
monthly_diagnostics_case = True
else:
fname_pattern = "pm*p"
pattern = re.compile(".*" + 8 * "0" + ".*")
flist = [f for f in glob.glob(str(output_path.joinpath("*").joinpath(fname_pattern))) if pattern.match(f) is None]
r = MultiRPN(flist)
date_to_field = r.get_all_time_records_for_name_and_level(varname=vname)
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
r.close()
# get the basemap object
bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest(
lons, lats, shp_path=basins_of_interest_shp, mask_margin=5)
station_to_model_data = {} # model data are the pandas timeseries
stations_to_ignore = []
for station, model_point in station_to_model_point.items():
assert isinstance(model_point, ModelPoint)
assert isinstance(cell_manager, CellManager)
assert isinstance(station, cehq_station.Station)
upstream_mask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices(model_point.ix,
model_point.jy)
# Skip model points and staions with small number of gridcells upstream
if upstream_mask.sum() <= 1:
stations_to_ignore.append(station)
print("Station {} is ignored, because the number of upstream cells is <= 1.".format(station.id))
continue
# Skip model points and stations outside the region of interest
if not data_mask[model_point.ix, model_point.jy]:
stations_to_ignore.append(station)
print("Station {} is ignored, because it is outside of the domain of interest.".format(station.id))
continue
# Plot station position
fig = plt.figure()
ax = plt.gca()
lons1, lats1 = lons[upstream_mask > 0.5], lats[upstream_mask > 0.5]
x1, y1 = bmp(lons1, lats1)
bmp.drawrivers()
bmp.drawcoastlines(ax=ax)
bmp.drawcountries(ax=ax, linewidth=0.2)
bmp.drawstates(linewidth=0.1)
bmp.readshapefile(basins_of_interest_shp[:-4], "basin", linewidth=2, color="m")
bmp.scatter(x1, y1, c="g", s=100)
bmp.scatter(*bmp(lons[model_point.ix, model_point.jy], lats[model_point.ix, model_point.jy]), c="b", s=250)
fig.tight_layout()
plt.savefig(str(img_folder.joinpath("{}_position_and_upstream.png".format(station.id))), bbox_inche="tight")
plt.close(fig)
res = pd.Series(index=sorted(date_to_field.keys()),
data=[date_to_field[d][model_point.ix, model_point.jy] for d in sorted(date_to_field.keys())])
# get monthly means
res = res.groupby(lambda d: d.replace(day=15, hour=0)).mean()
if monthly_diagnostics_case:
# shift to the end of the month before a previous month, and then shift 15 days to later
res = res.shift(-2, freq="M").shift(15, freq="D")
print(res.index[:20])
print(res.index[-20:])
station_to_model_data[station] = res
# Not enough drainage area
for s in stations_to_ignore:
del station_to_model_point[s]
return station_to_model_data
def plot_monthly_clim_in_a_panel(months=None, diag_folder="", vname="STFL",
grid_config=None, basins_of_interest_shp=""):
"""
Plots climatologies using diagnostics outputs, not samples
:param months:
:param diag_folder:
:param vname:
"""
if months is None:
months = list(range(1, 13))
diag_path = Path(diag_folder)
month_to_field = OrderedDict()
lons, lats, bmp = None, None, None
data_mask = None
for m in months:
r = MultiRPN(str(diag_path.joinpath("*{:02d}".format(m)).joinpath("pm*_moyenne")))
date_to_field = r.get_all_time_records_for_name_and_level()
the_mean = np.mean([f for f in date_to_field.values()], axis=0)
the_mean = np.ma.masked_where(the_mean < 0, the_mean)
month_to_field[m] = the_mean
if bmp is None:
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
# get the basemap object
bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest(
lons, lats, shp_path=basins_of_interest_shp, mask_margin=5)
r.close()
fig = plt.figure()
ncols = 3
nrows = len(months) // ncols + int(len(months) % ncols != 0)
gs = GridSpec(nrows=nrows, ncols=ncols + 1)
xx, yy = bmp(lons, lats)
clevs = [0, 20, 50, 100, 200, 500, 1000, 1500, 3000, 4500, 5000, 7000, 9000]
bn = BoundaryNorm(clevs, len(clevs) - 1)
cmap = cm.get_cmap("jet", len(clevs) - 1)
for m, field in month_to_field.items():
row = (m - 1) // ncols
col = (m - 1) % ncols
ax = fig.add_subplot(gs[row, col])
ax.set_title(calendar.month_name[m])
to_plot = np.ma.masked_where(~data_mask, field)
im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1])
bmp.colorbar(im, extend="max")
bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax)
bmp.drawcoastlines(ax=ax)
plt.close(fig)
# plot annual mean
ann_mean = np.mean([field for m, field in month_to_field.items()], axis=0)
fig = plt.figure()
ax = fig.add_subplot(gs[:, :])
ax.set_title("Annual mean")
to_plot = np.ma.masked_where(~data_mask, ann_mean)
im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1])
bmp.colorbar(im, extend="max")
bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax)
bmp.drawcoastlines(ax=ax)
plt.show()
plt.close(fig)
def main(plot_vals=False):
varname = "RFAC"
multiplier = 24 * 3600
data_folder = Path(
"/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/atm_data_for_Arman_simulations"
)
day_range = range(19, 22)
dates_of_interest = [datetime(2013, 6, d) for d in day_range]
img_folder = Path("calgary_flood/2D")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
lons, lats, bmp = None, None, None
the_mask = get_bow_river_basin_mask()
i_list, j_list = np.where(the_mask > 0.5)
imin, imax = i_list.min() - 2, i_list.max() + 5
jmin, jmax = j_list.min() - 2, j_list.max() + 5
# Calculate daily means
sim_label_to_date_to_mean = OrderedDict()
for sim_dir in data_folder.iterdir():
mr = MultiRPN(str(sim_dir.joinpath("pm*")))
print(str(sim_dir))
print(mr.get_number_of_records())
label = sim_dir.name.split("_")[-2].replace("NoDrain", "").replace(
"frozen", "Frozen").replace("Bow", "")
sim_label_to_date_to_mean[label] = OrderedDict()
data = mr.get_4d_field(varname)
data = {
d: list(v.items())[0][1]
for d, v in data.items() if d.day in day_range
}
for d in dates_of_interest:
sim_label_to_date_to_mean[label][d] = np.array([
field for d1, field in data.items() if d1.day == d.day
]).mean(axis=0) * multiplier
if lons is None:
lons, lats = mr.get_longitudes_and_latitudes_of_the_last_read_rec()
for f in sim_dir.iterdir():
if f.name.startswith("pm"):
r = RPN(str(f))
r.get_first_record_for_name(varname=varname)
rll = RotatedLatLon(
**r.get_proj_parameters_for_the_last_read_rec())
bmp = rll.get_basemap_object_for_lons_lats(
lons2d=lons[imin:imax, jmin:jmax],
lats2d=lats[imin:imax, jmin:jmax],
resolution="i")
r.close()
break
mr.close()
# reorder simulations
sim_label_to_date_to_mean = OrderedDict([
(k, sim_label_to_date_to_mean[k]) for k in sorted(
sim_label_to_date_to_mean, key=lambda z: len(z), reverse=True)
])
key_list = [k for k in sim_label_to_date_to_mean]
key_list[-2], key_list[-1] = key_list[-1], key_list[-2]
sim_label_to_date_to_mean = OrderedDict([(k, sim_label_to_date_to_mean[k])
for k in key_list])
# do the plots (subplots: vertically - simulations, horizontally - days)
plot_utils.apply_plot_params(width_cm=24, height_cm=28, font_size=10)
fig = plt.figure()
nrows = len(sim_label_to_date_to_mean)
ncols = len(day_range)
gs = GridSpec(nrows, ncols, wspace=0, hspace=0.1)
clevs_vals = [0, 0.1, 20, 50, 100, 150, 200, 300]
base_label = None
clevs_diff = [1, 5, 10, 20, 50, 100]
clevs_diff = [-c for c in reversed(clevs_diff)] + [0] + clevs_diff
cmap_diff = cm.get_cmap("bwr", len(clevs_diff) - 1)
cmap_vals = get_cmap_from_ncl_spec_file(
path="colormap_files/precip3_16lev.rgb", ncolors=len(clevs_vals) - 1)
xx, yy = bmp(lons, lats)
title = "Total runoff ({}, mm/day)".format(varname)
fig.suptitle(title)
for row, (sim_label,
date_to_field) in enumerate(sim_label_to_date_to_mean.items()):
if row == 0 or plot_vals:
base_sim = OrderedDict([(k, 0) for k, v in date_to_field.items()])
base_label = sim_label
plot_label = sim_label
clevs = clevs_vals
cmap = cmap_vals
extend = "max"
else:
base_sim = list(sim_label_to_date_to_mean.items())[0][1]
plot_label = "{}\n-\n{}".format(sim_label, base_label)
clevs = clevs_diff
cmap = cmap_diff
extend = "both"
bn = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1)
for col, (the_date, field) in enumerate(date_to_field.items()):
ax = fig.add_subplot(gs[row, col])
to_plot = np.ma.masked_where(the_mask < 0.5,
field - base_sim[the_date])
# cs = bmp.contourf(xx[~to_plot.mask], yy[~to_plot.mask], to_plot[~to_plot.mask], levels=clevs, extend="max", tri=True)
cs = bmp.pcolormesh(xx,
yy,
to_plot[:-1, :-1],
norm=bn,
vmin=clevs[0],
vmax=clevs[-1],
cmap=cmap)
bmp.drawcoastlines(ax=ax, linewidth=0.3)
assert isinstance(bmp, Basemap)
bmp.readshapefile(BOW_RIVER_SHP[:-4], "basin", zorder=5)
cb = bmp.colorbar(cs,
ax=ax,
ticks=clevs,
extend=extend,
pad="4%",
size="10%")
if plot_vals:
cb.ax.set_visible(col == ncols - 1 and row == 0)
else:
cb.ax.set_visible(col == ncols - 1 and row in (0, 1))
if col == 0:
ax.set_ylabel(plot_label)
if row == 0:
ax.set_title(the_date.strftime("%b %d"))
if plot_vals:
img_file = img_folder.joinpath("{}.png".format(varname))
else:
img_file = img_folder.joinpath("{}_diff.png".format(varname))
fig.savefig(str(img_file))
plt.close(fig)
def get_area_avg_timeseries(samples_dir, start_year=-np.Inf, end_year=np.Inf, filename_prefix="pm",
level=-1, level_kind=level_kinds.ARBITRARY,
varname="", mask=None, mask_lons2d=None, mask_lats2d=None, file_per_var=False) -> pd.Series:
"""
get the timeseries of area averaged ice fraction
:rtype : pd.Series
"""
yearly_ts = []
lons2d, lats2d = None, None
samples_dir_p = Path(samples_dir)
# interpolated mask
interpolated_mask = None
for y in range(start_year, end_year + 1):
files_for_year = []
mfolders = [f for f in samples_dir_p.iterdir() if f.name[:-2].endswith(str(y))]
for mfolder in mfolders:
# Select all files containing the varname in the filename
if file_per_var:
files_for_year += [str(f) for f in mfolder.iterdir() if varname in f.name]
else:
files_for_year += [str(f) for f in mfolder.iterdir() if f.name.startswith(filename_prefix) and f.name[-9:-1] != "0" * 8]
if len(files_for_year) == 0:
continue
mrpn = MultiRPN(files_for_year)
data = mrpn.get_all_time_records_for_name_and_level(varname=varname, level=level, level_kind=level_kind)
if lons2d is None:
lons2d, lats2d = mrpn.get_longitudes_and_latitudes_of_the_last_read_rec()
# interpolate the mask using nearest neighbour approach
if interpolated_mask is None:
xs, ys, zs = lat_lon.lon_lat_to_cartesian(mask_lons2d.flatten(), mask_lats2d.flatten())
ktree = KDTree(data=list(zip(xs, ys, zs)))
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1)
interpolated_mask = mask.flatten()[inds]
interpolated_mask.shape = lons2d.shape
for t, field in data.items():
data[t] = field[interpolated_mask].mean()
tlist = [t for t in data.keys()]
ser = pd.Series(index=tlist, data=[data[t] for t in tlist])
ser.sort_index(inplace=True)
yearly_ts.append(ser)
mrpn.close()
return pd.concat(yearly_ts), lons2d, lats2d
def plot_monthly_clim_in_a_panel(months=None,
diag_folder="",
vname="STFL",
grid_config=None,
basins_of_interest_shp=""):
"""
Plots climatologies using diagnostics outputs, not samples
:param months:
:param diag_folder:
:param vname:
"""
if months is None:
months = list(range(1, 13))
diag_path = Path(diag_folder)
month_to_field = OrderedDict()
lons, lats, bmp = None, None, None
data_mask = None
for m in months:
r = MultiRPN(
str(
diag_path.joinpath(
"*{:02d}".format(m)).joinpath("pm*_moyenne")))
date_to_field = r.get_all_time_records_for_name_and_level()
the_mean = np.mean([f for f in date_to_field.values()], axis=0)
the_mean = np.ma.masked_where(the_mean < 0, the_mean)
month_to_field[m] = the_mean
if bmp is None:
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
# get the basemap object
bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest(
lons, lats, shp_path=basins_of_interest_shp, mask_margin=5)
r.close()
fig = plt.figure()
ncols = 3
nrows = len(months) // ncols + int(len(months) % ncols != 0)
gs = GridSpec(nrows=nrows, ncols=ncols + 1)
xx, yy = bmp(lons, lats)
clevs = [
0, 20, 50, 100, 200, 500, 1000, 1500, 3000, 4500, 5000, 7000, 9000
]
bn = BoundaryNorm(clevs, len(clevs) - 1)
cmap = cm.get_cmap("jet", len(clevs) - 1)
for m, field in month_to_field.items():
row = (m - 1) // ncols
col = (m - 1) % ncols
ax = fig.add_subplot(gs[row, col])
ax.set_title(calendar.month_name[m])
to_plot = np.ma.masked_where(~data_mask, field)
im = bmp.pcolormesh(xx,
yy,
to_plot,
norm=bn,
cmap=cmap,
vmin=clevs[0],
vmax=clevs[-1])
bmp.colorbar(im, extend="max")
bmp.readshapefile(basins_of_interest_shp[:-4],
"basins",
linewidth=2,
color="m",
ax=ax)
bmp.drawcoastlines(ax=ax)
plt.close(fig)
# plot annual mean
ann_mean = np.mean([field for m, field in month_to_field.items()], axis=0)
fig = plt.figure()
ax = fig.add_subplot(gs[:, :])
ax.set_title("Annual mean")
to_plot = np.ma.masked_where(~data_mask, ann_mean)
im = bmp.pcolormesh(xx,
yy,
to_plot,
norm=bn,
cmap=cmap,
vmin=clevs[0],
vmax=clevs[-1])
bmp.colorbar(im, extend="max")
bmp.readshapefile(basins_of_interest_shp[:-4],
"basins",
linewidth=2,
color="m",
ax=ax)
bmp.drawcoastlines(ax=ax)
plt.show()
plt.close(fig)
def get_area_avg_timeseries(samples_dir,
start_year=-np.Inf,
end_year=np.Inf,
filename_prefix="pm",
level=-1,
level_kind=level_kinds.ARBITRARY,
varname="",
mask=None,
mask_lons2d=None,
mask_lats2d=None,
file_per_var=False) -> pd.Series:
"""
get the timeseries of area averaged ice fraction
:rtype : pd.Series
"""
yearly_ts = []
lons2d, lats2d = None, None
samples_dir_p = Path(samples_dir)
# interpolated mask
interpolated_mask = None
for y in range(start_year, end_year + 1):
files_for_year = []
mfolders = [
f for f in samples_dir_p.iterdir() if f.name[:-2].endswith(str(y))
]
for mfolder in mfolders:
# Select all files containing the varname in the filename
if file_per_var:
files_for_year += [
str(f) for f in mfolder.iterdir() if varname in f.name
]
else:
files_for_year += [
str(f) for f in mfolder.iterdir()
if f.name.startswith(filename_prefix)
and f.name[-9:-1] != "0" * 8
]
if len(files_for_year) == 0:
continue
mrpn = MultiRPN(files_for_year)
data = mrpn.get_all_time_records_for_name_and_level(
varname=varname, level=level, level_kind=level_kind)
if lons2d is None:
lons2d, lats2d = mrpn.get_longitudes_and_latitudes_of_the_last_read_rec(
)
# interpolate the mask using nearest neighbour approach
if interpolated_mask is None:
xs, ys, zs = lat_lon.lon_lat_to_cartesian(mask_lons2d.flatten(),
mask_lats2d.flatten())
ktree = KDTree(data=list(zip(xs, ys, zs)))
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons2d.flatten(),
lats2d.flatten())
dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1)
interpolated_mask = mask.flatten()[inds]
interpolated_mask.shape = lons2d.shape
for t, field in data.items():
data[t] = field[interpolated_mask].mean()
tlist = [t for t in data.keys()]
ser = pd.Series(index=tlist, data=[data[t] for t in tlist])
ser.sort_index(inplace=True)
yearly_ts.append(ser)
mrpn.close()
return pd.concat(yearly_ts), lons2d, lats2d
def main():
#path = "/RECH/data/Simulations/CRCM5/North_America/NorthAmerica_0.44deg_ERA40-Int_B1/Diagnostics/NorthAmerica_0.44deg_ERA40-Int_B1_2007{:02d}"
path = "/RESCUE/skynet3_rech1/huziy/from_guillimin/new_outputs/current_climate_30_yr_sims/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/Samples/quebec_crcm5-hcd-rl-intfl_1988{:02d}"
months = [6, 7, 8]
pm_list = []
dm_list = []
for m in months:
print(path.format(m))
month_folder = path.format(m)
for fn in os.listdir(month_folder):
# if not fn.endswith("moyenne"):
# continue
if fn.startswith("pm"):
pm_list.append(os.path.join(month_folder, fn))
elif fn.startswith("dm"):
dm_list.append(os.path.join(month_folder, fn))
pm = MultiRPN(pm_list)
dm = MultiRPN(dm_list)
tsurf_mean = np.mean([
field for field in pm.get_all_time_records_for_name_and_level(
varname="J8").values()
],
axis=0)
tair_mean = np.mean([
field for field in dm.get_all_time_records_for_name_and_level(
varname="TT", level=1, level_kind=level_kinds.HYBRID).values()
],
axis=0)
lons, lats = pm.get_longitudes_and_latitudes_of_the_last_read_rec()
projparams = pm.linked_robj_list[
0].get_proj_parameters_for_the_last_read_rec()
rll = RotatedLatLon(**projparams)
bmp = rll.get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats)
xx, yy = bmp(lons, lats)
plt.figure()
cs = bmp.contourf(xx, yy, tsurf_mean - 273.15, 40)
bmp.drawcoastlines()
plt.title("Tsurf")
plt.colorbar()
plt.figure()
bmp.contourf(xx,
yy,
tair_mean,
levels=cs.levels,
norm=cs.norm,
cmap=cs.cmap)
bmp.drawcoastlines()
plt.title("Tair")
plt.colorbar()
plt.figure()
bmp.contourf(xx,
yy,
tsurf_mean - 273.15 - tair_mean,
levels=np.arange(-2, 2.2, 0.2),
cmap=cs.cmap)
bmp.drawcoastlines()
plt.title("Tsurf - Tair")
plt.colorbar()
pm.close()
dm.close()
plt.show()
def main():
#path = "/RECH/data/Simulations/CRCM5/North_America/NorthAmerica_0.44deg_ERA40-Int_B1/Diagnostics/NorthAmerica_0.44deg_ERA40-Int_B1_2007{:02d}"
path = "/RESCUE/skynet3_rech1/huziy/from_guillimin/new_outputs/current_climate_30_yr_sims/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/Samples/quebec_crcm5-hcd-rl-intfl_1988{:02d}"
months = [6, 7, 8]
pm_list = []
dm_list = []
for m in months:
print(path.format(m))
month_folder = path.format(m)
for fn in os.listdir(month_folder):
# if not fn.endswith("moyenne"):
# continue
if fn.startswith("pm"):
pm_list.append(os.path.join(month_folder, fn))
elif fn.startswith("dm"):
dm_list.append(os.path.join(month_folder, fn))
pm = MultiRPN(pm_list)
dm = MultiRPN(dm_list)
tsurf_mean = np.mean([field for field in pm.get_all_time_records_for_name_and_level(varname="J8").values()], axis=0)
tair_mean = np.mean([field for field in dm.get_all_time_records_for_name_and_level(varname="TT", level=1, level_kind=level_kinds.HYBRID).values()], axis=0)
lons, lats = pm.get_longitudes_and_latitudes_of_the_last_read_rec()
projparams = pm.linked_robj_list[0].get_proj_parameters_for_the_last_read_rec()
rll = RotatedLatLon(**projparams)
bmp = rll.get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats)
xx, yy = bmp(lons, lats)
plt.figure()
cs = bmp.contourf(xx, yy, tsurf_mean - 273.15, 40)
bmp.drawcoastlines()
plt.title("Tsurf")
plt.colorbar()
plt.figure()
bmp.contourf(xx, yy, tair_mean, levels=cs.levels, norm=cs.norm, cmap=cs.cmap)
bmp.drawcoastlines()
plt.title("Tair")
plt.colorbar()
plt.figure()
bmp.contourf(xx, yy, tsurf_mean - 273.15 - tair_mean, levels=np.arange(-2, 2.2, 0.2), cmap=cs.cmap)
bmp.drawcoastlines()
plt.title("Tsurf - Tair")
plt.colorbar()
pm.close()
dm.close()
plt.show()
def main(plot_vals=False):
varname = "RFAC"
multiplier = 24 * 3600
data_folder = Path("/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/atm_data_for_Arman_simulations")
day_range = range(19, 22)
dates_of_interest = [datetime(2013, 6, d) for d in day_range]
img_folder = Path("calgary_flood/2D")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
lons, lats, bmp = None, None, None
the_mask = get_bow_river_basin_mask()
i_list, j_list = np.where(the_mask > 0.5)
imin, imax = i_list.min() - 2, i_list.max() + 5
jmin, jmax = j_list.min() - 2, j_list.max() + 5
# Calculate daily means
sim_label_to_date_to_mean = OrderedDict()
for sim_dir in data_folder.iterdir():
mr = MultiRPN(str(sim_dir.joinpath("pm*")))
print(str(sim_dir))
print(mr.get_number_of_records())
label = sim_dir.name.split("_")[-2].replace("NoDrain", "").replace("frozen", "Frozen").replace("Bow", "")
sim_label_to_date_to_mean[label] = OrderedDict()
data = mr.get_4d_field(varname)
data = {d: list(v.items())[0][1] for d, v in data.items() if d.day in day_range}
for d in dates_of_interest:
sim_label_to_date_to_mean[label][d] = (
np.array([field for d1, field in data.items() if d1.day == d.day]).mean(axis=0) * multiplier
)
if lons is None:
lons, lats = mr.get_longitudes_and_latitudes_of_the_last_read_rec()
for f in sim_dir.iterdir():
if f.name.startswith("pm"):
r = RPN(str(f))
r.get_first_record_for_name(varname=varname)
rll = RotatedLatLon(**r.get_proj_parameters_for_the_last_read_rec())
bmp = rll.get_basemap_object_for_lons_lats(
lons2d=lons[imin:imax, jmin:jmax], lats2d=lats[imin:imax, jmin:jmax], resolution="i"
)
r.close()
break
mr.close()
# reorder simulations
sim_label_to_date_to_mean = OrderedDict(
[
(k, sim_label_to_date_to_mean[k])
for k in sorted(sim_label_to_date_to_mean, key=lambda z: len(z), reverse=True)
]
)
key_list = [k for k in sim_label_to_date_to_mean]
key_list[-2], key_list[-1] = key_list[-1], key_list[-2]
sim_label_to_date_to_mean = OrderedDict([(k, sim_label_to_date_to_mean[k]) for k in key_list])
# do the plots (subplots: vertically - simulations, horizontally - days)
plot_utils.apply_plot_params(width_cm=24, height_cm=28, font_size=10)
fig = plt.figure()
nrows = len(sim_label_to_date_to_mean)
ncols = len(day_range)
gs = GridSpec(nrows, ncols, wspace=0, hspace=0.1)
clevs_vals = [0, 0.1, 20, 50, 100, 150, 200, 300]
base_label = None
clevs_diff = [1, 5, 10, 20, 50, 100]
clevs_diff = [-c for c in reversed(clevs_diff)] + [0] + clevs_diff
cmap_diff = cm.get_cmap("bwr", len(clevs_diff) - 1)
cmap_vals = get_cmap_from_ncl_spec_file(path="colormap_files/precip3_16lev.rgb", ncolors=len(clevs_vals) - 1)
xx, yy = bmp(lons, lats)
title = "Total runoff ({}, mm/day)".format(varname)
fig.suptitle(title)
for row, (sim_label, date_to_field) in enumerate(sim_label_to_date_to_mean.items()):
if row == 0 or plot_vals:
base_sim = OrderedDict([(k, 0) for k, v in date_to_field.items()])
base_label = sim_label
plot_label = sim_label
clevs = clevs_vals
cmap = cmap_vals
extend = "max"
else:
base_sim = list(sim_label_to_date_to_mean.items())[0][1]
plot_label = "{}\n-\n{}".format(sim_label, base_label)
clevs = clevs_diff
cmap = cmap_diff
extend = "both"
bn = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1)
for col, (the_date, field) in enumerate(date_to_field.items()):
ax = fig.add_subplot(gs[row, col])
to_plot = np.ma.masked_where(the_mask < 0.5, field - base_sim[the_date])
# cs = bmp.contourf(xx[~to_plot.mask], yy[~to_plot.mask], to_plot[~to_plot.mask], levels=clevs, extend="max", tri=True)
cs = bmp.pcolormesh(xx, yy, to_plot[:-1, :-1], norm=bn, vmin=clevs[0], vmax=clevs[-1], cmap=cmap)
bmp.drawcoastlines(ax=ax, linewidth=0.3)
assert isinstance(bmp, Basemap)
bmp.readshapefile(BOW_RIVER_SHP[:-4], "basin", zorder=5)
cb = bmp.colorbar(cs, ax=ax, ticks=clevs, extend=extend, pad="4%", size="10%")
if plot_vals:
cb.ax.set_visible(col == ncols - 1 and row == 0)
else:
cb.ax.set_visible(col == ncols - 1 and row in (0, 1))
if col == 0:
ax.set_ylabel(plot_label)
if row == 0:
ax.set_title(the_date.strftime("%b %d"))
if plot_vals:
img_file = img_folder.joinpath("{}.png".format(varname))
else:
img_file = img_folder.joinpath("{}_diff.png".format(varname))
fig.savefig(str(img_file))
plt.close(fig)
