def get_lake_fraction_for_date(self, the_date=None):
"""
Get the lake ice cover fraction for the specified date
:param the_date:
:return:
"""
if the_date not in self.cached_data:
month_date = datetime(the_date.year, the_date.month, 1)
self.cached_data = {}
mr = MultiRPN(self.year_month_date_to_file_list[month_date])
self.cached_data = mr.get_all_time_records_for_name_and_level(varname=self.varname, level=self.level,
level_kind=self.level_kind)
mr.close()
# if still it is not there try looking in the previous month
if the_date not in self.cached_data:
month = the_date.month - 1
year = the_date.year
if the_date.month == 0:
month = 12
year -= 1
month_date = datetime(year, month, 1)
mr = MultiRPN(self.year_month_date_to_file_list[month_date])
self.cached_data.update(mr.get_all_time_records_for_name_and_level(varname=self.varname, level=self.level,
level_kind=self.level_kind))
mr.close()
return self.cached_data[the_date]
def test_getting_coordinates_for_the_last_read_record_should_not_fail():
r = None
try:
r = MultiRPN(in_path)
rec = r.get_first_record_for_name("I5")
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
tools.assert_equals(lons.shape, lats.shape, "Shapes of lon and lat arrays should be the same.")
finally:
if r is not None:
r.close()
delete_files()
def test_get_number_of_records():
r = None
try:
create_files()
r = MultiRPN("test_?.rpn")
msg = "Number of records should be equal to the total number in all files"
tools.assert_equals(r.get_number_of_records(), len(FILE_NAMES), msg)
finally:
if r is not None:
r.close()
delete_files()
def test_can_link_many_files(nfiles=200):
many_fnames = ["test_{}.rpn".format(i) for i in range(nfiles)]
r = None
try:
create_files(fnames=many_fnames)
r = MultiRPN("test_*.rpn")
tools.assert_greater(r.get_number_of_records(), 0, msg="There should be more than 0 records")
finally:
if r is not None:
r.close()
delete_files(fnames=many_fnames)
def test_should_find_all_records():
r = None
try:
create_files()
r = MultiRPN("test_?.rpn")
for fn in range(len(FILE_NAMES)):
rec = r.get_first_record_for_name("T{}".format(fn))
print("file {} - OK".format(fn))
print(rec.mean(), rec.shape)
finally:
if r is not None:
r.close()
delete_files()
def calculate_seasonal_mean_winds(seasons=commons.default_seasons,
level_hpa=None,
samples_folder=None,
file_prefix="dp"):
cache_file_name = "seasonal_wind_components_{}_{}hPa_{}.npz".format(
"-".join(seasons.keys()), level_hpa,
hashlib.sha224(str(samples_folder).encode()).hexdigest())
cache_file = Path(cache_file_name)
if cache_file.is_file():
print("Using cached winds from {}".format(cache_file))
return np.load(cache_file_name)["arr_0"]
u_season_to_mean = OrderedDict()
v_season_to_mean = OrderedDict()
r = None
for sname, months in seasons.items():
# find files for the season
paths = []
for m in months:
paths.extend(
glob.glob(
str(
samples_folder.joinpath("*{:02d}/{}*".format(
m, file_prefix)))))
r = MultiRPN(paths)
u_data = r.get_4d_field(varname="UU")
v_data = r.get_4d_field(varname="VV")
u_season_to_mean[sname] = np.array(
[v[level_hpa] for k, v in u_data.items()]).mean(axis=0)
v_season_to_mean[sname] = np.array(
[v[level_hpa] for k, v in v_data.items()]).mean(axis=0)
print("Processed: {}".format(sname))
lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec()
np.savez_compressed(cache_file_name,
[lons2d, lats2d, u_season_to_mean, v_season_to_mean])
return lons2d, lats2d, u_season_to_mean, v_season_to_mean
def extract_runoff_to_nc_process(args):
in_path, out_path = args
if os.path.exists(out_path):
print("Nothing to do for: {}".format(out_path))
return # skip files that already exist
traf_name = "TRAF"
tdra_name = "TDRA"
r = MultiRPN(in_path)
traf_data = r.get_all_time_records_for_name_and_level(varname=traf_name, level=5, level_kind=level_kinds.ARBITRARY)
tdra_data = r.get_all_time_records_for_name_and_level(varname=tdra_name, level=5, level_kind=level_kinds.ARBITRARY)
r.close()
nx, ny = list(traf_data.items())[0][1].shape
with nc.Dataset(out_path, "w", format="NETCDF3_CLASSIC") as ds:
ds.createDimension("lon", nx)
ds.createDimension("lat", ny)
ds.createDimension("time", None)
varTraf = ds.createVariable(traf_name, "f4", dimensions=("time", "lon", "lat"))
varTraf.units = "kg/( m**2 * s )"
varTdra = ds.createVariable(tdra_name, "f4", dimensions=("time", "lon", "lat"))
varTdra.units = "kg/( m**2 * s )"
timeVar = ds.createVariable("time", "f4", dimensions=("time",))
sorted_dates = list(sorted(traf_data.keys()))
timeVar.units = "hours since {0}".format(sorted_dates[0])
timeVar[:] = nc.date2num(sorted_dates, timeVar.units)
varTraf[:] = np.array(
[traf_data[d] for d in sorted_dates]
)
varTdra[:] = np.array(
[tdra_data[d] for d in sorted_dates]
)
def extract_runoff_to_nc_process(args):
in_path, out_path = args
if os.path.exists(out_path):
print("Nothing to do for: {}".format(out_path))
return # skip files that already exist
traf_name = "TRAF"
tdra_name = "TDRA"
r = MultiRPN(in_path)
traf_data = r.get_all_time_records_for_name_and_level(varname=traf_name, level=5, level_kind=level_kinds.ARBITRARY)
tdra_data = r.get_all_time_records_for_name_and_level(varname=tdra_name, level=5, level_kind=level_kinds.ARBITRARY)
r.close()
nx, ny = list(traf_data.items())[0][1].shape
with nc.Dataset(out_path, "w", format="NETCDF3_CLASSIC") as ds:
ds.createDimension("lon", nx)
ds.createDimension("lat", ny)
ds.createDimension("time", None)
varTraf = ds.createVariable(traf_name, "f4", dimensions=("time", "lon", "lat"))
varTraf.units = "kg/( m**2 * s )"
varTdra = ds.createVariable(tdra_name, "f4", dimensions=("time", "lon", "lat"))
varTdra.units = "kg/( m**2 * s )"
timeVar = ds.createVariable("time", "f4", dimensions=("time",))
sorted_dates = list(sorted(traf_data.keys()))
timeVar.units = "hours since {0}".format(sorted_dates[0])
timeVar[:] = nc.date2num(sorted_dates, timeVar.units)
varTraf[:] = np.array(
[traf_data[d] for d in sorted_dates]
)
varTdra[:] = np.array(
[tdra_data[d] for d in sorted_dates]
)
def test_get_4d_field():
r = None
vname = "T"
try:
create_files_with_same_var_for_different_times(vname=vname)
r = MultiRPN("test_?.rpn")
recs = r.get_4d_field(vname)
msg = "Not all records for {} were found".format(vname)
print(recs.keys())
tools.assert_equals(len(FILE_NAMES), len(recs), msg)
finally:
if r is not None:
r.close()
delete_files()
def test_get_list_of_varnames():
r = None
vname = "T"
try:
create_files_with_same_var_for_different_times(vname=vname)
r = MultiRPN("test_?.rpn")
msg = "the list of varnames should contain {}".format(vname)
vnames = r.get_list_of_varnames()
tools.assert_in(vname, vnames, msg)
tools.assert_equal(len(vnames), 1, "There is only one unique field name in the files")
finally:
if r is not None:
r.close()
delete_files()
def get_seasonal_means_from_rpn_monthly_folders(samples_dir="", season_to_months=None, start_year=-np.Inf, end_year=np.Inf,
filename_prefix="pm", varname="", level=-1, level_kind=-1):
result = OrderedDict()
season_to_files = {s: [] for s in season_to_months}
smpls_dir = Path(samples_dir)
for month_folder in smpls_dir.iterdir():
y, m = get_year_and_month(month_folder.name)
# skip if the year is not in the selected range
if not start_year <= y <= end_year:
continue
for season, months in season_to_months.items():
if m in months:
for data_file in month_folder.iterdir():
if data_file.name[-9:-1] == 8 * "0":
continue
if data_file.name.startswith(filename_prefix):
season_to_files[season].append(str(data_file))
for season, months in season_to_months.items():
print("{} => {}".format(season, months))
mrpn = MultiRPN(season_to_files[season])
date_to_field = mrpn.get_all_time_records_for_name_and_level(varname=varname, level=level, level_kind=level_kind)
result[season] = np.mean([field for field in date_to_field.values()], axis=0)
return result
def main():
path_to_nemo_outputs = "/RESCUE/skynet3_rech1/huziy/one_way_coupled_nemo_outputs_1979_1985/GLK_1d_grid_T.nc"
vname_nemo = "sosstsst"
path_to_crcm5_outputs = "/home/huziy/skynet3_rech1/glk-oneway-coupled-crcm5-outputs/coupled-GL/Samples"
vname_crcm5 = "TT"
# month of interest
month = 4
year = 1981
mrpn = MultiRPN("{}/*_{}{}/dm*".format(path_to_crcm5_outputs, year, month))
data = mrpn.get_all_time_records_for_name_and_level(varname=vname_crcm5, level=1, level_kind=level_kinds.HYBRID)
# Calculate the monthly mean fields in both cases
assert isinstance(data, dict)
mm_crcm5 = np.array(list(data.values())).mean(axis=0)
print("crcm5-out-shape = ", mm_crcm5.shape)
with Dataset(path_to_nemo_outputs) as ds:
mm_nemo = ds.variables[vname_nemo][:].mean(axis=0)
print("nemo-out-shape = ", mm_nemo.shape)
def calculate_seasonal_mean_winds(seasons=commons.default_seasons, level_hpa=None, samples_folder=None, file_prefix="dp"):
cache_file_name = "seasonal_wind_components_{}_{}hPa_{}.npz".format("-".join(seasons.keys()), level_hpa,
hashlib.sha224(str(samples_folder).encode()).hexdigest())
cache_file = Path(cache_file_name)
if cache_file.is_file():
print("Using cached winds from {}".format(cache_file))
return np.load(cache_file_name)["arr_0"]
u_season_to_mean = OrderedDict()
v_season_to_mean = OrderedDict()
r = None
for sname, months in seasons.items():
# find files for the season
paths = []
for m in months:
paths.extend(glob.glob(str(samples_folder.joinpath("*{:02d}/{}*".format(m, file_prefix)))))
r = MultiRPN(paths)
u_data = r.get_4d_field(varname="UU")
v_data = r.get_4d_field(varname="VV")
u_season_to_mean[sname] = np.array([v[level_hpa] for k, v in u_data.items()]).mean(axis=0)
v_season_to_mean[sname] = np.array([v[level_hpa] for k, v in v_data.items()]).mean(axis=0)
print("Processed: {}".format(sname))
lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec()
np.savez_compressed(cache_file_name, [lons2d, lats2d, u_season_to_mean, v_season_to_mean])
return lons2d, lats2d, u_season_to_mean, v_season_to_mean
def main():
path_to_nemo_outputs = "/RESCUE/skynet3_rech1/huziy/one_way_coupled_nemo_outputs_1979_1985/GLK_1d_grid_T.nc"
vname_nemo = "sosstsst"
path_to_crcm5_outputs = "/home/huziy/skynet3_rech1/glk-oneway-coupled-crcm5-outputs/coupled-GL/Samples"
vname_crcm5 = "TT"
# month of interest
month = 4
year = 1981
mrpn = MultiRPN("{}/*_{}{}/dm*".format(path_to_crcm5_outputs, year, month))
data = mrpn.get_all_time_records_for_name_and_level(
varname=vname_crcm5, level=1, level_kind=level_kinds.HYBRID)
# Calculate the monthly mean fields in both cases
assert isinstance(data, dict)
mm_crcm5 = np.array(list(data.values())).mean(axis=0)
print("crcm5-out-shape = ", mm_crcm5.shape)
with Dataset(path_to_nemo_outputs) as ds:
mm_nemo = ds.variables[vname_nemo][:].mean(axis=0)
print("nemo-out-shape = ", mm_nemo.shape)
def get_lake_fraction_for_date(self, the_date=None):
"""
Get the lake ice cover fraction for the specified date
:param the_date:
:return:
"""
if the_date not in self.cached_data:
month_date = datetime(the_date.year, the_date.month, 1)
self.cached_data = {}
mr = MultiRPN(self.year_month_date_to_file_list[month_date])
self.cached_data = mr.get_all_time_records_for_name_and_level(
varname=self.varname, level=self.level, level_kind=self.level_kind
)
mr.close()
# if still it is not there try looking in the previous month
if the_date not in self.cached_data:
month = the_date.month - 1
year = the_date.year
if the_date.month == 0:
month = 12
year -= 1
month_date = datetime(year, month, 1)
mr = MultiRPN(self.year_month_date_to_file_list[month_date])
self.cached_data.update(
mr.get_all_time_records_for_name_and_level(
varname=self.varname, level=self.level, level_kind=self.level_kind
)
)
mr.close()
return self.cached_data[the_date]
def main():
plot_utils.apply_plot_params(width_cm=40, height_cm=25, font_size=14)
seasons = commons.default_seasons
dx_plotting = 0.5 # degrees
# units = "$^\circ$C"
# long_name = "SST, {}".format(units)
# var_name = "TM"
# clevs_diff = np.arange(-5, 5.5, 0.5)
units = ""
long_name = "Sea ice {}".format(units)
var_name = "LG"
clevs_diff = [v for v in np.arange(-1, 1.1, 0.1) if abs(v) > 1.0e-6]
label_to_folder = OrderedDict([
("ERA-Interim", "/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/SST_SeaIce/I_SST_SeaIce"),
("PreI-CanESM2", "/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce"),
("PreI-GFDL", "/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce"),
("PreI-GISS", "/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce")
])
bmp = Basemap(projection="robin", lon_0=180)
xx, yy = None, None
year = 2013
print(seasons)
label_to_season_to_field = OrderedDict()
for label, folder in label_to_folder.items():
label_to_season_to_field[label] = OrderedDict()
all_files = [f for f in os.listdir(folder) if label.lower() in f.lower() or label.split("-")[1].lower() in f.lower()]
for sname, months in seasons.items():
season_files = [f for f in all_files if _get_month_and_year_from_file_name(f)[0] in months and _get_month_and_year_from_file_name(f)[1] == year]
season_files = [os.path.join(folder, f) for f in season_files]
print(10 * "+" + sname + "+" * 10)
print(season_files)
r = MultiRPN(season_files)
data = r.get_4d_field(var_name)
the_mean = np.array([list(v.items())[0][1] for d, v in data.items()]).mean(axis=0)
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
print("lon range", lons.min(), lons.max())
print("lat range", lats.min(), lats.max())
# lons, lats, the_mean = commons.interpolate_to_uniform_global_grid(the_mean, lons_in=lons, lats_in=lats, out_dx=dx_plotting)
# mask land
lons1 = lons.copy()
lons1[lons1 > 180] -= 360
ocean_mask = maskoceans(lons1, lats, np.zeros_like(the_mean))
the_mean = np.ma.masked_where(~ocean_mask.mask, the_mean)
if xx is None:
xx, yy = bmp(lons, lats)
label_to_season_to_field[label][sname] = the_mean
# plotting------
fig = plt.figure()
gs = GridSpec(len(label_to_season_to_field), len(seasons) + 1, width_ratios=[1.0, ] * len(seasons) + [0.05, ])
#
base_fields = None
base_label = None
for row, (label, season_to_mean) in enumerate(label_to_season_to_field.items()):
print(gs.get_geometry())
axes = [fig.add_subplot(gs[row, col]) for col in range(len(season_to_mean) + 1)]
common_params = dict(axes=axes, bmp=bmp, xx=xx, yy=yy)
if base_fields is None:
plot_row(row, season_to_mean=season_to_mean, label=label, **common_params)
base_fields = season_to_mean
base_label = label
else:
plot_row(row, season_to_mean=OrderedDict([(k, v - base_fields[k]) for k, v in season_to_mean.items()]),
difference=True, clevs=clevs_diff, label="{}\n-\n{}".format(label, base_label),
**common_params)
fig.suptitle(long_name)
img_folder = Path("industrial_and_preindustrial_sst").joinpath("seasonal")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
img_format = "png"
img_file = img_folder.joinpath("{}.{}".format(var_name, img_format))
with img_file.open("wb") as f:
fig.savefig(f, bbox_inches="tight", format=img_format)
def plot_seasonal_integrated_water_vapour(samples_folder,
seasons=commons.default_seasons):
# TODO: make the integrations
vname = "HU"
file_prefix = "dm"
plot_units = "kg/kg"
mult_coeff = 1
add_offset = 0
out_dx = 0.5
season_to_mean = OrderedDict()
r = None
for sname, months in seasons.items():
# find files for the season
paths = []
for m in months:
paths.extend(
glob.glob(
str(
samples_folder.joinpath("*{:02d}/{}*".format(
m, file_prefix)))))
r = MultiRPN(paths)
data = r.get_4d_field(varname=vname)
season_to_mean[sname] = np.array(
[list(v.items())[0][1] for k, v in data.items()]).mean(axis=0)
print("Processed: {}".format(sname))
lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec()
img_folder = samples_folder.joinpath("images/seasonal")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
img_file = img_folder.joinpath("{}.png".format(vname))
plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18)
fig = plt.figure()
ncols = 2
gs = GridSpec(len(season_to_mean) // ncols +
int(not (len(season_to_mean) % ncols == 0)),
ncols,
wspace=0,
hspace=0)
xx, yy = None, None
bmp = Basemap(projection="robin", lon_0=0)
nlevs = 20
clevs = None
for i, (sname, field) in enumerate(season_to_mean.items()):
row = i // ncols
col = i % ncols
ax = fig.add_subplot(gs[row, col])
lons, lats, data_out = commons.interpolate_to_uniform_global_grid(
field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx)
if xx is None:
xx, yy = bmp(lons, lats)
cs = bmp.contourf(xx,
yy,
data_out * mult_coeff,
nlevs if clevs is None else clevs,
cmap=cm_basemap.s3pcpn_l,
extend="max")
# save color levels for next subplots
clevs = cs.levels
print(np.max(data_out * mult_coeff))
ax.set_title(sname)
cb = plt.colorbar(cs, ax=ax)
if not (row == 0 and col == ncols - 1):
# cb.ax.set_title(plot_units)
cb.ax.set_visible(False)
bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH)
fig.suptitle("Precipitation, {}".format(plot_units))
with img_file.open("wb") as f:
fig.savefig(f, bbox_inches="tight")
plt.close(fig)
def plot_seasonal_vertical_velocity(samples_folder,
seasons=commons.default_seasons):
level = 850 # millibars
vname = "WW"
long_name = "Vertical compoent of wind velocity ({}mb)".format(level)
file_prefix = "dp"
plot_units = "Pa/s"
mult_coeff = 1
add_offset = 0
out_dx = 0.5
file_format = "png"
season_to_mean = OrderedDict()
r = None
for sname, months in seasons.items():
# find files for the season
paths = []
for m in months:
paths.extend(
glob.glob(
str(
samples_folder.joinpath("*{:02d}/{}*".format(
m, file_prefix)))))
r = MultiRPN(paths)
data = r.get_4d_field(varname=vname)
season_to_mean[sname] = np.array([v[level] for k, v in data.items()
]).mean(axis=0)
print("Processed: {}".format(sname))
lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec()
img_folder = samples_folder.joinpath("images/seasonal")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
img_file = img_folder.joinpath("{}_{}hPa.{}".format(
vname, level, file_format))
plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18)
fig = plt.figure()
ncols = 2
gs = GridSpec(len(season_to_mean) // ncols +
int(not (len(season_to_mean) % ncols == 0)),
ncols,
wspace=0,
hspace=0)
xx, yy = None, None
bmp = Basemap(projection="robin", lon_0=0)
cmap = cm_basemap.GMT_polar
clevs = np.arange(-0.1, 0.11, 0.01)
for i, (sname, field) in enumerate(season_to_mean.items()):
row = i // ncols
col = i % ncols
ax = fig.add_subplot(gs[row, col])
lons, lats, data_out = commons.interpolate_to_uniform_global_grid(
field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx)
if xx is None:
xx, yy = bmp(lons, lats)
cs = bmp.contourf(xx,
yy,
data_out * mult_coeff,
20 if clevs is None else clevs,
cmap=cmap,
extend="both")
# save color levels for next subplots
clevs = cs.levels
print(np.max(data_out * mult_coeff))
ax.set_title(sname)
cb = plt.colorbar(cs, ax=ax)
if not (row == 0 and col == ncols - 1):
# cb.ax.set_title(plot_units)
cb.ax.set_visible(False)
bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH)
fig.suptitle("{}, {}".format(long_name, plot_units))
with img_file.open("wb") as f:
fig.savefig(f, bbox_inches="tight", format=file_format)
plt.close(fig)
def get_daily_clim_profiles(samples_dir, start_year=-np.Inf, end_year=np.Inf, filename_prefix="dp", varname="", mask=None):
"""
dates_2d, levels_2d, values - needed for plotting
:param start_year:
:param end_year:
:param filename_prefix:
:param varname:
"""
dates_sorted = None
levels_sorted = None
yearly_fields = []
day = timedelta(days=1)
stamp_dates = [datetime(2001, 1, 1) + i * day for i in range(365)]
for y in range(start_year, end_year + 1):
files_for_year = []
mfolders = [os.path.join(samples_dir, f) for f in os.listdir(samples_dir) if f[:-2].endswith(str(y))]
for mfolder in mfolders:
files_for_year += [os.path.join(mfolder, fn) for fn in os.listdir(mfolder) if fn.startswith(filename_prefix)]
mrpn = MultiRPN(files_for_year)
data = mrpn.get_4d_field(varname=varname)
# calculate the area-average
for d, lev_to_field in data.items():
for lev in lev_to_field:
lev_to_field[lev] = lev_to_field[lev][mask].mean()
#
dates_sorted = list(sorted([d for d in data if not (d.month == 2 and d.day == 29) and (d.year == y)]))
levels_sorted = list(sorted([lev for lev in data[dates_sorted[0]] if lev >= 500]))
#
year_field = np.zeros((len(dates_sorted), len(levels_sorted)))
for j, lev in enumerate(levels_sorted):
for i, the_date in enumerate(dates_sorted):
year_field[i, j] = data[the_date][lev]
panel = pd.DataFrame(data=year_field, index=dates_sorted, columns=range(len(levels_sorted)))
daily = panel.groupby(by=lambda d: datetime(d.year, d.month, d.day)).mean()
yearly_fields.append(daily.values)
values = np.mean(yearly_fields, axis=0)
assert not hasattr(values, "mask")
dates_num = date2num(stamp_dates)
levs_2d, dates_num_2d = np.meshgrid(levels_sorted, dates_num)
assert levs_2d.shape == values.shape, "levels and values shapes ({} and {}, respectively) are not consistent.".format(levs_2d.shape, values.shape)
return dates_num_2d, levs_2d, values
def plot_seasonal_integrated_water_vapour(samples_folder, seasons=commons.default_seasons):
# TODO: make the integrations
vname = "HU"
file_prefix = "dm"
plot_units = "kg/kg"
mult_coeff = 1
add_offset = 0
out_dx = 0.5
season_to_mean = OrderedDict()
r = None
for sname, months in seasons.items():
# find files for the season
paths = []
for m in months:
paths.extend(glob.glob(str(samples_folder.joinpath("*{:02d}/{}*".format(m, file_prefix)))))
r = MultiRPN(paths)
data = r.get_4d_field(varname=vname)
season_to_mean[sname] = np.array([list(v.items())[0][1] for k, v in data.items()]).mean(axis=0)
print("Processed: {}".format(sname))
lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec()
img_folder = samples_folder.joinpath("images/seasonal")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
img_file = img_folder.joinpath("{}.png".format(vname))
plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18)
fig = plt.figure()
ncols = 2
gs = GridSpec(len(season_to_mean) // ncols + int(not (len(season_to_mean) % ncols == 0)), ncols, wspace=0, hspace=0)
xx, yy = None, None
bmp = Basemap(projection="robin", lon_0=0)
nlevs = 20
clevs = None
for i, (sname, field) in enumerate(season_to_mean.items()):
row = i // ncols
col = i % ncols
ax = fig.add_subplot(gs[row, col])
lons, lats, data_out = commons.interpolate_to_uniform_global_grid(field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx)
if xx is None:
xx, yy = bmp(lons, lats)
cs = bmp.contourf(xx, yy, data_out * mult_coeff, nlevs if clevs is None else clevs, cmap=cm_basemap.s3pcpn_l, extend="max")
# save color levels for next subplots
clevs = cs.levels
print(np.max(data_out * mult_coeff))
ax.set_title(sname)
cb = plt.colorbar(cs, ax=ax)
if not (row == 0 and col == ncols - 1):
# cb.ax.set_title(plot_units)
cb.ax.set_visible(False)
bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH)
fig.suptitle("Precipitation, {}".format(plot_units))
with img_file.open("wb") as f:
fig.savefig(f, bbox_inches="tight")
plt.close(fig)
def get_area_avg_timeseries(samples_dir, start_year=-np.Inf, end_year=np.Inf, filename_prefix="pm",
level=-1, level_kind=level_kinds.ARBITRARY,
varname="", mask=None, mask_lons2d=None, mask_lats2d=None, file_per_var=False) -> pd.Series:
"""
get the timeseries of area averaged ice fraction
:rtype : pd.Series
"""
yearly_ts = []
lons2d, lats2d = None, None
samples_dir_p = Path(samples_dir)
# interpolated mask
interpolated_mask = None
for y in range(start_year, end_year + 1):
files_for_year = []
mfolders = [f for f in samples_dir_p.iterdir() if f.name[:-2].endswith(str(y))]
for mfolder in mfolders:
# Select all files containing the varname in the filename
if file_per_var:
files_for_year += [str(f) for f in mfolder.iterdir() if varname in f.name]
else:
files_for_year += [str(f) for f in mfolder.iterdir() if f.name.startswith(filename_prefix) and f.name[-9:-1] != "0" * 8]
if len(files_for_year) == 0:
continue
mrpn = MultiRPN(files_for_year)
data = mrpn.get_all_time_records_for_name_and_level(varname=varname, level=level, level_kind=level_kind)
if lons2d is None:
lons2d, lats2d = mrpn.get_longitudes_and_latitudes_of_the_last_read_rec()
# interpolate the mask using nearest neighbour approach
if interpolated_mask is None:
xs, ys, zs = lat_lon.lon_lat_to_cartesian(mask_lons2d.flatten(), mask_lats2d.flatten())
ktree = KDTree(data=list(zip(xs, ys, zs)))
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten())
dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1)
interpolated_mask = mask.flatten()[inds]
interpolated_mask.shape = lons2d.shape
for t, field in data.items():
data[t] = field[interpolated_mask].mean()
tlist = [t for t in data.keys()]
ser = pd.Series(index=tlist, data=[data[t] for t in tlist])
ser.sort_index(inplace=True)
yearly_ts.append(ser)
mrpn.close()
return pd.concat(yearly_ts), lons2d, lats2d
def main():
start_year = 1979
end_year = 1981
HL_LABEL = "CRCM5_HL"
NEMO_LABEL = "CRCM5_NEMO"
file_prefix = "dm"
level = 1
level_type = level_kinds.HYBRID
wind_comp_names = ["UU", "VV"]
sim_label_to_path = OrderedDict(
[(HL_LABEL, "/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples"),
(NEMO_LABEL, "/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples")]
)
# get a coord file ...
coord_file = ""
found_coord_file = False
for mdir in os.listdir(sim_label_to_path[HL_LABEL]):
mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir)
if not os.path.isdir(mdir_path):
continue
for fn in os.listdir(mdir_path):
print(fn)
if fn[:2] not in ["pm", "dm", "pp", "dp"]:
continue
coord_file = os.path.join(mdir_path, fn)
found_coord_file = True
if found_coord_file:
break
bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file(path=coord_file)
xx, yy = bmp(lons, lats)
lons[lons > 180] -= 360
# loop through all files rotate vaectors and save to netcdf
for sim_label, samples_dir in sim_label_to_path.items():
samples = Path(samples_dir)
po = samples.parent
monthdate_to_path_list = nemo_hl_util.get_monthyeardate_to_paths_map(file_prefix=file_prefix,
start_year=start_year, end_year=end_year,
samples_dir_path=samples)
# Netcdf output file to put rotated winds
po /= "rotated_wind_{}.nc".format(sim_label)
with Dataset(str(po), "w") as ds:
ds.createDimension("time", None)
ds.createDimension("lon", lons.shape[0])
ds.createDimension("lat", lons.shape[1])
# create the schema of the output file
vname_to_ncvar = {}
for vname in wind_comp_names:
vname_to_ncvar[vname] = ds.createVariable(vname, "f4", dimensions=("time", "lon", "lat"))
vname_to_ncvar[vname].units = "knots"
lons_var = ds.createVariable("lon", "f4", dimensions=("lon", "lat"))
lats_var = ds.createVariable("lat", "f4", dimensions=("lon", "lat"))
time_var = ds.createVariable("time", "i8", dimensions=("time",))
time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format(datetime(start_year, 1, 1))
lons_var[:] = lons
lats_var[:] = lats
# use sorted dates
record_count = 0
for month_date in sorted(monthdate_to_path_list):
# select only dm files
mr = MultiRPN(path=monthdate_to_path_list[month_date])
vname_to_fields = {}
for vname in wind_comp_names:
vname_to_fields[vname] = mr.get_all_time_records_for_name_and_level(varname=vname, level=level, level_kind=level_type)
for ti, t in enumerate(sorted(vname_to_fields[wind_comp_names[0]])):
time_var[record_count] = date2num(t, time_var.units)
uu = vname_to_fields[wind_comp_names[0]][t]
vv = vname_to_fields[wind_comp_names[1]][t]
uu_rot, vv_rot = rotate_vecs_from_geo_to_rotpole(uu, vv, lons, lats, bmp=bmp)
# in knots not in m/s
vname_to_ncvar[wind_comp_names[0]][record_count, :, :] = uu_rot
vname_to_ncvar[wind_comp_names[1]][record_count, :, :] = vv_rot
record_count += 1
def main(plot_vals=False):
varname = "RFAC"
multiplier = 24 * 3600
data_folder = Path(
"/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/atm_data_for_Arman_simulations"
)
day_range = range(19, 22)
dates_of_interest = [datetime(2013, 6, d) for d in day_range]
img_folder = Path("calgary_flood/2D")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
lons, lats, bmp = None, None, None
the_mask = get_bow_river_basin_mask()
i_list, j_list = np.where(the_mask > 0.5)
imin, imax = i_list.min() - 2, i_list.max() + 5
jmin, jmax = j_list.min() - 2, j_list.max() + 5
# Calculate daily means
sim_label_to_date_to_mean = OrderedDict()
for sim_dir in data_folder.iterdir():
mr = MultiRPN(str(sim_dir.joinpath("pm*")))
print(str(sim_dir))
print(mr.get_number_of_records())
label = sim_dir.name.split("_")[-2].replace("NoDrain", "").replace(
"frozen", "Frozen").replace("Bow", "")
sim_label_to_date_to_mean[label] = OrderedDict()
data = mr.get_4d_field(varname)
data = {
d: list(v.items())[0][1]
for d, v in data.items() if d.day in day_range
}
for d in dates_of_interest:
sim_label_to_date_to_mean[label][d] = np.array([
field for d1, field in data.items() if d1.day == d.day
]).mean(axis=0) * multiplier
if lons is None:
lons, lats = mr.get_longitudes_and_latitudes_of_the_last_read_rec()
for f in sim_dir.iterdir():
if f.name.startswith("pm"):
r = RPN(str(f))
r.get_first_record_for_name(varname=varname)
rll = RotatedLatLon(
**r.get_proj_parameters_for_the_last_read_rec())
bmp = rll.get_basemap_object_for_lons_lats(
lons2d=lons[imin:imax, jmin:jmax],
lats2d=lats[imin:imax, jmin:jmax],
resolution="i")
r.close()
break
mr.close()
# reorder simulations
sim_label_to_date_to_mean = OrderedDict([
(k, sim_label_to_date_to_mean[k]) for k in sorted(
sim_label_to_date_to_mean, key=lambda z: len(z), reverse=True)
])
key_list = [k for k in sim_label_to_date_to_mean]
key_list[-2], key_list[-1] = key_list[-1], key_list[-2]
sim_label_to_date_to_mean = OrderedDict([(k, sim_label_to_date_to_mean[k])
for k in key_list])
# do the plots (subplots: vertically - simulations, horizontally - days)
plot_utils.apply_plot_params(width_cm=24, height_cm=28, font_size=10)
fig = plt.figure()
nrows = len(sim_label_to_date_to_mean)
ncols = len(day_range)
gs = GridSpec(nrows, ncols, wspace=0, hspace=0.1)
clevs_vals = [0, 0.1, 20, 50, 100, 150, 200, 300]
base_label = None
clevs_diff = [1, 5, 10, 20, 50, 100]
clevs_diff = [-c for c in reversed(clevs_diff)] + [0] + clevs_diff
cmap_diff = cm.get_cmap("bwr", len(clevs_diff) - 1)
cmap_vals = get_cmap_from_ncl_spec_file(
path="colormap_files/precip3_16lev.rgb", ncolors=len(clevs_vals) - 1)
xx, yy = bmp(lons, lats)
title = "Total runoff ({}, mm/day)".format(varname)
fig.suptitle(title)
for row, (sim_label,
date_to_field) in enumerate(sim_label_to_date_to_mean.items()):
if row == 0 or plot_vals:
base_sim = OrderedDict([(k, 0) for k, v in date_to_field.items()])
base_label = sim_label
plot_label = sim_label
clevs = clevs_vals
cmap = cmap_vals
extend = "max"
else:
base_sim = list(sim_label_to_date_to_mean.items())[0][1]
plot_label = "{}\n-\n{}".format(sim_label, base_label)
clevs = clevs_diff
cmap = cmap_diff
extend = "both"
bn = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1)
for col, (the_date, field) in enumerate(date_to_field.items()):
ax = fig.add_subplot(gs[row, col])
to_plot = np.ma.masked_where(the_mask < 0.5,
field - base_sim[the_date])
# cs = bmp.contourf(xx[~to_plot.mask], yy[~to_plot.mask], to_plot[~to_plot.mask], levels=clevs, extend="max", tri=True)
cs = bmp.pcolormesh(xx,
yy,
to_plot[:-1, :-1],
norm=bn,
vmin=clevs[0],
vmax=clevs[-1],
cmap=cmap)
bmp.drawcoastlines(ax=ax, linewidth=0.3)
assert isinstance(bmp, Basemap)
bmp.readshapefile(BOW_RIVER_SHP[:-4], "basin", zorder=5)
cb = bmp.colorbar(cs,
ax=ax,
ticks=clevs,
extend=extend,
pad="4%",
size="10%")
if plot_vals:
cb.ax.set_visible(col == ncols - 1 and row == 0)
else:
cb.ax.set_visible(col == ncols - 1 and row in (0, 1))
if col == 0:
ax.set_ylabel(plot_label)
if row == 0:
ax.set_title(the_date.strftime("%b %d"))
if plot_vals:
img_file = img_folder.joinpath("{}.png".format(varname))
else:
img_file = img_folder.joinpath("{}_diff.png".format(varname))
fig.savefig(str(img_file))
plt.close(fig)
def main():
plot_utils.apply_plot_params(width_cm=40, height_cm=25, font_size=14)
seasons = commons.default_seasons
dx_plotting = 0.5 # degrees
# units = "$^\circ$C"
# long_name = "SST, {}".format(units)
# var_name = "TM"
# clevs_diff = np.arange(-5, 5.5, 0.5)
units = ""
long_name = "Sea ice {}".format(units)
var_name = "LG"
clevs_diff = [v for v in np.arange(-1, 1.1, 0.1) if abs(v) > 1.0e-6]
label_to_folder = OrderedDict([
("ERA-Interim",
"/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/SST_SeaIce/I_SST_SeaIce"
),
("PreI-CanESM2",
"/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce"
),
("PreI-GFDL",
"/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce"
),
("PreI-GISS",
"/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce"
)
])
bmp = Basemap(projection="robin", lon_0=180)
xx, yy = None, None
year = 2013
print(seasons)
label_to_season_to_field = OrderedDict()
for label, folder in label_to_folder.items():
label_to_season_to_field[label] = OrderedDict()
all_files = [
f for f in os.listdir(folder) if label.lower() in f.lower()
or label.split("-")[1].lower() in f.lower()
]
for sname, months in seasons.items():
season_files = [
f for f in all_files
if _get_month_and_year_from_file_name(f)[0] in months
and _get_month_and_year_from_file_name(f)[1] == year
]
season_files = [os.path.join(folder, f) for f in season_files]
print(10 * "+" + sname + "+" * 10)
print(season_files)
r = MultiRPN(season_files)
data = r.get_4d_field(var_name)
the_mean = np.array(
[list(v.items())[0][1] for d, v in data.items()]).mean(axis=0)
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
print("lon range", lons.min(), lons.max())
print("lat range", lats.min(), lats.max())
# lons, lats, the_mean = commons.interpolate_to_uniform_global_grid(the_mean, lons_in=lons, lats_in=lats, out_dx=dx_plotting)
# mask land
lons1 = lons.copy()
lons1[lons1 > 180] -= 360
ocean_mask = maskoceans(lons1, lats, np.zeros_like(the_mean))
the_mean = np.ma.masked_where(~ocean_mask.mask, the_mean)
if xx is None:
xx, yy = bmp(lons, lats)
label_to_season_to_field[label][sname] = the_mean
# plotting------
fig = plt.figure()
gs = GridSpec(len(label_to_season_to_field),
len(seasons) + 1,
width_ratios=[
1.0,
] * len(seasons) + [
0.05,
])
#
base_fields = None
base_label = None
for row, (label,
season_to_mean) in enumerate(label_to_season_to_field.items()):
print(gs.get_geometry())
axes = [
fig.add_subplot(gs[row, col])
for col in range(len(season_to_mean) + 1)
]
common_params = dict(axes=axes, bmp=bmp, xx=xx, yy=yy)
if base_fields is None:
plot_row(row,
season_to_mean=season_to_mean,
label=label,
**common_params)
base_fields = season_to_mean
base_label = label
else:
plot_row(row,
season_to_mean=OrderedDict([
(k, v - base_fields[k])
for k, v in season_to_mean.items()
]),
difference=True,
clevs=clevs_diff,
label="{}\n-\n{}".format(label, base_label),
**common_params)
fig.suptitle(long_name)
img_folder = Path("industrial_and_preindustrial_sst").joinpath("seasonal")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
img_format = "png"
img_file = img_folder.joinpath("{}.{}".format(var_name, img_format))
with img_file.open("wb") as f:
fig.savefig(f, bbox_inches="tight", format=img_format)
def main(plot_vals=False):
varname = "RFAC"
multiplier = 24 * 3600
data_folder = Path("/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/atm_data_for_Arman_simulations")
day_range = range(19, 22)
dates_of_interest = [datetime(2013, 6, d) for d in day_range]
img_folder = Path("calgary_flood/2D")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
lons, lats, bmp = None, None, None
the_mask = get_bow_river_basin_mask()
i_list, j_list = np.where(the_mask > 0.5)
imin, imax = i_list.min() - 2, i_list.max() + 5
jmin, jmax = j_list.min() - 2, j_list.max() + 5
# Calculate daily means
sim_label_to_date_to_mean = OrderedDict()
for sim_dir in data_folder.iterdir():
mr = MultiRPN(str(sim_dir.joinpath("pm*")))
print(str(sim_dir))
print(mr.get_number_of_records())
label = sim_dir.name.split("_")[-2].replace("NoDrain", "").replace("frozen", "Frozen").replace("Bow", "")
sim_label_to_date_to_mean[label] = OrderedDict()
data = mr.get_4d_field(varname)
data = {d: list(v.items())[0][1] for d, v in data.items() if d.day in day_range}
for d in dates_of_interest:
sim_label_to_date_to_mean[label][d] = (
np.array([field for d1, field in data.items() if d1.day == d.day]).mean(axis=0) * multiplier
)
if lons is None:
lons, lats = mr.get_longitudes_and_latitudes_of_the_last_read_rec()
for f in sim_dir.iterdir():
if f.name.startswith("pm"):
r = RPN(str(f))
r.get_first_record_for_name(varname=varname)
rll = RotatedLatLon(**r.get_proj_parameters_for_the_last_read_rec())
bmp = rll.get_basemap_object_for_lons_lats(
lons2d=lons[imin:imax, jmin:jmax], lats2d=lats[imin:imax, jmin:jmax], resolution="i"
)
r.close()
break
mr.close()
# reorder simulations
sim_label_to_date_to_mean = OrderedDict(
[
(k, sim_label_to_date_to_mean[k])
for k in sorted(sim_label_to_date_to_mean, key=lambda z: len(z), reverse=True)
]
)
key_list = [k for k in sim_label_to_date_to_mean]
key_list[-2], key_list[-1] = key_list[-1], key_list[-2]
sim_label_to_date_to_mean = OrderedDict([(k, sim_label_to_date_to_mean[k]) for k in key_list])
# do the plots (subplots: vertically - simulations, horizontally - days)
plot_utils.apply_plot_params(width_cm=24, height_cm=28, font_size=10)
fig = plt.figure()
nrows = len(sim_label_to_date_to_mean)
ncols = len(day_range)
gs = GridSpec(nrows, ncols, wspace=0, hspace=0.1)
clevs_vals = [0, 0.1, 20, 50, 100, 150, 200, 300]
base_label = None
clevs_diff = [1, 5, 10, 20, 50, 100]
clevs_diff = [-c for c in reversed(clevs_diff)] + [0] + clevs_diff
cmap_diff = cm.get_cmap("bwr", len(clevs_diff) - 1)
cmap_vals = get_cmap_from_ncl_spec_file(path="colormap_files/precip3_16lev.rgb", ncolors=len(clevs_vals) - 1)
xx, yy = bmp(lons, lats)
title = "Total runoff ({}, mm/day)".format(varname)
fig.suptitle(title)
for row, (sim_label, date_to_field) in enumerate(sim_label_to_date_to_mean.items()):
if row == 0 or plot_vals:
base_sim = OrderedDict([(k, 0) for k, v in date_to_field.items()])
base_label = sim_label
plot_label = sim_label
clevs = clevs_vals
cmap = cmap_vals
extend = "max"
else:
base_sim = list(sim_label_to_date_to_mean.items())[0][1]
plot_label = "{}\n-\n{}".format(sim_label, base_label)
clevs = clevs_diff
cmap = cmap_diff
extend = "both"
bn = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1)
for col, (the_date, field) in enumerate(date_to_field.items()):
ax = fig.add_subplot(gs[row, col])
to_plot = np.ma.masked_where(the_mask < 0.5, field - base_sim[the_date])
# cs = bmp.contourf(xx[~to_plot.mask], yy[~to_plot.mask], to_plot[~to_plot.mask], levels=clevs, extend="max", tri=True)
cs = bmp.pcolormesh(xx, yy, to_plot[:-1, :-1], norm=bn, vmin=clevs[0], vmax=clevs[-1], cmap=cmap)
bmp.drawcoastlines(ax=ax, linewidth=0.3)
assert isinstance(bmp, Basemap)
bmp.readshapefile(BOW_RIVER_SHP[:-4], "basin", zorder=5)
cb = bmp.colorbar(cs, ax=ax, ticks=clevs, extend=extend, pad="4%", size="10%")
if plot_vals:
cb.ax.set_visible(col == ncols - 1 and row == 0)
else:
cb.ax.set_visible(col == ncols - 1 and row in (0, 1))
if col == 0:
ax.set_ylabel(plot_label)
if row == 0:
ax.set_title(the_date.strftime("%b %d"))
if plot_vals:
img_file = img_folder.joinpath("{}.png".format(varname))
else:
img_file = img_folder.joinpath("{}_diff.png".format(varname))
fig.savefig(str(img_file))
plt.close(fig)
def main():
erainterim_075_folder = "/HOME/data/Validation/ERA-Interim_0.75/Offline_driving_data/3h_Forecast"
vname = "PR"
start_year = 1980
end_year = 2010
season_key = "summer"
season_labels = {season_key: "Summer"}
season_to_months = OrderedDict([
(season_key, [6, 7, 8])
])
# Validate temperature and precip
model_vars = ["TT", "PR"]
obs_vars = ["tmp", "pre"]
obs_paths = [
"/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc",
"/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc"
]
model_var_to_obs_var = dict(zip(model_vars, obs_vars))
model_var_to_obs_path = dict(zip(model_vars, obs_paths))
obs_path = model_var_to_obs_path[vname]
cru = CRUDataManager(var_name=model_var_to_obs_var[vname], path=obs_path)
# Calculate seasonal means for CRU
seasonal_clim_fields_cru = cru.get_seasonal_means(season_name_to_months=season_to_months,
start_year=start_year,
end_year=end_year)
# Calculate seasonal mean for erai
flist = get_files_for_season(erainterim_075_folder, start_year=start_year, end_year=end_year, months=season_to_months[season_key])
rpf = MultiRPN(flist)
date_to_field_erai075 = rpf.get_all_time_records_for_name_and_level(varname=vname, level=-1)
# Convert to mm/day
era075 = np.mean([field for field in date_to_field_erai075.values()], axis=0) * 24 * 3600 * 1000
lons_era, lats_era = rpf.get_longitudes_and_latitudes_of_the_last_read_rec()
seasonal_clim_fields_cru_interp = OrderedDict()
# Calculate biases
for season, cru_field in seasonal_clim_fields_cru.items():
seasonal_clim_fields_cru_interp[season] = cru.interpolate_data_to(cru_field,
lons2d=lons_era,
lats2d=lats_era,
nneighbours=1)
# Do the plotting ------------------------------------------------------------------------------
plot_utils.apply_plot_params()
fig = plt.figure()
b = Basemap()
gs = gridspec.GridSpec(nrows=3, ncols=1)
ax = fig.add_subplot(gs[0, 0])
xx, yy = b(cru.lons2d, cru.lats2d)
cs = b.contourf(xx, yy, seasonal_clim_fields_cru[season_key], 20)
b.drawcoastlines(ax=ax)
ax.set_title("CRU")
plt.colorbar(cs, ax=ax)
ax = fig.add_subplot(gs[1, 0])
lons_era[lons_era > 180] -= 360
lons_era, era075 = b.shiftdata(lons_era, datain=era075, lon_0=0)
xx, yy = b(lons_era, lats_era)
# mask oceans in the era plot as well
era075 = maskoceans(lons_era, lats_era, era075)
cs = b.contourf(xx, yy, era075, levels=cs.levels, norm=cs.norm, cmap=cs.cmap, ax=ax)
b.drawcoastlines(ax=ax)
ax.set_title("ERA-Interim 0.75")
plt.colorbar(cs, ax=ax)
# differences
ax = fig.add_subplot(gs[2, 0])
diff = era075 - seasonal_clim_fields_cru_interp[season_key]
delta = np.percentile(np.abs(diff)[~diff.mask], 90)
clevs = np.linspace(-delta, delta, 20)
cs = b.contourf(xx, yy, diff, levels=clevs, cmap="RdBu_r", extend="both")
b.drawcoastlines(ax=ax)
ax.set_title("ERA-Interim 0.75 - CRU")
plt.colorbar(cs, ax=ax)
plt.show()
fig.savefig(os.path.join(img_folder, "erai0.75_vs_cru_precip.png"), bbox_inches="tight")
def main():
#path = "/RECH/data/Simulations/CRCM5/North_America/NorthAmerica_0.44deg_ERA40-Int_B1/Diagnostics/NorthAmerica_0.44deg_ERA40-Int_B1_2007{:02d}"
path = "/RESCUE/skynet3_rech1/huziy/from_guillimin/new_outputs/current_climate_30_yr_sims/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/Samples/quebec_crcm5-hcd-rl-intfl_1988{:02d}"
months = [6, 7, 8]
pm_list = []
dm_list = []
for m in months:
print(path.format(m))
month_folder = path.format(m)
for fn in os.listdir(month_folder):
# if not fn.endswith("moyenne"):
# continue
if fn.startswith("pm"):
pm_list.append(os.path.join(month_folder, fn))
elif fn.startswith("dm"):
dm_list.append(os.path.join(month_folder, fn))
pm = MultiRPN(pm_list)
dm = MultiRPN(dm_list)
tsurf_mean = np.mean([
field for field in pm.get_all_time_records_for_name_and_level(
varname="J8").values()
],
axis=0)
tair_mean = np.mean([
field for field in dm.get_all_time_records_for_name_and_level(
varname="TT", level=1, level_kind=level_kinds.HYBRID).values()
],
axis=0)
lons, lats = pm.get_longitudes_and_latitudes_of_the_last_read_rec()
projparams = pm.linked_robj_list[
0].get_proj_parameters_for_the_last_read_rec()
rll = RotatedLatLon(**projparams)
bmp = rll.get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats)
xx, yy = bmp(lons, lats)
plt.figure()
cs = bmp.contourf(xx, yy, tsurf_mean - 273.15, 40)
bmp.drawcoastlines()
plt.title("Tsurf")
plt.colorbar()
plt.figure()
bmp.contourf(xx,
yy,
tair_mean,
levels=cs.levels,
norm=cs.norm,
cmap=cs.cmap)
bmp.drawcoastlines()
plt.title("Tair")
plt.colorbar()
plt.figure()
bmp.contourf(xx,
yy,
tsurf_mean - 273.15 - tair_mean,
levels=np.arange(-2, 2.2, 0.2),
cmap=cs.cmap)
bmp.drawcoastlines()
plt.title("Tsurf - Tair")
plt.colorbar()
pm.close()
dm.close()
plt.show()
def get_area_avg_timeseries(samples_dir,
start_year=-np.Inf,
end_year=np.Inf,
filename_prefix="pm",
level=-1,
level_kind=level_kinds.ARBITRARY,
varname="",
mask=None,
mask_lons2d=None,
mask_lats2d=None,
file_per_var=False) -> pd.Series:
"""
get the timeseries of area averaged ice fraction
:rtype : pd.Series
"""
yearly_ts = []
lons2d, lats2d = None, None
samples_dir_p = Path(samples_dir)
# interpolated mask
interpolated_mask = None
for y in range(start_year, end_year + 1):
files_for_year = []
mfolders = [
f for f in samples_dir_p.iterdir() if f.name[:-2].endswith(str(y))
]
for mfolder in mfolders:
# Select all files containing the varname in the filename
if file_per_var:
files_for_year += [
str(f) for f in mfolder.iterdir() if varname in f.name
]
else:
files_for_year += [
str(f) for f in mfolder.iterdir()
if f.name.startswith(filename_prefix)
and f.name[-9:-1] != "0" * 8
]
if len(files_for_year) == 0:
continue
mrpn = MultiRPN(files_for_year)
data = mrpn.get_all_time_records_for_name_and_level(
varname=varname, level=level, level_kind=level_kind)
if lons2d is None:
lons2d, lats2d = mrpn.get_longitudes_and_latitudes_of_the_last_read_rec(
)
# interpolate the mask using nearest neighbour approach
if interpolated_mask is None:
xs, ys, zs = lat_lon.lon_lat_to_cartesian(mask_lons2d.flatten(),
mask_lats2d.flatten())
ktree = KDTree(data=list(zip(xs, ys, zs)))
xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons2d.flatten(),
lats2d.flatten())
dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1)
interpolated_mask = mask.flatten()[inds]
interpolated_mask.shape = lons2d.shape
for t, field in data.items():
data[t] = field[interpolated_mask].mean()
tlist = [t for t in data.keys()]
ser = pd.Series(index=tlist, data=[data[t] for t in tlist])
ser.sort_index(inplace=True)
yearly_ts.append(ser)
mrpn.close()
return pd.concat(yearly_ts), lons2d, lats2d
def get_model_data(station_to_model_point, output_path=None, grid_config=None, basins_of_interest_shp="",
cell_manager=None, vname=None):
lons, lats, bmp = None, None, None
data_mask = None
monthly_diagnostics_case = False
if output_path.name.lower().endswith("diagnostics"):
fname_pattern = "pm*_moyenne"
monthly_diagnostics_case = True
else:
fname_pattern = "pm*p"
pattern = re.compile(".*" + 8 * "0" + ".*")
flist = [f for f in glob.glob(str(output_path.joinpath("*").joinpath(fname_pattern))) if pattern.match(f) is None]
r = MultiRPN(flist)
date_to_field = r.get_all_time_records_for_name_and_level(varname=vname)
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
r.close()
# get the basemap object
bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest(
lons, lats, shp_path=basins_of_interest_shp, mask_margin=5)
station_to_model_data = {} # model data are the pandas timeseries
stations_to_ignore = []
for station, model_point in station_to_model_point.items():
assert isinstance(model_point, ModelPoint)
assert isinstance(cell_manager, CellManager)
assert isinstance(station, cehq_station.Station)
upstream_mask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices(model_point.ix,
model_point.jy)
# Skip model points and staions with small number of gridcells upstream
if upstream_mask.sum() <= 1:
stations_to_ignore.append(station)
print("Station {} is ignored, because the number of upstream cells is <= 1.".format(station.id))
continue
# Skip model points and stations outside the region of interest
if not data_mask[model_point.ix, model_point.jy]:
stations_to_ignore.append(station)
print("Station {} is ignored, because it is outside of the domain of interest.".format(station.id))
continue
# Plot station position
fig = plt.figure()
ax = plt.gca()
lons1, lats1 = lons[upstream_mask > 0.5], lats[upstream_mask > 0.5]
x1, y1 = bmp(lons1, lats1)
bmp.drawrivers()
bmp.drawcoastlines(ax=ax)
bmp.drawcountries(ax=ax, linewidth=0.2)
bmp.drawstates(linewidth=0.1)
bmp.readshapefile(basins_of_interest_shp[:-4], "basin", linewidth=2, color="m")
bmp.scatter(x1, y1, c="g", s=100)
bmp.scatter(*bmp(lons[model_point.ix, model_point.jy], lats[model_point.ix, model_point.jy]), c="b", s=250)
fig.tight_layout()
plt.savefig(str(img_folder.joinpath("{}_position_and_upstream.png".format(station.id))), bbox_inche="tight")
plt.close(fig)
res = pd.Series(index=sorted(date_to_field.keys()),
data=[date_to_field[d][model_point.ix, model_point.jy] for d in sorted(date_to_field.keys())])
# get monthly means
res = res.groupby(lambda d: d.replace(day=15, hour=0)).mean()
if monthly_diagnostics_case:
# shift to the end of the month before a previous month, and then shift 15 days to later
res = res.shift(-2, freq="M").shift(15, freq="D")
print(res.index[:20])
print(res.index[-20:])
station_to_model_data[station] = res
# Not enough drainage area
for s in stations_to_ignore:
del station_to_model_point[s]
return station_to_model_data
def plot_monthly_clim_in_a_panel(months=None, diag_folder="", vname="STFL",
grid_config=None, basins_of_interest_shp=""):
"""
Plots climatologies using diagnostics outputs, not samples
:param months:
:param diag_folder:
:param vname:
"""
if months is None:
months = list(range(1, 13))
diag_path = Path(diag_folder)
month_to_field = OrderedDict()
lons, lats, bmp = None, None, None
data_mask = None
for m in months:
r = MultiRPN(str(diag_path.joinpath("*{:02d}".format(m)).joinpath("pm*_moyenne")))
date_to_field = r.get_all_time_records_for_name_and_level()
the_mean = np.mean([f for f in date_to_field.values()], axis=0)
the_mean = np.ma.masked_where(the_mean < 0, the_mean)
month_to_field[m] = the_mean
if bmp is None:
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
# get the basemap object
bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest(
lons, lats, shp_path=basins_of_interest_shp, mask_margin=5)
r.close()
fig = plt.figure()
ncols = 3
nrows = len(months) // ncols + int(len(months) % ncols != 0)
gs = GridSpec(nrows=nrows, ncols=ncols + 1)
xx, yy = bmp(lons, lats)
clevs = [0, 20, 50, 100, 200, 500, 1000, 1500, 3000, 4500, 5000, 7000, 9000]
bn = BoundaryNorm(clevs, len(clevs) - 1)
cmap = cm.get_cmap("jet", len(clevs) - 1)
for m, field in month_to_field.items():
row = (m - 1) // ncols
col = (m - 1) % ncols
ax = fig.add_subplot(gs[row, col])
ax.set_title(calendar.month_name[m])
to_plot = np.ma.masked_where(~data_mask, field)
im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1])
bmp.colorbar(im, extend="max")
bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax)
bmp.drawcoastlines(ax=ax)
plt.close(fig)
# plot annual mean
ann_mean = np.mean([field for m, field in month_to_field.items()], axis=0)
fig = plt.figure()
ax = fig.add_subplot(gs[:, :])
ax.set_title("Annual mean")
to_plot = np.ma.masked_where(~data_mask, ann_mean)
im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1])
bmp.colorbar(im, extend="max")
bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax)
bmp.drawcoastlines(ax=ax)
plt.show()
plt.close(fig)
def main():
start_year = 1979
end_year = 1981
HL_LABEL = "CRCM5_HL"
NEMO_LABEL = "CRCM5_NEMO"
file_prefix = "pm"
PR_level = -1
PR_level_type = level_kinds.ARBITRARY
tprecip_vname = "PR"
sprecip_vname = "SN"
TT_level = 1
TT_level_type = level_kinds.HYBRID
sim_label_to_path = OrderedDict([
(HL_LABEL,
"/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples"
),
(NEMO_LABEL,
"/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples"
)
])
# get a coord file ...
coord_file = ""
found_coord_file = False
for mdir in os.listdir(sim_label_to_path[HL_LABEL]):
mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir)
if not os.path.isdir(mdir_path):
continue
for fn in os.listdir(mdir_path):
print(fn)
if fn[:2] not in ["pm", "dm", "pp", "dp"]:
continue
coord_file = os.path.join(mdir_path, fn)
found_coord_file = True
if found_coord_file:
break
bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file(
path=coord_file)
xx, yy = bmp(lons, lats)
# read necessary input and calculate snowfall and save to the file
for sim_label, samples_dir in sim_label_to_path.items():
samples_dir_path = Path(samples_dir)
TT_monthdate_to_paths = get_monthyeardate_to_paths_map(
file_prefix="dm",
start_year=start_year,
end_year=end_year,
samples_dir_path=samples_dir_path)
PR_monthdate_to_paths = get_monthyeardate_to_paths_map(
file_prefix="pm",
start_year=start_year,
end_year=end_year,
samples_dir_path=samples_dir_path)
with Dataset(
str(samples_dir_path.parent / "{}_snow_fall_{}-{}.nc".format(
sim_label, start_year, end_year)), "w") as ds:
assert isinstance(ds, Dataset)
ds.createDimension("time", None)
ds.createDimension("lon", lons.shape[0])
ds.createDimension("lat", lons.shape[1])
# create the schema of the output file
snow_fall = ds.createVariable(sprecip_vname,
"f4",
dimensions=("time", "lon", "lat"))
lons_var = ds.createVariable("lon",
"f4",
dimensions=("lon", "lat"))
lats_var = ds.createVariable("lat",
"f4",
dimensions=("lon", "lat"))
time_var = ds.createVariable("time", "i8", dimensions=("time", ))
time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format(
datetime(start_year, 1, 1))
lons_var[:] = lons
lats_var[:] = lats
# use sorted dates
record_count = 0
for month_date in sorted(TT_monthdate_to_paths):
tt = MultiRPN(path=TT_monthdate_to_paths[month_date]
).get_all_time_records_for_name_and_level(
varname="TT",
level=TT_level,
level_kind=TT_level_type)
pr = MultiRPN(path=PR_monthdate_to_paths[month_date]
).get_all_time_records_for_name_and_level(
varname="PR",
level=PR_level,
level_kind=PR_level_type)
print("Processing {}".format(month_date))
for d in sorted(tt):
t_field = tt[d]
pr_field = pr[d]
sn = get_snow_fall_m_per_s(precip_m_per_s=pr_field,
tair_deg_c=t_field)
snow_fall[record_count, :, :] = sn
time_var[record_count] = date2num(d, time_var.units)
record_count += 1
def main():
vname = "VV"
start_year = 1980
end_year = 2010
crcm_data_path = "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
months_of_interest = [6, 7, 8] # summer
summer_crcm = analysis.get_seasonal_climatology(hdf_path=crcm_data_path, start_year=start_year, end_year=end_year,
level=0, var_name=vname, months=months_of_interest)
bmp_info = analysis.get_basemap_info_from_hdf(file_path=crcm_data_path)
erainterim_15_folder = "/RECH/data/Driving_data/Pilots/ERA-Interim_1.5/Pilots/"
flist_for_season = get_files_for_season(erainterim_15_folder, start_year=start_year, end_year=end_year, months=months_of_interest)
rpf = MultiRPN(flist_for_season)
date_to_hu_erai15 = rpf.get_all_time_records_for_name_and_level(varname=vname, level=1000, level_kind=level_kinds.PRESSURE)
summer_era15 = np.mean([field for field in date_to_hu_erai15.values()], axis=0)
lons_era, lats_era = rpf.get_longitudes_and_latitudes_of_the_last_read_rec()
# plotting
# ERA-Interim
plt.figure()
b = Basemap(lon_0=180)
xxg, yyg = b(lons_era, lats_era)
im = b.contourf(xxg, yyg, summer_era15, 40, zorder=1)
lonsr = bmp_info.lons.copy()
lonsr[lonsr < 180] += 360
xxr, yyr = b(lonsr, bmp_info.lats)
b.contourf(xxr, yyr, summer_crcm, levels=im.levels, norm=im.norm, cmap=im.cmap, zorder=2)
b.drawcoastlines(zorder=3)
plt.colorbar(im)
# CRCM (plot both crcm and era on the same plot)
fig = plt.figure()
xx, yy = bmp_info.get_proj_xy()
margin = 20
bext = bmp_info.basemap_for_extended_region(marginx=10 * margin, marginy=10 * margin)
bmiddle = bmp_info.basemap_for_extended_region(marginx=9 * margin, marginy=9 * margin)
xxg, yyg = bext(lons_era, lats_era)
outer_domain = (xxg <= bext.urcrnrx) & (xxg >= bext.llcrnrx) & (yyg <= bext.urcrnry) & (yyg >= bext.llcrnry)
summer_era15 = np.ma.masked_where(~outer_domain, summer_era15)
im = bext.contourf(xx, yy, summer_crcm, levels=im.levels, norm=im.norm, cmap=im.cmap, zorder=2)
bmiddle.contourf(xxg, yyg, summer_era15, levels=im.levels, norm=im.norm, cmap=im.cmap, zorder=1)
bext.drawcoastlines()
plt.colorbar(im)
# Add a polygon
ax = plt.gca()
coords = np.array([
[xx[0, 0], yy[0, 0]], [xx[0, -1], yy[0, -1]], [xx[-1, -1], yy[-1, -1]], [xx[-1, 0], yy[-1, 0]]
])
ax.add_patch(Polygon(coords, facecolor="none", lw=3, zorder=3, edgecolor="k"))
img_folder = "cc-paper-comments"
if not os.path.isdir(img_folder):
os.mkdir(img_folder)
fig.savefig(os.path.join(img_folder, "{}_era_1.5_and_crcm.png".format(vname)), bbox_inches="tight", transparent=True)
plt.show()
def plot_monthly_clim_in_a_panel(months=None,
diag_folder="",
vname="STFL",
grid_config=None,
basins_of_interest_shp=""):
"""
Plots climatologies using diagnostics outputs, not samples
:param months:
:param diag_folder:
:param vname:
"""
if months is None:
months = list(range(1, 13))
diag_path = Path(diag_folder)
month_to_field = OrderedDict()
lons, lats, bmp = None, None, None
data_mask = None
for m in months:
r = MultiRPN(
str(
diag_path.joinpath(
"*{:02d}".format(m)).joinpath("pm*_moyenne")))
date_to_field = r.get_all_time_records_for_name_and_level()
the_mean = np.mean([f for f in date_to_field.values()], axis=0)
the_mean = np.ma.masked_where(the_mean < 0, the_mean)
month_to_field[m] = the_mean
if bmp is None:
lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec()
# get the basemap object
bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest(
lons, lats, shp_path=basins_of_interest_shp, mask_margin=5)
r.close()
fig = plt.figure()
ncols = 3
nrows = len(months) // ncols + int(len(months) % ncols != 0)
gs = GridSpec(nrows=nrows, ncols=ncols + 1)
xx, yy = bmp(lons, lats)
clevs = [
0, 20, 50, 100, 200, 500, 1000, 1500, 3000, 4500, 5000, 7000, 9000
]
bn = BoundaryNorm(clevs, len(clevs) - 1)
cmap = cm.get_cmap("jet", len(clevs) - 1)
for m, field in month_to_field.items():
row = (m - 1) // ncols
col = (m - 1) % ncols
ax = fig.add_subplot(gs[row, col])
ax.set_title(calendar.month_name[m])
to_plot = np.ma.masked_where(~data_mask, field)
im = bmp.pcolormesh(xx,
yy,
to_plot,
norm=bn,
cmap=cmap,
vmin=clevs[0],
vmax=clevs[-1])
bmp.colorbar(im, extend="max")
bmp.readshapefile(basins_of_interest_shp[:-4],
"basins",
linewidth=2,
color="m",
ax=ax)
bmp.drawcoastlines(ax=ax)
plt.close(fig)
# plot annual mean
ann_mean = np.mean([field for m, field in month_to_field.items()], axis=0)
fig = plt.figure()
ax = fig.add_subplot(gs[:, :])
ax.set_title("Annual mean")
to_plot = np.ma.masked_where(~data_mask, ann_mean)
im = bmp.pcolormesh(xx,
yy,
to_plot,
norm=bn,
cmap=cmap,
vmin=clevs[0],
vmax=clevs[-1])
bmp.colorbar(im, extend="max")
bmp.readshapefile(basins_of_interest_shp[:-4],
"basins",
linewidth=2,
color="m",
ax=ax)
bmp.drawcoastlines(ax=ax)
plt.show()
plt.close(fig)
def main(sim_path="/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected/",
fpath_pattern="var_per_file",
fname_prefix=None, coords_filename="pm1979010100_00000000p"):
start_year = 1980
end_year = 1982
vname = "SD"
level = 6
level_kind = level_kinds.ARBITRARY
lkfr = get_lake_fraction(sim_path, fname=coords_filename)
data_series = []
year_to_paths_cache = defaultdict(list)
data = None
for y in range(start_year, end_year + 1):
if fpath_pattern == "var_per_file":
r = MultiRPN("{}/*{}*/*{}*".format(sim_path, y, vname))
data = r.get_all_time_records_for_name_and_level(vname, level=level, level_kind=level_kind)
elif fpath_pattern == "default":
sim_dir = Path(sim_path)
# Create the map in order to reuse
if len(year_to_paths_cache) == 0:
for month_dir in sim_dir.iterdir():
year_to_paths_cache[y].append(month_dir)
for month_dir in year_to_paths_cache[y]:
for fpath in month_dir.iterdir():
# print(fpath)
# Check if the prefix is OK
if not fpath.name.startswith(fname_prefix):
continue
# Check if this is not a timestep 0
if fpath.name[:-1].endswith(8 * "0"):
continue
with RPN(str(fpath)) as r:
data_tmp = r.get_all_time_records_for_name_and_level(vname, level=level, level_kind=level_kind)
if data is None:
data = data_tmp
else:
data.update(data_tmp)
else:
raise Exception("Unknown file path pattern: {}".format(fpath_pattern))
ts_for_year = {}
for d, field in data.items():
ts_for_year[d] = field[lkfr > 0.5].mean()
data_series.append(pd.Series(ts_for_year))
total_series = pd.concat(data_series)
assert isinstance(total_series, pd.Series)
ax = total_series.plot(title="{}-{}".format(start_year, end_year))
plt.show()
def plot_seasonal_vertical_velocity(samples_folder, seasons=commons.default_seasons):
level = 850 # millibars
vname = "WW"
long_name = "Vertical compoent of wind velocity ({}mb)".format(level)
file_prefix = "dp"
plot_units = "Pa/s"
mult_coeff = 1
add_offset = 0
out_dx = 0.5
file_format = "png"
season_to_mean = OrderedDict()
r = None
for sname, months in seasons.items():
# find files for the season
paths = []
for m in months:
paths.extend(glob.glob(str(samples_folder.joinpath("*{:02d}/{}*".format(m, file_prefix)))))
r = MultiRPN(paths)
data = r.get_4d_field(varname=vname)
season_to_mean[sname] = np.array([v[level] for k, v in data.items()]).mean(axis=0)
print("Processed: {}".format(sname))
lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec()
img_folder = samples_folder.joinpath("images/seasonal")
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
img_file = img_folder.joinpath("{}_{}hPa.{}".format(vname, level, file_format))
plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18)
fig = plt.figure()
ncols = 2
gs = GridSpec(len(season_to_mean) // ncols + int(not (len(season_to_mean) % ncols == 0)), ncols, wspace=0, hspace=0)
xx, yy = None, None
bmp = Basemap(projection="robin", lon_0=0)
cmap = cm_basemap.GMT_polar
clevs = np.arange(-0.1, 0.11, 0.01)
for i, (sname, field) in enumerate(season_to_mean.items()):
row = i // ncols
col = i % ncols
ax = fig.add_subplot(gs[row, col])
lons, lats, data_out = commons.interpolate_to_uniform_global_grid(field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx)
if xx is None:
xx, yy = bmp(lons, lats)
cs = bmp.contourf(xx, yy, data_out * mult_coeff, 20 if clevs is None else clevs, cmap=cmap, extend="both")
# save color levels for next subplots
clevs = cs.levels
print(np.max(data_out * mult_coeff))
ax.set_title(sname)
cb = plt.colorbar(cs, ax=ax)
if not (row == 0 and col == ncols - 1):
# cb.ax.set_title(plot_units)
cb.ax.set_visible(False)
bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH)
fig.suptitle("{}, {}".format(long_name, plot_units))
with img_file.open("wb") as f:
fig.savefig(f, bbox_inches="tight", format=file_format)
plt.close(fig)
def get_daily_clim_profiles(samples_dir,
start_year=-np.Inf,
end_year=np.Inf,
filename_prefix="dp",
varname="",
mask=None):
"""
dates_2d, levels_2d, values - needed for plotting
:param start_year:
:param end_year:
:param filename_prefix:
:param varname:
"""
dates_sorted = None
levels_sorted = None
yearly_fields = []
day = timedelta(days=1)
stamp_dates = [datetime(2001, 1, 1) + i * day for i in range(365)]
for y in range(start_year, end_year + 1):
files_for_year = []
mfolders = [
os.path.join(samples_dir, f) for f in os.listdir(samples_dir)
if f[:-2].endswith(str(y))
]
for mfolder in mfolders:
files_for_year += [
os.path.join(mfolder, fn) for fn in os.listdir(mfolder)
if fn.startswith(filename_prefix)
]
mrpn = MultiRPN(files_for_year)
data = mrpn.get_4d_field(varname=varname)
# calculate the area-average
for d, lev_to_field in data.items():
for lev in lev_to_field:
lev_to_field[lev] = lev_to_field[lev][mask].mean()
#
dates_sorted = list(
sorted([
d for d in data
if not (d.month == 2 and d.day == 29) and (d.year == y)
]))
levels_sorted = list(
sorted([lev for lev in data[dates_sorted[0]] if lev >= 500]))
#
year_field = np.zeros((len(dates_sorted), len(levels_sorted)))
for j, lev in enumerate(levels_sorted):
for i, the_date in enumerate(dates_sorted):
year_field[i, j] = data[the_date][lev]
panel = pd.DataFrame(data=year_field,
index=dates_sorted,
columns=range(len(levels_sorted)))
daily = panel.groupby(
by=lambda d: datetime(d.year, d.month, d.day)).mean()
yearly_fields.append(daily.values)
values = np.mean(yearly_fields, axis=0)
assert not hasattr(values, "mask")
dates_num = date2num(stamp_dates)
levs_2d, dates_num_2d = np.meshgrid(levels_sorted, dates_num)
assert levs_2d.shape == values.shape, "levels and values shapes ({} and {}, respectively) are not consistent.".format(
levs_2d.shape, values.shape)
return dates_num_2d, levs_2d, values
def main(
sim_path="/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected/",
fpath_pattern="var_per_file",
fname_prefix=None,
coords_filename="pm1979010100_00000000p"):
start_year = 1980
end_year = 1982
vname = "SD"
level = 6
level_kind = level_kinds.ARBITRARY
lkfr = get_lake_fraction(sim_path, fname=coords_filename)
data_series = []
year_to_paths_cache = defaultdict(list)
data = None
for y in range(start_year, end_year + 1):
if fpath_pattern == "var_per_file":
r = MultiRPN("{}/*{}*/*{}*".format(sim_path, y, vname))
data = r.get_all_time_records_for_name_and_level(
vname, level=level, level_kind=level_kind)
elif fpath_pattern == "default":
sim_dir = Path(sim_path)
# Create the map in order to reuse
if len(year_to_paths_cache) == 0:
for month_dir in sim_dir.iterdir():
year_to_paths_cache[y].append(month_dir)
for month_dir in year_to_paths_cache[y]:
for fpath in month_dir.iterdir():
# print(fpath)
# Check if the prefix is OK
if not fpath.name.startswith(fname_prefix):
continue
# Check if this is not a timestep 0
if fpath.name[:-1].endswith(8 * "0"):
continue
with RPN(str(fpath)) as r:
data_tmp = r.get_all_time_records_for_name_and_level(
vname, level=level, level_kind=level_kind)
if data is None:
data = data_tmp
else:
data.update(data_tmp)
else:
raise Exception(
"Unknown file path pattern: {}".format(fpath_pattern))
ts_for_year = {}
for d, field in data.items():
ts_for_year[d] = field[lkfr > 0.5].mean()
data_series.append(pd.Series(ts_for_year))
total_series = pd.concat(data_series)
assert isinstance(total_series, pd.Series)
ax = total_series.plot(title="{}-{}".format(start_year, end_year))
plt.show()
def main():
start_year = 1979
end_year = 1981
HL_LABEL = "CRCM5_HL"
NEMO_LABEL = "CRCM5_NEMO"
file_prefix = "dm"
level = 1
level_type = level_kinds.HYBRID
wind_comp_names = ["UU", "VV"]
sim_label_to_path = OrderedDict([
(HL_LABEL,
"/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples"
),
(NEMO_LABEL,
"/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples"
)
])
# get a coord file ...
coord_file = ""
found_coord_file = False
for mdir in os.listdir(sim_label_to_path[HL_LABEL]):
mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir)
if not os.path.isdir(mdir_path):
continue
for fn in os.listdir(mdir_path):
print(fn)
if fn[:2] not in ["pm", "dm", "pp", "dp"]:
continue
coord_file = os.path.join(mdir_path, fn)
found_coord_file = True
if found_coord_file:
break
bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file(
path=coord_file)
xx, yy = bmp(lons, lats)
lons[lons > 180] -= 360
# loop through all files rotate vaectors and save to netcdf
for sim_label, samples_dir in sim_label_to_path.items():
samples = Path(samples_dir)
po = samples.parent
monthdate_to_path_list = nemo_hl_util.get_monthyeardate_to_paths_map(
file_prefix=file_prefix,
start_year=start_year,
end_year=end_year,
samples_dir_path=samples)
# Netcdf output file to put rotated winds
po /= "rotated_wind_{}.nc".format(sim_label)
with Dataset(str(po), "w") as ds:
ds.createDimension("time", None)
ds.createDimension("lon", lons.shape[0])
ds.createDimension("lat", lons.shape[1])
# create the schema of the output file
vname_to_ncvar = {}
for vname in wind_comp_names:
vname_to_ncvar[vname] = ds.createVariable(vname,
"f4",
dimensions=("time",
"lon",
"lat"))
vname_to_ncvar[vname].units = "knots"
lons_var = ds.createVariable("lon",
"f4",
dimensions=("lon", "lat"))
lats_var = ds.createVariable("lat",
"f4",
dimensions=("lon", "lat"))
time_var = ds.createVariable("time", "i8", dimensions=("time", ))
time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format(
datetime(start_year, 1, 1))
lons_var[:] = lons
lats_var[:] = lats
# use sorted dates
record_count = 0
for month_date in sorted(monthdate_to_path_list):
# select only dm files
mr = MultiRPN(path=monthdate_to_path_list[month_date])
vname_to_fields = {}
for vname in wind_comp_names:
vname_to_fields[
vname] = mr.get_all_time_records_for_name_and_level(
varname=vname, level=level, level_kind=level_type)
for ti, t in enumerate(
sorted(vname_to_fields[wind_comp_names[0]])):
time_var[record_count] = date2num(t, time_var.units)
uu = vname_to_fields[wind_comp_names[0]][t]
vv = vname_to_fields[wind_comp_names[1]][t]
uu_rot, vv_rot = rotate_vecs_from_geo_to_rotpole(uu,
vv,
lons,
lats,
bmp=bmp)
# in knots not in m/s
vname_to_ncvar[wind_comp_names[0]][
record_count, :, :] = uu_rot
vname_to_ncvar[wind_comp_names[1]][
record_count, :, :] = vv_rot
record_count += 1
def main():
#path = "/RECH/data/Simulations/CRCM5/North_America/NorthAmerica_0.44deg_ERA40-Int_B1/Diagnostics/NorthAmerica_0.44deg_ERA40-Int_B1_2007{:02d}"
path = "/RESCUE/skynet3_rech1/huziy/from_guillimin/new_outputs/current_climate_30_yr_sims/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/Samples/quebec_crcm5-hcd-rl-intfl_1988{:02d}"
months = [6, 7, 8]
pm_list = []
dm_list = []
for m in months:
print(path.format(m))
month_folder = path.format(m)
for fn in os.listdir(month_folder):
# if not fn.endswith("moyenne"):
# continue
if fn.startswith("pm"):
pm_list.append(os.path.join(month_folder, fn))
elif fn.startswith("dm"):
dm_list.append(os.path.join(month_folder, fn))
pm = MultiRPN(pm_list)
dm = MultiRPN(dm_list)
tsurf_mean = np.mean([field for field in pm.get_all_time_records_for_name_and_level(varname="J8").values()], axis=0)
tair_mean = np.mean([field for field in dm.get_all_time_records_for_name_and_level(varname="TT", level=1, level_kind=level_kinds.HYBRID).values()], axis=0)
lons, lats = pm.get_longitudes_and_latitudes_of_the_last_read_rec()
projparams = pm.linked_robj_list[0].get_proj_parameters_for_the_last_read_rec()
rll = RotatedLatLon(**projparams)
bmp = rll.get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats)
xx, yy = bmp(lons, lats)
plt.figure()
cs = bmp.contourf(xx, yy, tsurf_mean - 273.15, 40)
bmp.drawcoastlines()
plt.title("Tsurf")
plt.colorbar()
plt.figure()
bmp.contourf(xx, yy, tair_mean, levels=cs.levels, norm=cs.norm, cmap=cs.cmap)
bmp.drawcoastlines()
plt.title("Tair")
plt.colorbar()
plt.figure()
bmp.contourf(xx, yy, tsurf_mean - 273.15 - tair_mean, levels=np.arange(-2, 2.2, 0.2), cmap=cs.cmap)
bmp.drawcoastlines()
plt.title("Tsurf - Tair")
plt.colorbar()
pm.close()
dm.close()
plt.show()
def main():
erainterim_075_folder = "/HOME/data/Validation/ERA-Interim_0.75/Offline_driving_data/3h_Forecast"
vname = "PR"
start_year = 1980
end_year = 2010
season_key = "summer"
season_labels = {season_key: "Summer"}
season_to_months = OrderedDict([(season_key, [6, 7, 8])])
# Validate temperature and precip
model_vars = ["TT", "PR"]
obs_vars = ["tmp", "pre"]
obs_paths = [
"/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc",
"/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc",
]
model_var_to_obs_var = dict(zip(model_vars, obs_vars))
model_var_to_obs_path = dict(zip(model_vars, obs_paths))
obs_path = model_var_to_obs_path[vname]
cru = CRUDataManager(var_name=model_var_to_obs_var[vname], path=obs_path)
# Calculate seasonal means for CRU
seasonal_clim_fields_cru = cru.get_seasonal_means(
season_name_to_months=season_to_months, start_year=start_year, end_year=end_year
)
# Calculate seasonal mean for erai
flist = get_files_for_season(
erainterim_075_folder, start_year=start_year, end_year=end_year, months=season_to_months[season_key]
)
rpf = MultiRPN(flist)
date_to_field_erai075 = rpf.get_all_time_records_for_name_and_level(varname=vname, level=-1)
# Convert to mm/day
era075 = np.mean([field for field in date_to_field_erai075.values()], axis=0) * 24 * 3600 * 1000
lons_era, lats_era = rpf.get_longitudes_and_latitudes_of_the_last_read_rec()
seasonal_clim_fields_cru_interp = OrderedDict()
# Calculate biases
for season, cru_field in seasonal_clim_fields_cru.items():
seasonal_clim_fields_cru_interp[season] = cru.interpolate_data_to(
cru_field, lons2d=lons_era, lats2d=lats_era, nneighbours=1
)
# Do the plotting ------------------------------------------------------------------------------
plot_utils.apply_plot_params()
fig = plt.figure()
b = Basemap()
gs = gridspec.GridSpec(nrows=3, ncols=1)
ax = fig.add_subplot(gs[0, 0])
xx, yy = b(cru.lons2d, cru.lats2d)
cs = b.contourf(xx, yy, seasonal_clim_fields_cru[season_key], 20)
b.drawcoastlines(ax=ax)
ax.set_title("CRU")
plt.colorbar(cs, ax=ax)
ax = fig.add_subplot(gs[1, 0])
lons_era[lons_era > 180] -= 360
lons_era, era075 = b.shiftdata(lons_era, datain=era075, lon_0=0)
xx, yy = b(lons_era, lats_era)
# mask oceans in the era plot as well
era075 = maskoceans(lons_era, lats_era, era075)
cs = b.contourf(xx, yy, era075, levels=cs.levels, norm=cs.norm, cmap=cs.cmap, ax=ax)
b.drawcoastlines(ax=ax)
ax.set_title("ERA-Interim 0.75")
plt.colorbar(cs, ax=ax)
# differences
ax = fig.add_subplot(gs[2, 0])
diff = era075 - seasonal_clim_fields_cru_interp[season_key]
delta = np.percentile(np.abs(diff)[~diff.mask], 90)
clevs = np.linspace(-delta, delta, 20)
cs = b.contourf(xx, yy, diff, levels=clevs, cmap="RdBu_r", extend="both")
b.drawcoastlines(ax=ax)
ax.set_title("ERA-Interim 0.75 - CRU")
plt.colorbar(cs, ax=ax)
plt.show()
fig.savefig(os.path.join(img_folder, "erai0.75_vs_cru_precip.png"), bbox_inches="tight")
