def get_lake_fraction_for_date(self, the_date=None): """ Get the lake ice cover fraction for the specified date :param the_date: :return: """ if the_date not in self.cached_data: month_date = datetime(the_date.year, the_date.month, 1) self.cached_data = {} mr = MultiRPN(self.year_month_date_to_file_list[month_date]) self.cached_data = mr.get_all_time_records_for_name_and_level(varname=self.varname, level=self.level, level_kind=self.level_kind) mr.close() # if still it is not there try looking in the previous month if the_date not in self.cached_data: month = the_date.month - 1 year = the_date.year if the_date.month == 0: month = 12 year -= 1 month_date = datetime(year, month, 1) mr = MultiRPN(self.year_month_date_to_file_list[month_date]) self.cached_data.update(mr.get_all_time_records_for_name_and_level(varname=self.varname, level=self.level, level_kind=self.level_kind)) mr.close() return self.cached_data[the_date]
def test_getting_coordinates_for_the_last_read_record_should_not_fail(): r = None try: r = MultiRPN(in_path) rec = r.get_first_record_for_name("I5") lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec() tools.assert_equals(lons.shape, lats.shape, "Shapes of lon and lat arrays should be the same.") finally: if r is not None: r.close() delete_files()
def test_get_number_of_records(): r = None try: create_files() r = MultiRPN("test_?.rpn") msg = "Number of records should be equal to the total number in all files" tools.assert_equals(r.get_number_of_records(), len(FILE_NAMES), msg) finally: if r is not None: r.close() delete_files()
def test_can_link_many_files(nfiles=200): many_fnames = ["test_{}.rpn".format(i) for i in range(nfiles)] r = None try: create_files(fnames=many_fnames) r = MultiRPN("test_*.rpn") tools.assert_greater(r.get_number_of_records(), 0, msg="There should be more than 0 records") finally: if r is not None: r.close() delete_files(fnames=many_fnames)
def test_should_find_all_records(): r = None try: create_files() r = MultiRPN("test_?.rpn") for fn in range(len(FILE_NAMES)): rec = r.get_first_record_for_name("T{}".format(fn)) print("file {} - OK".format(fn)) print(rec.mean(), rec.shape) finally: if r is not None: r.close() delete_files()
def calculate_seasonal_mean_winds(seasons=commons.default_seasons, level_hpa=None, samples_folder=None, file_prefix="dp"): cache_file_name = "seasonal_wind_components_{}_{}hPa_{}.npz".format( "-".join(seasons.keys()), level_hpa, hashlib.sha224(str(samples_folder).encode()).hexdigest()) cache_file = Path(cache_file_name) if cache_file.is_file(): print("Using cached winds from {}".format(cache_file)) return np.load(cache_file_name)["arr_0"] u_season_to_mean = OrderedDict() v_season_to_mean = OrderedDict() r = None for sname, months in seasons.items(): # find files for the season paths = [] for m in months: paths.extend( glob.glob( str( samples_folder.joinpath("*{:02d}/{}*".format( m, file_prefix))))) r = MultiRPN(paths) u_data = r.get_4d_field(varname="UU") v_data = r.get_4d_field(varname="VV") u_season_to_mean[sname] = np.array( [v[level_hpa] for k, v in u_data.items()]).mean(axis=0) v_season_to_mean[sname] = np.array( [v[level_hpa] for k, v in v_data.items()]).mean(axis=0) print("Processed: {}".format(sname)) lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec() np.savez_compressed(cache_file_name, [lons2d, lats2d, u_season_to_mean, v_season_to_mean]) return lons2d, lats2d, u_season_to_mean, v_season_to_mean
def extract_runoff_to_nc_process(args): in_path, out_path = args if os.path.exists(out_path): print("Nothing to do for: {}".format(out_path)) return # skip files that already exist traf_name = "TRAF" tdra_name = "TDRA" r = MultiRPN(in_path) traf_data = r.get_all_time_records_for_name_and_level(varname=traf_name, level=5, level_kind=level_kinds.ARBITRARY) tdra_data = r.get_all_time_records_for_name_and_level(varname=tdra_name, level=5, level_kind=level_kinds.ARBITRARY) r.close() nx, ny = list(traf_data.items())[0][1].shape with nc.Dataset(out_path, "w", format="NETCDF3_CLASSIC") as ds: ds.createDimension("lon", nx) ds.createDimension("lat", ny) ds.createDimension("time", None) varTraf = ds.createVariable(traf_name, "f4", dimensions=("time", "lon", "lat")) varTraf.units = "kg/( m**2 * s )" varTdra = ds.createVariable(tdra_name, "f4", dimensions=("time", "lon", "lat")) varTdra.units = "kg/( m**2 * s )" timeVar = ds.createVariable("time", "f4", dimensions=("time",)) sorted_dates = list(sorted(traf_data.keys())) timeVar.units = "hours since {0}".format(sorted_dates[0]) timeVar[:] = nc.date2num(sorted_dates, timeVar.units) varTraf[:] = np.array( [traf_data[d] for d in sorted_dates] ) varTdra[:] = np.array( [tdra_data[d] for d in sorted_dates] )
def test_get_4d_field(): r = None vname = "T" try: create_files_with_same_var_for_different_times(vname=vname) r = MultiRPN("test_?.rpn") recs = r.get_4d_field(vname) msg = "Not all records for {} were found".format(vname) print(recs.keys()) tools.assert_equals(len(FILE_NAMES), len(recs), msg) finally: if r is not None: r.close() delete_files()
def test_get_list_of_varnames(): r = None vname = "T" try: create_files_with_same_var_for_different_times(vname=vname) r = MultiRPN("test_?.rpn") msg = "the list of varnames should contain {}".format(vname) vnames = r.get_list_of_varnames() tools.assert_in(vname, vnames, msg) tools.assert_equal(len(vnames), 1, "There is only one unique field name in the files") finally: if r is not None: r.close() delete_files()
def get_seasonal_means_from_rpn_monthly_folders(samples_dir="", season_to_months=None, start_year=-np.Inf, end_year=np.Inf, filename_prefix="pm", varname="", level=-1, level_kind=-1): result = OrderedDict() season_to_files = {s: [] for s in season_to_months} smpls_dir = Path(samples_dir) for month_folder in smpls_dir.iterdir(): y, m = get_year_and_month(month_folder.name) # skip if the year is not in the selected range if not start_year <= y <= end_year: continue for season, months in season_to_months.items(): if m in months: for data_file in month_folder.iterdir(): if data_file.name[-9:-1] == 8 * "0": continue if data_file.name.startswith(filename_prefix): season_to_files[season].append(str(data_file)) for season, months in season_to_months.items(): print("{} => {}".format(season, months)) mrpn = MultiRPN(season_to_files[season]) date_to_field = mrpn.get_all_time_records_for_name_and_level(varname=varname, level=level, level_kind=level_kind) result[season] = np.mean([field for field in date_to_field.values()], axis=0) return result
def main(): path_to_nemo_outputs = "/RESCUE/skynet3_rech1/huziy/one_way_coupled_nemo_outputs_1979_1985/GLK_1d_grid_T.nc" vname_nemo = "sosstsst" path_to_crcm5_outputs = "/home/huziy/skynet3_rech1/glk-oneway-coupled-crcm5-outputs/coupled-GL/Samples" vname_crcm5 = "TT" # month of interest month = 4 year = 1981 mrpn = MultiRPN("{}/*_{}{}/dm*".format(path_to_crcm5_outputs, year, month)) data = mrpn.get_all_time_records_for_name_and_level(varname=vname_crcm5, level=1, level_kind=level_kinds.HYBRID) # Calculate the monthly mean fields in both cases assert isinstance(data, dict) mm_crcm5 = np.array(list(data.values())).mean(axis=0) print("crcm5-out-shape = ", mm_crcm5.shape) with Dataset(path_to_nemo_outputs) as ds: mm_nemo = ds.variables[vname_nemo][:].mean(axis=0) print("nemo-out-shape = ", mm_nemo.shape)
def calculate_seasonal_mean_winds(seasons=commons.default_seasons, level_hpa=None, samples_folder=None, file_prefix="dp"): cache_file_name = "seasonal_wind_components_{}_{}hPa_{}.npz".format("-".join(seasons.keys()), level_hpa, hashlib.sha224(str(samples_folder).encode()).hexdigest()) cache_file = Path(cache_file_name) if cache_file.is_file(): print("Using cached winds from {}".format(cache_file)) return np.load(cache_file_name)["arr_0"] u_season_to_mean = OrderedDict() v_season_to_mean = OrderedDict() r = None for sname, months in seasons.items(): # find files for the season paths = [] for m in months: paths.extend(glob.glob(str(samples_folder.joinpath("*{:02d}/{}*".format(m, file_prefix))))) r = MultiRPN(paths) u_data = r.get_4d_field(varname="UU") v_data = r.get_4d_field(varname="VV") u_season_to_mean[sname] = np.array([v[level_hpa] for k, v in u_data.items()]).mean(axis=0) v_season_to_mean[sname] = np.array([v[level_hpa] for k, v in v_data.items()]).mean(axis=0) print("Processed: {}".format(sname)) lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec() np.savez_compressed(cache_file_name, [lons2d, lats2d, u_season_to_mean, v_season_to_mean]) return lons2d, lats2d, u_season_to_mean, v_season_to_mean
def main(): path_to_nemo_outputs = "/RESCUE/skynet3_rech1/huziy/one_way_coupled_nemo_outputs_1979_1985/GLK_1d_grid_T.nc" vname_nemo = "sosstsst" path_to_crcm5_outputs = "/home/huziy/skynet3_rech1/glk-oneway-coupled-crcm5-outputs/coupled-GL/Samples" vname_crcm5 = "TT" # month of interest month = 4 year = 1981 mrpn = MultiRPN("{}/*_{}{}/dm*".format(path_to_crcm5_outputs, year, month)) data = mrpn.get_all_time_records_for_name_and_level( varname=vname_crcm5, level=1, level_kind=level_kinds.HYBRID) # Calculate the monthly mean fields in both cases assert isinstance(data, dict) mm_crcm5 = np.array(list(data.values())).mean(axis=0) print("crcm5-out-shape = ", mm_crcm5.shape) with Dataset(path_to_nemo_outputs) as ds: mm_nemo = ds.variables[vname_nemo][:].mean(axis=0) print("nemo-out-shape = ", mm_nemo.shape)
def get_lake_fraction_for_date(self, the_date=None): """ Get the lake ice cover fraction for the specified date :param the_date: :return: """ if the_date not in self.cached_data: month_date = datetime(the_date.year, the_date.month, 1) self.cached_data = {} mr = MultiRPN(self.year_month_date_to_file_list[month_date]) self.cached_data = mr.get_all_time_records_for_name_and_level( varname=self.varname, level=self.level, level_kind=self.level_kind ) mr.close() # if still it is not there try looking in the previous month if the_date not in self.cached_data: month = the_date.month - 1 year = the_date.year if the_date.month == 0: month = 12 year -= 1 month_date = datetime(year, month, 1) mr = MultiRPN(self.year_month_date_to_file_list[month_date]) self.cached_data.update( mr.get_all_time_records_for_name_and_level( varname=self.varname, level=self.level, level_kind=self.level_kind ) ) mr.close() return self.cached_data[the_date]
def main(): plot_utils.apply_plot_params(width_cm=40, height_cm=25, font_size=14) seasons = commons.default_seasons dx_plotting = 0.5 # degrees # units = "$^\circ$C" # long_name = "SST, {}".format(units) # var_name = "TM" # clevs_diff = np.arange(-5, 5.5, 0.5) units = "" long_name = "Sea ice {}".format(units) var_name = "LG" clevs_diff = [v for v in np.arange(-1, 1.1, 0.1) if abs(v) > 1.0e-6] label_to_folder = OrderedDict([ ("ERA-Interim", "/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/SST_SeaIce/I_SST_SeaIce"), ("PreI-CanESM2", "/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce"), ("PreI-GFDL", "/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce"), ("PreI-GISS", "/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce") ]) bmp = Basemap(projection="robin", lon_0=180) xx, yy = None, None year = 2013 print(seasons) label_to_season_to_field = OrderedDict() for label, folder in label_to_folder.items(): label_to_season_to_field[label] = OrderedDict() all_files = [f for f in os.listdir(folder) if label.lower() in f.lower() or label.split("-")[1].lower() in f.lower()] for sname, months in seasons.items(): season_files = [f for f in all_files if _get_month_and_year_from_file_name(f)[0] in months and _get_month_and_year_from_file_name(f)[1] == year] season_files = [os.path.join(folder, f) for f in season_files] print(10 * "+" + sname + "+" * 10) print(season_files) r = MultiRPN(season_files) data = r.get_4d_field(var_name) the_mean = np.array([list(v.items())[0][1] for d, v in data.items()]).mean(axis=0) lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec() print("lon range", lons.min(), lons.max()) print("lat range", lats.min(), lats.max()) # lons, lats, the_mean = commons.interpolate_to_uniform_global_grid(the_mean, lons_in=lons, lats_in=lats, out_dx=dx_plotting) # mask land lons1 = lons.copy() lons1[lons1 > 180] -= 360 ocean_mask = maskoceans(lons1, lats, np.zeros_like(the_mean)) the_mean = np.ma.masked_where(~ocean_mask.mask, the_mean) if xx is None: xx, yy = bmp(lons, lats) label_to_season_to_field[label][sname] = the_mean # plotting------ fig = plt.figure() gs = GridSpec(len(label_to_season_to_field), len(seasons) + 1, width_ratios=[1.0, ] * len(seasons) + [0.05, ]) # base_fields = None base_label = None for row, (label, season_to_mean) in enumerate(label_to_season_to_field.items()): print(gs.get_geometry()) axes = [fig.add_subplot(gs[row, col]) for col in range(len(season_to_mean) + 1)] common_params = dict(axes=axes, bmp=bmp, xx=xx, yy=yy) if base_fields is None: plot_row(row, season_to_mean=season_to_mean, label=label, **common_params) base_fields = season_to_mean base_label = label else: plot_row(row, season_to_mean=OrderedDict([(k, v - base_fields[k]) for k, v in season_to_mean.items()]), difference=True, clevs=clevs_diff, label="{}\n-\n{}".format(label, base_label), **common_params) fig.suptitle(long_name) img_folder = Path("industrial_and_preindustrial_sst").joinpath("seasonal") if not img_folder.is_dir(): img_folder.mkdir(parents=True) img_format = "png" img_file = img_folder.joinpath("{}.{}".format(var_name, img_format)) with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight", format=img_format)
def plot_seasonal_integrated_water_vapour(samples_folder, seasons=commons.default_seasons): # TODO: make the integrations vname = "HU" file_prefix = "dm" plot_units = "kg/kg" mult_coeff = 1 add_offset = 0 out_dx = 0.5 season_to_mean = OrderedDict() r = None for sname, months in seasons.items(): # find files for the season paths = [] for m in months: paths.extend( glob.glob( str( samples_folder.joinpath("*{:02d}/{}*".format( m, file_prefix))))) r = MultiRPN(paths) data = r.get_4d_field(varname=vname) season_to_mean[sname] = np.array( [list(v.items())[0][1] for k, v in data.items()]).mean(axis=0) print("Processed: {}".format(sname)) lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec() img_folder = samples_folder.joinpath("images/seasonal") if not img_folder.is_dir(): img_folder.mkdir(parents=True) img_file = img_folder.joinpath("{}.png".format(vname)) plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18) fig = plt.figure() ncols = 2 gs = GridSpec(len(season_to_mean) // ncols + int(not (len(season_to_mean) % ncols == 0)), ncols, wspace=0, hspace=0) xx, yy = None, None bmp = Basemap(projection="robin", lon_0=0) nlevs = 20 clevs = None for i, (sname, field) in enumerate(season_to_mean.items()): row = i // ncols col = i % ncols ax = fig.add_subplot(gs[row, col]) lons, lats, data_out = commons.interpolate_to_uniform_global_grid( field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx) if xx is None: xx, yy = bmp(lons, lats) cs = bmp.contourf(xx, yy, data_out * mult_coeff, nlevs if clevs is None else clevs, cmap=cm_basemap.s3pcpn_l, extend="max") # save color levels for next subplots clevs = cs.levels print(np.max(data_out * mult_coeff)) ax.set_title(sname) cb = plt.colorbar(cs, ax=ax) if not (row == 0 and col == ncols - 1): # cb.ax.set_title(plot_units) cb.ax.set_visible(False) bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH) fig.suptitle("Precipitation, {}".format(plot_units)) with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight") plt.close(fig)
def plot_seasonal_vertical_velocity(samples_folder, seasons=commons.default_seasons): level = 850 # millibars vname = "WW" long_name = "Vertical compoent of wind velocity ({}mb)".format(level) file_prefix = "dp" plot_units = "Pa/s" mult_coeff = 1 add_offset = 0 out_dx = 0.5 file_format = "png" season_to_mean = OrderedDict() r = None for sname, months in seasons.items(): # find files for the season paths = [] for m in months: paths.extend( glob.glob( str( samples_folder.joinpath("*{:02d}/{}*".format( m, file_prefix))))) r = MultiRPN(paths) data = r.get_4d_field(varname=vname) season_to_mean[sname] = np.array([v[level] for k, v in data.items() ]).mean(axis=0) print("Processed: {}".format(sname)) lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec() img_folder = samples_folder.joinpath("images/seasonal") if not img_folder.is_dir(): img_folder.mkdir(parents=True) img_file = img_folder.joinpath("{}_{}hPa.{}".format( vname, level, file_format)) plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18) fig = plt.figure() ncols = 2 gs = GridSpec(len(season_to_mean) // ncols + int(not (len(season_to_mean) % ncols == 0)), ncols, wspace=0, hspace=0) xx, yy = None, None bmp = Basemap(projection="robin", lon_0=0) cmap = cm_basemap.GMT_polar clevs = np.arange(-0.1, 0.11, 0.01) for i, (sname, field) in enumerate(season_to_mean.items()): row = i // ncols col = i % ncols ax = fig.add_subplot(gs[row, col]) lons, lats, data_out = commons.interpolate_to_uniform_global_grid( field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx) if xx is None: xx, yy = bmp(lons, lats) cs = bmp.contourf(xx, yy, data_out * mult_coeff, 20 if clevs is None else clevs, cmap=cmap, extend="both") # save color levels for next subplots clevs = cs.levels print(np.max(data_out * mult_coeff)) ax.set_title(sname) cb = plt.colorbar(cs, ax=ax) if not (row == 0 and col == ncols - 1): # cb.ax.set_title(plot_units) cb.ax.set_visible(False) bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH) fig.suptitle("{}, {}".format(long_name, plot_units)) with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight", format=file_format) plt.close(fig)
def get_daily_clim_profiles(samples_dir, start_year=-np.Inf, end_year=np.Inf, filename_prefix="dp", varname="", mask=None): """ dates_2d, levels_2d, values - needed for plotting :param start_year: :param end_year: :param filename_prefix: :param varname: """ dates_sorted = None levels_sorted = None yearly_fields = [] day = timedelta(days=1) stamp_dates = [datetime(2001, 1, 1) + i * day for i in range(365)] for y in range(start_year, end_year + 1): files_for_year = [] mfolders = [os.path.join(samples_dir, f) for f in os.listdir(samples_dir) if f[:-2].endswith(str(y))] for mfolder in mfolders: files_for_year += [os.path.join(mfolder, fn) for fn in os.listdir(mfolder) if fn.startswith(filename_prefix)] mrpn = MultiRPN(files_for_year) data = mrpn.get_4d_field(varname=varname) # calculate the area-average for d, lev_to_field in data.items(): for lev in lev_to_field: lev_to_field[lev] = lev_to_field[lev][mask].mean() # dates_sorted = list(sorted([d for d in data if not (d.month == 2 and d.day == 29) and (d.year == y)])) levels_sorted = list(sorted([lev for lev in data[dates_sorted[0]] if lev >= 500])) # year_field = np.zeros((len(dates_sorted), len(levels_sorted))) for j, lev in enumerate(levels_sorted): for i, the_date in enumerate(dates_sorted): year_field[i, j] = data[the_date][lev] panel = pd.DataFrame(data=year_field, index=dates_sorted, columns=range(len(levels_sorted))) daily = panel.groupby(by=lambda d: datetime(d.year, d.month, d.day)).mean() yearly_fields.append(daily.values) values = np.mean(yearly_fields, axis=0) assert not hasattr(values, "mask") dates_num = date2num(stamp_dates) levs_2d, dates_num_2d = np.meshgrid(levels_sorted, dates_num) assert levs_2d.shape == values.shape, "levels and values shapes ({} and {}, respectively) are not consistent.".format(levs_2d.shape, values.shape) return dates_num_2d, levs_2d, values
def plot_seasonal_integrated_water_vapour(samples_folder, seasons=commons.default_seasons): # TODO: make the integrations vname = "HU" file_prefix = "dm" plot_units = "kg/kg" mult_coeff = 1 add_offset = 0 out_dx = 0.5 season_to_mean = OrderedDict() r = None for sname, months in seasons.items(): # find files for the season paths = [] for m in months: paths.extend(glob.glob(str(samples_folder.joinpath("*{:02d}/{}*".format(m, file_prefix))))) r = MultiRPN(paths) data = r.get_4d_field(varname=vname) season_to_mean[sname] = np.array([list(v.items())[0][1] for k, v in data.items()]).mean(axis=0) print("Processed: {}".format(sname)) lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec() img_folder = samples_folder.joinpath("images/seasonal") if not img_folder.is_dir(): img_folder.mkdir(parents=True) img_file = img_folder.joinpath("{}.png".format(vname)) plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18) fig = plt.figure() ncols = 2 gs = GridSpec(len(season_to_mean) // ncols + int(not (len(season_to_mean) % ncols == 0)), ncols, wspace=0, hspace=0) xx, yy = None, None bmp = Basemap(projection="robin", lon_0=0) nlevs = 20 clevs = None for i, (sname, field) in enumerate(season_to_mean.items()): row = i // ncols col = i % ncols ax = fig.add_subplot(gs[row, col]) lons, lats, data_out = commons.interpolate_to_uniform_global_grid(field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx) if xx is None: xx, yy = bmp(lons, lats) cs = bmp.contourf(xx, yy, data_out * mult_coeff, nlevs if clevs is None else clevs, cmap=cm_basemap.s3pcpn_l, extend="max") # save color levels for next subplots clevs = cs.levels print(np.max(data_out * mult_coeff)) ax.set_title(sname) cb = plt.colorbar(cs, ax=ax) if not (row == 0 and col == ncols - 1): # cb.ax.set_title(plot_units) cb.ax.set_visible(False) bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH) fig.suptitle("Precipitation, {}".format(plot_units)) with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight") plt.close(fig)
def get_area_avg_timeseries(samples_dir, start_year=-np.Inf, end_year=np.Inf, filename_prefix="pm", level=-1, level_kind=level_kinds.ARBITRARY, varname="", mask=None, mask_lons2d=None, mask_lats2d=None, file_per_var=False) -> pd.Series: """ get the timeseries of area averaged ice fraction :rtype : pd.Series """ yearly_ts = [] lons2d, lats2d = None, None samples_dir_p = Path(samples_dir) # interpolated mask interpolated_mask = None for y in range(start_year, end_year + 1): files_for_year = [] mfolders = [f for f in samples_dir_p.iterdir() if f.name[:-2].endswith(str(y))] for mfolder in mfolders: # Select all files containing the varname in the filename if file_per_var: files_for_year += [str(f) for f in mfolder.iterdir() if varname in f.name] else: files_for_year += [str(f) for f in mfolder.iterdir() if f.name.startswith(filename_prefix) and f.name[-9:-1] != "0" * 8] if len(files_for_year) == 0: continue mrpn = MultiRPN(files_for_year) data = mrpn.get_all_time_records_for_name_and_level(varname=varname, level=level, level_kind=level_kind) if lons2d is None: lons2d, lats2d = mrpn.get_longitudes_and_latitudes_of_the_last_read_rec() # interpolate the mask using nearest neighbour approach if interpolated_mask is None: xs, ys, zs = lat_lon.lon_lat_to_cartesian(mask_lons2d.flatten(), mask_lats2d.flatten()) ktree = KDTree(data=list(zip(xs, ys, zs))) xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1) interpolated_mask = mask.flatten()[inds] interpolated_mask.shape = lons2d.shape for t, field in data.items(): data[t] = field[interpolated_mask].mean() tlist = [t for t in data.keys()] ser = pd.Series(index=tlist, data=[data[t] for t in tlist]) ser.sort_index(inplace=True) yearly_ts.append(ser) mrpn.close() return pd.concat(yearly_ts), lons2d, lats2d
def main(): start_year = 1979 end_year = 1981 HL_LABEL = "CRCM5_HL" NEMO_LABEL = "CRCM5_NEMO" file_prefix = "dm" level = 1 level_type = level_kinds.HYBRID wind_comp_names = ["UU", "VV"] sim_label_to_path = OrderedDict( [(HL_LABEL, "/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples"), (NEMO_LABEL, "/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples")] ) # get a coord file ... coord_file = "" found_coord_file = False for mdir in os.listdir(sim_label_to_path[HL_LABEL]): mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir) if not os.path.isdir(mdir_path): continue for fn in os.listdir(mdir_path): print(fn) if fn[:2] not in ["pm", "dm", "pp", "dp"]: continue coord_file = os.path.join(mdir_path, fn) found_coord_file = True if found_coord_file: break bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file(path=coord_file) xx, yy = bmp(lons, lats) lons[lons > 180] -= 360 # loop through all files rotate vaectors and save to netcdf for sim_label, samples_dir in sim_label_to_path.items(): samples = Path(samples_dir) po = samples.parent monthdate_to_path_list = nemo_hl_util.get_monthyeardate_to_paths_map(file_prefix=file_prefix, start_year=start_year, end_year=end_year, samples_dir_path=samples) # Netcdf output file to put rotated winds po /= "rotated_wind_{}.nc".format(sim_label) with Dataset(str(po), "w") as ds: ds.createDimension("time", None) ds.createDimension("lon", lons.shape[0]) ds.createDimension("lat", lons.shape[1]) # create the schema of the output file vname_to_ncvar = {} for vname in wind_comp_names: vname_to_ncvar[vname] = ds.createVariable(vname, "f4", dimensions=("time", "lon", "lat")) vname_to_ncvar[vname].units = "knots" lons_var = ds.createVariable("lon", "f4", dimensions=("lon", "lat")) lats_var = ds.createVariable("lat", "f4", dimensions=("lon", "lat")) time_var = ds.createVariable("time", "i8", dimensions=("time",)) time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format(datetime(start_year, 1, 1)) lons_var[:] = lons lats_var[:] = lats # use sorted dates record_count = 0 for month_date in sorted(monthdate_to_path_list): # select only dm files mr = MultiRPN(path=monthdate_to_path_list[month_date]) vname_to_fields = {} for vname in wind_comp_names: vname_to_fields[vname] = mr.get_all_time_records_for_name_and_level(varname=vname, level=level, level_kind=level_type) for ti, t in enumerate(sorted(vname_to_fields[wind_comp_names[0]])): time_var[record_count] = date2num(t, time_var.units) uu = vname_to_fields[wind_comp_names[0]][t] vv = vname_to_fields[wind_comp_names[1]][t] uu_rot, vv_rot = rotate_vecs_from_geo_to_rotpole(uu, vv, lons, lats, bmp=bmp) # in knots not in m/s vname_to_ncvar[wind_comp_names[0]][record_count, :, :] = uu_rot vname_to_ncvar[wind_comp_names[1]][record_count, :, :] = vv_rot record_count += 1
def main(plot_vals=False): varname = "RFAC" multiplier = 24 * 3600 data_folder = Path( "/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/atm_data_for_Arman_simulations" ) day_range = range(19, 22) dates_of_interest = [datetime(2013, 6, d) for d in day_range] img_folder = Path("calgary_flood/2D") if not img_folder.is_dir(): img_folder.mkdir(parents=True) lons, lats, bmp = None, None, None the_mask = get_bow_river_basin_mask() i_list, j_list = np.where(the_mask > 0.5) imin, imax = i_list.min() - 2, i_list.max() + 5 jmin, jmax = j_list.min() - 2, j_list.max() + 5 # Calculate daily means sim_label_to_date_to_mean = OrderedDict() for sim_dir in data_folder.iterdir(): mr = MultiRPN(str(sim_dir.joinpath("pm*"))) print(str(sim_dir)) print(mr.get_number_of_records()) label = sim_dir.name.split("_")[-2].replace("NoDrain", "").replace( "frozen", "Frozen").replace("Bow", "") sim_label_to_date_to_mean[label] = OrderedDict() data = mr.get_4d_field(varname) data = { d: list(v.items())[0][1] for d, v in data.items() if d.day in day_range } for d in dates_of_interest: sim_label_to_date_to_mean[label][d] = np.array([ field for d1, field in data.items() if d1.day == d.day ]).mean(axis=0) * multiplier if lons is None: lons, lats = mr.get_longitudes_and_latitudes_of_the_last_read_rec() for f in sim_dir.iterdir(): if f.name.startswith("pm"): r = RPN(str(f)) r.get_first_record_for_name(varname=varname) rll = RotatedLatLon( **r.get_proj_parameters_for_the_last_read_rec()) bmp = rll.get_basemap_object_for_lons_lats( lons2d=lons[imin:imax, jmin:jmax], lats2d=lats[imin:imax, jmin:jmax], resolution="i") r.close() break mr.close() # reorder simulations sim_label_to_date_to_mean = OrderedDict([ (k, sim_label_to_date_to_mean[k]) for k in sorted( sim_label_to_date_to_mean, key=lambda z: len(z), reverse=True) ]) key_list = [k for k in sim_label_to_date_to_mean] key_list[-2], key_list[-1] = key_list[-1], key_list[-2] sim_label_to_date_to_mean = OrderedDict([(k, sim_label_to_date_to_mean[k]) for k in key_list]) # do the plots (subplots: vertically - simulations, horizontally - days) plot_utils.apply_plot_params(width_cm=24, height_cm=28, font_size=10) fig = plt.figure() nrows = len(sim_label_to_date_to_mean) ncols = len(day_range) gs = GridSpec(nrows, ncols, wspace=0, hspace=0.1) clevs_vals = [0, 0.1, 20, 50, 100, 150, 200, 300] base_label = None clevs_diff = [1, 5, 10, 20, 50, 100] clevs_diff = [-c for c in reversed(clevs_diff)] + [0] + clevs_diff cmap_diff = cm.get_cmap("bwr", len(clevs_diff) - 1) cmap_vals = get_cmap_from_ncl_spec_file( path="colormap_files/precip3_16lev.rgb", ncolors=len(clevs_vals) - 1) xx, yy = bmp(lons, lats) title = "Total runoff ({}, mm/day)".format(varname) fig.suptitle(title) for row, (sim_label, date_to_field) in enumerate(sim_label_to_date_to_mean.items()): if row == 0 or plot_vals: base_sim = OrderedDict([(k, 0) for k, v in date_to_field.items()]) base_label = sim_label plot_label = sim_label clevs = clevs_vals cmap = cmap_vals extend = "max" else: base_sim = list(sim_label_to_date_to_mean.items())[0][1] plot_label = "{}\n-\n{}".format(sim_label, base_label) clevs = clevs_diff cmap = cmap_diff extend = "both" bn = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1) for col, (the_date, field) in enumerate(date_to_field.items()): ax = fig.add_subplot(gs[row, col]) to_plot = np.ma.masked_where(the_mask < 0.5, field - base_sim[the_date]) # cs = bmp.contourf(xx[~to_plot.mask], yy[~to_plot.mask], to_plot[~to_plot.mask], levels=clevs, extend="max", tri=True) cs = bmp.pcolormesh(xx, yy, to_plot[:-1, :-1], norm=bn, vmin=clevs[0], vmax=clevs[-1], cmap=cmap) bmp.drawcoastlines(ax=ax, linewidth=0.3) assert isinstance(bmp, Basemap) bmp.readshapefile(BOW_RIVER_SHP[:-4], "basin", zorder=5) cb = bmp.colorbar(cs, ax=ax, ticks=clevs, extend=extend, pad="4%", size="10%") if plot_vals: cb.ax.set_visible(col == ncols - 1 and row == 0) else: cb.ax.set_visible(col == ncols - 1 and row in (0, 1)) if col == 0: ax.set_ylabel(plot_label) if row == 0: ax.set_title(the_date.strftime("%b %d")) if plot_vals: img_file = img_folder.joinpath("{}.png".format(varname)) else: img_file = img_folder.joinpath("{}_diff.png".format(varname)) fig.savefig(str(img_file)) plt.close(fig)
def main(): plot_utils.apply_plot_params(width_cm=40, height_cm=25, font_size=14) seasons = commons.default_seasons dx_plotting = 0.5 # degrees # units = "$^\circ$C" # long_name = "SST, {}".format(units) # var_name = "TM" # clevs_diff = np.arange(-5, 5.5, 0.5) units = "" long_name = "Sea ice {}".format(units) var_name = "LG" clevs_diff = [v for v in np.arange(-1, 1.1, 0.1) if abs(v) > 1.0e-6] label_to_folder = OrderedDict([ ("ERA-Interim", "/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/SST_SeaIce/I_SST_SeaIce" ), ("PreI-CanESM2", "/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce" ), ("PreI-GFDL", "/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce" ), ("PreI-GISS", "/RESCUE/skynet3_rech1/huziy/CNRCWP/Calgary_flood/SST_SeaIce/PreI_SST_SeaIce" ) ]) bmp = Basemap(projection="robin", lon_0=180) xx, yy = None, None year = 2013 print(seasons) label_to_season_to_field = OrderedDict() for label, folder in label_to_folder.items(): label_to_season_to_field[label] = OrderedDict() all_files = [ f for f in os.listdir(folder) if label.lower() in f.lower() or label.split("-")[1].lower() in f.lower() ] for sname, months in seasons.items(): season_files = [ f for f in all_files if _get_month_and_year_from_file_name(f)[0] in months and _get_month_and_year_from_file_name(f)[1] == year ] season_files = [os.path.join(folder, f) for f in season_files] print(10 * "+" + sname + "+" * 10) print(season_files) r = MultiRPN(season_files) data = r.get_4d_field(var_name) the_mean = np.array( [list(v.items())[0][1] for d, v in data.items()]).mean(axis=0) lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec() print("lon range", lons.min(), lons.max()) print("lat range", lats.min(), lats.max()) # lons, lats, the_mean = commons.interpolate_to_uniform_global_grid(the_mean, lons_in=lons, lats_in=lats, out_dx=dx_plotting) # mask land lons1 = lons.copy() lons1[lons1 > 180] -= 360 ocean_mask = maskoceans(lons1, lats, np.zeros_like(the_mean)) the_mean = np.ma.masked_where(~ocean_mask.mask, the_mean) if xx is None: xx, yy = bmp(lons, lats) label_to_season_to_field[label][sname] = the_mean # plotting------ fig = plt.figure() gs = GridSpec(len(label_to_season_to_field), len(seasons) + 1, width_ratios=[ 1.0, ] * len(seasons) + [ 0.05, ]) # base_fields = None base_label = None for row, (label, season_to_mean) in enumerate(label_to_season_to_field.items()): print(gs.get_geometry()) axes = [ fig.add_subplot(gs[row, col]) for col in range(len(season_to_mean) + 1) ] common_params = dict(axes=axes, bmp=bmp, xx=xx, yy=yy) if base_fields is None: plot_row(row, season_to_mean=season_to_mean, label=label, **common_params) base_fields = season_to_mean base_label = label else: plot_row(row, season_to_mean=OrderedDict([ (k, v - base_fields[k]) for k, v in season_to_mean.items() ]), difference=True, clevs=clevs_diff, label="{}\n-\n{}".format(label, base_label), **common_params) fig.suptitle(long_name) img_folder = Path("industrial_and_preindustrial_sst").joinpath("seasonal") if not img_folder.is_dir(): img_folder.mkdir(parents=True) img_format = "png" img_file = img_folder.joinpath("{}.{}".format(var_name, img_format)) with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight", format=img_format)
def main(plot_vals=False): varname = "RFAC" multiplier = 24 * 3600 data_folder = Path("/home/huziy/skynet3_rech1/CNRCWP/Calgary_flood/atm_data_for_Arman_simulations") day_range = range(19, 22) dates_of_interest = [datetime(2013, 6, d) for d in day_range] img_folder = Path("calgary_flood/2D") if not img_folder.is_dir(): img_folder.mkdir(parents=True) lons, lats, bmp = None, None, None the_mask = get_bow_river_basin_mask() i_list, j_list = np.where(the_mask > 0.5) imin, imax = i_list.min() - 2, i_list.max() + 5 jmin, jmax = j_list.min() - 2, j_list.max() + 5 # Calculate daily means sim_label_to_date_to_mean = OrderedDict() for sim_dir in data_folder.iterdir(): mr = MultiRPN(str(sim_dir.joinpath("pm*"))) print(str(sim_dir)) print(mr.get_number_of_records()) label = sim_dir.name.split("_")[-2].replace("NoDrain", "").replace("frozen", "Frozen").replace("Bow", "") sim_label_to_date_to_mean[label] = OrderedDict() data = mr.get_4d_field(varname) data = {d: list(v.items())[0][1] for d, v in data.items() if d.day in day_range} for d in dates_of_interest: sim_label_to_date_to_mean[label][d] = ( np.array([field for d1, field in data.items() if d1.day == d.day]).mean(axis=0) * multiplier ) if lons is None: lons, lats = mr.get_longitudes_and_latitudes_of_the_last_read_rec() for f in sim_dir.iterdir(): if f.name.startswith("pm"): r = RPN(str(f)) r.get_first_record_for_name(varname=varname) rll = RotatedLatLon(**r.get_proj_parameters_for_the_last_read_rec()) bmp = rll.get_basemap_object_for_lons_lats( lons2d=lons[imin:imax, jmin:jmax], lats2d=lats[imin:imax, jmin:jmax], resolution="i" ) r.close() break mr.close() # reorder simulations sim_label_to_date_to_mean = OrderedDict( [ (k, sim_label_to_date_to_mean[k]) for k in sorted(sim_label_to_date_to_mean, key=lambda z: len(z), reverse=True) ] ) key_list = [k for k in sim_label_to_date_to_mean] key_list[-2], key_list[-1] = key_list[-1], key_list[-2] sim_label_to_date_to_mean = OrderedDict([(k, sim_label_to_date_to_mean[k]) for k in key_list]) # do the plots (subplots: vertically - simulations, horizontally - days) plot_utils.apply_plot_params(width_cm=24, height_cm=28, font_size=10) fig = plt.figure() nrows = len(sim_label_to_date_to_mean) ncols = len(day_range) gs = GridSpec(nrows, ncols, wspace=0, hspace=0.1) clevs_vals = [0, 0.1, 20, 50, 100, 150, 200, 300] base_label = None clevs_diff = [1, 5, 10, 20, 50, 100] clevs_diff = [-c for c in reversed(clevs_diff)] + [0] + clevs_diff cmap_diff = cm.get_cmap("bwr", len(clevs_diff) - 1) cmap_vals = get_cmap_from_ncl_spec_file(path="colormap_files/precip3_16lev.rgb", ncolors=len(clevs_vals) - 1) xx, yy = bmp(lons, lats) title = "Total runoff ({}, mm/day)".format(varname) fig.suptitle(title) for row, (sim_label, date_to_field) in enumerate(sim_label_to_date_to_mean.items()): if row == 0 or plot_vals: base_sim = OrderedDict([(k, 0) for k, v in date_to_field.items()]) base_label = sim_label plot_label = sim_label clevs = clevs_vals cmap = cmap_vals extend = "max" else: base_sim = list(sim_label_to_date_to_mean.items())[0][1] plot_label = "{}\n-\n{}".format(sim_label, base_label) clevs = clevs_diff cmap = cmap_diff extend = "both" bn = BoundaryNorm(boundaries=clevs, ncolors=len(clevs) - 1) for col, (the_date, field) in enumerate(date_to_field.items()): ax = fig.add_subplot(gs[row, col]) to_plot = np.ma.masked_where(the_mask < 0.5, field - base_sim[the_date]) # cs = bmp.contourf(xx[~to_plot.mask], yy[~to_plot.mask], to_plot[~to_plot.mask], levels=clevs, extend="max", tri=True) cs = bmp.pcolormesh(xx, yy, to_plot[:-1, :-1], norm=bn, vmin=clevs[0], vmax=clevs[-1], cmap=cmap) bmp.drawcoastlines(ax=ax, linewidth=0.3) assert isinstance(bmp, Basemap) bmp.readshapefile(BOW_RIVER_SHP[:-4], "basin", zorder=5) cb = bmp.colorbar(cs, ax=ax, ticks=clevs, extend=extend, pad="4%", size="10%") if plot_vals: cb.ax.set_visible(col == ncols - 1 and row == 0) else: cb.ax.set_visible(col == ncols - 1 and row in (0, 1)) if col == 0: ax.set_ylabel(plot_label) if row == 0: ax.set_title(the_date.strftime("%b %d")) if plot_vals: img_file = img_folder.joinpath("{}.png".format(varname)) else: img_file = img_folder.joinpath("{}_diff.png".format(varname)) fig.savefig(str(img_file)) plt.close(fig)
def main(): erainterim_075_folder = "/HOME/data/Validation/ERA-Interim_0.75/Offline_driving_data/3h_Forecast" vname = "PR" start_year = 1980 end_year = 2010 season_key = "summer" season_labels = {season_key: "Summer"} season_to_months = OrderedDict([ (season_key, [6, 7, 8]) ]) # Validate temperature and precip model_vars = ["TT", "PR"] obs_vars = ["tmp", "pre"] obs_paths = [ "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc", "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc" ] model_var_to_obs_var = dict(zip(model_vars, obs_vars)) model_var_to_obs_path = dict(zip(model_vars, obs_paths)) obs_path = model_var_to_obs_path[vname] cru = CRUDataManager(var_name=model_var_to_obs_var[vname], path=obs_path) # Calculate seasonal means for CRU seasonal_clim_fields_cru = cru.get_seasonal_means(season_name_to_months=season_to_months, start_year=start_year, end_year=end_year) # Calculate seasonal mean for erai flist = get_files_for_season(erainterim_075_folder, start_year=start_year, end_year=end_year, months=season_to_months[season_key]) rpf = MultiRPN(flist) date_to_field_erai075 = rpf.get_all_time_records_for_name_and_level(varname=vname, level=-1) # Convert to mm/day era075 = np.mean([field for field in date_to_field_erai075.values()], axis=0) * 24 * 3600 * 1000 lons_era, lats_era = rpf.get_longitudes_and_latitudes_of_the_last_read_rec() seasonal_clim_fields_cru_interp = OrderedDict() # Calculate biases for season, cru_field in seasonal_clim_fields_cru.items(): seasonal_clim_fields_cru_interp[season] = cru.interpolate_data_to(cru_field, lons2d=lons_era, lats2d=lats_era, nneighbours=1) # Do the plotting ------------------------------------------------------------------------------ plot_utils.apply_plot_params() fig = plt.figure() b = Basemap() gs = gridspec.GridSpec(nrows=3, ncols=1) ax = fig.add_subplot(gs[0, 0]) xx, yy = b(cru.lons2d, cru.lats2d) cs = b.contourf(xx, yy, seasonal_clim_fields_cru[season_key], 20) b.drawcoastlines(ax=ax) ax.set_title("CRU") plt.colorbar(cs, ax=ax) ax = fig.add_subplot(gs[1, 0]) lons_era[lons_era > 180] -= 360 lons_era, era075 = b.shiftdata(lons_era, datain=era075, lon_0=0) xx, yy = b(lons_era, lats_era) # mask oceans in the era plot as well era075 = maskoceans(lons_era, lats_era, era075) cs = b.contourf(xx, yy, era075, levels=cs.levels, norm=cs.norm, cmap=cs.cmap, ax=ax) b.drawcoastlines(ax=ax) ax.set_title("ERA-Interim 0.75") plt.colorbar(cs, ax=ax) # differences ax = fig.add_subplot(gs[2, 0]) diff = era075 - seasonal_clim_fields_cru_interp[season_key] delta = np.percentile(np.abs(diff)[~diff.mask], 90) clevs = np.linspace(-delta, delta, 20) cs = b.contourf(xx, yy, diff, levels=clevs, cmap="RdBu_r", extend="both") b.drawcoastlines(ax=ax) ax.set_title("ERA-Interim 0.75 - CRU") plt.colorbar(cs, ax=ax) plt.show() fig.savefig(os.path.join(img_folder, "erai0.75_vs_cru_precip.png"), bbox_inches="tight")
def main(): #path = "/RECH/data/Simulations/CRCM5/North_America/NorthAmerica_0.44deg_ERA40-Int_B1/Diagnostics/NorthAmerica_0.44deg_ERA40-Int_B1_2007{:02d}" path = "/RESCUE/skynet3_rech1/huziy/from_guillimin/new_outputs/current_climate_30_yr_sims/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/Samples/quebec_crcm5-hcd-rl-intfl_1988{:02d}" months = [6, 7, 8] pm_list = [] dm_list = [] for m in months: print(path.format(m)) month_folder = path.format(m) for fn in os.listdir(month_folder): # if not fn.endswith("moyenne"): # continue if fn.startswith("pm"): pm_list.append(os.path.join(month_folder, fn)) elif fn.startswith("dm"): dm_list.append(os.path.join(month_folder, fn)) pm = MultiRPN(pm_list) dm = MultiRPN(dm_list) tsurf_mean = np.mean([ field for field in pm.get_all_time_records_for_name_and_level( varname="J8").values() ], axis=0) tair_mean = np.mean([ field for field in dm.get_all_time_records_for_name_and_level( varname="TT", level=1, level_kind=level_kinds.HYBRID).values() ], axis=0) lons, lats = pm.get_longitudes_and_latitudes_of_the_last_read_rec() projparams = pm.linked_robj_list[ 0].get_proj_parameters_for_the_last_read_rec() rll = RotatedLatLon(**projparams) bmp = rll.get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats) xx, yy = bmp(lons, lats) plt.figure() cs = bmp.contourf(xx, yy, tsurf_mean - 273.15, 40) bmp.drawcoastlines() plt.title("Tsurf") plt.colorbar() plt.figure() bmp.contourf(xx, yy, tair_mean, levels=cs.levels, norm=cs.norm, cmap=cs.cmap) bmp.drawcoastlines() plt.title("Tair") plt.colorbar() plt.figure() bmp.contourf(xx, yy, tsurf_mean - 273.15 - tair_mean, levels=np.arange(-2, 2.2, 0.2), cmap=cs.cmap) bmp.drawcoastlines() plt.title("Tsurf - Tair") plt.colorbar() pm.close() dm.close() plt.show()
def get_area_avg_timeseries(samples_dir, start_year=-np.Inf, end_year=np.Inf, filename_prefix="pm", level=-1, level_kind=level_kinds.ARBITRARY, varname="", mask=None, mask_lons2d=None, mask_lats2d=None, file_per_var=False) -> pd.Series: """ get the timeseries of area averaged ice fraction :rtype : pd.Series """ yearly_ts = [] lons2d, lats2d = None, None samples_dir_p = Path(samples_dir) # interpolated mask interpolated_mask = None for y in range(start_year, end_year + 1): files_for_year = [] mfolders = [ f for f in samples_dir_p.iterdir() if f.name[:-2].endswith(str(y)) ] for mfolder in mfolders: # Select all files containing the varname in the filename if file_per_var: files_for_year += [ str(f) for f in mfolder.iterdir() if varname in f.name ] else: files_for_year += [ str(f) for f in mfolder.iterdir() if f.name.startswith(filename_prefix) and f.name[-9:-1] != "0" * 8 ] if len(files_for_year) == 0: continue mrpn = MultiRPN(files_for_year) data = mrpn.get_all_time_records_for_name_and_level( varname=varname, level=level, level_kind=level_kind) if lons2d is None: lons2d, lats2d = mrpn.get_longitudes_and_latitudes_of_the_last_read_rec( ) # interpolate the mask using nearest neighbour approach if interpolated_mask is None: xs, ys, zs = lat_lon.lon_lat_to_cartesian(mask_lons2d.flatten(), mask_lats2d.flatten()) ktree = KDTree(data=list(zip(xs, ys, zs))) xt, yt, zt = lat_lon.lon_lat_to_cartesian(lons2d.flatten(), lats2d.flatten()) dists, inds = ktree.query(list(zip(xt, yt, zt)), k=1) interpolated_mask = mask.flatten()[inds] interpolated_mask.shape = lons2d.shape for t, field in data.items(): data[t] = field[interpolated_mask].mean() tlist = [t for t in data.keys()] ser = pd.Series(index=tlist, data=[data[t] for t in tlist]) ser.sort_index(inplace=True) yearly_ts.append(ser) mrpn.close() return pd.concat(yearly_ts), lons2d, lats2d
def get_model_data(station_to_model_point, output_path=None, grid_config=None, basins_of_interest_shp="", cell_manager=None, vname=None): lons, lats, bmp = None, None, None data_mask = None monthly_diagnostics_case = False if output_path.name.lower().endswith("diagnostics"): fname_pattern = "pm*_moyenne" monthly_diagnostics_case = True else: fname_pattern = "pm*p" pattern = re.compile(".*" + 8 * "0" + ".*") flist = [f for f in glob.glob(str(output_path.joinpath("*").joinpath(fname_pattern))) if pattern.match(f) is None] r = MultiRPN(flist) date_to_field = r.get_all_time_records_for_name_and_level(varname=vname) lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec() r.close() # get the basemap object bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest( lons, lats, shp_path=basins_of_interest_shp, mask_margin=5) station_to_model_data = {} # model data are the pandas timeseries stations_to_ignore = [] for station, model_point in station_to_model_point.items(): assert isinstance(model_point, ModelPoint) assert isinstance(cell_manager, CellManager) assert isinstance(station, cehq_station.Station) upstream_mask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices(model_point.ix, model_point.jy) # Skip model points and staions with small number of gridcells upstream if upstream_mask.sum() <= 1: stations_to_ignore.append(station) print("Station {} is ignored, because the number of upstream cells is <= 1.".format(station.id)) continue # Skip model points and stations outside the region of interest if not data_mask[model_point.ix, model_point.jy]: stations_to_ignore.append(station) print("Station {} is ignored, because it is outside of the domain of interest.".format(station.id)) continue # Plot station position fig = plt.figure() ax = plt.gca() lons1, lats1 = lons[upstream_mask > 0.5], lats[upstream_mask > 0.5] x1, y1 = bmp(lons1, lats1) bmp.drawrivers() bmp.drawcoastlines(ax=ax) bmp.drawcountries(ax=ax, linewidth=0.2) bmp.drawstates(linewidth=0.1) bmp.readshapefile(basins_of_interest_shp[:-4], "basin", linewidth=2, color="m") bmp.scatter(x1, y1, c="g", s=100) bmp.scatter(*bmp(lons[model_point.ix, model_point.jy], lats[model_point.ix, model_point.jy]), c="b", s=250) fig.tight_layout() plt.savefig(str(img_folder.joinpath("{}_position_and_upstream.png".format(station.id))), bbox_inche="tight") plt.close(fig) res = pd.Series(index=sorted(date_to_field.keys()), data=[date_to_field[d][model_point.ix, model_point.jy] for d in sorted(date_to_field.keys())]) # get monthly means res = res.groupby(lambda d: d.replace(day=15, hour=0)).mean() if monthly_diagnostics_case: # shift to the end of the month before a previous month, and then shift 15 days to later res = res.shift(-2, freq="M").shift(15, freq="D") print(res.index[:20]) print(res.index[-20:]) station_to_model_data[station] = res # Not enough drainage area for s in stations_to_ignore: del station_to_model_point[s] return station_to_model_data
def plot_monthly_clim_in_a_panel(months=None, diag_folder="", vname="STFL", grid_config=None, basins_of_interest_shp=""): """ Plots climatologies using diagnostics outputs, not samples :param months: :param diag_folder: :param vname: """ if months is None: months = list(range(1, 13)) diag_path = Path(diag_folder) month_to_field = OrderedDict() lons, lats, bmp = None, None, None data_mask = None for m in months: r = MultiRPN(str(diag_path.joinpath("*{:02d}".format(m)).joinpath("pm*_moyenne"))) date_to_field = r.get_all_time_records_for_name_and_level() the_mean = np.mean([f for f in date_to_field.values()], axis=0) the_mean = np.ma.masked_where(the_mean < 0, the_mean) month_to_field[m] = the_mean if bmp is None: lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec() # get the basemap object bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest( lons, lats, shp_path=basins_of_interest_shp, mask_margin=5) r.close() fig = plt.figure() ncols = 3 nrows = len(months) // ncols + int(len(months) % ncols != 0) gs = GridSpec(nrows=nrows, ncols=ncols + 1) xx, yy = bmp(lons, lats) clevs = [0, 20, 50, 100, 200, 500, 1000, 1500, 3000, 4500, 5000, 7000, 9000] bn = BoundaryNorm(clevs, len(clevs) - 1) cmap = cm.get_cmap("jet", len(clevs) - 1) for m, field in month_to_field.items(): row = (m - 1) // ncols col = (m - 1) % ncols ax = fig.add_subplot(gs[row, col]) ax.set_title(calendar.month_name[m]) to_plot = np.ma.masked_where(~data_mask, field) im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1]) bmp.colorbar(im, extend="max") bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax) bmp.drawcoastlines(ax=ax) plt.close(fig) # plot annual mean ann_mean = np.mean([field for m, field in month_to_field.items()], axis=0) fig = plt.figure() ax = fig.add_subplot(gs[:, :]) ax.set_title("Annual mean") to_plot = np.ma.masked_where(~data_mask, ann_mean) im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1]) bmp.colorbar(im, extend="max") bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax) bmp.drawcoastlines(ax=ax) plt.show() plt.close(fig)
def main(): start_year = 1979 end_year = 1981 HL_LABEL = "CRCM5_HL" NEMO_LABEL = "CRCM5_NEMO" file_prefix = "pm" PR_level = -1 PR_level_type = level_kinds.ARBITRARY tprecip_vname = "PR" sprecip_vname = "SN" TT_level = 1 TT_level_type = level_kinds.HYBRID sim_label_to_path = OrderedDict([ (HL_LABEL, "/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples" ), (NEMO_LABEL, "/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples" ) ]) # get a coord file ... coord_file = "" found_coord_file = False for mdir in os.listdir(sim_label_to_path[HL_LABEL]): mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir) if not os.path.isdir(mdir_path): continue for fn in os.listdir(mdir_path): print(fn) if fn[:2] not in ["pm", "dm", "pp", "dp"]: continue coord_file = os.path.join(mdir_path, fn) found_coord_file = True if found_coord_file: break bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file( path=coord_file) xx, yy = bmp(lons, lats) # read necessary input and calculate snowfall and save to the file for sim_label, samples_dir in sim_label_to_path.items(): samples_dir_path = Path(samples_dir) TT_monthdate_to_paths = get_monthyeardate_to_paths_map( file_prefix="dm", start_year=start_year, end_year=end_year, samples_dir_path=samples_dir_path) PR_monthdate_to_paths = get_monthyeardate_to_paths_map( file_prefix="pm", start_year=start_year, end_year=end_year, samples_dir_path=samples_dir_path) with Dataset( str(samples_dir_path.parent / "{}_snow_fall_{}-{}.nc".format( sim_label, start_year, end_year)), "w") as ds: assert isinstance(ds, Dataset) ds.createDimension("time", None) ds.createDimension("lon", lons.shape[0]) ds.createDimension("lat", lons.shape[1]) # create the schema of the output file snow_fall = ds.createVariable(sprecip_vname, "f4", dimensions=("time", "lon", "lat")) lons_var = ds.createVariable("lon", "f4", dimensions=("lon", "lat")) lats_var = ds.createVariable("lat", "f4", dimensions=("lon", "lat")) time_var = ds.createVariable("time", "i8", dimensions=("time", )) time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format( datetime(start_year, 1, 1)) lons_var[:] = lons lats_var[:] = lats # use sorted dates record_count = 0 for month_date in sorted(TT_monthdate_to_paths): tt = MultiRPN(path=TT_monthdate_to_paths[month_date] ).get_all_time_records_for_name_and_level( varname="TT", level=TT_level, level_kind=TT_level_type) pr = MultiRPN(path=PR_monthdate_to_paths[month_date] ).get_all_time_records_for_name_and_level( varname="PR", level=PR_level, level_kind=PR_level_type) print("Processing {}".format(month_date)) for d in sorted(tt): t_field = tt[d] pr_field = pr[d] sn = get_snow_fall_m_per_s(precip_m_per_s=pr_field, tair_deg_c=t_field) snow_fall[record_count, :, :] = sn time_var[record_count] = date2num(d, time_var.units) record_count += 1
def main(): vname = "VV" start_year = 1980 end_year = 2010 crcm_data_path = "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5" months_of_interest = [6, 7, 8] # summer summer_crcm = analysis.get_seasonal_climatology(hdf_path=crcm_data_path, start_year=start_year, end_year=end_year, level=0, var_name=vname, months=months_of_interest) bmp_info = analysis.get_basemap_info_from_hdf(file_path=crcm_data_path) erainterim_15_folder = "/RECH/data/Driving_data/Pilots/ERA-Interim_1.5/Pilots/" flist_for_season = get_files_for_season(erainterim_15_folder, start_year=start_year, end_year=end_year, months=months_of_interest) rpf = MultiRPN(flist_for_season) date_to_hu_erai15 = rpf.get_all_time_records_for_name_and_level(varname=vname, level=1000, level_kind=level_kinds.PRESSURE) summer_era15 = np.mean([field for field in date_to_hu_erai15.values()], axis=0) lons_era, lats_era = rpf.get_longitudes_and_latitudes_of_the_last_read_rec() # plotting # ERA-Interim plt.figure() b = Basemap(lon_0=180) xxg, yyg = b(lons_era, lats_era) im = b.contourf(xxg, yyg, summer_era15, 40, zorder=1) lonsr = bmp_info.lons.copy() lonsr[lonsr < 180] += 360 xxr, yyr = b(lonsr, bmp_info.lats) b.contourf(xxr, yyr, summer_crcm, levels=im.levels, norm=im.norm, cmap=im.cmap, zorder=2) b.drawcoastlines(zorder=3) plt.colorbar(im) # CRCM (plot both crcm and era on the same plot) fig = plt.figure() xx, yy = bmp_info.get_proj_xy() margin = 20 bext = bmp_info.basemap_for_extended_region(marginx=10 * margin, marginy=10 * margin) bmiddle = bmp_info.basemap_for_extended_region(marginx=9 * margin, marginy=9 * margin) xxg, yyg = bext(lons_era, lats_era) outer_domain = (xxg <= bext.urcrnrx) & (xxg >= bext.llcrnrx) & (yyg <= bext.urcrnry) & (yyg >= bext.llcrnry) summer_era15 = np.ma.masked_where(~outer_domain, summer_era15) im = bext.contourf(xx, yy, summer_crcm, levels=im.levels, norm=im.norm, cmap=im.cmap, zorder=2) bmiddle.contourf(xxg, yyg, summer_era15, levels=im.levels, norm=im.norm, cmap=im.cmap, zorder=1) bext.drawcoastlines() plt.colorbar(im) # Add a polygon ax = plt.gca() coords = np.array([ [xx[0, 0], yy[0, 0]], [xx[0, -1], yy[0, -1]], [xx[-1, -1], yy[-1, -1]], [xx[-1, 0], yy[-1, 0]] ]) ax.add_patch(Polygon(coords, facecolor="none", lw=3, zorder=3, edgecolor="k")) img_folder = "cc-paper-comments" if not os.path.isdir(img_folder): os.mkdir(img_folder) fig.savefig(os.path.join(img_folder, "{}_era_1.5_and_crcm.png".format(vname)), bbox_inches="tight", transparent=True) plt.show()
def plot_monthly_clim_in_a_panel(months=None, diag_folder="", vname="STFL", grid_config=None, basins_of_interest_shp=""): """ Plots climatologies using diagnostics outputs, not samples :param months: :param diag_folder: :param vname: """ if months is None: months = list(range(1, 13)) diag_path = Path(diag_folder) month_to_field = OrderedDict() lons, lats, bmp = None, None, None data_mask = None for m in months: r = MultiRPN( str( diag_path.joinpath( "*{:02d}".format(m)).joinpath("pm*_moyenne"))) date_to_field = r.get_all_time_records_for_name_and_level() the_mean = np.mean([f for f in date_to_field.values()], axis=0) the_mean = np.ma.masked_where(the_mean < 0, the_mean) month_to_field[m] = the_mean if bmp is None: lons, lats = r.get_longitudes_and_latitudes_of_the_last_read_rec() # get the basemap object bmp, data_mask = grid_config.get_basemap_using_shape_with_polygons_of_interest( lons, lats, shp_path=basins_of_interest_shp, mask_margin=5) r.close() fig = plt.figure() ncols = 3 nrows = len(months) // ncols + int(len(months) % ncols != 0) gs = GridSpec(nrows=nrows, ncols=ncols + 1) xx, yy = bmp(lons, lats) clevs = [ 0, 20, 50, 100, 200, 500, 1000, 1500, 3000, 4500, 5000, 7000, 9000 ] bn = BoundaryNorm(clevs, len(clevs) - 1) cmap = cm.get_cmap("jet", len(clevs) - 1) for m, field in month_to_field.items(): row = (m - 1) // ncols col = (m - 1) % ncols ax = fig.add_subplot(gs[row, col]) ax.set_title(calendar.month_name[m]) to_plot = np.ma.masked_where(~data_mask, field) im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1]) bmp.colorbar(im, extend="max") bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax) bmp.drawcoastlines(ax=ax) plt.close(fig) # plot annual mean ann_mean = np.mean([field for m, field in month_to_field.items()], axis=0) fig = plt.figure() ax = fig.add_subplot(gs[:, :]) ax.set_title("Annual mean") to_plot = np.ma.masked_where(~data_mask, ann_mean) im = bmp.pcolormesh(xx, yy, to_plot, norm=bn, cmap=cmap, vmin=clevs[0], vmax=clevs[-1]) bmp.colorbar(im, extend="max") bmp.readshapefile(basins_of_interest_shp[:-4], "basins", linewidth=2, color="m", ax=ax) bmp.drawcoastlines(ax=ax) plt.show() plt.close(fig)
def main(sim_path="/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected/", fpath_pattern="var_per_file", fname_prefix=None, coords_filename="pm1979010100_00000000p"): start_year = 1980 end_year = 1982 vname = "SD" level = 6 level_kind = level_kinds.ARBITRARY lkfr = get_lake_fraction(sim_path, fname=coords_filename) data_series = [] year_to_paths_cache = defaultdict(list) data = None for y in range(start_year, end_year + 1): if fpath_pattern == "var_per_file": r = MultiRPN("{}/*{}*/*{}*".format(sim_path, y, vname)) data = r.get_all_time_records_for_name_and_level(vname, level=level, level_kind=level_kind) elif fpath_pattern == "default": sim_dir = Path(sim_path) # Create the map in order to reuse if len(year_to_paths_cache) == 0: for month_dir in sim_dir.iterdir(): year_to_paths_cache[y].append(month_dir) for month_dir in year_to_paths_cache[y]: for fpath in month_dir.iterdir(): # print(fpath) # Check if the prefix is OK if not fpath.name.startswith(fname_prefix): continue # Check if this is not a timestep 0 if fpath.name[:-1].endswith(8 * "0"): continue with RPN(str(fpath)) as r: data_tmp = r.get_all_time_records_for_name_and_level(vname, level=level, level_kind=level_kind) if data is None: data = data_tmp else: data.update(data_tmp) else: raise Exception("Unknown file path pattern: {}".format(fpath_pattern)) ts_for_year = {} for d, field in data.items(): ts_for_year[d] = field[lkfr > 0.5].mean() data_series.append(pd.Series(ts_for_year)) total_series = pd.concat(data_series) assert isinstance(total_series, pd.Series) ax = total_series.plot(title="{}-{}".format(start_year, end_year)) plt.show()
def plot_seasonal_vertical_velocity(samples_folder, seasons=commons.default_seasons): level = 850 # millibars vname = "WW" long_name = "Vertical compoent of wind velocity ({}mb)".format(level) file_prefix = "dp" plot_units = "Pa/s" mult_coeff = 1 add_offset = 0 out_dx = 0.5 file_format = "png" season_to_mean = OrderedDict() r = None for sname, months in seasons.items(): # find files for the season paths = [] for m in months: paths.extend(glob.glob(str(samples_folder.joinpath("*{:02d}/{}*".format(m, file_prefix))))) r = MultiRPN(paths) data = r.get_4d_field(varname=vname) season_to_mean[sname] = np.array([v[level] for k, v in data.items()]).mean(axis=0) print("Processed: {}".format(sname)) lons2d, lats2d = r.get_longitudes_and_latitudes_of_the_last_read_rec() img_folder = samples_folder.joinpath("images/seasonal") if not img_folder.is_dir(): img_folder.mkdir(parents=True) img_file = img_folder.joinpath("{}_{}hPa.{}".format(vname, level, file_format)) plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18) fig = plt.figure() ncols = 2 gs = GridSpec(len(season_to_mean) // ncols + int(not (len(season_to_mean) % ncols == 0)), ncols, wspace=0, hspace=0) xx, yy = None, None bmp = Basemap(projection="robin", lon_0=0) cmap = cm_basemap.GMT_polar clevs = np.arange(-0.1, 0.11, 0.01) for i, (sname, field) in enumerate(season_to_mean.items()): row = i // ncols col = i % ncols ax = fig.add_subplot(gs[row, col]) lons, lats, data_out = commons.interpolate_to_uniform_global_grid(field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx) if xx is None: xx, yy = bmp(lons, lats) cs = bmp.contourf(xx, yy, data_out * mult_coeff, 20 if clevs is None else clevs, cmap=cmap, extend="both") # save color levels for next subplots clevs = cs.levels print(np.max(data_out * mult_coeff)) ax.set_title(sname) cb = plt.colorbar(cs, ax=ax) if not (row == 0 and col == ncols - 1): # cb.ax.set_title(plot_units) cb.ax.set_visible(False) bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH) fig.suptitle("{}, {}".format(long_name, plot_units)) with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight", format=file_format) plt.close(fig)
def get_daily_clim_profiles(samples_dir, start_year=-np.Inf, end_year=np.Inf, filename_prefix="dp", varname="", mask=None): """ dates_2d, levels_2d, values - needed for plotting :param start_year: :param end_year: :param filename_prefix: :param varname: """ dates_sorted = None levels_sorted = None yearly_fields = [] day = timedelta(days=1) stamp_dates = [datetime(2001, 1, 1) + i * day for i in range(365)] for y in range(start_year, end_year + 1): files_for_year = [] mfolders = [ os.path.join(samples_dir, f) for f in os.listdir(samples_dir) if f[:-2].endswith(str(y)) ] for mfolder in mfolders: files_for_year += [ os.path.join(mfolder, fn) for fn in os.listdir(mfolder) if fn.startswith(filename_prefix) ] mrpn = MultiRPN(files_for_year) data = mrpn.get_4d_field(varname=varname) # calculate the area-average for d, lev_to_field in data.items(): for lev in lev_to_field: lev_to_field[lev] = lev_to_field[lev][mask].mean() # dates_sorted = list( sorted([ d for d in data if not (d.month == 2 and d.day == 29) and (d.year == y) ])) levels_sorted = list( sorted([lev for lev in data[dates_sorted[0]] if lev >= 500])) # year_field = np.zeros((len(dates_sorted), len(levels_sorted))) for j, lev in enumerate(levels_sorted): for i, the_date in enumerate(dates_sorted): year_field[i, j] = data[the_date][lev] panel = pd.DataFrame(data=year_field, index=dates_sorted, columns=range(len(levels_sorted))) daily = panel.groupby( by=lambda d: datetime(d.year, d.month, d.day)).mean() yearly_fields.append(daily.values) values = np.mean(yearly_fields, axis=0) assert not hasattr(values, "mask") dates_num = date2num(stamp_dates) levs_2d, dates_num_2d = np.meshgrid(levels_sorted, dates_num) assert levs_2d.shape == values.shape, "levels and values shapes ({} and {}, respectively) are not consistent.".format( levs_2d.shape, values.shape) return dates_num_2d, levs_2d, values
def main( sim_path="/RECH2/huziy/coupling/GL_440x260_0.1deg_GL_with_Hostetler/Samples_selected/", fpath_pattern="var_per_file", fname_prefix=None, coords_filename="pm1979010100_00000000p"): start_year = 1980 end_year = 1982 vname = "SD" level = 6 level_kind = level_kinds.ARBITRARY lkfr = get_lake_fraction(sim_path, fname=coords_filename) data_series = [] year_to_paths_cache = defaultdict(list) data = None for y in range(start_year, end_year + 1): if fpath_pattern == "var_per_file": r = MultiRPN("{}/*{}*/*{}*".format(sim_path, y, vname)) data = r.get_all_time_records_for_name_and_level( vname, level=level, level_kind=level_kind) elif fpath_pattern == "default": sim_dir = Path(sim_path) # Create the map in order to reuse if len(year_to_paths_cache) == 0: for month_dir in sim_dir.iterdir(): year_to_paths_cache[y].append(month_dir) for month_dir in year_to_paths_cache[y]: for fpath in month_dir.iterdir(): # print(fpath) # Check if the prefix is OK if not fpath.name.startswith(fname_prefix): continue # Check if this is not a timestep 0 if fpath.name[:-1].endswith(8 * "0"): continue with RPN(str(fpath)) as r: data_tmp = r.get_all_time_records_for_name_and_level( vname, level=level, level_kind=level_kind) if data is None: data = data_tmp else: data.update(data_tmp) else: raise Exception( "Unknown file path pattern: {}".format(fpath_pattern)) ts_for_year = {} for d, field in data.items(): ts_for_year[d] = field[lkfr > 0.5].mean() data_series.append(pd.Series(ts_for_year)) total_series = pd.concat(data_series) assert isinstance(total_series, pd.Series) ax = total_series.plot(title="{}-{}".format(start_year, end_year)) plt.show()
def main(): start_year = 1979 end_year = 1981 HL_LABEL = "CRCM5_HL" NEMO_LABEL = "CRCM5_NEMO" file_prefix = "dm" level = 1 level_type = level_kinds.HYBRID wind_comp_names = ["UU", "VV"] sim_label_to_path = OrderedDict([ (HL_LABEL, "/RESCUE/skynet3_rech1/huziy/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl_oneway/Samples" ), (NEMO_LABEL, "/HOME/huziy/skynet3_rech1/CNRCWP/C5/2016/2-year-runs/coupled-GL+stfl/Samples" ) ]) # get a coord file ... coord_file = "" found_coord_file = False for mdir in os.listdir(sim_label_to_path[HL_LABEL]): mdir_path = os.path.join(sim_label_to_path[HL_LABEL], mdir) if not os.path.isdir(mdir_path): continue for fn in os.listdir(mdir_path): print(fn) if fn[:2] not in ["pm", "dm", "pp", "dp"]: continue coord_file = os.path.join(mdir_path, fn) found_coord_file = True if found_coord_file: break bmp, lons, lats = nemo_hl_util.get_basemap_obj_and_coords_from_rpn_file( path=coord_file) xx, yy = bmp(lons, lats) lons[lons > 180] -= 360 # loop through all files rotate vaectors and save to netcdf for sim_label, samples_dir in sim_label_to_path.items(): samples = Path(samples_dir) po = samples.parent monthdate_to_path_list = nemo_hl_util.get_monthyeardate_to_paths_map( file_prefix=file_prefix, start_year=start_year, end_year=end_year, samples_dir_path=samples) # Netcdf output file to put rotated winds po /= "rotated_wind_{}.nc".format(sim_label) with Dataset(str(po), "w") as ds: ds.createDimension("time", None) ds.createDimension("lon", lons.shape[0]) ds.createDimension("lat", lons.shape[1]) # create the schema of the output file vname_to_ncvar = {} for vname in wind_comp_names: vname_to_ncvar[vname] = ds.createVariable(vname, "f4", dimensions=("time", "lon", "lat")) vname_to_ncvar[vname].units = "knots" lons_var = ds.createVariable("lon", "f4", dimensions=("lon", "lat")) lats_var = ds.createVariable("lat", "f4", dimensions=("lon", "lat")) time_var = ds.createVariable("time", "i8", dimensions=("time", )) time_var.units = "hours since {:%Y-%m-%d %H:%M:%S}".format( datetime(start_year, 1, 1)) lons_var[:] = lons lats_var[:] = lats # use sorted dates record_count = 0 for month_date in sorted(monthdate_to_path_list): # select only dm files mr = MultiRPN(path=monthdate_to_path_list[month_date]) vname_to_fields = {} for vname in wind_comp_names: vname_to_fields[ vname] = mr.get_all_time_records_for_name_and_level( varname=vname, level=level, level_kind=level_type) for ti, t in enumerate( sorted(vname_to_fields[wind_comp_names[0]])): time_var[record_count] = date2num(t, time_var.units) uu = vname_to_fields[wind_comp_names[0]][t] vv = vname_to_fields[wind_comp_names[1]][t] uu_rot, vv_rot = rotate_vecs_from_geo_to_rotpole(uu, vv, lons, lats, bmp=bmp) # in knots not in m/s vname_to_ncvar[wind_comp_names[0]][ record_count, :, :] = uu_rot vname_to_ncvar[wind_comp_names[1]][ record_count, :, :] = vv_rot record_count += 1
def main(): #path = "/RECH/data/Simulations/CRCM5/North_America/NorthAmerica_0.44deg_ERA40-Int_B1/Diagnostics/NorthAmerica_0.44deg_ERA40-Int_B1_2007{:02d}" path = "/RESCUE/skynet3_rech1/huziy/from_guillimin/new_outputs/current_climate_30_yr_sims/quebec_0.1_crcm5-hcd-rl-intfl_ITFS/Samples/quebec_crcm5-hcd-rl-intfl_1988{:02d}" months = [6, 7, 8] pm_list = [] dm_list = [] for m in months: print(path.format(m)) month_folder = path.format(m) for fn in os.listdir(month_folder): # if not fn.endswith("moyenne"): # continue if fn.startswith("pm"): pm_list.append(os.path.join(month_folder, fn)) elif fn.startswith("dm"): dm_list.append(os.path.join(month_folder, fn)) pm = MultiRPN(pm_list) dm = MultiRPN(dm_list) tsurf_mean = np.mean([field for field in pm.get_all_time_records_for_name_and_level(varname="J8").values()], axis=0) tair_mean = np.mean([field for field in dm.get_all_time_records_for_name_and_level(varname="TT", level=1, level_kind=level_kinds.HYBRID).values()], axis=0) lons, lats = pm.get_longitudes_and_latitudes_of_the_last_read_rec() projparams = pm.linked_robj_list[0].get_proj_parameters_for_the_last_read_rec() rll = RotatedLatLon(**projparams) bmp = rll.get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats) xx, yy = bmp(lons, lats) plt.figure() cs = bmp.contourf(xx, yy, tsurf_mean - 273.15, 40) bmp.drawcoastlines() plt.title("Tsurf") plt.colorbar() plt.figure() bmp.contourf(xx, yy, tair_mean, levels=cs.levels, norm=cs.norm, cmap=cs.cmap) bmp.drawcoastlines() plt.title("Tair") plt.colorbar() plt.figure() bmp.contourf(xx, yy, tsurf_mean - 273.15 - tair_mean, levels=np.arange(-2, 2.2, 0.2), cmap=cs.cmap) bmp.drawcoastlines() plt.title("Tsurf - Tair") plt.colorbar() pm.close() dm.close() plt.show()
def main(): erainterim_075_folder = "/HOME/data/Validation/ERA-Interim_0.75/Offline_driving_data/3h_Forecast" vname = "PR" start_year = 1980 end_year = 2010 season_key = "summer" season_labels = {season_key: "Summer"} season_to_months = OrderedDict([(season_key, [6, 7, 8])]) # Validate temperature and precip model_vars = ["TT", "PR"] obs_vars = ["tmp", "pre"] obs_paths = [ "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc", "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc", ] model_var_to_obs_var = dict(zip(model_vars, obs_vars)) model_var_to_obs_path = dict(zip(model_vars, obs_paths)) obs_path = model_var_to_obs_path[vname] cru = CRUDataManager(var_name=model_var_to_obs_var[vname], path=obs_path) # Calculate seasonal means for CRU seasonal_clim_fields_cru = cru.get_seasonal_means( season_name_to_months=season_to_months, start_year=start_year, end_year=end_year ) # Calculate seasonal mean for erai flist = get_files_for_season( erainterim_075_folder, start_year=start_year, end_year=end_year, months=season_to_months[season_key] ) rpf = MultiRPN(flist) date_to_field_erai075 = rpf.get_all_time_records_for_name_and_level(varname=vname, level=-1) # Convert to mm/day era075 = np.mean([field for field in date_to_field_erai075.values()], axis=0) * 24 * 3600 * 1000 lons_era, lats_era = rpf.get_longitudes_and_latitudes_of_the_last_read_rec() seasonal_clim_fields_cru_interp = OrderedDict() # Calculate biases for season, cru_field in seasonal_clim_fields_cru.items(): seasonal_clim_fields_cru_interp[season] = cru.interpolate_data_to( cru_field, lons2d=lons_era, lats2d=lats_era, nneighbours=1 ) # Do the plotting ------------------------------------------------------------------------------ plot_utils.apply_plot_params() fig = plt.figure() b = Basemap() gs = gridspec.GridSpec(nrows=3, ncols=1) ax = fig.add_subplot(gs[0, 0]) xx, yy = b(cru.lons2d, cru.lats2d) cs = b.contourf(xx, yy, seasonal_clim_fields_cru[season_key], 20) b.drawcoastlines(ax=ax) ax.set_title("CRU") plt.colorbar(cs, ax=ax) ax = fig.add_subplot(gs[1, 0]) lons_era[lons_era > 180] -= 360 lons_era, era075 = b.shiftdata(lons_era, datain=era075, lon_0=0) xx, yy = b(lons_era, lats_era) # mask oceans in the era plot as well era075 = maskoceans(lons_era, lats_era, era075) cs = b.contourf(xx, yy, era075, levels=cs.levels, norm=cs.norm, cmap=cs.cmap, ax=ax) b.drawcoastlines(ax=ax) ax.set_title("ERA-Interim 0.75") plt.colorbar(cs, ax=ax) # differences ax = fig.add_subplot(gs[2, 0]) diff = era075 - seasonal_clim_fields_cru_interp[season_key] delta = np.percentile(np.abs(diff)[~diff.mask], 90) clevs = np.linspace(-delta, delta, 20) cs = b.contourf(xx, yy, diff, levels=clevs, cmap="RdBu_r", extend="both") b.drawcoastlines(ax=ax) ax.set_title("ERA-Interim 0.75 - CRU") plt.colorbar(cs, ax=ax) plt.show() fig.savefig(os.path.join(img_folder, "erai0.75_vs_cru_precip.png"), bbox_inches="tight")