def get_seasonal_clim_obs_data(rconfig=None, vname="TT", bmp_info=None, season_to_months=None, obs_path=None, nx_agg=None, ny_agg=None): # Number of points for aggregation """ return aggregated BasemapInfo object corresponding to the CRU resolution :param rconfig: :param vname: :param bmp_info: BasemapInfo object for the model field (will be upscaled to the CRU resolution) :param season_to_months: """ if bmp_info is None: bmp_info = analysis.get_basemap_info_from_hdf( file_path=rconfig.data_path) if nx_agg is not None: bmp_info_agg = bmp_info.get_aggregated(nagg_x=nx_agg, nagg_y=ny_agg) else: bmp_info_agg = bmp_info # Validate temperature and precip model_vars = ["TT", "PR"] obs_vars = ["tmp", "pre"] obs_paths = [ "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc", "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc" ] model_var_to_obs_var = dict(zip(model_vars, obs_vars)) model_var_to_obs_path = dict(zip(model_vars, obs_paths)) if obs_path is None: obs_path = model_var_to_obs_path[vname] cru = CRUDataManager(var_name=model_var_to_obs_var[vname], path=obs_path) seasonal_clim_fields_obs = cru.get_seasonal_means( season_name_to_months=season_to_months, start_year=rconfig.start_year, end_year=rconfig.end_year) seasonal_clim_fields_obs_interp = OrderedDict() for season, cru_field in seasonal_clim_fields_obs.items(): seasonal_clim_fields_obs_interp[season] = cru.interpolate_data_to( cru_field, lons2d=bmp_info_agg.lons, lats2d=bmp_info_agg.lats, nneighbours=1) # assert hasattr(seasonal_clim_fields_obs_interp[season], "mask") return bmp_info_agg, seasonal_clim_fields_obs_interp
def main(dfs_var_name="t2", cru_var_name="tmp", dfs_folder="/home/huziy/skynet3_rech1/NEMO_OFFICIAL/DFS5.2_interpolated", cru_file = "data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.tmp.dat.nc"): if not os.path.isdir(NEMO_IMAGES_DIR): os.mkdir(NEMO_IMAGES_DIR) #year range is inclusive [start_year, end_year] start_year = 1981 end_year = 2009 season_name_to_months = OrderedDict([ ("Winter", (1, 2, 12)), ("Spring", list(range(3, 6))), ("Summer", list(range(6, 9))), ("Fall", list(range(9, 12)))]) cru_t_manager = CRUDataManager(var_name=cru_var_name, path=cru_file) cru_lons, cru_lats = cru_t_manager.lons2d, cru_t_manager.lats2d #get seasonal means (CRU) season_to_mean_cru = cru_t_manager.get_seasonal_means(season_name_to_months=season_name_to_months, start_year=start_year, end_year=end_year) #get seasonal means Drakkar dfs_manager = DFSDataManager(folder_path=dfs_folder, var_name=dfs_var_name) season_to_mean_dfs = dfs_manager.get_seasonal_means(season_name_to_months=season_name_to_months, start_year=start_year, end_year=end_year) dfs_lons, dfs_lats = dfs_manager.get_lons_and_lats_2d() xt, yt, zt = lat_lon.lon_lat_to_cartesian(dfs_lons.flatten(), dfs_lats.flatten()) xs, ys, zs = lat_lon.lon_lat_to_cartesian(cru_lons.flatten(), cru_lats.flatten()) ktree = cKDTree(data=list(zip(xs, ys, zs))) dists, inds = ktree.query(list(zip(xt, yt, zt))) season_to_err = OrderedDict() for k in season_to_mean_dfs: interpolated_cru = season_to_mean_cru[k].flatten()[inds].reshape(dfs_lons.shape) if dfs_var_name.lower() == "t2": #interpolated_cru += 273.15 season_to_mean_dfs[k] -= 273.15 elif dfs_var_name.lower() == "precip": # precipitation in mm/day season_to_mean_dfs[k] *= 24 * 60 * 60 season_to_err[k] = season_to_mean_dfs[k] #- interpolated_cru season_indicator = "-".join(sorted(season_to_err.keys())) fig_path = os.path.join(NEMO_IMAGES_DIR, "{3}_errors_{0}-{1}_{2}_dfs.jpeg".format(start_year, end_year, season_indicator, dfs_var_name)) basemap = nemo_commons.get_default_basemap_for_glk(dfs_lons, dfs_lats, resolution="l") x, y = basemap(dfs_lons, dfs_lats) coords_and_basemap = { "basemap": basemap, "x": x, "y": y } plot_errors_in_one_figure(season_to_err, fig_path=fig_path, **coords_and_basemap)
def __init__(self, path="data/swe_ross_brown/swe.nc", var_name=""): self.lons2d, self.lats2d = None, None self.times = None self.var_data = None BaseDataManager.__init__(self) CRUDataManager.__init__(self, path=path, var_name=var_name) print(list(self.nc_dataset.variables.keys()))
def get_cru_obs_mean_fields(target_lon_2d, target_lat_2d, nneighbours=1, months=None): tmpCruDataManager = CRUDataManager(var_name="tmp", lazy=True) preCruDataManager = CRUDataManager( path="data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.pre.dat.nc", var_name="pre", lazy=True) #get mean annual temperature from CRU dataset tmpCru = tmpCruDataManager.get_mean(start_year=start_year, end_year=end_year, months=months) #same thing for the precip preCru = preCruDataManager.get_mean(start_year=start_year, end_year=end_year, months=months) ##pre is in mm/month #convert to mm/day preCru *= 12.0 / 365.25 tmpCruI = tmpCruDataManager.interpolate_data_to(tmpCru, target_lon_2d, target_lat_2d, nneighbours=nneighbours) preCruI = preCruDataManager.interpolate_data_to(preCru, target_lon_2d, target_lat_2d, nneighbours=nneighbours) return tmpCruI, preCruI
def get_seasonal_clim_obs_data(rconfig=None, vname="TT", bmp_info=None, season_to_months=None, obs_path=None): # Number of points for aggregation """ return aggregated BasemapInfo object corresponding to the CRU resolution :param rconfig: :param vname: :param bmp_info: BasemapInfo object for the model field (will be upscaled to the CRU resolution) :param season_to_months: """ nx_agg = 5 ny_agg = 5 if bmp_info is None: bmp_info = analysis.get_basemap_info_from_hdf(file_path=rconfig.data_path) bmp_info_agg = bmp_info.get_aggregated(nagg_x=nx_agg, nagg_y=ny_agg) # Validate temperature and precip model_vars = ["TT", "PR"] obs_vars = ["tmp", "pre"] obs_paths = [ "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc", "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc" ] model_var_to_obs_var = dict(zip(model_vars, obs_vars)) model_var_to_obs_path = dict(zip(model_vars, obs_paths)) if obs_path is None: obs_path = model_var_to_obs_path[vname] cru = CRUDataManager(var_name=model_var_to_obs_var[vname], path=obs_path) seasonal_clim_fields_obs = cru.get_seasonal_means(season_name_to_months=season_to_months, start_year=rconfig.start_year, end_year=rconfig.end_year) seasonal_clim_fields_obs_interp = OrderedDict() for season, cru_field in seasonal_clim_fields_obs.items(): seasonal_clim_fields_obs_interp[season] = cru.interpolate_data_to(cru_field, lons2d=bmp_info_agg.lons, lats2d=bmp_info_agg.lats, nneighbours=1) # assert hasattr(seasonal_clim_fields_obs_interp[season], "mask") return bmp_info_agg, seasonal_clim_fields_obs_interp
def compare_vars(vname_model="TT", vname_obs="tmp", r_config=None, season_to_months=None, obs_path=None, nx_agg=5, ny_agg=5, bmp_info_agg=None, axes_list=None): seasonal_clim_fields_model = analysis.get_seasonal_climatology_for_runconfig(run_config=r_config, varname=vname_model, level=0, season_to_months=season_to_months) season_to_clim_fields_model_agg = OrderedDict() for season, field in seasonal_clim_fields_model.items(): print(field.shape) season_to_clim_fields_model_agg[season] = aggregate_array(field, nagg_x=nx_agg, nagg_y=ny_agg) if vname_model == "PR": season_to_clim_fields_model_agg[season] *= 1.0e3 * 24 * 3600 if obs_path is None: cru = CRUDataManager(var_name=vname_obs) else: cru = CRUDataManager(var_name=vname_obs, path=obs_path) seasonal_clim_fields_obs = cru.get_seasonal_means(season_name_to_months=season_to_months, start_year=r_config.start_year, end_year=r_config.end_year) seasonal_clim_fields_obs_interp = OrderedDict() for season, cru_field in seasonal_clim_fields_obs.items(): seasonal_clim_fields_obs_interp[season] = cru.interpolate_data_to(cru_field, lons2d=bmp_info_agg.lons, lats2d=bmp_info_agg.lats, nneighbours=1) assert hasattr(seasonal_clim_fields_obs_interp[season], "mask") season_to_err = OrderedDict() for season in seasonal_clim_fields_obs_interp: if vname_model == "PR": # * 10, because in the UDEL dataset, precipitation is in cm season_to_err[season] = season_to_clim_fields_model_agg[season] - \ seasonal_clim_fields_obs_interp[season] * MM_PER_CM else: season_to_err[season] = season_to_clim_fields_model_agg[season] - seasonal_clim_fields_obs_interp[season] plot_performance_err_with_cru.plot_seasonal_mean_biases(season_to_error_field=season_to_err, varname=vname_model, basemap_info=bmp_info_agg, axes_list=axes_list)
def main(): # seasons = OrderedDict([ # ("DJF", MonthPeriod(12, 3)), # ("MAM", MonthPeriod(3, 3)) # ]) season_to_month_period = OrderedDict([ ("DJF", MonthPeriod(12, 3)) ]) start_year = 1980 end_year = 1982 vname_to_path = { "pre": "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc", "tmp": "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc" } for vname, path in vname_to_path.items(): manager = CRUDataManager(path=path, var_name=vname) res = manager.get_seasonal_means_with_ttest_stats( season_to_monthperiod=season_to_month_period, start_year=start_year, end_year=end_year) plt.figure() mean_field, std_field, nobs = res["DJF"] im = plt.pcolormesh(mean_field.T) plt.colorbar(im) plt.show()
def compare_vars(vname_model="TT", vname_obs="tmp", r_config=None, season_to_months=None, obs_path=None, nx_agg=5, ny_agg=5, bmp_info_agg=None, axes_list=None): seasonal_clim_fields_model = analysis.get_seasonal_climatology_for_runconfig( run_config=r_config, varname=vname_model, level=0, season_to_months=season_to_months) season_to_clim_fields_model_agg = OrderedDict() for season, field in seasonal_clim_fields_model.items(): print(field.shape) season_to_clim_fields_model_agg[season] = aggregate_array( field, nagg_x=nx_agg, nagg_y=ny_agg) if vname_model == "PR": season_to_clim_fields_model_agg[season] *= 1.0e3 * 24 * 3600 if obs_path is None: cru = CRUDataManager(var_name=vname_obs) else: cru = CRUDataManager(var_name=vname_obs, path=obs_path) seasonal_clim_fields_obs = cru.get_seasonal_means( season_name_to_months=season_to_months, start_year=r_config.start_year, end_year=r_config.end_year) seasonal_clim_fields_obs_interp = OrderedDict() for season, cru_field in seasonal_clim_fields_obs.items(): seasonal_clim_fields_obs_interp[season] = cru.interpolate_data_to( cru_field, lons2d=bmp_info_agg.lons, lats2d=bmp_info_agg.lats, nneighbours=1) assert hasattr(seasonal_clim_fields_obs_interp[season], "mask") season_to_err = OrderedDict() for season in seasonal_clim_fields_obs_interp: if vname_model == "PR": # * 10, because in the UDEL dataset, precipitation is in cm season_to_err[season] = season_to_clim_fields_model_agg[season] - \ seasonal_clim_fields_obs_interp[season] * MM_PER_CM else: season_to_err[season] = season_to_clim_fields_model_agg[ season] - seasonal_clim_fields_obs_interp[season] plot_performance_err_with_cru.plot_seasonal_mean_biases( season_to_error_field=season_to_err, varname=vname_model, basemap_info=bmp_info_agg, axes_list=axes_list)
def get_cru_obs_mean_fields(target_lon_2d, target_lat_2d, nneighbours = 1, months = None): tmpCruDataManager = CRUDataManager(var_name="tmp", lazy = True) preCruDataManager = CRUDataManager(path = "data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.pre.dat.nc", var_name="pre", lazy = True) #get mean annual temperature from CRU dataset tmpCru = tmpCruDataManager.get_mean(start_year=start_year, end_year = end_year, months = months) #same thing for the precip preCru = preCruDataManager.get_mean(start_year=start_year, end_year = end_year, months = months) ##pre is in mm/month #convert to mm/day preCru *= 12.0 / 365.25 tmpCruI = tmpCruDataManager.interpolate_data_to(tmpCru, target_lon_2d, target_lat_2d, nneighbours=nneighbours) preCruI = preCruDataManager.interpolate_data_to(preCru, target_lon_2d, target_lat_2d, nneighbours=nneighbours) return tmpCruI, preCruI
def validate_using_monthly_diagnostics(): start_year = 1980 end_year = 1996 sim_data_folder = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1" sim_names = ["ERA40", "MPI", "CanESM"] simname_to_path = { "ERA40": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1_dm", "MPI": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_MPI_B1_dm", "CanESM": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_CanESM_B1_dm" } coord_file = os.path.join( sim_data_folder, "pmNorthAmerica_0.44deg_ERA40-Int_B1_200812_moyenne") basemap, lons2d, lats2d = draw_regions.get_basemap_and_coords( resolution="c", file_path=coord_file, llcrnrlat=45.0, llcrnrlon=-145, urcrnrlon=-20, urcrnrlat=74, anchor="W") assert isinstance(basemap, Basemap) lons2d[lons2d > 180] -= 360 obs_manager = CRUDataManager() obs = obs_manager.get_mean(start_year, end_year, months=[6, 7, 8]) obs = obs_manager.interpolate_data_to(obs, lons2d, lats2d, nneighbours=1) x, y = basemap(lons2d, lats2d) #x = (x[1:,1:] + x[:-1, :-1]) /2.0 permafrost_mask = draw_regions.get_permafrost_mask(lons2d, lats2d) mask_cond = (permafrost_mask <= 0) | (permafrost_mask >= 2) #plot_utils.apply_plot_params(width_pt=None, width_cm=35,height_cm=55, font_size=35) fig = plt.figure() assert isinstance(fig, Figure) cmap = my_colormaps.get_red_blue_colormap(ncolors=10, reversed=True) gs = gridspec.GridSpec(3, 2, width_ratios=[1, 0.1], hspace=0, wspace=0, left=0.05, bottom=0.01, top=0.95) all_axes = [] all_img = [] i = 0 for name in sim_names: path = simname_to_path[name] dm = CRCMDataManager(data_folder=path) mod = dm.get_mean_over_months_of_2d_var(start_year, end_year, months=[6, 7, 8], var_name="TT", level=1, level_kind=level_kinds.HYBRID) delta = mod - obs ax = fig.add_subplot(gs[i, 0]) assert isinstance(ax, Axes) delta = np.ma.masked_where(mask_cond, delta) img = basemap.pcolormesh(x, y, delta, cmap=cmap, vmin=-5.0, vmax=5.0) if not i: ax.set_title("T2m, Mod - Obs, ({0} - {1}), JJA \n".format( start_year, end_year)) i += 1 #ax.set_ylabel(name) all_axes.append(ax) all_img.append(img) i = 0 axs_to_hide = [] #zones and coastlines for the_ax, the_img in zip(all_axes, all_img): # divider = make_axes_locatable(the_ax) # cax = divider.append_axes("right", "5%", pad="3%") #cax.set_title("%\n") assert isinstance(the_ax, Axes) basemap.drawcoastlines(ax=the_ax, linewidth=0.5) basemap.readshapefile("data/pf_4/permafrost8_wgs84/permaice", name="zone", ax=the_ax, linewidth=1.5, drawbounds=False) for nshape, seg in enumerate(basemap.zone): if basemap.zone_info[nshape]["EXTENT"] != "C": continue poly = mpl.patches.Polygon(seg, edgecolor="k", facecolor="none", zorder=10, lw=1.5) the_ax.add_patch(poly) i += 1 cax = fig.add_subplot(gs[:, 1]) cax.set_anchor("W") cax.set_aspect(30) formatter = FuncFormatter(lambda x, pos: "{0: <6}".format(str(x))) cb = fig.colorbar(all_img[0], ax=cax, cax=cax, extend="both", format=formatter) cax.set_title("$^{\\circ} {\\rm C}$") #fig.tight_layout(h_pad=0) # for the_ax in axs_to_hide: # the_ax.set_visible(False) fig.savefig("tmp_validation_era_mpi_canesm.png")
def compare_for_season(start_year=1958, end_year=1974, the_months=None, period_str="djf"): """ Compare CRU, ERA40-driven and GCM-driven s """ #b, lons2d, lats2d = draw_regions.get_basemap_and_coords(llcrnrlat=40.0, llcrnrlon=-145, urcrnrlon=-10) b, lons2d, lats2d = draw_regions.get_basemap_and_coords() lons2d[lons2d > 180] -= 360 x, y = b(lons2d, lats2d) cru = CRUDataManager() cru_data = cru.get_mean(start_year, end_year, months=the_months) cru_data_interp = cru.interpolate_data_to(cru_data, lons2d, lats2d) temp_levels = np.arange(-40, 40, 5) diff_levels = np.arange(-10, 12, 2) gs = gridspec.GridSpec(3, 2) #plot_utils.apply_plot_params(width_pt=None, height_cm=20, width_cm=20, font_size=12) fig = plt.figure() coast_line_width = 0.25 axes_list = [] #plot CRU data ax = fig.add_subplot(gs[0, :]) axes_list.append(ax) cru_data_interp = maskoceans(lons2d, lats2d, cru_data_interp) img = b.contourf(x, y, cru_data_interp, ax=ax, levels=temp_levels) ax.set_title("CRU") plot_utils.draw_colorbar(fig, img, ax=ax) #era40 driven file_path = None era40_folder = "data/CORDEX/na/era40_1" file_prefix = "dm" for file_name in os.listdir(era40_folder): if period_str.upper() in file_name and file_name.startswith( file_prefix): file_path = os.path.join(era40_folder, file_name) break #get the temperature rpn_obj = RPN(file_path) t2m_era40 = rpn_obj.get_first_record_for_name_and_level( varname="TT", level=1, level_kind=level_kinds.HYBRID) t2m_era40 = maskoceans(lons2d, lats2d, t2m_era40) ax = fig.add_subplot(gs[1, 0]) axes_list.append(ax) img = b.contourf(x, y, t2m_era40, ax=ax, levels=temp_levels) ax.set_title("ERA40 driven 1 (1958-1961)") plot_utils.draw_colorbar(fig, img, ax=ax) rpn_obj.close() #era40 - cru ax = fig.add_subplot(gs[1, 1]) axes_list.append(ax) img = b.contourf(x, y, t2m_era40 - cru_data_interp, ax=ax, levels=diff_levels) ax.set_title("ERA40 driven 1 - CRU") plot_utils.draw_colorbar(fig, img, ax=ax) plot_e2_data = False if plot_e2_data: ##get and plot E2 data file_path = None e2_folder = "data/CORDEX/na/e2" prefix = "dm" #get file path for file_name in os.listdir(e2_folder): if file_name.endswith(period_str) and file_name.startswith(prefix): file_path = os.path.join(e2_folder, file_name) break pass #get the temperature rpn_obj = RPN(file_path) t2m = rpn_obj.get_first_record_for_name_and_level( varname="TT", level=1, level_kind=level_kinds.HYBRID) t2m = maskoceans(lons2d, lats2d, t2m) ax = fig.add_subplot(gs[2, 0]) axes_list.append(ax) img = b.contourf(x, y, t2m, ax=ax, levels=temp_levels) ax.set_title("E2, GCM driven") plot_utils.draw_colorbar(fig, img, ax=ax) #e2 - cru ax = fig.add_subplot(gs[2, 1]) axes_list.append(ax) img = b.contourf(x, y, t2m - cru_data_interp, ax=ax, levels=diff_levels) ax.set_title("E2, GCM driven - CRU") plot_utils.draw_colorbar(fig, img, ax=ax) ####Draw common elements pf_kinds = draw_regions.get_permafrost_mask(lons2d, lats2d) for the_ax in axes_list: b.drawcoastlines(ax=the_ax, linewidth=coast_line_width) b.contour(x, y, pf_kinds, ax=the_ax, colors="k") gs.tight_layout(fig, h_pad=5, w_pad=5, pad=2) fig.suptitle(period_str.upper(), y=0.03, x=0.5) fig.savefig("temperature_validation_{0}.png".format(period_str)) fig = plt.figure() ax = plt.gca() img = b.contourf(x, y, t2m_era40 - cru_data_interp, ax=ax, levels=diff_levels) ax.set_title("ERA40 driven 1 - CRU") plot_utils.draw_colorbar(fig, img, ax=ax) b.drawcoastlines(ax=ax, linewidth=coast_line_width) b.contour(x, y, pf_kinds, ax=ax, colors="k") fig.savefig("temperature_diff_{0}.png".format(period_str)) pass
def plot_seasonal_2m_temp_cru(samples_folder, seasons=commons.default_seasons): vname = "tmp" file_prefix = "dm" plot_units = "$^\circ$C" mult_coeff = 1 add_offset = 0 out_dx = 0.5 level = 1 # in hybrid coords img_file_name = "CRU_{}.png".format(vname) long_name = "CRU: 2m temperature, {}".format(plot_units) # get data from cru files data_manager = CRUDataManager(path="/RESCUE/skynet3_rech1/huziy/CRU/cru_ts3.23.2011.2014.tmp.dat.nc", var_name=vname, lazy=True) season_to_mean = data_manager.get_seasonal_means(season_name_to_months=seasons, start_year=2013, end_year=2013) lons2d, lats2d = data_manager.lons2d, data_manager.lats2d img_folder = samples_folder.joinpath("images/seasonal") if not img_folder.is_dir(): img_folder.mkdir(parents=True) img_file = img_folder.joinpath(img_file_name) plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18) fig = plt.figure() ncols = 2 gs = GridSpec(len(season_to_mean) // ncols + int(not (len(season_to_mean) % ncols == 0)), ncols, wspace=0, hspace=0) xx, yy = None, None bmp = Basemap(projection="robin", lon_0=0) cmap = plt.get_cmap('bwr') clevs = np.arange(-30, 32, 2) for i, (sname, field) in enumerate(season_to_mean.items()): row = i // ncols col = i % ncols ax = fig.add_subplot(gs[row, col]) lons, lats, data_out = commons.interpolate_to_uniform_global_grid(field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx) if xx is None: xx, yy = bmp(lons, lats) cs = bmp.contourf(xx, yy, data_out * mult_coeff, clevs, cmap=cmap, extend="both") # save color levels for next subplots clevs = cs.levels print(np.max(data_out * mult_coeff)) ax.set_title(sname) cb = plt.colorbar(cs, ax=ax) if not (row == 0 and col == ncols - 1): # cb.ax.set_title(plot_units) cb.ax.set_visible(False) bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH) fig.suptitle(long_name) with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight") plt.close(fig)
def validate_thawing_index(): start_year = 1980 end_year = 1996 sim_data_folder = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1" sim_names = ["ERA40", "MPI", "CanESM"] fold = "/home/samira/skynet/DailyTempData" simname_to_path = { "ERA40": os.path.join(fold, "TempERA40_b1_1981-2008.nc"), "MPI": os.path.join(fold, "TempMPI1981-2010.nc"), "CanESM": os.path.join(fold, "TempCanESM1981-2010.nc"), } coord_file = os.path.join( sim_data_folder, "pmNorthAmerica_0.44deg_ERA40-Int_B1_200812_moyenne") basemap, lons2d, lats2d = draw_regions.get_basemap_and_coords( resolution="c", file_path=coord_file, llcrnrlat=40.0, llcrnrlon=-145, urcrnrlon=-20, urcrnrlat=74) assert isinstance(basemap, Basemap) lons2d[lons2d > 180] -= 360 om = CRUDataManager() clim = om.get_daily_climatology(start_year, end_year) obs = om.get_thawing_index_from_climatology(clim) obs = om.interpolate_data_to( obs, lons2d, lats2d, nneighbours=1) #interpolatee to the model grid x, y = basemap(lons2d, lats2d) #x = (x[1:,1:] + x[:-1, :-1]) /2.0 permafrost_mask = draw_regions.get_permafrost_mask(lons2d, lats2d) mask_cond = (permafrost_mask <= 0) | (permafrost_mask >= 3) #plot_utils.apply_plot_params(width_pt=None, width_cm=35,height_cm=55, font_size=35) fig = plt.figure() assert isinstance(fig, Figure) cmap = my_colormaps.get_red_blue_colormap(ncolors=10, reversed=True) gs = gridspec.GridSpec(3, 1) all_axes = [] all_img = [] i = 0 for name in sim_names: path = simname_to_path[name] ds = Dataset(path) data_mod = ds.variables["air"][:] mod = _get_thawing_index() delta = mod - obs ax = fig.add_subplot(gs[i, 0]) assert isinstance(ax, Axes) delta = np.ma.masked_where(mask_cond, delta) img = basemap.pcolormesh(x, y, delta, cmap=cmap, vmin=None, vmax=None) if not i: ax.set_title("Thawing index, Mod - Obs, ({0} - {1}) \n".format( start_year, end_year)) i += 1 ax.set_ylabel(name) all_axes.append(ax) all_img.append(img) i = 0 axs_to_hide = [] #zones and coastlines for the_ax, the_img in zip(all_axes, all_img): divider = make_axes_locatable(the_ax) cax = divider.append_axes("right", "5%", pad="3%") cb = fig.colorbar(the_img, cax=cax, extend="both") cax.set_title("$^{\\circ} {\\rm C}$\n") #cax.set_title("%\n") assert isinstance(the_ax, Axes) basemap.drawcoastlines(ax=the_ax, linewidth=1.5) basemap.readshapefile("data/pf_4/permafrost8_wgs84/permaice", name="zone", ax=the_ax, linewidth=3) if i != 1: axs_to_hide.append(cax) i += 1 fig.tight_layout() for the_ax in axs_to_hide: the_ax.set_visible(False) fig.savefig("tmp_validation_era_mpi_canesm.png") pass
def plot_seasonal_2m_temp_cru(samples_folder, seasons=commons.default_seasons): vname = "tmp" file_prefix = "dm" plot_units = "$^\circ$C" mult_coeff = 1 add_offset = 0 out_dx = 0.5 level = 1 # in hybrid coords img_file_name = "CRU_{}.png".format(vname) long_name = "CRU: 2m temperature, {}".format(plot_units) # get data from cru files data_manager = CRUDataManager( path="/RESCUE/skynet3_rech1/huziy/CRU/cru_ts3.23.2011.2014.tmp.dat.nc", var_name=vname, lazy=True) season_to_mean = data_manager.get_seasonal_means( season_name_to_months=seasons, start_year=2013, end_year=2013) lons2d, lats2d = data_manager.lons2d, data_manager.lats2d img_folder = samples_folder.joinpath("images/seasonal") if not img_folder.is_dir(): img_folder.mkdir(parents=True) img_file = img_folder.joinpath(img_file_name) plot_utils.apply_plot_params(width_cm=25, height_cm=15, font_size=18) fig = plt.figure() ncols = 2 gs = GridSpec(len(season_to_mean) // ncols + int(not (len(season_to_mean) % ncols == 0)), ncols, wspace=0, hspace=0) xx, yy = None, None bmp = Basemap(projection="robin", lon_0=0) cmap = plt.get_cmap('bwr') clevs = np.arange(-30, 32, 2) for i, (sname, field) in enumerate(season_to_mean.items()): row = i // ncols col = i % ncols ax = fig.add_subplot(gs[row, col]) lons, lats, data_out = commons.interpolate_to_uniform_global_grid( field, lons_in=lons2d, lats_in=lats2d, out_dx=out_dx) if xx is None: xx, yy = bmp(lons, lats) cs = bmp.contourf(xx, yy, data_out * mult_coeff, clevs, cmap=cmap, extend="both") # save color levels for next subplots clevs = cs.levels print(np.max(data_out * mult_coeff)) ax.set_title(sname) cb = plt.colorbar(cs, ax=ax) if not (row == 0 and col == ncols - 1): # cb.ax.set_title(plot_units) cb.ax.set_visible(False) bmp.drawcoastlines(ax=ax, linewidth=LINEWIDTH) fig.suptitle(long_name) with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight") plt.close(fig)
def main( dfs_var_name="t2", cru_var_name="tmp", dfs_folder="/home/huziy/skynet3_rech1/NEMO_OFFICIAL/DFS5.2_interpolated", cru_file="data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.tmp.dat.nc"): if not os.path.isdir(NEMO_IMAGES_DIR): os.mkdir(NEMO_IMAGES_DIR) #year range is inclusive [start_year, end_year] start_year = 1981 end_year = 2009 season_name_to_months = OrderedDict([("Winter", (1, 2, 12)), ("Spring", list(range(3, 6))), ("Summer", list(range(6, 9))), ("Fall", list(range(9, 12)))]) cru_t_manager = CRUDataManager(var_name=cru_var_name, path=cru_file) cru_lons, cru_lats = cru_t_manager.lons2d, cru_t_manager.lats2d #get seasonal means (CRU) season_to_mean_cru = cru_t_manager.get_seasonal_means( season_name_to_months=season_name_to_months, start_year=start_year, end_year=end_year) #get seasonal means Drakkar dfs_manager = DFSDataManager(folder_path=dfs_folder, var_name=dfs_var_name) season_to_mean_dfs = dfs_manager.get_seasonal_means( season_name_to_months=season_name_to_months, start_year=start_year, end_year=end_year) dfs_lons, dfs_lats = dfs_manager.get_lons_and_lats_2d() xt, yt, zt = lat_lon.lon_lat_to_cartesian(dfs_lons.flatten(), dfs_lats.flatten()) xs, ys, zs = lat_lon.lon_lat_to_cartesian(cru_lons.flatten(), cru_lats.flatten()) ktree = cKDTree(data=list(zip(xs, ys, zs))) dists, inds = ktree.query(list(zip(xt, yt, zt))) season_to_err = OrderedDict() for k in season_to_mean_dfs: interpolated_cru = season_to_mean_cru[k].flatten()[inds].reshape( dfs_lons.shape) if dfs_var_name.lower() == "t2": #interpolated_cru += 273.15 season_to_mean_dfs[k] -= 273.15 elif dfs_var_name.lower() == "precip": # precipitation in mm/day season_to_mean_dfs[k] *= 24 * 60 * 60 season_to_err[k] = season_to_mean_dfs[k] #- interpolated_cru season_indicator = "-".join(sorted(season_to_err.keys())) fig_path = os.path.join( NEMO_IMAGES_DIR, "{3}_errors_{0}-{1}_{2}_dfs.jpeg".format(start_year, end_year, season_indicator, dfs_var_name)) basemap = nemo_commons.get_default_basemap_for_glk(dfs_lons, dfs_lats, resolution="l") x, y = basemap(dfs_lons, dfs_lats) coords_and_basemap = {"basemap": basemap, "x": x, "y": y} plot_errors_in_one_figure(season_to_err, fig_path=fig_path, **coords_and_basemap)
def validate_daily_climatology(): """ """ #years are inclusive start_year = 1979 end_year = 1988 #sim_name_list = ["crcm5-r", "crcm5-hcd-r", "crcm5-hcd-rl"] sim_name_list = ["crcm5-hcd-rl", "crcm5-hcd-rl-intfl"] rpn_folder_paths = [ "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup".format(sim_name_list[0]), "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup2/Samples_all_in_one_folder".format( sim_name_list[1]) ] nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb" #select stations selected_ids = None selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905", "041903", "040830", "093806", "090613", "081002", "093801", "080718"] selected_ids = ["074903", ] start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) selected_ids = None stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date ) stations_hydat = cehq_station.read_hydat_station_data(folder_path="/home/huziy/skynet3_rech1/HYDAT", start_date=start_date, end_date=end_date) stations.extend(stations_hydat) varname = "STFL" sim_name_to_manager = {} sim_name_to_station_to_model_point = {} day_stamps = Station.get_stamp_days(2001) sweManager = SweDataManager(var_name="SWE") cruTempManager = CRUDataManager(lazy=True) cruPreManager = CRUDataManager(var_name="pre", lazy=True, path="data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.pre.dat.nc") #common lake fractions when comparing simulations on the same grid all_model_points = [] cell_manager = None for sim_name, rpn_folder in zip(sim_name_list, rpn_folder_paths): dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="dm", all_files_in_samples_folder=True, need_cell_manager=cell_manager is None) #here using the fact that all the simulations are on the same grid if cell_manager is None: cell_manager = dmManager.cell_manager else: dmManager.cell_manager = cell_manager #determine comon lake fractions, so it is not taken from the trivial case lf = 0, but note #this has only sense when all the simulations were performed on the same grid sim_name_to_manager[sim_name] = dmManager nc_sim_folder = os.path.join(nc_db_folder, sim_name) nc_path = os.path.join(nc_sim_folder, "{0}_all.nc4".format(varname)) #In general there are several model points corresponding to a given station st_to_mp = dmManager.get_model_points_for_stations(stations, sim_name=sim_name, nc_path=nc_path, nc_sim_folder=nc_sim_folder, set_data_to_model_points=True) print("got model points for stations") sim_name_to_station_to_model_point[sim_name] = st_to_mp #save model points to a list of all points for s, mps in st_to_mp.items(): assert isinstance(s, Station) for mp in mps: assert isinstance(mp, ModelPoint) #calculate upstream swe if needed if s.mean_swe_upstream_daily_clim is None: s.mean_swe_upstream_daily_clim = sweManager.get_mean_upstream_timeseries_daily(mp, dmManager, stamp_dates=day_stamps) #These are taken from CRU dataset, only monthly data are available s.mean_temp_upstream_monthly_clim = cruTempManager.get_mean_upstream_timeseries_monthly(mp, dmManager) s.mean_prec_upstream_monthly_clim = cruPreManager.get_mean_upstream_timeseries_monthly(mp, dmManager) print("Calculated observed upstream mean values...") all_model_points.extend(mps) print("imported input data successfully, plotting ...") #for tests #test(sim_name_to_station_to_model_point) #select only stations which have corresponding model points stations = list(sim_name_to_station_to_model_point[sim_name_list[0]].keys()) from matplotlib.backends.backend_pdf import PdfPages for s in stations: years = s.get_list_of_complete_years() if len(years) < 6: continue #skip stations with less than 6 continuous years of data pp = PdfPages("nc_diagnose_{0}.pdf".format(s.id)) #plot hydrographs fig = plt.figure() gs = gridspec.GridSpec(3, 3, left=0.05, hspace=0.3, wspace=0.2) ax_stfl = fig.add_subplot(gs[0, 0]) labels, handles = plot_hydrographs(ax_stfl, s, sim_name_to_station_to_model_point, day_stamps=day_stamps, sim_names=sim_name_list ) plt.setp(ax_stfl.get_xticklabels(), visible=False) #do not show ticklabels for upper rows fig.legend(handles, labels, "lower right") #plot swe 1d compare with obs ax_swe = fig.add_subplot(gs[1, 0], sharex=ax_stfl) plot_swe_1d_compare_with_obs(ax_swe, s, sim_name_to_station_to_model_point, day_stamps=day_stamps, sim_names=sim_name_list) #plot mean temp 1d compare with obs -- here plot biases directly...?? ax_temp = fig.add_subplot(gs[0, 2]) plot_temp_1d_compare_with_obs(ax_temp, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) plt.setp(ax_temp.get_xticklabels(), visible=False) #do not show ticklabels for upper rows #plot mean precip 1d compare with obs -- here plot biases directly...?? ax = fig.add_subplot(gs[1, 2], sharex=ax_temp) plot_precip_1d_compare_with_obs(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) #plot mean Surface and subsurface runoff ax = fig.add_subplot(gs[0, 1], sharex=ax_stfl) plot_surf_runoff(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) plt.setp(ax.get_xticklabels(), visible=False) #do not show ticklabels for upper rows ax = fig.add_subplot(gs[1, 1], sharex=ax_stfl) plot_subsurf_runoff(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) plt.setp(ax.get_xticklabels(), visible=False) #do not show ticklabels for upper rows ax = fig.add_subplot(gs[2, 1], sharex=ax_stfl) plot_total_runoff(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) pp.savefig() #plot flow direction and basin boundaries fig = plt.figure() gs = gridspec.GridSpec(1, 2, right=0.99, bottom=0.001) ax = fig.add_subplot(gs[0, 1]) plot_flow_directions_and_basin_boundaries(ax, s, sim_name_to_station_to_model_point, sim_name_to_manager=sim_name_to_manager) pp.savefig() #plot 2d correlation between wind speed and measured streamflow at the station pp.close()
def validate_thawing_index(): start_year = 1980 end_year = 1996 sim_data_folder = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1" sim_names = ["ERA40","MPI","CanESM"] fold = "/home/samira/skynet/DailyTempData" simname_to_path = { "ERA40": os.path.join(fold, "TempERA40_b1_1981-2008.nc"), "MPI": os.path.join(fold, "TempMPI1981-2010.nc"), "CanESM":os.path.join(fold, "TempCanESM1981-2010.nc"), } coord_file = os.path.join(sim_data_folder, "pmNorthAmerica_0.44deg_ERA40-Int_B1_200812_moyenne") basemap, lons2d, lats2d = draw_regions.get_basemap_and_coords(resolution="c", file_path = coord_file, llcrnrlat=40.0, llcrnrlon=-145, urcrnrlon=-20, urcrnrlat=74 ) assert isinstance(basemap, Basemap) lons2d[lons2d > 180] -= 360 om = CRUDataManager() clim = om.get_daily_climatology(start_year, end_year) obs = om.get_thawing_index_from_climatology(clim) obs = om.interpolate_data_to(obs, lons2d, lats2d, nneighbours=1) #interpolatee to the model grid x, y = basemap(lons2d, lats2d) #x = (x[1:,1:] + x[:-1, :-1]) /2.0 permafrost_mask = draw_regions.get_permafrost_mask(lons2d, lats2d) mask_cond = (permafrost_mask <= 0) | (permafrost_mask >= 3) #plot_utils.apply_plot_params(width_pt=None, width_cm=35,height_cm=55, font_size=35) fig = plt.figure() assert isinstance(fig, Figure) cmap = my_colormaps.get_red_blue_colormap(ncolors=10, reversed=True) gs = gridspec.GridSpec(3,1) all_axes = [] all_img = [] i = 0 for name in sim_names: path = simname_to_path[name] ds = Dataset(path) data_mod = ds.variables["air"][:] mod = _get_thawing_index() delta = mod - obs ax = fig.add_subplot(gs[i,0]) assert isinstance(ax, Axes) delta = np.ma.masked_where(mask_cond, delta) img = basemap.pcolormesh(x, y, delta, cmap = cmap, vmin=None, vmax = None) if not i: ax.set_title("Thawing index, Mod - Obs, ({0} - {1}) \n".format(start_year, end_year)) i += 1 ax.set_ylabel(name) all_axes.append(ax) all_img.append(img) i = 0 axs_to_hide = [] #zones and coastlines for the_ax, the_img in zip(all_axes, all_img): divider = make_axes_locatable(the_ax) cax = divider.append_axes("right", "5%", pad="3%") cb = fig.colorbar(the_img, cax = cax, extend = "both") cax.set_title("$^{\\circ} {\\rm C}$\n") #cax.set_title("%\n") assert isinstance(the_ax, Axes) basemap.drawcoastlines(ax = the_ax, linewidth=1.5) basemap.readshapefile("data/pf_4/permafrost8_wgs84/permaice", name="zone", ax=the_ax, linewidth=3) if i != 1: axs_to_hide.append(cax) i += 1 fig.tight_layout() for the_ax in axs_to_hide: the_ax.set_visible(False) fig.savefig("tmp_validation_era_mpi_canesm.png") pass
def validate_using_monthly_diagnostics(): start_year = 1980 end_year = 1996 sim_data_folder = "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1" sim_names = ["ERA40","MPI","CanESM"] simname_to_path = { "ERA40": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/era40_driven_b1_dm", "MPI": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_MPI_B1_dm", "CanESM": "/home/huziy/skynet1_rech3/cordex/CORDEX_DIAG/NorthAmerica_0.44deg_CanESM_B1_dm" } coord_file = os.path.join(sim_data_folder, "pmNorthAmerica_0.44deg_ERA40-Int_B1_200812_moyenne") basemap, lons2d, lats2d = draw_regions.get_basemap_and_coords(resolution="c", file_path = coord_file, llcrnrlat=45.0, llcrnrlon=-145, urcrnrlon=-20, urcrnrlat=74, anchor = "W" ) assert isinstance(basemap, Basemap) lons2d[lons2d > 180] -= 360 obs_manager = CRUDataManager() obs = obs_manager.get_mean(start_year, end_year, months=[6,7,8]) obs = obs_manager.interpolate_data_to(obs, lons2d, lats2d, nneighbours=1) x, y = basemap(lons2d, lats2d) #x = (x[1:,1:] + x[:-1, :-1]) /2.0 permafrost_mask = draw_regions.get_permafrost_mask(lons2d, lats2d) mask_cond = (permafrost_mask <= 0) | (permafrost_mask >= 2) #plot_utils.apply_plot_params(width_pt=None, width_cm=35,height_cm=55, font_size=35) fig = plt.figure() assert isinstance(fig, Figure) cmap = my_colormaps.get_red_blue_colormap(ncolors=10, reversed=True) gs = gridspec.GridSpec(3,2, width_ratios=[1,0.1], hspace=0, wspace=0, left=0.05, bottom = 0.01, top=0.95) all_axes = [] all_img = [] i = 0 for name in sim_names: path = simname_to_path[name] dm = CRCMDataManager(data_folder=path) mod = dm.get_mean_over_months_of_2d_var(start_year, end_year, months = [6,7,8], var_name="TT", level=1, level_kind=level_kinds.HYBRID) delta = mod - obs ax = fig.add_subplot(gs[i,0]) assert isinstance(ax, Axes) delta = np.ma.masked_where(mask_cond, delta) img = basemap.pcolormesh(x, y, delta, cmap = cmap, vmin=-5.0, vmax = 5.0) if not i: ax.set_title("T2m, Mod - Obs, ({0} - {1}), JJA \n".format(start_year, end_year)) i += 1 #ax.set_ylabel(name) all_axes.append(ax) all_img.append(img) i = 0 axs_to_hide = [] #zones and coastlines for the_ax, the_img in zip(all_axes, all_img): # divider = make_axes_locatable(the_ax) # cax = divider.append_axes("right", "5%", pad="3%") #cax.set_title("%\n") assert isinstance(the_ax, Axes) basemap.drawcoastlines(ax = the_ax, linewidth=0.5) basemap.readshapefile("data/pf_4/permafrost8_wgs84/permaice", name="zone", ax=the_ax, linewidth=1.5, drawbounds=False) for nshape,seg in enumerate(basemap.zone): if basemap.zone_info[nshape]["EXTENT"] != "C": continue poly = mpl.patches.Polygon(seg,edgecolor = "k", facecolor="none", zorder = 10, lw = 1.5) the_ax.add_patch(poly) i += 1 cax = fig.add_subplot(gs[:,1]) cax.set_anchor("W") cax.set_aspect(30) formatter = FuncFormatter( lambda x, pos: "{0: <6}".format(str(x)) ) cb = fig.colorbar(all_img[0], ax = cax, cax = cax, extend = "both", format = formatter) cax.set_title("$^{\\circ} {\\rm C}$") #fig.tight_layout(h_pad=0) # for the_ax in axs_to_hide: # the_ax.set_visible(False) fig.savefig("tmp_validation_era_mpi_canesm.png")
def compare_for_season( start_year = 1958, end_year = 1974, the_months = None, period_str = "djf"): """ Compare CRU, ERA40-driven and GCM-driven s """ #b, lons2d, lats2d = draw_regions.get_basemap_and_coords(llcrnrlat=40.0, llcrnrlon=-145, urcrnrlon=-10) b, lons2d, lats2d = draw_regions.get_basemap_and_coords() lons2d[lons2d > 180] -= 360 x, y = b(lons2d, lats2d) cru = CRUDataManager() cru_data = cru.get_mean(start_year,end_year, months = the_months) cru_data_interp = cru.interpolate_data_to(cru_data, lons2d, lats2d) temp_levels = np.arange(-40, 40, 5) diff_levels = np.arange(-10, 12, 2) gs = gridspec.GridSpec(3,2) #plot_utils.apply_plot_params(width_pt=None, height_cm=20, width_cm=20, font_size=12) fig = plt.figure() coast_line_width = 0.25 axes_list = [] #plot CRU data ax = fig.add_subplot(gs[0,:]) axes_list.append(ax) cru_data_interp = maskoceans(lons2d, lats2d, cru_data_interp) img = b.contourf(x, y, cru_data_interp, ax = ax, levels = temp_levels) ax.set_title("CRU") plot_utils.draw_colorbar(fig, img, ax = ax) #era40 driven file_path = None era40_folder = "data/CORDEX/na/era40_1" file_prefix = "dm" for file_name in os.listdir(era40_folder): if period_str.upper() in file_name and file_name.startswith(file_prefix): file_path = os.path.join(era40_folder, file_name) break #get the temperature rpn_obj = RPN(file_path) t2m_era40 = rpn_obj.get_first_record_for_name_and_level(varname="TT", level=1, level_kind=level_kinds.HYBRID) t2m_era40 = maskoceans(lons2d, lats2d, t2m_era40) ax = fig.add_subplot(gs[1,0]) axes_list.append(ax) img = b.contourf(x, y, t2m_era40, ax = ax, levels = temp_levels) ax.set_title("ERA40 driven 1 (1958-1961)") plot_utils.draw_colorbar(fig, img, ax = ax) rpn_obj.close() #era40 - cru ax = fig.add_subplot(gs[1,1]) axes_list.append(ax) img = b.contourf(x, y, t2m_era40 - cru_data_interp, ax = ax, levels = diff_levels) ax.set_title("ERA40 driven 1 - CRU") plot_utils.draw_colorbar(fig, img, ax = ax) plot_e2_data = False if plot_e2_data: ##get and plot E2 data file_path = None e2_folder = "data/CORDEX/na/e2" prefix = "dm" #get file path for file_name in os.listdir(e2_folder): if file_name.endswith(period_str) and file_name.startswith(prefix): file_path = os.path.join(e2_folder, file_name) break pass #get the temperature rpn_obj = RPN(file_path) t2m = rpn_obj.get_first_record_for_name_and_level(varname="TT", level=1, level_kind=level_kinds.HYBRID) t2m = maskoceans(lons2d, lats2d, t2m) ax = fig.add_subplot(gs[2,0]) axes_list.append(ax) img = b.contourf(x, y, t2m, ax = ax, levels = temp_levels) ax.set_title("E2, GCM driven") plot_utils.draw_colorbar(fig, img, ax = ax) #e2 - cru ax = fig.add_subplot(gs[2,1]) axes_list.append(ax) img = b.contourf(x, y, t2m - cru_data_interp, ax = ax, levels = diff_levels) ax.set_title("E2, GCM driven - CRU") plot_utils.draw_colorbar(fig, img, ax = ax) ####Draw common elements pf_kinds = draw_regions.get_permafrost_mask(lons2d, lats2d) for the_ax in axes_list: b.drawcoastlines(ax = the_ax, linewidth = coast_line_width) b.contour(x, y, pf_kinds, ax = the_ax, colors = "k") gs.tight_layout(fig, h_pad = 5, w_pad = 5, pad=2) fig.suptitle(period_str.upper(), y = 0.03, x = 0.5) fig.savefig("temperature_validation_{0}.png".format(period_str)) fig = plt.figure() ax = plt.gca() img = b.contourf(x, y, t2m_era40 - cru_data_interp, ax = ax, levels = diff_levels) ax.set_title("ERA40 driven 1 - CRU") plot_utils.draw_colorbar(fig, img, ax = ax) b.drawcoastlines(ax = ax, linewidth = coast_line_width) b.contour(x, y, pf_kinds, ax = ax, colors = "k") fig.savefig("temperature_diff_{0}.png".format(period_str)) pass
def plot_total_precip_and_temp_re_1d(ax_pr, ax_temp, crcm5_manager, rot_latlon_projection, areas2d, model_data, mask=None): """ plot relative error of total precipitation in time """ assert isinstance(crcm5_manager, Crcm5ModelDataManager) assert isinstance(ax_pr, Axes) if mask is None: i_model0, j_model0 = model_data.metadata["ix"], model_data.metadata[ "jy"] mask = crcm5_manager.get_mask_for_cells_upstream(i_model0, j_model0) #print model_data.time[0], model_data.time[-1] #####Precipitation ts_prec_mod, dt_mod = crcm5_manager.get_monthly_sums_over_points( mask, "PR", areas2d=areas2d, start_date=model_data.time[0], end_date=model_data.time[-1]) cruManager = CRUDataManager( path="data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.pre.dat.nc", var_name="pre") ts_prec_obs = cruManager.get_monthly_timeseries_using_mask( mask, crcm5_manager.lons2D, crcm5_manager.lats2D, areas2d, start_date=model_data.time[0], end_date=model_data.time[-1]) mod = np.array( ts_prec_mod.data) * dt_mod.seconds * 1000.0 #converting to mm/month sta = np.array(ts_prec_obs.data) #CRU data used was initially in mm/month #print sta #print mod #print mod.shape #print sta.shape assert len(sta) == len(mod) ax_pr.annotate("r = {0:.2f}".format(float(np.corrcoef([mod, sta])[0, 1])), xy=(0.7, 0.8), xycoords="axes fraction") ax_pr.annotate("ns = {0:.2f}".format(scores.nash_sutcliffe(mod, sta)), xy=(0.7, 0.9), xycoords="axes fraction") ax_pr.plot(ts_prec_mod.time, (mod - sta) / sta, color="k", linewidth=2) ax_pr.xaxis.set_major_formatter(DateFormatter("%y/%m")) ax_pr.xaxis.set_major_locator(MonthLocator(bymonth=list(range(1, 13, 2)))) ax_pr.set_title("CRCM5 versus CRU") ax_pr.set_ylabel("Total Precip (Pmod - Pobs)/Pobs") #Temperature ts_temp_mod = crcm5_manager.get_spatial_integral_over_mask_of_dyn_field( mask, areas2d, var_name="TT", level=1000, level_kind=level_kinds.PRESSURE, path_to_folder=crcm5_manager.samples_folder, file_prefix="dp") print("ts_temp_mod time interval: ", ts_temp_mod.time[0], ts_temp_mod.time[-1]) ts_temp_mod = ts_temp_mod.time_slice(model_data.time[0], model_data.time[-1]) cruManager = CRUDataManager( path="data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.tmp.dat.nc", var_name="tmp") ts_temp_obs = cruManager.get_monthly_timeseries_using_mask( mask, crcm5_manager.lons2D, crcm5_manager.lats2D, areas2d, start_date=model_data.time[0], end_date=model_data.time[-1]) mod = np.array(ts_temp_mod.get_ts_of_monthly_means().data) / sum( areas2d[mask == 1]) #converting to mm/month sta = np.array(ts_temp_obs.data) / sum( areas2d[mask == 1]) #CRU data used was initially in mm/month assert len(sta) == len(mod) ax_temp.annotate("r = {0:.2f}".format(float(np.corrcoef([mod, sta])[0, 1])), xy=(0.7, 0.8), xycoords="axes fraction") ax_temp.annotate("ns = {0:.2f}".format(scores.nash_sutcliffe(mod, sta)), xy=(0.7, 0.9), xycoords="axes fraction") ax_temp.plot(ts_prec_mod.time, (mod - sta), color="k", linewidth=2) ax_temp.xaxis.set_major_formatter(DateFormatter("%y/%m")) ax_temp.xaxis.set_major_locator( MonthLocator(bymonth=list(range(1, 13, 2)))) ax_temp.set_title("CRCM5 versus CRU") ax_temp.set_ylabel("Temperature, deg., Tmod - Tobs") pass
def main(): erainterim_075_folder = "/HOME/data/Validation/ERA-Interim_0.75/Offline_driving_data/3h_Forecast" vname = "PR" start_year = 1980 end_year = 2010 season_key = "summer" season_labels = {season_key: "Summer"} season_to_months = OrderedDict([ (season_key, [6, 7, 8]) ]) # Validate temperature and precip model_vars = ["TT", "PR"] obs_vars = ["tmp", "pre"] obs_paths = [ "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc", "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc" ] model_var_to_obs_var = dict(zip(model_vars, obs_vars)) model_var_to_obs_path = dict(zip(model_vars, obs_paths)) obs_path = model_var_to_obs_path[vname] cru = CRUDataManager(var_name=model_var_to_obs_var[vname], path=obs_path) # Calculate seasonal means for CRU seasonal_clim_fields_cru = cru.get_seasonal_means(season_name_to_months=season_to_months, start_year=start_year, end_year=end_year) # Calculate seasonal mean for erai flist = get_files_for_season(erainterim_075_folder, start_year=start_year, end_year=end_year, months=season_to_months[season_key]) rpf = MultiRPN(flist) date_to_field_erai075 = rpf.get_all_time_records_for_name_and_level(varname=vname, level=-1) # Convert to mm/day era075 = np.mean([field for field in date_to_field_erai075.values()], axis=0) * 24 * 3600 * 1000 lons_era, lats_era = rpf.get_longitudes_and_latitudes_of_the_last_read_rec() seasonal_clim_fields_cru_interp = OrderedDict() # Calculate biases for season, cru_field in seasonal_clim_fields_cru.items(): seasonal_clim_fields_cru_interp[season] = cru.interpolate_data_to(cru_field, lons2d=lons_era, lats2d=lats_era, nneighbours=1) # Do the plotting ------------------------------------------------------------------------------ plot_utils.apply_plot_params() fig = plt.figure() b = Basemap() gs = gridspec.GridSpec(nrows=3, ncols=1) ax = fig.add_subplot(gs[0, 0]) xx, yy = b(cru.lons2d, cru.lats2d) cs = b.contourf(xx, yy, seasonal_clim_fields_cru[season_key], 20) b.drawcoastlines(ax=ax) ax.set_title("CRU") plt.colorbar(cs, ax=ax) ax = fig.add_subplot(gs[1, 0]) lons_era[lons_era > 180] -= 360 lons_era, era075 = b.shiftdata(lons_era, datain=era075, lon_0=0) xx, yy = b(lons_era, lats_era) # mask oceans in the era plot as well era075 = maskoceans(lons_era, lats_era, era075) cs = b.contourf(xx, yy, era075, levels=cs.levels, norm=cs.norm, cmap=cs.cmap, ax=ax) b.drawcoastlines(ax=ax) ax.set_title("ERA-Interim 0.75") plt.colorbar(cs, ax=ax) # differences ax = fig.add_subplot(gs[2, 0]) diff = era075 - seasonal_clim_fields_cru_interp[season_key] delta = np.percentile(np.abs(diff)[~diff.mask], 90) clevs = np.linspace(-delta, delta, 20) cs = b.contourf(xx, yy, diff, levels=clevs, cmap="RdBu_r", extend="both") b.drawcoastlines(ax=ax) ax.set_title("ERA-Interim 0.75 - CRU") plt.colorbar(cs, ax=ax) plt.show() fig.savefig(os.path.join(img_folder, "erai0.75_vs_cru_precip.png"), bbox_inches="tight")
def main(): erainterim_075_folder = "/HOME/data/Validation/ERA-Interim_0.75/Offline_driving_data/3h_Forecast" vname = "PR" start_year = 1980 end_year = 2010 season_key = "summer" season_labels = {season_key: "Summer"} season_to_months = OrderedDict([(season_key, [6, 7, 8])]) # Validate temperature and precip model_vars = ["TT", "PR"] obs_vars = ["tmp", "pre"] obs_paths = [ "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc", "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc", ] model_var_to_obs_var = dict(zip(model_vars, obs_vars)) model_var_to_obs_path = dict(zip(model_vars, obs_paths)) obs_path = model_var_to_obs_path[vname] cru = CRUDataManager(var_name=model_var_to_obs_var[vname], path=obs_path) # Calculate seasonal means for CRU seasonal_clim_fields_cru = cru.get_seasonal_means( season_name_to_months=season_to_months, start_year=start_year, end_year=end_year ) # Calculate seasonal mean for erai flist = get_files_for_season( erainterim_075_folder, start_year=start_year, end_year=end_year, months=season_to_months[season_key] ) rpf = MultiRPN(flist) date_to_field_erai075 = rpf.get_all_time_records_for_name_and_level(varname=vname, level=-1) # Convert to mm/day era075 = np.mean([field for field in date_to_field_erai075.values()], axis=0) * 24 * 3600 * 1000 lons_era, lats_era = rpf.get_longitudes_and_latitudes_of_the_last_read_rec() seasonal_clim_fields_cru_interp = OrderedDict() # Calculate biases for season, cru_field in seasonal_clim_fields_cru.items(): seasonal_clim_fields_cru_interp[season] = cru.interpolate_data_to( cru_field, lons2d=lons_era, lats2d=lats_era, nneighbours=1 ) # Do the plotting ------------------------------------------------------------------------------ plot_utils.apply_plot_params() fig = plt.figure() b = Basemap() gs = gridspec.GridSpec(nrows=3, ncols=1) ax = fig.add_subplot(gs[0, 0]) xx, yy = b(cru.lons2d, cru.lats2d) cs = b.contourf(xx, yy, seasonal_clim_fields_cru[season_key], 20) b.drawcoastlines(ax=ax) ax.set_title("CRU") plt.colorbar(cs, ax=ax) ax = fig.add_subplot(gs[1, 0]) lons_era[lons_era > 180] -= 360 lons_era, era075 = b.shiftdata(lons_era, datain=era075, lon_0=0) xx, yy = b(lons_era, lats_era) # mask oceans in the era plot as well era075 = maskoceans(lons_era, lats_era, era075) cs = b.contourf(xx, yy, era075, levels=cs.levels, norm=cs.norm, cmap=cs.cmap, ax=ax) b.drawcoastlines(ax=ax) ax.set_title("ERA-Interim 0.75") plt.colorbar(cs, ax=ax) # differences ax = fig.add_subplot(gs[2, 0]) diff = era075 - seasonal_clim_fields_cru_interp[season_key] delta = np.percentile(np.abs(diff)[~diff.mask], 90) clevs = np.linspace(-delta, delta, 20) cs = b.contourf(xx, yy, diff, levels=clevs, cmap="RdBu_r", extend="both") b.drawcoastlines(ax=ax) ax.set_title("ERA-Interim 0.75 - CRU") plt.colorbar(cs, ax=ax) plt.show() fig.savefig(os.path.join(img_folder, "erai0.75_vs_cru_precip.png"), bbox_inches="tight")
def main(): img_folder = Path("nei_validation") img_folder.mkdir(parents=True, exist_ok=True) var_names = ["TT", "PR"] seasons = OrderedDict([ ("DJF", MonthPeriod(12, 3)), ("MAM", MonthPeriod(3, 3)), ("JJA", MonthPeriod(6, 3)), ("SON", MonthPeriod(9, 3)), ]) sim_paths = OrderedDict() start_year = 1980 end_year = 2008 sim_paths["WC_0.44deg_default"] = Path \ ("/HOME/huziy/skynet3_rech1/CRCM5_outputs/NEI/diags/NEI_WC0.44deg_default/Diagnostics") sim_paths["WC_0.44deg_ctem+frsoil+dyngla"] = Path \ ("/HOME/huziy/skynet3_rech1/CRCM5_outputs/NEI/diags/debug_NEI_WC0.44deg_Crr1/Diagnostics") sim_paths["WC_0.11deg_ctem+frsoil+dyngla"] = Path( "/snow3/huziy/NEI/WC/NEI_WC0.11deg_Crr1/Diagnostics") cru_vname_to_path = { "pre": "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.pre.dat.nc", "tmp": "/HOME/data/Validation/CRU_TS_3.1/Original_files_gzipped/cru_ts_3_10.1901.2009.tmp.dat.nc" } plot_cru_data = True plot_model_data = True plot_naobs_data = True plot_daymet_data = True plot_utils.apply_plot_params(font_size=14) basemap_for_obs = None # plot simulation data for sim_label, sim_path in sim_paths.items(): manager = DiagCrcmManager(data_dir=sim_path) # get the basemap to be reused for plotting observation data if basemap_for_obs is None: basemap_for_obs = manager.get_basemap(resolution="i", area_thresh=area_thresh_km2) if not plot_model_data: break for vname in var_names: seas_to_clim = manager.get_seasonal_means_with_ttest_stats( season_to_monthperiod=seasons, start_year=start_year, end_year=end_year, vname=vname, vertical_level=var_name_to_level[vname], data_file_prefix=var_name_to_file_prefix[vname]) _plot_seasonal_data( seas_data=seas_to_clim, data_label="{}_{}-{}".format(sim_label, start_year, end_year), img_dir=img_folder, map=manager.get_basemap(resolution="i", area_thresh=area_thresh_km2), lons=manager.lons, lats=manager.lats, vname=vname) assert basemap_for_obs is not None # plot obs data # -- CRU for vname in var_names: if not plot_cru_data: break cru_vname = var_name_to_cru_name[vname] manager = CRUDataManager(path=cru_vname_to_path[cru_vname], var_name=cru_vname) seas_to_clim = manager.get_seasonal_means_with_ttest_stats( season_to_monthperiod=seasons, start_year=start_year, end_year=end_year) manager.close() _plot_seasonal_data(seas_data=seas_to_clim, data_label="{}_{}-{}".format( "CRU", start_year, end_year), img_dir=img_folder, map=basemap_for_obs, lons=manager.lons2d, lats=manager.lats2d, vname=vname, var_name_to_mul={ "TT": 1, "PR": 1 }) # -- NAOBS naobs_vname_to_path = { "TT": "/HOME/huziy/skynet3_rech1/obs_data/anuspl_uw_0.11_wc_domain/anusplin+_interpolated_tt_pr.nc", "PR": "/HOME/huziy/skynet3_rech1/obs_data/anuspl_uw_0.11_wc_domain/anusplin+_interpolated_tt_pr.nc" } for vname in var_names: if not plot_naobs_data: break manager = CRUDataManager(path=naobs_vname_to_path[vname], var_name=vname) seas_to_clim = manager.get_seasonal_means_with_ttest_stats( season_to_monthperiod=seasons, start_year=start_year, end_year=end_year) # mask no data points for s, data in seas_to_clim.items(): for i in [0, 1]: data[i] = np.ma.masked_where(manager.lats2d > 60, data[i]) data[i] = np.ma.masked_where(manager.lons2d < -150, data[i]) data[i] = maskoceans(manager.lons2d, manager.lats2d, datain=data[i]) _plot_seasonal_data(seas_data=seas_to_clim, data_label="{}_{}-{}".format( "NAOBS", start_year, end_year), img_dir=img_folder, map=basemap_for_obs, lons=manager.lons2d, lats=manager.lats2d, vname=vname) manager.close() # -- daymet monthly daymet_vname_to_path = { "prcp": "/HOME/data/Validation/Daymet/Monthly_means/NetCDF/daymet_v3_prcp_monttl_*_na.nc4", "tavg": "/HOME/huziy/skynet3_rech1/obs_data/daymet_tavg_monthly/daymet_v3_tavg_monavg_*_na_nc4classic.nc4" } vname_to_daymet_vname = {"PR": "prcp", "TT": "tavg"} for vname in var_names: if not plot_daymet_data: break daymet_vname = vname_to_daymet_vname[vname] manager = HighResDataManager(path=daymet_vname_to_path[daymet_vname], vname=daymet_vname) seas_to_clim = manager.get_seasonal_means_with_ttest_stats_dask( season_to_monthperiod=seasons, start_year=start_year, end_year=end_year, convert_monthly_accumulators_to_daily=(vname == "PR")) _plot_seasonal_data(seas_data=seas_to_clim, data_label="{}_{}-{}".format( "DAYMET", start_year, end_year), img_dir=img_folder, map=basemap_for_obs, lons=manager.lons, lats=manager.lats, vname=vname, var_name_to_mul={ "PR": 1, "TT": 1 }) manager.close()
def compare_vars(vname_model="TT", vname_obs="tmp", r_config=None, season_to_months=None, obs_path=None, nx_agg_model=5, ny_agg_model=5, bmp_info_agg=None, diff_axes_list=None, obs_axes_list=None, model_axes_list=None, bmp_info_model=None, mask_shape_file=None, nx_agg_obs=1, ny_agg_obs=1): """ if obs_axes_list is not None, plot observation data in those :param mask_shape_file: :param bmp_info_model: basemap info native to the model :param model_axes_list: Axes to plot model outputs :param vname_model: :param vname_obs: :param r_config: :param season_to_months: :param obs_path: :param nx_agg_model: :param ny_agg_model: :param bmp_info_agg: :param diff_axes_list: if it is None the plots for each variable is done in separate figures """ if vname_obs is None: vname_model_to_vname_obs = {"TT": "tmp", "PR": "pre"} vname_obs = vname_model_to_vname_obs[vname_model] seasonal_clim_fields_model = analysis.get_seasonal_climatology_for_runconfig( run_config=r_config, varname=vname_model, level=0, season_to_months=season_to_months) season_to_clim_fields_model_agg = OrderedDict() for season, field in seasonal_clim_fields_model.items(): print(field.shape) season_to_clim_fields_model_agg[season] = aggregate_array( field, nagg_x=nx_agg_model, nagg_y=ny_agg_model) if vname_model == "PR": season_to_clim_fields_model_agg[season] *= 1.0e3 * 24 * 3600 if vname_obs in [ "SWE", ]: obs_manager = SweDataManager(path=obs_path, var_name=vname_obs) elif obs_path is None: obs_manager = CRUDataManager(var_name=vname_obs) else: obs_manager = CRUDataManager(var_name=vname_obs, path=obs_path) seasonal_clim_fields_obs = obs_manager.get_seasonal_means( season_name_to_months=season_to_months, start_year=r_config.start_year, end_year=r_config.end_year) seasonal_clim_fields_obs_interp = OrderedDict() # Derive the mask from a shapefile if provided if mask_shape_file is not None: the_mask = get_mask(bmp_info_agg.lons, bmp_info_agg.lats, shp_path=mask_shape_file) else: the_mask = np.zeros_like(bmp_info_agg.lons) for season, obs_field in seasonal_clim_fields_obs.items(): obs_field = obs_manager.interpolate_data_to(obs_field, lons2d=bmp_info_agg.lons, lats2d=bmp_info_agg.lats, nneighbours=nx_agg_obs * ny_agg_obs) obs_field = np.ma.masked_where(the_mask > 0.5, obs_field) seasonal_clim_fields_obs_interp[season] = obs_field # assert hasattr(seasonal_clim_fields_obs_interp[season], "mask") season_to_err = OrderedDict() print("-------------var: {} (PE with CRU)---------------------".format( vname_model)) for season in seasonal_clim_fields_obs_interp: seasonal_clim_fields_obs_interp[season] = np.ma.masked_where( np.isnan(seasonal_clim_fields_obs_interp[season]), seasonal_clim_fields_obs_interp[season]) season_to_err[season] = season_to_clim_fields_model_agg[ season] - seasonal_clim_fields_obs_interp[season] if vname_model in ["I5"]: lons = bmp_info_agg.lons.copy() lons[lons > 180] -= 360 season_to_err[season] = maskoceans(lons, bmp_info_agg.lats, season_to_err[season]) good_vals = season_to_err[season] good_vals = good_vals[~good_vals.mask] print("{}: min={}; max={}; avg={}".format(season, good_vals.min(), good_vals.max(), good_vals.mean())) print("---------percetages --- CRU ---") print("{}: {} \%".format( season, good_vals.mean() / seasonal_clim_fields_obs_interp[season] [~season_to_err[season].mask].mean() * 100)) cs = plot_seasonal_mean_biases(season_to_error_field=season_to_err, varname=vname_model, basemap_info=bmp_info_agg, axes_list=diff_axes_list) if obs_axes_list is not None and vname_model in ["I5"]: clevs = [0, 50, 60, 70, 80, 90, 100, 150, 200, 250, 300, 350, 400, 500] cs_obs = None xx, yy = bmp_info_agg.get_proj_xy() lons = bmp_info_agg.lons.copy() lons[lons > 180] -= 360 lons_model = None xx_model, yy_model = None, None cs_mod = None norm = BoundaryNorm(clevs, 256) for col, (season, obs_field) in enumerate( seasonal_clim_fields_obs_interp.items()): # Obsrved fields ax = obs_axes_list[col] if bmp_info_agg.should_draw_basin_boundaries: bmp_info_agg.basemap.readshapefile(BASIN_BOUNDARIES_SHP[:-4], "basin", ax=ax) to_plot = maskoceans(lons, bmp_info_agg.lats, obs_field) cs_obs = bmp_info_agg.basemap.contourf(xx, yy, to_plot, levels=clevs, ax=ax, norm=norm, extend="max") bmp_info_agg.basemap.drawcoastlines(ax=ax, linewidth=0.3) ax.set_title(season) # Model outputs if model_axes_list is not None: ax = model_axes_list[col] if bmp_info_agg.should_draw_basin_boundaries: bmp_info_agg.basemap.readshapefile( BASIN_BOUNDARIES_SHP[:-4], "basin", ax=ax) if lons_model is None: lons_model = bmp_info_model.lons.copy() lons_model[lons_model > 180] -= 360 xx_model, yy_model = bmp_info_model.basemap( lons_model, bmp_info_model.lats) model_field = seasonal_clim_fields_model[season] to_plot = maskoceans(lons_model, bmp_info_model.lats, model_field) cs_mod = bmp_info_agg.basemap.contourf(xx_model, yy_model, to_plot, levels=cs_obs.levels, ax=ax, norm=cs_obs.norm, cmap=cs_obs.cmap, extend="max") bmp_info_agg.basemap.drawcoastlines(ax=ax, linewidth=0.3) plt.colorbar(cs_obs, cax=obs_axes_list[-1]) return cs