def main(): # Define the simulations to be validated r_config = RunConfig( data_path= "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5", start_year=1990, end_year=2010, label="CRCM5-L1") r_config_list = [r_config] r_config = RunConfig( data_path= "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r.hdf5", start_year=1990, end_year=2010, label="CRCM5-NL") r_config_list.append(r_config) bmp_info = analysis.get_basemap_info_from_hdf(file_path=r_config.data_path) bmp_info.should_draw_grey_map_background = True bmp_info.should_draw_basin_boundaries = False bmp_info.map_bg_color = "0.75" station_ids = ["104001", "093806", "093801", "081002", "081007", "080718"] # get river network information used in the model flow_directions = analysis.get_array_from_file( r_config.data_path, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) accumulation_area_km2 = analysis.get_array_from_file( path=r_config.data_path, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) cell_manager = CellManager(flow_dirs=flow_directions, lons2d=bmp_info.lons, lats2d=bmp_info.lats, accumulation_area_km2=accumulation_area_km2) # Get the list of stations to indicate on the bias map stations = cehq_station.read_station_data(start_date=None, end_date=None, selected_ids=station_ids) """:type : list[Station]""" xx, yy = bmp_info.get_proj_xy() station_to_modelpoint = cell_manager.get_model_points_for_stations( station_list=stations) upstream_edges = cell_manager.get_upstream_polygons_for_points( model_point_list=station_to_modelpoint.values(), xx=xx, yy=yy) bmp_info.draw_colorbar_for_each_subplot = True # Validate temperature, precip and swe obs_path_anusplin = "/home/huziy/skynet3_rech1/anusplin_links" obs_path_swe = "data/swe_ross_brown/swe.nc" model_var_to_obs_path = OrderedDict([("TT", obs_path_anusplin), ("I5", obs_path_swe)]) model_var_to_season = OrderedDict([ ("TT", OrderedDict([("Spring", range(3, 6))])), ("I5", OrderedDict([("Winter", [1, 2, 12])])) ]) vname_to_obs_data = {} # parameters that won't change in the loop over variable names params_const = dict(rconfig=r_config, bmp_info=bmp_info) for vname, obs_path in model_var_to_obs_path.items(): season_to_obs_data = get_seasonal_clim_obs_data( vname=vname, obs_path=obs_path, season_to_months=model_var_to_season[vname], **params_const) # Comment swe over lakes, since I5 calculated only for land if vname in [ "I5", ]: for season in season_to_obs_data: season_to_obs_data[season] = maskoceans( bmp_info.lons, bmp_info.lats, season_to_obs_data[season], inlands=True) vname_to_obs_data[vname] = season_to_obs_data # Plotting plot_all_vars_in_one_fig = True fig = None gs = None if plot_all_vars_in_one_fig: plot_utils.apply_plot_params(font_size=12, width_pt=None, width_cm=25, height_cm=20) fig = plt.figure() ncols = len(model_var_to_obs_path) + 1 gs = GridSpec(len(r_config_list), ncols, width_ratios=(ncols - 1) * [ 1., ] + [ 0.05, ]) else: plot_utils.apply_plot_params(font_size=12, width_pt=None, width_cm=25, height_cm=25) station_x_list = [] station_y_list = [] mvarname_to_cs = {} for row, r_config in enumerate(r_config_list): for col, mname in enumerate(model_var_to_obs_path): row_axes = [ fig.add_subplot(gs[row, col]), ] mvarname_to_cs[mname] = compare_vars( vname_model=mname, vname_to_obs=vname_to_obs_data, r_config=r_config, season_to_months=model_var_to_season[mname], bmp_info_agg=bmp_info, axes_list=row_axes) # -1 in order to exclude colorbars for the_ax in row_axes: the_ax.set_title(the_ax.get_title() + ", {}".format( infovar.get_long_display_label_for_var(mname))) # Need titles only for the first row if row > 0: the_ax.set_title("") if col == 0: the_ax.set_ylabel(r_config.label) else: the_ax.set_ylabel("") draw_upstream_area_bounds(the_ax, upstream_edges, color="g") if len(station_x_list) == 0: for the_station in stations: xst, yst = bmp_info.basemap(the_station.longitude, the_station.latitude) station_x_list.append(xst) station_y_list.append(yst) bmp_info.basemap.scatter(station_x_list, station_y_list, c="g", ax=the_ax, s=20, zorder=10, alpha=0.5) # Save the figure if necessary if plot_all_vars_in_one_fig: if not img_folder.is_dir(): img_folder.mkdir(parents=True) fig_path = img_folder.joinpath("{}.png".format( "_".join(model_var_to_obs_path))) with fig_path.open("wb") as figfile: fig.savefig(figfile, format="png", bbox_inches="tight") plt.close(fig)
def main(): start_year = 1980 end_year = 2010 # model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Diagnostics") model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Samples") static_data_file = "/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p" corrected_obs_data_folder = Path("mh/obs_data/") r = RPN(static_data_file) fldir = r.get_first_record_for_name("FLDR") faa = r.get_first_record_for_name("FAA") lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec() gc = default_domains.bc_mh_044 cell_manager = CellManager(fldir, nx=fldir.shape[0], ny=fldir.shape[1], lons2d=lons, lats2d=lats, accumulation_area_km2=faa) selected_station_ids = [ "05LM006", "05BN012", "05AK001", "05QB003" ] stations = cehq_station.load_from_hydat_db(province=None, selected_ids=selected_station_ids, natural=None, skip_data_checks=True) for s in stations: assert isinstance(s, cehq_station.Station) if s.id == "05AK001": s.drainage_km2 *= 2.5 if s.id == "05BN012": pass # Manitoba natural stations # statons_mnb = cehq_station.load_from_hydat_db(province="MB", natural=True, start_date=datetime(start_year, 1, 1), end_date=datetime(end_year,12,31)) # statons_ssk = cehq_station.load_from_hydat_db(province="SK", natural=True, start_date=datetime(start_year, 1, 1), end_date=datetime(end_year,12,31)) # statons_alb = cehq_station.load_from_hydat_db(province="AB", natural=True, start_date=datetime(start_year, 1, 1), end_date=datetime(end_year,12,31)) # for s in statons_mnb + statons_ssk + statons_alb: # if s not in stations: # stations.append(s) # (06EA002): CHURCHILL RIVER AT SANDY BAY at (-102.31832885742188,55.52333068847656), accum. area is 212000.0 km**2 # TODO: plot where is this station, compare modelled and observed hydrographs for s in stations: print(s) # assert len(stations) == len(selected_station_ids), "Could not find stations for some of the specified ids" station_to_model_point = cell_manager.get_model_points_for_stations(stations, drainaige_area_reldiff_limit=0.9, nneighbours=8) print("Established the station to model point mapping") plot_validations_for_stations(station_to_model_point, cell_manager=cell_manager, corrected_obs_data_folder=corrected_obs_data_folder, model_data_path=model_data_path, grid_config=gc, start_year=start_year, end_year=end_year)
def main(): direction_file_path = Path( "/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p" ) sim_label = "mh_0.44" start_year = 1981 end_year = 2010 streamflow_internal_name = "streamflow" selected_staion_ids = constants.selected_station_ids_for_streamflow_validation # ====================================================== day = timedelta(days=1) t0 = datetime(2001, 1, 1) stamp_dates = [t0 + i * day for i in range(365)] print("stamp dates range {} ... {}".format(stamp_dates[0], stamp_dates[-1])) lake_fraction = None # establish the correspondence between the stations and model grid points with RPN(str(direction_file_path)) as r: assert isinstance(r, RPN) fldir = r.get_first_record_for_name("FLDR") flow_acc_area = r.get_first_record_for_name("FAA") lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec() # lake_fraction = r.get_first_record_for_name("LF1") cell_manager = CellManager(fldir, lons2d=lons, lats2d=lats, accumulation_area_km2=flow_acc_area) stations = stfl_stations.load_stations_from_csv( selected_ids=selected_staion_ids) station_to_model_point = cell_manager.get_model_points_for_stations( station_list=stations, lake_fraction=lake_fraction, nneighbours=8) # Update the end year if required max_year_st = -1 for station in station_to_model_point: y = max(station.get_list_of_complete_years()) if y >= max_year_st: max_year_st = y if end_year > max_year_st: print("Updated end_year to {}, because no obs data after...".format( max_year_st)) end_year = max_year_st # read model data mod_data_manager = DataManager( store_config={ "varname_mapping": { streamflow_internal_name: "STFA" }, "base_folder": str(direction_file_path.parent.parent), "data_source_type": data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT, "level_mapping": { streamflow_internal_name: VerticalLevel(-1, level_type=level_kinds.ARBITRARY) }, "offset_mapping": vname_to_offset_CRCM5, "filename_prefix_mapping": { streamflow_internal_name: "pm" } }) station_to_model_data = defaultdict(list) for year in range(start_year, end_year + 1): start = Pendulum(year, 1, 1) p_test = Period(start, start.add(years=1).subtract(microseconds=1)) stfl_mod = mod_data_manager.read_data_for_period( p_test, streamflow_internal_name) # convert to daily stfl_mod = stfl_mod.resample("D", "t", how="mean", closed="left", keep_attrs=True) assert isinstance(stfl_mod, xr.DataArray) for station, model_point in station_to_model_point.items(): assert isinstance(model_point, ModelPoint) ts1 = stfl_mod[:, model_point.ix, model_point.jy].to_series() station_to_model_data[station].append( pd.Series(index=stfl_mod.t.values, data=ts1)) # concatenate the timeseries for each point, if required if end_year - start_year + 1 > 1: for station in station_to_model_data: station_to_model_data[station] = pd.concat( station_to_model_data[station]) else: for station in station_to_model_data: station_to_model_data[station] = station_to_model_data[station][0] # calculate observed climatology station_to_climatology = OrderedDict() for s in sorted(station_to_model_point, key=lambda st: st.latitude, reverse=True): assert isinstance(s, Station) print(s.id, len(s.get_list_of_complete_years())) # Check if there are continuous years for the selected period common_years = set(s.get_list_of_complete_years()).intersection( set(range(start_year, end_year + 1))) if len(common_years) > 0: _, station_to_climatology[ s] = s.get_daily_climatology_for_complete_years_with_pandas( stamp_dates=stamp_dates, years=common_years) _, station_to_model_data[ s] = pandas_utils.get_daily_climatology_from_pandas_series( station_to_model_data[s], stamp_dates, years_of_interest=common_years) else: print( "Skipping {}, since it does not have enough data during the period of interest" .format(s.id)) # ---- Do the plotting ---- ncols = 4 nrows = len(station_to_climatology) // ncols nrows += int(not (len(station_to_climatology) % ncols == 0)) axes_list = [] plot_utils.apply_plot_params(width_cm=8 * ncols, height_cm=8 * nrows, font_size=8) fig = plt.figure() gs = GridSpec(nrows=nrows, ncols=ncols) for i, (s, clim) in enumerate(station_to_climatology.items()): assert isinstance(s, Station) row = i // ncols col = i % ncols print(row, col, nrows, ncols) # normalize by the drainage area if s.drainage_km2 is not None: station_to_model_data[ s] *= s.drainage_km2 / station_to_model_point[ s].accumulation_area if s.id in constants.stations_to_greyout: ax = fig.add_subplot(gs[row, col], facecolor="0.45") else: ax = fig.add_subplot(gs[row, col]) assert isinstance(ax, Axes) ax.plot(stamp_dates, clim, color="k", lw=2, label="Obs.") ax.plot(stamp_dates, station_to_model_data[s], color="r", lw=2, label="Mod.") ax.xaxis.set_major_formatter(FuncFormatter(format_month_label)) ax.xaxis.set_major_locator(MonthLocator(bymonthday=15)) ax.xaxis.set_minor_locator(MonthLocator(bymonthday=1)) ax.grid() ax.annotate(s.get_pp_name(), xy=(1.02, 1), xycoords="axes fraction", horizontalalignment="left", verticalalignment="top", fontsize=8, rotation=-90) last_date = stamp_dates[-1] last_date = last_date.replace( day=calendar.monthrange(last_date.year, last_date.month)[1]) ax.set_xlim(stamp_dates[0].replace(day=1), last_date) ymin, ymax = ax.get_ylim() ax.set_ylim(0, ymax) if s.drainage_km2 is not None: ax.set_title( "{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA={:.0f} km$^2$)". format(s.id, s.longitude, s.latitude, s.drainage_km2)) else: ax.set_title( "{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA not used)".format( s.id, s.longitude, s.latitude)) axes_list.append(ax) # plot the legend axes_list[-1].legend() if not img_folder.exists(): img_folder.mkdir() fig.tight_layout() img_file = img_folder / "{}_{}-{}_{}.png".format( sim_label, start_year, end_year, "-".join( sorted(s.id for s in station_to_climatology))) print("Saving {}".format(img_file)) fig.savefig(str(img_file), bbox_inches="tight", dpi=300)
def draw_model_comparison(model_points=None, stations=None, sim_name_to_file_name=None, hdf_folder=None, start_year=None, end_year=None, cell_manager=None, stfl_name="STFA", drainage_area_reldiff_min=0.1, plot_upstream_area_averaged=True, sim_name_to_color=None): """ :param model_points: list of model point objects :param stations: list of stations corresponding to the list of model points :param cell_manager: is a CellManager instance which can be provided for better performance if necessary len(model_points) == len(stations) if stations is not None. if stations is None - then no measured streamflow will be plotted """ assert model_points is None or stations is None or len(stations) == len(model_points) label_list = list(sim_name_to_file_name.keys()) # Needed to keep the order the same for all subplots path0 = os.path.join(hdf_folder, list(sim_name_to_file_name.items())[0][1]) flow_directions = analysis.get_array_from_file(path=path0, var_name="flow_direction") lake_fraction = analysis.get_array_from_file(path=path0, var_name="lake_fraction") # mask lake fraction in the ocean lake_fraction = np.ma.masked_where((flow_directions <= 0) | (flow_directions > 128), lake_fraction) accumulation_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) area_m2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_M2) # Try to read cell areas im meters if it is not Ok then try in km2 if area_m2 is not None: cell_area_km2 = area_m2 * 1.0e-6 else: cell_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_KM2) print("cell area ranges from {} to {}".format(cell_area_km2.min(), cell_area_km2.max())) # print "plotting from {0}".format(path0) # plt.pcolormesh(lake_fraction.transpose()) # plt.colorbar() # plt.show() # exit() file_scores = open("scores_{0}_{1}-{2}.txt".format("_".join(label_list), start_year, end_year), "w") file_correlations = open("corr_{0}_{1}-{2}.txt".format("_".join(label_list), start_year, end_year), "w") file_annual_discharge = open("flow_{0}_{1}-{2}.txt".format("_".join(label_list), start_year, end_year), "w") text_files = [file_scores, file_correlations, file_annual_discharge] # write the following columns to the scores file header_format = "{0:10s}\t{1:10s}\t{2:10s}\t" + "\t".join(["{" + str(i + 3) + ":10s}" for i in range(len(sim_name_to_file_name))]) line_format = "{0:10s}\t{1:10.1f}\t{2:10.1f}\t" + "\t".join(["{" + str(i + 3) + ":10.1f}" for i in range(len(sim_name_to_file_name))]) header_ns = ("ID", "DAo", "DAm",) + tuple(["NS({0})".format(key) for key in sim_name_to_file_name]) file_scores.write(header_format.format(*header_ns) + "\n") header_qyear = ("ID", "DAo", "DAm",) + tuple(["Qyear({0})".format(key) for key in label_list]) + \ ("Qyear(obs)",) header_format_qyear = header_format + "\t{" + str(len(label_list) + 3) + ":10s}" file_annual_discharge.write(header_format_qyear.format(*header_qyear) + "\n") lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(file_path=path0) # Create a cell manager if it is not provided if cell_manager is None: cell_manager = CellManager(flow_directions, accumulation_area_km2=accumulation_area_km2, lons2d=lons2d, lats2d=lats2d) if stations is not None: # Get the list of the corresponding model points station_to_modelpoint = cell_manager.get_model_points_for_stations( station_list=stations, lake_fraction=lake_fraction, drainaige_area_reldiff_limit=drainage_area_reldiff_min) station_list = list(station_to_modelpoint.keys()) station_list.sort(key=lambda st1: st1.latitude, reverse=True) mp_list = [station_to_modelpoint[st] for st in station_list] else: mp_list = model_points station_list = None # sort so that the northernmost stations appear uppermost mp_list.sort(key=lambda mpt: mpt.latitude, reverse=True) # set ids to the model points so they can be distinguished easier model_point.set_model_point_ids(mp_list) # ###Uncomment the lines below for the validation plot in paper 2 # brewer2mpl.get_map args: set name set type number of colors # bmap = brewer2mpl.get_map("Set1", "qualitative", 9) # Change the default colors # mpl.rcParams["axes.color_cycle"] = bmap.mpl_colors # For the streamflow only plot ncols = 3 nrows = max(len(mp_list) // ncols, 1) if ncols * nrows < len(mp_list): nrows += 1 figure_stfl = plt.figure(figsize=(4 * ncols, 3 * nrows)) gs_stfl = gridspec.GridSpec(nrows=nrows, ncols=ncols) # a flag which signifies if a legend should be added to the plot, it is needed so we ahve only one legend per plot legend_added = False ax_stfl = None all_years = [y for y in range(start_year, end_year + 1)] if station_list is not None: processed_stations = station_list else: processed_stations = [None] * len(mp_list) processed_model_points = mp_list plot_point_positions_with_upstream_areas(processed_stations, processed_model_points, basemap, cell_manager, lake_fraction_field=lake_fraction) if plot_upstream_area_averaged: # create obs data managers anusplin_tmin = AnuSplinManager(variable="stmn") anusplin_tmax = AnuSplinManager(variable="stmx") anusplin_pcp = AnuSplinManager(variable="pcp") daily_dates, obs_tmin_fields = anusplin_tmin.get_daily_clim_fields_interpolated_to( start_year=start_year, end_year=end_year, lons_target=lons2d, lats_target=lats2d) _, obs_tmax_fields = anusplin_tmax.get_daily_clim_fields_interpolated_to( start_year=start_year, end_year=end_year, lons_target=lons2d, lats_target=lats2d) _, obs_pcp_fields = anusplin_pcp.get_daily_clim_fields_interpolated_to( start_year=start_year, end_year=end_year, lons_target=lons2d, lats_target=lats2d) swe_path = "/skynet3_rech1/huziy/swe_ross_brown/swe.nc4" if not os.path.isfile(os.path.realpath(swe_path)): raise IOError("SWE-obs file {} does not exist".format(swe_path)) swe_manager = SweDataManager(path=swe_path, var_name="SWE") obs_swe_daily_clim = swe_manager.get_daily_climatology(start_year, end_year) interpolated_obs_swe_clim = swe_manager.interpolate_daily_climatology_to(obs_swe_daily_clim, lons2d_target=lons2d, lats2d_target=lats2d) values_obs = None for i, the_model_point in enumerate(mp_list): ax_stfl = figure_stfl.add_subplot(gs_stfl[i // ncols, i % ncols], sharex=ax_stfl) assert isinstance(the_model_point, ModelPoint) # Check the number of years accessible for the station if the list of stations is given the_station = None if station_list is None else station_list[i] if the_station is not None: assert isinstance(the_station, Station) year_list = the_station.get_list_of_complete_years() year_list = list(filter(lambda yi: start_year <= yi <= end_year, year_list)) if len(year_list) < 1: continue else: year_list = all_years fig = plt.figure(figsize=(12, 15)) gs = gridspec.GridSpec(4, 4, wspace=1) # plot station position ax = fig.add_subplot(gs[3, 0:2]) upstream_mask = _plot_station_position(ax, the_station, basemap, cell_manager, the_model_point) # plot streamflows ax = fig.add_subplot(gs[0:2, 0:2]) dates = None model_daily_temp_clim = {} model_daily_precip_clim = {} model_daily_clim_surf_runoff = {} model_daily_clim_subsurf_runoff = {} model_daily_clim_swe = {} # get model data for the list of years simlabel_to_vals = {} for label in label_list: fname = sim_name_to_file_name[label] if hdf_folder is None: fpath = fname else: fpath = os.path.join(hdf_folder, fname) if plot_upstream_area_averaged: # read temperature data and calculate daily climatologic fileds _, model_daily_temp_clim[label] = analysis.get_daily_climatology( path_to_hdf_file=fpath, var_name="TT", level=0, start_year=start_year, end_year=end_year) # read modelled precip and calculate daily climatologic fields _, model_daily_precip_clim[label] = analysis.get_daily_climatology( path_to_hdf_file=fpath, var_name="PR", level=0, start_year=start_year, end_year=end_year) # read modelled surface runoff and calculate daily climatologic fields _, model_daily_clim_surf_runoff[label] = analysis.get_daily_climatology( path_to_hdf_file=fpath, var_name="TRAF", level=0, start_year=start_year, end_year=end_year) # read modelled subsurface runoff and calculate daily climatologic fields _, model_daily_clim_subsurf_runoff[label] = analysis.get_daily_climatology( path_to_hdf_file=fpath, var_name="TDRA", level=0, start_year=start_year, end_year=end_year) # read modelled swe and calculate daily climatologic fields _, model_daily_clim_swe[label] = analysis.get_daily_climatology( path_to_hdf_file=fpath, var_name="I5", level=0, start_year=start_year, end_year=end_year) dates, values_model = analysis.get_daily_climatology_for_a_point(path=fpath, var_name=stfl_name, years_of_interest=year_list, i_index=the_model_point.ix, j_index=the_model_point.jy) ax.plot(dates, values_model, label=label, lw=2) if sim_name_to_color is None: ax_stfl.plot(dates, values_model, label=label, lw=2) else: ax_stfl.plot(dates, values_model, sim_name_to_color[label], label=label, lw=2) print(20 * "!!!") print("{} -> {}".format(label, sim_name_to_color[label])) print(20 * "!!!") simlabel_to_vals[label] = values_model if the_station is not None: assert isinstance(the_station, Station) dates, values_obs = the_station.get_daily_climatology_for_complete_years_with_pandas(stamp_dates=dates, years=year_list) # To keep the colors consistent for all the variables, the obs Should be plotted last ax.plot(dates, values_obs, label="Obs.", lw=2) # no ticklabels for streamflow plot plt.setp(ax.get_xticklabels(), visible=False) if sim_name_to_color is None: ax_stfl.plot(dates, values_obs, label="Obs.", lw=2) else: ax_stfl.plot(dates, values_obs, label="Obs.", lw=2, color=sim_name_to_color["Obs."]) # Print excesss from streamflow validation for label, values_model in simlabel_to_vals.items(): calclulate_spring_peak_err(dates, values_obs, values_model, st_id="{}: {}".format(label, the_station.id), da_mod=the_model_point.accumulation_area, da_obs=the_station.drainage_km2) ax.set_ylabel(r"Streamflow: ${\rm m^3/s}$") assert isinstance(ax, Axes) assert isinstance(fig, Figure) upstream_area_km2 = np.sum(cell_area_km2[upstream_mask == 1]) # Put some information about the point if the_station is not None: lf_upstream = lake_fraction[upstream_mask == 1] point_info = "{0}".format(the_station.id) write_annual_flows_to_txt(label_list, simlabel_to_vals, values_obs, file_annual_discharge, station_id=the_station.id, da_obs=the_station.drainage_km2, da_mod=the_model_point.accumulation_area) else: point_info = "{0}".format(the_model_point.point_id) ax.annotate(point_info, (0.8, 0.8), xycoords="axes fraction", bbox=dict(facecolor="white", alpha=0.5), va="top", ha="right") ax.legend(loc=(0.0, 1.05), borderaxespad=0, ncol=3) ax.xaxis.set_minor_formatter(FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0])) ax.xaxis.set_minor_locator(MonthLocator(bymonthday=15)) ax.xaxis.set_major_locator(MonthLocator()) ax.grid() streamflow_axes = ax # save streamflow axes for later use if not legend_added: ax_stfl.legend(loc="lower left", bbox_to_anchor=(0, 1.15), borderaxespad=0, ncol=3) ax_stfl.xaxis.set_minor_formatter(FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0])) ax_stfl.xaxis.set_minor_locator(MonthLocator(bymonthday=15)) ax_stfl.xaxis.set_major_locator(MonthLocator()) ax_stfl.set_ylabel(r"Streamflow ${\rm m^3/s}$") legend_added = True plt.setp(ax_stfl.get_xmajorticklabels(), visible=False) ax_stfl.yaxis.set_major_locator(MaxNLocator(nbins=5)) sfmt = ScalarFormatter(useMathText=True) sfmt.set_powerlimits((-2, 2)) ax_stfl.yaxis.set_major_formatter(sfmt) ax_stfl.grid() # annotate streamflow-only panel plot ax_stfl.annotate(point_info, (0.05, 0.95), xycoords="axes fraction", bbox=dict(facecolor="white"), va="top", ha="left") if plot_upstream_area_averaged: # plot temperature comparisons (tmod - daily with anusplin tmin and tmax) ax = fig.add_subplot(gs[3, 2:], sharex=streamflow_axes) _validate_temperature_with_anusplin(ax, the_model_point, cell_area_km2=cell_area_km2, upstream_mask=upstream_mask, daily_dates=daily_dates, obs_tmin_clim_fields=obs_tmin_fields, obs_tmax_clim_fields=obs_tmax_fields, model_data_dict=model_daily_temp_clim, simlabel_list=label_list) # plot temperature comparisons (tmod - daily with anusplin tmin and tmax) ax = fig.add_subplot(gs[2, 2:], sharex=streamflow_axes) _validate_precip_with_anusplin(ax, the_model_point, cell_area_km2=cell_area_km2, upstream_mask=upstream_mask, daily_dates=daily_dates, obs_precip_clim_fields=obs_pcp_fields, model_data_dict=model_daily_precip_clim, simlabel_list=label_list) # plot mean upstream surface runoff ax = fig.add_subplot(gs[0, 2:], sharex=streamflow_axes) _plot_upstream_surface_runoff(ax, the_model_point, cell_area_km2=cell_area_km2, upstream_mask=upstream_mask, daily_dates=daily_dates, model_data_dict=model_daily_clim_surf_runoff, simlabel_list=label_list) # plot mean upstream subsurface runoff ax = fig.add_subplot(gs[1, 2:], sharex=streamflow_axes, sharey=ax) _plot_upstream_subsurface_runoff(ax, the_model_point, cell_area_km2=cell_area_km2, upstream_mask=upstream_mask, daily_dates=daily_dates, model_data_dict=model_daily_clim_subsurf_runoff, simlabel_list=label_list) # plot mean upstream swe comparison ax = fig.add_subplot(gs[2, 0:2], sharex=streamflow_axes) print("Validating SWE for ", the_station.id, "--" * 20) _validate_swe_with_ross_brown(ax, the_model_point, cell_area_km2=cell_area_km2, upstream_mask=upstream_mask, daily_dates=daily_dates, model_data_dict=model_daily_clim_swe, obs_swe_clim_fields=interpolated_obs_swe_clim, simlabel_list=label_list) if the_station is not None: im_name = "comp_point_with_obs_{0}_{1}_{2}.png".format(the_station.id, the_station.source, "_".join(label_list)) im_folder_path = os.path.join(images_folder, the_station.source) else: im_name = "comp_point_with_obs_{0}_{1}.png".format(the_model_point.point_id, "_".join(label_list)) im_folder_path = os.path.join(images_folder, "outlets_point_comp") # create a folder for a given source of observed streamflow if it does not exist yet if not os.path.isdir(im_folder_path): os.mkdir(im_folder_path) im_path = os.path.join(im_folder_path, im_name) if plot_upstream_area_averaged: fig.savefig(im_path, dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight", transparent=True) plt.close(fig) # return # temporary plot only one point assert isinstance(figure_stfl, Figure) figure_stfl.tight_layout() figure_stfl.savefig(os.path.join(images_folder, "comp_point_with_obs_{0}.png".format("_".join(label_list))), bbox_inches="tight", transparent=True, dpi=cpp.FIG_SAVE_DPI) plt.close(figure_stfl) # close information text files for f in text_files: f.close()
def main(directions_file_path: Path): """ compare drainage areas, longitudes and latitudes from the stations and model """ stations = stfl_stations.load_stations_from_csv() lake_fraction = None with Dataset(str(directions_file_path)) as ds: flow_dirs = ds.variables["flow_direction_value"][:] flow_acc_area = ds.variables["accumulation_area"][:] lons_2d, lats_2d = [ds.variables[k][:] for k in ["lon", "lat"]] # lake_fraction = ds.variables["lake_fraction"][:] cell_manager = CellManager(flow_dirs, lons2d=lons_2d, lats2d=lats_2d, accumulation_area_km2=flow_acc_area) station_to_mod_point = cell_manager.get_model_points_for_stations( station_list=stations, lake_fraction=lake_fraction, nneighbours=8) lons_m, lats_m, da_m = [], [], [] lons_o, lats_o, da_o = [], [], [] for s, mp in station_to_mod_point.items(): assert isinstance(s, Station) assert isinstance(mp, ModelPoint) # obs lons_o.append(s.longitude if s.longitude < 180 else s.longitude - 360) lats_o.append(s.latitude) da_o.append(s.drainage_km2) # model lons_m.append(mp.longitude if mp.longitude < 180 else mp.longitude - 360) lats_m.append(mp.latitude) da_m.append(mp.accumulation_area) print("m | s ({})".format(s.id)) print("{} | {}".format(mp.longitude, s.longitude)) print("{} | {}".format(mp.latitude, s.latitude)) print("{} | {}".format(mp.accumulation_area, s.drainage_km2)) axes_list = [] plot_utils.apply_plot_params(width_cm=25, height_cm=10, font_size=8) fig = plt.figure() gs = GridSpec(1, 3) ax = fig.add_subplot(gs[0, 0]) ax.set_title("Longitude") ax.scatter(lons_o, lons_m) axes_list.append(ax) ax.set_ylabel("Model") ax = fig.add_subplot(gs[0, 1]) ax.set_title("Latitude") ax.scatter(lats_o, lats_m) axes_list.append(ax) ax.set_xlabel("Obs") ax = fig.add_subplot(gs[0, 2]) ax.set_title("Drainage area (km$^2$)") ax.scatter(da_o, da_m) sf = ScalarFormatter(useMathText=True) sf.set_powerlimits((-2, 3)) ax.set_xscale("log") ax.set_yscale("log") axes_list.append(ax) # plot the 1-1 line for ax in axes_list: assert isinstance(ax, Axes) ax.plot(ax.get_xlim(), ax.get_xlim(), "--", color="gray") ax.grid() img_file = img_folder.joinpath("lon_lat_da_scatter_{}_{}.png".format( directions_file_path.name, "-".join(sorted(s.id for s in station_to_mod_point)))) fig.savefig(str(img_file), bbox_inches="tight")
def plot_station_positions(directions_file_path: Path, stations: list, ax: Axes, grid_config: GridConfig=default_domains.bc_mh_044, save_upstream_boundaries_to_shp=False): with Dataset(str(directions_file_path)) as ds: flow_dirs = ds.variables["flow_direction_value"][:] flow_acc_area = ds.variables["accumulation_area"][:] lons_2d, lats_2d = [ds.variables[k][:] for k in ["lon", "lat"]] basemap, reg_of_interest = grid_config.get_basemap_using_shape_with_polygons_of_interest(lons_2d, lats_2d, shp_path=default_domains.MH_BASINS_PATH, resolution="i") cell_manager = CellManager(flow_dirs, lons2d=lons_2d, lats2d=lats_2d, accumulation_area_km2=flow_acc_area) station_to_model_point = cell_manager.get_model_points_for_stations(station_list=stations, nneighbours=8) ##### xx, yy = basemap(lons_2d, lats_2d) upstream_edges = cell_manager.get_upstream_polygons_for_points( model_point_list=list(station_to_model_point.values()), xx=xx, yy=yy) upstream_edges_latlon = cell_manager.get_upstream_polygons_for_points( model_point_list=list(station_to_model_point.values()), xx=lons_2d, yy=lats_2d) plot_utils.draw_upstream_area_bounds(ax, upstream_edges=upstream_edges, color="r", linewidth=0.6) if save_upstream_boundaries_to_shp: plot_utils.save_to_shape_file(upstream_edges_latlon, folder_path="mh/engage_report/upstream_stations_areas/mh_{}".format(grid_config.dx), in_proj=None) basemap.drawrivers(linewidth=0.2) basemap.drawstates(linewidth=0.1) basemap.drawcountries(linewidth=0.1) basemap.drawcoastlines(linewidth=0.2) pos_ids, lons_pos, lats_pos = [], [], [] pos_labels = [] legend_lines = [] for i, (s, mp) in enumerate(sorted(station_to_model_point.items(), key=lambda p: p[0].latitude, reverse=True), start=1): pos_ids.append(s.id) pos_labels.append(i) lons_pos.append(mp.longitude) lats_pos.append(mp.latitude) legend_lines.append("{}: {}".format(i, s.id)) xm, ym = basemap(lons_pos, lats_pos) ax.scatter(xm, ym, c="g", s=20) for txt, x1, y1, pos_label in zip(pos_ids, xm, ym, pos_labels): ax.annotate(pos_label, xy=(x1, y1)) at = AnchoredText("\n".join(legend_lines), prop=dict(size=8), frameon=True, loc=1) at.patch.set_boxstyle("round,pad=0.,rounding_size=0.2") ax.add_artist(at)
def main(): start_year = 1980 end_year = 2010 start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) ids_with_lakes_upstream = [ "104001", "093806", "093801", "081002", "081007", "080718" ] selected_station_ids = [ "092715", "074903", "080104", "081007", "061905", "093806", "090613", "081002", "093801", "080718", "104001" ] selected_station_ids = ids_with_lakes_upstream # Get the list of stations to do the comparison with stations = cehq_station.read_station_data( start_date=start_date, end_date=end_date, selected_ids=selected_station_ids) # add hydat stations # province = "QC" # min_drainage_area_km2 = 10000.0 # stations_hd = cehq_station.load_from_hydat_db(start_date=start_date, end_date=end_date, # province=province, min_drainage_area_km2=min_drainage_area_km2) # if not len(stations_hd): # print "No hydat stations satisying the conditions: period {0}-{1}, province {2}".format( # str(start_date), str(end_date), province # ) # stations.extend(stations_hd) # brewer2mpl.get_map args: set name set type number of colors bmap = brewer2mpl.get_map("Set1", "qualitative", 9) path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5" label1 = "CRCM5-L1" path2 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5" label2 = "CRCM5-L2" color2, color1 = bmap.mpl_colors[:2] fldirs = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(path1) lake_fractions = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_LAKE_FRACTION_NAME) # cell_areas = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_CELL_AREA_NAME) acc_area = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) cell_manager = CellManager(fldirs, lons2d=lons2d, lats2d=lats2d, accumulation_area_km2=acc_area) station_to_mp = cell_manager.get_model_points_for_stations( station_list=stations, lake_fraction=lake_fractions, drainaige_area_reldiff_limit=0.3) fig, axes = plt.subplots(1, 2, gridspec_kw=dict(top=0.80, wspace=0.4)) q90_obs_list = [] q90_mod1_list = [] q90_mod2_list = [] q10_obs_list = [] q10_mod1_list = [] q10_mod2_list = [] for the_station, the_mp in station_to_mp.items(): assert isinstance(the_station, Station) compl_years = the_station.get_list_of_complete_years() if len(compl_years) < 3: continue t, stfl1 = analysis.get_daily_climatology_for_a_point( path=path1, years_of_interest=compl_years, i_index=the_mp.ix, j_index=the_mp.jy, var_name="STFA") _, stfl2 = analysis.get_daily_climatology_for_a_point( path=path2, years_of_interest=compl_years, i_index=the_mp.ix, j_index=the_mp.jy, var_name="STFA") _, stfl_obs = the_station.get_daily_climatology_for_complete_years( stamp_dates=t, years=compl_years) # Q90 q90_obs = np.percentile(stfl_obs, 90) q90_mod1 = np.percentile(stfl1, 90) q90_mod2 = np.percentile(stfl2, 90) # Q10 q10_obs = np.percentile(stfl_obs, 10) q10_mod1 = np.percentile(stfl1, 10) q10_mod2 = np.percentile(stfl2, 10) # save quantiles to lists for correlation calculation q90_obs_list.append(q90_obs) q90_mod1_list.append(q90_mod1) q90_mod2_list.append(q90_mod2) q10_mod1_list.append(q10_mod1) q10_mod2_list.append(q10_mod2) q10_obs_list.append(q10_obs) # axes[0].annotate(the_station.id, (q90_obs, np.percentile(stfl1, 90))) # axes[1].annotate(the_station.id, (q10_obs, np.percentile(stfl1, 10))) # Plot scatter plot of Q90 the_ax = axes[0] # the_ax.annotate(the_station.id, (q90_obs, np.percentile(stfl1, 90))) the_ax.scatter(q90_obs_list, q90_mod1_list, label=label1, c=color1) the_ax.scatter(q90_obs_list, q90_mod2_list, label=label2, c=color2) # plot scatter plot of Q10 the_ax = axes[1] # the_ax.annotate(the_station.id, (q10_obs, np.percentile(stfl1, 10))) h1 = the_ax.scatter(q10_obs_list, q10_mod1_list, label=label1, c=color1) h2 = the_ax.scatter(q10_obs_list, q10_mod2_list, label=label2, c=color2) # Add correlation coefficients to the axes fp = FontProperties(size=14, weight="bold") axes[0].annotate(r"$R^2 = {0:.2f}$".format( np.corrcoef(q90_mod1_list, q90_obs_list)[0, 1]**2), (0.1, 0.85), color=color1, xycoords="axes fraction", font_properties=fp) axes[0].annotate(r"$R^2 = {0:.2f}$".format( np.corrcoef(q90_mod2_list, q90_obs_list)[0, 1]**2), (0.1, 0.70), color=color2, xycoords="axes fraction", font_properties=fp) axes[1].annotate(r"$R^2 = {0:.2f}$".format( np.corrcoef(q10_mod1_list, q10_obs_list)[0, 1]**2), (0.1, 0.85), color=color1, xycoords="axes fraction", font_properties=fp) axes[1].annotate(r"$R^2 = {0:.2f}$".format( np.corrcoef(q10_mod2_list, q10_obs_list)[0, 1]**2), (0.1, 0.70), color=color2, xycoords="axes fraction", font_properties=fp) sf = ScalarFormatter(useMathText=True) sf.set_powerlimits((-2, 3)) for ind, the_ax in enumerate(axes): plot_one_to_one_line(the_ax) if ind == 0: the_ax.set_xlabel(r"Observed $\left({\rm m^3/s} \right)$") the_ax.set_ylabel(r"Modelled $\left({\rm m^3/s} \right)$") the_ax.annotate(r"$Q_{90}$" if ind == 0 else r"$Q_{10}$", (0.95, 0.95), xycoords="axes fraction", bbox=dict(facecolor="white"), va="top", ha="right") the_ax.xaxis.set_major_formatter(sf) the_ax.yaxis.set_major_formatter(sf) locator = MaxNLocator(nbins=5) the_ax.xaxis.set_major_locator(locator) the_ax.yaxis.set_major_locator(locator) x1, x2 = the_ax.get_xlim() # Since streamflow percentiles can only be positive the_ax.set_xlim(0, x2) the_ax.set_ylim(0, x2) fig.legend([h1, h2], [label1, label2], loc="upper center", ncol=2) figpath = os.path.join(images_folder, "percentiles_comparison.png") # plt.tight_layout() fig.savefig(figpath, dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight")
def main(start_year=1980, end_year=1989): soil_layer_widths = infovar.soil_layer_widths_26_to_60 soil_tops = np.cumsum(soil_layer_widths).tolist()[:-1] soil_tops = [ 0, ] + soil_tops selected_station_ids = [ "061905", "074903", "090613", "092715", "093801", "093806" ] # path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf" # label1 = "CRCM5-HCD-RL" path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5" label1 = "CRCM5-HCD-RL-INTFL" path2 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5" label2 = "CRCM5-HCD-RL-INTFL-improved" ############ images_folder = "images_for_lake-river_paper/comp_soil_profiles" if not os.path.isdir(images_folder): os.mkdir(images_folder) fldirs = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(path1) lake_fractions = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_LAKE_FRACTION_NAME) cell_areas = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_CELL_AREA_NAME_M2) acc_areakm2 = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) depth_to_bedrock = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_DEPTH_TO_BEDROCK_NAME) cell_manager = CellManager(fldirs, lons2d=lons2d, lats2d=lats2d, accumulation_area_km2=acc_areakm2) #get climatologic liquid soil moisture and convert fractions to mm t0 = time.clock() daily_dates, levels, i1_nointfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path1, var_name="I1", start_year=start_year, end_year=end_year) print("read I1 - 1") print("Spent {0} seconds ".format(time.clock() - t0)) _, _, i1_intfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path2, var_name="I1", start_year=start_year, end_year=end_year) print("read I1 - 2") #get climatologic frozen soil moisture and convert fractions to mm _, _, i2_nointfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path1, var_name="I2", start_year=start_year, end_year=end_year) print("read I2 - 1") _, _, i2_intfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path2, var_name="I2", start_year=start_year, end_year=end_year) print("read I2 - 2") # sm_intfl = i1_intfl + i2_intfl sm_nointfl = i1_nointfl + i2_nointfl #Get the list of stations to do the comparison with stations = cehq_station.read_station_data( start_date=datetime(start_year, 1, 1), end_date=datetime(end_year, 12, 31), selected_ids=selected_station_ids) print("sm_noinfl, min, max = {0}, {1}".format(sm_nointfl.min(), sm_nointfl.max())) print("sm_infl, min, max = {0}, {1}".format(sm_intfl.min(), sm_intfl.max())) diff = (sm_intfl - sm_nointfl) #diff *= soil_layer_widths[np.newaxis, :, np.newaxis, np.newaxis] * 1000 # to convert in mm #print "number of nans", np.isnan(diff).astype(int).sum() print("cell area min,max = {0}, {1}".format(cell_areas.min(), cell_areas.max())) print("acc area min,max = {0}, {1}".format(acc_areakm2.min(), acc_areakm2.max())) assert np.all(lake_fractions >= 0) print("lake fractions (min, max): ", lake_fractions.min(), lake_fractions.max()) #Non need to go very deep nlayers = 3 z, t = np.meshgrid(soil_tops[:nlayers], date2num(daily_dates)) station_to_mp = cell_manager.get_model_points_for_stations(stations) plotted_global = False for the_station, mp in station_to_mp.items(): assert isinstance(mp, ModelPoint) assert isinstance(the_station, Station) fig = plt.figure() umask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices( mp.ix, mp.jy) #exclude lake cells from the profiles sel = (umask == 1) & (depth_to_bedrock > 3) & (acc_areakm2 >= 0) umaskf = umask.astype(float) umaskf *= (1.0 - lake_fractions) * cell_areas umaskf[~sel] = 0.0 profiles = np.tensordot(diff, umaskf) / umaskf.sum() print(profiles.shape, profiles.min(), profiles.max(), umaskf.sum(), umaskf.min(), umaskf.max()) d = np.abs(profiles).max() print("d = {0}".format(d)) clevs = np.round(np.linspace(-d, d, 12), decimals=5) diff_cmap = cm.get_cmap("RdBu_r", lut=len(clevs) - 1) bn = BoundaryNorm(clevs, len(clevs) - 1) plt.title("({})-({})".format(label2, label2)) img = plt.contourf(t, z, profiles[:, :nlayers], cmap=diff_cmap, levels=clevs, norm=bn) plt.colorbar(img, ticks=clevs) ax = plt.gca() assert isinstance(ax, Axes) ax.invert_yaxis() ax.xaxis.set_major_formatter(DateFormatter("%b")) ax.xaxis.set_major_locator(MonthLocator()) fig.savefig(os.path.join( images_folder, "{0}_{1}_{2}.jpeg".format(the_station.id, label1, label2)), dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight") print("processed: {0}".format(the_station)) if not plotted_global: plotted_global = True fig = plt.figure() sel = (depth_to_bedrock >= 0.1) & (acc_areakm2 >= 0) umaskf = (1.0 - lake_fractions) * cell_areas umaskf[~sel] = 0.0 profiles = np.tensordot(diff, umaskf) / umaskf.sum() print(profiles.shape, profiles.min(), profiles.max(), umaskf.sum(), umaskf.min(), umaskf.max()) d = np.abs(profiles).max() print("d = {0}".format(d)) clevs = np.round(np.linspace(-d, d, 12), decimals=5) diff_cmap = cm.get_cmap("RdBu_r", lut=len(clevs) - 1) bn = BoundaryNorm(clevs, len(clevs) - 1) img = plt.contourf(t, z, profiles[:, :nlayers], cmap=diff_cmap, levels=clevs, norm=bn) plt.colorbar(img, ticks=clevs) ax = plt.gca() assert isinstance(ax, Axes) ax.invert_yaxis() ax.xaxis.set_major_formatter(DateFormatter("%b")) ax.xaxis.set_major_locator(MonthLocator()) fig.savefig(os.path.join(images_folder, "global_mean.jpeg"), dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight") pass
def main(): model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Diagnostics") # model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Samples") static_data_file = "/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p" r = RPN(static_data_file) fldir = r.get_first_record_for_name("FLDR") faa = r.get_first_record_for_name("FAA") lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec() gc = default_domains.bc_mh_044 cell_manager = CellManager(fldir, nx=fldir.shape[0], ny=fldir.shape[1], lons2d=lons, lats2d=lats, accumulation_area_km2=faa) selected_station_ids = ["06EA002", ] stations = cehq_station.load_from_hydat_db(province="SK", selected_ids=selected_station_ids, natural=None) # (06EA002): CHURCHILL RIVER AT SANDY BAY at (-102.31832885742188,55.52333068847656), accum. area is 212000.0 km**2 # TODO: plot where is this station, compare modelled and observed hydrographs # for s in stations: # assert isinstance(s, cehq_station.Station) # s.latitude += 0.9 # s.longitude -= 0.2 # print(s) station_to_model_point = cell_manager.get_model_points_for_stations(stations, drainaige_area_reldiff_limit=0.8, nneighbours=1) print(station_to_model_point[stations[0]]) station = stations[0] assert isinstance(station, cehq_station.Station) obs_not_corrected = pd.Series(index=station.dates, data=station.values).groupby( by=lambda d: d.replace(day=15)).mean() obs_corrected = pd.read_csv("mh/obs_data/Churchill Historic Monthly Apportionable Flow_06EA002.csv.bak.original", skiprows=2) print(obs_corrected.head()) print(obs_corrected.year.iloc[0], obs_corrected.year.iloc[-1]) date_index = pd.date_range(start=datetime(obs_corrected.year.iloc[0] - 1, 12, 15), end=datetime(obs_corrected.year.iloc[-1], 12, 15), freq="M") date_index = date_index.shift(15, freq=pd.datetools.day) print(date_index) data = np.concatenate([r for r in obs_corrected.values[:, 1:-1]]) factor = date_index.map(lambda d: 1000 / (calendar.monthrange(d.year, d.month)[1] * 24 * 3600)) print(factor[:10]) obs_corrected = pd.Series(index=date_index, data=data * factor) station_to_modelled_data = get_model_data(station_to_model_point, output_path=model_data_path, grid_config=gc, basins_of_interest_shp=default_domains.MH_BASINS_PATH, cell_manager=cell_manager, vname="STFL") modelled_data = station_to_modelled_data[station] fig = plt.figure() ax = obs_corrected.plot(label="obs corrected") obs_not_corrected.plot(label="obs not corrected", ax=ax, color="k") modelled_data.plot(label="CRCM5", ax=ax, color="r") ax.legend(loc="upper left") img_file = img_folder.joinpath("{}_validation_monthly.png".format(station.id)) fig.savefig(str(img_file)) plt.close(fig) # climatology start_year = 1980 end_year = 2010 date_selector = lambda d: (start_year <= d.year <= end_year) and not ((d.month == 2) and (d.day == 29)) fig = plt.figure() ax = obs_corrected.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(label="obs corrected") obs_not_corrected.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot( label="obs not corrected", ax=ax, color="k") modelled_data.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(label="CRCM5", ax=ax, color="r") ax.xaxis.set_major_locator(MonthLocator(bymonthday=15)) ax.xaxis.set_major_formatter(DateFormatter("%b")) ax.legend(loc="upper left") img_file = img_folder.joinpath("{}_validation_clim.png".format(station.id)) fig.savefig(str(img_file)) plt.close(fig) # Interannual variability fig = plt.figure() obs_corrected = obs_corrected.select(lambda d: start_year <= d.year <= end_year) modelled_data = modelled_data.select(lambda d: start_year <= d.year <= end_year) corr_list = [] for m in range(1, 13): obs = obs_corrected.select(lambda d: d.month == m) mod = modelled_data.select(lambda d: d.month == m) print(obs.head()) obs.index = obs.index.map(lambda d: d.year) mod.index = mod.index.map(lambda d: d.year) corr_list.append(obs.corr(mod)) ax = plt.gca() ax.plot(range(1, 13), corr_list) ax.set_xlabel("Month") ax.set_title("Inter-annual variability") img_file = img_folder.joinpath("{}_interannual.png".format(station.id)) fig.tight_layout() fig.savefig(str(img_file), bbox_inches="tight") plt.close(fig)
def main(hdf_folder="/home/huziy/skynet3_rech1/hdf_store", start_year=1980, end_year=2010): prepare() all_markers = ["*", "s", "p", "+", "x", "d", "h"] excluded = ["white", "w", "aliceblue", "azure"] excluded.extend([ci for ci in colors.cnames if "yellow" in ci]) all_colors = ["k", "b", "r", "g", "m"] + sorted([ci for ci in colors.cnames if ci not in excluded]) # Station ids to get from the CEHQ database ids_with_lakes_upstream = [ "104001", "093806", "093801", "081002", "081007", "080718" ] selected_ids = ids_with_lakes_upstream filedir = Path(hdf_folder) sim_name_to_file_path = OrderedDict([ # ("CRCM5-LI", filedir.joinpath("quebec_0.1_crcm5-hcd-r.hdf5").as_posix()), ("ERAI-CRCM5-L", filedir.joinpath("quebec_0.1_crcm5-hcd-rl.hdf5").as_posix()), # ("CanESM2-CRCM5-NL", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-r-cc-canesm2-1980-2010.hdf5").as_posix()), ("CanESM2-CRCM5-L", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5").as_posix()), # ("CanESM2-CRCM5-LI", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-intfl-cc-canesm2-1980-2010.hdf5").as_posix()), ]) obs_label = "Obs." labels = [obs_label, ] + list(sim_name_to_file_path.keys()) label_to_marker = dict(zip(labels, all_markers)) label_to_color = dict(zip(labels, all_colors)) # Get the list of stations to do the comparison with start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) stations = cehq_station.read_station_data( start_date=start_date, end_date=end_date, selected_ids=selected_ids ) # Get geophysical fields from one of the model simulations path0 = list(sim_name_to_file_path.values())[0] lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(file_path=path0) flow_directions = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) lake_fraction = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_LAKE_FRACTION_NAME) accumulation_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) area_m2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_M2) # Try to read cell areas im meters if it is not Ok then try in km2 if area_m2 is not None: cell_area_km2 = area_m2 * 1.0e-6 else: cell_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_KM2) # Create a cell manager if it is not provided cell_manager = CellManager(flow_directions, accumulation_area_km2=accumulation_area_km2, lons2d=lons2d, lats2d=lats2d) # Get the list of the corresponding model points station_to_modelpoint = cell_manager.get_model_points_for_stations( station_list=stations, lake_fraction=lake_fraction, drainaige_area_reldiff_limit=0.1) # plot_utils.apply_plot_params(font_size=10, width_cm=20, height_cm=18) fig = plt.figure() ncols = max([len(rp_list) for et, rp_list in ExtremeProperties.extreme_type_to_return_periods.items()]) nrows = len(ExtremeProperties.extreme_types) gs = GridSpec(nrows, ncols) ext_type_to_rp_to_ax = OrderedDict() ax_with_legend = None label_to_ax_to_xdata = {} label_to_ax_to_ydata = {} for row, ext_type in enumerate(ExtremeProperties.extreme_types): ext_type_to_rp_to_ax[ext_type] = OrderedDict() for col, rperiod in enumerate(ExtremeProperties.extreme_type_to_return_periods[ext_type]): ax = fig.add_subplot(gs[row, col]) ext_type_to_rp_to_ax[ext_type][rperiod] = ax if col == 0: ax.set_ylabel(ext_type) if row == nrows - 1 and col == ncols - 1: ax_with_legend = ax # Set axes labels if row == nrows - 1: ax.set_xlabel("Observations") if col == 0: ax.set_ylabel("Model") for label in sim_name_to_file_path: if label not in label_to_ax_to_xdata: label_to_ax_to_xdata[label] = {ax: []} label_to_ax_to_ydata[label] = {ax: []} else: label_to_ax_to_xdata[label][ax] = [] label_to_ax_to_ydata[label][ax] = [] ax.set_xscale("log") ax.set_yscale("log") print("Initial list of stations:") sim_label_to_handle = {} for s in stations: print("{0}".format(s)) assert isinstance(s, Station) print(len([y for y in s.get_list_of_complete_years() if start_year <= y <= end_year])) df_ext_obs = extreme_commons.get_annual_extrema(ts_times=s.dates, ts_vals=s.values, start_year=start_year, end_year=end_year) mp = station_to_modelpoint[s] assert isinstance(mp, ModelPoint) years_of_interest = df_ext_obs.index label_to_extrema_model = {} # label -> ext_type -> [return period -> ret level, return period -> std] label_to_return_levels = OrderedDict( [(obs_label, OrderedDict())] ) for sim_label, sim_path in sim_name_to_file_path.items(): label_to_return_levels[sim_label] = OrderedDict() label_to_extrema_model[sim_label] = OrderedDict() # Calculate the return levels and standard deviations for ext_type in ExtremeProperties.extreme_types: return_periods = ExtremeProperties.extreme_type_to_return_periods[ext_type] # fit GEV distribution and apply non-parametric bootstrap to get std label_to_return_levels[obs_label][ext_type] = gevfit.do_gevfit_for_a_point(df_ext_obs[ext_type].values, extreme_type=ext_type, return_periods=return_periods) return_levels_obs, rl_stds_obs = label_to_return_levels[obs_label][ext_type] # get annual extremas for the model output at the points colose to the stations for sim_label, sim_path in sim_name_to_file_path.items(): label_to_return_levels[sim_label] = OrderedDict() ext_field = analysis.get_annual_extrema( rconfig=RunConfig(data_path=sim_path, start_year=start_year, end_year=end_year), varname="STFL", months_of_interest=ExtremeProperties.extreme_type_to_month_of_interest[ext_type], n_avg_days=ExtremeProperties.extreme_type_to_n_agv_days[ext_type], high_flow=ext_type == ExtremeProperties.high) # Select only those years when obs are available ts_data = [v for y, v in zip(range(start_year, end_year + 1), ext_field[:, mp.ix, mp.jy]) if y in years_of_interest] ts_data = np.array(ts_data) return_levels, rl_stds = gevfit.do_gevfit_for_a_point(ts_data, extreme_type=ext_type, return_periods=return_periods) # Do the plotting for rp in return_periods: ax = ext_type_to_rp_to_ax[ext_type][rp] ax.set_title("T = {rp}-year".format(rp=rp)) # h = ax.errorbar(return_levels_obs[rp], return_levels[rp], # marker=label_to_marker[sim_label], color=label_to_color[sim_label], label=sim_label, # xerr=rl_stds_obs[rp] * 1.96, yerr=rl_stds[rp] * 1.96) h = ax.scatter(return_levels_obs[rp], return_levels[rp], marker=label_to_marker[sim_label], color=label_to_color[sim_label], label=sim_label) # save the data for maybe further calculation of the correlation coefficients label_to_ax_to_xdata[sim_label][ax].append(return_levels_obs[rp]) label_to_ax_to_ydata[sim_label][ax].append(return_levels[rp]) sim_label_to_handle[sim_label] = h # Calculate the biases for sim_label in sim_name_to_file_path: for ext_type in ExtremeProperties.extreme_types: ret_periods = ExtremeProperties.extreme_type_to_return_periods[ext_type] for rp in ret_periods: ax = ext_type_to_rp_to_ax[ext_type][rp] mod = np.asarray(label_to_ax_to_ydata[sim_label][ax]) obs = np.asarray(label_to_ax_to_xdata[sim_label][ax]) bias = np.mean((mod - obs)/obs) corr, pv = stats.pearsonr(mod, obs) print("({sim_label}) Mean bias for {rp}-year {ext_type}-flow return level is: {bias}; corr={corr:.2f}; corr_pval={corr_pval:2g}".format( sim_label=sim_label, rp=rp, bias=bias, corr=corr, corr_pval=pv, ext_type=ext_type )) sfmt = ScalarFormatter(useMathText=True) sfmt.set_powerlimits((-2, 2)) for et, rp_to_ax in ext_type_to_rp_to_ax.items(): for rp, ax in rp_to_ax.items(): xmin, xmax = ax.get_xlim() ymin, ymax = ax.get_ylim() x1 = min(xmin, ymin) x2 = min(xmax, ymax) ax.plot([x1, x2], [x1, x2], "k--") # ax.xaxis.set_major_locator(MaxNLocator(nbins=5)) # ax.yaxis.set_major_locator(MaxNLocator(nbins=5)) # ax.xaxis.set_major_formatter(sfmt) # ax.yaxis.set_major_formatter(sfmt) sim_labels = list(sim_name_to_file_path.keys()) ax_with_legend.legend([sim_label_to_handle[sl] for sl in sim_labels], sim_labels, bbox_to_anchor=(1, -0.25), borderaxespad=0.0, loc="upper right", ncol=2, scatterpoints=1, numpoints=1) # Save the plot img_file = "{}.eps".format("_".join(sorted(label_to_marker.keys()))) img_file = img_folder.joinpath(img_file) fig.tight_layout() with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight")