def main(): #url = "http://maps.googleapis.com/maps/api/staticmap?center=46.5,-72&zoom=15&size=800x800&maptype=satellite&sensor=false" params = { "center": "46.5,-72", "zoom": "5", "size": "800x800", "maptype": "satellite", "sensor": "false" } script = "http://maps.googleapis.com/maps/api/staticmap?" print(urllib.parse.urlencode(params)) url = script + urllib.parse.urlencode(params) stations = cehq_station.read_station_data(folder="data/cehq_levels")[:5] for s in stations: assert isinstance(s, cehq_station.Station) url += "&" + urllib.parse.urlencode({ "markers": "color:red|label:%s|%f,%f" % (s.id, s.latitude, s.longitude) }) print(url) urllib.request.urlretrieve(url, filename="gmap.png") #TODO: implement pass
def regenerate_station_to_gridcell_mapping(start_year, end_year, model_manager): """ should be called when grid or search algorithm change """ assert isinstance(model_manager, Crcm5ModelDataManager) ktree = model_manager.kdtree model_acc_area = model_manager.accumulation_area_km2 model_acc_area_1d = model_acc_area.flatten() # selected_ids = ["104001", "103715", # "093806", "093801", # "092715", # "081006", "040830"] selected_ids = None start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date) station_to_grid_point = {} for s in stations: assert isinstance(s, Station) x, y, z = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) dists, inds = ktree.query((x, y, z), k=8) deltaDaMin = np.min(np.abs(model_acc_area_1d[inds] - s.drainage_km2)) imin = np.where( np.abs(model_acc_area_1d[inds] - s.drainage_km2) == deltaDaMin)[0] deltaDa2D = np.abs(model_acc_area - s.drainage_km2) ij = np.where(deltaDa2D == deltaDaMin) mp = ModelPoint() mp.accumulation_area = model_acc_area[ij[0][0], ij[1][0]] mp.ix = ij[0][0] mp.jy = ij[1][0] mp.longitude = model_manager.lons2D[mp.ix, mp.jy] mp.latitude = model_manager.lats2D[mp.ix, mp.jy] #flow in mask mp.flow_in_mask = model_manager.get_mask_for_cells_upstream( mp.ix, mp.jy) station_to_grid_point[s] = mp print("da_diff (sel) = ", deltaDaMin) print("dist (sel) = ", dists[imin]) return station_to_grid_point
def main_for_cc_paper(start_date=None, end_date=None): # Station ids to get from the CEHQ database ids_with_lakes_upstream = [ "104001", "093806", "093801", "081002", "081007", "080718" ] # for a presentation # ids_with_lakes_upstream = [ # "104001", "093806", "081002", "081007", # ] selected_ids = ids_with_lakes_upstream sim_labels = [ # "CRCM5-R", # "CRCM5-L2", "ERAI-CRCM5-L", "CanESM2-CRCM5-L" # "CRCM5-HCD-RL-INTF-a" ] sim_file_names = [ "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5", "/RESCUE/skynet3_rech1/huziy/hdf_store/cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5" ] color_list = ["dodgerblue", "r", "g"] sim_name_to_color = OrderedDict([ ("Obs.", "k") ]) for sim_name, the_color in zip(sim_labels, color_list): sim_name_to_color[sim_name] = the_color sim_name_to_file_name = OrderedDict(zip(sim_labels, sim_file_names)) # Get the list of stations to do the comparison with stations = cehq_station.read_station_data( start_date=start_date, end_date=end_date, selected_ids=selected_ids ) print("Initial list of stations:") for s in stations: print("{0}".format(s)) plot_utils.apply_plot_params(font_size=16, width_pt=None, width_cm=25, height_cm=12) draw_model_comparison(model_points=None, sim_name_to_file_name=sim_name_to_file_name, hdf_folder=None, start_year=start_date.year, end_year=end_date.year, stations=stations, stfl_name="STFL", drainage_area_reldiff_min=0.1, plot_upstream_area_averaged=False, sim_name_to_color=sim_name_to_color)
def main(): start_date = datetime(1985,1,1) end_date = datetime(1990,12, 31) stations = cehq_station.read_station_data(start_date = start_date, end_date=end_date) print("Read {0} stations".format(len(stations))) for s in stations: if not len(s.dates): print(s.id, "=>", len(s.dates), "from: -- to -- ") continue print(s.id, "=>", len(s.dates), "from: ", s.dates[0], " to ", s.dates[-1]) pass
def main(): model_file = "" selected_ids = [ ] #Get the list of stations to plot stations = cehq_station.read_station_data( start_date=None, end_date=None, selected_ids=selected_ids )
def main(): data_dir = "data/cehq_data/MontrealFlood2017_station_data/streamflow/daily" basin_shapes = [ "/RESCUE/skynet3_rech1/huziy/Netbeans Projects/Python/RPN/data/shp/mtl_flood_2017_basins/02JKL_SDA_Ottawa.shp", #"/RESCUE/skynet3_rech1/huziy/Netbeans Projects/Python/RPN/data/shp/mtl_flood_2017_basins/02MOP_SDA_St_Lawrence.shp", # "/RESCUE/skynet3_rech1/huziy/Netbeans Projects/Python/RPN/data/shp/mtl_flood_2017_basins/02N_SDA_St_Maurice.shp" ] selected_ids = [""] excluded_ids = ["120201"] start_date = datetime(1980, 1, 1) end_date = datetime(2017, 6, 1) stations = cehq_station.read_station_data(start_date=start_date, end_date=end_date, folder=data_dir, min_number_of_complete_years=0, only_natural=None) for s in stations: assert isinstance(s, Station) print(s) print(s.get_list_of_complete_years()) stations = [s for s in stations if s.id not in excluded_ids] plot_utils.apply_plot_params(font_size=10) img_file = "stfl_station_positions.png" img_file = commons.img_folder / img_file plot_station_positions(stations, img_file=str(img_file), shp_paths=basin_shapes, min_lon=-81.5, max_lon=-70, min_lat=43.5, max_lat=49) pass
def main(): # url = "http://maps.googleapis.com/maps/api/staticmap?center=46.5,-72&zoom=15&size=800x800&maptype=satellite&sensor=false" params = {"center": "46.5,-72", "zoom": "5", "size": "800x800", "maptype": "satellite", "sensor": "false"} script = "http://maps.googleapis.com/maps/api/staticmap?" print(urllib.parse.urlencode(params)) url = script + urllib.parse.urlencode(params) stations = cehq_station.read_station_data(folder="data/cehq_levels")[:5] for s in stations: assert isinstance(s, cehq_station.Station) url += "&" + urllib.parse.urlencode({"markers": "color:red|label:%s|%f,%f" % (s.id, s.latitude, s.longitude)}) print(url) urllib.request.urlretrieve(url, filename="gmap.png") # TODO: implement pass
def main(hdf_folder="/home/huziy/skynet3_rech1/hdf_store", start_date=None, end_date=None): # Station ids to get from the CEHQ database # selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905", # "041903", "040830", "093806", "090613", "081002", "093801", "080718"] selected_ids = ["092715", "074903", "080104", "081007", "061905", "093806", "090613", "081002", "093801", "080718", "104001"] # selected_ids = [ # "074903", "061905", "090613", "092715", "093801", "093806", "081002" # ] # selected_ids = ["081002", "104001"] # selected_ids = ["090613", ] sim_labels = [ "CRCM5-L2", ] sim_file_names = [ "quebec_0.1_crcm5-hcd-rl.hdf5", ] sim_name_to_file_name = OrderedDict() for k, v in zip(sim_labels, sim_file_names): sim_name_to_file_name[k] = v #sim_name_to_file_name = { #"CRCM5-R": "quebec_0.1_crcm5-r.hdf5", #"CRCM5-HCD-R": "quebec_0.1_crcm5-hcd-r.hdf5", #"CRCM5-HCD-RL": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf", #"CRCM5-HCD-RL-INTFL": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf", #"SANI=10000, ignore THFC": # "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc.hdf", #"CRCM5-HCD-RL-ERA075": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075.hdf", #"SANI=10000": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000.hdf" #"CRCM5-HCD-RL-ECOCLIMAP": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap.hdf" #} skip_list = [ "061303", # regulated "061305", "061304", "051012", "050437", "050914", "030247", "030268" ] # Get the list of stations to do the comparison with stations = cehq_station.read_station_data( folder="/home/huziy/skynet3_rech1/CEHQ", start_date=start_date, end_date=end_date, min_number_of_complete_years=1 ) # stations.extend( # cehq_station.load_from_hydat_db(start_date=start_date, end_date=end_date, datavariable="level") # ) # Commented hydat station for performance during testing # province = "QC" # stations_hd = cehq_station.load_from_hydat_db(start_date=start_date, end_date=end_date, province=province) # if not len(stations_hd): # print "No hydat stations satisying the conditions: period {0}-{1}, province {2}".format( # str(start_date), str(end_date), province # ) # stations.extend(stations_hd) # province = "ON" # stations_hd = cehq_station.load_from_hydat_db(start_date=start_date, end_date=end_date, province=province) # stations.extend(stations_hd) # Do not process skip stations stations = [the_s for the_s in stations if the_s.id not in skip_list] # debug: for s in stations: assert isinstance(s, Station) print(s.get_list_of_complete_years()) print(s.drainage_km2) assert len(stations) draw_model_comparison(model_points=None, sim_name_to_file_name=sim_name_to_file_name, hdf_folder=hdf_folder, start_year=start_date.year, end_year=end_date.year, stations=stations, plot_upstream_averages=False)
def main(): start_year = 1980 end_year = 2010 start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) ids_with_lakes_upstream = [ "104001", "093806", "093801", "081002", "081007", "080718" ] selected_station_ids = ["092715", "074903", "080104", "081007", "061905", "093806", "090613", "081002", "093801", "080718", "104001"] selected_station_ids = ids_with_lakes_upstream # Get the list of stations to do the comparison with stations = cehq_station.read_station_data( start_date=start_date, end_date=end_date, selected_ids=selected_station_ids ) # add hydat stations # province = "QC" # min_drainage_area_km2 = 10000.0 # stations_hd = cehq_station.load_from_hydat_db(start_date=start_date, end_date=end_date, # province=province, min_drainage_area_km2=min_drainage_area_km2) # if not len(stations_hd): # print "No hydat stations satisying the conditions: period {0}-{1}, province {2}".format( # str(start_date), str(end_date), province # ) # stations.extend(stations_hd) # brewer2mpl.get_map args: set name set type number of colors bmap = brewer2mpl.get_map("Set1", "qualitative", 9) path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5" label1 = "CRCM5-L1" path2 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5" label2 = "CRCM5-L2" color2, color1 = bmap.mpl_colors[:2] fldirs = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(path1) lake_fractions = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_LAKE_FRACTION_NAME) # cell_areas = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_CELL_AREA_NAME) acc_area = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) cell_manager = CellManager(fldirs, lons2d=lons2d, lats2d=lats2d, accumulation_area_km2=acc_area) station_to_mp = cell_manager.get_model_points_for_stations(station_list=stations, lake_fraction=lake_fractions, drainaige_area_reldiff_limit=0.3) fig, axes = plt.subplots(1, 2, gridspec_kw=dict(top=0.80, wspace=0.4)) q90_obs_list = [] q90_mod1_list = [] q90_mod2_list = [] q10_obs_list = [] q10_mod1_list = [] q10_mod2_list = [] for the_station, the_mp in station_to_mp.items(): assert isinstance(the_station, Station) compl_years = the_station.get_list_of_complete_years() if len(compl_years) < 3: continue t, stfl1 = analysis.get_daily_climatology_for_a_point(path=path1, years_of_interest=compl_years, i_index=the_mp.ix, j_index=the_mp.jy, var_name="STFA") _, stfl2 = analysis.get_daily_climatology_for_a_point(path=path2, years_of_interest=compl_years, i_index=the_mp.ix, j_index=the_mp.jy, var_name="STFA") _, stfl_obs = the_station.get_daily_climatology_for_complete_years(stamp_dates=t, years=compl_years) # Q90 q90_obs = np.percentile(stfl_obs, 90) q90_mod1 = np.percentile(stfl1, 90) q90_mod2 = np.percentile(stfl2, 90) # Q10 q10_obs = np.percentile(stfl_obs, 10) q10_mod1 = np.percentile(stfl1, 10) q10_mod2 = np.percentile(stfl2, 10) # save quantiles to lists for correlation calculation q90_obs_list.append(q90_obs) q90_mod1_list.append(q90_mod1) q90_mod2_list.append(q90_mod2) q10_mod1_list.append(q10_mod1) q10_mod2_list.append(q10_mod2) q10_obs_list.append(q10_obs) # axes[0].annotate(the_station.id, (q90_obs, np.percentile(stfl1, 90))) # axes[1].annotate(the_station.id, (q10_obs, np.percentile(stfl1, 10))) # Plot scatter plot of Q90 the_ax = axes[0] # the_ax.annotate(the_station.id, (q90_obs, np.percentile(stfl1, 90))) the_ax.scatter(q90_obs_list, q90_mod1_list, label=label1, c=color1) the_ax.scatter(q90_obs_list, q90_mod2_list, label=label2, c=color2) # plot scatter plot of Q10 the_ax = axes[1] # the_ax.annotate(the_station.id, (q10_obs, np.percentile(stfl1, 10))) h1 = the_ax.scatter(q10_obs_list, q10_mod1_list, label=label1, c=color1) h2 = the_ax.scatter(q10_obs_list, q10_mod2_list, label=label2, c=color2) # Add correlation coefficients to the axes fp = FontProperties(size=14, weight="bold") axes[0].annotate(r"$R^2 = {0:.2f}$".format(np.corrcoef(q90_mod1_list, q90_obs_list)[0, 1] ** 2), (0.1, 0.85), color=color1, xycoords="axes fraction", font_properties=fp) axes[0].annotate(r"$R^2 = {0:.2f}$".format(np.corrcoef(q90_mod2_list, q90_obs_list)[0, 1] ** 2), (0.1, 0.70), color=color2, xycoords="axes fraction", font_properties=fp) axes[1].annotate(r"$R^2 = {0:.2f}$".format(np.corrcoef(q10_mod1_list, q10_obs_list)[0, 1] ** 2), (0.1, 0.85), color=color1, xycoords="axes fraction", font_properties=fp) axes[1].annotate(r"$R^2 = {0:.2f}$".format(np.corrcoef(q10_mod2_list, q10_obs_list)[0, 1] ** 2), (0.1, 0.70), color=color2, xycoords="axes fraction", font_properties=fp) sf = ScalarFormatter(useMathText=True) sf.set_powerlimits((-2, 3)) for ind, the_ax in enumerate(axes): plot_one_to_one_line(the_ax) if ind == 0: the_ax.set_xlabel(r"Observed $\left({\rm m^3/s} \right)$") the_ax.set_ylabel(r"Modelled $\left({\rm m^3/s} \right)$") the_ax.annotate(r"$Q_{90}$" if ind == 0 else r"$Q_{10}$", (0.95, 0.95), xycoords="axes fraction", bbox=dict(facecolor="white"), va="top", ha="right") the_ax.xaxis.set_major_formatter(sf) the_ax.yaxis.set_major_formatter(sf) locator = MaxNLocator(nbins=5) the_ax.xaxis.set_major_locator(locator) the_ax.yaxis.set_major_locator(locator) x1, x2 = the_ax.get_xlim() # Since streamflow percentiles can only be positive the_ax.set_xlim(0, x2) the_ax.set_ylim(0, x2) fig.legend([h1, h2], [label1, label2], loc="upper center", ncol=2) figpath = os.path.join(images_folder, "percentiles_comparison.png") # plt.tight_layout() fig.savefig(figpath, dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight")
def plot_at_indices(ix,jy): var_name_liquid = "I1" var_name_solid = "I2" #peirod of interest start_year = 1979 end_year = 1988 #simulation names corresponding to the paths sim_names = ["crcm5-hcd-rl", "crcm5-hcd-rl-intfl"] sim_labels = [x.upper() for x in sim_names] layer_widths = [0.1, 0.2, 0.3, 0.4, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0] layer_depths = np.cumsum(layer_widths) paths = [ "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup", "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup2/Samples_all_in_one" ] managers = [ Crcm5ModelDataManager(samples_folder_path=path, file_name_prefix="pm", all_files_in_samples_folder=True, need_cell_manager= (i == 0)) for i, path in enumerate(paths) ] #share the cell manager a_data_manager = managers[0] assert isinstance(a_data_manager, Crcm5ModelDataManager) cell_manager = a_data_manager.cell_manager assert isinstance(cell_manager, CellManager) for m in managers[1:]: assert isinstance(m, Crcm5ModelDataManager) m.cell_manager = cell_manager #share the lake fraction field lake_fraction = a_data_manager.lake_fraction selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905", "041903", "040830", "093806", "090613", "081002", "093801", "080718"] start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) stations = cehq_station.read_station_data(selected_ids = selected_ids, start_date=start_date, end_date=end_date ) #stations with corresponding model points station_to_mp = a_data_manager.get_dataless_model_points_for_stations(stations) #figure out levels in soil sim_label_to_profiles = {} fig = plt.figure() fmt = ScalarFormatter(useMathText=True) fmt.set_powerlimits([-2, 2]) for m, label in zip(managers, sim_labels): assert isinstance(m, Crcm5ModelDataManager) monthly_means_liquid = _get_cached_monthly_mean_fields(label, start_year, end_year, var_name_liquid) if monthly_means_liquid is None: monthly_means_liquid = m.get_monthly_climatology_of_3d_field(var_name=var_name_liquid, start_year=start_year , end_year=end_year) _cache_monthly_mean_fields(monthly_means_liquid, label, start_year, end_year, var_name_liquid) monthly_means_solid = _get_cached_monthly_mean_fields(label, start_year, end_year, var_name_solid) if monthly_means_solid is None: monthly_means_solid = m.get_monthly_climatology_of_3d_field(var_name=var_name_solid, start_year=start_year, end_year=end_year) _cache_monthly_mean_fields(monthly_means_solid, label, start_year, end_year, var_name_solid) profiles = [monthly_means_liquid[i][ix,jy, :] + monthly_means_solid[i][ix, jy, :] for i in range(12)] sim_label_to_profiles[label] = np.array(profiles) x = list(range(12)) y = layer_depths y2d, x2d = np.meshgrid(y, x) plt.contourf(x2d, y2d, sim_label_to_profiles[sim_labels[1]] - sim_label_to_profiles[sim_labels[0]]) plt.gca().invert_yaxis() plt.colorbar() #fig.tight_layout() fig.savefig("soil_profile_at_ix={0};jy={1}.pdf".format(ix, jy))
def main(hdf_folder="/home/huziy/skynet3_rech1/hdf_store", start_year=1980, end_year=2010): prepare() all_markers = ["*", "s", "p", "+", "x", "d", "h"] excluded = ["white", "w", "aliceblue", "azure"] excluded.extend([ci for ci in colors.cnames if "yellow" in ci]) all_colors = ["k", "b", "r", "g", "m"] + sorted([ci for ci in colors.cnames if ci not in excluded]) # Station ids to get from the CEHQ database ids_with_lakes_upstream = [ "104001", "093806", "093801", "081002", "081007", "080718" ] selected_ids = ids_with_lakes_upstream filedir = Path(hdf_folder) sim_name_to_file_path = OrderedDict([ # ("CRCM5-LI", filedir.joinpath("quebec_0.1_crcm5-hcd-r.hdf5").as_posix()), ("ERAI-CRCM5-L", filedir.joinpath("quebec_0.1_crcm5-hcd-rl.hdf5").as_posix()), # ("CanESM2-CRCM5-NL", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-r-cc-canesm2-1980-2010.hdf5").as_posix()), ("CanESM2-CRCM5-L", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5").as_posix()), # ("CanESM2-CRCM5-LI", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-intfl-cc-canesm2-1980-2010.hdf5").as_posix()), ]) obs_label = "Obs." labels = [obs_label, ] + list(sim_name_to_file_path.keys()) label_to_marker = dict(zip(labels, all_markers)) label_to_color = dict(zip(labels, all_colors)) # Get the list of stations to do the comparison with start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) stations = cehq_station.read_station_data( start_date=start_date, end_date=end_date, selected_ids=selected_ids ) # Get geophysical fields from one of the model simulations path0 = list(sim_name_to_file_path.values())[0] lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(file_path=path0) flow_directions = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) lake_fraction = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_LAKE_FRACTION_NAME) accumulation_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) area_m2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_M2) # Try to read cell areas im meters if it is not Ok then try in km2 if area_m2 is not None: cell_area_km2 = area_m2 * 1.0e-6 else: cell_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_KM2) # Create a cell manager if it is not provided cell_manager = CellManager(flow_directions, accumulation_area_km2=accumulation_area_km2, lons2d=lons2d, lats2d=lats2d) # Get the list of the corresponding model points station_to_modelpoint = cell_manager.get_model_points_for_stations( station_list=stations, lake_fraction=lake_fraction, drainaige_area_reldiff_limit=0.1) # plot_utils.apply_plot_params(font_size=10, width_cm=20, height_cm=18) fig = plt.figure() ncols = max([len(rp_list) for et, rp_list in ExtremeProperties.extreme_type_to_return_periods.items()]) nrows = len(ExtremeProperties.extreme_types) gs = GridSpec(nrows, ncols) ext_type_to_rp_to_ax = OrderedDict() ax_with_legend = None label_to_ax_to_xdata = {} label_to_ax_to_ydata = {} for row, ext_type in enumerate(ExtremeProperties.extreme_types): ext_type_to_rp_to_ax[ext_type] = OrderedDict() for col, rperiod in enumerate(ExtremeProperties.extreme_type_to_return_periods[ext_type]): ax = fig.add_subplot(gs[row, col]) ext_type_to_rp_to_ax[ext_type][rperiod] = ax if col == 0: ax.set_ylabel(ext_type) if row == nrows - 1 and col == ncols - 1: ax_with_legend = ax # Set axes labels if row == nrows - 1: ax.set_xlabel("Observations") if col == 0: ax.set_ylabel("Model") for label in sim_name_to_file_path: if label not in label_to_ax_to_xdata: label_to_ax_to_xdata[label] = {ax: []} label_to_ax_to_ydata[label] = {ax: []} else: label_to_ax_to_xdata[label][ax] = [] label_to_ax_to_ydata[label][ax] = [] ax.set_xscale("log") ax.set_yscale("log") print("Initial list of stations:") sim_label_to_handle = {} for s in stations: print("{0}".format(s)) assert isinstance(s, Station) print(len([y for y in s.get_list_of_complete_years() if start_year <= y <= end_year])) df_ext_obs = extreme_commons.get_annual_extrema(ts_times=s.dates, ts_vals=s.values, start_year=start_year, end_year=end_year) mp = station_to_modelpoint[s] assert isinstance(mp, ModelPoint) years_of_interest = df_ext_obs.index label_to_extrema_model = {} # label -> ext_type -> [return period -> ret level, return period -> std] label_to_return_levels = OrderedDict( [(obs_label, OrderedDict())] ) for sim_label, sim_path in sim_name_to_file_path.items(): label_to_return_levels[sim_label] = OrderedDict() label_to_extrema_model[sim_label] = OrderedDict() # Calculate the return levels and standard deviations for ext_type in ExtremeProperties.extreme_types: return_periods = ExtremeProperties.extreme_type_to_return_periods[ext_type] # fit GEV distribution and apply non-parametric bootstrap to get std label_to_return_levels[obs_label][ext_type] = gevfit.do_gevfit_for_a_point(df_ext_obs[ext_type].values, extreme_type=ext_type, return_periods=return_periods) return_levels_obs, rl_stds_obs = label_to_return_levels[obs_label][ext_type] # get annual extremas for the model output at the points colose to the stations for sim_label, sim_path in sim_name_to_file_path.items(): label_to_return_levels[sim_label] = OrderedDict() ext_field = analysis.get_annual_extrema( rconfig=RunConfig(data_path=sim_path, start_year=start_year, end_year=end_year), varname="STFL", months_of_interest=ExtremeProperties.extreme_type_to_month_of_interest[ext_type], n_avg_days=ExtremeProperties.extreme_type_to_n_agv_days[ext_type], high_flow=ext_type == ExtremeProperties.high) # Select only those years when obs are available ts_data = [v for y, v in zip(range(start_year, end_year + 1), ext_field[:, mp.ix, mp.jy]) if y in years_of_interest] ts_data = np.array(ts_data) return_levels, rl_stds = gevfit.do_gevfit_for_a_point(ts_data, extreme_type=ext_type, return_periods=return_periods) # Do the plotting for rp in return_periods: ax = ext_type_to_rp_to_ax[ext_type][rp] ax.set_title("T = {rp}-year".format(rp=rp)) # h = ax.errorbar(return_levels_obs[rp], return_levels[rp], # marker=label_to_marker[sim_label], color=label_to_color[sim_label], label=sim_label, # xerr=rl_stds_obs[rp] * 1.96, yerr=rl_stds[rp] * 1.96) h = ax.scatter(return_levels_obs[rp], return_levels[rp], marker=label_to_marker[sim_label], color=label_to_color[sim_label], label=sim_label) # save the data for maybe further calculation of the correlation coefficients label_to_ax_to_xdata[sim_label][ax].append(return_levels_obs[rp]) label_to_ax_to_ydata[sim_label][ax].append(return_levels[rp]) sim_label_to_handle[sim_label] = h # Calculate the biases for sim_label in sim_name_to_file_path: for ext_type in ExtremeProperties.extreme_types: ret_periods = ExtremeProperties.extreme_type_to_return_periods[ext_type] for rp in ret_periods: ax = ext_type_to_rp_to_ax[ext_type][rp] mod = np.asarray(label_to_ax_to_ydata[sim_label][ax]) obs = np.asarray(label_to_ax_to_xdata[sim_label][ax]) bias = np.mean((mod - obs)/obs) corr, pv = stats.pearsonr(mod, obs) print("({sim_label}) Mean bias for {rp}-year {ext_type}-flow return level is: {bias}; corr={corr:.2f}; corr_pval={corr_pval:2g}".format( sim_label=sim_label, rp=rp, bias=bias, corr=corr, corr_pval=pv, ext_type=ext_type )) sfmt = ScalarFormatter(useMathText=True) sfmt.set_powerlimits((-2, 2)) for et, rp_to_ax in ext_type_to_rp_to_ax.items(): for rp, ax in rp_to_ax.items(): xmin, xmax = ax.get_xlim() ymin, ymax = ax.get_ylim() x1 = min(xmin, ymin) x2 = min(xmax, ymax) ax.plot([x1, x2], [x1, x2], "k--") # ax.xaxis.set_major_locator(MaxNLocator(nbins=5)) # ax.yaxis.set_major_locator(MaxNLocator(nbins=5)) # ax.xaxis.set_major_formatter(sfmt) # ax.yaxis.set_major_formatter(sfmt) sim_labels = list(sim_name_to_file_path.keys()) ax_with_legend.legend([sim_label_to_handle[sl] for sl in sim_labels], sim_labels, bbox_to_anchor=(1, -0.25), borderaxespad=0.0, loc="upper right", ncol=2, scatterpoints=1, numpoints=1) # Save the plot img_file = "{}.eps".format("_".join(sorted(label_to_marker.keys()))) img_file = img_folder.joinpath(img_file) fig.tight_layout() with img_file.open("wb") as f: fig.savefig(f, bbox_inches="tight")
def compare_hydrographs_at_stations(manager_list, start_date=None, end_date=None, img_path="hydrographs.png", colors=None, fig=None): selected_ids = None stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date) if colors is None: colors = len(manager_list) * [None] skip_stations = ["080718", "095003", "094206", "090613", "092715"] # 090613 is skipped for the 0.5 deg resolution since the drainaige network is not fully # represented by the model lines_model = [] station_to_list_of_model_ts = {} run_id_list = [m.run_id for m in manager_list] filtered_stations = [] for s in stations: assert isinstance(s, Station) if s.id in skip_stations: continue # skip stations with smaller accumulation areas if s.drainage_km2 <= 4 * np.radians( 0.5)**2 * lat_lon.EARTH_RADIUS_METERS**2 * 1.0e-6: continue if not s.passes_rough_continuity_test(start_date, end_date): continue filtered_stations.append(s) stations = filtered_stations print(len(filtered_stations)) # if True: raise Exception() #save all run ids plot_utils.apply_plot_params(width_pt=None, height_cm=40.0, width_cm=30.0, font_size=10) run_id_to_dataframe = {} run_id_to_cell_props = {} for manager in manager_list: assert isinstance(manager, Crcm5ModelDataManager) df, station_to_cellprops = manager.get_streamflow_dataframe_for_stations( stations, start_date=start_date, end_date=end_date, var_name="STFL", nneighbours=9) assert isinstance(df, pandas.DataFrame) df = df.dropna(axis=1) run_id_to_cell_props[manager.run_id] = station_to_cellprops df = df.groupby(lambda i: datetime(2001, i.month + 1, 1) if i.month == 2 and i.day == 29 else datetime( 2001, i.month, i.day)).mean() print(df) #again filter the stations with data time interval overlapping with model time interval stations = list(filter(lambda s: s.id in df.columns, stations)) run_id_to_dataframe[manager.run_id] = df if fig is None: fig = plt.figure() #two columns ncols = 2 nrows = len(stations) / ncols if nrows * ncols < len(stations): nrows += 1 gs = GridSpec(nrows, ncols, hspace=0.4, wspace=0.4) line_model, line_obs = None, None stations.sort(key=lambda x: x.latitude, reverse=True) plot_station_positions(manager_list[0], stations) i = -1 ns_list = [] station_list = [] flow_acc_area_list = [] #one_day = timedelta(days = 1) one_day_sec = 24 * 60 * 60.0 for s in stations: i += 1 ax = fig.add_subplot(gs[i // ncols, i % ncols]) assert isinstance(s, Station) year_dates, sta_clims = s.get_daily_normals() #plot error limits ax.fill_between(year_dates, sta_clims * 1.256, sta_clims * 0.744, alpha=0.25, color="b") line_obs = ax.plot(year_dates, sta_clims, color="b", label="Observation", lw=3, alpha=0.5) ax.annotate( "{0:.3g}".format(sum(sta_clims) * one_day_sec), (0.1, 0.95), xycoords="axes fraction", color="b", alpha=0.5) #integral flow since those values are daily normals for run_id, color, color_index in zip(run_id_list, colors, list(range(len(colors)))): df = run_id_to_dataframe[run_id] the_line = ax.plot(year_dates, df[s.id], color=color, label=run_id, lw=1) ax.annotate("{0:.3g}".format(sum(df[s.id]) * one_day_sec), (0.1, 0.9 - color_index * 0.05), xycoords="axes fraction", color=color ) #integral flow since those values are daily normals if not i: #save the labels only for the first step lines_model.append(the_line) #dt = model_ts.time[1] - model_ts.time[0] #dt_sec = dt.days * 24 * 60 * 60 + dt.seconds #ax.annotate( "{0:g}".format( sum(mod_vals) * dt_sec ) + " ${\\rm m^3}$", xy = (0.7,0.7), xycoords= "axes fraction", color = "b") #ax.annotate( "{0:g}".format( sum(s.values) * dt_sec) + " ${\\rm m^3}$", xy = (0.7,0.6), xycoords= "axes fraction", color = "r") metadata = list(run_id_to_cell_props.items())[0][1][s.id] da_mod = metadata["acc_area_km2"] dist = metadata["distance_to_obs_km"] #ax.set_title("{0}: $\delta DA = {1:.1f}$ %, dist = {2:.1f} km".format(s.id, # (da_mod - s.drainage_km2) / s.drainage_km2 * 100.0, dist ) ) ax.set_title("{0}: $DA = {1:.1f}$ {2}".format(s.id, s.drainage_km2, "${\\rm km ^ 2}$")) ax.xaxis.set_major_formatter(DateFormatter("%m")) #ax.xaxis.set_major_locator(YearLocator()) assert isinstance(ax, Axes) ax.xaxis.axis_date() #ax.xaxis.tick_bottom().set_rotation(60) lines = lines_model + [ line_obs, ] labels = run_id_list + [ "Observation", ] fig.legend(lines, labels, ncol=5) if img_path is not None: fig.savefig(img_path)
def main(): season_to_months = DEFAULT_SEASON_TO_MONTHS r_config = RunConfig( data_path="/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5", start_year=1990, end_year=2010, label="CRCM5-L" ) bmp_info = analysis.get_basemap_info_from_hdf(file_path=r_config.data_path) bmp_info.should_draw_grey_map_background = True bmp_info.should_draw_basin_boundaries = False bmp_info.map_bg_color = "0.75" station_ids = [ "104001", "093806", "093801", "081002", "081007", "080718" ] # get river network information used in the model flow_directions = analysis.get_array_from_file(r_config.data_path, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) accumulation_area_km2 = analysis.get_array_from_file(path=r_config.data_path, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) cell_manager = CellManager(flow_dirs=flow_directions, lons2d=bmp_info.lons, lats2d=bmp_info.lats, accumulation_area_km2=accumulation_area_km2) # Get the list of stations to indicate on the bias map stations = cehq_station.read_station_data( start_date=None, end_date=None, selected_ids=station_ids ) """:type : list[Station]""" xx, yy = bmp_info.get_proj_xy() station_to_modelpoint = cell_manager.get_model_points_for_stations(station_list=stations) upstream_edges = cell_manager.get_upstream_polygons_for_points( model_point_list=station_to_modelpoint.values(), xx=xx, yy=yy) # Validate temperature, precip and swe obs_path_anusplin = "/home/huziy/skynet3_rech1/anusplin_links" obs_path_swe = "data/swe_ross_brown/swe.nc" model_var_to_obs_path = OrderedDict([ ("TT", obs_path_anusplin), # ("PR", obs_path_anusplin), ("I5", obs_path_swe) ]) vname_to_obs_data = {} # parameters that won't change in the loop over variable names params_const = dict(rconfig=r_config, bmp_info=bmp_info, season_to_months=season_to_months) for vname, obs_path in model_var_to_obs_path.items(): season_to_obs_data = get_seasonal_clim_obs_data(vname=vname, obs_path=obs_path, **params_const) # Comment swe over lakes, since I5 calculated only for land if vname in ["I5", ]: for season in season_to_obs_data: season_to_obs_data[season] = maskoceans(bmp_info.lons, bmp_info.lats, season_to_obs_data[season], inlands=True) vname_to_obs_data[vname] = season_to_obs_data # Plotting plot_all_vars_in_one_fig = True fig = None gs = None row_axes = [] ncols = None if plot_all_vars_in_one_fig: plot_utils.apply_plot_params(font_size=12, width_pt=None, width_cm=25, height_cm=20) fig = plt.figure() ncols = len(season_to_months) + 1 gs = GridSpec(len(model_var_to_obs_path), ncols, width_ratios=(ncols - 1) * [1., ] + [0.05, ]) else: plot_utils.apply_plot_params(font_size=12, width_pt=None, width_cm=25, height_cm=25) row = 0 station_x_list = [] station_y_list = [] for mname in model_var_to_obs_path: if plot_all_vars_in_one_fig: row_axes = [fig.add_subplot(gs[row, col]) for col in range(ncols)] compare_vars(vname_model=mname, vname_to_obs=vname_to_obs_data, r_config=r_config, season_to_months=season_to_months, bmp_info_agg=bmp_info, axes_list=row_axes) # -1 in order to exclude colorbars for the_ax in row_axes[:-1]: # Need titles only for the first row if row > 0: the_ax.set_title("") draw_upstream_area_bounds(the_ax, upstream_edges) if len(station_x_list) == 0: for the_station in stations: xst, yst = bmp_info.basemap(the_station.longitude, the_station.latitude) station_x_list.append(xst) station_y_list.append(yst) bmp_info.basemap.scatter(station_x_list, station_y_list, c="g", ax=the_ax, s=5, zorder=10, alpha=0.5) # Hide fall swe if mname in ["I5"]: row_axes[-2].set_visible(False) row += 1 # Save the figure if necessary if plot_all_vars_in_one_fig: fig_path = img_folder.joinpath("{}.png".format("_".join(model_var_to_obs_path))) with fig_path.open("wb") as figfile: fig.savefig(figfile, format="png", bbox_inches="tight") plt.close(fig)
def plot_at_indices(ix, jy): var_name_liquid = "I1" var_name_solid = "I2" #peirod of interest start_year = 1979 end_year = 1988 #simulation names corresponding to the paths sim_names = ["crcm5-hcd-rl", "crcm5-hcd-rl-intfl"] sim_labels = [x.upper() for x in sim_names] layer_widths = [ 0.1, 0.2, 0.3, 0.4, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0 ] layer_depths = np.cumsum(layer_widths) paths = [ "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup", "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup2/Samples_all_in_one" ] managers = [ Crcm5ModelDataManager(samples_folder_path=path, file_name_prefix="pm", all_files_in_samples_folder=True, need_cell_manager=(i == 0)) for i, path in enumerate(paths) ] #share the cell manager a_data_manager = managers[0] assert isinstance(a_data_manager, Crcm5ModelDataManager) cell_manager = a_data_manager.cell_manager assert isinstance(cell_manager, CellManager) for m in managers[1:]: assert isinstance(m, Crcm5ModelDataManager) m.cell_manager = cell_manager #share the lake fraction field lake_fraction = a_data_manager.lake_fraction selected_ids = [ "092715", "080101", "074903", "050304", "080104", "081007", "061905", "041903", "040830", "093806", "090613", "081002", "093801", "080718" ] start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date) #stations with corresponding model points station_to_mp = a_data_manager.get_dataless_model_points_for_stations( stations) #figure out levels in soil sim_label_to_profiles = {} fig = plt.figure() fmt = ScalarFormatter(useMathText=True) fmt.set_powerlimits([-2, 2]) for m, label in zip(managers, sim_labels): assert isinstance(m, Crcm5ModelDataManager) monthly_means_liquid = _get_cached_monthly_mean_fields( label, start_year, end_year, var_name_liquid) if monthly_means_liquid is None: monthly_means_liquid = m.get_monthly_climatology_of_3d_field( var_name=var_name_liquid, start_year=start_year, end_year=end_year) _cache_monthly_mean_fields(monthly_means_liquid, label, start_year, end_year, var_name_liquid) monthly_means_solid = _get_cached_monthly_mean_fields( label, start_year, end_year, var_name_solid) if monthly_means_solid is None: monthly_means_solid = m.get_monthly_climatology_of_3d_field( var_name=var_name_solid, start_year=start_year, end_year=end_year) _cache_monthly_mean_fields(monthly_means_solid, label, start_year, end_year, var_name_solid) profiles = [ monthly_means_liquid[i][ix, jy, :] + monthly_means_solid[i][ix, jy, :] for i in range(12) ] sim_label_to_profiles[label] = np.array(profiles) x = list(range(12)) y = layer_depths y2d, x2d = np.meshgrid(y, x) plt.contourf( x2d, y2d, sim_label_to_profiles[sim_labels[1]] - sim_label_to_profiles[sim_labels[0]]) plt.gca().invert_yaxis() plt.colorbar() #fig.tight_layout() fig.savefig("soil_profile_at_ix={0};jy={1}.pdf".format(ix, jy))
def validate_precip(model_file="", simlabel="", obs_manager=None, season_to_months=None, start_year=None, end_year=None, season_to_plot_indices=None, station_ids_list=None): """ :param model_file: :param obs_manager: should implement the method getMeanFieldForMonthsInterpolatedTo(self, months = None, lonsTarget = None, latsTarget = None) anusplin data is in mm/day model data is in m/s """ model_var_name = "PR" if obs_manager is None: print( "Skipping validation of {}, since the obs manager is None.".format( model_var_name)) return if station_ids_list is not None: # Get the list of stations to indicate on the bias map stations = cehq_station.read_station_data( start_date=None, end_date=None, selected_ids=station_ids_list) else: stations = [] model_level = 0 reasonable_error_mm_per_day = 1 assert isinstance(obs_manager, AnuSplinManager) fig = plt.figure() assert isinstance(fig, Figure) fig.suptitle("({0}) - ({1})".format(simlabel, "Obs.")) lon, lat, basemap = analysis.get_basemap_from_hdf(file_path=model_file) # do calculations and only after that do the plotting season_to_field = {} # calculate global min and max for plotting vmin = None vmax = None for season, months in season_to_months.items(): model_field = analysis.get_seasonal_climatology( start_year=start_year, end_year=end_year, months=months, level=model_level, var_name=model_var_name, hdf_path=model_file) # convert m/s to mm/day for comparison with anusplin data model_field *= 1000.0 * 60 * 60 * 24 obs_field = obs_manager.getMeanFieldForMonthsInterpolatedTo( months=months, lonstarget=lon, latstarget=lat, start_year=start_year, end_year=end_year) # calculate the difference between the modelled and observed fields the_diff = model_field - obs_field current_min = np.min(the_diff) current_max = np.max(the_diff) if vmin is not None: vmin = current_min if current_min < vmin else vmin vmax = current_max if current_max > vmax else vmax else: vmin = current_min vmax = current_max season_to_field[season] = the_diff ncolors = 12 gs = gridspec.GridSpec(2, 3, width_ratios=[1, 1, 0.05]) cmap = cm.get_cmap("RdBu_r", ncolors) x, y = basemap(lon, lat) im = None d = min(abs(vmin), abs(vmax)) vmin = -d vmax = d bn, bounds, _, _ = infovar.get_boundary_norm(vmin, vmax, ncolors, exclude_zero=False) print("bounds: ", bounds) cs = None for season, field in season_to_field.items(): row, col = season_to_plot_indices[season] ax = fig.add_subplot(gs[row, col]) ax.set_title(season) basemap.drawmapboundary(fill_color="gray", ax=ax) im = basemap.pcolormesh(x, y, season_to_field[season], vmin=vmin, vmax=vmax, cmap=cmap, norm=bn) basemap.drawcoastlines(ax=ax, linewidth=cpp.COASTLINE_WIDTH) for the_station in stations: assert isinstance(the_station, Station) xst, yst = basemap(the_station.longitude, the_station.latitude) # ax.annotate(the_station.id, (xst, yst), font_properties=FontProperties(size=6), # bbox=dict(facecolor="w"), va="top", ha="right") basemap.scatter(xst, yst, c="g", ax=ax) # small_error = (np.abs(season_to_field[season]) < reasonable_error_mm_per_day).astype(int) # nlevs = 1 # ax.contour(x, y, small_error, nlevs, colors = "black", linestyle = "-") # cs = ax.contourf(x, y, small_error, nlevs, colors="none", hatches=["/", None], extend="lower", linewidth=2) # artists, labels = cs.legend_elements() # plt.legend(artists, labels, handleheight=2) cax = fig.add_subplot(gs[:, 2]) cax.set_title("mm/day\n") plt.colorbar(im, cax=cax, extend="both") seasons_str = "_".join( sorted([str(s) for s in list(season_to_field.keys())])) atm_val_folder = os.path.join(images_folder, "validate_atm") if not os.path.isdir(atm_val_folder): os.mkdir(atm_val_folder) out_filename = "{3}/validate_2d_{0}_{1}_{2}.png".format( model_var_name, simlabel, seasons_str, atm_val_folder) fig.savefig(os.path.join(images_folder, out_filename), bbox_inches="tight")
def plot_for_different_months(start_date = None, end_date = None): lake_names = [ #"Matagami", "Mistassini", "Nemiscau"] lake_areas_km2 = [ #370.7, 2162.7, 148.3] level_st_ids = [ #"080716", "081003", "081001" ] level_stations = cehq.read_station_data(folder="data/cehq_levels", selected_ids=level_st_ids) stfl_st_ids = [ #"080707", "081007", "081002" ] stfl_stations = cehq.read_station_data(folder="data/cehq_measure_data", selected_ids=stfl_st_ids) for lev_station, stfl_station, lake_name, lake_area_km2, c in zip(level_stations, stfl_stations, lake_names, lake_areas_km2, ["r","g","b"] ): assert isinstance(lev_station, Station) assert isinstance(stfl_station, Station) all_q_obs = [] all_q_calc = [] all_k = [] all_b = [] intersection_dates = list( sorted( filter( lambda d: d in stfl_station.dates, lev_station.dates) ) ) q_vals = [stfl_station.get_value_for_date(d) for d in intersection_dates] h_vals = [lev_station.get_value_for_date(d) for d in intersection_dates] #change the way streamflow calculated here q_calc = get_streamflows_from_active_stores_bowling(get_active_storages(h_vals,lake_area_km2), lake_area_km2) q_min = min( min(q_vals), min(q_calc) ) q_max = max( max(q_vals), max(q_calc) ) plot_utils.apply_plot_params(width_pt=None, width_cm=30, height_cm=20, font_size=10) fig = plt.figure() gs = gridspec.GridSpec(3,4,wspace=0.5, hspace=0.5) if start_date is None: start_date = intersection_dates[0] if end_date is None: end_date = intersection_dates[-1] print("lake name: {0}".format(lake_name)) print("data are from " + str( intersection_dates[0] ) + " to " + str( intersection_dates[-1] )) ##As a base month we are using August (assuming that we don't have ice in August) base_month = 8 the_dates = list( filter( lambda d: (start_date <= d <= end_date) and d.month == base_month, intersection_dates) ) q_base = np.mean([stfl_station.get_value_for_date(d) for d in the_dates]) h_vals = [lev_station.get_value_for_date(d) for d in the_dates] store_base = np.mean(get_active_storages(h_vals,lake_area_km2)) ice_factors = [] for month in range(1, 13): ax = fig.add_subplot(gs[(month - 1)//4, (month - 1) % 4]) the_dates = list( filter( lambda d: (start_date <= d <= end_date) and d.month == month, intersection_dates) ) q_vals = [stfl_station.get_value_for_date(d) for d in the_dates] h_vals = [lev_station.get_value_for_date(d) for d in the_dates] s_vals = get_active_storages(h_vals,lake_area_km2) print(len(h_vals)) q_calc = get_streamflows_from_active_stores_bowling(s_vals, lake_area_km2) ice_factors.append(get_ice_factor(np.mean(q_vals), np.mean(s_vals), q_base, store_base)) all_q_obs.extend(q_vals) all_q_calc.extend(q_calc) ax.scatter(q_vals, q_calc, linewidths = 0) print("len(q_vals) = {0}".format(len(q_vals))) ax.set_xlabel("$Q_{\\rm obs}$") ax.set_ylabel("$Q_{\\rm mod}$") the_poly = np.polyfit(q_vals, q_calc, 1) k, b = the_poly all_k.append(k) all_b.append(b) ax.scatter(q_vals, [(x - b) / k for x in q_calc], c ="r", linewidth = 0, zorder = 6) #ax.annotate("k={0:.2f}; \nb={1:d}".format(k, int(b)), xy = (0.6, 0.05), # xycoords = "axes fraction", zorder = 5 #) ax.plot([q_min, q_max], [q_min, q_max], "k-",lw = 3, zorder = 5) d = datetime(2000, month, 1) ax.set_title(d.strftime("%b") + "( k={0:.7f}; b={1:.2f})".format(k, b)) ax.xaxis.set_major_locator(MultipleLocator(base = np.round((q_max - q_min )/ 10) * 10 / 2 )) ax.yaxis.set_major_locator(MultipleLocator(base = np.round((q_max - q_min ) / 10) * 10 / 2)) fig.suptitle(lake_name) fig.savefig("{0}.png".format(lake_name)) assert isinstance(fig, Figure) plot_utils.apply_plot_params(width_pt=None, width_cm=30, height_cm=20, font_size=25) #plot k and b as a function of month fig = plt.figure() ax = fig.add_subplot(1,1,1) assert isinstance(ax, Axes) ax.set_title("Correction coefficients applied \n q'=(1/k)*q-b/k") ax.plot(list(range(1,13)), 1.0 / np.array(all_k), lw = 3, label = "1/k") ax.legend() fig.savefig("{0}_coefs_1_k.png".format(lake_name)) fig = plt.figure() ax = fig.add_subplot(1,1,1) assert isinstance(ax, Axes) ax.set_title("Correction coefficients applied \n q'=(1/k)*q-b/k") ax.plot(list(range(1,13)), -np.array(all_b) / np.array(all_k), lw = 3, label = "-b/k") ax.legend() fig.savefig("{0}_coefs_b_k.png".format(lake_name)) plt.figure() plt.plot(list(range(1,13)), ice_factors, lw = 3) plt.title("Ice factor for {0}".format(lake_name)) plt.savefig("{0}_ice_factors.png".format(lake_name)) ###plot q-q for all season in the same plot plot_utils.apply_plot_params(width_pt=None, width_cm=30, height_cm=20, font_size=20) fig1 = plt.figure() ax = fig1.add_subplot(1,1,1) ax.set_title(lake_name) ax.scatter(all_q_obs, all_q_calc, c="b", linewidths=0) the_poly = np.polyfit(all_q_obs, all_q_calc, 1) k,b = the_poly print(k, b) print("len(all_q) = {0}".format(len(all_q_calc))) ax.scatter(all_q_obs, [(x - b) / k for x in all_q_calc], c ="r", linewidth = 0, zorder = 6) ax.annotate("k={0:.2f}; \nb={1:.2f}".format(k, b), xy = (0.6, 0.05), xycoords = "axes fraction", zorder = 7 ) print(np.polyfit(all_q_obs, [(x - b) / k for x in all_q_calc],1)) print("min(all_q_calc) = {0}, max(all_q_calc) = {1}".format(min(all_q_calc), max(all_q_calc))) # ax.plot([q_min, q_max], [k * q_min + b, k * q_max + b], "g" ) ax.plot([q_min, q_max], [q_min, q_max], "k-",lw = 1, zorder = 5) # for the_k, the_b in zip(all_k, all_b): # ax.plot([q_min, q_max], [the_k * q_min + the_b, the_k * q_max + the_b] ) ax.grid(b = True) ax.set_xlabel("$Q_{\\rm obs}$") ax.set_ylabel("$Q_{\\rm mod}$") ax.xaxis.set_major_locator(MultipleLocator(base = np.round((q_max - q_min )/ 10) * 10 / 2 )) ax.yaxis.set_major_locator(MultipleLocator(base = np.round((q_max - q_min ) / 10) * 10 / 2)) print("all_{0}.png".format(lake_name)) #plt.show() fig1.subplots_adjust(left = 0.2) fig1.savefig("all_{0}.png".format(lake_name))
def main(): var_name_liquid = "I1" var_name_solid = "I2" #peirod of interest start_year = 1979 end_year = 1988 #spatial averaging will be done over upstream points to the stations selected_ids = [ "092715", "080101", "074903", "050304", "080104", "081007", "061905", "041903", "040830", "093806", "090613", "081002", "093801", "080718" ] selected_ids = ["074903"] #simulation names corresponding to the paths sim_names = ["crcm5-hcd-rl", "crcm5-hcd-rl-intfl"] sim_labels = [x.upper() for x in sim_names] colors = ["blue", "violet"] layer_widths = [ 0.1, 0.2, 0.3, 0.4, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0 ] layer_depths = np.cumsum(layer_widths) paths = [ "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup", "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup2/Samples_all_in_one" ] seasons = [[12, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11]] season_names = ["DJF", "MAM", "JJA", "SON"] managers = [ Crcm5ModelDataManager(samples_folder_path=path, file_name_prefix="pm", all_files_in_samples_folder=True, need_cell_manager=(i == 0)) for i, path in enumerate(paths) ] #share the cell manager a_data_manager = managers[0] assert isinstance(a_data_manager, Crcm5ModelDataManager) cell_manager = a_data_manager.cell_manager assert isinstance(cell_manager, CellManager) for m in managers[1:]: assert isinstance(m, Crcm5ModelDataManager) m.cell_manager = cell_manager #share the lake fraction field lake_fraction = a_data_manager.lake_fraction #selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905", # "041903", "040830", "093806", "090613", "081002", "093801", "080718"] start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date) #stations with corresponding model points station_to_mp = a_data_manager.get_dataless_model_points_for_stations( stations) #figure out levels in soil sim_label_to_profiles = {} for s, mp in station_to_mp.items(): assert isinstance(mp, ModelPoint) mask = (mp.flow_in_mask == 1) & (lake_fraction < 0.6) fig = plt.figure() fmt = ScalarFormatter(useMathText=True) fmt.set_powerlimits([-2, 2]) print(mp.ix, mp.jy, s.id) for m, label, color in zip(managers, sim_labels, colors): assert isinstance(m, Crcm5ModelDataManager) monthly_means_liquid = _get_cached_monthly_mean_fields( label, start_year, end_year, var_name_liquid) if monthly_means_liquid is None: monthly_means_liquid = m.get_monthly_climatology_of_3d_field( var_name=var_name_liquid, start_year=start_year, end_year=end_year) _cache_monthly_mean_fields(monthly_means_liquid, label, start_year, end_year, var_name_liquid) monthly_means_solid = _get_cached_monthly_mean_fields( label, start_year, end_year, var_name_solid) if monthly_means_solid is None: monthly_means_solid = m.get_monthly_climatology_of_3d_field( var_name=var_name_solid, start_year=start_year, end_year=end_year) _cache_monthly_mean_fields(monthly_means_solid, label, start_year, end_year, var_name_solid) profiles = [ monthly_means_liquid[i][mask, :].mean(axis=0) + monthly_means_solid[i][mask, :].mean(axis=0) for i in range(12) ] sim_label_to_profiles[label] = np.array(profiles) x = [date2num(datetime(2001, month, 1)) for month in range(1, 13)] y = layer_depths y2d, x2d = np.meshgrid(y, x) delta = (sim_label_to_profiles[sim_labels[1]] - sim_label_to_profiles[sim_labels[0]] ) / sim_label_to_profiles[sim_labels[0]] * 100 #delta = np.ma.masked_where(delta < 0.1, delta) cmap = my_colormaps.get_cmap_from_ncl_spec_file( path="colormap_files/BlueRed.rgb", ncolors=10) the_min = -6.0 the_max = 6.0 step = (the_max - the_min) / float(cmap.N) plt.pcolormesh(x2d[:, :8], y2d[:, :8], delta[:, :8], cmap=cmap, vmin=the_min, vmax=the_max) #, levels = np.arange(-6,7,1)) plt.gca().invert_yaxis() plt.colorbar(ticks=np.arange(the_min, the_max + step, step)) plt.gca().set_ylabel("Depth (m)") plt.gca().xaxis.set_major_formatter(DateFormatter("%b")) #fig.tight_layout() fig.savefig("soil_profile_upstream_of_{0}.pdf".format(s.id)) pass
def main(): # data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_highres_spinup_12_month_with_lakes" data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_test_198501_198612_0.1deg" coord_file = os.path.join(data_path, "pm1985010100_00000000p") manager = Crcm5ModelDataManager( samples_folder_path=data_path, file_name_prefix="pm", all_files_in_samples_folder=True ) assert isinstance(manager, Crcm5ModelDataManager) selected_ids = ["104001", "103715", "093806", "093801", "092715", "081006", "061502", "040830", "080718"] start_date = datetime(1985, 1, 1) end_date = datetime(1985, 12, 31) stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date) plot_utils.apply_plot_params(width_pt=None, height_cm=30.0, width_cm=16, font_size=10) fig = plt.figure() # two columns gs = GridSpec(len(stations) // 2 + len(stations) % 2, 2, hspace=0.4, wspace=0.4) line_model, line_obs = None, None stations.sort(key=lambda x: x.latitude, reverse=True) for i, s in enumerate(stations): model_ts = manager.get_streamflow_timeseries_for_station( s, start_date=start_date, end_date=end_date, var_name="SWSR" ) ax = fig.add_subplot(gs[i // 2, i % 2]) assert isinstance(model_ts, TimeSeries) # [t, m_data] = model_ts.get_daily_normals() # [t, s_data] = s.get_daily_normals() assert isinstance(s, Station) # climatologies # line_model = ax.plot(t, m_data, label = "Model (CRCM5)", lw = 3, color = "b") # line_obs = ax.plot(t, s_data, label = "Observation", lw = 3, color = "r") model_ts = model_ts.get_ts_of_daily_means() print(model_ts.time[0], model_ts.time[-1]) print(model_ts.data[0:10]) mod_vals = model_ts.get_data_for_dates(s.dates) print(mod_vals[:20]) print("+" * 20) assert len(mod_vals) == len(s.dates) line_model = ax.plot(s.dates, mod_vals, label="Model (CRCM5)", lw=1, color="b") # line_obs = ax.plot(s.dates, s.values, label = "Observation", lw = 3, color = "r", alpha = 0.5) bf_store = model_ts.metadata["bankfull_store_m3"] ax.plot([s.dates[0], s.dates[-1]], [bf_store, bf_store], color="k") ax.set_title( "%s: da_diff=%.2f %%, dist = %.1f" % ( s.id, (-s.drainage_km2 + model_ts.metadata["acc_area_km2"]) / s.drainage_km2 * 100.0, model_ts.metadata["distance_to_obs"], ) ) ax.xaxis.set_major_formatter(DateFormatter("%Y")) ax.xaxis.set_major_locator(YearLocator()) fig.savefig("storage.png") pass
def main(): start_year = 1980 end_year = 2010 start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) ids_with_lakes_upstream = [ "104001", "093806", "093801", "081002", "081007", "080718" ] selected_station_ids = [ "092715", "074903", "080104", "081007", "061905", "093806", "090613", "081002", "093801", "080718", "104001" ] selected_station_ids = ids_with_lakes_upstream # Get the list of stations to do the comparison with stations = cehq_station.read_station_data( start_date=start_date, end_date=end_date, selected_ids=selected_station_ids) # add hydat stations # province = "QC" # min_drainage_area_km2 = 10000.0 # stations_hd = cehq_station.load_from_hydat_db(start_date=start_date, end_date=end_date, # province=province, min_drainage_area_km2=min_drainage_area_km2) # if not len(stations_hd): # print "No hydat stations satisying the conditions: period {0}-{1}, province {2}".format( # str(start_date), str(end_date), province # ) # stations.extend(stations_hd) # brewer2mpl.get_map args: set name set type number of colors bmap = brewer2mpl.get_map("Set1", "qualitative", 9) path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5" label1 = "CRCM5-L1" path2 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5" label2 = "CRCM5-L2" color2, color1 = bmap.mpl_colors[:2] fldirs = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(path1) lake_fractions = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_LAKE_FRACTION_NAME) # cell_areas = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_CELL_AREA_NAME) acc_area = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) cell_manager = CellManager(fldirs, lons2d=lons2d, lats2d=lats2d, accumulation_area_km2=acc_area) station_to_mp = cell_manager.get_model_points_for_stations( station_list=stations, lake_fraction=lake_fractions, drainaige_area_reldiff_limit=0.3) fig, axes = plt.subplots(1, 2, gridspec_kw=dict(top=0.80, wspace=0.4)) q90_obs_list = [] q90_mod1_list = [] q90_mod2_list = [] q10_obs_list = [] q10_mod1_list = [] q10_mod2_list = [] for the_station, the_mp in station_to_mp.items(): assert isinstance(the_station, Station) compl_years = the_station.get_list_of_complete_years() if len(compl_years) < 3: continue t, stfl1 = analysis.get_daily_climatology_for_a_point( path=path1, years_of_interest=compl_years, i_index=the_mp.ix, j_index=the_mp.jy, var_name="STFA") _, stfl2 = analysis.get_daily_climatology_for_a_point( path=path2, years_of_interest=compl_years, i_index=the_mp.ix, j_index=the_mp.jy, var_name="STFA") _, stfl_obs = the_station.get_daily_climatology_for_complete_years( stamp_dates=t, years=compl_years) # Q90 q90_obs = np.percentile(stfl_obs, 90) q90_mod1 = np.percentile(stfl1, 90) q90_mod2 = np.percentile(stfl2, 90) # Q10 q10_obs = np.percentile(stfl_obs, 10) q10_mod1 = np.percentile(stfl1, 10) q10_mod2 = np.percentile(stfl2, 10) # save quantiles to lists for correlation calculation q90_obs_list.append(q90_obs) q90_mod1_list.append(q90_mod1) q90_mod2_list.append(q90_mod2) q10_mod1_list.append(q10_mod1) q10_mod2_list.append(q10_mod2) q10_obs_list.append(q10_obs) # axes[0].annotate(the_station.id, (q90_obs, np.percentile(stfl1, 90))) # axes[1].annotate(the_station.id, (q10_obs, np.percentile(stfl1, 10))) # Plot scatter plot of Q90 the_ax = axes[0] # the_ax.annotate(the_station.id, (q90_obs, np.percentile(stfl1, 90))) the_ax.scatter(q90_obs_list, q90_mod1_list, label=label1, c=color1) the_ax.scatter(q90_obs_list, q90_mod2_list, label=label2, c=color2) # plot scatter plot of Q10 the_ax = axes[1] # the_ax.annotate(the_station.id, (q10_obs, np.percentile(stfl1, 10))) h1 = the_ax.scatter(q10_obs_list, q10_mod1_list, label=label1, c=color1) h2 = the_ax.scatter(q10_obs_list, q10_mod2_list, label=label2, c=color2) # Add correlation coefficients to the axes fp = FontProperties(size=14, weight="bold") axes[0].annotate(r"$R^2 = {0:.2f}$".format( np.corrcoef(q90_mod1_list, q90_obs_list)[0, 1]**2), (0.1, 0.85), color=color1, xycoords="axes fraction", font_properties=fp) axes[0].annotate(r"$R^2 = {0:.2f}$".format( np.corrcoef(q90_mod2_list, q90_obs_list)[0, 1]**2), (0.1, 0.70), color=color2, xycoords="axes fraction", font_properties=fp) axes[1].annotate(r"$R^2 = {0:.2f}$".format( np.corrcoef(q10_mod1_list, q10_obs_list)[0, 1]**2), (0.1, 0.85), color=color1, xycoords="axes fraction", font_properties=fp) axes[1].annotate(r"$R^2 = {0:.2f}$".format( np.corrcoef(q10_mod2_list, q10_obs_list)[0, 1]**2), (0.1, 0.70), color=color2, xycoords="axes fraction", font_properties=fp) sf = ScalarFormatter(useMathText=True) sf.set_powerlimits((-2, 3)) for ind, the_ax in enumerate(axes): plot_one_to_one_line(the_ax) if ind == 0: the_ax.set_xlabel(r"Observed $\left({\rm m^3/s} \right)$") the_ax.set_ylabel(r"Modelled $\left({\rm m^3/s} \right)$") the_ax.annotate(r"$Q_{90}$" if ind == 0 else r"$Q_{10}$", (0.95, 0.95), xycoords="axes fraction", bbox=dict(facecolor="white"), va="top", ha="right") the_ax.xaxis.set_major_formatter(sf) the_ax.yaxis.set_major_formatter(sf) locator = MaxNLocator(nbins=5) the_ax.xaxis.set_major_locator(locator) the_ax.yaxis.set_major_locator(locator) x1, x2 = the_ax.get_xlim() # Since streamflow percentiles can only be positive the_ax.set_xlim(0, x2) the_ax.set_ylim(0, x2) fig.legend([h1, h2], [label1, label2], loc="upper center", ncol=2) figpath = os.path.join(images_folder, "percentiles_comparison.png") # plt.tight_layout() fig.savefig(figpath, dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight")
def main(start_year=1980, end_year=1989): soil_layer_widths = infovar.soil_layer_widths_26_to_60 soil_tops = np.cumsum(soil_layer_widths).tolist()[:-1] soil_tops = [ 0, ] + soil_tops selected_station_ids = [ "061905", "074903", "090613", "092715", "093801", "093806" ] # path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf" # label1 = "CRCM5-HCD-RL" path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5" label1 = "CRCM5-HCD-RL-INTFL" path2 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5" label2 = "CRCM5-HCD-RL-INTFL-improved" ############ images_folder = "images_for_lake-river_paper/comp_soil_profiles" if not os.path.isdir(images_folder): os.mkdir(images_folder) fldirs = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(path1) lake_fractions = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_LAKE_FRACTION_NAME) cell_areas = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_CELL_AREA_NAME_M2) acc_areakm2 = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) depth_to_bedrock = analysis.get_array_from_file( path=path1, var_name=infovar.HDF_DEPTH_TO_BEDROCK_NAME) cell_manager = CellManager(fldirs, lons2d=lons2d, lats2d=lats2d, accumulation_area_km2=acc_areakm2) #get climatologic liquid soil moisture and convert fractions to mm t0 = time.clock() daily_dates, levels, i1_nointfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path1, var_name="I1", start_year=start_year, end_year=end_year) print("read I1 - 1") print("Spent {0} seconds ".format(time.clock() - t0)) _, _, i1_intfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path2, var_name="I1", start_year=start_year, end_year=end_year) print("read I1 - 2") #get climatologic frozen soil moisture and convert fractions to mm _, _, i2_nointfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path1, var_name="I2", start_year=start_year, end_year=end_year) print("read I2 - 1") _, _, i2_intfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path2, var_name="I2", start_year=start_year, end_year=end_year) print("read I2 - 2") # sm_intfl = i1_intfl + i2_intfl sm_nointfl = i1_nointfl + i2_nointfl #Get the list of stations to do the comparison with stations = cehq_station.read_station_data( start_date=datetime(start_year, 1, 1), end_date=datetime(end_year, 12, 31), selected_ids=selected_station_ids) print("sm_noinfl, min, max = {0}, {1}".format(sm_nointfl.min(), sm_nointfl.max())) print("sm_infl, min, max = {0}, {1}".format(sm_intfl.min(), sm_intfl.max())) diff = (sm_intfl - sm_nointfl) #diff *= soil_layer_widths[np.newaxis, :, np.newaxis, np.newaxis] * 1000 # to convert in mm #print "number of nans", np.isnan(diff).astype(int).sum() print("cell area min,max = {0}, {1}".format(cell_areas.min(), cell_areas.max())) print("acc area min,max = {0}, {1}".format(acc_areakm2.min(), acc_areakm2.max())) assert np.all(lake_fractions >= 0) print("lake fractions (min, max): ", lake_fractions.min(), lake_fractions.max()) #Non need to go very deep nlayers = 3 z, t = np.meshgrid(soil_tops[:nlayers], date2num(daily_dates)) station_to_mp = cell_manager.get_model_points_for_stations(stations) plotted_global = False for the_station, mp in station_to_mp.items(): assert isinstance(mp, ModelPoint) assert isinstance(the_station, Station) fig = plt.figure() umask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices( mp.ix, mp.jy) #exclude lake cells from the profiles sel = (umask == 1) & (depth_to_bedrock > 3) & (acc_areakm2 >= 0) umaskf = umask.astype(float) umaskf *= (1.0 - lake_fractions) * cell_areas umaskf[~sel] = 0.0 profiles = np.tensordot(diff, umaskf) / umaskf.sum() print(profiles.shape, profiles.min(), profiles.max(), umaskf.sum(), umaskf.min(), umaskf.max()) d = np.abs(profiles).max() print("d = {0}".format(d)) clevs = np.round(np.linspace(-d, d, 12), decimals=5) diff_cmap = cm.get_cmap("RdBu_r", lut=len(clevs) - 1) bn = BoundaryNorm(clevs, len(clevs) - 1) plt.title("({})-({})".format(label2, label2)) img = plt.contourf(t, z, profiles[:, :nlayers], cmap=diff_cmap, levels=clevs, norm=bn) plt.colorbar(img, ticks=clevs) ax = plt.gca() assert isinstance(ax, Axes) ax.invert_yaxis() ax.xaxis.set_major_formatter(DateFormatter("%b")) ax.xaxis.set_major_locator(MonthLocator()) fig.savefig(os.path.join( images_folder, "{0}_{1}_{2}.jpeg".format(the_station.id, label1, label2)), dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight") print("processed: {0}".format(the_station)) if not plotted_global: plotted_global = True fig = plt.figure() sel = (depth_to_bedrock >= 0.1) & (acc_areakm2 >= 0) umaskf = (1.0 - lake_fractions) * cell_areas umaskf[~sel] = 0.0 profiles = np.tensordot(diff, umaskf) / umaskf.sum() print(profiles.shape, profiles.min(), profiles.max(), umaskf.sum(), umaskf.min(), umaskf.max()) d = np.abs(profiles).max() print("d = {0}".format(d)) clevs = np.round(np.linspace(-d, d, 12), decimals=5) diff_cmap = cm.get_cmap("RdBu_r", lut=len(clevs) - 1) bn = BoundaryNorm(clevs, len(clevs) - 1) img = plt.contourf(t, z, profiles[:, :nlayers], cmap=diff_cmap, levels=clevs, norm=bn) plt.colorbar(img, ticks=clevs) ax = plt.gca() assert isinstance(ax, Axes) ax.invert_yaxis() ax.xaxis.set_major_formatter(DateFormatter("%b")) ax.xaxis.set_major_locator(MonthLocator()) fig.savefig(os.path.join(images_folder, "global_mean.jpeg"), dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight") pass
def main(): #data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_highres_spinup_12_month_with_lakes" data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_test_198501_198612_0.1deg" coord_file = os.path.join(data_path, "pm1985010100_00000000p") manager = Crcm5ModelDataManager(samples_folder_path=data_path, file_name_prefix="pm", all_files_in_samples_folder=True) assert isinstance(manager, Crcm5ModelDataManager) selected_ids = [ "104001", "103715", "093806", "093801", "092715", "081006", "061502", "040830", "080718" ] start_date = datetime(1985, 1, 1) end_date = datetime(1985, 12, 31) stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date) plot_utils.apply_plot_params(width_pt=None, height_cm=30.0, width_cm=16, font_size=10) fig = plt.figure() #two columns gs = GridSpec(len(stations) // 2 + len(stations) % 2, 2, hspace=0.4, wspace=0.4) line_model, line_obs = None, None stations.sort(key=lambda x: x.latitude, reverse=True) for i, s in enumerate(stations): model_ts = manager.get_streamflow_timeseries_for_station( s, start_date=start_date, end_date=end_date, var_name="SWSR") ax = fig.add_subplot(gs[i // 2, i % 2]) assert isinstance(model_ts, TimeSeries) #[t, m_data] = model_ts.get_daily_normals() #[t, s_data] = s.get_daily_normals() assert isinstance(s, Station) #climatologies #line_model = ax.plot(t, m_data, label = "Model (CRCM5)", lw = 3, color = "b") #line_obs = ax.plot(t, s_data, label = "Observation", lw = 3, color = "r") model_ts = model_ts.get_ts_of_daily_means() print(model_ts.time[0], model_ts.time[-1]) print(model_ts.data[0:10]) mod_vals = model_ts.get_data_for_dates(s.dates) print(mod_vals[:20]) print("+" * 20) assert len(mod_vals) == len(s.dates) line_model = ax.plot(s.dates, mod_vals, label="Model (CRCM5)", lw=1, color="b") #line_obs = ax.plot(s.dates, s.values, label = "Observation", lw = 3, color = "r", alpha = 0.5) bf_store = model_ts.metadata["bankfull_store_m3"] ax.plot([s.dates[0], s.dates[-1]], [bf_store, bf_store], color="k") ax.set_title( "%s: da_diff=%.2f %%, dist = %.1f" % (s.id, (-s.drainage_km2 + model_ts.metadata["acc_area_km2"]) / s.drainage_km2 * 100.0, model_ts.metadata["distance_to_obs"])) ax.xaxis.set_major_formatter(DateFormatter("%Y")) ax.xaxis.set_major_locator(YearLocator()) fig.savefig("storage.png") pass
def validate_swe(model_file, obs_manager, season_to_months, simlabel, season_to_plot_indices, start_year, end_year, lake_fraction=None, station_ids_list=None): model_var_name = "I5" model_level = None reasonable_error_mm = 100.0 assert isinstance(obs_manager, SweDataManager) if station_ids_list is not None: # Get the list of stations to indicate on the bias map stations = cehq_station.read_station_data( start_date=None, end_date=None, selected_ids=station_ids_list) else: stations = [] if lake_fraction is not None: print("lake fraction ranges: {0}, {1}".format(lake_fraction.min(), lake_fraction.max())) fig = plt.figure() obs_manager.name = "Obs." fig.suptitle("({0}) - ({1})".format(simlabel, obs_manager.name)) # 1. read model results # 2. plot the differences (model - obs) lon, lat, basemap = analysis.get_basemap_from_hdf(file_path=model_file) # do calculations and only after that do the plotting season_to_field = {} # calculate global min and max for plotting vmin = None vmax = None season_to_obs_field = {} for season, months in season_to_months.items(): model_field = analysis.get_seasonal_climatology( start_year=start_year, end_year=end_year, months=months, level=model_level, var_name=model_var_name, hdf_path=model_file) obs_field = obs_manager.getMeanFieldForMonthsInterpolatedTo( months=months, lons_target=lon, lats_target=lat, start_year=start_year, end_year=end_year) season_to_obs_field[season] = obs_field # calculate the difference between the modelled and observed fields the_diff = model_field - obs_field current_min = np.min(the_diff) current_max = np.max(the_diff) if vmin is not None: vmin = current_min if current_min < vmin else vmin vmax = current_max if current_max > vmax else vmax else: vmin = current_min vmax = current_max season_to_field[season] = the_diff ncolors = 11 gs = gridspec.GridSpec(2, 3, width_ratios=[1, 1, 0.05]) x, y = basemap(lon, lat) im = None d = min(abs(vmin), abs(vmax)) d = 100 # limit module of the difference to 200 mm vmin = -d vmax = d # bn, bounds, _, _ = infovar.get_boundary_norm(vmin, vmax, ncolors, exclude_zero=True) bounds = [-100, -80, -50, -20, -10, -5] bounds += [ 0, ] + [-b for b in reversed(bounds)] bn = BoundaryNorm(bounds, ncolors=len(bounds) - 1) cmap = cm.get_cmap("RdBu_r", len(bounds) - 1) print("bounds: ", bounds) cs = None for season, field in season_to_field.items(): if season.lower() == "summer": print("Warning: skipping summer season for SWE") continue row, col = season_to_plot_indices[season] ax = fig.add_subplot(gs[row, col]) ax.set_title(season) basemap.drawmapboundary(fill_color="gray") if lake_fraction is not None: to_plot = np.ma.masked_where((lake_fraction > 0.9), season_to_field[season]) else: to_plot = season_to_field[season] to_plot = maskoceans(lon, lat, to_plot) im = basemap.pcolormesh(x, y, to_plot, vmin=vmin, vmax=vmax, cmap=cmap, norm=bn) basemap.drawcoastlines(ax=ax, linewidth=cpp.COASTLINE_WIDTH) # small_error = ((np.abs(season_to_field[season]) < reasonable_error_mm) | to_plot.mask).astype(int) # nlevs = 1 # ax.contour(x, y, small_error, nlevs, colors = "black", linestyle = "-") # cs = ax.contourf(x, y, small_error, nlevs, colors="none", hatches=["/", None], extend="lower", linewidth=2) for the_station in stations: assert isinstance(the_station, Station) xst, yst = basemap(the_station.longitude, the_station.latitude) # ax.annotate(the_station.id, (xst, yst), font_properties=FontProperties(size=6), # bbox=dict(facecolor="w"), va="top", ha="right") basemap.scatter(xst, yst, c="g") # artists, labels = cs.legend_elements() # plt.legend(artists, labels, handleheight=2) cax = fig.add_subplot(gs[:, 2]) units_str = r"${\rm mm}$" var_str = r"SWE" cax.set_title("{0}\n".format(units_str)) plt.colorbar(im, cax=cax, ticks=bounds, extend="both") seasons_str = "_".join( sorted([str(s) for s in list(season_to_months.keys())])) atm_val_folder = os.path.join(images_folder, "validate_atm") if not os.path.isdir(atm_val_folder): os.mkdir(atm_val_folder) out_filename = "{3}/validate_2d_{0}_{1}_{2}.png".format( model_var_name, simlabel, seasons_str, atm_val_folder) fig.savefig(os.path.join(images_folder, out_filename), bbox_inches="tight")
def validate_as_is(): #years are inclusive start_year = 1979 end_year = 1988 sim_name_list = ["crcm5-r", "crcm5-hcd-r", "crcm5-hcd-rl"] rpn_folder_path_form = "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup" nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb" #select stations selected_ids = None start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date) varname = "STFL" sim_name_to_manager = {} sim_name_to_station_to_model_point = {} dmManager = None for sim_name in sim_name_list: rpn_folder = rpn_folder_path_form.format(sim_name) dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="dm", all_files_in_samples_folder=True, need_cell_manager=True) sim_name_to_manager[sim_name] = dmManager nc_path = os.path.join(nc_db_folder, sim_name) nc_path = os.path.join(nc_path, "{0}_all.nc".format(varname)) st_to_mp = dmManager.get_model_points_for_stations(stations, nc_path=nc_path, varname=varname) sim_name_to_station_to_model_point[sim_name] = st_to_mp common_lake_fractions = dmManager.lake_fraction from matplotlib.backends.backend_pdf import PdfPages pp = PdfPages('comp_with_obs_as_is.pdf') for s in stations: #check the availability of the data assert isinstance(s, Station) #s.get_longest_continuous_series() plt.figure() obs_data = [s.date_to_value[d] for d in s.dates] obs_ann_mean = np.mean(obs_data) plt.plot(s.dates, [s.date_to_value[d] for d in s.dates], label="Obs \n ann.mean = {0:.1f}".format(obs_ann_mean)) mp = None for sim_name in sim_name_list: manager = sim_name_to_manager[sim_name] if s not in sim_name_to_station_to_model_point[sim_name]: continue mp = sim_name_to_station_to_model_point[sim_name][s] plt.plot(mp.time, mp.data[:, 0], label="{0}: {1:.2f} \n ann.mean = {2:.1f}".format( sim_name, manager.lake_fraction[mp.flow_in_mask == 1].mean(), mp.data[:, 0].mean())) plt.legend() if mp is None: continue plt.title("{0}: point lake fraction={1:.4f}".format( s.id, common_lake_fractions[mp.ix, mp.jy])) pp.savefig() pp.close()
def main(start_year = 1980, end_year = 1989): soil_layer_widths = infovar.soil_layer_widths_26_to_60 soil_tops = np.cumsum(soil_layer_widths).tolist()[:-1] soil_tops = [0, ] + soil_tops selected_station_ids = [ "061905", "074903", "090613", "092715", "093801", "093806" ] # path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf" # label1 = "CRCM5-HCD-RL" path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5" label1 = "CRCM5-HCD-RL-INTFL" path2 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5" label2 = "CRCM5-HCD-RL-INTFL-improved" ############ images_folder = "images_for_lake-river_paper/comp_soil_profiles" if not os.path.isdir(images_folder): os.mkdir(images_folder) fldirs = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(path1) lake_fractions = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_LAKE_FRACTION_NAME) cell_areas = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_CELL_AREA_NAME_M2) acc_areakm2 = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) depth_to_bedrock = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_DEPTH_TO_BEDROCK_NAME) cell_manager = CellManager(fldirs, lons2d=lons2d, lats2d=lats2d, accumulation_area_km2=acc_areakm2) #get climatologic liquid soil moisture and convert fractions to mm t0 = time.clock() daily_dates, levels, i1_nointfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path1, var_name="I1", start_year=start_year, end_year=end_year ) print("read I1 - 1") print("Spent {0} seconds ".format(time.clock() - t0)) _, _, i1_intfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path2, var_name="I1", start_year=start_year, end_year=end_year ) print("read I1 - 2") #get climatologic frozen soil moisture and convert fractions to mm _, _, i2_nointfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path1, var_name="I2", start_year=start_year, end_year=end_year ) print("read I2 - 1") _, _, i2_intfl = analysis.get_daily_climatology_of_3d_field( path_to_hdf_file=path2, var_name="I2", start_year=start_year, end_year=end_year ) print("read I2 - 2") # sm_intfl = i1_intfl + i2_intfl sm_nointfl = i1_nointfl + i2_nointfl #Get the list of stations to do the comparison with stations = cehq_station.read_station_data( start_date=datetime(start_year, 1, 1), end_date=datetime(end_year, 12, 31), selected_ids=selected_station_ids ) print("sm_noinfl, min, max = {0}, {1}".format(sm_nointfl.min(), sm_nointfl.max())) print("sm_infl, min, max = {0}, {1}".format(sm_intfl.min(), sm_intfl.max())) diff = (sm_intfl - sm_nointfl) #diff *= soil_layer_widths[np.newaxis, :, np.newaxis, np.newaxis] * 1000 # to convert in mm #print "number of nans", np.isnan(diff).astype(int).sum() print("cell area min,max = {0}, {1}".format(cell_areas.min(), cell_areas.max())) print("acc area min,max = {0}, {1}".format(acc_areakm2.min(), acc_areakm2.max())) assert np.all(lake_fractions >= 0) print("lake fractions (min, max): ", lake_fractions.min(), lake_fractions.max()) #Non need to go very deep nlayers = 3 z, t = np.meshgrid(soil_tops[:nlayers], date2num(daily_dates)) station_to_mp = cell_manager.get_model_points_for_stations(stations) plotted_global = False for the_station, mp in station_to_mp.items(): assert isinstance(mp, ModelPoint) assert isinstance(the_station, Station) fig = plt.figure() umask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices(mp.ix, mp.jy) #exclude lake cells from the profiles sel = (umask == 1) & (depth_to_bedrock > 3) & (acc_areakm2 >= 0) umaskf = umask.astype(float) umaskf *= (1.0 - lake_fractions) * cell_areas umaskf[~sel] = 0.0 profiles = np.tensordot(diff, umaskf) / umaskf.sum() print(profiles.shape, profiles.min(), profiles.max(), umaskf.sum(), umaskf.min(), umaskf.max()) d = np.abs(profiles).max() print("d = {0}".format(d)) clevs = np.round(np.linspace(-d, d, 12), decimals=5) diff_cmap = cm.get_cmap("RdBu_r", lut=len(clevs) - 1) bn = BoundaryNorm(clevs, len(clevs) - 1) plt.title("({})-({})".format(label2, label2)) img = plt.contourf(t, z, profiles[:, :nlayers], cmap = diff_cmap, levels = clevs, norm = bn) plt.colorbar(img, ticks = clevs) ax = plt.gca() assert isinstance(ax, Axes) ax.invert_yaxis() ax.xaxis.set_major_formatter(DateFormatter("%b")) ax.xaxis.set_major_locator(MonthLocator()) fig.savefig(os.path.join(images_folder, "{0}_{1}_{2}.jpeg".format(the_station.id, label1, label2)), dpi = cpp.FIG_SAVE_DPI, bbox_inches = "tight") print("processed: {0}".format(the_station)) if not plotted_global: plotted_global = True fig = plt.figure() sel = (depth_to_bedrock >= 0.1) & (acc_areakm2 >= 0) umaskf = (1.0 - lake_fractions) * cell_areas umaskf[~sel] = 0.0 profiles = np.tensordot(diff, umaskf) / umaskf.sum() print(profiles.shape, profiles.min(), profiles.max(), umaskf.sum(), umaskf.min(), umaskf.max()) d = np.abs(profiles).max() print("d = {0}".format(d)) clevs = np.round(np.linspace(-d, d, 12), decimals=5) diff_cmap = cm.get_cmap("RdBu_r", lut=len(clevs) - 1) bn = BoundaryNorm(clevs, len(clevs) - 1) img = plt.contourf(t, z, profiles[:, :nlayers], cmap = diff_cmap, levels = clevs, norm = bn) plt.colorbar(img, ticks = clevs) ax = plt.gca() assert isinstance(ax, Axes) ax.invert_yaxis() ax.xaxis.set_major_formatter(DateFormatter("%b")) ax.xaxis.set_major_locator(MonthLocator()) fig.savefig(os.path.join(images_folder, "global_mean.jpeg"), dpi = cpp.FIG_SAVE_DPI, bbox_inches = "tight") pass
def validate_daily_climatology(): """ """ #years are inclusive start_year = 1979 end_year = 1988 #sim_name_list = ["crcm5-r", "crcm5-hcd-r", "crcm5-hcd-rl"] sim_name_list = ["crcm5-hcd-rl", "crcm5-hcd-rl-intfl"] rpn_folder_paths = [ "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup".format(sim_name_list[0]), "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_{0}_spinup2/Samples_all_in_one_folder".format( sim_name_list[1]) ] nc_db_folder = "/home/huziy/skynet3_rech1/crcm_data_ncdb" #select stations selected_ids = None selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905", "041903", "040830", "093806", "090613", "081002", "093801", "080718"] selected_ids = ["074903", ] start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) selected_ids = None stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date ) stations_hydat = cehq_station.read_hydat_station_data(folder_path="/home/huziy/skynet3_rech1/HYDAT", start_date=start_date, end_date=end_date) stations.extend(stations_hydat) varname = "STFL" sim_name_to_manager = {} sim_name_to_station_to_model_point = {} day_stamps = Station.get_stamp_days(2001) sweManager = SweDataManager(var_name="SWE") cruTempManager = CRUDataManager(lazy=True) cruPreManager = CRUDataManager(var_name="pre", lazy=True, path="data/cru_data/CRUTS3.1/cru_ts_3_10.1901.2009.pre.dat.nc") #common lake fractions when comparing simulations on the same grid all_model_points = [] cell_manager = None for sim_name, rpn_folder in zip(sim_name_list, rpn_folder_paths): dmManager = Crcm5ModelDataManager(samples_folder_path=rpn_folder, file_name_prefix="dm", all_files_in_samples_folder=True, need_cell_manager=cell_manager is None) #here using the fact that all the simulations are on the same grid if cell_manager is None: cell_manager = dmManager.cell_manager else: dmManager.cell_manager = cell_manager #determine comon lake fractions, so it is not taken from the trivial case lf = 0, but note #this has only sense when all the simulations were performed on the same grid sim_name_to_manager[sim_name] = dmManager nc_sim_folder = os.path.join(nc_db_folder, sim_name) nc_path = os.path.join(nc_sim_folder, "{0}_all.nc4".format(varname)) #In general there are several model points corresponding to a given station st_to_mp = dmManager.get_model_points_for_stations(stations, sim_name=sim_name, nc_path=nc_path, nc_sim_folder=nc_sim_folder, set_data_to_model_points=True) print("got model points for stations") sim_name_to_station_to_model_point[sim_name] = st_to_mp #save model points to a list of all points for s, mps in st_to_mp.items(): assert isinstance(s, Station) for mp in mps: assert isinstance(mp, ModelPoint) #calculate upstream swe if needed if s.mean_swe_upstream_daily_clim is None: s.mean_swe_upstream_daily_clim = sweManager.get_mean_upstream_timeseries_daily(mp, dmManager, stamp_dates=day_stamps) #These are taken from CRU dataset, only monthly data are available s.mean_temp_upstream_monthly_clim = cruTempManager.get_mean_upstream_timeseries_monthly(mp, dmManager) s.mean_prec_upstream_monthly_clim = cruPreManager.get_mean_upstream_timeseries_monthly(mp, dmManager) print("Calculated observed upstream mean values...") all_model_points.extend(mps) print("imported input data successfully, plotting ...") #for tests #test(sim_name_to_station_to_model_point) #select only stations which have corresponding model points stations = list(sim_name_to_station_to_model_point[sim_name_list[0]].keys()) from matplotlib.backends.backend_pdf import PdfPages for s in stations: years = s.get_list_of_complete_years() if len(years) < 6: continue #skip stations with less than 6 continuous years of data pp = PdfPages("nc_diagnose_{0}.pdf".format(s.id)) #plot hydrographs fig = plt.figure() gs = gridspec.GridSpec(3, 3, left=0.05, hspace=0.3, wspace=0.2) ax_stfl = fig.add_subplot(gs[0, 0]) labels, handles = plot_hydrographs(ax_stfl, s, sim_name_to_station_to_model_point, day_stamps=day_stamps, sim_names=sim_name_list ) plt.setp(ax_stfl.get_xticklabels(), visible=False) #do not show ticklabels for upper rows fig.legend(handles, labels, "lower right") #plot swe 1d compare with obs ax_swe = fig.add_subplot(gs[1, 0], sharex=ax_stfl) plot_swe_1d_compare_with_obs(ax_swe, s, sim_name_to_station_to_model_point, day_stamps=day_stamps, sim_names=sim_name_list) #plot mean temp 1d compare with obs -- here plot biases directly...?? ax_temp = fig.add_subplot(gs[0, 2]) plot_temp_1d_compare_with_obs(ax_temp, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) plt.setp(ax_temp.get_xticklabels(), visible=False) #do not show ticklabels for upper rows #plot mean precip 1d compare with obs -- here plot biases directly...?? ax = fig.add_subplot(gs[1, 2], sharex=ax_temp) plot_precip_1d_compare_with_obs(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) #plot mean Surface and subsurface runoff ax = fig.add_subplot(gs[0, 1], sharex=ax_stfl) plot_surf_runoff(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) plt.setp(ax.get_xticklabels(), visible=False) #do not show ticklabels for upper rows ax = fig.add_subplot(gs[1, 1], sharex=ax_stfl) plot_subsurf_runoff(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) plt.setp(ax.get_xticklabels(), visible=False) #do not show ticklabels for upper rows ax = fig.add_subplot(gs[2, 1], sharex=ax_stfl) plot_total_runoff(ax, s, sim_name_to_station_to_model_point, sim_names=sim_name_list) pp.savefig() #plot flow direction and basin boundaries fig = plt.figure() gs = gridspec.GridSpec(1, 2, right=0.99, bottom=0.001) ax = fig.add_subplot(gs[0, 1]) plot_flow_directions_and_basin_boundaries(ax, s, sim_name_to_station_to_model_point, sim_name_to_manager=sim_name_to_manager) pp.savefig() #plot 2d correlation between wind speed and measured streamflow at the station pp.close()
def diagnose(station_ids=None, model_data_path=None): manager = Crcm5ModelDataManager(samples_folder_path=model_data_path, file_name_prefix="pm", all_files_in_samples_folder=True, need_cell_manager=True) nx, ny = manager.lons2D.shape rot_lat_lon = RotatedLatLon(lon1=-68, lat1=52, lon2=16.65, lat2=0.0) x00, y00 = rot_lat_lon.toProjectionXY(manager.lons2D[0, 0], manager.lats2D[0, 0]) x10, y10 = rot_lat_lon.toProjectionXY(manager.lons2D[1, 0], manager.lats2D[1, 0]) x01, y01 = rot_lat_lon.toProjectionXY(manager.lons2D[0, 1], manager.lats2D[0, 1]) dx = x10 - x00 dy = y01 - y00 print("dx, dy = {0}, {1}".format(dx, dy)) areas = rot_lat_lon.get_areas_of_gridcells( dx, dy, nx, ny, y00, 1) #1 -since the index is starting from 1 print(areas[0, 0]) start_date = datetime(1986, 1, 1) end_date = datetime(1986, 12, 31) stations = cehq_station.read_station_data(selected_ids=station_ids, start_date=start_date, end_date=end_date) stations.sort(key=lambda x: x.latitude, reverse=True) for i, s in enumerate(stations): fig = plt.figure() #3 columns gs = GridSpec(5, 3, hspace=0.2, wspace=0.2, right=0.98, left=0.1, top=0.98) model_ts = manager.get_streamflow_timeseries_for_station( s, start_date=start_date, end_date=end_date, nneighbours=9) print(model_ts.time[0], model_ts.time[-1]) i_model0, j_model0 = model_ts.metadata["ix"], model_ts.metadata["jy"] mask = manager.get_mask_for_cells_upstream(i_model0, j_model0) #hydrographs ax = fig.add_subplot(gs[0, 0]) plot_streamflows(ax, s, model_ts) #relative error ax = fig.add_subplot(gs[1, 0]) plot_streamflow_re(ax, s, model_ts) #directions plot_directions_and_positions(fig.add_subplot(gs[:2, 1]), s, model_ts, manager, rot_lat_lon, mask=mask) #runoff ax = fig.add_subplot(gs[2, 0]) plot_runoff(ax, manager, areas, model_ts, mask=mask) #runoff from gldas ax = fig.add_subplot(gs[2, 1]) #plot_gldas_runoff(ax, manager, areas, model_ts, mask = mask) #temperature ax_temp = fig.add_subplot(gs[3, 0]) ax_prec = fig.add_subplot(gs[4, 0]) plot_total_precip_and_temp_re_1d(ax_prec, ax_temp, manager, rot_lat_lon, areas, model_ts, mask=mask) #swe timeseries ax = fig.add_subplot(gs[3, 1]) plot_swe_timeseries(ax, manager, areas, model_ts, mask=mask) #print np.where(mask == 1) print("(i, j) = ({0}, {1})".format(model_ts.metadata["ix"], model_ts.metadata["jy"])) fig.savefig("diagnose_{0}_{1:.2f}deg.pdf".format(s.id, dx))
def main(): data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_test_198501_198612_0.1deg" coord_file = os.path.join(data_path, "pm1985010100_00000000p") manager = Crcm5ModelDataManager(samples_folder_path=data_path, file_name_prefix="pm", all_files_in_samples_folder=True) selected_ids = [ "104001", "103715", "093806", "093801", "092715", "081006", "061502", "040830", "080718" ] start_date = datetime(1986, 1, 1) end_date = datetime(1986, 12, 31) stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date) plot_utils.apply_plot_params(width_pt=None, height_cm=30.0, width_cm=16, font_size=10) fig = plt.figure() #two columns gs = GridSpec(len(stations) // 2 + len(stations) % 2, 2, hspace=0.4, wspace=0.4) line_model, line_obs = None, None stations.sort(key=lambda x: x.latitude, reverse=True) for i, s in enumerate(stations): model_ts = manager.get_streamflow_timeseries_for_station( s, start_date=start_date, end_date=end_date) ax = fig.add_subplot(gs[i // 2, i % 2]) assert isinstance(model_ts, TimeSeries) #[t, m_data] = model_ts.get_daily_normals() #[t, s_data] = s.get_daily_normals() assert isinstance(s, Station) #climatologies #line_model = ax.plot(t, m_data, label = "Model (CRCM5)", lw = 3, color = "b") #line_obs = ax.plot(t, s_data, label = "Observation", lw = 3, color = "r") model_ts = model_ts.get_ts_of_daily_means() print(model_ts.time[0], model_ts.time[-1]) print(model_ts.data[0:10]) mod_vals = model_ts.get_data_for_dates(s.dates) print(mod_vals[:20]) print("+" * 20) assert len(mod_vals) == len(s.dates) #model_acorr = [1] + [ np.corrcoef([mod_vals[i:], mod_vals[:-i]])[0,1] for i in range(1,len(mod_vals)) ] #obs_acorr = [1] + [ np.corrcoef([s.values[i:], s.values[:-i]])[0,1] for i in range(1,len(mod_vals)) ] npoints = np.array(list(range(len(mod_vals), 0, -1))) model_acorr = np.correlate(mod_vals, mod_vals, mode="full") model_acorr = model_acorr[len(model_acorr) / 2:] / max(model_acorr) model_acorr /= npoints obs_acorr = np.correlate(s.values, s.values, mode="full") obs_acorr = obs_acorr[len(obs_acorr) / 2:] / max(obs_acorr) obs_acorr /= npoints print(len(model_acorr), len(s.dates)) line_model = ax.plot(s.dates, model_acorr, label="Model (CRCM5)", lw=1, color="b") line_obs = ax.plot(s.dates, obs_acorr, label="Observation", lw=3, color="r", alpha=0.5) #ax.annotate( "r = {0:.2f}".format( float( np.corrcoef([mod_vals, s.values])[0,1] )), xy = (0.7,0.8), xycoords= "axes fraction") ax.set_title( "%s: da_diff=%.2f %%, dist = %.1f" % (s.id, (-s.drainage_km2 + model_ts.metadata["acc_area_km2"]) / s.drainage_km2 * 100.0, model_ts.metadata["distance_to_obs"])) ax.xaxis.set_major_formatter(DateFormatter("%b")) ax.xaxis.set_major_locator(MonthLocator(bymonth=list(range(1, 13, 2)))) lines = (line_model, line_obs) labels = ("Model (CRCM5)", "Observation") fig.legend(lines, labels) fig.savefig("acorr_without_lakes_0.1deg_1year.png") pass
def main(): # Define the simulations to be validated r_config = RunConfig( data_path= "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5", start_year=1990, end_year=2010, label="CRCM5-L1") r_config_list = [r_config] r_config = RunConfig( data_path= "/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r.hdf5", start_year=1990, end_year=2010, label="CRCM5-NL") r_config_list.append(r_config) bmp_info = analysis.get_basemap_info_from_hdf(file_path=r_config.data_path) bmp_info.should_draw_grey_map_background = True bmp_info.should_draw_basin_boundaries = False bmp_info.map_bg_color = "0.75" station_ids = ["104001", "093806", "093801", "081002", "081007", "080718"] # get river network information used in the model flow_directions = analysis.get_array_from_file( r_config.data_path, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME) accumulation_area_km2 = analysis.get_array_from_file( path=r_config.data_path, var_name=infovar.HDF_ACCUMULATION_AREA_NAME) cell_manager = CellManager(flow_dirs=flow_directions, lons2d=bmp_info.lons, lats2d=bmp_info.lats, accumulation_area_km2=accumulation_area_km2) # Get the list of stations to indicate on the bias map stations = cehq_station.read_station_data(start_date=None, end_date=None, selected_ids=station_ids) """:type : list[Station]""" xx, yy = bmp_info.get_proj_xy() station_to_modelpoint = cell_manager.get_model_points_for_stations( station_list=stations) upstream_edges = cell_manager.get_upstream_polygons_for_points( model_point_list=station_to_modelpoint.values(), xx=xx, yy=yy) bmp_info.draw_colorbar_for_each_subplot = True # Validate temperature, precip and swe obs_path_anusplin = "/home/huziy/skynet3_rech1/anusplin_links" obs_path_swe = "data/swe_ross_brown/swe.nc" model_var_to_obs_path = OrderedDict([("TT", obs_path_anusplin), ("I5", obs_path_swe)]) model_var_to_season = OrderedDict([ ("TT", OrderedDict([("Spring", range(3, 6))])), ("I5", OrderedDict([("Winter", [1, 2, 12])])) ]) vname_to_obs_data = {} # parameters that won't change in the loop over variable names params_const = dict(rconfig=r_config, bmp_info=bmp_info) for vname, obs_path in model_var_to_obs_path.items(): season_to_obs_data = get_seasonal_clim_obs_data( vname=vname, obs_path=obs_path, season_to_months=model_var_to_season[vname], **params_const) # Comment swe over lakes, since I5 calculated only for land if vname in [ "I5", ]: for season in season_to_obs_data: season_to_obs_data[season] = maskoceans( bmp_info.lons, bmp_info.lats, season_to_obs_data[season], inlands=True) vname_to_obs_data[vname] = season_to_obs_data # Plotting plot_all_vars_in_one_fig = True fig = None gs = None if plot_all_vars_in_one_fig: plot_utils.apply_plot_params(font_size=12, width_pt=None, width_cm=25, height_cm=20) fig = plt.figure() ncols = len(model_var_to_obs_path) + 1 gs = GridSpec(len(r_config_list), ncols, width_ratios=(ncols - 1) * [ 1., ] + [ 0.05, ]) else: plot_utils.apply_plot_params(font_size=12, width_pt=None, width_cm=25, height_cm=25) station_x_list = [] station_y_list = [] mvarname_to_cs = {} for row, r_config in enumerate(r_config_list): for col, mname in enumerate(model_var_to_obs_path): row_axes = [ fig.add_subplot(gs[row, col]), ] mvarname_to_cs[mname] = compare_vars( vname_model=mname, vname_to_obs=vname_to_obs_data, r_config=r_config, season_to_months=model_var_to_season[mname], bmp_info_agg=bmp_info, axes_list=row_axes) # -1 in order to exclude colorbars for the_ax in row_axes: the_ax.set_title(the_ax.get_title() + ", {}".format( infovar.get_long_display_label_for_var(mname))) # Need titles only for the first row if row > 0: the_ax.set_title("") if col == 0: the_ax.set_ylabel(r_config.label) else: the_ax.set_ylabel("") draw_upstream_area_bounds(the_ax, upstream_edges, color="g") if len(station_x_list) == 0: for the_station in stations: xst, yst = bmp_info.basemap(the_station.longitude, the_station.latitude) station_x_list.append(xst) station_y_list.append(yst) bmp_info.basemap.scatter(station_x_list, station_y_list, c="g", ax=the_ax, s=20, zorder=10, alpha=0.5) # Save the figure if necessary if plot_all_vars_in_one_fig: if not img_folder.is_dir(): img_folder.mkdir(parents=True) fig_path = img_folder.joinpath("{}.png".format( "_".join(model_var_to_obs_path))) with fig_path.open("wb") as figfile: fig.savefig(figfile, format="png", bbox_inches="tight") plt.close(fig)
def main(): data_path = "/home/huziy/skynet3_exec1/from_guillimin/quebec_test_198501_198612_0.1deg" coord_file = os.path.join(data_path, "pm1985010100_00000000p") manager = Crcm5ModelDataManager(samples_folder_path=data_path, file_name_prefix="pm", all_files_in_samples_folder=True ) selected_ids = ["104001", "103715", "093806", "093801", "092715", "081006", "061502", "040830", "080718"] start_date = datetime(1986, 1, 1) end_date = datetime(1986, 12, 31) stations = cehq_station.read_station_data(selected_ids = selected_ids, start_date=start_date, end_date=end_date ) plot_utils.apply_plot_params(width_pt=None, height_cm =30.0, width_cm=16, font_size=10) fig = plt.figure() #two columns gs = GridSpec( len(stations) // 2 + len(stations) % 2, 2, hspace=0.4, wspace=0.4 ) line_model, line_obs = None, None stations.sort(key=lambda x: x.latitude, reverse=True) for i, s in enumerate(stations): model_ts = manager.get_streamflow_timeseries_for_station(s, start_date = start_date, end_date = end_date) ax = fig.add_subplot( gs[i // 2, i % 2] ) assert isinstance(model_ts, TimeSeries) #[t, m_data] = model_ts.get_daily_normals() #[t, s_data] = s.get_daily_normals() assert isinstance(s, Station) #climatologies #line_model = ax.plot(t, m_data, label = "Model (CRCM5)", lw = 3, color = "b") #line_obs = ax.plot(t, s_data, label = "Observation", lw = 3, color = "r") model_ts = model_ts.get_ts_of_daily_means() print(model_ts.time[0], model_ts.time[-1]) print(model_ts.data[0:10]) mod_vals = model_ts.get_data_for_dates(s.dates) print(mod_vals[:20]) print("+" * 20) assert len(mod_vals) == len(s.dates) #model_acorr = [1] + [ np.corrcoef([mod_vals[i:], mod_vals[:-i]])[0,1] for i in range(1,len(mod_vals)) ] #obs_acorr = [1] + [ np.corrcoef([s.values[i:], s.values[:-i]])[0,1] for i in range(1,len(mod_vals)) ] npoints = np.array(list(range(len(mod_vals), 0, -1))) model_acorr = np.correlate(mod_vals, mod_vals, mode="full") model_acorr = model_acorr[len(model_acorr) / 2:] / max(model_acorr) model_acorr /= npoints obs_acorr = np.correlate(s.values, s.values, mode = "full") obs_acorr = obs_acorr[len(obs_acorr) / 2 :] / max(obs_acorr) obs_acorr /= npoints print(len(model_acorr), len(s.dates)) line_model = ax.plot(s.dates, model_acorr, label = "Model (CRCM5)", lw = 1, color = "b") line_obs = ax.plot(s.dates, obs_acorr, label = "Observation", lw = 3, color = "r", alpha = 0.5) #ax.annotate( "r = {0:.2f}".format( float( np.corrcoef([mod_vals, s.values])[0,1] )), xy = (0.7,0.8), xycoords= "axes fraction") ax.set_title("%s: da_diff=%.2f %%, dist = %.1f" % (s.id, (-s.drainage_km2+ model_ts.metadata["acc_area_km2"]) / s.drainage_km2 * 100.0, model_ts.metadata["distance_to_obs"])) ax.xaxis.set_major_formatter(DateFormatter("%b")) ax.xaxis.set_major_locator(MonthLocator(bymonth=list(range(1,13, 2)))) lines = (line_model, line_obs) labels = ("Model (CRCM5)", "Observation" ) fig.legend(lines, labels) fig.savefig("acorr_without_lakes_0.1deg_1year.png") pass
def validate_precip(model_file="", simlabel="", obs_manager=None, season_to_months=None, start_year=None, end_year=None, season_to_plot_indices=None, station_ids_list=None): """ :param model_file: :param obs_manager: should implement the method getMeanFieldForMonthsInterpolatedTo(self, months = None, lonsTarget = None, latsTarget = None) anusplin data is in mm/day model data is in m/s """ model_var_name = "PR" if obs_manager is None: print("Skipping validation of {}, since the obs manager is None.".format(model_var_name)) return if station_ids_list is not None: # Get the list of stations to indicate on the bias map stations = cehq_station.read_station_data( start_date=None, end_date=None, selected_ids=station_ids_list ) else: stations = [] model_level = 0 reasonable_error_mm_per_day = 1 assert isinstance(obs_manager, AnuSplinManager) fig = plt.figure() assert isinstance(fig, Figure) fig.suptitle("({0}) - ({1})".format(simlabel, "Obs.")) lon, lat, basemap = analysis.get_basemap_from_hdf(file_path=model_file) # do calculations and only after that do the plotting season_to_field = {} # calculate global min and max for plotting vmin = None vmax = None for season, months in season_to_months.items(): model_field = analysis.get_seasonal_climatology(start_year=start_year, end_year=end_year, months=months, level=model_level, var_name=model_var_name, hdf_path=model_file) # convert m/s to mm/day for comparison with anusplin data model_field *= 1000.0 * 60 * 60 * 24 obs_field = obs_manager.getMeanFieldForMonthsInterpolatedTo(months=months, lonstarget=lon, latstarget=lat, start_year=start_year, end_year=end_year) # calculate the difference between the modelled and observed fields the_diff = model_field - obs_field current_min = np.min(the_diff) current_max = np.max(the_diff) if vmin is not None: vmin = current_min if current_min < vmin else vmin vmax = current_max if current_max > vmax else vmax else: vmin = current_min vmax = current_max season_to_field[season] = the_diff ncolors = 12 gs = gridspec.GridSpec(2, 3, width_ratios=[1, 1, 0.05]) cmap = cm.get_cmap("RdBu_r", ncolors) x, y = basemap(lon, lat) im = None d = min(abs(vmin), abs(vmax)) vmin = -d vmax = d bn, bounds, _, _ = infovar.get_boundary_norm(vmin, vmax, ncolors, exclude_zero=False) print("bounds: ", bounds) cs = None for season, field in season_to_field.items(): row, col = season_to_plot_indices[season] ax = fig.add_subplot(gs[row, col]) ax.set_title(season) basemap.drawmapboundary(fill_color="gray", ax=ax) im = basemap.pcolormesh(x, y, season_to_field[season], vmin=vmin, vmax=vmax, cmap=cmap, norm=bn) basemap.drawcoastlines(ax=ax, linewidth=cpp.COASTLINE_WIDTH) for the_station in stations: assert isinstance(the_station, Station) xst, yst = basemap(the_station.longitude, the_station.latitude) # ax.annotate(the_station.id, (xst, yst), font_properties=FontProperties(size=6), # bbox=dict(facecolor="w"), va="top", ha="right") basemap.scatter(xst, yst, c="g", ax=ax) # small_error = (np.abs(season_to_field[season]) < reasonable_error_mm_per_day).astype(int) # nlevs = 1 # ax.contour(x, y, small_error, nlevs, colors = "black", linestyle = "-") # cs = ax.contourf(x, y, small_error, nlevs, colors="none", hatches=["/", None], extend="lower", linewidth=2) # artists, labels = cs.legend_elements() # plt.legend(artists, labels, handleheight=2) cax = fig.add_subplot(gs[:, 2]) cax.set_title("mm/day\n") plt.colorbar(im, cax=cax, extend="both") seasons_str = "_".join(sorted([str(s) for s in list(season_to_field.keys())])) atm_val_folder = os.path.join(images_folder, "validate_atm") if not os.path.isdir(atm_val_folder): os.mkdir(atm_val_folder) out_filename = "{3}/validate_2d_{0}_{1}_{2}.png".format(model_var_name, simlabel, seasons_str, atm_val_folder) fig.savefig(os.path.join(images_folder, out_filename), bbox_inches="tight")
def main(): var_name_liquid = "I1" var_name_solid = "I2" #peirod of interest start_year = 1979 end_year = 1988 #spatial averaging will be done over upstream points to the stations selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905", "041903", "040830", "093806", "090613", "081002", "093801", "080718"] selected_ids = ["074903"] #simulation names corresponding to the paths sim_names = ["crcm5-hcd-rl", "crcm5-hcd-rl-intfl"] sim_labels = [x.upper() for x in sim_names] colors = ["blue", "violet"] layer_widths = [0.1, 0.2, 0.3, 0.4, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 0.5, 1.0, 3.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0, 5.0] layer_depths = np.cumsum(layer_widths) paths = [ "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl_spinup", "/home/huziy/skynet3_rech1/from_guillimin/new_outputs/quebec_0.1_crcm5-hcd-rl-intfl_spinup2/Samples_all_in_one" ] seasons = [ [12, 1, 2], [3, 4, 5], [6, 7, 8], [9, 10, 11] ] season_names = [ "DJF", "MAM", "JJA", "SON" ] managers = [ Crcm5ModelDataManager(samples_folder_path=path, file_name_prefix="pm", all_files_in_samples_folder=True, need_cell_manager= (i == 0)) for i, path in enumerate(paths) ] #share the cell manager a_data_manager = managers[0] assert isinstance(a_data_manager, Crcm5ModelDataManager) cell_manager = a_data_manager.cell_manager assert isinstance(cell_manager, CellManager) for m in managers[1:]: assert isinstance(m, Crcm5ModelDataManager) m.cell_manager = cell_manager #share the lake fraction field lake_fraction = a_data_manager.lake_fraction #selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905", # "041903", "040830", "093806", "090613", "081002", "093801", "080718"] start_date = datetime(start_year, 1, 1) end_date = datetime(end_year, 12, 31) stations = cehq_station.read_station_data(selected_ids = selected_ids, start_date=start_date, end_date=end_date ) #stations with corresponding model points station_to_mp = a_data_manager.get_dataless_model_points_for_stations(stations) #figure out levels in soil sim_label_to_profiles = {} for s, mp in station_to_mp.items(): assert isinstance(mp, ModelPoint) mask = (mp.flow_in_mask == 1) & (lake_fraction < 0.6) fig = plt.figure() fmt = ScalarFormatter(useMathText=True) fmt.set_powerlimits([-2, 2]) print(mp.ix, mp.jy, s.id) for m, label, color in zip(managers, sim_labels, colors): assert isinstance(m, Crcm5ModelDataManager) monthly_means_liquid = _get_cached_monthly_mean_fields(label, start_year, end_year, var_name_liquid) if monthly_means_liquid is None: monthly_means_liquid = m.get_monthly_climatology_of_3d_field(var_name=var_name_liquid, start_year=start_year, end_year=end_year) _cache_monthly_mean_fields(monthly_means_liquid, label, start_year, end_year, var_name_liquid) monthly_means_solid = _get_cached_monthly_mean_fields(label, start_year, end_year, var_name_solid) if monthly_means_solid is None: monthly_means_solid = m.get_monthly_climatology_of_3d_field(var_name=var_name_solid, start_year=start_year, end_year=end_year) _cache_monthly_mean_fields(monthly_means_solid, label, start_year, end_year, var_name_solid) profiles = [ monthly_means_liquid[i][mask,:].mean(axis = 0) + monthly_means_solid[i][mask,:].mean(axis = 0) for i in range(12) ] sim_label_to_profiles[label] = np.array( profiles ) x = [ date2num( datetime(2001,month,1) ) for month in range(1,13)] y = layer_depths y2d, x2d = np.meshgrid(y, x) delta = (sim_label_to_profiles[sim_labels[1]] - sim_label_to_profiles[sim_labels[0]]) / sim_label_to_profiles[sim_labels[0]] * 100 #delta = np.ma.masked_where(delta < 0.1, delta) cmap = my_colormaps.get_cmap_from_ncl_spec_file(path="colormap_files/BlueRed.rgb", ncolors=10) the_min = -6.0 the_max = 6.0 step = (the_max - the_min) / float(cmap.N) plt.pcolormesh(x2d[:,:8], y2d[:,:8], delta[:,:8], cmap = cmap, vmin = the_min, vmax = the_max) #, levels = np.arange(-6,7,1)) plt.gca().invert_yaxis() plt.colorbar(ticks = np.arange(the_min, the_max + step, step)) plt.gca().set_ylabel("Depth (m)") plt.gca().xaxis.set_major_formatter(DateFormatter("%b")) #fig.tight_layout() fig.savefig("soil_profile_upstream_of_{0}.pdf".format(s.id)) pass
def main(hdf_folder="/home/huziy/skynet3_rech1/hdf_store", start_date=None, end_date=None, min_station_accumulation_area_km2=1000.0): # Station ids to get from the CEHQ database # selected_ids = ["092715", "080101", "074903", "050304", "080104", "081007", "061905", # "041903", "040830", "093806", "090613", "081002", "093801", "080718"] ids_with_lakes_upstream = [ "104001", "093806", "093801", "081002", "081007", "080718" ] # for presentation # ids_with_lakes_upstream = [ # "104001", "093806", # ] selected_ids = ["092715", "074903", "080104", "081007", "061905", "093806", "090613", "081002", "093801", "080718", "104001"] selected_ids = ids_with_lakes_upstream # selected_ids = [] # Do not use CEHQ stations temporarily # selected_ids = [ # "074903", "061905", "090613", "092715", "093801", "093806", "081002" # ] # selected_ids = ["081002", "093801"] # selected_ids = ["090613", ] sim_labels = [ # "CRCM5-R", "CRCM5-L2", # "CRCM5-L1", # "CRCM5-NL" # "CRCM5-HCD-RL-INTF-a" ] sim_file_names = [ # "quebec_0.1_crcm5-r.hdf5", "quebec_0.1_crcm5-hcd-rl.hdf5", # "quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5", # "quebec_0.1_crcm5-hcd-r.hdf5", # "quebec_0.1_crcm5-r.hdf5", # "quebec_0.1_crcm5-hcd-rl-intfl_ITFS.hdf5" # "quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5" ] simname_to_color = { "CRCM5-NL": "b", "CRCM5-L2": "r", "Obs.": "g" } sim_name_to_file_name = OrderedDict() for k, v in zip(sim_labels, sim_file_names): sim_name_to_file_name[k] = v # sim_name_to_file_name = { # "CRCM5-R": "quebec_0.1_crcm5-r.hdf5", # "CRCM5-HCD-R": "quebec_0.1_crcm5-hcd-r.hdf5", # "CRCM5-HCD-RL": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf", # "CRCM5-HCD-RL-INTFL": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_do_not_discard_small.hdf", # "SANI=10000, ignore THFC": # "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000_not_care_about_thfc.hdf", # # "CRCM5-HCD-RL-ERA075": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap_era075.hdf", # "SANI=10000": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_sani-10000.hdf" # "CRCM5-HCD-RL-ECOCLIMAP": "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ecoclimap.hdf" # } # Get the list of stations to do the comparison with stations = cehq_station.read_station_data( start_date=start_date, end_date=end_date, selected_ids=selected_ids ) print("Initial list of stations:") for s in stations: print("{0}".format(s)) assert isinstance(s, Station) cy = s.get_list_of_complete_years() print("{} of complete years: {}".format(len(cy), cy)) # Commented hydat station for performance during testing # province = "QC" # selected_ids_hydat = None # stations_hd = cehq_station.load_from_hydat_db(start_date=start_date, end_date=end_date, # province=province, # min_drainage_area_km2=min_station_accumulation_area_km2, # selected_ids=selected_ids_hydat) # if not len(stations_hd): # print "No hydat stations satisying the conditions: period {0}-{1}, province {2}".format( # str(start_date), str(end_date), province # ) # # stations.extend(stations_hd) # # province = "ON" # stations_hd = cehq_station.load_from_hydat_db(start_date=start_date, end_date=end_date, # province=province, # min_drainage_area_km2=min_station_accumulation_area_km2) # stations.extend(stations_hd) draw_model_comparison(model_points=None, sim_name_to_file_name=sim_name_to_file_name, hdf_folder=hdf_folder, start_year=start_date.year, end_year=end_date.year, stations=stations, stfl_name="STFA", drainage_area_reldiff_min=0.1, plot_upstream_area_averaged=False, sim_name_to_color=simname_to_color)
def plot_station_positions(basemap): sel_ids = [ "080707", "080717", "093801", "093806", "093808", "102701", "102704", "072301", "072302", "074902", "074903", "103702", "103703", "103715", "041902", "041903", "042103", "042607", "043012", "040212", "040814", "040830", "073801", "073802", "081002", "081006", "081007", "081008", "061502", "061801", "061901", "061905", "061906", "062102", "050119", "050135", "080704", "080718" ] stations = cehq_station.read_station_data(folder="data/cehq_measure_data_all", read_only_headers=True) stations = itertools.ifilter(lambda s: s.id in sel_ids, stations) stations = list(stations) assert len(stations) == len(sel_ids), "{0} != {1}".format(len(stations), len(sel_ids)) xsta = [] ysta = [] for s in stations: xsta.append(s.longitude) ysta.append(s.latitude) xsta, ysta = basemap(xsta, ysta) basemap.scatter(xsta, ysta, c = "r", s = 80 , zorder = 10) #color stations with yellow yellow_ids = [ "103702", "093808", "061502", "041903", "093806", "042607", "081002", "073801", "080718", "081006", "093801", "103715", "050119", "040830"] xsta = [] ysta = [] for s in stations: if s.id not in yellow_ids: continue xsta.append(s.longitude) ysta.append(s.latitude) xsta, ysta = basemap(xsta, ysta) #basemap.scatter(xsta, ysta, c = "#DCBD34", s = 80 , zorder = 10) basemap.scatter(xsta, ysta, c = "#40dae6", s = 80 , zorder = 10)
def main(): lake_names = ["Matagami", "Mistassini", "Nemiscau"] lake_areas_km2 = [370.7, 2162.7, 148.3] level_st_ids = [ "080716", "081003", "081001" ] level_stations = cehq.read_station_data(folder="data/cehq_levels", selected_ids=level_st_ids) stfl_st_ids = [ "080707", "081007", "081002" ] stfl_stations = cehq.read_station_data(folder="data/cehq_measure_data", selected_ids=stfl_st_ids) q_obs = [] q_calc = [] plt.figure() for lev_station, stfl_station, lake_name, lake_area_km2, c in zip(level_stations, stfl_stations, lake_names, lake_areas_km2, ["r","g","b"] ): assert isinstance(lev_station, Station) assert isinstance(stfl_station, Station) count_intersection = sum( [int(d in stfl_station.dates) for d in lev_station.dates] ) intersection_dates = list( filter( lambda d: d in stfl_station.dates, lev_station.dates) ) q_vals = [stfl_station.get_value_for_date(d) for d in intersection_dates] h_vals = [lev_station.get_value_for_date(d) for d in intersection_dates] q_obs.append(q_vals) q_calc.append(get_streamflows_from_active_stores_bowling(get_active_storages(h_vals,lake_area_km2), lake_area_km2)) #Calculate correlation between Q and H print(10 * "-" + lake_name) print("r = {0}".format(np.corrcoef([q_vals, h_vals])[0,1])) print((lev_station.latitude, lev_station.longitude)) print("dist_m = {0} km ".format( 1.0e-3 * lat_lon.get_distance_in_meters(lev_station.longitude, lev_station.latitude, stfl_station.longitude, stfl_station.latitude))) print("{0} and {1} have {2} measurements at the same time ({3}).".format(lev_station.id, stfl_station.id, count_intersection, lake_name )) #plot storage-discharge relation plt.title(lake_name) plt.scatter(get_active_storages(h_vals,lake_area_km2), q_vals, c = c , label = lake_name, linewidths=0) #plt.plot(intersection_dates, q_vals, c, label = lake_name ) #plt.plot(intersection_dates, get_active_storages(h_vals,lake_area_km2), c+"-", label = lake_name ) #plt.xlabel("S-active (obs)") #plt.ylabel("Q (obs)") plt.legend() #Compare observed and theoretical lake outflows plt.figure() title = "" for qo, qc, name, c in zip(q_obs, q_calc,lake_names, ["r","g","b"]): qc_a = np.array(qc) qo_a = np.array(qo) title = "ME = {0:.2f}".format(1 - sum((qc_a - qo_a) ** 2) / sum( (qo_a - qo_a.mean()) ** 2)) plt.scatter(qo, qc, c= c, linewidths=0,label = name + ", " + title) #plt.title(title) plt.xlabel("$Q_{\\rm obs}$") plt.ylabel("$Q_{\\rm mod}$") xmin, xmax = plt.xlim() print(plt.xlim()) plt.plot([xmin, xmax], [xmin, xmax], "k-",lw = 3, zorder = 5) plt.legend() plt.show() #TODO: implement pass
def main(): start_year = 1970 end_year = 1999 stations = cehq_station.read_station_data(folder="data/cehq_measure_data_all") stations = list( itertools.ifilter( lambda s: s.is_natural, stations) ) for s in stations: s.delete_data_after_year(end_year) s.delete_data_before_year(start_year) pass stations = list( itertools.ifilter(lambda s: s.get_num_of_years_with_continuous_data() >= 10, stations) ) s = stations[0] assert isinstance(s, Station) #stations = list( itertools.ifilter(lambda s: s.is_natural, stations) ) x, y = polar_stereographic.lons, polar_stereographic.lats basemap = polar_stereographic.basemap x, y = basemap(x,y) sx = [s.longitude for s in stations] sy = [s.latitude for s in stations] sx, sy = basemap(sx, sy) #read model data model_file_path = "data/streamflows/hydrosheds_euler9/aex_discharge_1970_01_01_00_00.nc" acc_area = data_select.get_field_from_file(path=model_file_path, field_name="drainage") i_indices, j_indices = data_select.get_indices_from_file(path=model_file_path) lons_1d = data_select.get_field_from_file(path=model_file_path, field_name="longitude") lats_1d = data_select.get_field_from_file(path=model_file_path, field_name="latitude") x1d, y1d, z1d = lat_lon.lon_lat_to_cartesian(lons_1d, lats_1d) kdtree = KDTree(zip(x1d, y1d, z1d)) print "Id: 4 DA (km2) <-> 4 dist (km) <-> 4 (i,j)" #basemap.scatter(sx, sy, c = "r", zorder = 5) for s, isx, isy in zip( stations, sx, sy ): assert isinstance(s, Station) plt.annotate(s.id, xy=(isx, isy), bbox = dict(facecolor = 'white'), weight = "bold", font_properties = FontProperties(size=0.5)) #get model drainaige areas for the four closest gridcells to the station x0, y0, z0 = lat_lon.lon_lat_to_cartesian(s.longitude, s.latitude) dists, indices = kdtree.query([x0, y0, z0], k = 4) dists /= 1000 print("{0}: {1:.1f}; {2:.1f}; {3:.1f}; {4:.1f} <-> {5:.1f}; {6:.1f}; {7:.1f}; {8:.1f} <-> {9};{10};{11};{12}".format( "{0} (S_DA = {1:.1f})".format(s.id, s.drainage_km2), float(acc_area[indices[0]]), float(acc_area[indices[1]]), float(acc_area[indices[2]]), float(acc_area[indices[3]]), float( dists[0] ), float(dists[1]), float(dists[2]), float(dists[3]), "({0}, {1})".format(i_indices[indices[0]] + 1, j_indices[indices[0]] + 1), "({0}, {1})".format(i_indices[indices[1]] + 1, j_indices[indices[1]] + 1), "({0}, {1})".format(i_indices[indices[2]] + 1, j_indices[indices[2]] + 1), "({0}, {1})".format(i_indices[indices[3]] + 1, j_indices[indices[3]] + 1) )) basemap.drawcoastlines(linewidth=0.5) xmin, xmax = min(sx), max(sx) ymin, ymax = min(sy), max(sy) marginx = (xmax - xmin) * 0.1 marginy = (ymax - ymin) * 0.1 xmin -= marginx * 1.5 xmax += marginx * 2 ymin -= marginy ymax += marginy * 2 plt.xlim(xmin, xmax) plt.ylim(ymin, ymax) plt.tight_layout() basin_boundaries.plot_basin_boundaries_from_shape(basemap=basemap, plotter=plt, linewidth=1) plt.savefig("10yr_cont_stations_natural_fs0.5.pdf") #plt.show() pass
def compare_hydrographs_at_stations(manager_list, start_date=None, end_date=None, img_path="hydrographs.png", colors=None, fig=None): selected_ids = None stations = cehq_station.read_station_data(selected_ids=selected_ids, start_date=start_date, end_date=end_date ) if colors is None: colors = len(manager_list) * [None] skip_stations = ["080718", "095003", "094206", "090613", "092715"] # 090613 is skipped for the 0.5 deg resolution since the drainaige network is not fully # represented by the model lines_model = [] station_to_list_of_model_ts = {} run_id_list = [m.run_id for m in manager_list] filtered_stations = [] for s in stations: assert isinstance(s, Station) if s.id in skip_stations: continue # skip stations with smaller accumulation areas if s.drainage_km2 <= 4 * np.radians(0.5) ** 2 * lat_lon.EARTH_RADIUS_METERS ** 2 * 1.0e-6: continue if not s.passes_rough_continuity_test(start_date, end_date): continue filtered_stations.append(s) stations = filtered_stations print(len(filtered_stations)) # if True: raise Exception() #save all run ids plot_utils.apply_plot_params(width_pt=None, height_cm=40.0, width_cm=30.0, font_size=10) run_id_to_dataframe = {} run_id_to_cell_props = {} for manager in manager_list: assert isinstance(manager, Crcm5ModelDataManager) df, station_to_cellprops = manager.get_streamflow_dataframe_for_stations(stations, start_date=start_date, end_date=end_date, var_name="STFL", nneighbours=9) assert isinstance(df, pandas.DataFrame) df = df.dropna(axis=1) run_id_to_cell_props[manager.run_id] = station_to_cellprops df = df.groupby(lambda i: datetime(2001, i.month + 1, 1) if i.month == 2 and i.day == 29 else datetime(2001, i.month, i.day)).mean() print(df) #again filter the stations with data time interval overlapping with model time interval stations = list(filter(lambda s: s.id in df.columns, stations)) run_id_to_dataframe[manager.run_id] = df if fig is None: fig = plt.figure() #two columns ncols = 2 nrows = len(stations) / ncols if nrows * ncols < len(stations): nrows += 1 gs = GridSpec(nrows, ncols, hspace=0.4, wspace=0.4) line_model, line_obs = None, None stations.sort(key=lambda x: x.latitude, reverse=True) plot_station_positions(manager_list[0], stations) i = -1 ns_list = [] station_list = [] flow_acc_area_list = [] #one_day = timedelta(days = 1) one_day_sec = 24 * 60 * 60.0 for s in stations: i += 1 ax = fig.add_subplot(gs[i // ncols, i % ncols]) assert isinstance(s, Station) year_dates, sta_clims = s.get_daily_normals() #plot error limits ax.fill_between(year_dates, sta_clims * 1.256, sta_clims * 0.744, alpha=0.25, color="b") line_obs = ax.plot(year_dates, sta_clims, color="b", label="Observation", lw=3, alpha=0.5) ax.annotate("{0:.3g}".format(sum(sta_clims) * one_day_sec), (0.1, 0.95), xycoords="axes fraction", color="b", alpha=0.5 ) #integral flow since those values are daily normals for run_id, color, color_index in zip(run_id_list, colors, list(range(len(colors)))): df = run_id_to_dataframe[run_id] the_line = ax.plot(year_dates, df[s.id], color=color, label=run_id, lw=1) ax.annotate("{0:.3g}".format(sum(df[s.id]) * one_day_sec), (0.1, 0.9 - color_index * 0.05), xycoords="axes fraction", color=color ) #integral flow since those values are daily normals if not i: #save the labels only for the first step lines_model.append(the_line) #dt = model_ts.time[1] - model_ts.time[0] #dt_sec = dt.days * 24 * 60 * 60 + dt.seconds #ax.annotate( "{0:g}".format( sum(mod_vals) * dt_sec ) + " ${\\rm m^3}$", xy = (0.7,0.7), xycoords= "axes fraction", color = "b") #ax.annotate( "{0:g}".format( sum(s.values) * dt_sec) + " ${\\rm m^3}$", xy = (0.7,0.6), xycoords= "axes fraction", color = "r") metadata = list(run_id_to_cell_props.items())[0][1][s.id] da_mod = metadata["acc_area_km2"] dist = metadata["distance_to_obs_km"] #ax.set_title("{0}: $\delta DA = {1:.1f}$ %, dist = {2:.1f} km".format(s.id, # (da_mod - s.drainage_km2) / s.drainage_km2 * 100.0, dist ) ) ax.set_title("{0}: $DA = {1:.1f}$ {2}".format(s.id, s.drainage_km2, "${\\rm km ^ 2}$")) ax.xaxis.set_major_formatter(DateFormatter("%m")) #ax.xaxis.set_major_locator(YearLocator()) assert isinstance(ax, Axes) ax.xaxis.axis_date() #ax.xaxis.tick_bottom().set_rotation(60) lines = lines_model + [line_obs, ] labels = run_id_list + ["Observation", ] fig.legend(lines, labels, ncol=5) if img_path is not None: fig.savefig(img_path)
def create_kml_file_for_level_stations( data_path="data/cehq_levels", kml_file_name="mon.kml", title="Water levels in meters", icon_color="ffffccee", icon_link="http://dl.dropbox.com/u/4629759/blue-L.png", data_url_format="", plot_daily_normals=False, plot_monthly_normals=False): stations = cehq_station.read_station_data(folder=data_path) width = 250 height = 100 kmlBody = ("") for s in stations: assert isinstance(s, cehq_station.Station) print(s.id) ##Monthly normals if plot_monthly_normals: values_monthly = s.get_monthly_normals() if values_monthly is None: print( "Skipping {0} since the data series is not continuous enough" .format(s.id)) continue # skip stations with incomplete data low = min(values_monthly) up = max(values_monthly) xy_monthly = Line((values_monthly - low) / (up - low) * 100.0) xy_monthly.axes.type("xyx") xy_monthly.size(width, height) xy_monthly.axes.range(0, 1, 12) xy_monthly.axes.range(1, low, up) xy_monthly.axes.label(2, None, "Month") #Daily normals if plot_daily_normals: times, values_daily = s.get_daily_normals() if values_daily is None: print( "Skipping {0} since the data series is not continuous enough" .format(s.id)) continue low = min(values_daily) up = max(values_daily) xy_daily = Line((values_daily - low) / (up - low) * 100.0) xy_daily.axes.type("xyx") xy_daily.size(width, height) xy_daily.axes.range(0, 1, 365) xy_daily.axes.range(1, low, up) xy_daily.axes.label(2, None, "Day") kml = (""" <Placemark>\n <name>%s</name>\n <Style> <IconStyle> <color>%s</color> <Icon> <href>%s</href> </Icon> </IconStyle> </Style> <description>\n <![CDATA[\n <p> <b> %s </b> </p> <p> Flow acc. area is %.1f km<sup>2<sup> </p> ]]>\n </description>\n <Point>\n <coordinates>%f, %f</coordinates>\n </Point>\n </Placemark>\n""") % (s.id, icon_color, icon_link, title, s.drainage_km2, s.longitude, s.latitude) kmlBody += kml #"morceaux" du fichier KML kmlHeader = ('<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n' '<kml xmlns=\"http://earth.google.com/kml/2.2\">\n' '<Document>\n') kmlFooter = ('</Document>\n' '</kml>\n') kmlFull = kmlHeader + kmlBody + kmlFooter open(kml_file_name, 'wb').write(kmlFull)
def put_selected_stations(ax, the_basemap, i_list, j_list): """ :type the_basemap: Basemap """ stations_dump = 'stations_dump.bin' global stations if stations is not None: pass elif os.path.isfile(stations_dump): print 'unpickling' stations = pickle.load(open(stations_dump)) else: stations = read_station_data() pickle.dump(stations, open(stations_dump, 'w')) #get selected stations sel_stations = list( itertools.ifilter( lambda x: x.id in selected_station_ids, stations ) ) xs, ys = polar_stereographic.xs, polar_stereographic.ys dx = 0.01 * ( xs[i_list, j_list].max() - xs[i_list, j_list].min() ) dy = 0.01 * ( ys[i_list, j_list].max() - ys[i_list, j_list].min() ) the_xs = [] the_ys = [] for station in sel_stations: x, y = the_basemap(station.longitude, station.latitude) the_xs.append(x) the_ys.append(y) xtext = 1.005 * x ytext = y if station.id in ['061906']: xtext = 1.00 * x ytext = 0.97 * y if station.id in ['103603', '081002']: ytext = 0.98 * y if station.id in ['081007']: xtext = 0.97 * x if station.id in ["090602"]: ytext -= 7 * dy xtext -= 5 * dx if station.id in ["090613"]: ytext += 4 * dy xtext -= 6 * dx # the_id = station.id # # xtext = None # ytext = None # if the_id.startswith("0807"): # xtext, ytext = 0.1, 0.2 # # if the_id.startswith("081006"): # xtext, ytext = 0.1, 0.3 # # if the_id.startswith("093801"): # xtext, ytext = 0.1, 0.5 # # if the_id.startswith("093806"): # xtext, ytext = 0.1, 0.6 # # if the_id.startswith("103715"): # xtext, ytext = 0.1, 0.95 # # if the_id.startswith("104001"): # xtext, ytext = 0.4, 0.95 # # if the_id.startswith("061502"): # xtext, ytext = 0.8, 0.4 # # if the_id.startswith("040830"): # xtext, ytext = 0.8, 0.2 # # if the_id.startswith("092715"): # xtext, ytext = 0.1, 0.4 # ax.annotate(station.id, xy = (x, y), xytext = (xtext, ytext), #textcoords = "axes fraction", # bbox = dict(facecolor = 'white'), weight = "bold", #arrowprops=dict(facecolor='black', width = 1, headwidth = 1.5) # ) the_basemap.scatter(the_xs,the_ys, c = 'c', s = 60, marker='^', linewidth = 0.5, alpha = 1, zorder = 5, ax = ax) pass
def create_kml_file_for_level_stations(data_path = "data/cehq_levels", kml_file_name = "mon.kml", title = "Water levels in meters", icon_color = "ffffccee", icon_link = "http://dl.dropbox.com/u/4629759/blue-L.png", data_url_format = "", plot_daily_normals = False, plot_monthly_normals = False ): stations = cehq_station.read_station_data(folder=data_path) width = 250 height = 100 kmlBody = ("") for s in stations: assert isinstance(s, cehq_station.Station) print(s.id) ##Monthly normals if plot_monthly_normals: values_monthly = s.get_monthly_normals() if values_monthly is None: print("Skipping {0} since the data series is not continuous enough".format(s.id)) continue # skip stations with incomplete data low = min(values_monthly) up = max(values_monthly) xy_monthly = Line((values_monthly - low) / (up - low) * 100.0) xy_monthly.axes.type("xyx") xy_monthly.size(width, height) xy_monthly.axes.range(0, 1,12) xy_monthly.axes.range(1, low, up) xy_monthly.axes.label(2, None, "Month") #Daily normals if plot_daily_normals: times, values_daily = s.get_daily_normals() if values_daily is None: print("Skipping {0} since the data series is not continuous enough".format(s.id)) continue low = min(values_daily) up = max(values_daily) xy_daily = Line((values_daily - low) / (up - low) * 100.0) xy_daily.axes.type("xyx") xy_daily.size(width, height) xy_daily.axes.range(0, 1,365) xy_daily.axes.range(1, low, up) xy_daily.axes.label(2, None, "Day") kml = ( """ <Placemark>\n <name>%s</name>\n <Style> <IconStyle> <color>%s</color> <Icon> <href>%s</href> </Icon> </IconStyle> </Style> <description>\n <![CDATA[\n <p> <b> %s </b> </p> <p> Flow acc. area is %.1f km<sup>2<sup> </p> ]]>\n </description>\n <Point>\n <coordinates>%f, %f</coordinates>\n </Point>\n </Placemark>\n""" ) % ( s.id, icon_color, icon_link, title, s.drainage_km2, s.longitude, s.latitude) kmlBody += kml #"morceaux" du fichier KML kmlHeader = ('<?xml version=\"1.0\" encoding=\"UTF-8\"?>\n' '<kml xmlns=\"http://earth.google.com/kml/2.2\">\n' '<Document>\n') kmlFooter = ('</Document>\n' '</kml>\n') kmlFull = kmlHeader + kmlBody + kmlFooter open(kml_file_name,'wb').write(kmlFull)
def validate_swe(model_file, obs_manager, season_to_months, simlabel, season_to_plot_indices, start_year, end_year, lake_fraction=None, station_ids_list=None): model_var_name = "I5" model_level = None reasonable_error_mm = 100.0 assert isinstance(obs_manager, SweDataManager) if station_ids_list is not None: # Get the list of stations to indicate on the bias map stations = cehq_station.read_station_data( start_date=None, end_date=None, selected_ids=station_ids_list ) else: stations = [] if lake_fraction is not None: print("lake fraction ranges: {0}, {1}".format(lake_fraction.min(), lake_fraction.max())) fig = plt.figure() obs_manager.name = "Obs." fig.suptitle("({0}) - ({1})".format(simlabel, obs_manager.name)) # 1. read model results # 2. plot the differences (model - obs) lon, lat, basemap = analysis.get_basemap_from_hdf(file_path=model_file) # do calculations and only after that do the plotting season_to_field = {} # calculate global min and max for plotting vmin = None vmax = None season_to_obs_field = {} for season, months in season_to_months.items(): model_field = analysis.get_seasonal_climatology(start_year=start_year, end_year=end_year, months=months, level=model_level, var_name=model_var_name, hdf_path=model_file) obs_field = obs_manager.getMeanFieldForMonthsInterpolatedTo(months=months, lons_target=lon, lats_target=lat, start_year=start_year, end_year=end_year) season_to_obs_field[season] = obs_field # calculate the difference between the modelled and observed fields the_diff = model_field - obs_field current_min = np.min(the_diff) current_max = np.max(the_diff) if vmin is not None: vmin = current_min if current_min < vmin else vmin vmax = current_max if current_max > vmax else vmax else: vmin = current_min vmax = current_max season_to_field[season] = the_diff ncolors = 11 gs = gridspec.GridSpec(2, 3, width_ratios=[1, 1, 0.05]) x, y = basemap(lon, lat) im = None d = min(abs(vmin), abs(vmax)) d = 100 # limit module of the difference to 200 mm vmin = -d vmax = d # bn, bounds, _, _ = infovar.get_boundary_norm(vmin, vmax, ncolors, exclude_zero=True) bounds = [-100, -80, -50, -20, -10, -5] bounds += [0, ] + [-b for b in reversed(bounds)] bn = BoundaryNorm(bounds, ncolors=len(bounds) - 1) cmap = cm.get_cmap("RdBu_r", len(bounds) - 1) print("bounds: ", bounds) cs = None for season, field in season_to_field.items(): if season.lower() == "summer": print("Warning: skipping summer season for SWE") continue row, col = season_to_plot_indices[season] ax = fig.add_subplot(gs[row, col]) ax.set_title(season) basemap.drawmapboundary(fill_color="gray") if lake_fraction is not None: to_plot = np.ma.masked_where((lake_fraction > 0.9), season_to_field[season]) else: to_plot = season_to_field[season] to_plot = maskoceans(lon, lat, to_plot) im = basemap.pcolormesh(x, y, to_plot, vmin=vmin, vmax=vmax, cmap=cmap, norm=bn) basemap.drawcoastlines(ax=ax, linewidth=cpp.COASTLINE_WIDTH) # small_error = ((np.abs(season_to_field[season]) < reasonable_error_mm) | to_plot.mask).astype(int) # nlevs = 1 # ax.contour(x, y, small_error, nlevs, colors = "black", linestyle = "-") # cs = ax.contourf(x, y, small_error, nlevs, colors="none", hatches=["/", None], extend="lower", linewidth=2) for the_station in stations: assert isinstance(the_station, Station) xst, yst = basemap(the_station.longitude, the_station.latitude) # ax.annotate(the_station.id, (xst, yst), font_properties=FontProperties(size=6), # bbox=dict(facecolor="w"), va="top", ha="right") basemap.scatter(xst, yst, c="g") # artists, labels = cs.legend_elements() # plt.legend(artists, labels, handleheight=2) cax = fig.add_subplot(gs[:, 2]) units_str = r"${\rm mm}$" var_str = r"SWE" cax.set_title("{0}\n".format(units_str)) plt.colorbar(im, cax=cax, ticks=bounds, extend="both") seasons_str = "_".join(sorted([str(s) for s in list(season_to_months.keys())])) atm_val_folder = os.path.join(images_folder, "validate_atm") if not os.path.isdir(atm_val_folder): os.mkdir(atm_val_folder) out_filename = "{3}/validate_2d_{0}_{1}_{2}.png".format(model_var_name, simlabel, seasons_str, atm_val_folder) fig.savefig(os.path.join(images_folder, out_filename), bbox_inches="tight")