def main():
# Define the simulations to be validated
r_config = RunConfig(
data_path=
"/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5",
start_year=1990,
end_year=2010,
label="CRCM5-L1")
r_config_list = [r_config]
r_config = RunConfig(
data_path=
"/RESCUE/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-r.hdf5",
start_year=1990,
end_year=2010,
label="CRCM5-NL")
r_config_list.append(r_config)
bmp_info = analysis.get_basemap_info_from_hdf(file_path=r_config.data_path)
bmp_info.should_draw_grey_map_background = True
bmp_info.should_draw_basin_boundaries = False
bmp_info.map_bg_color = "0.75"
station_ids = ["104001", "093806", "093801", "081002", "081007", "080718"]
# get river network information used in the model
flow_directions = analysis.get_array_from_file(
r_config.data_path, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME)
accumulation_area_km2 = analysis.get_array_from_file(
path=r_config.data_path, var_name=infovar.HDF_ACCUMULATION_AREA_NAME)
cell_manager = CellManager(flow_dirs=flow_directions,
lons2d=bmp_info.lons,
lats2d=bmp_info.lats,
accumulation_area_km2=accumulation_area_km2)
# Get the list of stations to indicate on the bias map
stations = cehq_station.read_station_data(start_date=None,
end_date=None,
selected_ids=station_ids)
""":type : list[Station]"""
xx, yy = bmp_info.get_proj_xy()
station_to_modelpoint = cell_manager.get_model_points_for_stations(
station_list=stations)
upstream_edges = cell_manager.get_upstream_polygons_for_points(
model_point_list=station_to_modelpoint.values(), xx=xx, yy=yy)
bmp_info.draw_colorbar_for_each_subplot = True
# Validate temperature, precip and swe
obs_path_anusplin = "/home/huziy/skynet3_rech1/anusplin_links"
obs_path_swe = "data/swe_ross_brown/swe.nc"
model_var_to_obs_path = OrderedDict([("TT", obs_path_anusplin),
("I5", obs_path_swe)])
model_var_to_season = OrderedDict([
("TT", OrderedDict([("Spring", range(3, 6))])),
("I5", OrderedDict([("Winter", [1, 2, 12])]))
])
vname_to_obs_data = {}
# parameters that won't change in the loop over variable names
params_const = dict(rconfig=r_config, bmp_info=bmp_info)
for vname, obs_path in model_var_to_obs_path.items():
season_to_obs_data = get_seasonal_clim_obs_data(
vname=vname,
obs_path=obs_path,
season_to_months=model_var_to_season[vname],
**params_const)
# Comment swe over lakes, since I5 calculated only for land
if vname in [
"I5",
]:
for season in season_to_obs_data:
season_to_obs_data[season] = maskoceans(
bmp_info.lons,
bmp_info.lats,
season_to_obs_data[season],
inlands=True)
vname_to_obs_data[vname] = season_to_obs_data
# Plotting
plot_all_vars_in_one_fig = True
fig = None
gs = None
if plot_all_vars_in_one_fig:
plot_utils.apply_plot_params(font_size=12,
width_pt=None,
width_cm=25,
height_cm=20)
fig = plt.figure()
ncols = len(model_var_to_obs_path) + 1
gs = GridSpec(len(r_config_list),
ncols,
width_ratios=(ncols - 1) * [
1.,
] + [
0.05,
])
else:
plot_utils.apply_plot_params(font_size=12,
width_pt=None,
width_cm=25,
height_cm=25)
station_x_list = []
station_y_list = []
mvarname_to_cs = {}
for row, r_config in enumerate(r_config_list):
for col, mname in enumerate(model_var_to_obs_path):
row_axes = [
fig.add_subplot(gs[row, col]),
]
mvarname_to_cs[mname] = compare_vars(
vname_model=mname,
vname_to_obs=vname_to_obs_data,
r_config=r_config,
season_to_months=model_var_to_season[mname],
bmp_info_agg=bmp_info,
axes_list=row_axes)
# -1 in order to exclude colorbars
for the_ax in row_axes:
the_ax.set_title(the_ax.get_title() + ", {}".format(
infovar.get_long_display_label_for_var(mname)))
# Need titles only for the first row
if row > 0:
the_ax.set_title("")
if col == 0:
the_ax.set_ylabel(r_config.label)
else:
the_ax.set_ylabel("")
draw_upstream_area_bounds(the_ax, upstream_edges, color="g")
if len(station_x_list) == 0:
for the_station in stations:
xst, yst = bmp_info.basemap(the_station.longitude,
the_station.latitude)
station_x_list.append(xst)
station_y_list.append(yst)
bmp_info.basemap.scatter(station_x_list,
station_y_list,
c="g",
ax=the_ax,
s=20,
zorder=10,
alpha=0.5)
# Save the figure if necessary
if plot_all_vars_in_one_fig:
if not img_folder.is_dir():
img_folder.mkdir(parents=True)
fig_path = img_folder.joinpath("{}.png".format(
"_".join(model_var_to_obs_path)))
with fig_path.open("wb") as figfile:
fig.savefig(figfile, format="png", bbox_inches="tight")
plt.close(fig)
def main():
start_year = 1980
end_year = 2010
# model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Diagnostics")
model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Samples")
static_data_file = "/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p"
corrected_obs_data_folder = Path("mh/obs_data/")
r = RPN(static_data_file)
fldir = r.get_first_record_for_name("FLDR")
faa = r.get_first_record_for_name("FAA")
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
gc = default_domains.bc_mh_044
cell_manager = CellManager(fldir, nx=fldir.shape[0], ny=fldir.shape[1],
lons2d=lons, lats2d=lats, accumulation_area_km2=faa)
selected_station_ids = [
"05LM006",
"05BN012",
"05AK001",
"05QB003"
]
stations = cehq_station.load_from_hydat_db(province=None, selected_ids=selected_station_ids, natural=None, skip_data_checks=True)
for s in stations:
assert isinstance(s, cehq_station.Station)
if s.id == "05AK001":
s.drainage_km2 *= 2.5
if s.id == "05BN012":
pass
# Manitoba natural stations
# statons_mnb = cehq_station.load_from_hydat_db(province="MB", natural=True, start_date=datetime(start_year, 1, 1), end_date=datetime(end_year,12,31))
# statons_ssk = cehq_station.load_from_hydat_db(province="SK", natural=True, start_date=datetime(start_year, 1, 1), end_date=datetime(end_year,12,31))
# statons_alb = cehq_station.load_from_hydat_db(province="AB", natural=True, start_date=datetime(start_year, 1, 1), end_date=datetime(end_year,12,31))
# for s in statons_mnb + statons_ssk + statons_alb:
# if s not in stations:
# stations.append(s)
# (06EA002): CHURCHILL RIVER AT SANDY BAY at (-102.31832885742188,55.52333068847656), accum. area is 212000.0 km**2
# TODO: plot where is this station, compare modelled and observed hydrographs
for s in stations:
print(s)
# assert len(stations) == len(selected_station_ids), "Could not find stations for some of the specified ids"
station_to_model_point = cell_manager.get_model_points_for_stations(stations, drainaige_area_reldiff_limit=0.9,
nneighbours=8)
print("Established the station to model point mapping")
plot_validations_for_stations(station_to_model_point,
cell_manager=cell_manager,
corrected_obs_data_folder=corrected_obs_data_folder,
model_data_path=model_data_path,
grid_config=gc, start_year=start_year, end_year=end_year)
def main():
direction_file_path = Path(
"/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p"
)
sim_label = "mh_0.44"
start_year = 1981
end_year = 2010
streamflow_internal_name = "streamflow"
selected_staion_ids = constants.selected_station_ids_for_streamflow_validation
# ======================================================
day = timedelta(days=1)
t0 = datetime(2001, 1, 1)
stamp_dates = [t0 + i * day for i in range(365)]
print("stamp dates range {} ... {}".format(stamp_dates[0],
stamp_dates[-1]))
lake_fraction = None
# establish the correspondence between the stations and model grid points
with RPN(str(direction_file_path)) as r:
assert isinstance(r, RPN)
fldir = r.get_first_record_for_name("FLDR")
flow_acc_area = r.get_first_record_for_name("FAA")
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
# lake_fraction = r.get_first_record_for_name("LF1")
cell_manager = CellManager(fldir,
lons2d=lons,
lats2d=lats,
accumulation_area_km2=flow_acc_area)
stations = stfl_stations.load_stations_from_csv(
selected_ids=selected_staion_ids)
station_to_model_point = cell_manager.get_model_points_for_stations(
station_list=stations, lake_fraction=lake_fraction, nneighbours=8)
# Update the end year if required
max_year_st = -1
for station in station_to_model_point:
y = max(station.get_list_of_complete_years())
if y >= max_year_st:
max_year_st = y
if end_year > max_year_st:
print("Updated end_year to {}, because no obs data after...".format(
max_year_st))
end_year = max_year_st
# read model data
mod_data_manager = DataManager(
store_config={
"varname_mapping": {
streamflow_internal_name: "STFA"
},
"base_folder": str(direction_file_path.parent.parent),
"data_source_type":
data_source_types.SAMPLES_FOLDER_FROM_CRCM_OUTPUT,
"level_mapping": {
streamflow_internal_name:
VerticalLevel(-1, level_type=level_kinds.ARBITRARY)
},
"offset_mapping": vname_to_offset_CRCM5,
"filename_prefix_mapping": {
streamflow_internal_name: "pm"
}
})
station_to_model_data = defaultdict(list)
for year in range(start_year, end_year + 1):
start = Pendulum(year, 1, 1)
p_test = Period(start, start.add(years=1).subtract(microseconds=1))
stfl_mod = mod_data_manager.read_data_for_period(
p_test, streamflow_internal_name)
# convert to daily
stfl_mod = stfl_mod.resample("D",
"t",
how="mean",
closed="left",
keep_attrs=True)
assert isinstance(stfl_mod, xr.DataArray)
for station, model_point in station_to_model_point.items():
assert isinstance(model_point, ModelPoint)
ts1 = stfl_mod[:, model_point.ix, model_point.jy].to_series()
station_to_model_data[station].append(
pd.Series(index=stfl_mod.t.values, data=ts1))
# concatenate the timeseries for each point, if required
if end_year - start_year + 1 > 1:
for station in station_to_model_data:
station_to_model_data[station] = pd.concat(
station_to_model_data[station])
else:
for station in station_to_model_data:
station_to_model_data[station] = station_to_model_data[station][0]
# calculate observed climatology
station_to_climatology = OrderedDict()
for s in sorted(station_to_model_point,
key=lambda st: st.latitude,
reverse=True):
assert isinstance(s, Station)
print(s.id, len(s.get_list_of_complete_years()))
# Check if there are continuous years for the selected period
common_years = set(s.get_list_of_complete_years()).intersection(
set(range(start_year, end_year + 1)))
if len(common_years) > 0:
_, station_to_climatology[
s] = s.get_daily_climatology_for_complete_years_with_pandas(
stamp_dates=stamp_dates, years=common_years)
_, station_to_model_data[
s] = pandas_utils.get_daily_climatology_from_pandas_series(
station_to_model_data[s],
stamp_dates,
years_of_interest=common_years)
else:
print(
"Skipping {}, since it does not have enough data during the period of interest"
.format(s.id))
# ---- Do the plotting ----
ncols = 4
nrows = len(station_to_climatology) // ncols
nrows += int(not (len(station_to_climatology) % ncols == 0))
axes_list = []
plot_utils.apply_plot_params(width_cm=8 * ncols,
height_cm=8 * nrows,
font_size=8)
fig = plt.figure()
gs = GridSpec(nrows=nrows, ncols=ncols)
for i, (s, clim) in enumerate(station_to_climatology.items()):
assert isinstance(s, Station)
row = i // ncols
col = i % ncols
print(row, col, nrows, ncols)
# normalize by the drainage area
if s.drainage_km2 is not None:
station_to_model_data[
s] *= s.drainage_km2 / station_to_model_point[
s].accumulation_area
if s.id in constants.stations_to_greyout:
ax = fig.add_subplot(gs[row, col], facecolor="0.45")
else:
ax = fig.add_subplot(gs[row, col])
assert isinstance(ax, Axes)
ax.plot(stamp_dates, clim, color="k", lw=2, label="Obs.")
ax.plot(stamp_dates,
station_to_model_data[s],
color="r",
lw=2,
label="Mod.")
ax.xaxis.set_major_formatter(FuncFormatter(format_month_label))
ax.xaxis.set_major_locator(MonthLocator(bymonthday=15))
ax.xaxis.set_minor_locator(MonthLocator(bymonthday=1))
ax.grid()
ax.annotate(s.get_pp_name(),
xy=(1.02, 1),
xycoords="axes fraction",
horizontalalignment="left",
verticalalignment="top",
fontsize=8,
rotation=-90)
last_date = stamp_dates[-1]
last_date = last_date.replace(
day=calendar.monthrange(last_date.year, last_date.month)[1])
ax.set_xlim(stamp_dates[0].replace(day=1), last_date)
ymin, ymax = ax.get_ylim()
ax.set_ylim(0, ymax)
if s.drainage_km2 is not None:
ax.set_title(
"{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA={:.0f} km$^2$)".
format(s.id, s.longitude, s.latitude, s.drainage_km2))
else:
ax.set_title(
"{}: ({:.1f}$^\circ$E, {:.1f}$^\circ$N, DA not used)".format(
s.id, s.longitude, s.latitude))
axes_list.append(ax)
# plot the legend
axes_list[-1].legend()
if not img_folder.exists():
img_folder.mkdir()
fig.tight_layout()
img_file = img_folder / "{}_{}-{}_{}.png".format(
sim_label, start_year, end_year, "-".join(
sorted(s.id for s in station_to_climatology)))
print("Saving {}".format(img_file))
fig.savefig(str(img_file), bbox_inches="tight", dpi=300)
def draw_model_comparison(model_points=None, stations=None, sim_name_to_file_name=None, hdf_folder=None,
start_year=None, end_year=None, cell_manager=None, stfl_name="STFA",
drainage_area_reldiff_min=0.1, plot_upstream_area_averaged=True,
sim_name_to_color=None):
"""
:param model_points: list of model point objects
:param stations: list of stations corresponding to the list of model points
:param cell_manager: is a CellManager instance which can be provided for better performance if necessary
len(model_points) == len(stations) if stations is not None.
if stations is None - then no measured streamflow will be plotted
"""
assert model_points is None or stations is None or len(stations) == len(model_points)
label_list = list(sim_name_to_file_name.keys()) # Needed to keep the order the same for all subplots
path0 = os.path.join(hdf_folder, list(sim_name_to_file_name.items())[0][1])
flow_directions = analysis.get_array_from_file(path=path0, var_name="flow_direction")
lake_fraction = analysis.get_array_from_file(path=path0, var_name="lake_fraction")
# mask lake fraction in the ocean
lake_fraction = np.ma.masked_where((flow_directions <= 0) | (flow_directions > 128), lake_fraction)
accumulation_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_ACCUMULATION_AREA_NAME)
area_m2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_M2)
# Try to read cell areas im meters if it is not Ok then try in km2
if area_m2 is not None:
cell_area_km2 = area_m2 * 1.0e-6
else:
cell_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_KM2)
print("cell area ranges from {} to {}".format(cell_area_km2.min(), cell_area_km2.max()))
# print "plotting from {0}".format(path0)
# plt.pcolormesh(lake_fraction.transpose())
# plt.colorbar()
# plt.show()
# exit()
file_scores = open("scores_{0}_{1}-{2}.txt".format("_".join(label_list), start_year, end_year), "w")
file_correlations = open("corr_{0}_{1}-{2}.txt".format("_".join(label_list), start_year, end_year), "w")
file_annual_discharge = open("flow_{0}_{1}-{2}.txt".format("_".join(label_list), start_year, end_year), "w")
text_files = [file_scores, file_correlations, file_annual_discharge]
# write the following columns to the scores file
header_format = "{0:10s}\t{1:10s}\t{2:10s}\t" + "\t".join(["{" + str(i + 3) + ":10s}"
for i in range(len(sim_name_to_file_name))])
line_format = "{0:10s}\t{1:10.1f}\t{2:10.1f}\t" + "\t".join(["{" + str(i + 3) + ":10.1f}"
for i in range(len(sim_name_to_file_name))])
header_ns = ("ID", "DAo", "DAm",) + tuple(["NS({0})".format(key) for key in sim_name_to_file_name])
file_scores.write(header_format.format(*header_ns) + "\n")
header_qyear = ("ID", "DAo", "DAm",) + tuple(["Qyear({0})".format(key) for key in label_list]) + \
("Qyear(obs)",)
header_format_qyear = header_format + "\t{" + str(len(label_list) + 3) + ":10s}"
file_annual_discharge.write(header_format_qyear.format(*header_qyear) + "\n")
lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(file_path=path0)
# Create a cell manager if it is not provided
if cell_manager is None:
cell_manager = CellManager(flow_directions, accumulation_area_km2=accumulation_area_km2,
lons2d=lons2d, lats2d=lats2d)
if stations is not None:
# Get the list of the corresponding model points
station_to_modelpoint = cell_manager.get_model_points_for_stations(
station_list=stations,
lake_fraction=lake_fraction,
drainaige_area_reldiff_limit=drainage_area_reldiff_min)
station_list = list(station_to_modelpoint.keys())
station_list.sort(key=lambda st1: st1.latitude, reverse=True)
mp_list = [station_to_modelpoint[st] for st in station_list]
else:
mp_list = model_points
station_list = None
# sort so that the northernmost stations appear uppermost
mp_list.sort(key=lambda mpt: mpt.latitude, reverse=True)
# set ids to the model points so they can be distinguished easier
model_point.set_model_point_ids(mp_list)
# ###Uncomment the lines below for the validation plot in paper 2
# brewer2mpl.get_map args: set name set type number of colors
# bmap = brewer2mpl.get_map("Set1", "qualitative", 9)
# Change the default colors
# mpl.rcParams["axes.color_cycle"] = bmap.mpl_colors
# For the streamflow only plot
ncols = 3
nrows = max(len(mp_list) // ncols, 1)
if ncols * nrows < len(mp_list):
nrows += 1
figure_stfl = plt.figure(figsize=(4 * ncols, 3 * nrows))
gs_stfl = gridspec.GridSpec(nrows=nrows, ncols=ncols)
# a flag which signifies if a legend should be added to the plot, it is needed so we ahve only one legend per plot
legend_added = False
ax_stfl = None
all_years = [y for y in range(start_year, end_year + 1)]
if station_list is not None:
processed_stations = station_list
else:
processed_stations = [None] * len(mp_list)
processed_model_points = mp_list
plot_point_positions_with_upstream_areas(processed_stations, processed_model_points, basemap,
cell_manager, lake_fraction_field=lake_fraction)
if plot_upstream_area_averaged:
# create obs data managers
anusplin_tmin = AnuSplinManager(variable="stmn")
anusplin_tmax = AnuSplinManager(variable="stmx")
anusplin_pcp = AnuSplinManager(variable="pcp")
daily_dates, obs_tmin_fields = anusplin_tmin.get_daily_clim_fields_interpolated_to(
start_year=start_year, end_year=end_year,
lons_target=lons2d, lats_target=lats2d)
_, obs_tmax_fields = anusplin_tmax.get_daily_clim_fields_interpolated_to(
start_year=start_year, end_year=end_year,
lons_target=lons2d, lats_target=lats2d)
_, obs_pcp_fields = anusplin_pcp.get_daily_clim_fields_interpolated_to(
start_year=start_year, end_year=end_year,
lons_target=lons2d, lats_target=lats2d)
swe_path = "/skynet3_rech1/huziy/swe_ross_brown/swe.nc4"
if not os.path.isfile(os.path.realpath(swe_path)):
raise IOError("SWE-obs file {} does not exist".format(swe_path))
swe_manager = SweDataManager(path=swe_path, var_name="SWE")
obs_swe_daily_clim = swe_manager.get_daily_climatology(start_year, end_year)
interpolated_obs_swe_clim = swe_manager.interpolate_daily_climatology_to(obs_swe_daily_clim,
lons2d_target=lons2d,
lats2d_target=lats2d)
values_obs = None
for i, the_model_point in enumerate(mp_list):
ax_stfl = figure_stfl.add_subplot(gs_stfl[i // ncols, i % ncols], sharex=ax_stfl)
assert isinstance(the_model_point, ModelPoint)
# Check the number of years accessible for the station if the list of stations is given
the_station = None if station_list is None else station_list[i]
if the_station is not None:
assert isinstance(the_station, Station)
year_list = the_station.get_list_of_complete_years()
year_list = list(filter(lambda yi: start_year <= yi <= end_year, year_list))
if len(year_list) < 1:
continue
else:
year_list = all_years
fig = plt.figure(figsize=(12, 15))
gs = gridspec.GridSpec(4, 4, wspace=1)
# plot station position
ax = fig.add_subplot(gs[3, 0:2])
upstream_mask = _plot_station_position(ax, the_station, basemap, cell_manager, the_model_point)
# plot streamflows
ax = fig.add_subplot(gs[0:2, 0:2])
dates = None
model_daily_temp_clim = {}
model_daily_precip_clim = {}
model_daily_clim_surf_runoff = {}
model_daily_clim_subsurf_runoff = {}
model_daily_clim_swe = {}
# get model data for the list of years
simlabel_to_vals = {}
for label in label_list:
fname = sim_name_to_file_name[label]
if hdf_folder is None:
fpath = fname
else:
fpath = os.path.join(hdf_folder, fname)
if plot_upstream_area_averaged:
# read temperature data and calculate daily climatologic fileds
_, model_daily_temp_clim[label] = analysis.get_daily_climatology(
path_to_hdf_file=fpath, var_name="TT", level=0, start_year=start_year, end_year=end_year)
# read modelled precip and calculate daily climatologic fields
_, model_daily_precip_clim[label] = analysis.get_daily_climatology(
path_to_hdf_file=fpath, var_name="PR", level=0, start_year=start_year, end_year=end_year)
# read modelled surface runoff and calculate daily climatologic fields
_, model_daily_clim_surf_runoff[label] = analysis.get_daily_climatology(
path_to_hdf_file=fpath, var_name="TRAF", level=0, start_year=start_year, end_year=end_year)
# read modelled subsurface runoff and calculate daily climatologic fields
_, model_daily_clim_subsurf_runoff[label] = analysis.get_daily_climatology(
path_to_hdf_file=fpath, var_name="TDRA", level=0, start_year=start_year, end_year=end_year)
# read modelled swe and calculate daily climatologic fields
_, model_daily_clim_swe[label] = analysis.get_daily_climatology(
path_to_hdf_file=fpath, var_name="I5", level=0, start_year=start_year, end_year=end_year)
dates, values_model = analysis.get_daily_climatology_for_a_point(path=fpath,
var_name=stfl_name,
years_of_interest=year_list,
i_index=the_model_point.ix,
j_index=the_model_point.jy)
ax.plot(dates, values_model, label=label, lw=2)
if sim_name_to_color is None:
ax_stfl.plot(dates, values_model, label=label, lw=2)
else:
ax_stfl.plot(dates, values_model, sim_name_to_color[label], label=label, lw=2)
print(20 * "!!!")
print("{} -> {}".format(label, sim_name_to_color[label]))
print(20 * "!!!")
simlabel_to_vals[label] = values_model
if the_station is not None:
assert isinstance(the_station, Station)
dates, values_obs = the_station.get_daily_climatology_for_complete_years_with_pandas(stamp_dates=dates,
years=year_list)
# To keep the colors consistent for all the variables, the obs Should be plotted last
ax.plot(dates, values_obs, label="Obs.", lw=2)
# no ticklabels for streamflow plot
plt.setp(ax.get_xticklabels(), visible=False)
if sim_name_to_color is None:
ax_stfl.plot(dates, values_obs, label="Obs.", lw=2)
else:
ax_stfl.plot(dates, values_obs, label="Obs.", lw=2, color=sim_name_to_color["Obs."])
# Print excesss from streamflow validation
for label, values_model in simlabel_to_vals.items():
calclulate_spring_peak_err(dates, values_obs, values_model,
st_id="{}: {}".format(label, the_station.id),
da_mod=the_model_point.accumulation_area,
da_obs=the_station.drainage_km2)
ax.set_ylabel(r"Streamflow: ${\rm m^3/s}$")
assert isinstance(ax, Axes)
assert isinstance(fig, Figure)
upstream_area_km2 = np.sum(cell_area_km2[upstream_mask == 1])
# Put some information about the point
if the_station is not None:
lf_upstream = lake_fraction[upstream_mask == 1]
point_info = "{0}".format(the_station.id)
write_annual_flows_to_txt(label_list, simlabel_to_vals, values_obs, file_annual_discharge,
station_id=the_station.id,
da_obs=the_station.drainage_km2, da_mod=the_model_point.accumulation_area)
else:
point_info = "{0}".format(the_model_point.point_id)
ax.annotate(point_info, (0.8, 0.8), xycoords="axes fraction",
bbox=dict(facecolor="white", alpha=0.5),
va="top", ha="right")
ax.legend(loc=(0.0, 1.05), borderaxespad=0, ncol=3)
ax.xaxis.set_minor_formatter(FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0]))
ax.xaxis.set_minor_locator(MonthLocator(bymonthday=15))
ax.xaxis.set_major_locator(MonthLocator())
ax.grid()
streamflow_axes = ax # save streamflow axes for later use
if not legend_added:
ax_stfl.legend(loc="lower left", bbox_to_anchor=(0, 1.15), borderaxespad=0, ncol=3)
ax_stfl.xaxis.set_minor_formatter(FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0]))
ax_stfl.xaxis.set_minor_locator(MonthLocator(bymonthday=15))
ax_stfl.xaxis.set_major_locator(MonthLocator())
ax_stfl.set_ylabel(r"Streamflow ${\rm m^3/s}$")
legend_added = True
plt.setp(ax_stfl.get_xmajorticklabels(), visible=False)
ax_stfl.yaxis.set_major_locator(MaxNLocator(nbins=5))
sfmt = ScalarFormatter(useMathText=True)
sfmt.set_powerlimits((-2, 2))
ax_stfl.yaxis.set_major_formatter(sfmt)
ax_stfl.grid()
# annotate streamflow-only panel plot
ax_stfl.annotate(point_info, (0.05, 0.95), xycoords="axes fraction",
bbox=dict(facecolor="white"),
va="top", ha="left")
if plot_upstream_area_averaged:
# plot temperature comparisons (tmod - daily with anusplin tmin and tmax)
ax = fig.add_subplot(gs[3, 2:], sharex=streamflow_axes)
_validate_temperature_with_anusplin(ax, the_model_point, cell_area_km2=cell_area_km2,
upstream_mask=upstream_mask,
daily_dates=daily_dates,
obs_tmin_clim_fields=obs_tmin_fields,
obs_tmax_clim_fields=obs_tmax_fields,
model_data_dict=model_daily_temp_clim,
simlabel_list=label_list)
# plot temperature comparisons (tmod - daily with anusplin tmin and tmax)
ax = fig.add_subplot(gs[2, 2:], sharex=streamflow_axes)
_validate_precip_with_anusplin(ax, the_model_point, cell_area_km2=cell_area_km2,
upstream_mask=upstream_mask,
daily_dates=daily_dates,
obs_precip_clim_fields=obs_pcp_fields,
model_data_dict=model_daily_precip_clim,
simlabel_list=label_list)
# plot mean upstream surface runoff
ax = fig.add_subplot(gs[0, 2:], sharex=streamflow_axes)
_plot_upstream_surface_runoff(ax, the_model_point, cell_area_km2=cell_area_km2,
upstream_mask=upstream_mask,
daily_dates=daily_dates,
model_data_dict=model_daily_clim_surf_runoff,
simlabel_list=label_list)
# plot mean upstream subsurface runoff
ax = fig.add_subplot(gs[1, 2:], sharex=streamflow_axes, sharey=ax)
_plot_upstream_subsurface_runoff(ax, the_model_point, cell_area_km2=cell_area_km2,
upstream_mask=upstream_mask,
daily_dates=daily_dates,
model_data_dict=model_daily_clim_subsurf_runoff,
simlabel_list=label_list)
# plot mean upstream swe comparison
ax = fig.add_subplot(gs[2, 0:2], sharex=streamflow_axes)
print("Validating SWE for ", the_station.id, "--" * 20)
_validate_swe_with_ross_brown(ax, the_model_point, cell_area_km2=cell_area_km2,
upstream_mask=upstream_mask,
daily_dates=daily_dates,
model_data_dict=model_daily_clim_swe,
obs_swe_clim_fields=interpolated_obs_swe_clim,
simlabel_list=label_list)
if the_station is not None:
im_name = "comp_point_with_obs_{0}_{1}_{2}.png".format(the_station.id,
the_station.source,
"_".join(label_list))
im_folder_path = os.path.join(images_folder, the_station.source)
else:
im_name = "comp_point_with_obs_{0}_{1}.png".format(the_model_point.point_id,
"_".join(label_list))
im_folder_path = os.path.join(images_folder, "outlets_point_comp")
# create a folder for a given source of observed streamflow if it does not exist yet
if not os.path.isdir(im_folder_path):
os.mkdir(im_folder_path)
im_path = os.path.join(im_folder_path, im_name)
if plot_upstream_area_averaged:
fig.savefig(im_path, dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight", transparent=True)
plt.close(fig)
# return # temporary plot only one point
assert isinstance(figure_stfl, Figure)
figure_stfl.tight_layout()
figure_stfl.savefig(os.path.join(images_folder,
"comp_point_with_obs_{0}.png".format("_".join(label_list))),
bbox_inches="tight", transparent=True, dpi=cpp.FIG_SAVE_DPI)
plt.close(figure_stfl)
# close information text files
for f in text_files:
f.close()
def main(directions_file_path: Path):
"""
compare drainage areas, longitudes and latitudes from the stations and model
"""
stations = stfl_stations.load_stations_from_csv()
lake_fraction = None
with Dataset(str(directions_file_path)) as ds:
flow_dirs = ds.variables["flow_direction_value"][:]
flow_acc_area = ds.variables["accumulation_area"][:]
lons_2d, lats_2d = [ds.variables[k][:] for k in ["lon", "lat"]]
# lake_fraction = ds.variables["lake_fraction"][:]
cell_manager = CellManager(flow_dirs,
lons2d=lons_2d,
lats2d=lats_2d,
accumulation_area_km2=flow_acc_area)
station_to_mod_point = cell_manager.get_model_points_for_stations(
station_list=stations, lake_fraction=lake_fraction, nneighbours=8)
lons_m, lats_m, da_m = [], [], []
lons_o, lats_o, da_o = [], [], []
for s, mp in station_to_mod_point.items():
assert isinstance(s, Station)
assert isinstance(mp, ModelPoint)
# obs
lons_o.append(s.longitude if s.longitude < 180 else s.longitude - 360)
lats_o.append(s.latitude)
da_o.append(s.drainage_km2)
# model
lons_m.append(mp.longitude if mp.longitude < 180 else mp.longitude -
360)
lats_m.append(mp.latitude)
da_m.append(mp.accumulation_area)
print("m | s ({})".format(s.id))
print("{} | {}".format(mp.longitude, s.longitude))
print("{} | {}".format(mp.latitude, s.latitude))
print("{} | {}".format(mp.accumulation_area, s.drainage_km2))
axes_list = []
plot_utils.apply_plot_params(width_cm=25, height_cm=10, font_size=8)
fig = plt.figure()
gs = GridSpec(1, 3)
ax = fig.add_subplot(gs[0, 0])
ax.set_title("Longitude")
ax.scatter(lons_o, lons_m)
axes_list.append(ax)
ax.set_ylabel("Model")
ax = fig.add_subplot(gs[0, 1])
ax.set_title("Latitude")
ax.scatter(lats_o, lats_m)
axes_list.append(ax)
ax.set_xlabel("Obs")
ax = fig.add_subplot(gs[0, 2])
ax.set_title("Drainage area (km$^2$)")
ax.scatter(da_o, da_m)
sf = ScalarFormatter(useMathText=True)
sf.set_powerlimits((-2, 3))
ax.set_xscale("log")
ax.set_yscale("log")
axes_list.append(ax)
# plot the 1-1 line
for ax in axes_list:
assert isinstance(ax, Axes)
ax.plot(ax.get_xlim(), ax.get_xlim(), "--", color="gray")
ax.grid()
img_file = img_folder.joinpath("lon_lat_da_scatter_{}_{}.png".format(
directions_file_path.name,
"-".join(sorted(s.id for s in station_to_mod_point))))
fig.savefig(str(img_file), bbox_inches="tight")
def plot_station_positions(directions_file_path: Path, stations: list, ax: Axes, grid_config: GridConfig=default_domains.bc_mh_044,
save_upstream_boundaries_to_shp=False):
with Dataset(str(directions_file_path)) as ds:
flow_dirs = ds.variables["flow_direction_value"][:]
flow_acc_area = ds.variables["accumulation_area"][:]
lons_2d, lats_2d = [ds.variables[k][:] for k in ["lon", "lat"]]
basemap, reg_of_interest = grid_config.get_basemap_using_shape_with_polygons_of_interest(lons_2d, lats_2d,
shp_path=default_domains.MH_BASINS_PATH,
resolution="i")
cell_manager = CellManager(flow_dirs, lons2d=lons_2d, lats2d=lats_2d, accumulation_area_km2=flow_acc_area)
station_to_model_point = cell_manager.get_model_points_for_stations(station_list=stations, nneighbours=8)
#####
xx, yy = basemap(lons_2d, lats_2d)
upstream_edges = cell_manager.get_upstream_polygons_for_points(
model_point_list=list(station_to_model_point.values()), xx=xx, yy=yy)
upstream_edges_latlon = cell_manager.get_upstream_polygons_for_points(
model_point_list=list(station_to_model_point.values()), xx=lons_2d, yy=lats_2d)
plot_utils.draw_upstream_area_bounds(ax, upstream_edges=upstream_edges, color="r", linewidth=0.6)
if save_upstream_boundaries_to_shp:
plot_utils.save_to_shape_file(upstream_edges_latlon, folder_path="mh/engage_report/upstream_stations_areas/mh_{}".format(grid_config.dx), in_proj=None)
basemap.drawrivers(linewidth=0.2)
basemap.drawstates(linewidth=0.1)
basemap.drawcountries(linewidth=0.1)
basemap.drawcoastlines(linewidth=0.2)
pos_ids, lons_pos, lats_pos = [], [], []
pos_labels = []
legend_lines = []
for i, (s, mp) in enumerate(sorted(station_to_model_point.items(), key=lambda p: p[0].latitude, reverse=True), start=1):
pos_ids.append(s.id)
pos_labels.append(i)
lons_pos.append(mp.longitude)
lats_pos.append(mp.latitude)
legend_lines.append("{}: {}".format(i, s.id))
xm, ym = basemap(lons_pos, lats_pos)
ax.scatter(xm, ym, c="g", s=20)
for txt, x1, y1, pos_label in zip(pos_ids, xm, ym, pos_labels):
ax.annotate(pos_label, xy=(x1, y1))
at = AnchoredText("\n".join(legend_lines), prop=dict(size=8), frameon=True, loc=1)
at.patch.set_boxstyle("round,pad=0.,rounding_size=0.2")
ax.add_artist(at)
def main():
start_year = 1980
end_year = 2010
start_date = datetime(start_year, 1, 1)
end_date = datetime(end_year, 12, 31)
ids_with_lakes_upstream = [
"104001", "093806", "093801", "081002", "081007", "080718"
]
selected_station_ids = [
"092715", "074903", "080104", "081007", "061905", "093806", "090613",
"081002", "093801", "080718", "104001"
]
selected_station_ids = ids_with_lakes_upstream
# Get the list of stations to do the comparison with
stations = cehq_station.read_station_data(
start_date=start_date,
end_date=end_date,
selected_ids=selected_station_ids)
# add hydat stations
# province = "QC"
# min_drainage_area_km2 = 10000.0
# stations_hd = cehq_station.load_from_hydat_db(start_date=start_date, end_date=end_date,
# province=province, min_drainage_area_km2=min_drainage_area_km2)
# if not len(stations_hd):
# print "No hydat stations satisying the conditions: period {0}-{1}, province {2}".format(
# str(start_date), str(end_date), province
# )
# stations.extend(stations_hd)
# brewer2mpl.get_map args: set name set type number of colors
bmap = brewer2mpl.get_map("Set1", "qualitative", 9)
path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-r.hdf5"
label1 = "CRCM5-L1"
path2 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl.hdf5"
label2 = "CRCM5-L2"
color2, color1 = bmap.mpl_colors[:2]
fldirs = analysis.get_array_from_file(
path=path1, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME)
lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(path1)
lake_fractions = analysis.get_array_from_file(
path=path1, var_name=infovar.HDF_LAKE_FRACTION_NAME)
# cell_areas = analysis.get_array_from_file(path=path1, var_name=infovar.HDF_CELL_AREA_NAME)
acc_area = analysis.get_array_from_file(
path=path1, var_name=infovar.HDF_ACCUMULATION_AREA_NAME)
cell_manager = CellManager(fldirs,
lons2d=lons2d,
lats2d=lats2d,
accumulation_area_km2=acc_area)
station_to_mp = cell_manager.get_model_points_for_stations(
station_list=stations,
lake_fraction=lake_fractions,
drainaige_area_reldiff_limit=0.3)
fig, axes = plt.subplots(1, 2, gridspec_kw=dict(top=0.80, wspace=0.4))
q90_obs_list = []
q90_mod1_list = []
q90_mod2_list = []
q10_obs_list = []
q10_mod1_list = []
q10_mod2_list = []
for the_station, the_mp in station_to_mp.items():
assert isinstance(the_station, Station)
compl_years = the_station.get_list_of_complete_years()
if len(compl_years) < 3:
continue
t, stfl1 = analysis.get_daily_climatology_for_a_point(
path=path1,
years_of_interest=compl_years,
i_index=the_mp.ix,
j_index=the_mp.jy,
var_name="STFA")
_, stfl2 = analysis.get_daily_climatology_for_a_point(
path=path2,
years_of_interest=compl_years,
i_index=the_mp.ix,
j_index=the_mp.jy,
var_name="STFA")
_, stfl_obs = the_station.get_daily_climatology_for_complete_years(
stamp_dates=t, years=compl_years)
# Q90
q90_obs = np.percentile(stfl_obs, 90)
q90_mod1 = np.percentile(stfl1, 90)
q90_mod2 = np.percentile(stfl2, 90)
# Q10
q10_obs = np.percentile(stfl_obs, 10)
q10_mod1 = np.percentile(stfl1, 10)
q10_mod2 = np.percentile(stfl2, 10)
# save quantiles to lists for correlation calculation
q90_obs_list.append(q90_obs)
q90_mod1_list.append(q90_mod1)
q90_mod2_list.append(q90_mod2)
q10_mod1_list.append(q10_mod1)
q10_mod2_list.append(q10_mod2)
q10_obs_list.append(q10_obs)
# axes[0].annotate(the_station.id, (q90_obs, np.percentile(stfl1, 90)))
# axes[1].annotate(the_station.id, (q10_obs, np.percentile(stfl1, 10)))
# Plot scatter plot of Q90
the_ax = axes[0]
# the_ax.annotate(the_station.id, (q90_obs, np.percentile(stfl1, 90)))
the_ax.scatter(q90_obs_list, q90_mod1_list, label=label1, c=color1)
the_ax.scatter(q90_obs_list, q90_mod2_list, label=label2, c=color2)
# plot scatter plot of Q10
the_ax = axes[1]
# the_ax.annotate(the_station.id, (q10_obs, np.percentile(stfl1, 10)))
h1 = the_ax.scatter(q10_obs_list, q10_mod1_list, label=label1, c=color1)
h2 = the_ax.scatter(q10_obs_list, q10_mod2_list, label=label2, c=color2)
# Add correlation coefficients to the axes
fp = FontProperties(size=14, weight="bold")
axes[0].annotate(r"$R^2 = {0:.2f}$".format(
np.corrcoef(q90_mod1_list, q90_obs_list)[0, 1]**2), (0.1, 0.85),
color=color1,
xycoords="axes fraction",
font_properties=fp)
axes[0].annotate(r"$R^2 = {0:.2f}$".format(
np.corrcoef(q90_mod2_list, q90_obs_list)[0, 1]**2), (0.1, 0.70),
color=color2,
xycoords="axes fraction",
font_properties=fp)
axes[1].annotate(r"$R^2 = {0:.2f}$".format(
np.corrcoef(q10_mod1_list, q10_obs_list)[0, 1]**2), (0.1, 0.85),
color=color1,
xycoords="axes fraction",
font_properties=fp)
axes[1].annotate(r"$R^2 = {0:.2f}$".format(
np.corrcoef(q10_mod2_list, q10_obs_list)[0, 1]**2), (0.1, 0.70),
color=color2,
xycoords="axes fraction",
font_properties=fp)
sf = ScalarFormatter(useMathText=True)
sf.set_powerlimits((-2, 3))
for ind, the_ax in enumerate(axes):
plot_one_to_one_line(the_ax)
if ind == 0:
the_ax.set_xlabel(r"Observed $\left({\rm m^3/s} \right)$")
the_ax.set_ylabel(r"Modelled $\left({\rm m^3/s} \right)$")
the_ax.annotate(r"$Q_{90}$" if ind == 0 else r"$Q_{10}$", (0.95, 0.95),
xycoords="axes fraction",
bbox=dict(facecolor="white"),
va="top",
ha="right")
the_ax.xaxis.set_major_formatter(sf)
the_ax.yaxis.set_major_formatter(sf)
locator = MaxNLocator(nbins=5)
the_ax.xaxis.set_major_locator(locator)
the_ax.yaxis.set_major_locator(locator)
x1, x2 = the_ax.get_xlim()
# Since streamflow percentiles can only be positive
the_ax.set_xlim(0, x2)
the_ax.set_ylim(0, x2)
fig.legend([h1, h2], [label1, label2], loc="upper center", ncol=2)
figpath = os.path.join(images_folder, "percentiles_comparison.png")
# plt.tight_layout()
fig.savefig(figpath, dpi=cpp.FIG_SAVE_DPI, bbox_inches="tight")
def main(start_year=1980, end_year=1989):
soil_layer_widths = infovar.soil_layer_widths_26_to_60
soil_tops = np.cumsum(soil_layer_widths).tolist()[:-1]
soil_tops = [
0,
] + soil_tops
selected_station_ids = [
"061905", "074903", "090613", "092715", "093801", "093806"
]
# path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl_spinup.hdf"
# label1 = "CRCM5-HCD-RL"
path1 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_spinup_ITFS.hdf5"
label1 = "CRCM5-HCD-RL-INTFL"
path2 = "/skynet3_rech1/huziy/hdf_store/quebec_0.1_crcm5-hcd-rl-intfl_ITFS_avoid_truncation1979-1989.hdf5"
label2 = "CRCM5-HCD-RL-INTFL-improved"
############
images_folder = "images_for_lake-river_paper/comp_soil_profiles"
if not os.path.isdir(images_folder):
os.mkdir(images_folder)
fldirs = analysis.get_array_from_file(
path=path1, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME)
lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(path1)
lake_fractions = analysis.get_array_from_file(
path=path1, var_name=infovar.HDF_LAKE_FRACTION_NAME)
cell_areas = analysis.get_array_from_file(
path=path1, var_name=infovar.HDF_CELL_AREA_NAME_M2)
acc_areakm2 = analysis.get_array_from_file(
path=path1, var_name=infovar.HDF_ACCUMULATION_AREA_NAME)
depth_to_bedrock = analysis.get_array_from_file(
path=path1, var_name=infovar.HDF_DEPTH_TO_BEDROCK_NAME)
cell_manager = CellManager(fldirs,
lons2d=lons2d,
lats2d=lats2d,
accumulation_area_km2=acc_areakm2)
#get climatologic liquid soil moisture and convert fractions to mm
t0 = time.clock()
daily_dates, levels, i1_nointfl = analysis.get_daily_climatology_of_3d_field(
path_to_hdf_file=path1,
var_name="I1",
start_year=start_year,
end_year=end_year)
print("read I1 - 1")
print("Spent {0} seconds ".format(time.clock() - t0))
_, _, i1_intfl = analysis.get_daily_climatology_of_3d_field(
path_to_hdf_file=path2,
var_name="I1",
start_year=start_year,
end_year=end_year)
print("read I1 - 2")
#get climatologic frozen soil moisture and convert fractions to mm
_, _, i2_nointfl = analysis.get_daily_climatology_of_3d_field(
path_to_hdf_file=path1,
var_name="I2",
start_year=start_year,
end_year=end_year)
print("read I2 - 1")
_, _, i2_intfl = analysis.get_daily_climatology_of_3d_field(
path_to_hdf_file=path2,
var_name="I2",
start_year=start_year,
end_year=end_year)
print("read I2 - 2")
#
sm_intfl = i1_intfl + i2_intfl
sm_nointfl = i1_nointfl + i2_nointfl
#Get the list of stations to do the comparison with
stations = cehq_station.read_station_data(
start_date=datetime(start_year, 1, 1),
end_date=datetime(end_year, 12, 31),
selected_ids=selected_station_ids)
print("sm_noinfl, min, max = {0}, {1}".format(sm_nointfl.min(),
sm_nointfl.max()))
print("sm_infl, min, max = {0}, {1}".format(sm_intfl.min(),
sm_intfl.max()))
diff = (sm_intfl - sm_nointfl)
#diff *= soil_layer_widths[np.newaxis, :, np.newaxis, np.newaxis] * 1000 # to convert in mm
#print "number of nans", np.isnan(diff).astype(int).sum()
print("cell area min,max = {0}, {1}".format(cell_areas.min(),
cell_areas.max()))
print("acc area min,max = {0}, {1}".format(acc_areakm2.min(),
acc_areakm2.max()))
assert np.all(lake_fractions >= 0)
print("lake fractions (min, max): ", lake_fractions.min(),
lake_fractions.max())
#Non need to go very deep
nlayers = 3
z, t = np.meshgrid(soil_tops[:nlayers], date2num(daily_dates))
station_to_mp = cell_manager.get_model_points_for_stations(stations)
plotted_global = False
for the_station, mp in station_to_mp.items():
assert isinstance(mp, ModelPoint)
assert isinstance(the_station, Station)
fig = plt.figure()
umask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices(
mp.ix, mp.jy)
#exclude lake cells from the profiles
sel = (umask == 1) & (depth_to_bedrock > 3) & (acc_areakm2 >= 0)
umaskf = umask.astype(float)
umaskf *= (1.0 - lake_fractions) * cell_areas
umaskf[~sel] = 0.0
profiles = np.tensordot(diff, umaskf) / umaskf.sum()
print(profiles.shape, profiles.min(), profiles.max(), umaskf.sum(),
umaskf.min(), umaskf.max())
d = np.abs(profiles).max()
print("d = {0}".format(d))
clevs = np.round(np.linspace(-d, d, 12), decimals=5)
diff_cmap = cm.get_cmap("RdBu_r", lut=len(clevs) - 1)
bn = BoundaryNorm(clevs, len(clevs) - 1)
plt.title("({})-({})".format(label2, label2))
img = plt.contourf(t,
z,
profiles[:, :nlayers],
cmap=diff_cmap,
levels=clevs,
norm=bn)
plt.colorbar(img, ticks=clevs)
ax = plt.gca()
assert isinstance(ax, Axes)
ax.invert_yaxis()
ax.xaxis.set_major_formatter(DateFormatter("%b"))
ax.xaxis.set_major_locator(MonthLocator())
fig.savefig(os.path.join(
images_folder, "{0}_{1}_{2}.jpeg".format(the_station.id, label1,
label2)),
dpi=cpp.FIG_SAVE_DPI,
bbox_inches="tight")
print("processed: {0}".format(the_station))
if not plotted_global:
plotted_global = True
fig = plt.figure()
sel = (depth_to_bedrock >= 0.1) & (acc_areakm2 >= 0)
umaskf = (1.0 - lake_fractions) * cell_areas
umaskf[~sel] = 0.0
profiles = np.tensordot(diff, umaskf) / umaskf.sum()
print(profiles.shape, profiles.min(), profiles.max(), umaskf.sum(),
umaskf.min(), umaskf.max())
d = np.abs(profiles).max()
print("d = {0}".format(d))
clevs = np.round(np.linspace(-d, d, 12), decimals=5)
diff_cmap = cm.get_cmap("RdBu_r", lut=len(clevs) - 1)
bn = BoundaryNorm(clevs, len(clevs) - 1)
img = plt.contourf(t,
z,
profiles[:, :nlayers],
cmap=diff_cmap,
levels=clevs,
norm=bn)
plt.colorbar(img, ticks=clevs)
ax = plt.gca()
assert isinstance(ax, Axes)
ax.invert_yaxis()
ax.xaxis.set_major_formatter(DateFormatter("%b"))
ax.xaxis.set_major_locator(MonthLocator())
fig.savefig(os.path.join(images_folder, "global_mean.jpeg"),
dpi=cpp.FIG_SAVE_DPI,
bbox_inches="tight")
pass
def main():
model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Diagnostics")
# model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Samples")
static_data_file = "/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p"
r = RPN(static_data_file)
fldir = r.get_first_record_for_name("FLDR")
faa = r.get_first_record_for_name("FAA")
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
gc = default_domains.bc_mh_044
cell_manager = CellManager(fldir, nx=fldir.shape[0], ny=fldir.shape[1],
lons2d=lons, lats2d=lats, accumulation_area_km2=faa)
selected_station_ids = ["06EA002", ]
stations = cehq_station.load_from_hydat_db(province="SK", selected_ids=selected_station_ids, natural=None)
# (06EA002): CHURCHILL RIVER AT SANDY BAY at (-102.31832885742188,55.52333068847656), accum. area is 212000.0 km**2
# TODO: plot where is this station, compare modelled and observed hydrographs
# for s in stations:
# assert isinstance(s, cehq_station.Station)
# s.latitude += 0.9
# s.longitude -= 0.2
# print(s)
station_to_model_point = cell_manager.get_model_points_for_stations(stations, drainaige_area_reldiff_limit=0.8,
nneighbours=1)
print(station_to_model_point[stations[0]])
station = stations[0]
assert isinstance(station, cehq_station.Station)
obs_not_corrected = pd.Series(index=station.dates, data=station.values).groupby(
by=lambda d: d.replace(day=15)).mean()
obs_corrected = pd.read_csv("mh/obs_data/Churchill Historic Monthly Apportionable Flow_06EA002.csv.bak.original", skiprows=2)
print(obs_corrected.head())
print(obs_corrected.year.iloc[0], obs_corrected.year.iloc[-1])
date_index = pd.date_range(start=datetime(obs_corrected.year.iloc[0] - 1, 12, 15),
end=datetime(obs_corrected.year.iloc[-1], 12, 15),
freq="M")
date_index = date_index.shift(15, freq=pd.datetools.day)
print(date_index)
data = np.concatenate([r for r in obs_corrected.values[:, 1:-1]])
factor = date_index.map(lambda d: 1000 / (calendar.monthrange(d.year, d.month)[1] * 24 * 3600))
print(factor[:10])
obs_corrected = pd.Series(index=date_index, data=data * factor)
station_to_modelled_data = get_model_data(station_to_model_point, output_path=model_data_path,
grid_config=gc, basins_of_interest_shp=default_domains.MH_BASINS_PATH,
cell_manager=cell_manager, vname="STFL")
modelled_data = station_to_modelled_data[station]
fig = plt.figure()
ax = obs_corrected.plot(label="obs corrected")
obs_not_corrected.plot(label="obs not corrected", ax=ax, color="k")
modelled_data.plot(label="CRCM5", ax=ax, color="r")
ax.legend(loc="upper left")
img_file = img_folder.joinpath("{}_validation_monthly.png".format(station.id))
fig.savefig(str(img_file))
plt.close(fig)
# climatology
start_year = 1980
end_year = 2010
date_selector = lambda d: (start_year <= d.year <= end_year) and not ((d.month == 2) and (d.day == 29))
fig = plt.figure()
ax = obs_corrected.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(label="obs corrected")
obs_not_corrected.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(
label="obs not corrected", ax=ax, color="k")
modelled_data.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(label="CRCM5", ax=ax,
color="r")
ax.xaxis.set_major_locator(MonthLocator(bymonthday=15))
ax.xaxis.set_major_formatter(DateFormatter("%b"))
ax.legend(loc="upper left")
img_file = img_folder.joinpath("{}_validation_clim.png".format(station.id))
fig.savefig(str(img_file))
plt.close(fig)
# Interannual variability
fig = plt.figure()
obs_corrected = obs_corrected.select(lambda d: start_year <= d.year <= end_year)
modelled_data = modelled_data.select(lambda d: start_year <= d.year <= end_year)
corr_list = []
for m in range(1, 13):
obs = obs_corrected.select(lambda d: d.month == m)
mod = modelled_data.select(lambda d: d.month == m)
print(obs.head())
obs.index = obs.index.map(lambda d: d.year)
mod.index = mod.index.map(lambda d: d.year)
corr_list.append(obs.corr(mod))
ax = plt.gca()
ax.plot(range(1, 13), corr_list)
ax.set_xlabel("Month")
ax.set_title("Inter-annual variability")
img_file = img_folder.joinpath("{}_interannual.png".format(station.id))
fig.tight_layout()
fig.savefig(str(img_file), bbox_inches="tight")
plt.close(fig)
def main(hdf_folder="/home/huziy/skynet3_rech1/hdf_store", start_year=1980, end_year=2010):
prepare()
all_markers = ["*", "s", "p", "+", "x", "d", "h"]
excluded = ["white", "w", "aliceblue", "azure"]
excluded.extend([ci for ci in colors.cnames if "yellow" in ci])
all_colors = ["k", "b", "r", "g", "m"] + sorted([ci for ci in colors.cnames if ci not in excluded])
# Station ids to get from the CEHQ database
ids_with_lakes_upstream = [
"104001", "093806", "093801", "081002", "081007", "080718"
]
selected_ids = ids_with_lakes_upstream
filedir = Path(hdf_folder)
sim_name_to_file_path = OrderedDict([
# ("CRCM5-LI", filedir.joinpath("quebec_0.1_crcm5-hcd-r.hdf5").as_posix()),
("ERAI-CRCM5-L", filedir.joinpath("quebec_0.1_crcm5-hcd-rl.hdf5").as_posix()),
# ("CanESM2-CRCM5-NL", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-r-cc-canesm2-1980-2010.hdf5").as_posix()),
("CanESM2-CRCM5-L",
filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-cc-canesm2-1980-2010.hdf5").as_posix()),
# ("CanESM2-CRCM5-LI", filedir.joinpath("cc-canesm2-driven/quebec_0.1_crcm5-hcd-rl-intfl-cc-canesm2-1980-2010.hdf5").as_posix()),
])
obs_label = "Obs."
labels = [obs_label, ] + list(sim_name_to_file_path.keys())
label_to_marker = dict(zip(labels, all_markers))
label_to_color = dict(zip(labels, all_colors))
# Get the list of stations to do the comparison with
start_date = datetime(start_year, 1, 1)
end_date = datetime(end_year, 12, 31)
stations = cehq_station.read_station_data(
start_date=start_date, end_date=end_date, selected_ids=selected_ids
)
# Get geophysical fields from one of the model simulations
path0 = list(sim_name_to_file_path.values())[0]
lons2d, lats2d, basemap = analysis.get_basemap_from_hdf(file_path=path0)
flow_directions = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_FLOW_DIRECTIONS_NAME)
lake_fraction = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_LAKE_FRACTION_NAME)
accumulation_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_ACCUMULATION_AREA_NAME)
area_m2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_M2)
# Try to read cell areas im meters if it is not Ok then try in km2
if area_m2 is not None:
cell_area_km2 = area_m2 * 1.0e-6
else:
cell_area_km2 = analysis.get_array_from_file(path=path0, var_name=infovar.HDF_CELL_AREA_NAME_KM2)
# Create a cell manager if it is not provided
cell_manager = CellManager(flow_directions, accumulation_area_km2=accumulation_area_km2,
lons2d=lons2d, lats2d=lats2d)
# Get the list of the corresponding model points
station_to_modelpoint = cell_manager.get_model_points_for_stations(
station_list=stations,
lake_fraction=lake_fraction,
drainaige_area_reldiff_limit=0.1)
# plot_utils.apply_plot_params(font_size=10, width_cm=20, height_cm=18)
fig = plt.figure()
ncols = max([len(rp_list) for et, rp_list in ExtremeProperties.extreme_type_to_return_periods.items()])
nrows = len(ExtremeProperties.extreme_types)
gs = GridSpec(nrows, ncols)
ext_type_to_rp_to_ax = OrderedDict()
ax_with_legend = None
label_to_ax_to_xdata = {}
label_to_ax_to_ydata = {}
for row, ext_type in enumerate(ExtremeProperties.extreme_types):
ext_type_to_rp_to_ax[ext_type] = OrderedDict()
for col, rperiod in enumerate(ExtremeProperties.extreme_type_to_return_periods[ext_type]):
ax = fig.add_subplot(gs[row, col])
ext_type_to_rp_to_ax[ext_type][rperiod] = ax
if col == 0:
ax.set_ylabel(ext_type)
if row == nrows - 1 and col == ncols - 1:
ax_with_legend = ax
# Set axes labels
if row == nrows - 1:
ax.set_xlabel("Observations")
if col == 0:
ax.set_ylabel("Model")
for label in sim_name_to_file_path:
if label not in label_to_ax_to_xdata:
label_to_ax_to_xdata[label] = {ax: []}
label_to_ax_to_ydata[label] = {ax: []}
else:
label_to_ax_to_xdata[label][ax] = []
label_to_ax_to_ydata[label][ax] = []
ax.set_xscale("log")
ax.set_yscale("log")
print("Initial list of stations:")
sim_label_to_handle = {}
for s in stations:
print("{0}".format(s))
assert isinstance(s, Station)
print(len([y for y in s.get_list_of_complete_years() if start_year <= y <= end_year]))
df_ext_obs = extreme_commons.get_annual_extrema(ts_times=s.dates, ts_vals=s.values,
start_year=start_year, end_year=end_year)
mp = station_to_modelpoint[s]
assert isinstance(mp, ModelPoint)
years_of_interest = df_ext_obs.index
label_to_extrema_model = {}
# label -> ext_type -> [return period -> ret level, return period -> std]
label_to_return_levels = OrderedDict(
[(obs_label, OrderedDict())]
)
for sim_label, sim_path in sim_name_to_file_path.items():
label_to_return_levels[sim_label] = OrderedDict()
label_to_extrema_model[sim_label] = OrderedDict()
# Calculate the return levels and standard deviations
for ext_type in ExtremeProperties.extreme_types:
return_periods = ExtremeProperties.extreme_type_to_return_periods[ext_type]
# fit GEV distribution and apply non-parametric bootstrap to get std
label_to_return_levels[obs_label][ext_type] = gevfit.do_gevfit_for_a_point(df_ext_obs[ext_type].values,
extreme_type=ext_type,
return_periods=return_periods)
return_levels_obs, rl_stds_obs = label_to_return_levels[obs_label][ext_type]
# get annual extremas for the model output at the points colose to the stations
for sim_label, sim_path in sim_name_to_file_path.items():
label_to_return_levels[sim_label] = OrderedDict()
ext_field = analysis.get_annual_extrema(
rconfig=RunConfig(data_path=sim_path, start_year=start_year, end_year=end_year),
varname="STFL", months_of_interest=ExtremeProperties.extreme_type_to_month_of_interest[ext_type],
n_avg_days=ExtremeProperties.extreme_type_to_n_agv_days[ext_type],
high_flow=ext_type == ExtremeProperties.high)
# Select only those years when obs are available
ts_data = [v for y, v in zip(range(start_year, end_year + 1), ext_field[:, mp.ix, mp.jy]) if
y in years_of_interest]
ts_data = np.array(ts_data)
return_levels, rl_stds = gevfit.do_gevfit_for_a_point(ts_data, extreme_type=ext_type,
return_periods=return_periods)
# Do the plotting
for rp in return_periods:
ax = ext_type_to_rp_to_ax[ext_type][rp]
ax.set_title("T = {rp}-year".format(rp=rp))
# h = ax.errorbar(return_levels_obs[rp], return_levels[rp],
# marker=label_to_marker[sim_label], color=label_to_color[sim_label], label=sim_label,
# xerr=rl_stds_obs[rp] * 1.96, yerr=rl_stds[rp] * 1.96)
h = ax.scatter(return_levels_obs[rp], return_levels[rp],
marker=label_to_marker[sim_label], color=label_to_color[sim_label], label=sim_label)
# save the data for maybe further calculation of the correlation coefficients
label_to_ax_to_xdata[sim_label][ax].append(return_levels_obs[rp])
label_to_ax_to_ydata[sim_label][ax].append(return_levels[rp])
sim_label_to_handle[sim_label] = h
# Calculate the biases
for sim_label in sim_name_to_file_path:
for ext_type in ExtremeProperties.extreme_types:
ret_periods = ExtremeProperties.extreme_type_to_return_periods[ext_type]
for rp in ret_periods:
ax = ext_type_to_rp_to_ax[ext_type][rp]
mod = np.asarray(label_to_ax_to_ydata[sim_label][ax])
obs = np.asarray(label_to_ax_to_xdata[sim_label][ax])
bias = np.mean((mod - obs)/obs)
corr, pv = stats.pearsonr(mod, obs)
print("({sim_label}) Mean bias for {rp}-year {ext_type}-flow return level is: {bias}; corr={corr:.2f}; corr_pval={corr_pval:2g}".format(
sim_label=sim_label, rp=rp, bias=bias, corr=corr, corr_pval=pv,
ext_type=ext_type
))
sfmt = ScalarFormatter(useMathText=True)
sfmt.set_powerlimits((-2, 2))
for et, rp_to_ax in ext_type_to_rp_to_ax.items():
for rp, ax in rp_to_ax.items():
xmin, xmax = ax.get_xlim()
ymin, ymax = ax.get_ylim()
x1 = min(xmin, ymin)
x2 = min(xmax, ymax)
ax.plot([x1, x2], [x1, x2], "k--")
# ax.xaxis.set_major_locator(MaxNLocator(nbins=5))
# ax.yaxis.set_major_locator(MaxNLocator(nbins=5))
# ax.xaxis.set_major_formatter(sfmt)
# ax.yaxis.set_major_formatter(sfmt)
sim_labels = list(sim_name_to_file_path.keys())
ax_with_legend.legend([sim_label_to_handle[sl] for sl in sim_labels], sim_labels,
bbox_to_anchor=(1, -0.25), borderaxespad=0.0, loc="upper right",
ncol=2, scatterpoints=1, numpoints=1)
# Save the plot
img_file = "{}.eps".format("_".join(sorted(label_to_marker.keys())))
img_file = img_folder.joinpath(img_file)
fig.tight_layout()
with img_file.open("wb") as f:
fig.savefig(f, bbox_inches="tight")
