def get_station_objects(db_path="/home/san/Downloads/Hydat.sqlite"):
selected_ids = ["05BB001", "05BH005", "05BH004", "05BM004"]
stations = cehq_station.load_from_hydat_db(natural=None, province=PROVINCE, path=db_path, selected_ids=selected_ids)
for s in stations:
print(s)
# add an additional station here, if required
return stations
def main():
"""
:return:
"""
s = """
06DA002
06CD002
06EA002
05FE004
05EF001
05BN012
05CK004
06AG006
05AK001
05QB003
05LM006
05KJ001
05MD004
05JU001
"""
selected_ids = [tok.strip().upper() for tok in s.split("\n") if tok != ""]
print(selected_ids)
stations = cehq_station.load_from_hydat_db(
province=None,
selected_ids=selected_ids,
skip_data_checks=True,
natural=None
)
print(20 * "---")
for station in stations:
assert isinstance(station, Station)
print("{}\t{:.4f}\t{:.4f}\t{}".format(station.id, station.longitude, station.latitude, station.drainage_km2))
print(20 * "---")
def main():
stations = cehq_station.load_from_hydat_db(natural=True, province="QC")
dm = Crcm5ModelDataManager(
samples_folder_path="/skynet3_rech1/huziy/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup",
all_files_in_samples_folder=True)
basemap = dm.get_rotpole_basemap()
lons = [s.longitude for s in stations]
lats = [s.latitude for s in stations]
n_cont_years = [len(s.get_list_of_complete_years()) for s in stations]
x, y = basemap(lons, lats)
basemap.scatter(x, y, c=n_cont_years)
basemap.drawcoastlines()
basemap.colorbar()
plt.show()
def main():
stations = cehq_station.load_from_hydat_db(natural=True, province="QC")
dm = Crcm5ModelDataManager(
samples_folder_path=
"/skynet3_rech1/huziy/from_guillimin/new_outputs/quebec_0.1_crcm5-r_spinup",
all_files_in_samples_folder=True)
basemap = dm.get_rotpole_basemap()
lons = [s.longitude for s in stations]
lats = [s.latitude for s in stations]
n_cont_years = [len(s.get_list_of_complete_years()) for s in stations]
x, y = basemap(lons, lats)
basemap.scatter(x, y, c=n_cont_years)
basemap.drawcoastlines()
basemap.colorbar()
plt.show()
def main():
"""
:return:
"""
s = """
06DA002
06CD002
06EA002
05FE004
05EF001
05BN012
05CK004
06AG006
05AK001
05QB003
05LM006
05KJ001
05MD004
05JU001
"""
selected_ids = [tok.strip().upper() for tok in s.split("\n") if tok != ""]
print(selected_ids)
stations = cehq_station.load_from_hydat_db(province=None,
selected_ids=selected_ids,
skip_data_checks=True,
natural=None)
print(20 * "---")
for station in stations:
assert isinstance(station, Station)
print("{}\t{:.4f}\t{:.4f}\t{}".format(station.id, station.longitude,
station.latitude,
station.drainage_km2))
print(20 * "---")
def main():
# stations = cehq_station.read_grdc_stations(st_id_list=["2903430", "2909150", "2912600", "4208025"])
selected_ids = [
"08MH001", "08NE074", "08NG065", "08NJ013", "08NK002", "08NK016",
"08NL004", "08NL007", "08NL024", "08NL038", "08NN002"
]
stations = cehq_station.load_from_hydat_db(natural=True,
province="BC",
selected_ids=selected_ids)
stations_to_mp = None
import matplotlib.pyplot as plt
# labels = ["CanESM", "MPI"]
# paths = ["/skynet3_rech1/huziy/offline_stfl/canesm/discharge_1958_01_01_00_00.nc",
# "/skynet3_rech1/huziy/offline_stfl/mpi/discharge_1958_01_01_00_00.nc"]
#
# colors = ["r", "b"]
# labels = ["ERA", ]
# colors = ["r", ]
# paths = ["/skynet3_rech1/huziy/arctic_routing/era40/discharge_1958_01_01_00_00.nc"]
labels = ["Glacier-only", "All"]
colors = ["r", "b"]
paths = [
"/skynet3_exec2/aganji/glacier_katja/watroute_gemera/discharge_stat_glac_00_99_2000_01_01_00_00.nc",
"/skynet3_exec2/aganji/glacier_katja/watroute_gemera/discharge_stat_both_00_992000_01_01_00_00.nc"
]
start_year_current = 2000
end_year_current = 2013
plot_future = False
start_year_future = 2071 # ignored when plot future is false
end_year_future = 2100
if not plot_future:
start_year = start_year_current
end_year = end_year_current
else:
start_year = start_year_future
end_year = end_year_future
stations_filtered = []
for s in stations:
# Also filter out stations with small accumulation areas
if s.drainage_km2 < 1000:
continue
if s.latitude > 49.4:
continue
# Filter stations with data out of the required time frame
year_list = s.get_list_of_complete_years()
if max(year_list) < start_year or min(year_list) > end_year:
continue
stations_filtered.append(s)
stations = stations_filtered
min_lon = min(s.longitude for s in stations)
stations = [s for s in stations if s.longitude == min_lon]
print("Retained {} stations.".format(len(stations)))
sim_to_time = {}
monthly_dates = [datetime(2001, m, 15) for m in range(1, 13)]
fmt = FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0])
locator = MonthLocator()
fig = plt.figure()
axes = []
row_indices = []
col_indices = []
ncols = 1
shiftrow = 0 if len(stations) % ncols == 0 else 1
nrows = len(stations) // ncols + shiftrow
shared_ax = None
gs = gridspec.GridSpec(ncols=ncols, nrows=nrows)
for i, s in enumerate(stations):
row = i // ncols
col = i % ncols
row_indices.append(row)
col_indices.append(col)
if shared_ax is None:
ax = fig.add_subplot(gs[row, col])
shared_ax = ax
assert isinstance(shared_ax, Axes)
else:
ax = fig.add_subplot(gs[row, col])
ax.xaxis.set_major_locator(locator)
ax.yaxis.set_major_locator(MaxNLocator(nbins=4))
ax.xaxis.set_major_formatter(fmt)
sfmt = ScalarFormatter(useMathText=True)
sfmt.set_powerlimits((-3, 4))
ax.yaxis.set_major_formatter(sfmt)
assert isinstance(ax, Axes)
axes.append(ax)
# generate daily stamp dates
d0 = datetime(2001, 1, 1)
stamp_dates = [d0 + timedelta(days=i) for i in range(365)]
# plot a panel for each station
for s, ax, row, col in zip(stations, axes, row_indices, col_indices):
assert isinstance(s, Station)
assert isinstance(ax, Axes)
if s.grdc_monthly_clim_max is not None:
ax.fill_between(monthly_dates,
s.grdc_monthly_clim_min,
s.grdc_monthly_clim_max,
color="0.6",
alpha=0.5)
avail_years = s.get_list_of_complete_years()
print("{}: {}".format(s.id, ",".join([str(y) for y in avail_years])))
years = [y for y in avail_years if start_year <= y <= end_year]
_, obs_clim_stfl = s.get_daily_climatology_for_complete_years_with_pandas(
stamp_dates=stamp_dates, years=years)
if obs_clim_stfl is None:
continue
ax.plot(stamp_dates, obs_clim_stfl, "k", lw=3, label="Obs")
if s.river_name is not None and s.river_name != "":
ax.set_title(s.river_name)
else:
ax.set_title(s.id)
for path, sim_label, color in zip(paths, labels, colors):
ds = Dataset(path)
if stations_to_mp is None:
acc_area_2d = ds.variables["accumulation_area"][:]
lons2d, lats2d = ds.variables["longitude"][:], ds.variables[
"latitude"][:]
x_index, y_index = ds.variables["x_index"][:], ds.variables[
"y_index"][:]
stations_to_mp = get_dataless_model_points_for_stations(
stations, acc_area_2d, lons2d, lats2d, x_index, y_index)
# read dates only once for a given simulation
if sim_label not in sim_to_time:
time_str = ds.variables["time"][:].astype(str)
times = [
datetime.strptime("".join(t_s), TIME_FORMAT)
for t_s in time_str
]
sim_to_time[sim_label] = times
mp = stations_to_mp[s]
data = ds.variables["water_discharge_accumulated"][:,
mp.cell_index]
print(path)
df = DataFrame(data=data,
index=sim_to_time[sim_label],
columns=["value"])
df["year"] = df.index.map(lambda d: d.year)
df = df.ix[df.year.isin(years), :]
df = df.select(lambda d: not (d.month == 2 and d.day == 29))
df = df.groupby(lambda d: datetime(stamp_dates[0].year, d.month, d.
day)).mean()
daily_model_data = [df.ix[d, "value"] for d in stamp_dates]
# print np.mean( monthly_model ), s.river_name, sim_label
ax.plot(stamp_dates,
daily_model_data,
color,
lw=3,
label=sim_label + "(C)")
if plot_future:
ax.plot(stamp_dates,
daily_model_data,
color + "--",
lw=3,
label=sim_label + "(F2)")
ds.close()
if row < nrows - 1:
ax.set_xticklabels([])
axes[0].legend(fontsize=17, loc=2)
plt.tight_layout()
plt.savefig("offline_validation.png", dpi=400)
plt.close(fig)
r = RPN(
"/RESCUE/skynet3_rech1/huziy/CNRCWP/C3/Depth_to_bedrock_WestNA_0.25")
r.get_first_record_for_name("8L")
proj_params = r.get_proj_parameters_for_the_last_read_rec()
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
bsmp = RotatedLatLon(**proj_params).get_basemap_object_for_lons_lats(
lons2d=lons, lats2d=lats)
plot_utils.apply_plot_params(width_pt=None,
width_cm=19,
height_cm=19,
font_size=12)
plot_station_positions(manager=None, station_list=stations, bsmp=bsmp)
def main():
model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Diagnostics")
# model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Samples")
static_data_file = "/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p"
r = RPN(static_data_file)
fldir = r.get_first_record_for_name("FLDR")
faa = r.get_first_record_for_name("FAA")
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
gc = default_domains.bc_mh_044
cell_manager = CellManager(
fldir, nx=fldir.shape[0], ny=fldir.shape[1], lons2d=lons, lats2d=lats, accumulation_area_km2=faa
)
selected_station_ids = ["06EA002"]
stations = cehq_station.load_from_hydat_db(province="SK", selected_ids=selected_station_ids, natural=None)
# (06EA002): CHURCHILL RIVER AT SANDY BAY at (-102.31832885742188,55.52333068847656), accum. area is 212000.0 km**2
# TODO: plot where is this station, compare modelled and observed hydrographs
# for s in stations:
# assert isinstance(s, cehq_station.Station)
# s.latitude += 0.9
# s.longitude -= 0.2
# print(s)
station_to_model_point = cell_manager.get_model_points_for_stations(
stations, drainaige_area_reldiff_limit=0.8, nneighbours=1
)
print(station_to_model_point[stations[0]])
station = stations[0]
assert isinstance(station, cehq_station.Station)
obs_not_corrected = (
pd.Series(index=station.dates, data=station.values).groupby(by=lambda d: d.replace(day=15)).mean()
)
obs_corrected = pd.read_csv(
"mh/obs_data/Churchill Historic Monthly Apportionable Flow_06EA002.csv.bak.original", skiprows=2
)
print(obs_corrected.head())
print(obs_corrected.year.iloc[0], obs_corrected.year.iloc[-1])
date_index = pd.date_range(
start=datetime(obs_corrected.year.iloc[0] - 1, 12, 15),
end=datetime(obs_corrected.year.iloc[-1], 12, 15),
freq="M",
)
date_index = date_index.shift(15, freq=pd.datetools.day)
print(date_index)
data = np.concatenate([r for r in obs_corrected.values[:, 1:-1]])
factor = date_index.map(lambda d: 1000 / (calendar.monthrange(d.year, d.month)[1] * 24 * 3600))
print(factor[:10])
obs_corrected = pd.Series(index=date_index, data=data * factor)
station_to_modelled_data = get_model_data(
station_to_model_point,
output_path=model_data_path,
grid_config=gc,
basins_of_interest_shp=default_domains.MH_BASINS_PATH,
cell_manager=cell_manager,
vname="STFL",
)
modelled_data = station_to_modelled_data[station]
fig = plt.figure()
ax = obs_corrected.plot(label="obs corrected")
obs_not_corrected.plot(label="obs not corrected", ax=ax, color="k")
modelled_data.plot(label="CRCM5", ax=ax, color="r")
ax.legend(loc="upper left")
img_file = img_folder.joinpath("{}_validation_monthly.png".format(station.id))
fig.savefig(str(img_file))
plt.close(fig)
# climatology
start_year = 1980
end_year = 2010
date_selector = lambda d: (start_year <= d.year <= end_year) and not ((d.month == 2) and (d.day == 29))
fig = plt.figure()
ax = obs_corrected.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(label="obs corrected")
obs_not_corrected.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(
label="obs not corrected", ax=ax, color="k"
)
modelled_data.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(
label="CRCM5", ax=ax, color="r"
)
ax.xaxis.set_major_locator(MonthLocator(bymonthday=15))
ax.xaxis.set_major_formatter(DateFormatter("%b"))
ax.legend(loc="upper left")
img_file = img_folder.joinpath("{}_validation_clim.png".format(station.id))
fig.savefig(str(img_file))
plt.close(fig)
# Interannual variability
fig = plt.figure()
obs_corrected = obs_corrected.select(lambda d: start_year <= d.year <= end_year)
modelled_data = modelled_data.select(lambda d: start_year <= d.year <= end_year)
corr_list = []
for m in range(1, 13):
obs = obs_corrected.select(lambda d: d.month == m)
mod = modelled_data.select(lambda d: d.month == m)
print(obs.head())
obs.index = obs.index.map(lambda d: d.year)
mod.index = mod.index.map(lambda d: d.year)
corr_list.append(obs.corr(mod))
ax = plt.gca()
ax.plot(range(1, 13), corr_list)
ax.set_xlabel("Month")
ax.set_title("Inter-annual variability")
img_file = img_folder.joinpath("{}_interannual.png".format(station.id))
fig.tight_layout()
fig.savefig(str(img_file), bbox_inches="tight")
plt.close(fig)
def main():
# stations = cehq_station.read_grdc_stations(st_id_list=["2903430", "2909150", "2912600", "4208025"])
selected_station_ids = [
"05LM006", "05BN012", "05AK001", "05QB003", "06EA002"
]
stations = cehq_station.load_from_hydat_db(
natural=None,
province=None,
selected_ids=selected_station_ids,
skip_data_checks=True)
stations_mh = cehq_station.get_manitoba_hydro_stations()
# copy metadata from the corresponding hydat stations
for s in stations:
assert isinstance(s, Station)
for s_mh in stations_mh:
assert isinstance(s_mh, Station)
if s == s_mh:
s_mh.copy_metadata(s)
break
stations = [
s for s in stations_mh
if s.id in selected_station_ids and s.longitude is not None
]
stations_to_mp = None
import matplotlib.pyplot as plt
# labels = ["CanESM", "MPI"]
# paths = ["/skynet3_rech1/huziy/offline_stfl/canesm/discharge_1958_01_01_00_00.nc",
# "/skynet3_rech1/huziy/offline_stfl/mpi/discharge_1958_01_01_00_00.nc"]
#
# colors = ["r", "b"]
# labels = ["ERA", ]
# colors = ["r", ]
# paths = ["/skynet3_rech1/huziy/arctic_routing/era40/discharge_1958_01_01_00_00.nc"]
labels = [
"Model",
]
colors = [
"r",
]
paths = [
"/RESCUE/skynet3_rech1/huziy/water_route_mh_bc_011deg_wc/discharge_1980_01_01_12_00.nc"
]
infocell_path = "/RESCUE/skynet3_rech1/huziy/water_route_mh_bc_011deg_wc/infocell.nc"
start_year = 1980
end_year = 2014
stations_filtered = []
for s in stations:
# Also filter out stations with small accumulation areas
# if s.drainage_km2 is not None and s.drainage_km2 < 100:
# continue
# Filter stations with data out of the required time frame
year_list = s.get_list_of_complete_years()
print("Complete years for {}: {}".format(s.id, year_list))
stations_filtered.append(s)
stations = stations_filtered
print("Retained {} stations.".format(len(stations)))
sim_to_time = {}
monthly_dates = [datetime(2001, m, 15) for m in range(1, 13)]
fmt = FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0])
locator = MonthLocator(bymonthday=15)
fig = plt.figure()
axes = []
row_indices = []
col_indices = []
ncols = 1
shiftrow = 0 if len(stations) % ncols == 0 else 1
nrows = len(stations) // ncols + shiftrow
shared_ax = None
gs = gridspec.GridSpec(ncols=ncols, nrows=nrows)
for i, s in enumerate(stations):
row = i // ncols
col = i % ncols
row_indices.append(row)
col_indices.append(col)
if shared_ax is None:
ax = fig.add_subplot(gs[row, col])
shared_ax = ax
assert isinstance(shared_ax, Axes)
else:
ax = fig.add_subplot(gs[row, col])
ax.xaxis.set_major_locator(locator)
ax.yaxis.set_major_locator(MaxNLocator(nbins=4))
ax.xaxis.set_major_formatter(fmt)
sfmt = ScalarFormatter(useMathText=True)
sfmt.set_powerlimits((-3, 4))
ax.yaxis.set_major_formatter(sfmt)
assert isinstance(ax, Axes)
axes.append(ax)
# generate daily stamp dates
d0 = datetime(2001, 1, 1)
stamp_dates = [d0 + timedelta(days=i) for i in range(365)]
# plot a panel for each station
for s, ax, row, col in zip(stations, axes, row_indices, col_indices):
assert isinstance(s, Station)
assert isinstance(ax, Axes)
if s.grdc_monthly_clim_max is not None:
ax.fill_between(monthly_dates,
s.grdc_monthly_clim_min,
s.grdc_monthly_clim_max,
color="0.6",
alpha=0.5)
avail_years = s.get_list_of_complete_years()
print("{}: {}".format(s.id, ",".join([str(y) for y in avail_years])))
years = [y for y in avail_years if start_year <= y <= end_year]
obs_clim_stfl = s.get_monthly_climatology(years_list=years)
if obs_clim_stfl is None:
continue
print(obs_clim_stfl.head())
obs_clim_stfl.plot(color="k", lw=3, label="Obs", ax=ax)
if s.river_name is not None and s.river_name != "":
ax.set_title(s.river_name)
else:
ax.set_title(s.id)
for path, sim_label, color in zip(paths, labels, colors):
ds = Dataset(path)
if stations_to_mp is None:
acc_area_2d = ds.variables["accumulation_area"][:]
lons2d, lats2d = ds.variables["longitude"][:], ds.variables[
"latitude"][:]
x_index, y_index = ds.variables["x_index"][:], ds.variables[
"y_index"][:]
stations_to_mp = get_dataless_model_points_for_stations(
stations, acc_area_2d, lons2d, lats2d, x_index, y_index)
# read dates only once for a given simulation
if sim_label not in sim_to_time:
time_str = ds.variables["time"][:].astype(str)
times = [
datetime.strptime("".join(t_s), TIME_FORMAT)
for t_s in time_str
]
sim_to_time[sim_label] = times
mp = stations_to_mp[s]
data = ds.variables["water_discharge_accumulated"][:,
mp.cell_index]
print(path)
df = DataFrame(data=data,
index=sim_to_time[sim_label],
columns=["value"])
df["year"] = df.index.map(lambda d: d.year)
df = df.ix[df.year.isin(years), :]
df = df.groupby(lambda d: datetime(2001, d.month, 15)).mean()
# print np.mean( monthly_model ), s.river_name, sim_label
df.plot(color=color, lw=3, label=sim_label, ax=ax, y="value")
ds.close()
if row < nrows - 1:
ax.set_xticklabels([])
axes[0].legend(fontsize=17, loc=2)
plt.tight_layout()
plt.savefig("mh/offline_validation_mh.png", dpi=400)
plt.close(fig)
with Dataset(infocell_path) as ds:
fldir = ds.variables["flow_direction_value"][:]
faa = ds.variables["accumulation_area"][:]
lon, lat = [ds.variables[k][:] for k in ["lon", "lat"]]
# plot station positions and upstream areas
cell_manager = CellManager(fldir,
nx=fldir.shape[0],
ny=fldir.shape[1],
lons2d=lon,
lats2d=lat,
accumulation_area_km2=faa)
fig = plt.figure()
from crcm5.mh_domains import default_domains
gc = default_domains.bc_mh_011
# get the basemap object
bmp, data_mask = gc.get_basemap_using_shape_with_polygons_of_interest(
lon, lat, shp_path=default_domains.MH_BASINS_PATH, mask_margin=5)
xx, yy = bmp(lon, lat)
ax = plt.gca()
colors = ["g", "r", "m", "c", "y", "violet"]
i = 0
for s, mp in stations_to_mp.items():
assert isinstance(mp, ModelPoint)
upstream_mask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices(
mp.ix, mp.jy)
current_points = upstream_mask > 0.5
bmp.drawcoastlines()
bmp.drawrivers()
bmp.scatter(xx[current_points],
yy[current_points],
c=colors[i % len(colors)])
i += 1
va = "top"
if s.id in ["05AK001", "05LM006"]:
va = "bottom"
ha = "left"
if s.id in ["05QB003"]:
ha = "right"
bmp.scatter(xx[mp.ix, mp.jy], yy[mp.ix, mp.jy], c="b")
ax.annotate(s.id,
xy=(xx[mp.ix, mp.jy], yy[mp.ix, mp.jy]),
horizontalalignment=ha,
verticalalignment=va,
bbox=dict(boxstyle='round', fc='gray', alpha=0.5))
fig.savefig("mh/offline_stations_{}.png".format("positions"))
plt.close(fig)
def main():
start_year = 1980
end_year = 2010
# model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Diagnostics")
model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Samples")
static_data_file = "/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p"
corrected_obs_data_folder = Path("mh/obs_data/")
r = RPN(static_data_file)
fldir = r.get_first_record_for_name("FLDR")
faa = r.get_first_record_for_name("FAA")
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
gc = default_domains.bc_mh_044
cell_manager = CellManager(fldir, nx=fldir.shape[0], ny=fldir.shape[1],
lons2d=lons, lats2d=lats, accumulation_area_km2=faa)
selected_station_ids = [
"05LM006",
"05BN012",
"05AK001",
"05QB003"
]
stations = cehq_station.load_from_hydat_db(province=None, selected_ids=selected_station_ids, natural=None, skip_data_checks=True)
for s in stations:
assert isinstance(s, cehq_station.Station)
if s.id == "05AK001":
s.drainage_km2 *= 2.5
if s.id == "05BN012":
pass
# Manitoba natural stations
# statons_mnb = cehq_station.load_from_hydat_db(province="MB", natural=True, start_date=datetime(start_year, 1, 1), end_date=datetime(end_year,12,31))
# statons_ssk = cehq_station.load_from_hydat_db(province="SK", natural=True, start_date=datetime(start_year, 1, 1), end_date=datetime(end_year,12,31))
# statons_alb = cehq_station.load_from_hydat_db(province="AB", natural=True, start_date=datetime(start_year, 1, 1), end_date=datetime(end_year,12,31))
# for s in statons_mnb + statons_ssk + statons_alb:
# if s not in stations:
# stations.append(s)
# (06EA002): CHURCHILL RIVER AT SANDY BAY at (-102.31832885742188,55.52333068847656), accum. area is 212000.0 km**2
# TODO: plot where is this station, compare modelled and observed hydrographs
for s in stations:
print(s)
# assert len(stations) == len(selected_station_ids), "Could not find stations for some of the specified ids"
station_to_model_point = cell_manager.get_model_points_for_stations(stations, drainaige_area_reldiff_limit=0.9,
nneighbours=8)
print("Established the station to model point mapping")
plot_validations_for_stations(station_to_model_point,
cell_manager=cell_manager,
corrected_obs_data_folder=corrected_obs_data_folder,
model_data_path=model_data_path,
grid_config=gc, start_year=start_year, end_year=end_year)
def main():
# stations = cehq_station.read_grdc_stations(st_id_list=["2903430", "2909150", "2912600", "4208025"])
selected_ids = ["08MH001", "08NE074", "08NG065", "08NJ013", "08NK002", "08NK016",
"08NL004", "08NL007", "08NL024", "08NL038", "08NN002"]
stations = cehq_station.load_from_hydat_db(natural=True, province="BC", selected_ids=selected_ids)
stations_to_mp = None
import matplotlib.pyplot as plt
# labels = ["CanESM", "MPI"]
# paths = ["/skynet3_rech1/huziy/offline_stfl/canesm/discharge_1958_01_01_00_00.nc",
# "/skynet3_rech1/huziy/offline_stfl/mpi/discharge_1958_01_01_00_00.nc"]
#
# colors = ["r", "b"]
# labels = ["ERA", ]
# colors = ["r", ]
# paths = ["/skynet3_rech1/huziy/arctic_routing/era40/discharge_1958_01_01_00_00.nc"]
labels = ["Glacier-only", "All"]
colors = ["r", "b"]
paths = [
"/skynet3_exec2/aganji/glacier_katja/watroute_gemera/discharge_stat_glac_00_99_2000_01_01_00_00.nc",
"/skynet3_exec2/aganji/glacier_katja/watroute_gemera/discharge_stat_both_00_992000_01_01_00_00.nc"]
start_year_current = 2000
end_year_current = 2013
plot_future = False
start_year_future = 2071 # ignored when plot future is false
end_year_future = 2100
if not plot_future:
start_year = start_year_current
end_year = end_year_current
else:
start_year = start_year_future
end_year = end_year_future
stations_filtered = []
for s in stations:
# Also filter out stations with small accumulation areas
if s.drainage_km2 < 1000:
continue
if s.latitude > 49.4:
continue
# Filter stations with data out of the required time frame
year_list = s.get_list_of_complete_years()
if max(year_list) < start_year or min(year_list) > end_year:
continue
stations_filtered.append(s)
stations = stations_filtered
min_lon = min(s.longitude for s in stations)
stations = [s for s in stations if s.longitude == min_lon]
print("Retained {} stations.".format(len(stations)))
sim_to_time = {}
monthly_dates = [datetime(2001, m, 15) for m in range(1, 13)]
fmt = FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0])
locator = MonthLocator()
fig = plt.figure()
axes = []
row_indices = []
col_indices = []
ncols = 1
shiftrow = 0 if len(stations) % ncols == 0 else 1
nrows = len(stations) // ncols + shiftrow
shared_ax = None
gs = gridspec.GridSpec(ncols=ncols, nrows=nrows)
for i, s in enumerate(stations):
row = i // ncols
col = i % ncols
row_indices.append(row)
col_indices.append(col)
if shared_ax is None:
ax = fig.add_subplot(gs[row, col])
shared_ax = ax
assert isinstance(shared_ax, Axes)
else:
ax = fig.add_subplot(gs[row, col])
ax.xaxis.set_major_locator(locator)
ax.yaxis.set_major_locator(MaxNLocator(nbins=4))
ax.xaxis.set_major_formatter(fmt)
sfmt = ScalarFormatter(useMathText=True)
sfmt.set_powerlimits((-3, 4))
ax.yaxis.set_major_formatter(sfmt)
assert isinstance(ax, Axes)
axes.append(ax)
# generate daily stamp dates
d0 = datetime(2001, 1, 1)
stamp_dates = [d0 + timedelta(days=i) for i in range(365)]
# plot a panel for each station
for s, ax, row, col in zip(stations, axes, row_indices, col_indices):
assert isinstance(s, Station)
assert isinstance(ax, Axes)
if s.grdc_monthly_clim_max is not None:
ax.fill_between(monthly_dates, s.grdc_monthly_clim_min, s.grdc_monthly_clim_max, color="0.6", alpha=0.5)
avail_years = s.get_list_of_complete_years()
print("{}: {}".format(s.id, ",".join([str(y) for y in avail_years])))
years = [y for y in avail_years if start_year <= y <= end_year]
_, obs_clim_stfl = s.get_daily_climatology_for_complete_years_with_pandas(stamp_dates=stamp_dates, years=years)
if obs_clim_stfl is None:
continue
ax.plot(stamp_dates, obs_clim_stfl, "k", lw=3, label="Obs")
if s.river_name is not None and s.river_name != "":
ax.set_title(s.river_name)
else:
ax.set_title(s.id)
for path, sim_label, color in zip(paths, labels, colors):
ds = Dataset(path)
if stations_to_mp is None:
acc_area_2d = ds.variables["accumulation_area"][:]
lons2d, lats2d = ds.variables["longitude"][:], ds.variables["latitude"][:]
x_index, y_index = ds.variables["x_index"][:], ds.variables["y_index"][:]
stations_to_mp = get_dataless_model_points_for_stations(stations, acc_area_2d,
lons2d, lats2d, x_index, y_index)
# read dates only once for a given simulation
if sim_label not in sim_to_time:
time_str = ds.variables["time"][:].astype(str)
times = [datetime.strptime("".join(t_s), TIME_FORMAT) for t_s in time_str]
sim_to_time[sim_label] = times
mp = stations_to_mp[s]
data = ds.variables["water_discharge_accumulated"][:, mp.cell_index]
print(path)
df = DataFrame(data=data, index=sim_to_time[sim_label], columns=["value"])
df["year"] = df.index.map(lambda d: d.year)
df = df.ix[df.year.isin(years), :]
df = df.select(lambda d: not (d.month == 2 and d.day == 29))
df = df.groupby(lambda d: datetime(stamp_dates[0].year, d.month, d.day)).mean()
daily_model_data = [df.ix[d, "value"] for d in stamp_dates]
# print np.mean( monthly_model ), s.river_name, sim_label
ax.plot(stamp_dates, daily_model_data, color, lw=3, label=sim_label + "(C)")
if plot_future:
ax.plot(stamp_dates, daily_model_data, color + "--", lw=3, label=sim_label + "(F2)")
ds.close()
if row < nrows - 1:
ax.set_xticklabels([])
axes[0].legend(fontsize=17, loc=2)
plt.tight_layout()
plt.savefig("offline_validation.png", dpi=400)
plt.close(fig)
r = RPN("/RESCUE/skynet3_rech1/huziy/CNRCWP/C3/Depth_to_bedrock_WestNA_0.25")
r.get_first_record_for_name("8L")
proj_params = r.get_proj_parameters_for_the_last_read_rec()
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
bsmp = RotatedLatLon(**proj_params).get_basemap_object_for_lons_lats(lons2d=lons, lats2d=lats)
plot_utils.apply_plot_params(width_pt=None, width_cm=19, height_cm=19, font_size=12)
plot_station_positions(manager=None, station_list=stations, bsmp=bsmp)
def main():
# stations = cehq_station.read_grdc_stations(st_id_list=["2903430", "2909150", "2912600", "4208025"])
selected_station_ids = [
"05LM006",
"05BN012",
"05AK001",
"05QB003",
"06EA002"
]
stations = cehq_station.load_from_hydat_db(natural=None, province=None, selected_ids=selected_station_ids, skip_data_checks=True)
stations_mh = cehq_station.get_manitoba_hydro_stations()
# copy metadata from the corresponding hydat stations
for s in stations:
assert isinstance(s, Station)
for s_mh in stations_mh:
assert isinstance(s_mh, Station)
if s == s_mh:
s_mh.copy_metadata(s)
break
stations = [s for s in stations_mh if s.id in selected_station_ids and s.longitude is not None]
stations_to_mp = None
import matplotlib.pyplot as plt
# labels = ["CanESM", "MPI"]
# paths = ["/skynet3_rech1/huziy/offline_stfl/canesm/discharge_1958_01_01_00_00.nc",
# "/skynet3_rech1/huziy/offline_stfl/mpi/discharge_1958_01_01_00_00.nc"]
#
# colors = ["r", "b"]
# labels = ["ERA", ]
# colors = ["r", ]
# paths = ["/skynet3_rech1/huziy/arctic_routing/era40/discharge_1958_01_01_00_00.nc"]
labels = ["Model", ]
colors = ["r", ]
paths = [
"/RESCUE/skynet3_rech1/huziy/water_route_mh_bc_011deg_wc/discharge_1980_01_01_12_00.nc"
]
infocell_path = "/RESCUE/skynet3_rech1/huziy/water_route_mh_bc_011deg_wc/infocell.nc"
start_year = 1980
end_year = 2014
stations_filtered = []
for s in stations:
# Also filter out stations with small accumulation areas
# if s.drainage_km2 is not None and s.drainage_km2 < 100:
# continue
# Filter stations with data out of the required time frame
year_list = s.get_list_of_complete_years()
print("Complete years for {}: {}".format(s.id, year_list))
stations_filtered.append(s)
stations = stations_filtered
print("Retained {} stations.".format(len(stations)))
sim_to_time = {}
monthly_dates = [datetime(2001, m, 15) for m in range(1, 13)]
fmt = FuncFormatter(lambda x, pos: num2date(x).strftime("%b")[0])
locator = MonthLocator(bymonthday=15)
fig = plt.figure()
axes = []
row_indices = []
col_indices = []
ncols = 1
shiftrow = 0 if len(stations) % ncols == 0 else 1
nrows = len(stations) // ncols + shiftrow
shared_ax = None
gs = gridspec.GridSpec(ncols=ncols, nrows=nrows)
for i, s in enumerate(stations):
row = i // ncols
col = i % ncols
row_indices.append(row)
col_indices.append(col)
if shared_ax is None:
ax = fig.add_subplot(gs[row, col])
shared_ax = ax
assert isinstance(shared_ax, Axes)
else:
ax = fig.add_subplot(gs[row, col])
ax.xaxis.set_major_locator(locator)
ax.yaxis.set_major_locator(MaxNLocator(nbins=4))
ax.xaxis.set_major_formatter(fmt)
sfmt = ScalarFormatter(useMathText=True)
sfmt.set_powerlimits((-3, 4))
ax.yaxis.set_major_formatter(sfmt)
assert isinstance(ax, Axes)
axes.append(ax)
# generate daily stamp dates
d0 = datetime(2001, 1, 1)
stamp_dates = [d0 + timedelta(days=i) for i in range(365)]
# plot a panel for each station
for s, ax, row, col in zip(stations, axes, row_indices, col_indices):
assert isinstance(s, Station)
assert isinstance(ax, Axes)
if s.grdc_monthly_clim_max is not None:
ax.fill_between(monthly_dates, s.grdc_monthly_clim_min, s.grdc_monthly_clim_max, color="0.6", alpha=0.5)
avail_years = s.get_list_of_complete_years()
print("{}: {}".format(s.id, ",".join([str(y) for y in avail_years])))
years = [y for y in avail_years if start_year <= y <= end_year]
obs_clim_stfl = s.get_monthly_climatology(years_list=years)
if obs_clim_stfl is None:
continue
print(obs_clim_stfl.head())
obs_clim_stfl.plot(color="k", lw=3, label="Obs", ax=ax)
if s.river_name is not None and s.river_name != "":
ax.set_title(s.river_name)
else:
ax.set_title(s.id)
for path, sim_label, color in zip(paths, labels, colors):
ds = Dataset(path)
if stations_to_mp is None:
acc_area_2d = ds.variables["accumulation_area"][:]
lons2d, lats2d = ds.variables["longitude"][:], ds.variables["latitude"][:]
x_index, y_index = ds.variables["x_index"][:], ds.variables["y_index"][:]
stations_to_mp = get_dataless_model_points_for_stations(stations, acc_area_2d,
lons2d, lats2d, x_index, y_index)
# read dates only once for a given simulation
if sim_label not in sim_to_time:
time_str = ds.variables["time"][:].astype(str)
times = [datetime.strptime("".join(t_s), TIME_FORMAT) for t_s in time_str]
sim_to_time[sim_label] = times
mp = stations_to_mp[s]
data = ds.variables["water_discharge_accumulated"][:, mp.cell_index]
print(path)
df = DataFrame(data=data, index=sim_to_time[sim_label], columns=["value"])
df["year"] = df.index.map(lambda d: d.year)
df = df.ix[df.year.isin(years), :]
df = df.groupby(lambda d: datetime(2001, d.month, 15)).mean()
# print np.mean( monthly_model ), s.river_name, sim_label
df.plot(color=color, lw=3, label=sim_label, ax=ax, y="value")
ds.close()
if row < nrows - 1:
ax.set_xticklabels([])
axes[0].legend(fontsize=17, loc=2)
plt.tight_layout()
plt.savefig("mh/offline_validation_mh.png", dpi=400)
plt.close(fig)
with Dataset(infocell_path) as ds:
fldir = ds.variables["flow_direction_value"][:]
faa = ds.variables["accumulation_area"][:]
lon, lat = [ds.variables[k][:] for k in ["lon", "lat"]]
# plot station positions and upstream areas
cell_manager = CellManager(fldir, nx=fldir.shape[0], ny=fldir.shape[1],
lons2d=lon, lats2d=lat, accumulation_area_km2=faa)
fig = plt.figure()
from crcm5.mh_domains import default_domains
gc = default_domains.bc_mh_011
# get the basemap object
bmp, data_mask = gc.get_basemap_using_shape_with_polygons_of_interest(
lon, lat, shp_path=default_domains.MH_BASINS_PATH, mask_margin=5)
xx, yy = bmp(lon, lat)
ax = plt.gca()
colors = ["g", "r", "m", "c", "y", "violet"]
i = 0
for s, mp in stations_to_mp.items():
assert isinstance(mp, ModelPoint)
upstream_mask = cell_manager.get_mask_of_upstream_cells_connected_with_by_indices(mp.ix, mp.jy)
current_points = upstream_mask > 0.5
bmp.drawcoastlines()
bmp.drawrivers()
bmp.scatter(xx[current_points], yy[current_points], c=colors[i % len(colors)])
i += 1
va = "top"
if s.id in ["05AK001", "05LM006"]:
va = "bottom"
ha = "left"
if s.id in ["05QB003"]:
ha = "right"
bmp.scatter(xx[mp.ix, mp.jy], yy[mp.ix, mp.jy], c="b")
ax.annotate(s.id, xy=(xx[mp.ix, mp.jy], yy[mp.ix, mp.jy]), horizontalalignment=ha,
verticalalignment=va, bbox=dict(boxstyle='round', fc='gray', alpha=0.5))
fig.savefig("mh/offline_stations_{}.png".format("positions"))
plt.close(fig)
def main():
stations = cehq_station.load_from_hydat_db(natural=True, province="SK")
for s in stations:
assert isinstance(s, Station)
print("{}; {}; {}; {}; ".format(s.name, s.longitude, s.latitude, s.drainage_km2))
def main():
model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Diagnostics")
# model_data_path = Path("/RECH2/huziy/BC-MH/bc_mh_044deg/Samples")
static_data_file = "/RECH2/huziy/BC-MH/bc_mh_044deg/Samples/bc_mh_044deg_198001/pm1980010100_00000000p"
r = RPN(static_data_file)
fldir = r.get_first_record_for_name("FLDR")
faa = r.get_first_record_for_name("FAA")
lons, lats = r.get_longitudes_and_latitudes_for_the_last_read_rec()
gc = default_domains.bc_mh_044
cell_manager = CellManager(fldir, nx=fldir.shape[0], ny=fldir.shape[1],
lons2d=lons, lats2d=lats, accumulation_area_km2=faa)
selected_station_ids = ["06EA002", ]
stations = cehq_station.load_from_hydat_db(province="SK", selected_ids=selected_station_ids, natural=None)
# (06EA002): CHURCHILL RIVER AT SANDY BAY at (-102.31832885742188,55.52333068847656), accum. area is 212000.0 km**2
# TODO: plot where is this station, compare modelled and observed hydrographs
# for s in stations:
# assert isinstance(s, cehq_station.Station)
# s.latitude += 0.9
# s.longitude -= 0.2
# print(s)
station_to_model_point = cell_manager.get_model_points_for_stations(stations, drainaige_area_reldiff_limit=0.8,
nneighbours=1)
print(station_to_model_point[stations[0]])
station = stations[0]
assert isinstance(station, cehq_station.Station)
obs_not_corrected = pd.Series(index=station.dates, data=station.values).groupby(
by=lambda d: d.replace(day=15)).mean()
obs_corrected = pd.read_csv("mh/obs_data/Churchill Historic Monthly Apportionable Flow_06EA002.csv.bak.original", skiprows=2)
print(obs_corrected.head())
print(obs_corrected.year.iloc[0], obs_corrected.year.iloc[-1])
date_index = pd.date_range(start=datetime(obs_corrected.year.iloc[0] - 1, 12, 15),
end=datetime(obs_corrected.year.iloc[-1], 12, 15),
freq="M")
date_index = date_index.shift(15, freq=pd.datetools.day)
print(date_index)
data = np.concatenate([r for r in obs_corrected.values[:, 1:-1]])
factor = date_index.map(lambda d: 1000 / (calendar.monthrange(d.year, d.month)[1] * 24 * 3600))
print(factor[:10])
obs_corrected = pd.Series(index=date_index, data=data * factor)
station_to_modelled_data = get_model_data(station_to_model_point, output_path=model_data_path,
grid_config=gc, basins_of_interest_shp=default_domains.MH_BASINS_PATH,
cell_manager=cell_manager, vname="STFL")
modelled_data = station_to_modelled_data[station]
fig = plt.figure()
ax = obs_corrected.plot(label="obs corrected")
obs_not_corrected.plot(label="obs not corrected", ax=ax, color="k")
modelled_data.plot(label="CRCM5", ax=ax, color="r")
ax.legend(loc="upper left")
img_file = img_folder.joinpath("{}_validation_monthly.png".format(station.id))
fig.savefig(str(img_file))
plt.close(fig)
# climatology
start_year = 1980
end_year = 2010
date_selector = lambda d: (start_year <= d.year <= end_year) and not ((d.month == 2) and (d.day == 29))
fig = plt.figure()
ax = obs_corrected.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(label="obs corrected")
obs_not_corrected.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(
label="obs not corrected", ax=ax, color="k")
modelled_data.select(date_selector).groupby(lambda d: d.replace(year=2001)).mean().plot(label="CRCM5", ax=ax,
color="r")
ax.xaxis.set_major_locator(MonthLocator(bymonthday=15))
ax.xaxis.set_major_formatter(DateFormatter("%b"))
ax.legend(loc="upper left")
img_file = img_folder.joinpath("{}_validation_clim.png".format(station.id))
fig.savefig(str(img_file))
plt.close(fig)
# Interannual variability
fig = plt.figure()
obs_corrected = obs_corrected.select(lambda d: start_year <= d.year <= end_year)
modelled_data = modelled_data.select(lambda d: start_year <= d.year <= end_year)
corr_list = []
for m in range(1, 13):
obs = obs_corrected.select(lambda d: d.month == m)
mod = modelled_data.select(lambda d: d.month == m)
print(obs.head())
obs.index = obs.index.map(lambda d: d.year)
mod.index = mod.index.map(lambda d: d.year)
corr_list.append(obs.corr(mod))
ax = plt.gca()
ax.plot(range(1, 13), corr_list)
ax.set_xlabel("Month")
ax.set_title("Inter-annual variability")
img_file = img_folder.joinpath("{}_interannual.png".format(station.id))
fig.tight_layout()
fig.savefig(str(img_file), bbox_inches="tight")
plt.close(fig)