def test_mon_U(self, mon_series, series, mon_triangular, add_dims, kind,
name, use_dask):
"""
Train on
hist: U
ref: U + monthly cycle
Predict on hist to get ref
"""
u = np.random.rand(10000)
# Define distributions
xd = uniform(loc=1, scale=1)
yd = uniform(loc=2, scale=2)
noise = uniform(loc=0, scale=1e-7)
# Generate random numbers
x = xd.ppf(u)
y = yd.ppf(u) + noise.ppf(u)
# Test train
hist = sim = series(x, name)
if use_dask:
sim = sim.chunk({"time": -1})
if add_dims:
hist = hist.expand_dims(site=[0, 1, 2, 3, 4])
sim = sim.expand_dims(site=[0, 1, 2, 3, 4])
sel = {"site": 0}
else:
sel = {}
ref = mon_series(y, name)
QDM = QuantileDeltaMapping(kind=kind,
group="time.month",
nquantiles=40)
QDM.train(ref, hist)
p = QDM.adjust(sim, interp="linear" if kind == "+" else "nearest")
q = QDM.ds.coords["quantiles"]
expected = get_correction(xd.ppf(q), yd.ppf(q), kind)
expected = apply_correction(mon_triangular[:, np.newaxis],
expected[np.newaxis, :], kind)
np.testing.assert_array_almost_equal(QDM.ds.af.sel(quantiles=q, **sel),
expected, 1)
# Test predict
np.testing.assert_allclose(p.isel(**sel), ref, rtol=0.1, atol=0.2)
def test_cannon_and_diagnostics(self, cannon_2015_dist, cannon_2015_rvs):
ref, hist, sim = cannon_2015_rvs(15000, random=False)
# Quantile mapping
with set_options(sdba_extra_output=True):
QDM = QuantileDeltaMapping.train(ref,
hist,
kind="*",
group="time",
nquantiles=50)
scends = QDM.adjust(sim)
assert isinstance(scends, xr.Dataset)
sim_q_exp = sim.rank(dim="time", pct=True)
np.testing.assert_array_equal(sim_q_exp, scends.sim_q)
# Theoretical results
# ref, hist, sim = cannon_2015_dist
# u1 = equally_spaced_nodes(1001, None)
# u = np.convolve(u1, [0.5, 0.5], mode="valid")
# pu = ref.ppf(u) * sim.ppf(u) / hist.ppf(u)
# pu1 = ref.ppf(u1) * sim.ppf(u1) / hist.ppf(u1)
# pdf = np.diff(u1) / np.diff(pu1)
# mean = np.trapz(pdf * pu, pu)
# mom2 = np.trapz(pdf * pu ** 2, pu)
# std = np.sqrt(mom2 - mean ** 2)
bc_sim = scends.scen
np.testing.assert_almost_equal(bc_sim.mean(), 41.5, 1)
np.testing.assert_almost_equal(bc_sim.std(), 16.7, 0)
def test_train_qdm(kind):
"""Test that train_qdm outputs store giving sdba.adjustment.QuantileDeltaMapping
Checks that output is consistent if we do "additive" or "multiplicative"
QDM kinds.
"""
# Setup input data.
n_years = 10
n = n_years * 365
model_bias = 2
ts = np.sin(np.linspace(-10 * 3.14, 10 * 3.14, n)) * 0.5
hist = _datafactory(ts + model_bias)
ref = _datafactory(ts)
output_key = "memory://test_train_qdm/test_output.zarr"
hist_key = "memory://test_train_qdm/hist.zarr"
ref_key = "memory://test_train_qdm/ref.zarr"
# Load up a fake repo with our input data in the place of big data and cloud
# storage.
repository.write(hist_key, hist)
repository.write(ref_key, ref)
train_qdm(
historical=hist_key,
reference=ref_key,
out=output_key,
variable="fakevariable",
kind=kind,
)
assert QuantileDeltaMapping.from_dataset(repository.read(output_key))
def test_cannon(self, cannon_2015_dist, cannon_2015_rvs):
ref, hist, sim = cannon_2015_rvs(15000, random=False)
# Quantile mapping
QDM = QuantileDeltaMapping(kind="*", group="time", nquantiles=50)
QDM.train(ref, hist)
bc_sim = QDM.adjust(sim)
# Theoretical results
# ref, hist, sim = cannon_2015_dist
# u1 = equally_spaced_nodes(1001, None)
# u = np.convolve(u1, [0.5, 0.5], mode="valid")
# pu = ref.ppf(u) * sim.ppf(u) / hist.ppf(u)
# pu1 = ref.ppf(u1) * sim.ppf(u1) / hist.ppf(u1)
# pdf = np.diff(u1) / np.diff(pu1)
# mean = np.trapz(pdf * pu, pu)
# mom2 = np.trapz(pdf * pu ** 2, pu)
# std = np.sqrt(mom2 - mean ** 2)
np.testing.assert_almost_equal(bc_sim.mean(), 41.5, 1)
np.testing.assert_almost_equal(bc_sim.std(), 16.7, 0)
def test_mon_U(self, mon_series, series, mon_triangular, kind, name):
"""
Train on
hist: U
ref: U + monthly cycle
Predict on hist to get ref
"""
u = np.random.rand(10000)
# Define distributions
xd = uniform(loc=1, scale=1)
yd = uniform(loc=2, scale=2)
noise = uniform(loc=0, scale=1e-7)
# Generate random numbers
x = xd.ppf(u)
y = yd.ppf(u) + noise.ppf(u)
# Test train
hist = sim = series(x, name)
ref = mon_series(y, name)
QDM = QuantileDeltaMapping(kind=kind,
group="time.month",
nquantiles=40)
QDM.train(ref, hist)
p = QDM.adjust(sim)
q = QDM.ds.coords["quantiles"]
expected = get_correction(xd.ppf(q), yd.ppf(q), kind)
expected = apply_correction(mon_triangular[:, np.newaxis],
expected[np.newaxis, :], kind)
np.testing.assert_array_almost_equal(QDM.ds.af.sel(quantiles=q),
expected, 1)
# Test predict
np.testing.assert_array_almost_equal(p, ref, 1)
def test_quantiles(self, series, kind, name):
"""Train on
x : U(1,1)
y : U(1,2)
"""
u = np.random.rand(10000)
# Define distributions
xd = uniform(loc=1, scale=1)
yd = uniform(loc=2, scale=4)
# Generate random numbers with u so we get exact results for comparison
x = xd.ppf(u)
y = yd.ppf(u)
# Test train
hist = sim = series(x, name)
ref = series(y, name)
QDM = QuantileDeltaMapping(
kind=kind,
group="time",
nquantiles=10,
)
QDM.train(ref, hist)
p = QDM.adjust(sim, interp="linear")
q = QDM.ds.coords["quantiles"]
expected = get_correction(xd.ppf(q), yd.ppf(q), kind)
# Results are not so good at the endpoints
np.testing.assert_array_almost_equal(QDM.ds.af.T, expected, 1)
# Test predict
# Accept discrepancies near extremes
middle = (u > 1e-2) * (u < 0.99)
np.testing.assert_array_almost_equal(p[middle], ref[middle], 1)
def adapt_freq_graph():
"""
Create a graphic with the additive adjustment factors estimated after applying the adapt_freq method.
"""
n = 10000
x = tu.series(synth_rainfall(2, 2, wet_freq=0.25, size=n), "pr") # sim
y = tu.series(synth_rainfall(2, 2, wet_freq=0.5, size=n), "pr") # ref
xp = adapt_freq(x, y, thresh=0).sim_ad
fig, (ax1, ax2) = plt.subplots(2, 1)
sx = x.sortby(x)
sy = y.sortby(y)
sxp = xp.sortby(xp)
# Original and corrected series
ax1.plot(sx.values, color="blue", lw=1.5, label="x : sim")
ax1.plot(sxp.values, color="pink", label="xp : sim corrected")
ax1.plot(sy.values, color="k", label="y : ref")
ax1.legend()
# Compute qm factors
qm_add = QuantileDeltaMapping(kind="+", group="time").train(y, x).ds
qm_mul = QuantileDeltaMapping(kind="*", group="time").train(y, x).ds
qm_add_p = QuantileDeltaMapping(kind="+", group="time").train(y, xp).ds
qm_mul_p = QuantileDeltaMapping(kind="*", group="time").train(y, xp).ds
qm_add.cf.plot(ax=ax2, color="cyan", ls="--", label="+: y-x")
qm_add_p.cf.plot(ax=ax2, color="cyan", label="+: y-xp")
qm_mul.cf.plot(ax=ax2, color="brown", ls="--", label="*: y/x")
qm_mul_p.cf.plot(ax=ax2, color="brown", label="*: y/xp")
ax2.legend(loc="upper left", frameon=False)
return fig
def test_train_qdm_isel_slice():
"""Test that train_qdm outputs subset data when passed isel_slice"""
# Setup input data.
n_years = 10
n = n_years * 365
# Lazy way to make fake data for 2 latitudes...
model_bias = 2
ts = np.sin(np.linspace(-10 * 3.14, 10 * 3.14, n)) * 0.5
hist1 = _datafactory(ts + model_bias)
hist2 = _datafactory(ts + model_bias).assign_coords(
{"lat": hist1["lat"].data + 1.0}
)
hist = xr.concat([hist1, hist2], dim="lat")
ref1 = _datafactory(ts)
ref2 = _datafactory(ts + model_bias).assign_coords({"lat": ref1["lat"].data + 1.0})
ref = xr.concat([ref1, ref2], dim="lat")
output_key = "memory://test_train_qdm_isel_slice/test_output.zarr"
hist_key = "memory://test_train_qdm_isel_slice/hist.zarr"
ref_key = "memory://test_train_qdm_isel_slice/ref.zarr"
repository.write(hist_key, hist)
repository.write(ref_key, ref)
train_qdm(
historical=hist_key,
reference=ref_key,
out=output_key,
variable="fakevariable",
kind="additive",
isel_slice={"lat": slice(0, 1)}, # select only 1 of 2 lats by idx...
)
# Check we can read output and it's the selected value, only.
ds_result = repository.read(output_key)
np.testing.assert_equal(ds_result["lat"].data, ref["lat"].data[0])
assert QuantileDeltaMapping.from_dataset(ds_result)
def cannon_2015_figure_2():
n = 10000
ref, hist, sim = tu.cannon_2015_rvs(n, random=False)
QM = EmpiricalQuantileMapping(kind="*", group="time", interp="linear")
QM.train(ref, hist)
sim_eqm = QM.predict(sim)
DQM = DetrendedQuantileMapping(kind="*", group="time", interp="linear")
DQM.train(ref, hist)
sim_dqm = DQM.predict(sim, degree=0)
QDM = QuantileDeltaMapping(kind="*", group="time", interp="linear")
QDM.train(ref, hist)
sim_qdm = QDM.predict(sim)
fig, (ax1, ax2) = plt.subplots(1, 2, figsize=(11, 4))
x = np.linspace(0, 105, 50)
ax1.plot(x, gaussian_kde(ref)(x), color="r", label="Obs hist")
ax1.plot(x, gaussian_kde(hist)(x), color="k", label="GCM hist")
ax1.plot(x, gaussian_kde(sim)(x), color="blue", label="GCM simure")
ax1.plot(x, gaussian_kde(sim_qdm)(x), color="lime", label="QDM future")
ax1.plot(x,
gaussian_kde(sim_eqm)(x),
color="darkgreen",
ls="--",
label="QM future")
ax1.plot(x,
gaussian_kde(sim_dqm)(x),
color="lime",
ls=":",
label="DQM future")
ax1.legend(frameon=False)
ax1.set_xlabel("Value")
ax1.set_ylabel("Density")
tau = np.array([0.25, 0.5, 0.75, 0.95, 0.99]) * 100
bc_gcm = (scoreatpercentile(sim, tau) -
scoreatpercentile(hist, tau)) / scoreatpercentile(hist, tau)
bc_qdm = (scoreatpercentile(sim_qdm, tau) -
scoreatpercentile(ref, tau)) / scoreatpercentile(ref, tau)
bc_eqm = (scoreatpercentile(sim_eqm, tau) -
scoreatpercentile(ref, tau)) / scoreatpercentile(ref, tau)
bc_dqm = (scoreatpercentile(sim_dqm, tau) -
scoreatpercentile(ref, tau)) / scoreatpercentile(ref, tau)
ax2.plot([0, 1], [0, 1], ls=":", color="blue")
ax2.plot(bc_gcm, bc_gcm, "-", color="blue", label="GCM")
ax2.plot(bc_gcm, bc_qdm, marker="o", mfc="lime", label="QDM")
ax2.plot(
bc_gcm,
bc_eqm,
marker="o",
mfc="darkgreen",
ls=":",
color="darkgreen",
label="QM",
)
ax2.plot(
bc_gcm,
bc_dqm,
marker="s",
mec="lime",
mfc="w",
ls="--",
color="lime",
label="DQM",
)
for i, s in enumerate(tau / 100):
ax2.text(bc_gcm[i],
bc_eqm[i],
f"{s} ",
ha="right",
va="center",
fontsize=9)
ax2.set_xlabel("GCM relative change")
ax2.set_ylabel("Bias adjusted relative change")
ax2.legend(loc="upper left", frameon=False)
ax2.set_aspect("equal")
plt.tight_layout()
return fig
