def test_evaluation(self):
model = GR4JCN()
model.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
model(
TS,
area=4250.6,
elevation=843.0,
latitude=54.4848,
longitude=-123.3659,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
suppress_output=False,
evaluation_metrics=["RMSE", "KLING_GUPTA"],
evaluation_periods=[
EvaluationPeriod("period1", "2000-01-01", "2000-12-31"),
EvaluationPeriod("period2", "2001-01-01", "2001-12-31"),
],
)
d = model.diagnostics
assert "DIAG_RMSE" in d
assert "DIAG_KLING_GUPTA" in d
assert len(d["DIAG_RMSE"]) == 3 # ALL, period1, period2
def test_update_soil_water(self):
params = (0.529, -3.396, 407.29, 1.072, 16.9, 0.947)
# Reference run
model = GR4JCN()
model.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
model(
TS,
run_name="run_a",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2000, 2, 1),
params=params,
)
s_0 = float(model.storage["Soil Water[0]"].isel(time=-1).values)
s_1 = float(model.storage["Soil Water[1]"].isel(time=-1).values)
# hru_state = replace(model.rvc.hru_state, soil0=s_0, soil1=s_1)
model.config.rvc.hru_states[1] = replace(
model.config.rvc.hru_states[1], soil0=s_0, soil1=s_1)
model(
TS,
run_name="run_b",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2000, 2, 1),
# hru_state=hru_state,
params=params,
)
assert s_0 != model.storage["Soil Water[0]"].isel(time=-1)
assert s_1 != model.storage["Soil Water[1]"].isel(time=-1)
def test_hindcasting_GEPS(self, tmpdir):
# Prepare a RAVEN model run using historical data, GR4JCN in this case.
# This is a dummy run to get initial states. In a real forecast situation,
# this run would end on the day before the forecast, but process is the same.
ts = get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
)
model = GR4JCN(workdir=tmpdir)
model(
ts,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 6, 1),
area=44250.6,
elevation=843.0,
latitude=54.4848,
longitude=-123.3659,
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
)
# Extract the final states that will be used as the next initial states
rvc = model.outputs["solution"]
ts20 = get_local_testdata("caspar_eccc_hindcasts/geps_watershed.nc")
nm = 20
# It is necessary to clean the model state because the input variables of the previous
# model are not the same as the ones provided in the forecast model. therefore, if we
# do not clean, the model will simply add the hindcast file to the list of available
# data provided in the testdata above. Then the dates will not work, and the model errors.
model = GR4JCN()
model.rvc.parse(rvc.read_text())
# And run the model with the forecast data.
model(
ts=ts20,
nc_index=range(nm),
start_date=dt.datetime(2018, 6, 1),
end_date=dt.datetime(2018, 6, 10),
area=44250.6,
elevation=843.0,
latitude=54.4848,
longitude=-123.3659,
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
overwrite=True,
pr={
"linear_transform": (1000.0, 0.0),
"time_shift": -0.25,
"deaccumulate": True,
},
tas={"time_shift": -0.25},
)
# The model now has the forecast data generated and it has 10 days of forecasts.
assert len(model.q_sim.values) == 10
# Also see if GEPS has 20 members produced.
assert model.q_sim.values.shape[1] == nm
def test_race():
model1 = GR4JCN()
model1.config.rvi.suppress_output = True
model2 = GR4JCN()
ost = GR4JCN_OST()
assert model1.config.rvi.suppress_output.startswith(":SuppressOutput")
assert model2.config.rvi.suppress_output == ""
assert ost.config.rvi.suppress_output.startswith(":SuppressOutput")
def test_forecasting_GEPS(self):
# Prepare a RAVEN model run using historical data, GR4JCN in this case.
# This is a dummy run to get initial states. In a real forecast situation,
# this run would end on the day before the forecast, but process is the same.
ts = get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
)
model = GR4JCN()
model(
ts,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 6, 1),
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
hrus=(hru,),
)
# Extract the final states that will be used as the next initial states
rvc = model.outputs["solution"]
# Collect test forecast data for location and climate model (20 members)
ts20 = get_local_testdata("eccc_forecasts/geps_watershed.nc")
nm = 20
# It is necessary to clean the model state because the input variables of the previous
# model are not the same as the ones provided in the forecast model. therefore, if we
# do not clean, the model will simply add the hindcast file to the list of available
# data provided in the testdata above. Then the dates will not work, and the model errors.
model = GR4JCN()
model.config.rvc.parse_solution(rvc.read_text())
model(
ts=(ts20,),
duration=9,
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
hrus=(hru,),
overwrite=True,
pr={"time_shift": -0.25, "deaccumulate": True},
tas={"time_shift": -0.25},
parallel=dict(nc_index=range(nm)),
)
# The model now has the forecast data generated and it has 10 days of forecasts.
assert len(model.q_sim.values) == 10
# Also see if GEPS has 20 members produced.
assert model.q_sim.values.shape[1] == nm
# Check all members are different (checking snow because data in winter)
assert len(set(model.storage.Snow.isel(time=-1).values)) == nm
def test_overwrite(self):
model = GR4JCN()
model.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
model(
TS,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
)
assert model.config.rvi.suppress_output == ""
qsim1 = model.q_sim.copy(deep=True)
m1 = qsim1.mean()
# This is only needed temporarily while we fix this: https://github.com/CSHS-CWRA/RavenPy/issues/4
# Please remove when fixed!
model.hydrograph.close() # Needed with xarray 0.16.1
model(TS,
params=(0.5289, -3.397, 407.3, 1.071, 16.89, 0.948),
overwrite=True)
qsim2 = model.q_sim.copy(deep=True)
m2 = qsim2.mean()
# This is only needed temporarily while we fix this: https://github.com/CSHS-CWRA/RavenPy/issues/4
# Please remove when fixed!
model.hydrograph.close() # Needed with xarray 0.16.1
assert m1 != m2
np.testing.assert_almost_equal(m1, m2, 1)
d = model.diagnostics
np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], -0.117315, 4)
model.config.rvc.hru_states[1] = HRUStateVariableTableCommand.Record(
soil0=0)
# Set initial conditions explicitly
model(
TS,
end_date=dt.datetime(2001, 2, 1),
# hru_state=HRUStateVariableTableCommand.Record(soil0=0),
overwrite=True,
)
assert model.q_sim.isel(time=1).values[0] < qsim2.isel(
time=1).values[0]
def test_run_new_hrus_param(self):
model = GR4JCN()
model(
TS,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
suppress_output=False,
hrus=(GR4JCN.LandHRU(**salmon_land_hru_1), ),
)
d = model.diagnostics
np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], -0.117301, 4)
def test_dap(self):
"""Test Raven with DAP link instead of local netCDF file."""
model = GR4JCN()
config = dict(
start_date=dt.datetime(2000, 6, 1),
end_date=dt.datetime(2000, 6, 10),
run_name="test",
hrus=(GR4JCN.LandHRU(**salmon_land_hru_1), ),
params=model.Params(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
)
ts = (
f"{TDS}/raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
)
model(ts, **config)
def test_update_soil_water(self):
kwargs = dict(
area=4250.6,
elevation=843.0,
latitude=54.4848,
longitude=-123.3659,
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
)
# Reference run
model = GR4JCN()
model(
TS,
run_name="run_a",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2000, 2, 1),
**kwargs,
)
s_0 = float(model.storage["Soil Water[0]"].isel(time=-1).values)
s_1 = float(model.storage["Soil Water[1]"].isel(time=-1).values)
hru_state = replace(model.rvc.hru_state, soil0=s_0, soil1=s_1)
model(
TS,
run_name="run_b",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2000, 2, 1),
hru_state=hru_state,
**kwargs,
)
assert s_0 != model.storage["Soil Water[0]"].isel(time=-1)
assert s_1 != model.storage["Soil Water[1]"].isel(time=-1)
def test_config_update(self):
model = GR4JCN()
# This is a regular attribute member
model.config.update("run_name", "test")
assert model.config.rvi.run_name == "test"
# This is a computed property
model.config.update("evaporation", "PET_FROMMONTHLY")
assert model.config.rvi.evaporation == "PET_FROMMONTHLY"
# Existing property but wrong value (the enum cast should throw an error)
with pytest.raises(ValueError):
model.config.update("routing", "WRONG")
# Non-existing attribute
with pytest.raises(AttributeError):
model.config.update("why", "not?")
# Params
model.config.update(
"params", np.array([0.529, -3.396, 407.29, 1.072, 16.9, 0.947]))
assert model.config.rvp.params.GR4J_X1 == 0.529
model.config.update("params",
[0.529, -3.396, 407.29, 1.072, 16.9, 0.947])
assert model.config.rvp.params.GR4J_X1 == 0.529
model.config.update("params",
(0.529, -3.396, 407.29, 1.072, 16.9, 0.947))
assert model.config.rvp.params.GR4J_X1 == 0.529
def test_simple(self):
model = GR4JCN(tempfile.mkdtemp())
model.rvi.start_date = dt.datetime(2000, 1, 1)
model.rvi.end_date = dt.datetime(2002, 1, 1)
model.rvi.run_name = "test"
model.rvh.name = "Salmon"
model.rvh.area = "4250.6"
model.rvh.elevation = "843.0"
model.rvh.latitude = 54.4848
model.rvh.longitude = -123.3659
model.rvt.pr.deaccumulate = False
model.rvp.params = model.params(0.529, -3.396, 407.29, 1.072, 16.9,
0.947)
assert model.rvi.suppress_output == ""
model(TS)
d = model.diagnostics
# yields NSE=0.???? for full period 1954-2010
assert model.rvi.calendar == "GREGORIAN"
# Check parser
assert 1 in model.solution["HRUStateVariableTable"]["data"]
np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], -0.117301, 2)
hds = model.q_sim
assert hds.attrs["long_name"] == "Simulated outflows"
# Check attributes
assert model.hydrograph.attrs["model_id"] == "gr4jcn"
def test_run(self):
model = GR4JCN()
model.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
model(
TS,
area=4250.6,
elevation=843.0,
latitude=54.4848,
longitude=-123.3659,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
suppress_output=False,
)
d = model.diagnostics
np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], -0.117301, 4)
def test_error(self):
model = GR4JCN()
model.config.rvp.params = model.Params(0.529, -3.396, 407.29, 1.072,
16.9, 0.947)
with pytest.raises(RavenError) as exc:
model(TS)
assert "CHydroUnit constructor:: HRU 1 has a negative or zero area" in str(
exc.value)
def test_parallel_basins(self, input2d):
ts = input2d
model = GR4JCN()
model.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
model(
ts,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
params=[0.529, -3.396, 407.29, 1.072, 16.9, 0.947],
# name=["basin1", "basin2"], # Not sure about this..
suppress_output=False,
parallel={"nc_index": [0, 0]},
)
assert len(model.diagnostics) == 2
assert len(model.hydrograph.nbasins) == 2
np.testing.assert_array_equal(model.hydrograph.basin_name[:],
["sub_001", "sub_001"])
z = zipfile.ZipFile(model.outputs["rv_config"])
assert len(z.filelist) == 10
def test_parallel_params(self):
model = GR4JCN()
model.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
model(
TS,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
suppress_output=False,
parallel={
"params": [
(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
(0.528, -3.4, 407.3, 1.07, 17, 0.95),
]
},
)
assert len(model.diagnostics) == 2
assert model.hydrograph.dims["params"] == 2
z = zipfile.ZipFile(model.outputs["rv_config"])
assert len(z.filelist) == 10
def test_resume(self):
model_ab = GR4JCN()
model_ab.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
kwargs = dict(params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947), )
# Reference run
model_ab(
TS,
run_name="run_ab",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2001, 1, 1),
**kwargs,
)
model_a = GR4JCN()
model_a.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
model_a(
TS,
run_name="run_a",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2000, 7, 1),
**kwargs,
)
# Path to solution file from run A
rvc = model_a.outputs["solution"]
# Resume with final state from live model
model_a.resume()
model_a(
TS,
run_name="run_2",
start_date=dt.datetime(2000, 7, 1),
end_date=dt.datetime(2001, 1, 1),
**kwargs,
)
for key in ["Soil Water[0]", "Soil Water[1]"]:
np.testing.assert_array_almost_equal(
model_a.storage[key] - model_ab.storage[key], 0, 5)
# Resume with final state from saved solution file
model_b = GR4JCN()
model_b.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
model_b.resume(
rvc
) # <--------- And this is how you feed it to a brand new model.
model_b(
TS,
run_name="run_2",
start_date=dt.datetime(2000, 7, 1),
end_date=dt.datetime(2001, 1, 1),
**kwargs,
)
for key in ["Soil Water[0]", "Soil Water[1]"]:
np.testing.assert_array_almost_equal(
model_b.storage[key] - model_ab.storage[key], 0, 5)
def test_resume_earlier(self):
"""Check that we can resume a run with the start date set at another date than the time stamp in the
solution."""
params = (0.529, -3.396, 407.29, 1.072, 16.9, 0.947)
# Reference run
model = GR4JCN()
model.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
model(
TS,
run_name="run_a",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2000, 2, 1),
params=params,
)
s_a = model.storage["Soil Water[0]"].isel(time=-1)
# Path to solution file from run A
rvc = model.outputs["solution"]
# Resume with final state from live model
# We have two options to do this:
# 1. Replace model template by solution file as is: model.resume()
# 2. Replace variable in RVC class by parsed values: model.rvc.parse(rvc.read_text())
# I think in many cases option 2 will prove simpler.
model.config.rvc.parse_solution(rvc.read_text())
model(
TS,
run_name="run_b",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2000, 2, 1),
params=params,
)
s_b = model.storage["Soil Water[0]"].isel(time=-1)
assert s_a != s_b
def test_assign(self):
model = GR4JCN()
model.assign("run_name", "test")
assert model.rvi.run_name == "test"
model.assign("params",
np.array([0.529, -3.396, 407.29, 1.072, 16.9, 0.947]))
assert model.rvp.params.GR4J_X1 == 0.529
model.assign("params", [0.529, -3.396, 407.29, 1.072, 16.9, 0.947])
assert model.rvp.params.GR4J_X1 == 0.529
model.assign("params", (0.529, -3.396, 407.29, 1.072, 16.9, 0.947))
assert model.rvp.params.GR4J_X1 == 0.529
def test_resume_earlier(self):
"""Check that we can resume a run with the start date set at another date than the time stamp in the
solution."""
kwargs = dict(
area=4250.6,
elevation=843.0,
latitude=54.4848,
longitude=-123.3659,
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
)
# Reference run
model = GR4JCN()
model(
TS,
run_name="run_a",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2000, 2, 1),
**kwargs,
)
s_a = model.storage["Soil Water[0]"].isel(time=-1)
# Path to solution file from run A
rvc = model.outputs["solution"]
# Resume with final state from live model
# We have two options to do this:
# 1. Replace model template by solution file as is: model.resume()
# 2. Replace variable in RVC class by parsed values: model.rvc.parse(rvc.read_text())
# I think in many cases option 2 will prove simpler.
model.rvc.parse(rvc.read_text())
model(
TS,
run_name="run_b",
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2000, 2, 1),
**kwargs,
)
s_b = model.storage["Soil Water[0]"].isel(time=-1)
assert s_a != s_b
def test_parallel_params(self):
model = GR4JCN()
model(
TS,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
area=4250.6,
elevation=843.0,
latitude=54.4848,
longitude=-123.3659,
params=[
(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
(0.528, -3.4, 407.3, 1.07, 17, 0.95),
],
suppress_output=False,
)
assert len(model.diagnostics) == 2
assert model.hydrograph.dims["params"] == 2
z = zipfile.ZipFile(model.outputs["rv_config"])
assert len(z.filelist) == 10
def test_full_example(self):
ts = get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
)
model = "GR4JCN"
nash, params = reg.read_gauged_params(model)
variables = ["latitude", "longitude", "area", "forest"]
props = reg.read_gauged_properties(variables)
ungauged_props = {
"latitude": 40.4848,
"longitude": -103.3659,
"area": 4250.6,
"forest": 0.4,
}
hrus = (GR4JCN.LandHRU(area=4250.6,
elevation=843.0,
latitude=40.4848,
longitude=-103.3659), )
qsim, ens = reg.regionalize(
"SP_IDW",
model,
nash,
params,
props,
ungauged_props,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
hrus=hrus,
longitude=-103.3659,
min_NSE=0.6,
size=2,
ts=ts,
)
assert qsim.max() > 1
assert len(ens) == 2
assert "realization" in ens.dims
assert "param" in ens.dims
def test_parallel_basins(self, input2d):
ts = input2d
model = GR4JCN()
model(
ts,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 1, 1),
area=4250.6,
elevation=843.0,
latitude=54.4848,
longitude=-123.3659,
params=[0.529, -3.396, 407.29, 1.072, 16.9, 0.947],
nc_index=[0, 0],
name=["basin1", "basin2"],
suppress_output=False,
)
assert len(model.diagnostics) == 2
assert len(model.hydrograph.nbasins) == 2
np.testing.assert_array_equal(model.hydrograph.basin_name[:],
["basin1", "basin2"])
z = zipfile.ZipFile(model.outputs["rv_config"])
assert len(z.filelist) == 10
def test_canopex(self):
CANOPEX_DAP = (
"https://pavics.ouranos.ca/twitcher/ows/proxy/thredds/dodsC/birdhouse/ets"
"/Watersheds_5797_cfcompliant.nc")
model = GR4JCN()
config = dict(
start_date=dt.datetime(2010, 6, 1),
end_date=dt.datetime(2010, 6, 10),
nc_index=5600,
run_name="Test_run",
rain_snow_fraction="RAINSNOW_DINGMAN",
tasmax={"offset": -273.15},
tasmin={"offset": -273.15},
pr={"scale": 86400.0},
hrus=[
model.LandHRU(area=3650.47,
latitude=49.51,
longitude=-95.72,
elevation=330.59)
],
params=model.Params(108.02, 2.8693, 25.352, 1.3696, 1.2483,
0.30679),
)
model(ts=CANOPEX_DAP, **config)
import datetime as dt
from ravenpy.models import GR4JCN
from ravenpy.utilities.testdata import get_local_testdata
"""
Test to perform a hindcast using auto-queried ECCC data aggregated on THREDDS.
Currently only runs GEPS, eventually will run GEPS, GDPS, REPS and RDPS.
To do so will need to add the actual data from ECCC but this is a proof of concept.
"""
hru = GR4JCN.LandHRU(
area=44250.6, elevation=843.0, latitude=54.4848, longitude=-123.3659
)
class TestECCCForecast:
def test_forecasting_GEPS(self):
# Prepare a RAVEN model run using historical data, GR4JCN in this case.
# This is a dummy run to get initial states. In a real forecast situation,
# this run would end on the day before the forecast, but process is the same.
ts = get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
)
model = GR4JCN()
model(
ts,
start_date=dt.datetime(2000, 1, 1),
end_date=dt.datetime(2002, 6, 1),
params=(0.529, -3.396, 407.29, 1.072, 16.9, 0.947),
from .wps_raven import RavenProcess
LOGGER = logging.getLogger("PYWPS")
"""
Notes
-----
The configuration files for RAVEN's GR4J-Cemaneige model and in models/raven-gr4j-cemaneige.
All parameters that could potentially be user-defined are tagged using {}. These tags need to be replaced by
actual values before the model is launched.
"""
params_defaults = GR4JCN.Params(
GR4J_X1=0.529,
GR4J_X2=-3.396,
GR4J_X3=407.29,
GR4J_X4=1.072,
CEMANEIGE_X1=16.9,
CEMANEIGE_X2=0.947,
)
params = LiteralInput(
"params",
"Comma separated list of model parameters",
abstract="Parameters: " + ", ".join(f.name
for f in fields(params_defaults)),
data_type="string",
default=", ".join(map(str, astuple(params_defaults))),
min_occurs=0,
max_occurs=config.max_parallel_processes,
)
def test_simple(self):
# get timeseries
ts = get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
)
# set number of members. Using 7 here to make it easier to find and debug.
n_members = 7
# Perturbation parameters for the assimilation, keyed by standard_name
std = {
"rainfall": 0.30,
"prsn": 0.30,
"tasmin": 2.0,
"tasmax": 2.0,
"water_volume_transport_in_river_channel": 0.10,
}
# Perturbation distribution
dists = {
"pr": "gamma",
"rainfall": "gamma",
"prsn": "gamma",
"water_volume_transport_in_river_channel": "rnorm",
}
qkey = "water_volume_transport_in_river_channel"
if qkey not in std:
raise ValueError(
"Assimilation requires perturbing the flow variable.")
# Assimilation variables (from HRUStateVariable)
assim_var = ("soil0", "soil1")
# Assimilation period (days between each assimilation step)
assim_step_days = 3
# GR4JCN model instance
model = GR4JCN()
# set the start and end dates for the first assimilation period, warm-up
start_date = dt.datetime(1996, 9, 1)
end_date = dt.datetime(1996, 9, 30)
# Catchment properties to populate model
area = 4250.6
elevation = 843.0
latitude = 54.4848
longitude = -123.3659
params = (0.1353389, -0.005067198, 576.8007, 6.986121, 1.102917,
0.9224778)
# Do the first assimilation pass to get hru_states and basin_states.
# Can be skipped if there is already this data from a previous run.
model, xa, hru_states, basin_states = assimilation_initialization(
model,
ts,
start_date=start_date,
end_date=start_date + dt.timedelta(days=assim_step_days - 1),
area=area,
elevation=elevation,
latitude=latitude,
longitude=longitude,
params=params,
assim_var=assim_var,
n_members=n_members,
)
# Perturb the inputs for the rest of the assimilation
perturbed = perturb_full_series(
model,
std=std,
start_date=start_date,
end_date=end_date,
dists=dists,
n_members=n_members,
)
# Get observed streamflow for computing results later
q_obs = xr.open_dataset(ts)["qobs"].sel(
time=slice(start_date, end_date))
# Create netcdf for the model.
p_fn = model.workdir / "perturbed_forcing.nc"
perturbed = xr.Dataset(perturbed)
perturbed.to_netcdf(p_fn, mode="w")
# Run the sequential assimilation for the entire period.
q_assim, hru_states, basin_states = sequential_assimilation(
model,
hru_states,
basin_states,
p_fn,
q_obs,
assim_var,
start_date=start_date + dt.timedelta(days=assim_step_days),
end_date=end_date,
n_members=n_members,
assim_step_days=assim_step_days,
)
# ==== Reference run ====
model.config.rvi.run_name = "ref"
model.config.rvi.start_date = start_date
model.config.rvi.end_date = end_date
model.config.rvc.hru_states = {}
model.config.rvc.basin_states = {}
model.config.rvc.soil0 = None
model.config.rvc.soil1 = 15
model([ts])
# We can now plot everything!
plt.plot(q_assim.T, "r",
label="Assimilated") # plot the assimilated flows
plt.plot(q_obs.T, "b", label="Observed") # plot the observed flows
plt.plot(model.q_sim, "g", label="Simulated"
) # plot the open_loop (simulation with no assimilation)
# plt.legend()
# plt.show()
# print('RMSE - Assimilated: ' + str(xss.rmse(q_assim.mean(dim='state').T,q_obs[0:q_assim.shape[1]].T).data))
# print('RMSE - Open-Loop: ' + str(xss.rmse(model.q_sim[0:q_assim.shape[1],0],q_obs[0:q_assim.shape[1]].T).data))
assert q_assim.shape[0] == n_members
def test_simple(self):
# get timeseries
ts = get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily.nc"
)
# set number of members. Using 7 here to make it easier to find and debug.
n_members = 7
# Perturbation parameters for the assimilation, keyed by standard_name
std = {
"rainfall": 0.30,
"prsn": 0.30,
"tasmin": 2.0,
"tasmax": 2.0,
"water_volume_transport_in_river_channel": 0.15,
}
# Use the same random seed for both tasmin and tasmax
rs = np.random.SeedSequence(None).generate_state(1)[0]
seed = {"tasmin": rs, "tasmax": rs}
# Perturbation distribution
dists = {
"pr": "gamma",
"rainfall": "gamma",
"prsn": "gamma",
"water_volume_transport_in_river_channel": "rnorm",
}
qkey = "water_volume_transport_in_river_channel"
if qkey not in std:
raise ValueError("Assimilation requires perturbing the flow variable.")
# Assimilation variables (from HRUStateVariable)
assim_var = ("soil0", "soil1")
# Assimilation periods
assim_days = [10] + 4 * [3]
# GR4JCN model instance
model = GR4JCN()
# set the start and end dates for the first assimilation period, warm-up
start_date = dt.datetime(2000, 6, 1)
end_date = start_date + dt.timedelta(days=sum(assim_days))
# Set model options
model.rvh.name = "Salmon"
model.rvh.area = "4250.6"
model.rvh.elevation = "843.0"
model.rvh.latitude = 54.4848
model.rvh.longitude = -123.3659
model.rvp.params = model.params(
0.1353389, -0.005067198, 576.8007, 6.986121, 1.102917, 0.9224778
) # SALMON
# ==== Initialization (just to get reasonable states) ====
# Set initialization run options
model.rvi.run_name = "init"
model.rvi.start_date = start_date
# Richard: what is the end date policy for the init run ?
model.rvi.end_date = start_date + dt.timedelta(days=assim_days[0])
# Run the model
model([ts])
# Extract final model states
hru_state, basin_state = model.get_final_state()
xa = n_members * [getattr(hru_state, key) for key in assim_var]
hru_states = n_members * [hru_state]
basin_states = n_members * [basin_state]
# === Create perturbed time series for full assimilation period ====
perturbed = {}
for key, s in std.items():
nc = model.rvt.get(key)
with xr.open_dataset(nc.path) as ds:
da = ds.get(nc.var_name).sel(time=slice(start_date, end_date))
perturbed[key] = perturbation(
da,
dists.get(key, "norm"),
std=s,
seed=seed.get(key, None),
member=n_members,
)
# Save flow for later
if key == qkey:
q_obs = da
# Write to disk
p_fn = model.workdir / "perturbed_forcing.nc"
perturbed = xr.Dataset(perturbed)
perturbed.to_netcdf(p_fn, mode="w")
# ==== Assimilation ====
q_assim = []
sd = start_date
for i, ndays in enumerate(assim_days):
dates = [sd + dt.timedelta(days=x) for x in range(ndays)]
model.rvi.end_date = dates[-1]
model.rvi.run_name = f"assim_{i}"
# Perform the first assimilation step here
[xa, model] = assimilate(
model, p_fn, q_obs, assim_var, basin_states, hru_states, dates
)
# Save streamflow simulation
q_assim.append(model.q_sim.isel(nbasins=0))
# Update the start-time for the next loop
sd += dt.timedelta(days=ndays)
model.rvi.start_date = sd
# Get new initial conditions and feed assimilated values
hru_states, basin_states = model.get_final_state()
hru_states = [
replace(hru_states[i], **dict(zip(assim_var, xa[:, i])))
for i in range(n_members)
]
q_assim = xr.concat(q_assim, dim="time")
# ==== Reference run ====
model.rvi.run_name = "ref"
model.rvi.start_date = start_date
model.rvi.end_date = end_date
model.rvc = RVC(soil0=None, soil1=15, basin_state=BasinIndexCommand())
model([ts])
# We can now plot everything!
plt.plot(q_assim.T, "r", label="Assimilated") # plot the assimilated flows
plt.plot(q_obs.T, "b", label="Observed") # plot the observed flows
plt.plot(
model.q_sim, "g", label="Simulated"
) # plot the open_loop (simulation with no assimilation)
# plt.legend()
# plt.show()
assert q_assim.shape[0] == n_members
assert q_assim.shape[1] == sum(assim_days)
def test_simple(self):
model = GR4JCN_OST()
model.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
# Parameter bounds
low = (0.01, -15.0, 10.0, 0.0, 1.0, 0.0)
high = (2.5, 10.0, 700.0, 7.0, 30.0, 1.0)
model.configure(
get_local_testdata("ostrich-gr4j-cemaneige/OstRandomNumbers.txt"))
model(
TS,
start_date=dt.datetime(1954, 1, 1),
duration=208,
lowerBounds=low,
upperBounds=high,
algorithm="DDS",
random_seed=0,
max_iterations=10,
)
d = model.diagnostics
np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], 0.50717, 4)
# Random number seed: 123
# Budget: 10
# Algorithm: DDS
# :StartDate 1954-01-01 00:00:00
# :Duration 208
opt_para = astuple(model.calibrated_params)
opt_func = model.obj_func
np.testing.assert_almost_equal(
opt_para,
[2.424726, 3.758972, 204.3856, 5.866946, 16.60408, 0.3728098],
4,
err_msg="calibrated parameter set is not matching expected value",
)
np.testing.assert_almost_equal(
opt_func,
-0.50717,
4,
err_msg="calibrated NSE is not matching expected value",
)
# # Random number seed: 123
# # Budget: 50
# # Algorithm: DDS
# # :StartDate 1954-01-01 00:00:00
# # :Duration 20819
# np.testing.assert_almost_equal( opt_para, [0.3243268,3.034247,407.2890,2.722774,12.18124,0.9468769], 4,
# err_msg='calibrated parameter set is not matching expected value')
# np.testing.assert_almost_equal( opt_func, -0.5779910, 4,
# err_msg='calibrated NSE is not matching expected value')
gr4j = GR4JCN()
gr4j.config.rvh.hrus = (GR4JCN.LandHRU(**salmon_land_hru_1), )
gr4j(
TS,
start_date=dt.datetime(1954, 1, 1),
duration=208,
params=model.calibrated_params,
)
np.testing.assert_almost_equal(gr4j.diagnostics["DIAG_NASH_SUTCLIFFE"],
d["DIAG_NASH_SUTCLIFFE"])
def test_routing(self):
"""We need at least 2 subbasins to activate routing."""
model = GR4JCN()
ts_2d = get_local_testdata(
"raven-gr4j-cemaneige/Salmon-River-Near-Prince-George_meteo_daily_2d.nc"
)
#########
# R V I #
#########
model.config.rvi.start_date = dt.datetime(2000, 1, 1)
model.config.rvi.end_date = dt.datetime(2002, 1, 1)
model.config.rvi.run_name = "test_gr4jcn_routing"
model.config.rvi.routing = "ROUTE_DIFFUSIVE_WAVE"
#########
# R V H #
#########
# Here we assume that we have two subbasins. The first one (subbasin_id=10)
# has a lake (hru_id=2; area-100km2) and the rest is covered by land (hru_id=1;
# area=4250.6km2). The second subbasin (subbasin_id=20) does not contain a
# lake and is hence only land (hru_id=3; area=2000km2).
#
# Later the routing product will tell us which basin flows into which. Here
# we assume that the first subbasin (subbasin_id=10) drains into the second
# (subbasin_id=20). At the outlet of this second one we have an observation
# station (see :ObservationData in RVT). We will compare these observations
# with the simulated streamflow. That is the reason why "gauged=True" for
# the second basin.
# HRU IDs are 1 to 3
model.config.rvh.hrus = (
GR4JCN.LandHRU(hru_id=1, subbasin_id=10, **salmon_land_hru_1),
GR4JCN.LakeHRU(hru_id=2, subbasin_id=10, **salmon_lake_hru_1),
GR4JCN.LandHRU(hru_id=3, subbasin_id=20, **salmon_land_hru_2),
)
# Sub-basin IDs are 10 and 20 (not 1 and 2), to help disambiguate
model.config.rvh.subbasins = (
# gauged = False:
# Usually this output would only be written for user's convenience.
# There is usually no observation of streamflow available within
# catchments; only at the outlet. That's most commonly the reason
# why a catchment is defined as it is defined.
Sub(
name="upstream",
subbasin_id=10,
downstream_id=20,
profile="chn_10",
gauged=False,
),
# gauged = True:
# Since this is the outlet, this would usually be what we calibrate
# against (i.e. we try to match this to Qobs).
Sub(
name="downstream",
subbasin_id=20,
downstream_id=-1,
profile="chn_20",
gauged=True,
),
)
model.config.rvh.land_subbasin_property_multiplier = (
SBGroupPropertyMultiplierCommand("Land", "MANNINGS_N", 1.0))
model.config.rvh.lake_subbasin_property_multiplier = (
SBGroupPropertyMultiplierCommand("Lakes", "RESERVOIR_CREST_WIDTH",
1.0))
#########
# R V T #
#########
gws = GridWeightsCommand(
number_hrus=3,
number_grid_cells=1,
# Here we have a special case: station is 0 for every row because the example NC
# has only one region/station (which is column 0)
data=((1, 0, 1.0), (2, 0, 1.0), (3, 0, 1.0)),
)
# These will be shared (inline) to all the StationForcing commands in the RVT
model.config.rvt.grid_weights = gws
#########
# R V P #
#########
model.config.rvp.params = model.Params(0.529, -3.396, 407.29, 1.072,
16.9, 0.947)
total_area_in_km2 = sum(hru.area for hru in model.config.rvh.hrus)
total_area_in_m2 = total_area_in_km2 * 1000 * 1000
model.config.rvp.avg_annual_runoff = get_average_annual_runoff(
ts_2d, total_area_in_m2)
np.testing.assert_almost_equal(model.config.rvp.avg_annual_runoff,
139.5407534171111)
# These channel profiles describe the geometry of the actual river crossection.
# The eight points (x) to describe the following geometry are given in each
# profile:
#
# ----x x---
# \ FLOODPLAIN /
# x----x x----x
# \ /
# \ RIVERBED /
# x----------x
#
model.config.rvp.channel_profiles = [
ChannelProfileCommand(
name="chn_10",
bed_slope=7.62066e-05,
survey_points=[
(0, 463.647),
(16.0, 459.647),
(90.9828, 459.647),
(92.9828, 458.647),
(126.4742, 458.647),
(128.4742, 459.647),
(203.457, 459.647),
(219.457, 463.647),
],
roughness_zones=[
(0, 0.0909167),
(90.9828, 0.035),
(128.4742, 0.0909167),
],
),
ChannelProfileCommand(
name="chn_20",
bed_slope=9.95895e-05,
survey_points=[
(0, 450.657),
(16.0, 446.657),
(85.0166, 446.657),
(87.0166, 445.657),
(117.5249, 445.657),
(119.5249, 446.657),
(188.54149999999998, 446.657),
(204.54149999999998, 450.657),
],
roughness_zones=[
(0, 0.0915769),
(85.0166, 0.035),
(119.5249, 0.0915769),
],
),
]
#############
# Run model #
#############
model(ts_2d)
###########
# Verify #
###########
hds = model.q_sim
assert len(hds.nbasins) == 1 # number of "gauged" basins is 1
# We only have one SB with gauged=True, so the output has a single column.
# The number of time steps simulated between (2000, 1, 1) and
# (2002, 1, 1) is 732.
assert hds.shape == (732, 1)
# Check simulated streamflow at first three timesteps and three simulated
# timesteps in the middle of the simulation period.
dates = (
"2000-01-01",
"2000-01-02",
"2000-01-03",
"2001-01-30",
"2001-01-31",
"2001-02-01",
)
target_q_sim = [
0.0, 0.304073, 0.980807, 17.54049, 17.409493, 17.437954
]
for t in range(6):
np.testing.assert_almost_equal(hds.sel(nbasins=0, time=dates[t]),
target_q_sim[t], 4)
# For lumped GR4J model we have 1 subbasin and 1 HRU as well as no routing, no
# channel profiles, and the area of the entire basin is 4250.6 [km2]. Comparison
# of simulated and observed streamflow at outlet yielded:
# np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], -0.116971, 4)
#
# This is now a different value due to:
# - basin we have here is larger (4250.6 [km2] + 100 [km2] + 2000.0 [km2])
# - we do routing: so water from subbasin 1 needs some time to arrive at the
# outlet of subbasin 2
d = model.diagnostics
np.testing.assert_almost_equal(d["DIAG_NASH_SUTCLIFFE"], -0.0141168, 4)
def test_version(self):
model = Raven()
assert model.raven_version == "3.0.4"
model = GR4JCN()
assert model.raven_version == "3.0.4"
