def _check_forcing(self, forcing):
"""Check forcing argument and get path, start and end time of forcing data."""
if isinstance(forcing, MarrmotForcing):
forcing_dir = to_absolute_path(forcing.directory)
self.forcing_file = str(forcing_dir / forcing.forcing_file)
# convert date_strings to datetime objects
self.forcing_start_time = get_time(forcing.start_time)
self.forcing_end_time = get_time(forcing.end_time)
else:
raise TypeError(
f"Unknown forcing type: {forcing}. Please supply a "
" MarrmotForcing object.")
# parse start/end time
forcing_data = sio.loadmat(self.forcing_file, mat_dtype=True)
if "parameters" in forcing_data:
self._parameters = forcing_data["parameters"][0]
if "store_ini" in forcing_data:
self.store_ini = forcing_data["store_ini"][0]
if "solver" in forcing_data:
forcing_solver = forcing_data["solver"]
self.solver.name = forcing_solver["name"][0][0][0]
self.solver.resnorm_tolerance = forcing_solver[
"resnorm_tolerance"][0][0][0]
self.solver.resnorm_maxiter = forcing_solver["resnorm_maxiter"][0][
0][0]
def _update_config(self, **kwargs):
cfg = self.config
if "start_time" in kwargs:
cfg.set(
"globalOptions",
"startTime",
get_time(kwargs["start_time"]).strftime("%Y-%m-%d"),
)
if "end_time" in kwargs:
cfg.set(
"globalOptions",
"endTime",
get_time(kwargs["end_time"]).strftime("%Y-%m-%d"),
)
if "routing_method" in kwargs:
cfg.set("routingOptions", "routingMethod",
kwargs["routing_method"])
if "dynamic_flood_plain" in kwargs:
cfg.set(
"routingOptions",
"dynamicFloodPlain",
kwargs["dynamic_flood_plain"],
)
if "max_spinups_in_years" in kwargs:
cfg.set(
"globalOptions",
"maxSpinUpsInYears",
str(kwargs["max_spinups_in_years"]),
)
def _check_forcing(self, forcing):
"""Check forcing argument and get path, start/end time of forcing data."""
# TODO check if mask has same grid as forcing files,
# if not warn users to run reindex_forcings
if isinstance(forcing, LisfloodForcing):
self.forcing = forcing
self.forcing_dir = to_absolute_path(forcing.directory)
# convert date_strings to datetime objects
self._start = get_time(forcing.start_time)
self._end = get_time(forcing.end_time)
else:
raise TypeError(f"Unknown forcing type: {forcing}. "
"Please supply a LisfloodForcing object.")
def _setup_default_config(self):
config_file = self.parameter_set.config
input_dir = self.parameter_set.directory
cfg = CaseConfigParser()
cfg.read(config_file)
cfg.set("globalOptions", "inputDir", str(input_dir))
if self.forcing:
cfg.set(
"globalOptions",
"startTime",
get_time(self.forcing.start_time).strftime("%Y-%m-%d"),
)
cfg.set(
"globalOptions",
"endTime",
get_time(self.forcing.start_time).strftime("%Y-%m-%d"),
)
cfg.set(
"meteoOptions",
"temperatureNC",
str(
to_absolute_path(
self.forcing.temperatureNC,
parent=self.forcing.directory,
)),
)
cfg.set(
"meteoOptions",
"precipitationNC",
str(
to_absolute_path(
self.forcing.precipitationNC,
parent=self.forcing.directory,
)),
)
self.config = cfg
def _check_forcing(self, forcing):
"""Check forcing argument and get path, start and end time of forcing data."""
if isinstance(forcing, MarrmotForcing):
forcing_dir = to_absolute_path(forcing.directory)
self.forcing_file = str(forcing_dir / forcing.forcing_file)
# convert date_strings to datetime objects
self.forcing_start_time = get_time(forcing.start_time)
self.forcing_end_time = get_time(forcing.end_time)
else:
raise TypeError(f"Unknown forcing type: {forcing}. "
"Please supply a MarrmotForcing object.")
# parse start/end time
forcing_data = sio.loadmat(self.forcing_file, mat_dtype=True)
if "parameters" in forcing_data:
if len(forcing_data["parameters"]) == len(self._parameters):
self._parameters = forcing_data["parameters"]
else:
message = ("The length of parameters in forcing "
f"{self.forcing_file} does not match "
"the length of M14 parameters that is seven.")
logger.warning("%s", message)
if "store_ini" in forcing_data:
if len(forcing_data["store_ini"]) == len(self.store_ini):
self.store_ini = forcing_data["store_ini"]
else:
message = ("The length of initial stores in forcing "
f"{self.forcing_file} does not match "
"the length of M14 iniatial stores that is two.")
logger.warning("%s", message)
if "solver" in forcing_data:
forcing_solver = forcing_data["solver"]
self.solver.name = forcing_solver["name"][0][0][0]
self.solver.resnorm_tolerance = forcing_solver[
"resnorm_tolerance"][0][0][0]
self.solver.resnorm_maxiter = forcing_solver["resnorm_maxiter"][0][
0][0]
def _iso_to_wflow(time):
dt = get_time(time)
return dt.strftime("%Y-%m-%d %H:%M:%S")
def test_get_time_without_tz():
with pytest.raises(ValueError) as excinfo:
get_time("1989-01-02T00:00:00")
assert "not in UTC" in str(excinfo.value)
def test_get_time_with_utc():
dt = get_time("1989-01-02T00:00:00Z")
assert dt == datetime(1989, 1, 2, tzinfo=timezone.utc)
def _create_marrmot_config(self,
cfg_dir: Path,
start_time_iso: str = None,
end_time_iso: str = None) -> Path:
"""Write model configuration file.
Adds the model parameters to forcing file for the given period
and writes this information to a model configuration file.
Args:
cfg_dir: a run directory given by user or created for user.
start_time_iso: Start time of model in UTC and ISO format string
e.g. 'YYYY-MM-DDTHH:MM:SSZ'.
If not given then forcing start time is used.
end_time_iso: End time of model in UTC and ISO format string
e.g. 'YYYY-MM-DDTHH:MM:SSZ'.
If not given then forcing end time is used.
Returns:
Path for Marrmot config file
"""
forcing_data = sio.loadmat(self.forcing_file, mat_dtype=True)
# overwrite dates if given
if start_time_iso is not None:
start_time = get_time(start_time_iso)
if self.forcing_start_time <= start_time <= self.forcing_end_time:
forcing_data["time_start"][0][0:6] = [
start_time.year,
start_time.month,
start_time.day,
start_time.hour,
start_time.minute,
start_time.second,
]
self.forcing_start_time = start_time
else:
raise ValueError("start_time outside forcing time range")
if end_time_iso is not None:
end_time = get_time(end_time_iso)
if self.forcing_start_time <= end_time <= self.forcing_end_time:
forcing_data["time_end"][0][0:6] = [
end_time.year,
end_time.month,
end_time.day,
end_time.hour,
end_time.minute,
end_time.second,
]
self.forcing_end_time = end_time
else:
raise ValueError("end_time outside forcing time range")
# combine forcing and model parameters
forcing_data.update(
model_name=self.model_name,
parameters=self._parameters,
solver=asdict(self.solver),
store_ini=self.store_ini,
)
config_file = cfg_dir / "marrmot-m14_config.mat"
sio.savemat(config_file, forcing_data)
return config_file
def _create_lisflood_config(
self,
cfg_dir: Path,
start_time_iso: str = None,
end_time_iso: str = None,
IrrigationEfficiency: str = None, # noqa: N803
MaskMap: str = None,
) -> Path:
"""Create lisflood config file."""
assert self.parameter_set is not None
assert self.forcing is not None
# overwrite dates if given
if start_time_iso is not None:
start_time = get_time(start_time_iso)
if self._start <= start_time <= self._end:
self._start = start_time
else:
raise ValueError("start_time outside forcing time range")
if end_time_iso is not None:
end_time = get_time(end_time_iso)
if self._start <= end_time <= self._end:
self._end = end_time
else:
raise ValueError("end_time outside forcing time range")
settings = {
"CalendarDayStart": self._start.strftime("%d/%m/%Y 00:00"),
"StepStart": "1",
"StepEnd": str((self._end - self._start).days),
"PathRoot": str(self.parameter_set.directory),
"PathMeteo": str(self.forcing_dir),
"PathOut": str(cfg_dir),
}
if IrrigationEfficiency is not None:
settings["IrrigationEfficiency"] = IrrigationEfficiency
if MaskMap is not None:
mask_map = to_absolute_path(MaskMap)
settings["MaskMap"] = str(mask_map.with_suffix(""))
for textvar in self.cfg.config.iter("textvar"):
textvar_name = textvar.attrib["name"]
# general settings
for key, value in settings.items():
if key in textvar_name:
textvar.set("value", value)
# input for lisflood
if "PrefixPrecipitation" in textvar_name:
textvar.set("value",
Path(self.forcing.PrefixPrecipitation).stem)
if "PrefixTavg" in textvar_name:
textvar.set("value", Path(self.forcing.PrefixTavg).stem)
# maps_prefixes dictionary contains lisvap filenames in lisflood config
maps_prefixes = {
"E0Maps": {
"name": "PrefixE0",
"value": Path(self.forcing.PrefixE0).stem,
},
"ES0Maps": {
"name": "PrefixES0",
"value": Path(self.forcing.PrefixES0).stem,
},
"ET0Maps": {
"name": "PrefixET0",
"value": Path(self.forcing.PrefixET0).stem,
},
}
# output of lisvap
for map_var, prefix in maps_prefixes.items():
if prefix["name"] in textvar_name:
textvar.set("value", prefix["value"])
if map_var in textvar_name:
textvar.set("value", f"$(PathMeteo)/$({prefix['name']})")
# Write to new setting file
lisflood_file = cfg_dir / "lisflood_setting.xml"
self.cfg.save(str(lisflood_file))
return lisflood_file
def get_grdc_data(
station_id: str,
start_time: str,
end_time: str,
parameter: str = "Q",
data_home: str = None,
column: str = "streamflow",
) -> Tuple[pd.core.frame.DataFrame, MetaDataType]:
"""Get river discharge data from Global Runoff Data Centre (GRDC).
Requires the GRDC daily data files in a local directory. The GRDC daily data
files can be ordered at
https://www.bafg.de/GRDC/EN/02_srvcs/21_tmsrs/riverdischarge_node.html
Args:
station_id: The station id to get. The station id can be found in the
catalogues at
https://www.bafg.de/GRDC/EN/02_srvcs/21_tmsrs/212_prjctlgs/project_catalogue_node.html
start_time: Start time of model in UTC and ISO format string e.g.
'YYYY-MM-DDTHH:MM:SSZ'.
end_time: End time of model in UTC and ISO format string e.g.
'YYYY-MM-DDTHH:MM:SSZ'.
parameter: optional. The parameter code to get, e.g. ('Q') discharge,
cubic meters per second.
data_home : optional. The directory where the daily grdc data is
located. If left out will use the grdc_location in the eWaterCycle
configuration file.
column: optional. Name of column in dataframe. Default: "streamflow".
Returns:
grdc data in a dataframe and metadata.
Examples:
.. code-block:: python
from ewatercycle.observation.grdc import get_grdc_data
df, meta = get_grdc_data('6335020',
'2000-01-01T00:00Z',
'2001-01-01T00:00Z')
df.describe()
streamflow
count 4382.000000
mean 2328.992469
std 1190.181058
min 881.000000
25% 1550.000000
50% 2000.000000
75% 2730.000000
max 11300.000000
meta
{'grdc_file_name': '/home/myusername/git/eWaterCycle/ewatercycle/6335020_Q_Day.Cmd.txt',
'id_from_grdc': 6335020,
'file_generation_date': '2019-03-27',
'river_name': 'RHINE RIVER',
'station_name': 'REES',
'country_code': 'DE',
'grdc_latitude_in_arc_degree': 51.756918,
'grdc_longitude_in_arc_degree': 6.395395,
'grdc_catchment_area_in_km2': 159300.0,
'altitude_masl': 8.0,
'dataSetContent': 'MEAN DAILY DISCHARGE (Q)',
'units': 'm³/s',
'time_series': '1814-11 - 2016-12',
'no_of_years': 203,
'last_update': '2018-05-24',
'nrMeasurements': 'NA',
'UserStartTime': '2000-01-01T00:00Z',
'UserEndTime': '2001-01-01T00:00Z',
'nrMissingData': 0}
""" # noqa: E501
if data_home:
data_path = to_absolute_path(data_home)
elif CFG["grdc_location"]:
data_path = to_absolute_path(CFG["grdc_location"])
else:
raise ValueError(
"Provide the grdc path using `data_home` argument"
"or using `grdc_location` in ewatercycle configuration file.")
if not data_path.exists():
raise ValueError(f"The grdc directory {data_path} does not exist!")
# Read the raw data
raw_file = data_path / f"{station_id}_{parameter}_Day.Cmd.txt"
if not raw_file.exists():
raise ValueError(f"The grdc file {raw_file} does not exist!")
# Convert the raw data to an xarray
metadata, df = _grdc_read(
raw_file,
start=get_time(start_time).date(),
end=get_time(end_time).date(),
column=column,
)
# Add start/end_time to metadata
metadata["UserStartTime"] = start_time
metadata["UserEndTime"] = end_time
# Add number of missing data to metadata
metadata["nrMissingData"] = _count_missing_data(df, column)
# Shpw info about data
_log_metadata(metadata)
return df, metadata
