def _start_container(self):
if CFG["container_engine"] == "docker":
self.bmi = BmiClientDocker(
image=self.docker_image,
image_port=55555,
work_dir=str(self.work_dir),
timeout=300,
)
elif CFG["container_engine"] == "singularity":
self.bmi = BmiClientSingularity(
image=self._singularity_image(CFG["singularity_dir"]),
work_dir=str(self.work_dir),
timeout=300,
)
else:
raise ValueError(
f"Unknown container technology: {CFG['container_engine']}")
def _start_container(self):
additional_input_dirs = [str(self.parameter_set.directory)]
if self.forcing:
additional_input_dirs.append(self.forcing.directory)
if CFG["container_engine"] == "docker":
self.bmi = BmiClientDocker(
image=self.docker_image,
image_port=55555,
work_dir=str(self.work_dir),
input_dirs=additional_input_dirs,
timeout=300,
)
elif CFG["container_engine"] == "singularity":
self.bmi = BmiClientSingularity(
image=self._singularity_image(CFG["singularity_dir"]),
work_dir=str(self.work_dir),
input_dirs=additional_input_dirs,
timeout=300,
)
else:
raise ValueError(
f"Unknown container technology in CFG: {CFG['container_engine']}"
)
class Wflow(AbstractModel[WflowForcing]):
"""Create an instance of the Wflow model class.
Args:
version: pick a version from :py:attr:`~available_versions`
parameter_set: instance of
:py:class:`~ewatercycle.parameter_sets.default.ParameterSet`.
forcing: instance of :py:class:`~WflowForcing` or None.
If None, it is assumed that forcing is included with the parameter_set.
"""
available_versions = ("2020.1.1", "2020.1.2", "2020.1.3")
"""Show supported WFlow versions in eWaterCycle"""
def __init__( # noqa: D107
self,
version: str,
parameter_set: ParameterSet,
forcing: Optional[WflowForcing] = None,
):
super().__init__(version, parameter_set, forcing)
self._set_docker_image()
self._setup_default_config()
def _set_docker_image(self):
images = {
# "2019.1": "ewatercycle/wflow-grpc4bmi:2019.1", # no good ini file
"2020.1.1": "ewatercycle/wflow-grpc4bmi:2020.1.1",
"2020.1.2": "ewatercycle/wflow-grpc4bmi:2020.1.2",
"2020.1.3": "ewatercycle/wflow-grpc4bmi:2020.1.3",
}
self.docker_image = images[self.version]
def _singularity_image(self, singularity_dir):
images = {
"2020.1.1": "ewatercycle-wflow-grpc4bmi_2020.1.1.sif",
"2020.1.2": "ewatercycle-wflow-grpc4bmi_2020.1.2.sif",
"2020.1.3": "ewatercycle-wflow-grpc4bmi_2020.1.3.sif",
}
image = singularity_dir / images[self.version]
return str(image)
def _setup_default_config(self):
config_file = self.parameter_set.config
forcing = self.forcing
cfg = CaseConfigParser()
cfg.read(config_file)
if forcing:
cfg.set("framework", "netcdfinput", Path(forcing.netcdfinput).name)
cfg.set("inputmapstacks", "Precipitation", forcing.Precipitation)
cfg.set(
"inputmapstacks",
"EvapoTranspiration",
forcing.EvapoTranspiration,
)
cfg.set("inputmapstacks", "Temperature", forcing.Temperature)
cfg.set("run", "starttime", _iso_to_wflow(forcing.start_time))
cfg.set("run", "endtime", _iso_to_wflow(forcing.end_time))
if self.version in self.available_versions:
if not cfg.has_section("API"):
logger.warning(
"Config file from parameter set is missing API section, "
"adding section")
cfg.add_section("API")
if not cfg.has_option("API", "RiverRunoff"):
logger.warning(
"Config file from parameter set is missing RiverRunoff "
"option in API section, added it with value '2, m/s option'"
)
cfg.set("API", "RiverRunoff", "2, m/s")
self.config = cfg
def setup(self,
cfg_dir: str = None,
**kwargs) -> Tuple[str, str]: # type: ignore
"""Start the model inside a container and return a valid config file.
Args:
cfg_dir: a run directory given by user or created for user.
**kwargs (optional, dict): see :py:attr:`~parameters` for all
configurable model parameters.
Returns:
Path to config file and working directory
"""
self._setup_working_directory(cfg_dir)
cfg = self.config
if "start_time" in kwargs:
cfg.set("run", "starttime", _iso_to_wflow(kwargs["start_time"]))
if "end_time" in kwargs:
cfg.set("run", "endtime", _iso_to_wflow(kwargs["end_time"]))
updated_cfg_file = to_absolute_path("wflow_ewatercycle.ini",
parent=self.work_dir)
with updated_cfg_file.open("w") as filename:
cfg.write(filename)
try:
self._start_container()
except FutureTimeoutError as exc:
# https://github.com/eWaterCycle/grpc4bmi/issues/95
# https://github.com/eWaterCycle/grpc4bmi/issues/100
raise ValueError(
"Couldn't spawn container within allocated time limit "
"(300 seconds). You may try pulling the docker image with"
f" `docker pull {self.docker_image}` or call `singularity "
f"build {self._singularity_image(CFG['singularity_dir'])} "
f"docker://{self.docker_image}` if you're using singularity,"
" and then try again.") from exc
return (
str(updated_cfg_file),
str(self.work_dir),
)
def _setup_working_directory(self, cfg_dir: str = None):
if cfg_dir:
self.work_dir = to_absolute_path(cfg_dir)
else:
timestamp = datetime.datetime.now(
datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
self.work_dir = to_absolute_path(f"wflow_{timestamp}",
parent=CFG["output_dir"])
# Make sure parents exist
self.work_dir.parent.mkdir(parents=True, exist_ok=True)
assert self.parameter_set
shutil.copytree(src=self.parameter_set.directory, dst=self.work_dir)
if self.forcing:
forcing_path = to_absolute_path(self.forcing.netcdfinput,
parent=self.forcing.directory)
shutil.copy(src=forcing_path, dst=self.work_dir)
def _start_container(self):
if CFG["container_engine"] == "docker":
self.bmi = BmiClientDocker(
image=self.docker_image,
image_port=55555,
work_dir=str(self.work_dir),
timeout=300,
)
elif CFG["container_engine"] == "singularity":
self.bmi = BmiClientSingularity(
image=self._singularity_image(CFG["singularity_dir"]),
work_dir=str(self.work_dir),
timeout=300,
)
else:
raise ValueError(
f"Unknown container technology: {CFG['container_engine']}")
def _coords_to_indices(self, name: str, lat: Iterable[float],
lon: Iterable[float]) -> Iterable[int]:
"""Convert lat/lon values to index.
Args:
lat: Latitudinal value
lon: Longitudinal value
"""
grid_id = self.bmi.get_var_grid(name)
shape = self.bmi.get_grid_shape(grid_id) # (len(x), len(y))
grid_lat = self.bmi.get_grid_x(grid_id) # x is latitude
grid_lon = self.bmi.get_grid_y(grid_id) # y is longitude
indices = []
for point_lon, point_lat in zip(lon, lat):
idx_lon, idx_lat = find_closest_point(grid_lon, grid_lat,
point_lon, point_lat)
idx_flat = cast(int, np.ravel_multi_index((idx_lat, idx_lon),
shape))
indices.append(idx_flat)
logger.debug(
f"Requested point was lon: {point_lon}, lat: {point_lat}; "
"closest grid point is "
f"{grid_lon[idx_lon]:.2f}, {grid_lat[idx_lat]:.2f}.")
return indices
def get_value_as_xarray(self, name: str) -> xr.DataArray:
"""Return the value as xarray object."""
# Get time information
time_units = self.bmi.get_time_units()
grid = self.bmi.get_var_grid(name)
shape = self.bmi.get_grid_shape(grid)
# Extract the data and store it in an xarray DataArray
da = xr.DataArray(
data=np.reshape(self.bmi.get_value(name), shape),
coords={
"longitude": self.bmi.get_grid_y(grid),
"latitude": self.bmi.get_grid_x(grid),
"time": num2date(self.bmi.get_current_time(), time_units),
},
dims=["latitude", "longitude"],
name=name,
attrs={"units": self.bmi.get_var_units(name)},
)
return da.where(da != -999)
@property
def parameters(self) -> Iterable[Tuple[str, Any]]:
"""List the configurable parameters for this model."""
# An opiniated list of configurable parameters.
cfg = self.config
return [
("start_time", _wflow_to_iso(cfg.get("run", "starttime"))),
("end_time", _wflow_to_iso(cfg.get("run", "endtime"))),
]
def walrus_model_with_extra_volume(walrus_input_on_extra_volume):
(input_dir, docker_extra_volumes) = walrus_input_on_extra_volume
extra_volumes = {str(k): str(v['bind']) for k, v in docker_extra_volumes.items()}
model = BmiClientSingularity(image=IMAGE_NAME, input_dir=str(input_dir), extra_volumes=extra_volumes)
yield model
del model
def walrus_model(tmp_path, walrus_input):
model = BmiClientSingularity(image=IMAGE_NAME, input_dir=str(tmp_path))
yield model
del model
def setup( # type: ignore
self,
maximum_soil_moisture_storage: float = None,
initial_soil_moisture_storage: float = None,
start_time: str = None,
end_time: str = None,
solver: Solver = None,
cfg_dir: str = None,
) -> Tuple[str, str]:
"""Configure model run.
1. Creates config file and config directory based on the forcing
variables and time range
2. Start bmi container and store as :py:attr:`bmi`
Args:
maximum_soil_moisture_storage: in mm. Range is specfied in `model
parameter range file
<https://github.com/wknoben/MARRMoT/blob/master/MARRMoT/Models/Parameter%20range%20files/m_01_collie1_1p_1s_parameter_ranges.m>`_.
initial_soil_moisture_storage: in mm.
start_time: Start time of model in UTC and ISO format string e.g.
'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is
used.
end_time: End time of model in UTC and ISO format string e.g.
'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used.
solver: Solver settings
cfg_dir: a run directory given by user or created for user.
Returns:
Path to config file and path to config directory
"""
if maximum_soil_moisture_storage:
self._parameters = [maximum_soil_moisture_storage]
if initial_soil_moisture_storage:
self.store_ini = [initial_soil_moisture_storage]
if solver:
self.solver = solver
cfg_dir_as_path = None
if cfg_dir:
cfg_dir_as_path = to_absolute_path(cfg_dir)
cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path)
config_file = self._create_marrmot_config(cfg_dir_as_path, start_time,
end_time)
if CFG["container_engine"].lower() == "singularity":
message = f"The singularity image {self.singularity_image} does not exist."
assert self.singularity_image.exists(), message
self.bmi = BmiClientSingularity(
image=str(self.singularity_image),
work_dir=str(cfg_dir_as_path),
timeout=300,
)
elif CFG["container_engine"].lower() == "docker":
self.bmi = BmiClientDocker(
image=self.docker_image,
image_port=55555,
work_dir=str(cfg_dir_as_path),
timeout=300,
)
else:
raise ValueError(
f"Unknown container technology in CFG: {CFG['container_engine']}"
)
return str(config_file), str(cfg_dir_as_path)
def setup( # type: ignore
self,
maximum_soil_moisture_storage: float = None,
threshold_flow_generation_evap_change: float = None,
leakage_saturated_zone_flow_coefficient: float = None,
zero_deficit_base_flow_speed: float = None,
baseflow_coefficient: float = None,
gamma_distribution_chi_parameter: float = None,
gamma_distribution_phi_parameter: float = None,
initial_upper_zone_storage: float = None,
initial_saturated_zone_storage: float = None,
start_time: str = None,
end_time: str = None,
solver: Solver = None,
cfg_dir: str = None,
) -> Tuple[str, str]:
"""Configure model run.
1. Creates config file and config directory based on the forcing
variables and time range
2. Start bmi container and store as :py:attr:`bmi`
Args:
maximum_soil_moisture_storage: in mm. Range is specfied in `model
parameter range file
<https://github.com/wknoben/MARRMoT/blob/master/MARRMoT/Models/Parameter%20range%20files/m_01_collie1_1p_1s_parameter_ranges.m>`_.
threshold_flow_generation_evap_change.
leakage_saturated_zone_flow_coefficient: in mm/d.
zero_deficit_base_flow_speed: in mm/d.
baseflow_coefficient: in mm-1.
gamma_distribution_chi_parameter.
gamma_distribution_phi_parameter.
initial_upper_zone_storage: in mm.
initial_saturated_zone_storage: in mm.
start_time: Start time of model in UTC and ISO format string e.g.
'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing start time is
used.
end_time: End time of model in UTC and ISO format string e.g.
'YYYY-MM-DDTHH:MM:SSZ'. If not given then forcing end time is used.
solver: Solver settings
cfg_dir: a run directory given by user or created for user.
Returns:
Path to config file and path to config directory
"""
arguments = vars()
arguments_subset = {key: arguments[key] for key in M14_PARAMS}
for index, key in enumerate(M14_PARAMS):
if arguments_subset[key] is not None:
self._parameters[index] = arguments_subset[key]
if initial_upper_zone_storage:
self.store_ini[0] = initial_upper_zone_storage
if initial_saturated_zone_storage:
self.store_ini[1] = initial_saturated_zone_storage
if solver:
self.solver = solver
cfg_dir_as_path = None
if cfg_dir:
cfg_dir_as_path = to_absolute_path(cfg_dir)
cfg_dir_as_path = _generate_cfg_dir(cfg_dir_as_path)
config_file = self._create_marrmot_config(cfg_dir_as_path, start_time,
end_time)
if CFG["container_engine"].lower() == "singularity":
message = f"The singularity image {self.singularity_image} does not exist."
assert self.singularity_image.exists(), message
self.bmi = BmiClientSingularity(
image=str(self.singularity_image),
work_dir=str(cfg_dir_as_path),
timeout=300,
)
elif CFG["container_engine"].lower() == "docker":
self.bmi = BmiClientDocker(
image=self.docker_image,
image_port=55555,
work_dir=str(cfg_dir_as_path),
timeout=300,
)
else:
raise ValueError(
f"Unknown container technology in CFG: {CFG['container_engine']}"
)
return str(config_file), str(cfg_dir_as_path)
class PCRGlobWB(AbstractModel[PCRGlobWBForcing]):
"""eWaterCycle implementation of PCRGlobWB hydrological model.
Args:
version: pick a version from :py:attr:`~available_versions`
parameter_set: instance of
:py:class:`~ewatercycle.parameter_sets.default.ParameterSet`.
forcing: ewatercycle forcing container;
see :py:mod:`ewatercycle.forcing`.
"""
available_versions = ("setters", )
def __init__( # noqa: D107
self,
version: str,
parameter_set: ParameterSet,
forcing: Optional[PCRGlobWBForcing] = None,
):
super().__init__(version, parameter_set, forcing)
self._set_docker_image()
self._setup_default_config()
def _set_docker_image(self):
images = {
"setters": "ewatercycle/pcrg-grpc4bmi:setters",
}
self.docker_image = images[self.version]
def _singularity_image(self, singularity_dir):
images = {
"setters": "ewatercycle-pcrg-grpc4bmi_setters.sif",
}
image = singularity_dir / images[self.version]
return str(image)
def _setup_work_dir(self, cfg_dir: str = None):
if cfg_dir:
self.work_dir = to_absolute_path(cfg_dir)
else:
# Must exist before setting up default config
timestamp = datetime.datetime.now(
datetime.timezone.utc).strftime("%Y%m%d_%H%M%S")
self.work_dir = to_absolute_path(f"pcrglobwb_{timestamp}",
parent=CFG["output_dir"])
self.work_dir.mkdir(parents=True, exist_ok=True)
def _setup_default_config(self):
config_file = self.parameter_set.config
input_dir = self.parameter_set.directory
cfg = CaseConfigParser()
cfg.read(config_file)
cfg.set("globalOptions", "inputDir", str(input_dir))
if self.forcing:
cfg.set(
"globalOptions",
"startTime",
get_time(self.forcing.start_time).strftime("%Y-%m-%d"),
)
cfg.set(
"globalOptions",
"endTime",
get_time(self.forcing.start_time).strftime("%Y-%m-%d"),
)
cfg.set(
"meteoOptions",
"temperatureNC",
str(
to_absolute_path(
self.forcing.temperatureNC,
parent=self.forcing.directory,
)),
)
cfg.set(
"meteoOptions",
"precipitationNC",
str(
to_absolute_path(
self.forcing.precipitationNC,
parent=self.forcing.directory,
)),
)
self.config = cfg
def setup(self,
cfg_dir: str = None,
**kwargs) -> Tuple[str, str]: # type: ignore
"""Start model inside container and return config file and work dir.
Args:
cfg_dir: a run directory given by user or created for user.
**kwargs: Use :py:meth:`parameters` to see the current values
configurable options for this model,
Returns: Path to config file and work dir
"""
self._setup_work_dir(cfg_dir)
self._update_config(**kwargs)
cfg_file = self._export_config()
work_dir = self.work_dir
try:
self._start_container()
except FutureTimeoutError as exc:
# https://github.com/eWaterCycle/grpc4bmi/issues/95
# https://github.com/eWaterCycle/grpc4bmi/issues/100
raise ValueError(
"Couldn't spawn container within allocated time limit "
"(300 seconds). You may try pulling the docker image with"
f" `docker pull {self.docker_image}` or call `singularity "
f"build {self._singularity_image(CFG['singularity_dir'])} "
f"docker://{self.docker_image}` if you're using singularity,"
" and then try again.") from exc
return str(cfg_file), str(work_dir)
def _update_config(self, **kwargs):
cfg = self.config
if "start_time" in kwargs:
cfg.set(
"globalOptions",
"startTime",
get_time(kwargs["start_time"]).strftime("%Y-%m-%d"),
)
if "end_time" in kwargs:
cfg.set(
"globalOptions",
"endTime",
get_time(kwargs["end_time"]).strftime("%Y-%m-%d"),
)
if "routing_method" in kwargs:
cfg.set("routingOptions", "routingMethod",
kwargs["routing_method"])
if "dynamic_flood_plain" in kwargs:
cfg.set(
"routingOptions",
"dynamicFloodPlain",
kwargs["dynamic_flood_plain"],
)
if "max_spinups_in_years" in kwargs:
cfg.set(
"globalOptions",
"maxSpinUpsInYears",
str(kwargs["max_spinups_in_years"]),
)
def _export_config(self) -> PathLike:
self.config.set("globalOptions", "outputDir", str(self.work_dir))
new_cfg_file = to_absolute_path("pcrglobwb_ewatercycle.ini",
parent=self.work_dir)
with new_cfg_file.open("w") as filename:
self.config.write(filename)
self.cfg_file = new_cfg_file
return self.cfg_file
def _start_container(self):
additional_input_dirs = [str(self.parameter_set.directory)]
if self.forcing:
additional_input_dirs.append(self.forcing.directory)
if CFG["container_engine"] == "docker":
self.bmi = BmiClientDocker(
image=self.docker_image,
image_port=55555,
work_dir=str(self.work_dir),
input_dirs=additional_input_dirs,
timeout=300,
)
elif CFG["container_engine"] == "singularity":
self.bmi = BmiClientSingularity(
image=self._singularity_image(CFG["singularity_dir"]),
work_dir=str(self.work_dir),
input_dirs=additional_input_dirs,
timeout=300,
)
else:
raise ValueError(
f"Unknown container technology in CFG: {CFG['container_engine']}"
)
def _coords_to_indices(self, name: str, lat: Iterable[float],
lon: Iterable[float]) -> Iterable[int]:
"""Convert lat/lon values to index.
Args:
lat: Latitudinal value
lon: Longitudinal value
"""
grid_id = self.bmi.get_var_grid(name)
shape = self.bmi.get_grid_shape(grid_id) # (len(x), len(y))
grid_lat = self.bmi.get_grid_x(grid_id) # x is latitude
grid_lon = self.bmi.get_grid_y(grid_id) # y is longitude
indices = []
for point_lon, point_lat in zip(lon, lat):
idx_lon, idx_lat = find_closest_point(grid_lon, grid_lat,
point_lon, point_lat)
idx_flat = cast(int, np.ravel_multi_index((idx_lat, idx_lon),
shape))
indices.append(idx_flat)
logger.debug(
f"Requested point was lon: {point_lon}, lat: {point_lat}; "
"closest grid point is "
f"{grid_lon[idx_lon]:.2f}, {grid_lat[idx_lat]:.2f}.")
return indices
def get_value_as_xarray(self, name: str) -> xr.DataArray:
"""Return the value as xarray object."""
# Get time information
time_units = self.bmi.get_time_units()
grid = self.bmi.get_var_grid(name)
shape = self.bmi.get_grid_shape(grid)
# Extract the data and store it in an xarray DataArray
da = xr.DataArray(
data=np.reshape(self.bmi.get_value(name), shape),
coords={
"longitude": self.bmi.get_grid_y(grid),
"latitude": self.bmi.get_grid_x(grid),
"time": num2date(self.bmi.get_current_time(), time_units),
},
dims=["latitude", "longitude"],
name=name,
attrs={"units": self.bmi.get_var_units(name)},
)
return da.where(da != -999)
@property
def parameters(self) -> Iterable[Tuple[str, Any]]:
"""List the configurable parameters for this model."""
# An opiniated list of configurable parameters.
cfg = self.config
return [
(
"start_time",
f"{cfg.get('globalOptions', 'startTime')}T00:00:00Z",
),
("end_time", f"{cfg.get('globalOptions', 'endTime')}T00:00:00Z"),
("routing_method", cfg.get("routingOptions", "routingMethod")),
(
"max_spinups_in_years",
cfg.get("globalOptions", "maxSpinUpsInYears"),
),
]
def setup( # type: ignore
self,
IrrigationEfficiency: str = None, # noqa: N803
start_time: str = None,
end_time: str = None,
MaskMap: str = None,
cfg_dir: str = None,
) -> Tuple[str, str]:
"""Configure model run.
1. Creates config file and config directory
based on the forcing variables and time range.
2. Start bmi container and store as :py:attr:`bmi`
Args:
IrrigationEfficiency: Field application irrigation efficiency.
max 1, ~0.90 drip irrigation, ~0.75 sprinkling
start_time: Start time of model in UTC and ISO format string
e.g. 'YYYY-MM-DDTHH:MM:SSZ'.
If not given then forcing start time is used.
end_time: End time of model in UTC and ISO format string
e.g. 'YYYY-MM-DDTHH:MM:SSZ'.
If not given then forcing end time is used.
MaskMap: Mask map to use instead of one supplied in parameter set.
Path to a NetCDF or pcraster file with
same dimensions as parameter set map files and a boolean variable.
cfg_dir: a run directory given by user or created for user.
Returns:
Path to config file and path to config directory
"""
# TODO forcing can be a part of parameter_set
cfg_dir_as_path = None
if cfg_dir:
cfg_dir_as_path = to_absolute_path(cfg_dir)
cfg_dir_as_path = _generate_workdir(cfg_dir_as_path)
config_file = self._create_lisflood_config(
cfg_dir_as_path,
start_time,
end_time,
IrrigationEfficiency,
MaskMap,
)
assert self.parameter_set is not None
input_dirs = [str(self.parameter_set.directory), str(self.forcing_dir)]
if MaskMap is not None:
mask_map = to_absolute_path(MaskMap)
try:
mask_map.relative_to(self.parameter_set.directory)
except ValueError:
# If not relative add dir
input_dirs.append(str(mask_map.parent))
if CFG["container_engine"].lower() == "singularity":
image = get_singularity_image(self.version, CFG["singularity_dir"])
self.bmi = BmiClientSingularity(
image=str(image),
input_dirs=input_dirs,
work_dir=str(cfg_dir_as_path),
timeout=300,
)
elif CFG["container_engine"].lower() == "docker":
image = get_docker_image(self.version)
self.bmi = BmiClientDocker(
image=image,
image_port=55555,
input_dirs=input_dirs,
work_dir=str(cfg_dir_as_path),
timeout=300,
)
else:
raise ValueError(
f"Unknown container technology in CFG: {CFG['container_engine']}"
)
return str(config_file), str(cfg_dir_as_path)
def walrus_model(tmp_path, walrus_input):
model = BmiClientSingularity(
image="docker://ewatercycle/walrus-grpc4bmi:v0.2.0",
input_dir=str(tmp_path))
yield model
del model