def test_final_output_path_no_staging_dir():
# check final output file in correct location with a staging directory is not used
ds = _open(CMIP5_TAS)
get_output(ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")())
assert os.path.isfile("./output_001.nc")
os.remove("output_001.nc")
def test_SimpleFileNamer_no_fmt():
s = get_file_namer("simple")()
checks = (("my.stuff", None),)
for args in checks:
with pytest.raises(KeyError):
s.get_file_name(*args)
def test_final_output_path_staging_dir():
# check final output file in correct location with a staging directory used
CONFIG["clisops:write"]["output_staging_dir"] = "tests/"
ds = _open(CMIP5_TAS)
get_output(ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")())
assert os.path.isfile("./output_001.nc")
os.remove("output_001.nc")
def subset(
ds,
time=None,
area=None,
level=None,
output_dir=None,
output_type="netcdf",
split_method="time:auto",
file_namer="standard",
):
"""
Example:
ds: Xarray Dataset
time: ("1999-01-01T00:00:00", "2100-12-30T00:00:00")
area: (-5.,49.,10.,65)
level: (1000.,)
output_dir: "/cache/wps/procs/req0111"
output_type: "netcdf"
split_method: "time:auto"
file_namer: "standard"
:param ds:
:param time:
:param area:
:param level:
:param output_dir:
:param output_type:
:param split_method:
:param file_namer:
:return:
"""
# Convert all inputs to Xarray Datasets
if isinstance(ds, str):
ds = xr.open_mfdataset(ds, use_cftime=True, combine="by_coords")
LOGGER.debug(f"Mapping parameters: time: {time}, area: {area}, level: {level}")
args = utils.map_params(ds, time, area, level)
subset_ds = _subset(ds, args)
outputs = []
namer = get_file_namer(file_namer)()
time_slices = get_time_slices(subset_ds, split_method)
for tslice in time_slices:
result_ds = subset_ds.sel(time=slice(tslice[0], tslice[1]))
LOGGER.info(f"Processing subset for times: {tslice}")
output = get_output(result_ds, output_type, output_dir, namer)
outputs.append(output)
return outputs
def test_StandardFileNamer_no_project_match():
s = get_file_namer("standard")()
class Thing(object):
pass
mock_ds = Thing()
mock_ds.attrs = {}
with pytest.raises(KeyError):
s.get_file_name(mock_ds)
def test_SimpleFileNamer():
s = get_file_namer("simple")()
checks = [
(("my.stuff", "netcdf"), "output_001.nc"),
(("other", "netcdf"), "output_002.nc"),
]
for args, expected in checks:
resp = s.get_file_name(*args)
assert resp == expected
def _get_file_namer(self):
if self.params.get("dims", None):
dims = [convert_coord_to_axis(dim) for dim in self.params["dims"]]
extra = f"_avg-{''.join(sorted(dims))}"
else:
extra = ""
namer = get_file_namer(self._file_namer)(extra=extra)
return namer
def test_tmp_dir_deleted():
# check temporary directory under stagin dir gets deleted after data has bee staged
CONFIG["clisops:write"]["output_staging_dir"] = "tests/"
ds = _open(CMIP5_TAS)
get_output(ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")())
# check that no tmpdir directories exist
assert glob("tests/tmp*") == []
os.remove("output_001.nc")
def test_no_staging_dir(caplog):
CONFIG["clisops:write"]["output_staging_dir"] = ""
ds = _open(CMIP5_TAS)
output_path = get_output(
ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")()
)
assert "Writing to temporary path: " not in caplog.text
assert output_path == "output_001.nc"
os.remove("output_001.nc")
def test_invalid_staging_dir(caplog):
# check stagin dir not used with invalid directory
CONFIG["clisops:write"]["output_staging_dir"] = "test/not/real/dir/"
ds = _open(CMIP5_TAS)
output_path = get_output(
ds, output_type="nc", output_dir=".", namer=get_file_namer("simple")()
)
assert "Writing to temporary path: " not in caplog.text
assert output_path == "output_001.nc"
os.remove("output_001.nc")
def test_StandardFileNamer_cmip6(load_esgf_test_data):
s = get_file_namer("standard")()
_ds = xr.open_mfdataset(
CMIP6_SICONC,
use_cftime=True,
combine="by_coords",
)
checks = [(_ds, "siconc_SImon_CanESM5_historical_r1i1p1f1_gn_18500116-20141216.nc")]
for ds, expected in checks:
resp = s.get_file_name(ds)
assert resp == expected
def test_StandardFileNamer_cmip5(load_esgf_test_data):
s = get_file_namer("standard")()
_ds = xr.open_mfdataset(
CMIP5_TAS,
use_cftime=True,
combine="by_coords",
)
checks = [(_ds, "tas_mon_HadGEM2-ES_rcp85_r1i1p1_20051216-22991216.nc")]
for ds, expected in checks:
resp = s.get_file_name(ds)
assert resp == expected
def test_StandardFileNamer_cmip5_use_default_attr_names(load_esgf_test_data):
s = get_file_namer("standard")()
_ds = xr.open_mfdataset(
CMIP5_TAS,
use_cftime=True,
combine="by_coords",
)
checks = [(_ds, "tas_mon_no-model_rcp85_r1i1p1_20051216-22991216.nc")]
del _ds.attrs["model_id"]
for ds, expected in checks:
resp = s.get_file_name(ds)
assert resp == expected
def test_StandardFileNamer_cmip6_use_default_attr_names(load_esgf_test_data):
s = get_file_namer("standard")()
_ds = xr.open_mfdataset(
CMIP6_SICONC,
use_cftime=True,
combine="by_coords",
)
checks = [
(_ds, "siconc_SImon_no-model_historical_r1i1p1f1_no-grid_18500116-20141216.nc")
]
del _ds.attrs["source_id"]
del _ds.attrs["grid_label"]
for ds, expected in checks:
resp = s.get_file_name(ds)
assert resp == expected
def test_StandardFileNamer_c3s_cordex(load_esgf_test_data):
s = get_file_namer("standard")()
_ds = xr.open_mfdataset(
C3S_CORDEX_PSL,
use_cftime=True,
combine="by_coords",
)
checks = [
(
_ds,
"psl_EUR-11_MOHC-HadGEM2-ES_rcp85_r1i1p1_IPSL-WRF381P_v1_day_20060101-20991201.nc",
)
]
for ds, expected in checks:
resp = s.get_file_name(ds)
assert resp == expected
def test_StandardFileNamer_c3s_cordex_use_default_attr_names(load_esgf_test_data):
s = get_file_namer("standard")()
_ds = xr.open_mfdataset(
C3S_CORDEX_PSL,
use_cftime=True,
combine="by_coords",
)
checks = [
(
_ds,
"psl_no-domain_MOHC-HadGEM2-ES_rcp85_rXiXpX_IPSL-WRF381P_v1_day_20060101-20991201.nc",
)
]
del _ds.attrs["CORDEX_domain"]
del _ds.attrs["driving_model_ensemble_member"]
for ds, expected in checks:
resp = s.get_file_name(ds)
assert resp == expected
def _get_file_namer(self):
"""
Return the appropriate file namer object.
"""
namer = get_file_namer(self._file_namer)()
return namer
