def test_delete(tmp_path, key, data, serializer):
store = CacheStore(str(tmp_path))
store.put(key=key, value=data, serializer=serializer)
assert key in store.keys()
store.delete(key=key, dry_run=True)
del store[key]
assert key not in store.keys()
def test_on_duplicate_key(tmp_path, on_duplicate_key):
store = CacheStore(str(tmp_path), on_duplicate_key=on_duplicate_key)
key, data, new_data = 'foo', 'my_data', 'hello'
store.put(key=key, value=data)
if on_duplicate_key == 'raise_error':
with pytest.raises(ValueError):
store.put(key=key, value=new_data)
else:
store.put(key=key, value=new_data)
if on_duplicate_key == 'overwrite':
assert store.get(key) == new_data
else:
assert store.get(key) == data
def test_result(data, serializer):
r = XpersistResult(
CacheStore(),
serializer=serializer,
)
new = r.write(data)
assert new.read(new.location).value == data
assert r.cache_store.get(new.location) == data
assert r.exists(new.location)
def test_result_flow(executor):
os.environ['PREFECT__FLOWS__CHECKPOINTING'] = 'True'
r = XpersistResult(
CacheStore(),
serializer='xarray.netcdf',
)
@task(target='testing.nc', result=r)
def xarray_task():
time.sleep(2)
return xr.tutorial.open_dataset('rasm').isel(time=0)
with Flow('test') as flow:
xarray_task()
state = flow.run(executor=executor)
assert not state.is_failed()
def test_put_and_get(tmp_path, key, data, serializer):
store = CacheStore(str(tmp_path))
store.put(key=key, value=data, serializer=serializer)
assert key in store.keys()
assert isinstance(store.get_artifact(key), Artifact)
results = store[key]
if isinstance(data, (xr.Dataset, xr.DataArray)):
xr.testing.assert_equal(results, data)
elif isinstance(data, pd.DataFrame):
pd.testing.assert_frame_equal(results, data)
else:
assert results == data
pyramid_path_annual = make_annual_pyramid_path(gcm_identifier)
return (
gcm_grid_spec,
obs_identifier,
gcm_identifier,
pyramid_path_daily,
pyramid_path_monthly,
pyramid_path_annual,
)
@task(
checkpoint=True,
result=XpersistResult(
CacheStore(config.get('storage.intermediate.uri')),
serializer='xarray.zarr',
),
target=make_interpolated_obs_path,
)
def coarsen_and_interpolate_obs_task(obs, train_period, predict_period,
variables, gcm, scenario,
chunking_approach, bbox, **kwargs):
"""
Coarsen the observation dataset to the grid of the GCM model specified in inputs then
interpolate back into the observation grid. Rechunk the final output according to chunking approach.
Parameters
----------
obs: str
Name of obs dataset
gcm: str
def test_delete_error(tmp_path):
store = CacheStore(str(tmp_path))
with pytest.raises(KeyError):
store.delete(key='foo', dry_run=False)
def test_initialization(tmp_path, readonly):
store = CacheStore(str(tmp_path), readonly=readonly)
assert isinstance(store.mapper, fsspec.mapping.FSMap)
assert isinstance(store.raw_path, str)
assert store.raw_path == str(tmp_path)
assert store.readonly == readonly
from cmip6_downscaling.workflows.paths import (
make_annual_summary_path,
make_bcsd_output_path,
make_bias_corrected_path,
make_coarse_obs_path,
make_gcm_predict_path,
make_monthly_summary_path,
make_rechunked_gcm_path,
make_return_obs_path,
make_spatial_anomalies_path,
)
runtime = runtimes.get_runtime()
intermediate_cache_store = CacheStore(
config.get("storage.intermediate.uri"),
storage_options=config.get("storage.intermediate.storage_options"),
)
results_cache_store = CacheStore(
config.get("storage.results.uri"),
storage_options=config.get("storage.results.storage_options"),
)
# Transform Functions into Tasks -----------------------------------------------------------
return_obs_task = task(
return_obs,
result=XpersistResult(intermediate_cache_store, serializer="xarray.zarr"),
target=make_return_obs_path,
)
get_coarse_obs_task = task(
get_coarse_obs,