def test_delete(tmp_path, key, data, serializer): store = CacheStore(str(tmp_path)) store.put(key=key, value=data, serializer=serializer) assert key in store.keys() store.delete(key=key, dry_run=True) del store[key] assert key not in store.keys()
def test_on_duplicate_key(tmp_path, on_duplicate_key): store = CacheStore(str(tmp_path), on_duplicate_key=on_duplicate_key) key, data, new_data = 'foo', 'my_data', 'hello' store.put(key=key, value=data) if on_duplicate_key == 'raise_error': with pytest.raises(ValueError): store.put(key=key, value=new_data) else: store.put(key=key, value=new_data) if on_duplicate_key == 'overwrite': assert store.get(key) == new_data else: assert store.get(key) == data
def test_result(data, serializer): r = XpersistResult( CacheStore(), serializer=serializer, ) new = r.write(data) assert new.read(new.location).value == data assert r.cache_store.get(new.location) == data assert r.exists(new.location)
def test_result_flow(executor): os.environ['PREFECT__FLOWS__CHECKPOINTING'] = 'True' r = XpersistResult( CacheStore(), serializer='xarray.netcdf', ) @task(target='testing.nc', result=r) def xarray_task(): time.sleep(2) return xr.tutorial.open_dataset('rasm').isel(time=0) with Flow('test') as flow: xarray_task() state = flow.run(executor=executor) assert not state.is_failed()
def test_put_and_get(tmp_path, key, data, serializer): store = CacheStore(str(tmp_path)) store.put(key=key, value=data, serializer=serializer) assert key in store.keys() assert isinstance(store.get_artifact(key), Artifact) results = store[key] if isinstance(data, (xr.Dataset, xr.DataArray)): xr.testing.assert_equal(results, data) elif isinstance(data, pd.DataFrame): pd.testing.assert_frame_equal(results, data) else: assert results == data
pyramid_path_annual = make_annual_pyramid_path(gcm_identifier) return ( gcm_grid_spec, obs_identifier, gcm_identifier, pyramid_path_daily, pyramid_path_monthly, pyramid_path_annual, ) @task( checkpoint=True, result=XpersistResult( CacheStore(config.get('storage.intermediate.uri')), serializer='xarray.zarr', ), target=make_interpolated_obs_path, ) def coarsen_and_interpolate_obs_task(obs, train_period, predict_period, variables, gcm, scenario, chunking_approach, bbox, **kwargs): """ Coarsen the observation dataset to the grid of the GCM model specified in inputs then interpolate back into the observation grid. Rechunk the final output according to chunking approach. Parameters ---------- obs: str Name of obs dataset gcm: str
def test_delete_error(tmp_path): store = CacheStore(str(tmp_path)) with pytest.raises(KeyError): store.delete(key='foo', dry_run=False)
def test_initialization(tmp_path, readonly): store = CacheStore(str(tmp_path), readonly=readonly) assert isinstance(store.mapper, fsspec.mapping.FSMap) assert isinstance(store.raw_path, str) assert store.raw_path == str(tmp_path) assert store.readonly == readonly
from cmip6_downscaling.workflows.paths import ( make_annual_summary_path, make_bcsd_output_path, make_bias_corrected_path, make_coarse_obs_path, make_gcm_predict_path, make_monthly_summary_path, make_rechunked_gcm_path, make_return_obs_path, make_spatial_anomalies_path, ) runtime = runtimes.get_runtime() intermediate_cache_store = CacheStore( config.get("storage.intermediate.uri"), storage_options=config.get("storage.intermediate.storage_options"), ) results_cache_store = CacheStore( config.get("storage.results.uri"), storage_options=config.get("storage.results.storage_options"), ) # Transform Functions into Tasks ----------------------------------------------------------- return_obs_task = task( return_obs, result=XpersistResult(intermediate_cache_store, serializer="xarray.zarr"), target=make_return_obs_path, ) get_coarse_obs_task = task( get_coarse_obs,