def open_ml_dataset_from_local_fs(
ctx: ServiceContext,
dataset_descriptor: DatasetDescriptor) -> MultiLevelDataset:
ds_id = dataset_descriptor.get('Identifier')
path = dataset_descriptor.get('Path')
if not path:
raise ServiceConfigError(
f"Missing 'path' entry in dataset descriptor {ds_id}")
if not os.path.isabs(path):
path = os.path.join(ctx.base_dir, path)
data_format = dataset_descriptor.get('Format', guess_cube_format(path))
if data_format == FORMAT_NAME_NETCDF4:
with measure_time(tag=f"opened local NetCDF dataset {path}"):
ds = assert_cube(xr.open_dataset(path))
return BaseMultiLevelDataset(ds)
if data_format == FORMAT_NAME_ZARR:
with measure_time(tag=f"opened local zarr dataset {path}"):
ds = assert_cube(xr.open_zarr(path))
return BaseMultiLevelDataset(ds)
if data_format == FORMAT_NAME_LEVELS:
with measure_time(tag=f"opened local levels dataset {path}"):
return FileStorageMultiLevelDataset(path)
raise ServiceConfigError(
f"Illegal data format {data_format!r} for dataset {ds_id}")
def test_dry_run(self):
result = self.invoke_cli(['prune', self.TEST_CUBE, "--dry-run"])
self.assertEqual(0, result.exit_code)
self.assertEqual(
"Opening cube from 'test.zarr'...\n"
"Identifying empty blocks...\n"
"Deleting 24 empty block file(s) for variable 'precipitation'...\n"
"Deleting 24 empty block file(s) for variable 'temperature'...\n"
"Done, 48 block file(s) deleted.\n", result.stdout)
expected_file_names = sorted([
'.zarray', '.zattrs', '0.0.0', '0.0.1', '0.0.2', '0.0.3', '0.1.0',
'0.1.1', '0.1.2', '0.1.3', '1.0.0', '1.0.1', '1.0.2', '1.0.3',
'1.1.0', '1.1.1', '1.1.2', '1.1.3', '2.0.0', '2.0.1', '2.0.2',
'2.0.3', '2.1.0', '2.1.1', '2.1.2', '2.1.3'
])
self.assertEqual(expected_file_names,
sorted(os.listdir('test.zarr/precipitation')))
self.assertEqual(expected_file_names,
sorted(os.listdir('test.zarr/temperature')))
ds = xr.open_zarr('test.zarr')
assert_cube(ds)
self.assertIn('precipitation', ds)
self.assertEqual((3, 180, 360), ds.precipitation.shape)
self.assertEqual(('time', 'lat', 'lon'), ds.precipitation.dims)
self.assertIn('temperature', ds)
self.assertEqual((3, 180, 360), ds.temperature.shape)
self.assertEqual(('time', 'lat', 'lon'), ds.temperature.dims)
def test_with_vars(self):
result = self.invoke_cli(['resample', TEST_ZARR_DIR, '--vars', 'temperature,precipitation'])
self.assertEqual(0, result.exit_code)
self.assertTrue(os.path.isdir('out.zarr'))
ds = xr.open_zarr('out.zarr')
assert_cube(ds)
self.assertIn('precipitation_mean', ds)
self.assertIn('temperature_mean', ds)
self.assertNotIn('soil_moisture_mean', ds)
def test_all_defaults(self):
result = self.invoke_cli(['resample', TEST_ZARR_DIR])
self.assertEqual(0, result.exit_code)
self.assertEqual("Opening cube from 'test.zarr'...\n"
"Resampling...\n"
"Writing resampled cube to 'out.zarr'...\n"
"Done.\n",
result.stdout)
self.assertTrue(os.path.isdir('out.zarr'))
ds = xr.open_zarr('out.zarr')
assert_cube(ds)
self.assertIn('precipitation_mean', ds)
self.assertIn('temperature_mean', ds)
self.assertIn('soil_moisture_mean', ds)
def test_upsample_with_multiple_methods(self):
result = self.invoke_cli(['resample',
'--variables', 'temperature',
'-F', '12H',
'-T', '6H',
# '-K', 'quadratic',
# '-M', 'interpolate',
'-M', 'nearest',
TEST_ZARR_DIR])
self.assertEqual(0, result.exit_code)
self.assertTrue(os.path.isdir('out.zarr'))
ds = xr.open_zarr('out.zarr')
assert_cube(ds)
# self.assertIn('temperature_interpolate', ds)
self.assertIn('temperature_nearest', ds)
def test_downsample_with_multiple_methods(self):
result = self.invoke_cli(['resample',
'--variables', 'temperature',
'-F', '3D',
'-M', 'mean',
'-M', 'count',
'-M', 'prod',
TEST_ZARR_DIR])
self.assertEqual(0, result.exit_code)
self.assertTrue(os.path.isdir('out.zarr'))
ds = xr.open_zarr('out.zarr')
assert_cube(ds)
self.assertIn('temperature_mean', ds)
self.assertIn('temperature_count', ds)
self.assertIn('temperature_prod', ds)