def open_ml_dataset_from_local_fs( ctx: ServiceContext, dataset_descriptor: DatasetDescriptor) -> MultiLevelDataset: ds_id = dataset_descriptor.get('Identifier') path = dataset_descriptor.get('Path') if not path: raise ServiceConfigError( f"Missing 'path' entry in dataset descriptor {ds_id}") if not os.path.isabs(path): path = os.path.join(ctx.base_dir, path) data_format = dataset_descriptor.get('Format', guess_cube_format(path)) if data_format == FORMAT_NAME_NETCDF4: with measure_time(tag=f"opened local NetCDF dataset {path}"): ds = assert_cube(xr.open_dataset(path)) return BaseMultiLevelDataset(ds) if data_format == FORMAT_NAME_ZARR: with measure_time(tag=f"opened local zarr dataset {path}"): ds = assert_cube(xr.open_zarr(path)) return BaseMultiLevelDataset(ds) if data_format == FORMAT_NAME_LEVELS: with measure_time(tag=f"opened local levels dataset {path}"): return FileStorageMultiLevelDataset(path) raise ServiceConfigError( f"Illegal data format {data_format!r} for dataset {ds_id}")
def test_dry_run(self): result = self.invoke_cli(['prune', self.TEST_CUBE, "--dry-run"]) self.assertEqual(0, result.exit_code) self.assertEqual( "Opening cube from 'test.zarr'...\n" "Identifying empty blocks...\n" "Deleting 24 empty block file(s) for variable 'precipitation'...\n" "Deleting 24 empty block file(s) for variable 'temperature'...\n" "Done, 48 block file(s) deleted.\n", result.stdout) expected_file_names = sorted([ '.zarray', '.zattrs', '0.0.0', '0.0.1', '0.0.2', '0.0.3', '0.1.0', '0.1.1', '0.1.2', '0.1.3', '1.0.0', '1.0.1', '1.0.2', '1.0.3', '1.1.0', '1.1.1', '1.1.2', '1.1.3', '2.0.0', '2.0.1', '2.0.2', '2.0.3', '2.1.0', '2.1.1', '2.1.2', '2.1.3' ]) self.assertEqual(expected_file_names, sorted(os.listdir('test.zarr/precipitation'))) self.assertEqual(expected_file_names, sorted(os.listdir('test.zarr/temperature'))) ds = xr.open_zarr('test.zarr') assert_cube(ds) self.assertIn('precipitation', ds) self.assertEqual((3, 180, 360), ds.precipitation.shape) self.assertEqual(('time', 'lat', 'lon'), ds.precipitation.dims) self.assertIn('temperature', ds) self.assertEqual((3, 180, 360), ds.temperature.shape) self.assertEqual(('time', 'lat', 'lon'), ds.temperature.dims)
def test_with_vars(self): result = self.invoke_cli(['resample', TEST_ZARR_DIR, '--vars', 'temperature,precipitation']) self.assertEqual(0, result.exit_code) self.assertTrue(os.path.isdir('out.zarr')) ds = xr.open_zarr('out.zarr') assert_cube(ds) self.assertIn('precipitation_mean', ds) self.assertIn('temperature_mean', ds) self.assertNotIn('soil_moisture_mean', ds)
def test_all_defaults(self): result = self.invoke_cli(['resample', TEST_ZARR_DIR]) self.assertEqual(0, result.exit_code) self.assertEqual("Opening cube from 'test.zarr'...\n" "Resampling...\n" "Writing resampled cube to 'out.zarr'...\n" "Done.\n", result.stdout) self.assertTrue(os.path.isdir('out.zarr')) ds = xr.open_zarr('out.zarr') assert_cube(ds) self.assertIn('precipitation_mean', ds) self.assertIn('temperature_mean', ds) self.assertIn('soil_moisture_mean', ds)
def test_upsample_with_multiple_methods(self): result = self.invoke_cli(['resample', '--variables', 'temperature', '-F', '12H', '-T', '6H', # '-K', 'quadratic', # '-M', 'interpolate', '-M', 'nearest', TEST_ZARR_DIR]) self.assertEqual(0, result.exit_code) self.assertTrue(os.path.isdir('out.zarr')) ds = xr.open_zarr('out.zarr') assert_cube(ds) # self.assertIn('temperature_interpolate', ds) self.assertIn('temperature_nearest', ds)
def test_downsample_with_multiple_methods(self): result = self.invoke_cli(['resample', '--variables', 'temperature', '-F', '3D', '-M', 'mean', '-M', 'count', '-M', 'prod', TEST_ZARR_DIR]) self.assertEqual(0, result.exit_code) self.assertTrue(os.path.isdir('out.zarr')) ds = xr.open_zarr('out.zarr') assert_cube(ds) self.assertIn('temperature_mean', ds) self.assertIn('temperature_count', ds) self.assertIn('temperature_prod', ds)