def test_chunks_are_larger_than_sizes(self): cube1 = new_cube(variables=dict(chl=0.6, tsm=0.9, flags=16)) for var in cube1.variables.values(): self.assertIsNone(var.chunks) rc = CubeRechunker() cube2, gm, cc = rc.transform_cube( cube1, GridMapping.from_dataset(cube1), CubeConfig(chunks=dict(time=64, lat=512, lon=512))) self.assertIsInstance(cube2, xr.Dataset) self.assertIsInstance(gm, GridMapping) self.assertIsInstance(cc, CubeConfig) self.assertEqual(cube1.attrs, cube2.attrs) self.assertEqual(set(cube1.coords), set(cube2.coords)) self.assertEqual(set(cube1.data_vars), set(cube2.data_vars)) for k, v in cube2.coords.items(): if v.chunks is not None: self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}') self.assertNotIn('chunks', v.encoding) for k, v in cube2.data_vars.items(): self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}') self.assertEqual(((5, ), (180, ), (360, )), v.chunks) self.assertNotIn('chunks', v.encoding)
def test_chunks_are_smaller_than_sizes(self): cube1 = new_cube(variables=dict(chl=0.6, tsm=0.9, flags=16)) for var in cube1.variables.values(): self.assertIsNone(var.chunks) rc = CubeRechunker() cube2, gm, cc = rc.transform_cube( cube1, GridMapping.from_dataset(cube1), CubeConfig(chunks=dict(time=2, lat=100, lon=200))) self.assertIsInstance(cube2, xr.Dataset) self.assertIsInstance(gm, GridMapping) self.assertIsInstance(cc, CubeConfig) self.assertEqual(cube1.attrs, cube2.attrs) self.assertEqual({'time': 2, 'lat': 100, 'lon': 200}, cc.chunks) self.assertEqual(set(cube1.coords), set(cube2.coords)) self.assertEqual(set(cube1.data_vars), set(cube2.data_vars)) for k, v in cube2.coords.items(): if v.chunks is not None: self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}') self.assertNotIn('chunks', v.encoding) # self.assertIn('chunks', v.encoding) # self.assertEqual([v.sizes[d] for d in v.dims], # v.encoding['chunks']) for k, v in cube2.data_vars.items(): self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}') self.assertEqual(((2, 2, 1), (100, 80), (200, 160)), v.chunks) self.assertNotIn('chunks', v.encoding)
def transform_cube(self, cube: xr.Dataset, gm: GridMapping, cube_config: CubeConfig) -> TransformedCube: cube = cube.chunk(dict(lon=cube_config.tile_size[0], lat=cube_config.tile_size[1])) cube_config = cube_config.drop_props('tile_size') return cube, gm, cube_config
def test_easter_egg(self): cube = new_cube() md_adjuster = CubeMetadataAdjuster() with self.assertRaises(ValueError) as cm: md_adjuster.transform_cube( cube, GridMapping.from_dataset(cube), CubeConfig(metadata=dict(inverse_fine_structure_constant=136))) self.assertEqual(('inverse_fine_structure_constant must be 137' ' or running in wrong universe', ), cm.exception.args)
def test_it(self): cube = new_cube(variables=dict(a=0.5)) gm = GridMapping.from_dataset(cube) cube_config = CubeConfig() identity = CubeIdentity() t_cube = identity.transform_cube(cube, gm, cube_config) self.assertIsInstance(t_cube, tuple) self.assertEqual(3, len(t_cube)) self.assertIs(cube, t_cube[0]) self.assertIs(gm, t_cube[1]) self.assertIs(cube_config, t_cube[2])
def assertCallableWorks(self, user_code_callable): code_config = CodeConfig(_callable=user_code_callable, callable_params=self.good_params) executor = CubeUserCodeExecutor(code_config) ds_input = new_cube(variables=dict(a=1)) ds_output, gm, cc = executor.transform_cube( ds_input, GridMapping.from_dataset(ds_input), CubeConfig()) self.assertIsInstance(ds_output, xr.Dataset) self.assertIsInstance(gm, GridMapping) self.assertIsInstance(cc, CubeConfig) self.assertIsNot(ds_output, ds_input) self.assertIn('X', ds_output) self.assertEqual(42, ds_output.X)
def test_metadata_adjusted_geo_crs(self): (x1, x2), (y1, y2) = (53, 54), (11, 12) cube1 = new_cube( width=1000, height=1000, variables=dict(chl=0.6, tsm=0.9, flags=16), x_start=x1, y_start=y1, x_res=(x2 - x1) / 1000, y_res=(x2 - x1) / 1000, ) cube1.attrs = {} md_adjuster = CubeMetadataAdjuster() cube2, gm, cc = md_adjuster.transform_cube( cube1, GridMapping.from_dataset(cube1), CubeConfig(metadata=dict(title='S2L2A subset'), variable_metadata=dict( chl=dict(long_name='Chlorophyll'), tsm=dict(long_name='Total suspended matter'), flags=dict(long_name='Quality flags'), ))) self.assertIsNot(cube1, cube2) self.assertIsInstance(gm, GridMapping) self.assertIsInstance(cc, CubeConfig) date_created = cube2.attrs.pop('date_created', None) self.assertIsInstance(date_created, str) history = cube2.attrs.pop('history', None) self.assertIsInstance(history, list) self.assertEqual( { 'Conventions': 'CF-1.7', 'title': 'S2L2A subset', 'geospatial_bounds_crs': 'CRS84', 'geospatial_bounds': 'POLYGON((53 11, 53 12,' ' 54 12, 54 11, 53 11))', 'geospatial_lat_max': 12, 'geospatial_lat_min': 11, 'geospatial_lat_resolution': 0.001, 'geospatial_lat_units': 'degrees_north', 'geospatial_lon_max': 54, 'geospatial_lon_min': 53, 'geospatial_lon_resolution': 0.001, 'geospatial_lon_units': 'degrees_east', }, cube2.attrs) self.assertEqual({'long_name': 'Chlorophyll'}, cube2.chl.attrs) self.assertEqual({'long_name': 'Total suspended matter'}, cube2.tsm.attrs) self.assertEqual({'long_name': 'Quality flags'}, cube2.flags.attrs)
def test_empty_cube(self): cube = new_cube() gm = GridMapping.from_dataset(cube) cube_config = CubeConfig(tile_size=180) t_cube = transform_cube((cube, gm, cube_config), MyTiler()) self.assertIsInstance(t_cube, tuple) self.assertEqual(3, len(t_cube)) cube2, gm2, cc2 = t_cube self.assertIs(cube, cube2) self.assertIs(gm, gm2) self.assertIs(cube_config, cc2) self.assertEqual((180, 180), cc2.tile_size)
def test_non_empty_cube(self): cube = new_cube(variables=dict(a=0.5)) gm = GridMapping.from_dataset(cube) cube_config = CubeConfig(tile_size=180) t_cube = transform_cube((cube, gm, cube_config), MyTiler()) self.assertIsInstance(t_cube, tuple) self.assertEqual(3, len(t_cube)) cube2, gm2, cc2 = t_cube self.assertIsNot(cube, cube2) self.assertEqual(((5,), (180,), (180, 180)), cube2.a.chunks) self.assertIs(gm, gm2) self.assertEqual(None, cc2.tile_size)
def test_metadata_adjusted_other_crs(self): crs = pyproj.CRS.from_string('epsg:25832') t = pyproj.Transformer.from_crs(crs_from=CRS_CRS84, crs_to=crs) ((x1, x2), (y1, y2)) = t.transform((53, 54), (10, 11)) cube1 = new_cube( width=1000, height=1000, variables=dict(chl=0.6, tsm=0.9, flags=16), x_start=x1, y_start=y1, x_res=(x2 - x1) / 1000, y_res=(x2 - x1) / 1000, x_name='x', y_name='y', crs=crs, ) cube1.attrs = {} md_adjuster = CubeMetadataAdjuster() cube2, gm, cc = md_adjuster.transform_cube( cube1, GridMapping.from_dataset(cube1), CubeConfig(metadata=dict(title='S2L2A subset'), variable_metadata=dict( chl=dict(long_name='Chlorophyll'), tsm=dict(long_name='Total suspended matter'), flags=dict(long_name='Quality flags'), ))) self.assertIsNot(cube1, cube2) self.assertIsInstance(gm, GridMapping) self.assertIsInstance(cc, CubeConfig) self.assertAlmostEqual(53., cube2.attrs.get('geospatial_lon_min'), delta=0.001) self.assertAlmostEqual(10., cube2.attrs.get('geospatial_lat_min'), delta=0.001) self.assertAlmostEqual(0.001, cube2.attrs.get('geospatial_lon_resolution'), delta=0.0005) self.assertAlmostEqual(0.001, cube2.attrs.get('geospatial_lat_resolution'), delta=0.0005)