def test_chunks_are_larger_than_sizes(self):
cube1 = new_cube(variables=dict(chl=0.6, tsm=0.9, flags=16))
for var in cube1.variables.values():
self.assertIsNone(var.chunks)
rc = CubeRechunker()
cube2, gm, cc = rc.transform_cube(
cube1, GridMapping.from_dataset(cube1),
CubeConfig(chunks=dict(time=64, lat=512, lon=512)))
self.assertIsInstance(cube2, xr.Dataset)
self.assertIsInstance(gm, GridMapping)
self.assertIsInstance(cc, CubeConfig)
self.assertEqual(cube1.attrs, cube2.attrs)
self.assertEqual(set(cube1.coords), set(cube2.coords))
self.assertEqual(set(cube1.data_vars), set(cube2.data_vars))
for k, v in cube2.coords.items():
if v.chunks is not None:
self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}')
self.assertNotIn('chunks', v.encoding)
for k, v in cube2.data_vars.items():
self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}')
self.assertEqual(((5, ), (180, ), (360, )), v.chunks)
self.assertNotIn('chunks', v.encoding)
def test_chunks_are_smaller_than_sizes(self):
cube1 = new_cube(variables=dict(chl=0.6, tsm=0.9, flags=16))
for var in cube1.variables.values():
self.assertIsNone(var.chunks)
rc = CubeRechunker()
cube2, gm, cc = rc.transform_cube(
cube1, GridMapping.from_dataset(cube1),
CubeConfig(chunks=dict(time=2, lat=100, lon=200)))
self.assertIsInstance(cube2, xr.Dataset)
self.assertIsInstance(gm, GridMapping)
self.assertIsInstance(cc, CubeConfig)
self.assertEqual(cube1.attrs, cube2.attrs)
self.assertEqual({'time': 2, 'lat': 100, 'lon': 200}, cc.chunks)
self.assertEqual(set(cube1.coords), set(cube2.coords))
self.assertEqual(set(cube1.data_vars), set(cube2.data_vars))
for k, v in cube2.coords.items():
if v.chunks is not None:
self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}')
self.assertNotIn('chunks', v.encoding)
# self.assertIn('chunks', v.encoding)
# self.assertEqual([v.sizes[d] for d in v.dims],
# v.encoding['chunks'])
for k, v in cube2.data_vars.items():
self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}')
self.assertEqual(((2, 2, 1), (100, 80), (200, 160)), v.chunks)
self.assertNotIn('chunks', v.encoding)
def transform_cube(self,
cube: xr.Dataset,
gm: GridMapping,
cube_config: CubeConfig) -> TransformedCube:
cube = cube.chunk(dict(lon=cube_config.tile_size[0],
lat=cube_config.tile_size[1]))
cube_config = cube_config.drop_props('tile_size')
return cube, gm, cube_config
def test_easter_egg(self):
cube = new_cube()
md_adjuster = CubeMetadataAdjuster()
with self.assertRaises(ValueError) as cm:
md_adjuster.transform_cube(
cube, GridMapping.from_dataset(cube),
CubeConfig(metadata=dict(inverse_fine_structure_constant=136)))
self.assertEqual(('inverse_fine_structure_constant must be 137'
' or running in wrong universe', ),
cm.exception.args)
def test_it(self):
cube = new_cube(variables=dict(a=0.5))
gm = GridMapping.from_dataset(cube)
cube_config = CubeConfig()
identity = CubeIdentity()
t_cube = identity.transform_cube(cube,
gm,
cube_config)
self.assertIsInstance(t_cube, tuple)
self.assertEqual(3, len(t_cube))
self.assertIs(cube, t_cube[0])
self.assertIs(gm, t_cube[1])
self.assertIs(cube_config, t_cube[2])
def assertCallableWorks(self, user_code_callable):
code_config = CodeConfig(_callable=user_code_callable,
callable_params=self.good_params)
executor = CubeUserCodeExecutor(code_config)
ds_input = new_cube(variables=dict(a=1))
ds_output, gm, cc = executor.transform_cube(
ds_input, GridMapping.from_dataset(ds_input), CubeConfig())
self.assertIsInstance(ds_output, xr.Dataset)
self.assertIsInstance(gm, GridMapping)
self.assertIsInstance(cc, CubeConfig)
self.assertIsNot(ds_output, ds_input)
self.assertIn('X', ds_output)
self.assertEqual(42, ds_output.X)
def test_metadata_adjusted_geo_crs(self):
(x1, x2), (y1, y2) = (53, 54), (11, 12)
cube1 = new_cube(
width=1000,
height=1000,
variables=dict(chl=0.6, tsm=0.9, flags=16),
x_start=x1,
y_start=y1,
x_res=(x2 - x1) / 1000,
y_res=(x2 - x1) / 1000,
)
cube1.attrs = {}
md_adjuster = CubeMetadataAdjuster()
cube2, gm, cc = md_adjuster.transform_cube(
cube1, GridMapping.from_dataset(cube1),
CubeConfig(metadata=dict(title='S2L2A subset'),
variable_metadata=dict(
chl=dict(long_name='Chlorophyll'),
tsm=dict(long_name='Total suspended matter'),
flags=dict(long_name='Quality flags'),
)))
self.assertIsNot(cube1, cube2)
self.assertIsInstance(gm, GridMapping)
self.assertIsInstance(cc, CubeConfig)
date_created = cube2.attrs.pop('date_created', None)
self.assertIsInstance(date_created, str)
history = cube2.attrs.pop('history', None)
self.assertIsInstance(history, list)
self.assertEqual(
{
'Conventions': 'CF-1.7',
'title': 'S2L2A subset',
'geospatial_bounds_crs': 'CRS84',
'geospatial_bounds': 'POLYGON((53 11, 53 12,'
' 54 12, 54 11, 53 11))',
'geospatial_lat_max': 12,
'geospatial_lat_min': 11,
'geospatial_lat_resolution': 0.001,
'geospatial_lat_units': 'degrees_north',
'geospatial_lon_max': 54,
'geospatial_lon_min': 53,
'geospatial_lon_resolution': 0.001,
'geospatial_lon_units': 'degrees_east',
}, cube2.attrs)
self.assertEqual({'long_name': 'Chlorophyll'}, cube2.chl.attrs)
self.assertEqual({'long_name': 'Total suspended matter'},
cube2.tsm.attrs)
self.assertEqual({'long_name': 'Quality flags'}, cube2.flags.attrs)
def test_empty_cube(self):
cube = new_cube()
gm = GridMapping.from_dataset(cube)
cube_config = CubeConfig(tile_size=180)
t_cube = transform_cube((cube, gm, cube_config), MyTiler())
self.assertIsInstance(t_cube, tuple)
self.assertEqual(3, len(t_cube))
cube2, gm2, cc2 = t_cube
self.assertIs(cube, cube2)
self.assertIs(gm, gm2)
self.assertIs(cube_config, cc2)
self.assertEqual((180, 180), cc2.tile_size)
def test_non_empty_cube(self):
cube = new_cube(variables=dict(a=0.5))
gm = GridMapping.from_dataset(cube)
cube_config = CubeConfig(tile_size=180)
t_cube = transform_cube((cube, gm, cube_config), MyTiler())
self.assertIsInstance(t_cube, tuple)
self.assertEqual(3, len(t_cube))
cube2, gm2, cc2 = t_cube
self.assertIsNot(cube, cube2)
self.assertEqual(((5,), (180,), (180, 180)), cube2.a.chunks)
self.assertIs(gm, gm2)
self.assertEqual(None, cc2.tile_size)
def test_metadata_adjusted_other_crs(self):
crs = pyproj.CRS.from_string('epsg:25832')
t = pyproj.Transformer.from_crs(crs_from=CRS_CRS84, crs_to=crs)
((x1, x2), (y1, y2)) = t.transform((53, 54), (10, 11))
cube1 = new_cube(
width=1000,
height=1000,
variables=dict(chl=0.6, tsm=0.9, flags=16),
x_start=x1,
y_start=y1,
x_res=(x2 - x1) / 1000,
y_res=(x2 - x1) / 1000,
x_name='x',
y_name='y',
crs=crs,
)
cube1.attrs = {}
md_adjuster = CubeMetadataAdjuster()
cube2, gm, cc = md_adjuster.transform_cube(
cube1, GridMapping.from_dataset(cube1),
CubeConfig(metadata=dict(title='S2L2A subset'),
variable_metadata=dict(
chl=dict(long_name='Chlorophyll'),
tsm=dict(long_name='Total suspended matter'),
flags=dict(long_name='Quality flags'),
)))
self.assertIsNot(cube1, cube2)
self.assertIsInstance(gm, GridMapping)
self.assertIsInstance(cc, CubeConfig)
self.assertAlmostEqual(53.,
cube2.attrs.get('geospatial_lon_min'),
delta=0.001)
self.assertAlmostEqual(10.,
cube2.attrs.get('geospatial_lat_min'),
delta=0.001)
self.assertAlmostEqual(0.001,
cube2.attrs.get('geospatial_lon_resolution'),
delta=0.0005)
self.assertAlmostEqual(0.001,
cube2.attrs.get('geospatial_lat_resolution'),
delta=0.0005)