def test_select_all(self):
ds1 = create_highroc_dataset()
# noinspection PyTypeChecker
ds2 = select_variables(ds1, None)
self.assertIs(ds2, ds1)
ds2 = select_variables(ds1, ds1.data_vars.keys())
self.assertIs(ds2, ds1)
def test_vectorize_spectra(self):
dataset = create_highroc_dataset()
self.assertEqual(36, len(dataset.data_vars))
vectorized_dataset = vectorize_wavebands(dataset)
self.assertEqual(36 - 2 * 16 + 2, len(vectorized_dataset.data_vars))
self.assertIn('band', vectorized_dataset.dims)
self.assertIn('band', vectorized_dataset.coords)
band_var = vectorized_dataset.coords['band']
assert_array_almost_equal(
np.array([
400., 412.5, 442.5, 490., 510., 560., 620., 665., 673.75,
681.25, 708.75, 753.75, 778.75, 865., 885., 940.
]), band_var.values)
self.assertEqual((16, ), band_var.shape)
self.assertEqual('nm', band_var.attrs.get('units'))
self.assertIn('rtoa', vectorized_dataset)
rtoa_var = vectorized_dataset['rtoa']
self.assertEqual((16, 3, 4), rtoa_var.shape)
self.assertEqual(('band', 'y', 'x'), rtoa_var.dims)
self.assertEqual('1', rtoa_var.attrs.get('units'))
self.assertEqual('Top-of-atmosphere reflectance',
rtoa_var.attrs.get('long_name'))
self.assertIn('rrs', vectorized_dataset)
rrs_var = vectorized_dataset['rrs']
self.assertEqual((16, 3, 4), rrs_var.shape)
self.assertEqual(('band', 'y', 'x'), rrs_var.dims)
self.assertEqual('sr^-1', rrs_var.attrs.get('units'))
self.assertEqual(
'Atmospherically corrected angular dependent remote sensing reflectances',
rrs_var.attrs.get('long_name'))
def test_no_change(self):
ds1 = create_highroc_dataset()
# noinspection PyTypeChecker
ds2 = update_variable_props(ds1, None)
self.assertIs(ds2, ds1)
ds2 = update_variable_props(ds1, [])
self.assertIs(ds2, ds1)
def test_get_mask_sets(self):
dataset = create_highroc_dataset()
mask_sets = MaskSet.get_mask_sets(dataset)
self.assertIsNotNone(mask_sets)
self.assertEqual(len(mask_sets), 1)
self.assertIn('c2rcc_flags', mask_sets)
mask_set = mask_sets['c2rcc_flags']
self.assertIsInstance(mask_set, MaskSet)
def test_add_time_coords_point(self):
dataset = create_highroc_dataset()
dataset_with_time = add_time_coords(dataset,
(365 * 47 + 20, 365 * 47 + 20))
self.assertIsNot(dataset_with_time, dataset)
self.assertIn('time', dataset_with_time)
self.assertEqual(dataset_with_time.time.shape, (1, ))
self.assertNotIn('time_bnds', dataset_with_time)
def test_change_all_or_none(self):
ds1 = create_highroc_dataset()
ds2 = update_variable_props(ds1, [(var_name, {
'marker': True
}) for var_name in ds1.data_vars])
self.assertEqual(len(ds1.data_vars), len(ds2.data_vars))
self.assertTrue(all(['marker' in ds2[n].attrs for n in ds2.variables]))
with self.assertRaises(KeyError):
update_variable_props(ds1, [('bibo', {'marker': True})])
def test_get_mask_sets(self):
dataset = create_highroc_dataset()
mask_sets = MaskSet.get_mask_sets(dataset)
self.assertIsNotNone(mask_sets)
self.assertEqual(len(mask_sets), 1)
self.assertIn('c2rcc_flags', mask_sets)
mask_set = mask_sets['c2rcc_flags']
self.assertIsInstance(mask_set, MaskSet)
# TODO: add tests according to http://cfconventions.org/Data/cf-conventions/cf-conventions-1.7/build/ch03s05.html
def test_select_spatial_subset_invalid_params(self):
ds1 = create_highroc_dataset()
with self.assertRaises(ValueError) as cm:
select_spatial_subset(ds1,
ij_bbox=(5, 6, 7, 8),
xy_bbox=(0., 0., 1., 2.))
self.assertEqual("Only one of ij_bbox and xy_bbox can be given",
f'{cm.exception}')
with self.assertRaises(ValueError) as cm:
select_spatial_subset(ds1)
self.assertEqual("One of ij_bbox and xy_bbox must be given",
f'{cm.exception}')
def test_change_some(self):
ds1 = create_highroc_dataset()
ds2 = update_variable_props(ds1, [('conc_chl', {
'name': 'chl_c2rcc'
}), ('c2rcc_flags', {
'name': 'flags',
'marker': True
}), ('rtoa_10', None)])
self.assertEqual(len(ds1.data_vars), len(ds2.data_vars))
self.assertNotIn('conc_chl', ds2.data_vars)
self.assertNotIn('c2rcc_flags', ds2.data_vars)
self.assertIn('chl_c2rcc', ds2.data_vars)
self.assertIn('original_name', ds2.chl_c2rcc.attrs)
self.assertEqual('conc_chl', ds2.chl_c2rcc.attrs['original_name'])
self.assertIn('flags', ds2.data_vars)
self.assertIn('original_name', ds2.flags.attrs)
self.assertEqual('c2rcc_flags', ds2.flags.attrs['original_name'])
self.assertIn('marker', ds2.flags.attrs)
self.assertEqual(True, ds2.flags.attrs['marker'])
self.assertIn('rtoa_10', ds2.data_vars)
with self.assertRaises(ValueError) as cm:
update_variable_props(ds1, [('conc_chl', None),
('c2rcc_flags', None),
('rtoa_1', {
'name': 'refl_toa'
}), ('rtoa_2', {
'name': 'refl_toa'
}), ('rtoa_3', {
'name': 'refl_toa'
})])
self.assertEqual(
"variable 'rtoa_2' cannot be renamed into 'refl_toa' because the name is already in use",
f'{cm.exception}')
def _test_pre_process(self):
# FIXME: this test raises because create_highroc_dataset() does not return compatible SNAP L2 DS.
ds1 = create_highroc_dataset()
ds2 = self.processor.pre_process(ds1)
self.assertIsNot(ds1, ds2)
def test_select_spatial_subset_some_ij_bbox(self):
ds1 = create_highroc_dataset()
ds2 = select_spatial_subset(ds1, ij_bbox=(1, 1, 4, 3))
self.assertEqual((2, 3), ds2.conc_chl.shape)
def test_select_variables_for_some(self):
ds1 = create_highroc_dataset()
self.assertEqual(36, len(ds1.data_vars))
ds2 = select_variables(ds1, ['conc_chl', 'c2rcc_flags', 'rtoa_10'])
self.assertEqual(3, len(ds2.data_vars))
def test_post_process(self):
ds1 = create_highroc_dataset()
ds2 = self.processor.post_process(ds1)
self.assertIsNot(ds1, ds2)
import unittest
import yaml
import zarr
from test.sampledata import create_highroc_dataset
from xcube.core.new import new_cube
from xcube.core.chunk import chunk_dataset
from xcube.constants import FORMAT_NAME_ZARR
from xcube.core.dsio import rimraf
from xcube.core.edit import edit_metadata
from xcube.core.optimize import optimize_dataset
TEST_CUBE = create_highroc_dataset()
TEST_CUBE_COORDS = chunk_dataset(new_cube(time_periods=3, variables=dict(A=0.5, B=-1.5)),
chunk_sizes=dict(time=1, lat=180, lon=360), format_name=FORMAT_NAME_ZARR)
TEST_CUBE_ZARR = 'test.zarr'
TEST_CUBE_ZARR_COORDS = 'test_coords.zarr'
TEST_CUBE_ZARR_OPTIMIZED = 'test-optimized.zarr'
TEST_CUBE_ZARR_EDIT = 'test_edited_meta.zarr'
TEST_CUBE_ZARR_OPTIMIZED_EDIT = 'test_optimized_edited_meta.zarr'
TEST_NEW_META = {'output_metadata': {'creator_name': 'Brockmann Consult GmbH with love',
'creator_url': 'www.some_very_nice_url.com',
'geospatial_lat_max': 'something around the north pole.'},
def test_select_none(self):
ds1 = create_highroc_dataset()
ds2 = select_variables(ds1, [])
self.assertEqual(0, len(ds2.data_vars))
ds2 = select_variables(ds1, ['bibo'])
self.assertEqual(0, len(ds2.data_vars))
def test_select_spatial_subset_some_xy_bbox(self):
ds1 = create_highroc_dataset()
ds2 = select_spatial_subset(ds1, xy_bbox=(8., 55, 10., 56.))
self.assertEqual((3, 3), ds2.conc_chl.shape)
def test_select_spatial_subset_none_xy_bbox(self):
ds1 = create_highroc_dataset()
ds2 = select_spatial_subset(ds1, xy_bbox=(13., 57., 15., 60.))
self.assertEqual(None, ds2)
ds2 = select_spatial_subset(ds1, xy_bbox=(5.5, 55, 6.5, 56))
self.assertEqual(None, ds2)
def test_select_spatial_subset_all_xy_bbox(self):
ds1 = create_highroc_dataset()
ds2 = select_spatial_subset(ds1, xy_bbox=(7.9, 53.9, 12., 56.4))
self.assertIs(ds2, ds1)
def test_reprojection_info(self):
reprojection_info = self.processor.get_reprojection_info(create_highroc_dataset())
self.assertEqual(('lon', 'lat'), reprojection_info.xy_var_names)
self.assertEqual(5, reprojection_info.xy_gcp_step)
def test_reproject_xy_to_wgs84_highroc(self):
dst_width = 12
dst_height = 9
dataset = create_highroc_dataset()
proj_dataset = reproject_xy_to_wgs84(
dataset,
src_xy_var_names=('lon', 'lat'),
src_xy_tp_var_names=('TP_longitude', 'TP_latitude'),
src_xy_gcp_step=1,
src_xy_tp_gcp_step=1,
dst_size=(dst_width, dst_height))
self.assertIsNotNone(proj_dataset)
self.assertEqual(dict(lon=dst_width, lat=dst_height, bnds=2),
proj_dataset.sizes)
self.assertIn('lon', proj_dataset)
self.assertEqual(proj_dataset.lon.shape, (dst_width, ))
self.assertIn('lat', proj_dataset)
self.assertEqual(proj_dataset.lat.shape, (dst_height, ))
self.assertIn('lon_bnds', proj_dataset)
self.assertEqual(proj_dataset.lon_bnds.shape, (dst_width, 2))
self.assertIn('lat_bnds', proj_dataset)
self.assertEqual(proj_dataset.lat_bnds.shape, (dst_height, 2))
expected_conc_chl = np.array(
[[7., 7., 11., 11., 11., 11., nan, nan, nan, nan, 5., 5.],
[7., 7., 11., 11., 11., 11., nan, nan, nan, 21., 21., 21.],
[5., 5., 10., 10., 10., 10., 2., 2., 2., 21., 21., 21.],
[5., 5., 10., 10., 10., 2., 2., 2., 2., 21., 17., 17.],
[5., 5., 10., 10., 10., 20., 20., 20., 20., 17., 17., 17.],
[5., 16., 6., 6., 6., 20., 20., 20., 17., 17., nan, nan],
[16., 16., 6., 6., 6., 20., nan, nan, nan, nan, nan, nan],
[16., 16., 6., nan, nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]],
dtype=np.float64)
self.assertIn('conc_chl', proj_dataset)
# print(proj_dataset.conc_chl)
self.assertEqual(proj_dataset.conc_chl.shape, (dst_height, dst_width))
self.assertEqual(proj_dataset.conc_chl.dtype, np.float64)
assert_array_almost_equal(proj_dataset.conc_chl, expected_conc_chl)
expected_c2rcc_flags = np.array(
[[1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1., 1.],
[1., 1., 1., 1., 1., 1., 1., 1., 1., 2., 2., 2.],
[1., 1., 4., 4., 4., 4., 1., 1., 1., 2., 2., 2.],
[1., 1., 4., 4., 4., 1., 1., 1., 1., 2., 1., 1.],
[1., 1., 4., 4., 4., 1., 1., 1., 1., 1., 1., 1.],
[1., 8., 1., 1., 1., 1., 1., 1., 1., 1., nan, nan],
[8., 8., 1., 1., 1., 1., nan, nan, nan, nan, nan, nan],
[8., 8., 1., nan, nan, nan, nan, nan, nan, nan, nan, nan],
[nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan, nan]],
dtype=np.float64)
self.assertIn('c2rcc_flags', proj_dataset)
# print(proj_dataset.c2rcc_flags)
self.assertEqual(proj_dataset.c2rcc_flags.shape,
(dst_height, dst_width))
self.assertEqual(proj_dataset.c2rcc_flags.dtype, np.float64)
assert_array_almost_equal(proj_dataset.c2rcc_flags,
expected_c2rcc_flags)
def test_select_spatial_subset_all_ij_bbox(self):
ds1 = create_highroc_dataset()
ds2 = select_spatial_subset(ds1, ij_bbox=(0, 0, 4, 3))
self.assertIs(ds2, ds1)
def test_vectorize_nothing(self):
dataset = create_highroc_dataset(no_spectra=True)
vectorized_dataset = vectorize_wavebands(dataset)
self.assertIs(dataset, vectorized_dataset)
def test_select_spatial_subset_none_ij_bbox(self):
ds1 = create_highroc_dataset()
ds2 = select_spatial_subset(ds1, ij_bbox=(5, 6, 7, 8))
self.assertEqual(None, ds2)
ds2 = select_spatial_subset(ds1, ij_bbox=(-6, -4, 2, 2))
self.assertEqual(None, ds2)
def test_it(self):
ds1 = create_highroc_dataset()
ds2 = translate_snap_expr_attributes(ds1)
self.assertIsNot(ds1, ds2)