def test_write_and_delete_data(self):
cube_3 = new_cube(variables=dict(B05=0.1, B06=0.2))
cube_3_id = self.store.write_data(cube_3, data_id='cube_3')
self.assertEqual('cube_3', cube_3_id)
self.assertIs(cube_3, self.store.open_data(cube_3_id))
cube_4 = new_cube(variables=dict(B07=0.1, B08=0.2))
cube_4_id = self.store.write_data(cube_4)
self.assertIsInstance(cube_4_id, str)
self.assertIs(cube_4, self.store.open_data(cube_4_id))
with self.assertRaises(DataStoreError) as cm:
self.store.write_data(cube_4, tile_size=1000, spatial_res=0.5)
self.assertEqual('Unsupported write_params "tile_size", "spatial_res"',
f'{cm.exception}')
with self.assertRaises(DataStoreError) as cm:
self.store.write_data(cube_4, data_id='cube_3')
self.assertEqual('Data resource "cube_3" already exist in store',
f'{cm.exception}')
self.store.delete_data(cube_3_id)
self.store.delete_data(cube_4_id)
with self.assertRaises(DataStoreError) as cm:
self.store.delete_data(cube_3_id)
self.assertEqual('Data resource "cube_3" does not exist in store',
f'{cm.exception}')
with self.assertRaises(DataStoreError) as cm:
self.store.delete_data(cube_4_id)
self.assertEqual(
f'Data resource "{cube_4_id}" does not exist in store',
f'{cm.exception}')
def _init_local_store():
"""
Initialize a "directory" data store with test datasets.
"""
from xcube.core.new import new_cube
local_base_dir = new_temp_dir(suffix='_local_store')
dataset_1 = new_cube(width=36, height=18, variables={'A': 0.1, 'B': 0.2})
dataset_2 = new_cube(width=36, height=18, variables={'C': 0.2, 'D': 0.3})
dataset_1.to_zarr(os.path.join(local_base_dir, 'DATASET-1.zarr'))
dataset_2.to_zarr(os.path.join(local_base_dir, 'DATASET-2.zarr'))
global STORES_CONFIG_PATH
_, STORES_CONFIG_PATH = new_temp_file(suffix='_stores.yaml')
with open(STORES_CONFIG_PATH, 'w') as stream:
yaml.dump(
{
'test': {
'title': 'Local test store',
'store_id': 'file',
'store_params': {
'root': local_base_dir
}
}
}, stream)
print(f' * Store base directory: {local_base_dir}')
print(f' * Store configuration: {STORES_CONFIG_PATH}')
def setUp(self) -> None:
self.data_store = MemoryDataStore()
self.data_store.data_dict.update({
'cube_1':
new_cube(variables=dict(B01=0.4, B02=0.5)),
'cube_2':
new_cube(variables=dict(B03=0.4, B04=0.5))
})
def test_write_and_read_and_delete(self):
self.store.s3.mkdir(BUCKET_NAME)
dataset_1 = new_cube(variables=dict(a=4.1, b=7.4))
dataset_2 = new_cube(variables=dict(c=5.2, d=8.5))
dataset_3 = new_cube(variables=dict(e=6.3, f=9.6))
# Write 3 cubes
self.store.write_data(dataset_1, data_id='cube-1.zarr')
self.store.write_data(dataset_2, data_id='cube-2.zarr')
self.store.write_data(dataset_3, data_id='cube-3.zarr')
self.assertTrue(self.store.has_data('cube-1.zarr'))
self.assertTrue(self.store.has_data('cube-2.zarr'))
self.assertTrue(self.store.has_data('cube-3.zarr'))
self.assertIn(('cube-1.zarr', None), set(self.store.get_data_ids()))
self.assertIn(('cube-2.zarr', None), set(self.store.get_data_ids()))
self.assertIn(('cube-3.zarr', None), set(self.store.get_data_ids()))
self.assertEqual(3, len(set(self.store.get_data_ids())))
# Open the 3 written cubes
opened_dataset_1 = self.store.open_data('cube-1.zarr')
opened_dataset_2 = self.store.open_data('cube-2.zarr')
opened_dataset_3 = self.store.open_data('cube-3.zarr')
self.assertIsInstance(opened_dataset_1, xr.Dataset)
self.assertIsInstance(opened_dataset_2, xr.Dataset)
self.assertIsInstance(opened_dataset_3, xr.Dataset)
self.assertEqual(set(dataset_1.data_vars),
set(opened_dataset_1.data_vars))
self.assertEqual(set(dataset_2.data_vars),
set(opened_dataset_2.data_vars))
self.assertEqual(set(dataset_3.data_vars),
set(opened_dataset_3.data_vars))
# Try overwriting existing cube 1
dataset_4 = new_cube(variables=dict(g=7.4, h=10.7))
with self.assertRaises(DataStoreError) as cm:
self.store.write_data(dataset_4, data_id='cube-1.zarr')
self.assertEqual("path '' contains a group", f'{cm.exception}')
# replace=True should do the trick
self.store.write_data(dataset_4, data_id='cube-1.zarr', replace=True)
opened_dataset_4 = self.store.open_data('cube-1.zarr')
self.assertEqual(set(dataset_4.data_vars),
set(opened_dataset_4.data_vars))
# Try deleting cube 1
self.store.delete_data('cube-1.zarr')
self.assertEqual({('cube-2.zarr', None), ('cube-3.zarr', None)},
set(self.store.get_data_ids()))
self.assertFalse(self.store.has_data('cube-1.zarr'))
# Now it should be save to also write with replace=False
self.store.write_data(dataset_1, data_id='cube-1.zarr', replace=False)
def setUp(self) -> None:
self.old_global_data_dict = MemoryDataStore.replace_global_data_dict({
'cube_1':
new_cube(variables=dict(B01=0.4, B02=0.5)),
'cube_2':
new_cube(variables=dict(B03=0.4, B04=0.5)),
'ds_1':
xr.Dataset()
})
self._store = new_data_store('memory')
self.assertIsInstance(self.store, MemoryDataStore)
def test_strip_cube(self):
cube = new_cube(variables=dict(chl=0.6, tsm=0.9, flags=16))
self.assertIs(cube, strip_cube(cube))
cube = new_cube(variables=dict(chl=0.6, tsm=0.9, flags=16))
cube_subset = cube.sel(time=slice('1990-01-01', '1991-01-01'))
stripped_cube = strip_cube(cube_subset)
self.assertEqual(set(), set(stripped_cube.data_vars))
cube = new_cube(variables=dict(chl=0.6, tsm=0.9, flags=16))
cube_subset = cube.sel(lat=0, lon=0, method='nearest')
stripped_cube = strip_cube(cube_subset)
self.assertEqual(set(), set(stripped_cube.data_vars))
def new(cls, **kwargs) -> xr.Dataset:
"""
Create a new empty cube. Useful for testing.
Refer to :func:`xcube.core.new.new_cube` for details.
"""
return new_cube(**kwargs)
def setUp(self) -> None:
cube = new_cube(width=360,
height=180,
time_periods=6,
variables=dict(analysed_sst=275.3, analysis_error=2.1))
cube = chunk_dataset(cube, dict(time=3, lat=90, lon=90))
self.cube = cube
def test_new_ml_dataset_descriptor(self):
cube = new_cube(variables=dict(a=4.1, b=7.4))
ml_cube = BaseMultiLevelDataset(cube)
descriptor = new_data_descriptor('cube', ml_cube)
self.assertExpectedDescriptor(descriptor,
MultiLevelDatasetDescriptor,
'mldataset')
def setUp(self) -> None:
with open('_request.json', 'w') as fp:
json.dump(MainTest.REQUEST, fp)
with open('_request.yaml', 'w') as fp:
yaml.dump(MainTest.REQUEST, fp)
self.saved_cube_memory = MemoryDataStore.replace_global_data_dict(
{'S2L2A': new_cube(variables=dict(B01=0.1, B02=0.2, B03=0.3))})
def test_chunks_are_smaller_than_sizes(self):
cube1 = new_cube(variables=dict(chl=0.6, tsm=0.9, flags=16))
for var in cube1.variables.values():
self.assertIsNone(var.chunks)
rc = CubeRechunker()
cube2, gm, cc = rc.transform_cube(
cube1, GridMapping.from_dataset(cube1),
CubeConfig(chunks=dict(time=2, lat=100, lon=200)))
self.assertIsInstance(cube2, xr.Dataset)
self.assertIsInstance(gm, GridMapping)
self.assertIsInstance(cc, CubeConfig)
self.assertEqual(cube1.attrs, cube2.attrs)
self.assertEqual({'time': 2, 'lat': 100, 'lon': 200}, cc.chunks)
self.assertEqual(set(cube1.coords), set(cube2.coords))
self.assertEqual(set(cube1.data_vars), set(cube2.data_vars))
for k, v in cube2.coords.items():
if v.chunks is not None:
self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}')
self.assertNotIn('chunks', v.encoding)
# self.assertIn('chunks', v.encoding)
# self.assertEqual([v.sizes[d] for d in v.dims],
# v.encoding['chunks'])
for k, v in cube2.data_vars.items():
self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}')
self.assertEqual(((2, 2, 1), (100, 80), (200, 160)), v.chunks)
self.assertNotIn('chunks', v.encoding)
def test_non_empty_cube_subset(self):
dataset = new_cube(variables=dict(a=9, b=0.2))
cube = dataset.xcube.cube
self.assertIsInstance(cube, xr.Dataset)
self.assertEqual(set(dataset.data_vars), set(cube.data_vars))
gm = dataset.xcube.gm
self.assertIsInstance(gm, GridMapping)
def test_chunks_are_larger_than_sizes(self):
cube1 = new_cube(variables=dict(chl=0.6, tsm=0.9, flags=16))
for var in cube1.variables.values():
self.assertIsNone(var.chunks)
rc = CubeRechunker()
cube2, gm, cc = rc.transform_cube(
cube1, GridMapping.from_dataset(cube1),
CubeConfig(chunks=dict(time=64, lat=512, lon=512)))
self.assertIsInstance(cube2, xr.Dataset)
self.assertIsInstance(gm, GridMapping)
self.assertIsInstance(cc, CubeConfig)
self.assertEqual(cube1.attrs, cube2.attrs)
self.assertEqual(set(cube1.coords), set(cube2.coords))
self.assertEqual(set(cube1.data_vars), set(cube2.data_vars))
for k, v in cube2.coords.items():
if v.chunks is not None:
self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}')
self.assertNotIn('chunks', v.encoding)
for k, v in cube2.data_vars.items():
self.assertIsInstance(v.chunks, tuple, msg=f'{k!r}={v!r}')
self.assertEqual(((5, ), (180, ), (360, )), v.chunks)
self.assertNotIn('chunks', v.encoding)
def setUpClass(cls) -> None:
rimraf(S3_BUCKET)
os.mkdir(S3_BUCKET)
cube = new_cube(time_periods=3,
variables=dict(precipitation=0.9,
temperature=278.3)).chunk(dict(time=1, lat=90, lon=90))
write_cube(cube, TEST_CUBE_1, "zarr", cube_asserted=True)
write_cube(cube, TEST_CUBE_2, "zarr", cube_asserted=True)
def test_get_time_range_from_data_with_irregular_data(self):
cube = new_cube(drop_bounds=True, time_freq='M')
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-31T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-06-30T00:00:00',
pd.Timestamp(time_range[1]).isoformat())
def test_get_time_range_from_data(self):
cube = new_cube(drop_bounds=True)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())
def test_get_time_range_from_attrs(self):
cube = new_cube()
time_range = get_time_range_from_attrs(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())
def setUp(self):
self._rm_outputs()
dataset = new_cube(variables=dict(precipitation=0.4,
temperature=275.2,
soil_moisture=0.5),
time_periods=self.time_periods())
dataset.to_netcdf(TEST_NC_FILE, mode="w")
dataset.to_zarr(TEST_ZARR_DIR, mode="w")
def test_cube_stays_cube(self):
dataset = new_cube(variables=dict(a=1, b=2, c=3))
cube, grid_mapping, rest = decode_cube(dataset)
self.assertIs(dataset, cube)
self.assertIsInstance(grid_mapping, GridMapping)
self.assertTrue(grid_mapping.crs.is_geographic)
self.assertIsInstance(rest, xr.Dataset)
self.assertEqual(set(), set(rest.data_vars))
def test_get_dataset_indexes_for_single_cell(self):
dataset = new_cube(width=360, height=180, drop_bounds=True)
cell = dataset.isel(time=2, lat=20, lon=30)
with self.assertRaises(ValueError) as cm:
get_dataset_indexes(cell, "lon", np.array([-149.5]))
self.assertEqual(
"cannot determine cell boundaries"
" for coordinate variable 'lon' of size 1", f"{cm.exception}")
def setUp(self) -> None:
rimraf(self.TEST_CUBE)
cube = new_cube(time_periods=3,
variables=dict(precipitation=np.nan,
temperature=np.nan)).chunk(
dict(time=1, lat=90, lon=90))
write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
def test_verify_cube(self):
cube = new_cube()
self.assertEqual([], verify_cube(cube))
ds = cube.drop("time")
self.assertEqual(["missing time coordinate variable"], verify_cube(ds))
ds = ds.drop("lat")
self.assertEqual(["missing spatial x,y coordinate variables",
"missing time coordinate variable"], verify_cube(ds))
def test_get_time_range_from_data_start_and_end_time_arrays(self):
cube = new_cube(drop_bounds=True, use_cftime=True, time_dtype=None)
time_range = get_time_range_from_data(cube)
self.assertIsNotNone(time_range)
self.assertEqual('2010-01-01T00:00:00',
pd.Timestamp(time_range[0]).isoformat())
self.assertEqual('2010-01-06T00:00:00',
pd.Timestamp(time_range[1]).isoformat())
def test_verify_ok(self):
cube = new_cube(variables=dict(precipitation=0.5))
write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
result = self.invoke_cli(['verify', self.TEST_CUBE])
self.assertEqual(0, result.exit_code)
self.assertEqual(
"Opening cube from 'test.zarr'...\n"
"INPUT is a valid cube.\n", result.stdout)
def setUp(self) -> None:
self.cube = new_cube(width=2000,
height=1000,
x_start=0,
x_res=0.0025,
y_start=50,
y_res=0.0025,
inverse_y=True,
variables={'a': 0.5})
def _new_test_cube(self):
return new_cube(width=2000,
height=1000,
x_start=0.0,
y_start=50.0,
x_res=4.0 / 2000,
time_start="2010-01-01",
time_periods=20,
variables=dict(precipitation=0.6, temperature=276.2))
def test_write_dataset(self):
dataset = new_cube()
try:
write_dataset(dataset, TEST_NC_FILE_2)
self.assertTrue(os.path.isfile(TEST_NC_FILE_2))
finally:
if os.path.isfile(TEST_NC_FILE_2):
os.remove(TEST_NC_FILE_2)
def test_new_cube_with_const_vars(self):
cube = new_cube(variables=dict(sst=274.4, chl=10.31))
self.assertIn('sst', cube)
self.assertIn('chl', cube)
import numpy as np
np.testing.assert_almost_equal(cube.sst.values,
np.full((5, 180, 360), 274.4))
np.testing.assert_almost_equal(cube.chl.values,
np.full((5, 180, 360), 10.31))
def test_verify_cube_coord_equidistance_reverse_lat(self):
ds = new_cube()
ds['lat'] = np.flip(ds.lat)
ds['lat_bnds'] = (['lat', 'bnds'], np.flip(ds.lat_bnds))
ds['lon'] = np.roll(ds.lon, 90)
ds['lon_bnds'] = (['lon', 'bnds'], np.roll(ds.lon_bnds, 180))
self.assertEqual([], verify_cube(ds))
def test_verify_cube_coord_equidistance(self):
ds = new_cube()
ds['lat'] = ds['lat'] + np.random.rand(len(ds['lat']))
ds['lat_bnds'][:, 0] = ds['lat_bnds'][:, 0] + np.random.rand(len(ds['lat_bnds'][:, 0]))
ds['lat_bnds'][:, 1] = ds['lat_bnds'][:, 1] + np.random.rand(len(ds['lat_bnds'][:, 1]))
self.assertEqual(["coordinate variable 'lat' is not equidistant",
"coordinate variable 'lat_bnds' is not equidistant"],
verify_cube(ds))
