def test_verify_ok(self): cube = new_cube(variables=dict(precipitation=0.5)) write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True) result = self.invoke_cli(['verify', self.TEST_CUBE]) self.assertEqual(0, result.exit_code) self.assertEqual( "Opening cube from 'test.zarr'...\n" "INPUT is a valid cube.\n", result.stdout)
def setUp(self) -> None: rimraf(self.TEST_CUBE) cube = new_cube(time_periods=3, variables=dict(precipitation=np.nan, temperature=np.nan)).chunk( dict(time=1, lat=90, lon=90)) write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
def setUpClass(cls) -> None: rimraf(S3_BUCKET) os.mkdir(S3_BUCKET) cube = new_cube(time_periods=3, variables=dict(precipitation=0.9, temperature=278.3)).chunk(dict(time=1, lat=90, lon=90)) write_cube(cube, TEST_CUBE_1, "zarr", cube_asserted=True) write_cube(cube, TEST_CUBE_2, "zarr", cube_asserted=True)
def test_new_cube_with_const_vars(self): cube = new_cube(variables=dict(sst=274.4, chl=10.31)) self.assertIn('sst', cube) self.assertIn('chl', cube) import numpy as np np.testing.assert_almost_equal(cube.sst.values, np.full((5, 180, 360), 274.4)) np.testing.assert_almost_equal(cube.chl.values, np.full((5, 180, 360), 10.31))
def make_cube(self, start_date, num_days: int) -> xr.Dataset: cube = new_cube(time_periods=num_days, time_freq='1D', time_start=start_date, variables=dict(precipitation=0.1, temperature=270.5, soil_moisture=0.2)) chunk_sizes = dict(time=1, lat=90, lon=90) cube = chunk_dataset(cube, chunk_sizes, format_name='zarr') return cube
def setUp(self): shape = 25, 180, 360 dims = 'time', 'lat', 'lon' self.ts_a = np.linspace(1, 25, 25) self.ts_a_count = np.array(25 * [100]) self.ts_a_stdev = np.array(25 * [0.0]) self.ts_b = np.linspace(0, 1, 25) self.ts_b_count = np.array(25 * [100]) self.ts_b_stdev = np.array(25 * [0.0]) self.cube = new_cube(time_periods=25, variables=dict( A=xr.DataArray(np.broadcast_to(self.ts_a.reshape(25, 1, 1), shape), dims=dims), B=xr.DataArray(np.broadcast_to(self.ts_b.reshape(25, 1, 1), shape), dims=dims) )) self.cube = self.cube.chunk(chunks=dict(time=1, lat=180, lon=180))
def test_local(self): cube = new_cube(time_periods=10, time_start='2019-01-01', variables=dict(precipitation=0.1, temperature=270.5, soil_moisture=0.2)) cube = chunk_dataset(cube, dict(time=1, lat=90, lon=90), format_name='zarr') cube.to_zarr(self.CUBE_PATH) cube.close() diagnostic_store = DiagnosticStore( zarr.DirectoryStore(self.CUBE_PATH), logging_observer(log_path='local-cube.log')) xr.open_zarr(diagnostic_store)
def test_vars_to_dim(self): dataset = new_cube( variables=dict(precipitation=0.4, temperature=275.2)) ds = vars_to_dim(dataset) self.assertIn("var", ds.dims) self.assertEqual(2, ds.dims["var"]) self.assertIn("var", ds.coords) self.assertIn("data", ds.data_vars) var_names = ds["var"] self.assertEqual(("var", ), var_names.dims) self.assertTrue(hasattr(var_names, "encoding")) self.assertEqual(2, len(var_names)) self.assertIn("precipitation", str(var_names[0])) self.assertIn("temperature", str(var_names[1]))
def setUp(self) -> None: width = 16 height = 8 spatial_res = 360 / width lon_min = -2 * spatial_res + 0.5 * spatial_res lat_min = -2 * spatial_res + 0.5 * spatial_res lon_max = lon_min + 6 * spatial_res lat_max = lat_min + 3 * spatial_res self.triangle = shapely.geometry.Polygon( ((lon_min, lat_min), (lon_max, lat_min), (0.5 * (lon_max + lon_min), lat_max), (lon_min, lat_min))) self.cube = new_cube(width=width, height=height, spatial_res=spatial_res, drop_bounds=True, variables=dict(temp=273.9, precip=0.9))
def test_verify_failure(self): cube = new_cube(variables=dict(precipitation=0.5)) cube["chl"] = xr.DataArray(np.random.rand(cube.dims["lat"], cube.dims["lon"]), dims=("lat", "lon"), coords=dict(lat=cube.lat, lon=cube.lon)) write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True) result = self.invoke_cli(['verify', self.TEST_CUBE]) self.assertEqual(3, result.exit_code) self.assertEqual( "Opening cube from 'test.zarr'...\n" "INPUT is not a valid cube due to the following reasons:\n" "- dimensions of data variable 'chl' must be ('time', ..., 'lat', 'lon')," " but were ('lat', 'lon') for 'chl'\n" "- dimensions of all data variables must be same, but found ('lat', 'lon')" " for 'chl' and ('time', 'lat', 'lon') for 'precipitation'\n" "- all data variables must have same chunk sizes, but found ((90, 90), (360,))" " for 'chl' and ((3, 2), (90, 90), (180, 180)) for 'precipitation'\n", result.stdout)
def test_new_cube_with_bounds(self): cube = new_cube() self.assertEqual({ 'lon': 360, 'lat': 180, 'time': 5, 'bnds': 2 }, cube.dims) self.assertEqual(-179.5, float(cube.lon[0])) self.assertEqual(179.5, float(cube.lon[-1])) self.assertEqual(-89.5, float(cube.lat[0])) self.assertEqual(89.5, float(cube.lat[-1])) self.assertEqual(-180., float(cube.lon_bnds[0, 0])) self.assertEqual(-179., float(cube.lon_bnds[0, 1])) self.assertEqual(179., float(cube.lon_bnds[-1, 0])) self.assertEqual(180., float(cube.lon_bnds[-1, 1])) self.assertEqual(-90., float(cube.lat_bnds[0, 0])) self.assertEqual(-89., float(cube.lat_bnds[0, 1])) self.assertEqual(89., float(cube.lat_bnds[-1, 0])) self.assertEqual(90., float(cube.lat_bnds[-1, 1]))
def setUp(self): rimraf(self.TEST_ZARR) cube = new_cube(variables=dict(A=0.5, B=-1.5)) cube = chunk_dataset(cube, chunk_sizes=dict(time=1, lat=90, lon=90), format_name=FORMAT_NAME_ZARR) cube.to_zarr(self.TEST_ZARR) self.chunked_a_files = { '.zarray', '.zattrs', '0.0.0', '0.0.1', '0.0.2', '0.0.3', '0.1.0', '0.1.1', '0.1.2', '0.1.3', '1.0.0', '1.0.1', '1.0.2', '1.0.3', '1.1.0', '1.1.1', '1.1.2', '1.1.3', '2.0.0', '2.0.1', '2.0.2', '2.0.3', '2.1.0', '2.1.1', '2.1.2', '2.1.3', '3.0.0', '3.0.1', '3.0.2', '3.0.3', '3.1.0', '3.1.1', '3.1.2', '3.1.3', '4.0.0', '4.0.1', '4.0.2', '4.0.3', '4.1.0', '4.1.1', '4.1.2', '4.1.3' } self.chunked_b_files = self.chunked_a_files self.chunked_time_files = { '.zarray', '.zattrs', '0', '1', '2', '3', '4' } self.chunked_lat_files = {'.zattrs', '.zarray', '0', '1'} self.chunked_lon_files = {'.zattrs', '.zarray', '0', '1', '2', '3'}
def test_new_cube_with_func_vars(self): def sst_func(t, y, x): return 274.4 + t * 0.5 def chl_func(t, y, x): return 10.31 + y * 0.1 def aot_func(t, y, x): return 0.88 + x * 0.05 cube = new_cube(width=16, height=8, variables=dict(sst=sst_func, chl=chl_func, aot=aot_func)) self.assertIn('sst', cube) self.assertIn('chl', cube) self.assertIn('aot', cube) np.testing.assert_almost_equal(cube.sst[0, :, :].values, np.full((8, 16), 274.4)) np.testing.assert_almost_equal(cube.chl[:, 0, :].values, np.full((5, 16), 10.31)) np.testing.assert_almost_equal(cube.aot[:, :, 0].values, np.full((5, 8), 0.88)) np.testing.assert_almost_equal( cube.sst.isel(lon=3, lat=2).values, np.array([274.4, 274.9, 275.4, 275.9, 276.4])) np.testing.assert_almost_equal( cube.chl.isel(time=3, lon=4).values, np.array([10.31, 10.41, 10.51, 10.61, 10.71, 10.81, 10.91, 11.01])) np.testing.assert_almost_equal( cube.aot.isel(time=1, lat=2).values, np.array([ 0.88, 0.93, 0.98, 1.03, 1.08, 1.13, 1.18, 1.23, 1.28, 1.33, 1.38, 1.43, 1.48, 1.53, 1.58, 1.63 ]))
import os import os.path import unittest from typing import Set from xcube.api import new_cube from xcube.util.chunk import chunk_dataset from xcube.util.constants import FORMAT_NAME_ZARR from xcube.util.dsio import rimraf from xcube.util.optimize import optimize_dataset TEST_CUBE = chunk_dataset(new_cube(time_periods=3, variables=dict(A=0.5, B=-1.5)), chunk_sizes=dict(time=1, lat=180, lon=360), format_name=FORMAT_NAME_ZARR) TEST_CUBE_ZARR = 'test.zarr' TEST_CUBE_FILE_SET = { '.zattrs', '.zgroup', 'A/.zarray', 'A/.zattrs', 'A/0.0.0', 'A/1.0.0', 'A/2.0.0', 'B/.zarray', 'B/.zattrs', 'B/0.0.0', 'B/1.0.0', 'B/2.0.0', 'lat/.zarray', 'lat/.zattrs', 'lat/0', 'lat_bnds/.zarray', 'lat_bnds/.zattrs', 'lat_bnds/0.0', 'lon/.zarray', 'lon/.zattrs', 'lon/0', 'lon_bnds/.zarray', 'lon_bnds/.zattrs', 'lon_bnds/0.0', 'time/.zarray', 'time/.zattrs', 'time/0', 'time/1', 'time/2', 'time_bnds/.zarray', 'time_bnds/.zattrs', 'time_bnds/0.0', 'time_bnds/1.0', 'time_bnds/2.0' } class OptimizeDatasetTest(unittest.TestCase): def setUp(self):
def test_new_cube_without_bounds(self): cube = new_cube(drop_bounds=True) self.assertEqual({'lon': 360, 'lat': 180, 'time': 5}, cube.dims)