def test_verify_ok(self):
cube = new_cube(variables=dict(precipitation=0.5))
write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
result = self.invoke_cli(['verify', self.TEST_CUBE])
self.assertEqual(0, result.exit_code)
self.assertEqual(
"Opening cube from 'test.zarr'...\n"
"INPUT is a valid cube.\n", result.stdout)
def setUp(self) -> None:
rimraf(self.TEST_CUBE)
cube = new_cube(time_periods=3,
variables=dict(precipitation=np.nan,
temperature=np.nan)).chunk(
dict(time=1, lat=90, lon=90))
write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
def setUpClass(cls) -> None:
rimraf(S3_BUCKET)
os.mkdir(S3_BUCKET)
cube = new_cube(time_periods=3,
variables=dict(precipitation=0.9,
temperature=278.3)).chunk(dict(time=1, lat=90, lon=90))
write_cube(cube, TEST_CUBE_1, "zarr", cube_asserted=True)
write_cube(cube, TEST_CUBE_2, "zarr", cube_asserted=True)
def test_new_cube_with_const_vars(self):
cube = new_cube(variables=dict(sst=274.4, chl=10.31))
self.assertIn('sst', cube)
self.assertIn('chl', cube)
import numpy as np
np.testing.assert_almost_equal(cube.sst.values,
np.full((5, 180, 360), 274.4))
np.testing.assert_almost_equal(cube.chl.values,
np.full((5, 180, 360), 10.31))
def make_cube(self, start_date, num_days: int) -> xr.Dataset:
cube = new_cube(time_periods=num_days,
time_freq='1D',
time_start=start_date,
variables=dict(precipitation=0.1,
temperature=270.5,
soil_moisture=0.2))
chunk_sizes = dict(time=1, lat=90, lon=90)
cube = chunk_dataset(cube, chunk_sizes, format_name='zarr')
return cube
def setUp(self):
shape = 25, 180, 360
dims = 'time', 'lat', 'lon'
self.ts_a = np.linspace(1, 25, 25)
self.ts_a_count = np.array(25 * [100])
self.ts_a_stdev = np.array(25 * [0.0])
self.ts_b = np.linspace(0, 1, 25)
self.ts_b_count = np.array(25 * [100])
self.ts_b_stdev = np.array(25 * [0.0])
self.cube = new_cube(time_periods=25,
variables=dict(
A=xr.DataArray(np.broadcast_to(self.ts_a.reshape(25, 1, 1), shape), dims=dims),
B=xr.DataArray(np.broadcast_to(self.ts_b.reshape(25, 1, 1), shape), dims=dims)
))
self.cube = self.cube.chunk(chunks=dict(time=1, lat=180, lon=180))
def test_local(self):
cube = new_cube(time_periods=10,
time_start='2019-01-01',
variables=dict(precipitation=0.1,
temperature=270.5,
soil_moisture=0.2))
cube = chunk_dataset(cube,
dict(time=1, lat=90, lon=90),
format_name='zarr')
cube.to_zarr(self.CUBE_PATH)
cube.close()
diagnostic_store = DiagnosticStore(
zarr.DirectoryStore(self.CUBE_PATH),
logging_observer(log_path='local-cube.log'))
xr.open_zarr(diagnostic_store)
def test_vars_to_dim(self):
dataset = new_cube(
variables=dict(precipitation=0.4, temperature=275.2))
ds = vars_to_dim(dataset)
self.assertIn("var", ds.dims)
self.assertEqual(2, ds.dims["var"])
self.assertIn("var", ds.coords)
self.assertIn("data", ds.data_vars)
var_names = ds["var"]
self.assertEqual(("var", ), var_names.dims)
self.assertTrue(hasattr(var_names, "encoding"))
self.assertEqual(2, len(var_names))
self.assertIn("precipitation", str(var_names[0]))
self.assertIn("temperature", str(var_names[1]))
def setUp(self) -> None:
width = 16
height = 8
spatial_res = 360 / width
lon_min = -2 * spatial_res + 0.5 * spatial_res
lat_min = -2 * spatial_res + 0.5 * spatial_res
lon_max = lon_min + 6 * spatial_res
lat_max = lat_min + 3 * spatial_res
self.triangle = shapely.geometry.Polygon(
((lon_min, lat_min), (lon_max, lat_min),
(0.5 * (lon_max + lon_min), lat_max), (lon_min, lat_min)))
self.cube = new_cube(width=width,
height=height,
spatial_res=spatial_res,
drop_bounds=True,
variables=dict(temp=273.9, precip=0.9))
def test_verify_failure(self):
cube = new_cube(variables=dict(precipitation=0.5))
cube["chl"] = xr.DataArray(np.random.rand(cube.dims["lat"],
cube.dims["lon"]),
dims=("lat", "lon"),
coords=dict(lat=cube.lat, lon=cube.lon))
write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
result = self.invoke_cli(['verify', self.TEST_CUBE])
self.assertEqual(3, result.exit_code)
self.assertEqual(
"Opening cube from 'test.zarr'...\n"
"INPUT is not a valid cube due to the following reasons:\n"
"- dimensions of data variable 'chl' must be ('time', ..., 'lat', 'lon'),"
" but were ('lat', 'lon') for 'chl'\n"
"- dimensions of all data variables must be same, but found ('lat', 'lon')"
" for 'chl' and ('time', 'lat', 'lon') for 'precipitation'\n"
"- all data variables must have same chunk sizes, but found ((90, 90), (360,))"
" for 'chl' and ((3, 2), (90, 90), (180, 180)) for 'precipitation'\n",
result.stdout)
def test_new_cube_with_bounds(self):
cube = new_cube()
self.assertEqual({
'lon': 360,
'lat': 180,
'time': 5,
'bnds': 2
}, cube.dims)
self.assertEqual(-179.5, float(cube.lon[0]))
self.assertEqual(179.5, float(cube.lon[-1]))
self.assertEqual(-89.5, float(cube.lat[0]))
self.assertEqual(89.5, float(cube.lat[-1]))
self.assertEqual(-180., float(cube.lon_bnds[0, 0]))
self.assertEqual(-179., float(cube.lon_bnds[0, 1]))
self.assertEqual(179., float(cube.lon_bnds[-1, 0]))
self.assertEqual(180., float(cube.lon_bnds[-1, 1]))
self.assertEqual(-90., float(cube.lat_bnds[0, 0]))
self.assertEqual(-89., float(cube.lat_bnds[0, 1]))
self.assertEqual(89., float(cube.lat_bnds[-1, 0]))
self.assertEqual(90., float(cube.lat_bnds[-1, 1]))
def setUp(self):
rimraf(self.TEST_ZARR)
cube = new_cube(variables=dict(A=0.5, B=-1.5))
cube = chunk_dataset(cube,
chunk_sizes=dict(time=1, lat=90, lon=90),
format_name=FORMAT_NAME_ZARR)
cube.to_zarr(self.TEST_ZARR)
self.chunked_a_files = {
'.zarray', '.zattrs', '0.0.0', '0.0.1', '0.0.2', '0.0.3', '0.1.0',
'0.1.1', '0.1.2', '0.1.3', '1.0.0', '1.0.1', '1.0.2', '1.0.3',
'1.1.0', '1.1.1', '1.1.2', '1.1.3', '2.0.0', '2.0.1', '2.0.2',
'2.0.3', '2.1.0', '2.1.1', '2.1.2', '2.1.3', '3.0.0', '3.0.1',
'3.0.2', '3.0.3', '3.1.0', '3.1.1', '3.1.2', '3.1.3', '4.0.0',
'4.0.1', '4.0.2', '4.0.3', '4.1.0', '4.1.1', '4.1.2', '4.1.3'
}
self.chunked_b_files = self.chunked_a_files
self.chunked_time_files = {
'.zarray', '.zattrs', '0', '1', '2', '3', '4'
}
self.chunked_lat_files = {'.zattrs', '.zarray', '0', '1'}
self.chunked_lon_files = {'.zattrs', '.zarray', '0', '1', '2', '3'}
def test_new_cube_with_func_vars(self):
def sst_func(t, y, x):
return 274.4 + t * 0.5
def chl_func(t, y, x):
return 10.31 + y * 0.1
def aot_func(t, y, x):
return 0.88 + x * 0.05
cube = new_cube(width=16,
height=8,
variables=dict(sst=sst_func,
chl=chl_func,
aot=aot_func))
self.assertIn('sst', cube)
self.assertIn('chl', cube)
self.assertIn('aot', cube)
np.testing.assert_almost_equal(cube.sst[0, :, :].values,
np.full((8, 16), 274.4))
np.testing.assert_almost_equal(cube.chl[:, 0, :].values,
np.full((5, 16), 10.31))
np.testing.assert_almost_equal(cube.aot[:, :, 0].values,
np.full((5, 8), 0.88))
np.testing.assert_almost_equal(
cube.sst.isel(lon=3, lat=2).values,
np.array([274.4, 274.9, 275.4, 275.9, 276.4]))
np.testing.assert_almost_equal(
cube.chl.isel(time=3, lon=4).values,
np.array([10.31, 10.41, 10.51, 10.61, 10.71, 10.81, 10.91, 11.01]))
np.testing.assert_almost_equal(
cube.aot.isel(time=1, lat=2).values,
np.array([
0.88, 0.93, 0.98, 1.03, 1.08, 1.13, 1.18, 1.23, 1.28, 1.33,
1.38, 1.43, 1.48, 1.53, 1.58, 1.63
]))
import os
import os.path
import unittest
from typing import Set
from xcube.api import new_cube
from xcube.util.chunk import chunk_dataset
from xcube.util.constants import FORMAT_NAME_ZARR
from xcube.util.dsio import rimraf
from xcube.util.optimize import optimize_dataset
TEST_CUBE = chunk_dataset(new_cube(time_periods=3,
variables=dict(A=0.5, B=-1.5)),
chunk_sizes=dict(time=1, lat=180, lon=360),
format_name=FORMAT_NAME_ZARR)
TEST_CUBE_ZARR = 'test.zarr'
TEST_CUBE_FILE_SET = {
'.zattrs', '.zgroup', 'A/.zarray', 'A/.zattrs', 'A/0.0.0', 'A/1.0.0',
'A/2.0.0', 'B/.zarray', 'B/.zattrs', 'B/0.0.0', 'B/1.0.0', 'B/2.0.0',
'lat/.zarray', 'lat/.zattrs', 'lat/0', 'lat_bnds/.zarray',
'lat_bnds/.zattrs', 'lat_bnds/0.0', 'lon/.zarray', 'lon/.zattrs', 'lon/0',
'lon_bnds/.zarray', 'lon_bnds/.zattrs', 'lon_bnds/0.0', 'time/.zarray',
'time/.zattrs', 'time/0', 'time/1', 'time/2', 'time_bnds/.zarray',
'time_bnds/.zattrs', 'time_bnds/0.0', 'time_bnds/1.0', 'time_bnds/2.0'
}
class OptimizeDatasetTest(unittest.TestCase):
def setUp(self):
def test_new_cube_without_bounds(self):
cube = new_cube(drop_bounds=True)
self.assertEqual({'lon': 360, 'lat': 180, 'time': 5}, cube.dims)
