def test_chunk_dataset(self):
dataset = new_test_dataset([
"2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04",
"2010-01-05"
],
precipitation=0.4,
temperature=275.2)
chunked_dataset = chunk_dataset(dataset,
chunk_sizes=dict(time=1,
lat=10,
lon=20),
format_name="zarr")
self.assertEqual({'chunks': (1, 10, 20)},
chunked_dataset.precipitation.encoding)
self.assertEqual({'chunks': (1, 10, 20)},
chunked_dataset.temperature.encoding)
chunked_dataset = chunk_dataset(dataset,
chunk_sizes=dict(time=1,
lat=20,
lon=40),
format_name="netcdf4")
self.assertEqual({'chunksizes': (1, 20, 40)},
chunked_dataset.precipitation.encoding)
self.assertEqual({'chunksizes': (1, 20, 40)},
chunked_dataset.temperature.encoding)
chunked_dataset = chunk_dataset(dataset,
chunk_sizes=dict(time=1,
lat=20,
lon=40))
self.assertEqual({}, chunked_dataset.precipitation.encoding)
self.assertEqual({}, chunked_dataset.temperature.encoding)
def test_mask_dataset_for_chunked_input(self):
cube = chunk_dataset(self.cube,
chunk_sizes=dict(time=1, lat=90, lon=90))
cube = mask_dataset_by_geometry(cube, self.triangle)
self._assert_clipped_dataset_has_basic_props(cube)
self.assertEqual(((1, 1, 1, 1, 1), (4, ), (7, )), cube.temp.chunks)
self.assertEqual(((1, 1, 1, 1, 1), (4, ), (7, )), cube.precip.chunks)
def test_unchunk_dataset(self):
dataset = new_test_dataset([
"2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04",
"2010-01-05"
],
precipitation=0.4,
temperature=275.2)
for var in dataset.data_vars.values():
var.encoding.update({
"chunks": (5, 180, 360),
"_FillValue": -999.0
})
chunked_dataset = chunk_dataset(dataset, format_name="zarr")
self.assertEqual({"_FillValue": -999.0},
chunked_dataset.precipitation.encoding)
self.assertEqual({"_FillValue": -999.0},
chunked_dataset.temperature.encoding)
def setUp(self):
rimraf(self.TEST_ZARR)
cube = new_cube(variables=dict(A=0.5, B=-1.5))
cube = chunk_dataset(cube,
chunk_sizes=dict(time=1, lat=90, lon=90),
format_name=FORMAT_NAME_ZARR)
cube.to_zarr(self.TEST_ZARR)
self.chunked_a_files = {
'.zarray', '.zattrs', '0.0.0', '0.0.1', '0.0.2', '0.0.3', '0.1.0',
'0.1.1', '0.1.2', '0.1.3', '1.0.0', '1.0.1', '1.0.2', '1.0.3',
'1.1.0', '1.1.1', '1.1.2', '1.1.3', '2.0.0', '2.0.1', '2.0.2',
'2.0.3', '2.1.0', '2.1.1', '2.1.2', '2.1.3', '3.0.0', '3.0.1',
'3.0.2', '3.0.3', '3.1.0', '3.1.1', '3.1.2', '3.1.3', '4.0.0',
'4.0.1', '4.0.2', '4.0.3', '4.1.0', '4.1.1', '4.1.2', '4.1.3'
}
self.chunked_b_files = self.chunked_a_files
self.chunked_time_files = {
'.zarray', '.zattrs', '0', '1', '2', '3', '4'
}
self.chunked_lat_files = {'.zattrs', '.zarray', '0', '1'}
self.chunked_lon_files = {'.zattrs', '.zarray', '0', '1', '2', '3'}
import os
import os.path
import unittest
from typing import Set
from xcube.api import new_cube
from xcube.util.chunk import chunk_dataset
from xcube.util.constants import FORMAT_NAME_ZARR
from xcube.util.dsio import rimraf
from xcube.util.optimize import optimize_dataset
TEST_CUBE = chunk_dataset(new_cube(time_periods=3,
variables=dict(A=0.5, B=-1.5)),
chunk_sizes=dict(time=1, lat=180, lon=360),
format_name=FORMAT_NAME_ZARR)
TEST_CUBE_ZARR = 'test.zarr'
TEST_CUBE_FILE_SET = {
'.zattrs', '.zgroup', 'A/.zarray', 'A/.zattrs', 'A/0.0.0', 'A/1.0.0',
'A/2.0.0', 'B/.zarray', 'B/.zattrs', 'B/0.0.0', 'B/1.0.0', 'B/2.0.0',
'lat/.zarray', 'lat/.zattrs', 'lat/0', 'lat_bnds/.zarray',
'lat_bnds/.zattrs', 'lat_bnds/0.0', 'lon/.zarray', 'lon/.zattrs', 'lon/0',
'lon_bnds/.zarray', 'lon_bnds/.zattrs', 'lon_bnds/0.0', 'time/.zarray',
'time/.zattrs', 'time/0', 'time/1', 'time/2', 'time_bnds/.zarray',
'time_bnds/.zattrs', 'time_bnds/0.0', 'time_bnds/1.0', 'time_bnds/2.0'
}
class OptimizeDatasetTest(unittest.TestCase):
def setUp(self):