def test_chunk_dataset(self): dataset = new_test_dataset([ "2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05" ], precipitation=0.4, temperature=275.2) chunked_dataset = chunk_dataset(dataset, chunk_sizes=dict(time=1, lat=10, lon=20), format_name="zarr") self.assertEqual({'chunks': (1, 10, 20)}, chunked_dataset.precipitation.encoding) self.assertEqual({'chunks': (1, 10, 20)}, chunked_dataset.temperature.encoding) chunked_dataset = chunk_dataset(dataset, chunk_sizes=dict(time=1, lat=20, lon=40), format_name="netcdf4") self.assertEqual({'chunksizes': (1, 20, 40)}, chunked_dataset.precipitation.encoding) self.assertEqual({'chunksizes': (1, 20, 40)}, chunked_dataset.temperature.encoding) chunked_dataset = chunk_dataset(dataset, chunk_sizes=dict(time=1, lat=20, lon=40)) self.assertEqual({}, chunked_dataset.precipitation.encoding) self.assertEqual({}, chunked_dataset.temperature.encoding)
def test_mask_dataset_for_chunked_input(self): cube = chunk_dataset(self.cube, chunk_sizes=dict(time=1, lat=90, lon=90)) cube = mask_dataset_by_geometry(cube, self.triangle) self._assert_clipped_dataset_has_basic_props(cube) self.assertEqual(((1, 1, 1, 1, 1), (4, ), (7, )), cube.temp.chunks) self.assertEqual(((1, 1, 1, 1, 1), (4, ), (7, )), cube.precip.chunks)
def test_unchunk_dataset(self): dataset = new_test_dataset([ "2010-01-01", "2010-01-02", "2010-01-03", "2010-01-04", "2010-01-05" ], precipitation=0.4, temperature=275.2) for var in dataset.data_vars.values(): var.encoding.update({ "chunks": (5, 180, 360), "_FillValue": -999.0 }) chunked_dataset = chunk_dataset(dataset, format_name="zarr") self.assertEqual({"_FillValue": -999.0}, chunked_dataset.precipitation.encoding) self.assertEqual({"_FillValue": -999.0}, chunked_dataset.temperature.encoding)
def setUp(self): rimraf(self.TEST_ZARR) cube = new_cube(variables=dict(A=0.5, B=-1.5)) cube = chunk_dataset(cube, chunk_sizes=dict(time=1, lat=90, lon=90), format_name=FORMAT_NAME_ZARR) cube.to_zarr(self.TEST_ZARR) self.chunked_a_files = { '.zarray', '.zattrs', '0.0.0', '0.0.1', '0.0.2', '0.0.3', '0.1.0', '0.1.1', '0.1.2', '0.1.3', '1.0.0', '1.0.1', '1.0.2', '1.0.3', '1.1.0', '1.1.1', '1.1.2', '1.1.3', '2.0.0', '2.0.1', '2.0.2', '2.0.3', '2.1.0', '2.1.1', '2.1.2', '2.1.3', '3.0.0', '3.0.1', '3.0.2', '3.0.3', '3.1.0', '3.1.1', '3.1.2', '3.1.3', '4.0.0', '4.0.1', '4.0.2', '4.0.3', '4.1.0', '4.1.1', '4.1.2', '4.1.3' } self.chunked_b_files = self.chunked_a_files self.chunked_time_files = { '.zarray', '.zattrs', '0', '1', '2', '3', '4' } self.chunked_lat_files = {'.zattrs', '.zarray', '0', '1'} self.chunked_lon_files = {'.zattrs', '.zarray', '0', '1', '2', '3'}
import os import os.path import unittest from typing import Set from xcube.api import new_cube from xcube.util.chunk import chunk_dataset from xcube.util.constants import FORMAT_NAME_ZARR from xcube.util.dsio import rimraf from xcube.util.optimize import optimize_dataset TEST_CUBE = chunk_dataset(new_cube(time_periods=3, variables=dict(A=0.5, B=-1.5)), chunk_sizes=dict(time=1, lat=180, lon=360), format_name=FORMAT_NAME_ZARR) TEST_CUBE_ZARR = 'test.zarr' TEST_CUBE_FILE_SET = { '.zattrs', '.zgroup', 'A/.zarray', 'A/.zattrs', 'A/0.0.0', 'A/1.0.0', 'A/2.0.0', 'B/.zarray', 'B/.zattrs', 'B/0.0.0', 'B/1.0.0', 'B/2.0.0', 'lat/.zarray', 'lat/.zattrs', 'lat/0', 'lat_bnds/.zarray', 'lat_bnds/.zattrs', 'lat_bnds/0.0', 'lon/.zarray', 'lon/.zattrs', 'lon/0', 'lon_bnds/.zarray', 'lon_bnds/.zattrs', 'lon_bnds/0.0', 'time/.zarray', 'time/.zattrs', 'time/0', 'time/1', 'time/2', 'time_bnds/.zarray', 'time_bnds/.zattrs', 'time_bnds/0.0', 'time_bnds/1.0', 'time_bnds/2.0' } class OptimizeDatasetTest(unittest.TestCase): def setUp(self):