def test_verify_ok(self): cube = new_cube(variables=dict(precipitation=0.5)) write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True) result = self.invoke_cli(['verify', self.TEST_CUBE]) self.assertEqual(0, result.exit_code) self.assertEqual( "Opening cube from 'test.zarr'...\n" "INPUT is a valid cube.\n", result.stdout)
def setUpClass(cls) -> None: rimraf(S3_BUCKET) os.mkdir(S3_BUCKET) cube = new_cube(time_periods=3, variables=dict(precipitation=0.9, temperature=278.3)).chunk(dict(time=1, lat=90, lon=90)) write_cube(cube, TEST_CUBE_1, "zarr", cube_asserted=True) write_cube(cube, TEST_CUBE_2, "zarr", cube_asserted=True)
def setUp(self) -> None: rimraf(self.TEST_CUBE) cube = new_cube(time_periods=3, variables=dict(precipitation=np.nan, temperature=np.nan)).chunk( dict(time=1, lat=90, lon=90)) write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
def test_upload_to_s3(self): with moto.mock_s3(): s3_conn = boto3.client('s3') s3_conn.create_bucket(Bucket='upload_bucket', ACL='public-read') client_kwargs = {'provider_access_key_id': 'test_fake_id', 'provider_secret_access_key': 'test_fake_secret'} ds1 = xr.open_zarr('examples/serve/demo/cube-1-250-250.zarr') write_cube(ds1, 'https://s3.amazonaws.com/upload_bucket/cube-1-250-250.zarr', 'zarr', client_kwargs=client_kwargs) self.assertIn('cube-1-250-250.zarr/.zattrs', s3_conn.list_objects(Bucket='upload_bucket')['Contents'][0]['Key'])
def write_levels(ml_dataset: MultiLevelDataset, levels_path: str, client_kwargs: Dict[str, Any] = None): tile_w, tile_h = ml_dataset.tile_grid.tile_size chunks = dict(time=1, lat=tile_h, lon=tile_w) for level in range(ml_dataset.num_levels): level_dataset = ml_dataset.get_dataset(level) level_dataset = level_dataset.chunk(chunks) print(f'writing level {level + 1}...') write_cube(level_dataset, f'{levels_path}/{level}.zarr', 'zarr', client_kwargs=client_kwargs) print(f'written level {level + 1}')
def _resample_in_time(input_path: str = None, variables: Sequence[str] = None, metadata: Dict[str, Any] = None, output_path: str = DEFAULT_OUTPUT_PATH, output_format: str = None, methods: Sequence[str] = (DEFAULT_RESAMPLING_METHOD,), frequency: str = DEFAULT_RESAMPLING_FREQUENCY, offset: str = None, base: int = DEFAULT_RESAMPLING_BASE, interp_kind: str = DEFAULT_INTERPOLATION_KIND, tolerance: str = None, dry_run: bool = False, monitor=None): from xcube.core.dsio import guess_dataset_format from xcube.core.dsio import open_cube from xcube.core.dsio import write_cube from xcube.core.resample import resample_in_time from xcube.core.update import update_dataset_chunk_encoding if not output_format: output_format = guess_dataset_format(output_path) monitor(f'Opening cube from {input_path!r}...') with open_cube(input_path) as ds: monitor('Resampling...') agg_ds = resample_in_time(ds, frequency=frequency, method=methods, offset=offset, base=base, interp_kind=interp_kind, tolerance=tolerance, time_chunk_size=1, var_names=variables, metadata=metadata) agg_ds = update_dataset_chunk_encoding(agg_ds, chunk_sizes={}, format_name=output_format, in_place=True) monitor(f'Writing resampled cube to {output_path!r}...') if not dry_run: write_cube(agg_ds, output_path, output_format, cube_asserted=True) monitor(f'Done.')
def test_upload_to_s3_via_bucket_path(self): with moto.mock_s3(): s3_conn = boto3.client('s3') s3_conn.create_bucket(Bucket='upload_bucket', ACL='public-read') client_kwargs = { 'provider_access_key_id': 'test_fake_id', 'provider_secret_access_key': 'test_fake_secret' } zarr_path = os.path.join( os.path.dirname(__file__), '../../examples/serve/demo/cube-1-250-250.zarr') ds1 = xr.open_zarr(zarr_path) write_cube(ds1, 's3://upload_bucket/cube-1-250-250.zarr', 'zarr', client_kwargs=client_kwargs) self.assertIn( 'cube-1-250-250.zarr/.zattrs', s3_conn.list_objects( Bucket='upload_bucket')['Contents'][0]['Key'])
def test_verify_failure(self): cube = new_cube(variables=dict(precipitation=0.5)) cube["chl"] = xr.DataArray(np.random.rand(cube.dims["lat"], cube.dims["lon"]), dims=("lat", "lon"), coords=dict(lat=cube.lat, lon=cube.lon)) write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True) result = self.invoke_cli(['verify', self.TEST_CUBE]) self.assertEqual(3, result.exit_code) self.assertEqual( "Opening cube from 'test.zarr'...\n" "INPUT is not a valid cube due to the following reasons:\n" "- dimensions of data variable 'chl' must be ('time', ..., 'lat', 'lon')," " but were ('lat', 'lon') for 'chl'\n" "- dimensions of all data variables must be same, but found ('lat', 'lon')" " for 'chl' and ('time', 'lat', 'lon') for 'precipitation'\n" "- all data variables must have same chunk sizes, but found ((90, 90), (360,))" " for 'chl' and ((3, 2), (90, 90), (180, 180)) for 'precipitation'\n", result.stdout)
def write(self, output_path: str, format_name: str = None, **kwargs) -> xr.Dataset: """ Write this cube to *output_path*. If *format* is not provided it will be guessed from *output_path*. :param output_path: output path :param format_name: format, e.g. "zarr" or "netcdf4" :param kwargs: format-specific keyword arguments :return: the input dataset """ return write_cube(self._dataset, output_path, format_name=format_name, **kwargs)
def test_write_to_and_read_from_s3(self): s3_conn = boto3.client('s3', endpoint_url=MOTO_SERVER_ENDPOINT_URL) s3_conn.create_bucket(Bucket='upload_bucket', ACL='public-read') s3_kwargs = dict(key='test_fake_id', secret='test_fake_secret') s3_client_kwargs = {'endpoint_url': MOTO_SERVER_ENDPOINT_URL} ds1 = new_cube(width=36, height=18, variables=dict(chl=0.5, tsm=0.2)) write_cube(ds1, 'upload_bucket/cube-1-250-250.zarr', format_name='zarr', s3_kwargs=s3_kwargs, s3_client_kwargs=s3_client_kwargs) ds2 = open_cube('upload_bucket/cube-1-250-250.zarr', format_name='zarr', s3_kwargs=s3_kwargs, s3_client_kwargs=s3_client_kwargs) self.assertEqual(set(ds1.coords), set(ds2.coords)) self.assertEqual(set(ds1.data_vars), set(ds2.data_vars))