def test_verify_ok(self):
cube = new_cube(variables=dict(precipitation=0.5))
write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
result = self.invoke_cli(['verify', self.TEST_CUBE])
self.assertEqual(0, result.exit_code)
self.assertEqual(
"Opening cube from 'test.zarr'...\n"
"INPUT is a valid cube.\n", result.stdout)
def setUpClass(cls) -> None:
rimraf(S3_BUCKET)
os.mkdir(S3_BUCKET)
cube = new_cube(time_periods=3,
variables=dict(precipitation=0.9,
temperature=278.3)).chunk(dict(time=1, lat=90, lon=90))
write_cube(cube, TEST_CUBE_1, "zarr", cube_asserted=True)
write_cube(cube, TEST_CUBE_2, "zarr", cube_asserted=True)
def setUp(self) -> None:
rimraf(self.TEST_CUBE)
cube = new_cube(time_periods=3,
variables=dict(precipitation=np.nan,
temperature=np.nan)).chunk(
dict(time=1, lat=90, lon=90))
write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
def test_upload_to_s3(self):
with moto.mock_s3():
s3_conn = boto3.client('s3')
s3_conn.create_bucket(Bucket='upload_bucket', ACL='public-read')
client_kwargs = {'provider_access_key_id': 'test_fake_id', 'provider_secret_access_key': 'test_fake_secret'}
ds1 = xr.open_zarr('examples/serve/demo/cube-1-250-250.zarr')
write_cube(ds1, 'https://s3.amazonaws.com/upload_bucket/cube-1-250-250.zarr', 'zarr',
client_kwargs=client_kwargs)
self.assertIn('cube-1-250-250.zarr/.zattrs',
s3_conn.list_objects(Bucket='upload_bucket')['Contents'][0]['Key'])
def write_levels(ml_dataset: MultiLevelDataset,
levels_path: str,
client_kwargs: Dict[str, Any] = None):
tile_w, tile_h = ml_dataset.tile_grid.tile_size
chunks = dict(time=1, lat=tile_h, lon=tile_w)
for level in range(ml_dataset.num_levels):
level_dataset = ml_dataset.get_dataset(level)
level_dataset = level_dataset.chunk(chunks)
print(f'writing level {level + 1}...')
write_cube(level_dataset,
f'{levels_path}/{level}.zarr',
'zarr',
client_kwargs=client_kwargs)
print(f'written level {level + 1}')
def _resample_in_time(input_path: str = None,
variables: Sequence[str] = None,
metadata: Dict[str, Any] = None,
output_path: str = DEFAULT_OUTPUT_PATH,
output_format: str = None,
methods: Sequence[str] = (DEFAULT_RESAMPLING_METHOD,),
frequency: str = DEFAULT_RESAMPLING_FREQUENCY,
offset: str = None,
base: int = DEFAULT_RESAMPLING_BASE,
interp_kind: str = DEFAULT_INTERPOLATION_KIND,
tolerance: str = None,
dry_run: bool = False,
monitor=None):
from xcube.core.dsio import guess_dataset_format
from xcube.core.dsio import open_cube
from xcube.core.dsio import write_cube
from xcube.core.resample import resample_in_time
from xcube.core.update import update_dataset_chunk_encoding
if not output_format:
output_format = guess_dataset_format(output_path)
monitor(f'Opening cube from {input_path!r}...')
with open_cube(input_path) as ds:
monitor('Resampling...')
agg_ds = resample_in_time(ds,
frequency=frequency,
method=methods,
offset=offset,
base=base,
interp_kind=interp_kind,
tolerance=tolerance,
time_chunk_size=1,
var_names=variables,
metadata=metadata)
agg_ds = update_dataset_chunk_encoding(agg_ds,
chunk_sizes={},
format_name=output_format,
in_place=True)
monitor(f'Writing resampled cube to {output_path!r}...')
if not dry_run:
write_cube(agg_ds, output_path, output_format, cube_asserted=True)
monitor(f'Done.')
def test_upload_to_s3_via_bucket_path(self):
with moto.mock_s3():
s3_conn = boto3.client('s3')
s3_conn.create_bucket(Bucket='upload_bucket', ACL='public-read')
client_kwargs = {
'provider_access_key_id': 'test_fake_id',
'provider_secret_access_key': 'test_fake_secret'
}
zarr_path = os.path.join(
os.path.dirname(__file__),
'../../examples/serve/demo/cube-1-250-250.zarr')
ds1 = xr.open_zarr(zarr_path)
write_cube(ds1,
's3://upload_bucket/cube-1-250-250.zarr',
'zarr',
client_kwargs=client_kwargs)
self.assertIn(
'cube-1-250-250.zarr/.zattrs',
s3_conn.list_objects(
Bucket='upload_bucket')['Contents'][0]['Key'])
def test_verify_failure(self):
cube = new_cube(variables=dict(precipitation=0.5))
cube["chl"] = xr.DataArray(np.random.rand(cube.dims["lat"],
cube.dims["lon"]),
dims=("lat", "lon"),
coords=dict(lat=cube.lat, lon=cube.lon))
write_cube(cube, self.TEST_CUBE, "zarr", cube_asserted=True)
result = self.invoke_cli(['verify', self.TEST_CUBE])
self.assertEqual(3, result.exit_code)
self.assertEqual(
"Opening cube from 'test.zarr'...\n"
"INPUT is not a valid cube due to the following reasons:\n"
"- dimensions of data variable 'chl' must be ('time', ..., 'lat', 'lon'),"
" but were ('lat', 'lon') for 'chl'\n"
"- dimensions of all data variables must be same, but found ('lat', 'lon')"
" for 'chl' and ('time', 'lat', 'lon') for 'precipitation'\n"
"- all data variables must have same chunk sizes, but found ((90, 90), (360,))"
" for 'chl' and ((3, 2), (90, 90), (180, 180)) for 'precipitation'\n",
result.stdout)
def write(self, output_path: str, format_name: str = None, **kwargs) -> xr.Dataset:
"""
Write this cube to *output_path*.
If *format* is not provided it will be guessed from *output_path*.
:param output_path: output path
:param format_name: format, e.g. "zarr" or "netcdf4"
:param kwargs: format-specific keyword arguments
:return: the input dataset
"""
return write_cube(self._dataset, output_path, format_name=format_name, **kwargs)
def test_write_to_and_read_from_s3(self):
s3_conn = boto3.client('s3', endpoint_url=MOTO_SERVER_ENDPOINT_URL)
s3_conn.create_bucket(Bucket='upload_bucket', ACL='public-read')
s3_kwargs = dict(key='test_fake_id', secret='test_fake_secret')
s3_client_kwargs = {'endpoint_url': MOTO_SERVER_ENDPOINT_URL}
ds1 = new_cube(width=36, height=18, variables=dict(chl=0.5, tsm=0.2))
write_cube(ds1,
'upload_bucket/cube-1-250-250.zarr',
format_name='zarr',
s3_kwargs=s3_kwargs,
s3_client_kwargs=s3_client_kwargs)
ds2 = open_cube('upload_bucket/cube-1-250-250.zarr',
format_name='zarr',
s3_kwargs=s3_kwargs,
s3_client_kwargs=s3_client_kwargs)
self.assertEqual(set(ds1.coords), set(ds2.coords))
self.assertEqual(set(ds1.data_vars), set(ds2.data_vars))
