def test_from_and_to_dict(self):
config = CubeConfig.from_dict(
dict(dataset_name='S2L2A',
band_names=('B01', 'B02', 'B03'),
geometry=(10.11, 54.17, 10.14, 54.19),
spatial_res=0.00001,
tile_size=(512, 512),
time_range=('2019-01-01', '2019-01-02')))
self.assertEqual(
{
'band_names': ('B01', 'B02', 'B03'),
'band_sample_types':
None,
'band_units':
None,
'collection_id':
None,
'crs':
'http://www.opengis.net/def/crs/EPSG/0/4326',
'dataset_name':
'S2L2A',
'four_d':
False,
'geometry': (10.11, 54.17, 10.14072, 54.19048),
'spatial_res':
1e-05,
'tile_size': (512, 512),
'time_period':
None,
'time_range':
('2019-01-01T00:00:00+00:00', '2019-01-02T00:00:00+00:00'),
'time_tolerance':
'0 days 00:10:00'
}, config.as_dict())
def test_time_deltas(self):
config = CubeConfig.from_dict(
dict(dataset_name='S2L2A',
band_names=('B01', 'B02', 'B03'),
bbox=(10.11, 54.17, 10.14, 54.19),
spatial_res=0.00001,
time_range=('2019-01-01', '2019-01-02')))
self.assertEqual(None, config.time_period)
self.assertEqual(pd.Timedelta('0 days 00:10:00'),
config.time_tolerance)
config = CubeConfig.from_dict(
dict(dataset_name='S2L2A',
band_names=('B01', 'B02', 'B03'),
bbox=(10.11, 54.17, 10.14, 54.19),
spatial_res=0.00001,
time_period='8D',
time_range=('2019-01-01', '2019-01-02')))
self.assertEqual(pd.Timedelta('8 days 00:00:00'), config.time_period)
self.assertEqual(None, config.time_tolerance)
config = CubeConfig.from_dict(
dict(dataset_name='S2L2A',
band_names=('B01', 'B02', 'B03'),
bbox=(10.11, 54.17, 10.14, 54.19),
spatial_res=0.00001,
time_tolerance='1H',
time_range=('2019-01-01', '2019-01-02')))
self.assertEqual(None, config.time_period)
self.assertEqual(pd.Timedelta('0 days 01:00:00'),
config.time_tolerance)
config = CubeConfig.from_dict(
dict(dataset_name='S2L2A',
band_names=('B01', 'B02', 'B03'),
bbox=(10.11, 54.17, 10.14, 54.19),
spatial_res=0.00001,
time_period='8D',
time_tolerance='1H',
time_range=('2019-01-01', '2019-01-02')))
self.assertEqual(pd.Timedelta('8 days 00:00:00'), config.time_period)
self.assertEqual(pd.Timedelta('0 days 01:00:00'),
config.time_tolerance)
def test_from_dict_invalids(self):
with self.assertRaises(ValueError) as cm:
CubeConfig.from_dict(
dict(dataset_name='S2L2A',
band_names=('B01', 'B02', 'B03'),
geometry=(10.11, 54.17, 10.14, 54.19),
special_res=0.00001,
tile_size=(512, 512),
time_range=('2019-01-01', '2019-01-02')))
self.assertEqual(
"Found invalid parameter 'special_res' in cube configuration",
f'{cm.exception}')
with self.assertRaises(ValueError) as cm:
CubeConfig.from_dict(
dict(dataset_name='S2L2A',
band_names=('B01', 'B02', 'B03'),
geometrix=(10.11, 54.17, 10.14, 54.19),
special_res=0.00001,
tile_size=(512, 512),
time_range=('2019-01-01', '2019-01-02')))
self.assertEqual(
"Found invalid parameters in cube configuration: 'geometrix', 'special_res'",
f'{cm.exception}')
def gen(dataset, output_path, cube_config_path, source_config_path,
dest_config_path, band_names, tile_size, geometry, spatial_res, crs,
time_range, time_period, time_tolerance, four_d, verbose):
"""
Generate a data cube from SentinelHub.
By default, the command will create a ZARR dataset with 3D arrays
for each band e.g. "B01", "B02" with dimensions "time", "lat", "lon".
Use option "--4d" to write a single 4D array "band_data"
with dimensions "time", "lat", "lon", "band".
"""
import os.path
import time
import xarray as xr
from xcube_sh.config import CubeConfig
from xcube_sh.observers import Observers
from xcube_sh.sentinelhub import SentinelHub
from xcube_sh.store import SentinelHubStore
if os.path.exists(output_path):
raise click.ClickException(
f'Output {output_path} already exists. Move it away first.')
cube_config_dict = _load_config_dict(cube_config_path)
source_config_dict = _load_config_dict(source_config_path)
dest_config_dict = _load_config_dict(dest_config_path)
cube_config_dict.update({
k: v
for k, v in dict(dataset_name=dataset,
band_names=band_names,
tile_size=tile_size,
geometry=geometry,
spatial_res=spatial_res,
crs=crs,
time_range=time_range,
time_period=time_period,
time_tolerance=time_tolerance,
four_d=four_d).items() if v is not None
})
cube_config = CubeConfig.from_dict(cube_config_dict,
exception_type=click.ClickException)
# TODO: validate source_config_dict
sentinel_hub = SentinelHub(**source_config_dict)
print(f'Writing cube to {output_path}...')
# TODO: validate dest_config_dict
# TODO: use dest_config_dict and output_path to determine actuial output, which may be AWS S3
t0 = time.perf_counter()
store = SentinelHubStore(sentinel_hub, cube_config)
request_collector = Observers.request_collector()
store.add_observer(request_collector)
if verbose:
store.add_observer(Observers.request_dumper())
cube = xr.open_zarr(store)
cube.to_zarr(output_path, **dest_config_dict)
duration = time.perf_counter() - t0
print(f"Cube written to {output_path}, took {'%.2f' % duration} seconds.")
if verbose:
request_collector.stats.dump()
def gen(request: Optional[str],
dataset_name: Optional[str],
band_names: Optional[Tuple],
tile_size: Optional[str],
geometry: Optional[str],
spatial_res: Optional[float],
crs: Optional[str],
time_range: Optional[str],
time_period: Optional[str],
time_tolerance: Optional[str],
output_path: Optional[str],
four_d: bool,
verbose: bool):
"""
Generate a data cube from SENTINEL Hub.
By default, the command will create a Zarr dataset with 3D arrays
for each band e.g. "B01", "B02" with dimensions "time", "lat", "lon".
Use option "--4d" to write a single 4D array "band_data"
with dimensions "time", "lat", "lon", "band".
Please use command "xcube sh req" to generate example request files that can be passed as REQUEST.
REQUEST may have JSON or YAML format.
You can also pipe a JSON request into this command. In this case
"""
import json
import os.path
import sys
import xarray as xr
from xcube.core.dsio import write_dataset
from xcube.util.perf import measure_time
from xcube_sh.config import CubeConfig
from xcube_sh.observers import Observers
from xcube_sh.sentinelhub import SentinelHub
from xcube_sh.chunkstore import SentinelHubChunkStore
if request:
request_dict = _load_request(request)
elif not sys.stdin.isatty():
request_dict = json.load(sys.stdin)
else:
request_dict = {}
cube_config_dict = request_dict.get('cube_config', {})
_overwrite_config_params(cube_config_dict,
dataset_name=dataset_name,
band_names=band_names if band_names else None, # because of multiple=True
tile_size=tile_size,
geometry=geometry,
spatial_res=spatial_res,
crs=crs,
time_range=time_range,
time_period=time_period,
time_tolerance=time_tolerance,
four_d=four_d)
input_config_dict = request_dict.get('input_config', {})
if 'datastore_id' in input_config_dict:
input_config_dict = dict(input_config_dict)
datastore_id = input_config_dict.pop('datastore_id')
if datastore_id != 'sentinelhub':
warnings.warn(f'Unknown datastore_id={datastore_id!r} encountered in request. Ignoring it...')
# _overwrite_config_params(input_config_dict, ...)
# TODO: validate input_config_dict
output_config_dict = request_dict.get('output_config', {})
_overwrite_config_params(output_config_dict,
path=output_path)
# TODO: validate output_config_dict
cube_config = CubeConfig.from_dict(cube_config_dict,
exception_type=click.ClickException)
if 'path' in output_config_dict:
output_path = output_config_dict.pop('path')
else:
output_path = DEFAULT_GEN_OUTPUT_PATH
if not _is_bucket_url(output_path) and os.path.exists(output_path):
raise click.ClickException(f'Output {output_path} already exists. Move it away first.')
sentinel_hub = SentinelHub(**input_config_dict)
print(f'Writing cube to {output_path}...')
with measure_time() as cm:
store = SentinelHubChunkStore(sentinel_hub, cube_config)
request_collector = Observers.request_collector()
store.add_observer(request_collector)
if verbose:
store.add_observer(Observers.request_dumper())
cube = xr.open_zarr(store)
if _is_bucket_url(output_path):
client_kwargs = {k: output_config_dict.pop(k)
for k in ('provider_access_key_id', 'provider_secret_access_key')
if k in output_config_dict}
write_dataset(cube, output_path, format_name='zarr', client_kwargs=client_kwargs, **output_config_dict)
else:
write_dataset(cube, output_path, **output_config_dict)
print(f"Cube written to {output_path}, took {'%.2f' % cm.duration} seconds.")
if verbose:
request_collector.stats.dump()