def test_get_data_multi_binary(self):
with open(REQUEST_MULTI_JSON, 'r') as fp:
request = json.load(fp)
sentinel_hub = SentinelHub()
# TODO (forman): discuss with Primoz how to effectively do multi-bands request
t1 = time.perf_counter()
response = sentinel_hub.get_data(request,
mime_type='application/octet-stream')
t2 = time.perf_counter()
print(f"test_get_data_multi_binary: took {t2 - t1} secs")
_write_zarr_array(self.RESPONSE_MULTI_ZARR, response.content, 0,
(512, 512, 4), '<f4')
sentinel_hub.close()
zarr_array = zarr.open_array(self.RESPONSE_MULTI_ZARR)
self.assertEqual((1, 512, 512, 4), zarr_array.shape)
self.assertEqual((1, 512, 512, 4), zarr_array.chunks)
np_array = np.array(zarr_array).astype(np.float32)
self.assertEqual(np.float32, np_array.dtype)
np.testing.assert_almost_equal(
np.array([
0.6425, 0.6676, 0.5922, 0.5822, 0.5735, 0.4921, 0.5902, 0.6518,
0.5825, 0.5321
],
dtype=np.float32), np_array[0, 0, 0:10, 0])
np.testing.assert_almost_equal(
np.array([
0.8605, 0.8528, 0.8495, 0.8378, 0.8143, 0.7959, 0.7816, 0.7407,
0.7182, 0.7326
],
dtype=np.float32), np_array[0, 511, -10:, 0])
def test_it(self):
sentinel_hub = SentinelHub(api_url='https://creodias.sentinel-hub.com')
# sentinel_hub = SentinelHub(api_url='https://services-uswest2.sentinel-hub.com')
# sentinel_hub = SentinelHub()
collections = sentinel_hub.collections()
self.assertIsInstance(collections, list)
self.assertTrue(len(collections) >= 1)
sentinel_hub.close()
def test_get_data_multi(self):
with open(REQUEST_MULTI_JSON, 'r') as fp:
request = json.load(fp)
sentinel_hub = SentinelHub()
t1 = time.perf_counter()
response = sentinel_hub.get_data(request)
t2 = time.perf_counter()
print(f"test_get_data_multi: took {t2 - t1} secs")
with open(self.RESPONSE_MULTI_TAR, 'wb') as fp:
fp.write(response.content)
sentinel_hub.close()
def test_variable_names(self):
expected_band_names = [
'B01', 'B02', 'B03', 'B04', 'B05', 'B06', 'B07', 'B08', 'B8A',
'B09', 'B10', 'B11', 'B12', 'viewZenithMean', 'viewAzimuthMean',
'sunZenithAngles', 'sunAzimuthAngles'
]
sentinel_hub = SentinelHub(session=SessionMock({
'get': {
'https://services.sentinel-hub.com/api/v1/process/dataset/S2L2A/bands':
{
'data': expected_band_names
}
}
}))
self.assertEqual(expected_band_names, sentinel_hub.band_names('S2L2A'))
sentinel_hub.close()
def test_fetch_tiles(self):
instance_id = os.environ.get('SH_INSTANCE_ID')
x1 = 10.00 # degree
y1 = 54.27 # degree
x2 = 11.00 # degree
y2 = 54.60 # degree
t1 = '2019-09-17'
t2 = '2019-10-17'
tile_features = SentinelHub.fetch_tile_features(
instance_id=instance_id,
feature_type_name='S2.TILE',
bbox=(x1, y1, x2, y2),
time_range=(t1, t2))
self.assertEqual(32, len(tile_features))
for feature in tile_features:
self.assertEqual('Feature', feature.get('type'))
self.assertIn('geometry', feature)
self.assertIn('properties', feature)
properties = feature['properties']
self.assertIn('id', properties)
self.assertIn('path', properties)
self.assertIn('date', properties)
self.assertIn('time', properties)
def test_open_cube_with_illegal_kwargs(self):
with self.assertRaises(ValueError) as cm:
open_cube(
cube_config=cube_config,
sentinel_hub=SentinelHub(),
api_url=
"https://creodias.sentinel-hub.com/api/v1/catalog/collections")
self.assertEqual('unexpected keyword-arguments: api_url',
f'{cm.exception}')
def test_token_info(self):
expected_token_info = {
'name': 'Norman Fomferra',
'email': '*****@*****.**',
'active': True
}
sentinel_hub = SentinelHub(session=SessionMock({
'get': {
'https://services.sentinel-hub.com/oauth/tokeninfo':
expected_token_info
}
}))
self.assertEqual(
expected_token_info, {
k: v
for k, v in sentinel_hub.token_info.items()
if k in ['name', 'email', 'active']
})
sentinel_hub.close()
def test_dataset_names(self):
expected_dataset_names = ["DEM", "S2L1C", "S2L2A", "CUSTOM", "S1GRD"]
sentinel_hub = SentinelHub(session=SessionMock({
'get': {
'https://services.sentinel-hub.com/api/v1/process/dataset': {
"data": expected_dataset_names
}
}
}))
self.assertEqual(expected_dataset_names, sentinel_hub.dataset_names)
def test_get_features(self):
features = SentinelHub().get_features(
collection_name='sentinel-1-grd',
bbox=(13, 45, 14, 46),
time_range=('2019-12-10T00:00:00Z', '2019-12-11T00:00:00Z'))
# print(json.dumps(features, indent=2))
self.assertEqual(8, len(features))
for feature in features:
self.assertIn('properties', feature)
properties = feature['properties']
self.assertIn('datetime', properties)
def info(datasets: List[str] = None):
"""
Print SentinelHub metadata info. If DATASETS (names of datasets) are not present,
the list of available dataset names are returned. Otherwise,
the the variables of the given datasets are returned.
"""
from xcube_sh.sentinelhub import SentinelHub
sentinel_hub = SentinelHub()
import json
if not datasets:
response = dict(datasets=sentinel_hub.dataset_names)
else:
response = dict()
for dataset_name in datasets:
band_names = sentinel_hub.band_names(dataset_name)
bands = dict()
for band_name in band_names:
bands[band_name] = sentinel_hub.METADATA.dataset_band(dataset_name, band_name, default={})
response[dataset_name] = bands
print(json.dumps(response, indent=2))
def gen(dataset, output_path, band_names, tile_size, geometry, spatial_res,
crs, time_range, time_period, time_tolerance, four_d, verbose):
"""
Generate a data cube from SentinelHub.
By default, the command will create a ZARR dataset with 3D arrays
for each band e.g. "B01", "B02" with dimensions "time", "lat", "lon".
Use option "--4d" to write a single 4D array "band_data"
with dimensions "time", "lat", "lon", "band".
"""
import os.path
import time
import xarray as xr
from xcube_sh.config import CubeConfig
from xcube_sh.observers import Observers
from xcube_sh.sentinelhub import SentinelHub
from xcube_sh.store import SentinelHubStore
if os.path.exists(output_path):
raise click.ClickException(
f'Output {output_path} already exists. Move it away first.')
cube_config = CubeConfig(dataset_name=dataset,
band_names=band_names,
tile_size=tile_size,
geometry=geometry,
spatial_res=spatial_res,
crs=crs,
time_range=time_range,
time_period=time_period,
time_tolerance=time_tolerance,
four_d=four_d,
exception_type=click.ClickException)
sentinel_hub = SentinelHub()
print(f'Writing cube to {output_path}...')
t0 = time.perf_counter()
store = SentinelHubStore(sentinel_hub, cube_config)
request_collector = Observers.request_collector()
store.add_observer(request_collector)
if verbose:
store.add_observer(Observers.request_dumper())
cube = xr.open_zarr(store)
cube.to_zarr(output_path)
duration = time.perf_counter() - t0
print(f"Cube written to {output_path}, took {'%.2f' % duration} seconds.")
if verbose:
request_collector.stats.dump()
def test_get_features(self):
properties = [{
'datetime': '2019-10-02T10:35:47Z'
}, {
'datetime': '2019-10-04T10:25:47Z'
}, {
'datetime': '2019-10-05T10:45:36Z'
}, {
'datetime': '2019-10-05T10:45:44Z'
}]
expected_features = [dict(properties=p) for p in properties]
sentinel_hub = SentinelHub(session=SessionMock({
'post': {
'https://services.sentinel-hub.com/api/v1/catalog/search':
dict(type='FeatureCollection', features=expected_features)
}
}))
self.assertEqual(
expected_features,
sentinel_hub.get_features(collection_name='sentinel-2-l2a',
bbox=(12, 53, 13, 54),
time_range=('2019-10-02', '2019-10-05')))
sentinel_hub.close()
def test_new_data_request_multi_byod(self):
request = SentinelHub.new_data_request(
'CUSTOM', ['RED', 'GREEN', 'BLUE'], (512, 305),
bbox=(1545577, 5761986, 1705367, 5857046),
crs='http://www.opengis.net/def/crs/EPSG/0/3857',
band_sample_types='UINT8',
collection_id='1a3ab057-3c51-447c-9f85-27d4b633b3f5')
# with open(REQUEST_MULTI_JSON), 'w') as fp:
# json.dump(request, fp, indent=2)
with open(REQUEST_MULTI_BYOD_JSON, 'r') as fp:
expected_request = json.load(fp)
self.assertEqual(expected_request, request)
def test_new_data_request_single_byod(self):
request = SentinelHub.new_data_request(
'CUSTOM', ['RED'], (512, 305),
crs="http://www.opengis.net/def/crs/EPSG/0/3857",
bbox=(1545577, 5761986, 1705367, 5857046),
band_sample_types="UINT8",
collection_id='1a3ab057-3c51-447c-9f85-27d4b633b3f5')
# with open(os.path.join(REQUEST_SINGLE_JSON, 'w') as fp:
# json.dump(request, fp, indent=2)
with open(REQUEST_SINGLE_BYOD_JSON, 'r') as fp:
expected_request = json.load(fp)
self.assertEqual(expected_request, request)
def test_dataset_names(self):
expected_dataset_names = ["DEM", "S2L1C", "S2L2A", "CUSTOM", "S1GRD"]
sentinel_hub = SentinelHub(session=SessionMock({
'get': {
'https://services.sentinel-hub.com/configuration/v1/datasets':
[{
'id': "DEM"
}, {
'id': "S2L1C"
}, {
'id': "S2L2A"
}, {
'id': "CUSTOM"
}, {
'id': "S1GRD"
}]
}
}))
self.assertEqual(expected_dataset_names, sentinel_hub.dataset_names)
def test_new_data_request_multi(self):
request = SentinelHub.new_data_request(
'S2L1C', ['B02', 'B03', 'B04', 'B08'], (512, 512),
time_range=("2018-10-01T00:00:00.000Z",
"2018-10-10T00:00:00.000Z"),
bbox=(
13.822,
45.850,
14.559,
46.291,
),
band_sample_types="FLOAT32",
band_units="reflectance")
# with open(REQUEST_MULTI_JSON), 'w') as fp:
# json.dump(request, fp, indent=2)
with open(REQUEST_MULTI_JSON, 'r') as fp:
expected_request = json.load(fp)
self.assertEqual(expected_request, request)
def gen(dataset, output_path, cube_config_path, source_config_path,
dest_config_path, band_names, tile_size, geometry, spatial_res, crs,
time_range, time_period, time_tolerance, four_d, verbose):
"""
Generate a data cube from SentinelHub.
By default, the command will create a ZARR dataset with 3D arrays
for each band e.g. "B01", "B02" with dimensions "time", "lat", "lon".
Use option "--4d" to write a single 4D array "band_data"
with dimensions "time", "lat", "lon", "band".
"""
import os.path
import time
import xarray as xr
from xcube_sh.config import CubeConfig
from xcube_sh.observers import Observers
from xcube_sh.sentinelhub import SentinelHub
from xcube_sh.store import SentinelHubStore
if os.path.exists(output_path):
raise click.ClickException(
f'Output {output_path} already exists. Move it away first.')
cube_config_dict = _load_config_dict(cube_config_path)
source_config_dict = _load_config_dict(source_config_path)
dest_config_dict = _load_config_dict(dest_config_path)
cube_config_dict.update({
k: v
for k, v in dict(dataset_name=dataset,
band_names=band_names,
tile_size=tile_size,
geometry=geometry,
spatial_res=spatial_res,
crs=crs,
time_range=time_range,
time_period=time_period,
time_tolerance=time_tolerance,
four_d=four_d).items() if v is not None
})
cube_config = CubeConfig.from_dict(cube_config_dict,
exception_type=click.ClickException)
# TODO: validate source_config_dict
sentinel_hub = SentinelHub(**source_config_dict)
print(f'Writing cube to {output_path}...')
# TODO: validate dest_config_dict
# TODO: use dest_config_dict and output_path to determine actuial output, which may be AWS S3
t0 = time.perf_counter()
store = SentinelHubStore(sentinel_hub, cube_config)
request_collector = Observers.request_collector()
store.add_observer(request_collector)
if verbose:
store.add_observer(Observers.request_dumper())
cube = xr.open_zarr(store)
cube.to_zarr(output_path, **dest_config_dict)
duration = time.perf_counter() - t0
print(f"Cube written to {output_path}, took {'%.2f' % duration} seconds.")
if verbose:
request_collector.stats.dump()
def open_data(self, data_id: str, **open_params) -> xr.Dataset:
"""
Opens the dataset with given *data_id* and *open_params*.
Possible values for *data_id* can be retrieved from the :meth:SentinelHubDataStore::get_data_ids method.
Possible keyword-arguments in *open_params* are:
* ``variable_names: Sequence[str]`` - optional list of variable names.
If not given, all variables are included.
* ``variable_units: Union[str, Sequence[str]]`` - units for all or each variable
* ``variable_sample_types: Union[str, Sequence[str]]`` - sample types for all or each variable
* ``crs: str`` - spatial CRS identifier. must be a valid OGC CRS URI.
* ``tile_size: Tuple[int, int]`` - optional tuple of spatial tile sizes in pixels.
* ``bbox: Tuple[float, float, float, float]`` - spatial coverage given as (minx, miny, maxx, maxy)
in units of the CRS. Required parameter.
* ``spatial_res: float`` - spatial resolution in unsits of the CRS^.
Required parameter.
* ``time_range: Tuple[Optional[str], Optional[str]]`` - tuple (start-time, end-time).
Both start-time and end-time, if given, should use ISO 8601 format.
Required parameter.
* ``time_period: str`` - Pandas-compatible time period/frequency, e.g. "4D", "2W"
* ``time_tolerance: str`` - Maximum time tolerance. Pandas-compatible time period/frequency.
* ``collection_id: str`` - An identifier used by Sentinel HUB to identify BYOC datasets.
* ``four_d: bool`` - If True, variables will represented as fourth dimension.
In addition, all store parameters can be used, if the data opener is used on its own.
See :meth:SentinelHubDataStore::get_data_store_params_schema method.
:param data_id: The data identifier.
:param open_params: Open parameters.
:return: An xarray.Dataset instance
"""
assert_not_none(data_id, 'data_id')
schema = self.get_open_data_params_schema(data_id)
schema.validate_instance(open_params)
sentinel_hub = self._sentinel_hub
if sentinel_hub is None:
sh_kwargs, open_params = schema.process_kwargs_subset(
open_params, (
'client_id',
'client_secret',
'api_url',
'oauth2_url',
'enable_warnings',
'error_policy',
'num_retries',
'retry_backoff_max',
'retry_backoff_base',
))
sentinel_hub = SentinelHub(**sh_kwargs)
cube_config_kwargs, open_params = schema.process_kwargs_subset(
open_params, (
'variable_names',
'variable_units',
'variable_sample_types',
'crs',
'tile_size',
'bbox',
'spatial_res',
'time_range',
'time_period',
'time_tolerance',
'collection_id',
'four_d',
))
chunk_store_kwargs, open_params = schema.process_kwargs_subset(
open_params, ('observer', 'trace_store_calls'))
band_names = cube_config_kwargs.pop('variable_names', None)
band_units = cube_config_kwargs.pop('variable_units', None)
band_sample_types = cube_config_kwargs.pop('variable_sample_types',
None)
cube_config = CubeConfig(dataset_name=data_id,
band_names=band_names,
band_units=band_units,
band_sample_types=band_sample_types,
**cube_config_kwargs)
chunk_store = SentinelHubChunkStore(sentinel_hub, cube_config,
**chunk_store_kwargs)
max_cache_size = open_params.pop('max_cache_size', None)
if max_cache_size:
chunk_store = zarr.LRUStoreCache(chunk_store,
max_size=max_cache_size)
return xr.open_zarr(chunk_store, **open_params)
def gen(request: Optional[str],
dataset_name: Optional[str],
band_names: Optional[Tuple],
tile_size: Optional[str],
geometry: Optional[str],
spatial_res: Optional[float],
crs: Optional[str],
time_range: Optional[str],
time_period: Optional[str],
time_tolerance: Optional[str],
output_path: Optional[str],
four_d: bool,
verbose: bool):
"""
Generate a data cube from SENTINEL Hub.
By default, the command will create a Zarr dataset with 3D arrays
for each band e.g. "B01", "B02" with dimensions "time", "lat", "lon".
Use option "--4d" to write a single 4D array "band_data"
with dimensions "time", "lat", "lon", "band".
Please use command "xcube sh req" to generate example request files that can be passed as REQUEST.
REQUEST may have JSON or YAML format.
You can also pipe a JSON request into this command. In this case
"""
import json
import os.path
import sys
import xarray as xr
from xcube.core.dsio import write_dataset
from xcube.util.perf import measure_time
from xcube_sh.config import CubeConfig
from xcube_sh.observers import Observers
from xcube_sh.sentinelhub import SentinelHub
from xcube_sh.chunkstore import SentinelHubChunkStore
if request:
request_dict = _load_request(request)
elif not sys.stdin.isatty():
request_dict = json.load(sys.stdin)
else:
request_dict = {}
cube_config_dict = request_dict.get('cube_config', {})
_overwrite_config_params(cube_config_dict,
dataset_name=dataset_name,
band_names=band_names if band_names else None, # because of multiple=True
tile_size=tile_size,
geometry=geometry,
spatial_res=spatial_res,
crs=crs,
time_range=time_range,
time_period=time_period,
time_tolerance=time_tolerance,
four_d=four_d)
input_config_dict = request_dict.get('input_config', {})
if 'datastore_id' in input_config_dict:
input_config_dict = dict(input_config_dict)
datastore_id = input_config_dict.pop('datastore_id')
if datastore_id != 'sentinelhub':
warnings.warn(f'Unknown datastore_id={datastore_id!r} encountered in request. Ignoring it...')
# _overwrite_config_params(input_config_dict, ...)
# TODO: validate input_config_dict
output_config_dict = request_dict.get('output_config', {})
_overwrite_config_params(output_config_dict,
path=output_path)
# TODO: validate output_config_dict
cube_config = CubeConfig.from_dict(cube_config_dict,
exception_type=click.ClickException)
if 'path' in output_config_dict:
output_path = output_config_dict.pop('path')
else:
output_path = DEFAULT_GEN_OUTPUT_PATH
if not _is_bucket_url(output_path) and os.path.exists(output_path):
raise click.ClickException(f'Output {output_path} already exists. Move it away first.')
sentinel_hub = SentinelHub(**input_config_dict)
print(f'Writing cube to {output_path}...')
with measure_time() as cm:
store = SentinelHubChunkStore(sentinel_hub, cube_config)
request_collector = Observers.request_collector()
store.add_observer(request_collector)
if verbose:
store.add_observer(Observers.request_dumper())
cube = xr.open_zarr(store)
if _is_bucket_url(output_path):
client_kwargs = {k: output_config_dict.pop(k)
for k in ('provider_access_key_id', 'provider_secret_access_key')
if k in output_config_dict}
write_dataset(cube, output_path, format_name='zarr', client_kwargs=client_kwargs, **output_config_dict)
else:
write_dataset(cube, output_path, **output_config_dict)
print(f"Cube written to {output_path}, took {'%.2f' % cm.duration} seconds.")
if verbose:
request_collector.stats.dump()
def __init__(self, **sh_kwargs):
super().__init__(SentinelHub(**sh_kwargs))
def test_features_to_time_ranges(self):
properties = [
{
'datetime': '2019-09-17T10:35:42Z'
},
{
'datetime': '2019-09-17T10:35:46Z'
},
{
'datetime': '2019-10-09T10:25:46Z'
},
{
'datetime': '2019-10-10T10:45:38Z'
},
{
'datetime': '2019-09-19T10:25:44Z'
},
{
'datetime': '2019-09-20T10:45:35Z'
},
{
'datetime': '2019-09-20T10:45:43Z'
},
{
'datetime': '2019-09-22T10:35:42Z'
},
{
'datetime': '2019-09-27T10:35:44Z'
},
{
'datetime': '2019-09-27T10:35:48Z'
},
{
'datetime': '2019-10-02T10:35:47Z'
},
{
'datetime': '2019-10-04T10:25:47Z'
},
{
'datetime': '2019-10-05T10:45:36Z'
},
{
'datetime': '2019-10-05T10:45:44Z'
},
{
'datetime': '2019-10-07T10:35:45Z'
},
{
'datetime': '2019-10-07T10:35:49Z'
},
{
'datetime': '2019-09-29T10:25:46Z'
},
{
'datetime': '2019-09-30T10:45:37Z'
},
{
'datetime': '2019-09-25T10:45:35Z'
},
{
'datetime': '2019-09-25T10:45:43Z'
},
{
'datetime': '2019-09-30T10:45:45Z'
},
{
'datetime': '2019-10-02T10:35:43Z'
},
{
'datetime': '2019-10-10T10:45:46Z'
},
{
'datetime': '2019-10-12T10:35:44Z'
},
{
'datetime': '2019-09-22T10:35:46Z'
},
{
'datetime': '2019-09-24T10:25:46Z'
},
{
'datetime': '2019-10-12T10:35:48Z'
},
{
'datetime': '2019-10-14T10:25:48Z'
},
{
'datetime': '2019-10-15T10:45:36Z'
},
{
'datetime': '2019-10-15T10:45:44Z'
},
{
'datetime': '2019-10-17T10:35:46Z'
},
{
'datetime': '2019-10-17T10:35:50Z'
},
]
features = [dict(properties=p) for p in properties]
time_ranges = SentinelHub.features_to_time_ranges(features)
self.assertEqual(
[('2019-09-17T10:35:42+00:00', '2019-09-17T10:35:46+00:00'),
('2019-09-19T10:25:44+00:00', '2019-09-19T10:25:44+00:00'),
('2019-09-20T10:45:35+00:00', '2019-09-20T10:45:43+00:00'),
('2019-09-22T10:35:42+00:00', '2019-09-22T10:35:46+00:00'),
('2019-09-24T10:25:46+00:00', '2019-09-24T10:25:46+00:00'),
('2019-09-25T10:45:35+00:00', '2019-09-25T10:45:43+00:00'),
('2019-09-27T10:35:44+00:00', '2019-09-27T10:35:48+00:00'),
('2019-09-29T10:25:46+00:00', '2019-09-29T10:25:46+00:00'),
('2019-09-30T10:45:37+00:00', '2019-09-30T10:45:45+00:00'),
('2019-10-02T10:35:43+00:00', '2019-10-02T10:35:47+00:00'),
('2019-10-04T10:25:47+00:00', '2019-10-04T10:25:47+00:00'),
('2019-10-05T10:45:36+00:00', '2019-10-05T10:45:44+00:00'),
('2019-10-07T10:35:45+00:00', '2019-10-07T10:35:49+00:00'),
('2019-10-09T10:25:46+00:00', '2019-10-09T10:25:46+00:00'),
('2019-10-10T10:45:38+00:00', '2019-10-10T10:45:46+00:00'),
('2019-10-12T10:35:44+00:00', '2019-10-12T10:35:48+00:00'),
('2019-10-14T10:25:48+00:00', '2019-10-14T10:25:48+00:00'),
('2019-10-15T10:45:36+00:00', '2019-10-15T10:45:44+00:00'),
('2019-10-17T10:35:46+00:00', '2019-10-17T10:35:50+00:00')],
[(tr[0].isoformat(), tr[1].isoformat()) for tr in time_ranges])
