def test_is_wgs84_crs(self): config = CubeConfig(dataset_name='S2L2A', geometry=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, time_range=('2019-01-01', '2019-01-02')) self.assertEqual(True, config.is_wgs84_crs) config = CubeConfig(dataset_name='S2L2A', geometry=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, crs='http://www.opengis.net/def/crs/OGC/1.3/CRS84', time_range=('2019-01-01', '2019-01-02')) self.assertEqual(True, config.is_wgs84_crs) config = CubeConfig(dataset_name='S2L2A', geometry=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, crs='http://www.opengis.net/def/crs/EPSG/0/4326', time_range=('2019-01-01', '2019-01-02')) self.assertEqual(True, config.is_wgs84_crs) config = CubeConfig(dataset_name='S2L2A', geometry=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, crs='http://www.opengis.net/def/crs/EPSG/0/3035', time_range=('2019-01-01', '2019-01-02')) self.assertEqual(False, config.is_wgs84_crs)
def test_adjust_sizes(self): spatial_res = 0.00018 common_kwargs = dict(dataset_name='S2L2A', band_names=('B01', 'B02', 'B03'), spatial_res=spatial_res, tile_size=(512, 512), time_range=('2019-01-01', '2019-01-02')) # size will be smaller than chunk sizes config = CubeConfig(geometry=(10.11, 54.17, 10.14, 54.19), **common_kwargs) w, h = config.size x1, y1, x2, y2 = config.geometry self.assertEqual((167, 111), (w, h)) self.assertEqual((167, 111), config.tile_size) self.assertEqual((1, 1), config.num_tiles) self.assertAlmostEqual(10.11, x1) self.assertAlmostEqual(10.14006, x2, places=4) self.assertAlmostEqual(54.17, y1) self.assertAlmostEqual(54.18998, y2, places=4) self.assertEqual(w, round((x2 - x1) / spatial_res)) self.assertEqual(h, round((y2 - y1) / spatial_res)) # size will be smaller than 2x chunk sizes config = CubeConfig(geometry=(10.11, 54.17, 10.2025, 54.3), **common_kwargs) w, h = config.size x1, y1, x2, y2 = config.geometry self.assertEqual((514, 722), (w, h)) self.assertEqual((514, 722), config.tile_size) self.assertEqual((1, 1), config.num_tiles) self.assertAlmostEqual(10.11, x1) self.assertAlmostEqual(10.20252, x2, places=4) self.assertAlmostEqual(54.17, y1) self.assertAlmostEqual(54.29996, y2, places=4) self.assertEqual(w, round((x2 - x1) / spatial_res)) self.assertEqual(h, round((y2 - y1) / spatial_res)) # size will be larger than or equal 2x chunk sizes config = CubeConfig(geometry=(10.11, 54.17, 10.5, 54.5), **common_kwargs) w, h = config.size x1, y1, x2, y2 = config.geometry self.assertEqual((2560, 2048), (w, h)) self.assertEqual((512, 512), config.tile_size) self.assertEqual((5, 4), config.num_tiles) self.assertAlmostEqual(10.11, x1) self.assertAlmostEqual(10.57080, x2, places=4) self.assertAlmostEqual(54.17, y1) self.assertAlmostEqual(54.53864, y2, places=4) self.assertEqual(w, round((x2 - x1) / spatial_res)) self.assertEqual(h, round((y2 - y1) / spatial_res))
def test_from_and_to_dict(self): config = CubeConfig.from_dict( dict(dataset_name='S2L2A', band_names=('B01', 'B02', 'B03'), geometry=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, tile_size=(512, 512), time_range=('2019-01-01', '2019-01-02'))) self.assertEqual( { 'band_names': ('B01', 'B02', 'B03'), 'band_sample_types': None, 'band_units': None, 'collection_id': None, 'crs': 'http://www.opengis.net/def/crs/EPSG/0/4326', 'dataset_name': 'S2L2A', 'four_d': False, 'geometry': (10.11, 54.17, 10.14072, 54.19048), 'spatial_res': 1e-05, 'tile_size': (512, 512), 'time_period': None, 'time_range': ('2019-01-01T00:00:00+00:00', '2019-01-02T00:00:00+00:00'), 'time_tolerance': '0 days 00:10:00' }, config.as_dict())
def get_cube_config(self): return CubeConfig(dataset_name='S2L2A', bbox=(10.2, 53.5, 10.3, 53.6), spatial_res=0.1 / 4000, time_range=('2017-08-01', '2017-08-31'), time_period=None, four_d=False)
def test_band_names_may_be_null(self): config = CubeConfig(dataset_name='S2L2A', band_names=None, bbox=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, time_range=('2019-01-01', '2019-01-02')) self.assertEqual(None, config.band_names)
def get_cube_config(self): return CubeConfig(dataset_name='S2L1C', band_names=['B01', 'B08', 'B12'], bbox=(10.2, 53.5, 10.3, 53.6), spatial_res=0.1 / 4000, time_range=('2017-08-01', '2017-08-31'), time_period='1D', four_d=False)
def test_time_deltas(self): config = CubeConfig.from_dict( dict(dataset_name='S2L2A', band_names=('B01', 'B02', 'B03'), bbox=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, time_range=('2019-01-01', '2019-01-02'))) self.assertEqual(None, config.time_period) self.assertEqual(pd.Timedelta('0 days 00:10:00'), config.time_tolerance) config = CubeConfig.from_dict( dict(dataset_name='S2L2A', band_names=('B01', 'B02', 'B03'), bbox=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, time_period='8D', time_range=('2019-01-01', '2019-01-02'))) self.assertEqual(pd.Timedelta('8 days 00:00:00'), config.time_period) self.assertEqual(None, config.time_tolerance) config = CubeConfig.from_dict( dict(dataset_name='S2L2A', band_names=('B01', 'B02', 'B03'), bbox=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, time_tolerance='1H', time_range=('2019-01-01', '2019-01-02'))) self.assertEqual(None, config.time_period) self.assertEqual(pd.Timedelta('0 days 01:00:00'), config.time_tolerance) config = CubeConfig.from_dict( dict(dataset_name='S2L2A', band_names=('B01', 'B02', 'B03'), bbox=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, time_period='8D', time_tolerance='1H', time_range=('2019-01-01', '2019-01-02'))) self.assertEqual(pd.Timedelta('8 days 00:00:00'), config.time_period) self.assertEqual(pd.Timedelta('0 days 01:00:00'), config.time_tolerance)
def gen(dataset, output_path, band_names, tile_size, geometry, spatial_res, crs, time_range, time_period, time_tolerance, four_d, verbose): """ Generate a data cube from SentinelHub. By default, the command will create a ZARR dataset with 3D arrays for each band e.g. "B01", "B02" with dimensions "time", "lat", "lon". Use option "--4d" to write a single 4D array "band_data" with dimensions "time", "lat", "lon", "band". """ import os.path import time import xarray as xr from xcube_sh.config import CubeConfig from xcube_sh.observers import Observers from xcube_sh.sentinelhub import SentinelHub from xcube_sh.store import SentinelHubStore if os.path.exists(output_path): raise click.ClickException( f'Output {output_path} already exists. Move it away first.') cube_config = CubeConfig(dataset_name=dataset, band_names=band_names, tile_size=tile_size, geometry=geometry, spatial_res=spatial_res, crs=crs, time_range=time_range, time_period=time_period, time_tolerance=time_tolerance, four_d=four_d, exception_type=click.ClickException) sentinel_hub = SentinelHub() print(f'Writing cube to {output_path}...') t0 = time.perf_counter() store = SentinelHubStore(sentinel_hub, cube_config) request_collector = Observers.request_collector() store.add_observer(request_collector) if verbose: store.add_observer(Observers.request_dumper()) cube = xr.open_zarr(store) cube.to_zarr(output_path) duration = time.perf_counter() - t0 print(f"Cube written to {output_path}, took {'%.2f' % duration} seconds.") if verbose: request_collector.stats.dump()
def test_from_dict_invalids(self): with self.assertRaises(ValueError) as cm: CubeConfig.from_dict( dict(dataset_name='S2L2A', band_names=('B01', 'B02', 'B03'), geometry=(10.11, 54.17, 10.14, 54.19), special_res=0.00001, tile_size=(512, 512), time_range=('2019-01-01', '2019-01-02'))) self.assertEqual( "Found invalid parameter 'special_res' in cube configuration", f'{cm.exception}') with self.assertRaises(ValueError) as cm: CubeConfig.from_dict( dict(dataset_name='S2L2A', band_names=('B01', 'B02', 'B03'), geometrix=(10.11, 54.17, 10.14, 54.19), special_res=0.00001, tile_size=(512, 512), time_range=('2019-01-01', '2019-01-02'))) self.assertEqual( "Found invalid parameters in cube configuration: 'geometrix', 'special_res'", f'{cm.exception}')
"2020-05-07" ] dates = dates_ruhr aoi = aoi_ruhr IPython.display.GeoJSON(shapely.geometry.box(*aoi).__geo_interface__) ## Do Calculation for one time stamp # Get Sentinel-2 L2A data cube_con = CubeConfig(dataset_name=dataset, band_names=band_names, tile_size=[512, 512], geometry=aoi, spatial_res=spatial_res, time_range=[dates[0], dates[1]], time_period=time_period) cube = open_cube(cube_con) scl = MaskSet(cube.SCL) cube = cube.where((scl.clouds_high_probability) == 0) #date = dates[-1] # do exmaple calculation for last timestamp of dates date = dates[0] #timestamp1 = cube.sel(time = cube.time[-1]) timestamp1 = cube.sel(time=cube.time[0]) timestamp1.B02.plot.imshow(vmin=0, vmax=0.2, figsize=[16, 8]) # Compute a roads mask using band ratios and thresholds
def open_data(self, data_id: str, **open_params) -> xr.Dataset: """ Opens the dataset with given *data_id* and *open_params*. Possible values for *data_id* can be retrieved from the :meth:SentinelHubDataStore::get_data_ids method. Possible keyword-arguments in *open_params* are: * ``variable_names: Sequence[str]`` - optional list of variable names. If not given, all variables are included. * ``variable_units: Union[str, Sequence[str]]`` - units for all or each variable * ``variable_sample_types: Union[str, Sequence[str]]`` - sample types for all or each variable * ``crs: str`` - spatial CRS identifier. must be a valid OGC CRS URI. * ``tile_size: Tuple[int, int]`` - optional tuple of spatial tile sizes in pixels. * ``bbox: Tuple[float, float, float, float]`` - spatial coverage given as (minx, miny, maxx, maxy) in units of the CRS. Required parameter. * ``spatial_res: float`` - spatial resolution in unsits of the CRS^. Required parameter. * ``time_range: Tuple[Optional[str], Optional[str]]`` - tuple (start-time, end-time). Both start-time and end-time, if given, should use ISO 8601 format. Required parameter. * ``time_period: str`` - Pandas-compatible time period/frequency, e.g. "4D", "2W" * ``time_tolerance: str`` - Maximum time tolerance. Pandas-compatible time period/frequency. * ``collection_id: str`` - An identifier used by Sentinel HUB to identify BYOC datasets. * ``four_d: bool`` - If True, variables will represented as fourth dimension. In addition, all store parameters can be used, if the data opener is used on its own. See :meth:SentinelHubDataStore::get_data_store_params_schema method. :param data_id: The data identifier. :param open_params: Open parameters. :return: An xarray.Dataset instance """ assert_not_none(data_id, 'data_id') schema = self.get_open_data_params_schema(data_id) schema.validate_instance(open_params) sentinel_hub = self._sentinel_hub if sentinel_hub is None: sh_kwargs, open_params = schema.process_kwargs_subset( open_params, ( 'client_id', 'client_secret', 'api_url', 'oauth2_url', 'enable_warnings', 'error_policy', 'num_retries', 'retry_backoff_max', 'retry_backoff_base', )) sentinel_hub = SentinelHub(**sh_kwargs) cube_config_kwargs, open_params = schema.process_kwargs_subset( open_params, ( 'variable_names', 'variable_units', 'variable_sample_types', 'crs', 'tile_size', 'bbox', 'spatial_res', 'time_range', 'time_period', 'time_tolerance', 'collection_id', 'four_d', )) chunk_store_kwargs, open_params = schema.process_kwargs_subset( open_params, ('observer', 'trace_store_calls')) band_names = cube_config_kwargs.pop('variable_names', None) band_units = cube_config_kwargs.pop('variable_units', None) band_sample_types = cube_config_kwargs.pop('variable_sample_types', None) cube_config = CubeConfig(dataset_name=data_id, band_names=band_names, band_units=band_units, band_sample_types=band_sample_types, **cube_config_kwargs) chunk_store = SentinelHubChunkStore(sentinel_hub, cube_config, **chunk_store_kwargs) max_cache_size = open_params.pop('max_cache_size', None) if max_cache_size: chunk_store = zarr.LRUStoreCache(chunk_store, max_size=max_cache_size) return xr.open_zarr(chunk_store, **open_params)
# OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE # SOFTWARE. import unittest import xarray as xr from test.test_sentinelhub import HAS_SH_CREDENTIALS from test.test_sentinelhub import REQUIRE_SH_CREDENTIALS from xcube_sh.sentinelhub import SentinelHub from xcube_sh.config import CubeConfig from xcube_sh.cube import open_cube cube_config = CubeConfig(dataset_name='S2L1C', band_names=['B04'], bbox=(10.00, 54.27, 11.00, 54.60), spatial_res=0.00018, time_range=('2018-05-14', '2018-07-31'), time_tolerance='30M') @unittest.skipUnless(HAS_SH_CREDENTIALS, REQUIRE_SH_CREDENTIALS) class CubeTest(unittest.TestCase): def test_open_cube(self): cube = open_cube(cube_config=cube_config) self.assertIsInstance(cube, xr.Dataset) class CubeTest2(unittest.TestCase): def test_open_cube_with_illegal_kwargs(self): with self.assertRaises(ValueError) as cm: open_cube(
def gen(dataset, output_path, cube_config_path, source_config_path, dest_config_path, band_names, tile_size, geometry, spatial_res, crs, time_range, time_period, time_tolerance, four_d, verbose): """ Generate a data cube from SentinelHub. By default, the command will create a ZARR dataset with 3D arrays for each band e.g. "B01", "B02" with dimensions "time", "lat", "lon". Use option "--4d" to write a single 4D array "band_data" with dimensions "time", "lat", "lon", "band". """ import os.path import time import xarray as xr from xcube_sh.config import CubeConfig from xcube_sh.observers import Observers from xcube_sh.sentinelhub import SentinelHub from xcube_sh.store import SentinelHubStore if os.path.exists(output_path): raise click.ClickException( f'Output {output_path} already exists. Move it away first.') cube_config_dict = _load_config_dict(cube_config_path) source_config_dict = _load_config_dict(source_config_path) dest_config_dict = _load_config_dict(dest_config_path) cube_config_dict.update({ k: v for k, v in dict(dataset_name=dataset, band_names=band_names, tile_size=tile_size, geometry=geometry, spatial_res=spatial_res, crs=crs, time_range=time_range, time_period=time_period, time_tolerance=time_tolerance, four_d=four_d).items() if v is not None }) cube_config = CubeConfig.from_dict(cube_config_dict, exception_type=click.ClickException) # TODO: validate source_config_dict sentinel_hub = SentinelHub(**source_config_dict) print(f'Writing cube to {output_path}...') # TODO: validate dest_config_dict # TODO: use dest_config_dict and output_path to determine actuial output, which may be AWS S3 t0 = time.perf_counter() store = SentinelHubStore(sentinel_hub, cube_config) request_collector = Observers.request_collector() store.add_observer(request_collector) if verbose: store.add_observer(Observers.request_dumper()) cube = xr.open_zarr(store) cube.to_zarr(output_path, **dest_config_dict) duration = time.perf_counter() - t0 print(f"Cube written to {output_path}, took {'%.2f' % duration} seconds.") if verbose: request_collector.stats.dump()
place = area for i in range(2): if (date2020 == False): # 2019 data = "2019" date_x = dates[0] date_y = dates[1] else: data = "2020" date_x = dates[2] date_y = dates[3] IPython.display.GeoJSON(shapely.geometry.box(*aoi).__geo_interface__) cube_con = CubeConfig( dataset_name=dataset, band_names=band_names, tile_size=[512, 512], geometry=aoi, # area of interest spatial_res=spatial_res, #0.00009 time_range=[date_x, date_y], time_period=time_period) # what is time tolerance cube = open_cube(cube_con, **hc) scl = MaskSet(cube.SCL) #cube = cube.where((scl.clouds_high_probability) == 0) #shows weird errors cube = cube.where( (scl.clouds_high_probability + scl.clouds_medium_probability + scl.clouds_low_probability_or_unclassified + scl.cirrus) == 0) date = dates[0] t1 = cube.sel(time=cube.time[0]) B02 = t1.B02
dates = ["2020-01-01", "2020-05-10"] aoi = aoi_ruhr IPython.display.GeoJSON(shapely.geometry.box(*aoi).__geo_interface__) ## Do Calculation for one time stamp pixels = [] # Get Sentinel-2 L2A data cube_con = CubeConfig(dataset_name=dataset, band_names=band_names, tile_size=[512, 512], geometry=aoi, spatial_res=spatial_res, time_range=[dates[0], dates[1]], time_period=time_period) cube = open_cube(cube_con) # Mask out clouds scl = MaskSet(cube.SCL) cube = cube.where( (scl.clouds_high_probability + scl.clouds_medium_probability + scl.clouds_low_probability_or_unclassified + scl.cirrus) == 0) dates = cube.time.values for date in dates:
def test_deprecated_geometry_still_works(self): config = CubeConfig(dataset_name='S2L2A', geometry=(10.11, 54.17, 10.14, 54.19), spatial_res=0.00001, time_range=('2019-01-01', '2019-01-02')) self.assertEqual(config.geometry, config.bbox)
def gen(request: Optional[str], dataset_name: Optional[str], band_names: Optional[Tuple], tile_size: Optional[str], geometry: Optional[str], spatial_res: Optional[float], crs: Optional[str], time_range: Optional[str], time_period: Optional[str], time_tolerance: Optional[str], output_path: Optional[str], four_d: bool, verbose: bool): """ Generate a data cube from SENTINEL Hub. By default, the command will create a Zarr dataset with 3D arrays for each band e.g. "B01", "B02" with dimensions "time", "lat", "lon". Use option "--4d" to write a single 4D array "band_data" with dimensions "time", "lat", "lon", "band". Please use command "xcube sh req" to generate example request files that can be passed as REQUEST. REQUEST may have JSON or YAML format. You can also pipe a JSON request into this command. In this case """ import json import os.path import sys import xarray as xr from xcube.core.dsio import write_dataset from xcube.util.perf import measure_time from xcube_sh.config import CubeConfig from xcube_sh.observers import Observers from xcube_sh.sentinelhub import SentinelHub from xcube_sh.chunkstore import SentinelHubChunkStore if request: request_dict = _load_request(request) elif not sys.stdin.isatty(): request_dict = json.load(sys.stdin) else: request_dict = {} cube_config_dict = request_dict.get('cube_config', {}) _overwrite_config_params(cube_config_dict, dataset_name=dataset_name, band_names=band_names if band_names else None, # because of multiple=True tile_size=tile_size, geometry=geometry, spatial_res=spatial_res, crs=crs, time_range=time_range, time_period=time_period, time_tolerance=time_tolerance, four_d=four_d) input_config_dict = request_dict.get('input_config', {}) if 'datastore_id' in input_config_dict: input_config_dict = dict(input_config_dict) datastore_id = input_config_dict.pop('datastore_id') if datastore_id != 'sentinelhub': warnings.warn(f'Unknown datastore_id={datastore_id!r} encountered in request. Ignoring it...') # _overwrite_config_params(input_config_dict, ...) # TODO: validate input_config_dict output_config_dict = request_dict.get('output_config', {}) _overwrite_config_params(output_config_dict, path=output_path) # TODO: validate output_config_dict cube_config = CubeConfig.from_dict(cube_config_dict, exception_type=click.ClickException) if 'path' in output_config_dict: output_path = output_config_dict.pop('path') else: output_path = DEFAULT_GEN_OUTPUT_PATH if not _is_bucket_url(output_path) and os.path.exists(output_path): raise click.ClickException(f'Output {output_path} already exists. Move it away first.') sentinel_hub = SentinelHub(**input_config_dict) print(f'Writing cube to {output_path}...') with measure_time() as cm: store = SentinelHubChunkStore(sentinel_hub, cube_config) request_collector = Observers.request_collector() store.add_observer(request_collector) if verbose: store.add_observer(Observers.request_dumper()) cube = xr.open_zarr(store) if _is_bucket_url(output_path): client_kwargs = {k: output_config_dict.pop(k) for k in ('provider_access_key_id', 'provider_secret_access_key') if k in output_config_dict} write_dataset(cube, output_path, format_name='zarr', client_kwargs=client_kwargs, **output_config_dict) else: write_dataset(cube, output_path, **output_config_dict) print(f"Cube written to {output_path}, took {'%.2f' % cm.duration} seconds.") if verbose: request_collector.stats.dump()