def main(): # set configuration CLIENT_ID = '' CLIENT_SECRET = '' config = SHConfig() if CLIENT_ID and CLIENT_SECRET: config.sh_client_id = CLIENT_ID config.sh_client_secret = CLIENT_SECRET if config.sh_client_id == '' or config.sh_client_secret == '': print( "Warning! To use Sentinel Hub services, please provide the credentials (client ID and client secret)." ) # set coordinates, bounding box and a resolution (Naklo) TODO -> use input coordinates # [Longitude (x1), Latitude (y1) ... ] # betsiboka_coords_wgs84 = [14.2864, 46.2335, 14.3741, 46.2912] # Naklo betsiboka_coords_wgs84 = [14.3964, 46.2369, 14.4555, 46.2744] resolution = 10 betsiboka_bbox = BBox(bbox=betsiboka_coords_wgs84, crs=CRS.WGS84) betsiboka_size = bbox_to_dimensions(betsiboka_bbox, resolution=resolution) print(f'Image shape at {resolution} m resolution: {betsiboka_size} pixels') # get snow and vegetation data weatherDragons(betsiboka_coords_wgs84)
def config_fixture(): config = SHConfig() for param in config.get_params(): env_variable = param.upper() if os.environ.get(env_variable): setattr(config, param, os.environ.get(env_variable)) return config
def test_configuration(self): SHConfig().save() config_file = os.path.join(os.path.dirname(os.path.realpath(__file__)), '..', 'sentinelhub', 'config.json') if not os.path.isfile(config_file): self.fail(msg='Config file does not exist: {}'.format( os.path.abspath(config_file))) with open(config_file, 'r') as fp: config = json.load(fp) for attr in config: if attr not in [ 'instance_id', 'aws_access_key_id', 'aws_secret_access_key', 'sh_client_id', 'sh_client_secret' ]: value = config[attr] if isinstance(value, str): value = value.rstrip('/') self.assertEqual( SHConfig()[attr], value, "Expected value {}, got {}".format(config[attr], SHConfig()[attr]))
def set_sh_config(): config = SHConfig() expected_base_url = 'https://www.geopedia.world/rest/' if GeopediaConfig.is_production() else \ 'https://test.geopedia.world/rest/' if config.geopedia_rest_url != expected_base_url: config.geopedia_rest_url = expected_base_url config.save()
def __init__(self): # Set environment config = SHConfig() if os.getenv("CLIENT_ID") and os.getenv("CLIENT_SECRET"): config.sh_client_id = os.getenv("CLIENT_ID") config.sh_client_secret = os.getenv("CLIENT_SECRET") else: raise Exception("Client id and client secret is empty.") self.config = config
def test_get_available_collections(self): collections = DataCollection.get_available_collections() self._check_collection_list(collections) config = SHConfig() config.sh_base_url = ServiceUrl.EOCLOUD eocloud_collections = DataCollection.get_available_collections( config=config) self._check_collection_list(eocloud_collections) self.assertNotEqual(eocloud_collections, collections)
def login_config(CLIENT_ID, CLIENT_SECRET, instance_id): config = SHConfig() config.instance_id = instance_id if CLIENT_ID and CLIENT_SECRET: config.sh_client_id = CLIENT_ID config.sh_client_secret = CLIENT_SECRET if config.sh_client_id == '' or config.sh_client_secret == '' or config.instance_id == '': print( "Warning! To use Sentinel Hub services, please provide the credentials (client ID and client secret)." ) # config.save() return config
def __init__(self, data_source, size=None, resolution=None, cache_folder=None, config=None, max_threads=None): """ :param data_source: Source of requested satellite data. :type data_source: DataSource :param size: Number of pixels in x and y dimension. :type size: tuple(int, int) :type resolution: Resolution in meters, passed as a tuple for X and Y axis. :type resolution: tuple(int, int) :param cache_folder: Path to cache_folder. If set to None (default) requests will not be cached. :type cache_folder: str :param config: An instance of SHConfig defining the service :type config: SHConfig or None :param max_threads: Maximum threads to be used when downloading data. :type max_threads: int """ if (size is None) == (resolution is None): raise ValueError( "Exactly one of the parameters 'size' and 'resolution' should be given." ) self.size = size self.resolution = resolution self.config = config or SHConfig() self.max_threads = max_threads self.data_source = data_source self.cache_folder = cache_folder
def load_s3_filesystem(path, strict=False, config=None): """ Loads AWS s3 filesystem from a path :param path: A path to a folder on s3 bucket that will be the base folder in this filesystem :type path: str :param strict: If `True` the filesystem will be making additional checks to the s3. Default is `False`. :type strict: bool :param config: A configuration object with AWS credentials. By default is set to None and in this case the default configuration will be taken. :type config: SHConfig or None :return: A S3 filesystem object :rtype: fs_s3fs.S3FS """ if not path.startswith('s3://'): raise ValueError( "AWS path has to start with s3:// but found '{}'".format(path)) if config is None: config = SHConfig() path_chunks = path.split('/', 3)[2:] bucket_name = path_chunks[0] dir_path = path_chunks[1] if len(path_chunks) > 1 else '/' return S3FS(bucket_name=bucket_name, dir_path=dir_path, aws_access_key_id=config.aws_access_key_id if config.aws_access_key_id else None, aws_secret_access_key=config.aws_secret_access_key if config.aws_secret_access_key else None, strict=strict)
def __init__(self, data_source, size=None, resolution=None, cache_folder=None, config=None, max_threads=None): """ :param data_source: Source of requested satellite data. :type data_source: DataSource :param size: Number of pixels in x and y dimension. :type size: tuple(int, int) :type resolution: Resolution in meters, passed as a tuple for X and Y axis. :type resolution: tuple(int, int) :param cache_folder: Path to cache_folder. If set to None (default) requests will not be cached. :type cache_folder: str :param config: An instance of SHConfig defining the service :type config: SHConfig or None :param max_threads: Maximum threads to be used when downloading data. :type max_threads: int """ if (size is None) == (resolution is None): raise ValueError("Exactly one of the parameters 'size' and 'resolution' should be given.") self.size = size self.resolution = resolution self.config = config or SHConfig() self.max_threads = max_threads self.data_source = data_source self.request_args = dict( url=self.config.get_sh_processing_api_url(), headers={"accept": "application/tar", 'content-type': 'application/json'}, data_folder=cache_folder, hash_save=bool(cache_folder), request_type='POST', data_type=MimeType.TAR )
def __init__(self, data_collection, size=None, resolution=None, cache_folder=None, config=None, max_threads=None): """ :param data_collection: A collection of requested satellite data. :type data_collection: DataCollection :param size: Number of pixels in x and y dimension. :type size: tuple(int, int) :param resolution: Resolution in meters, passed as a single number or a tuple of two numbers - resolution in horizontal and resolution in vertical direction. :type resolution: float or (float, float) :param cache_folder: Path to cache_folder. If set to None (default) requests will not be cached. :type cache_folder: str :param config: An instance of SHConfig defining the service :type config: SHConfig or None :param max_threads: Maximum threads to be used when downloading data. :type max_threads: int """ if (size is None) == (resolution is None): raise ValueError( "Exactly one of the parameters 'size' and 'resolution' should be given." ) self.size = size self.resolution = resolution self.config = config or SHConfig() self.max_threads = max_threads self.data_collection = DataCollection(data_collection) self.cache_folder = cache_folder
def test_raise_for_missing_instance_id(self): config = SHConfig() config.instance_id = 'xxx' config.raise_for_missing_instance_id() config.instance_id = '' with self.assertRaises(ValueError): config.raise_for_missing_instance_id()
def download_from_aws(scene_id: str, destination: str): """Download the Sentinel Scene from AWS. It uses the library `sentinelhub-py <https://sentinelhub-py.readthedocs.io>`_ to download the Sentinel-2 SAFE folder. Once downloaded, it compressed into a `zip`. Notes: Make sure to set both `AWS_ACCESS_KEY_ID` and `AWS_SECRET_ACCESS_KEY` in environment variable. This method does not raise Exception. Args: scene_id - Sentinel-2 Product Id (We call as scene_id) destination - Path to store data. We recommend to use python `tempfile.TemporaryDirectory` and then move. Returns: Path to the downloaded file when success or None when an error occurred. """ try: config = SHConfig() config.aws_access_key_id = Config.AWS_ACCESS_KEY_ID config.aws_secret_access_key = Config.AWS_SECRET_ACCESS_KEY logging.info(f'Downloading {scene_id} From AWS...') request = AwsProductRequest(product_id=scene_id, data_folder=destination, safe_format=True, config=config) _ = request.get_data(save_data=True) file_name = '{}.SAFE'.format(scene_id) logging.info(f'Compressing {scene_id}.SAFE...') with working_directory(destination): shutil.make_archive(base_dir=file_name, format='zip', base_name=scene_id) return Path(destination) / file_name except BaseException as e: logging.error(f'Error downloading from AWS. {scene_id} - {str(e)}') return None
def authorize(): """ Set cridentials for sentinel hub services. :return: sentinel hub service configurations """ sh_config = SHConfig() if config.CLIENT_ID and config.CLIENT_SECRET: sh_config.sh_client_id = config.CLIENT_ID sh_config.sh_client_secret = config.CLIENT_SECRET if sh_config.sh_client_id == '' or sh_config.sh_client_secret == '': print( "Warning! To use Sentinel Hub services, please provide the credentials (client ID and client secret)." ) return sh_config
def test_s3_filesystem(self): folder_name = 'my_folder' s3_url = 's3://test-eo-bucket/{}'.format(folder_name) filesystem = get_filesystem(s3_url) self.assertTrue(isinstance(filesystem, S3FS)) self.assertEqual(filesystem.dir_path, folder_name) custom_config = SHConfig() custom_config.aws_access_key_id = 'fake-key' custom_config.aws_secret_access_key = 'fake-secret' filesystem = load_s3_filesystem(s3_url, strict=False, config=custom_config) self.assertTrue(isinstance(filesystem, S3FS)) self.assertEqual(filesystem.aws_access_key_id, custom_config.aws_access_key_id) self.assertEqual(filesystem.aws_secret_access_key, custom_config.aws_secret_access_key)
def test_reset(self): config = SHConfig() old_value = config.instance_id new_value = 'new' config.instance_id = new_value self.assertEqual(config.instance_id, new_value, 'New value was not set') self.assertEqual(config['instance_id'], new_value, 'New value was not set') self.assertEqual(config._instance.instance_id, old_value, 'Private value has changed') config.reset('ogc_base_url') config.reset(['aws_access_key_id', 'aws_secret_access_key']) self.assertEqual(config.instance_id, new_value, 'Instance ID should not reset yet') config.reset() self.assertEqual(config.instance_id, config._instance.CONFIG_PARAMS['instance_id'], 'Instance ID should reset')
def do_magic(lat, lng): payload = {} # set configuration CLIENT_ID = '' CLIENT_SECRET = '' config = SHConfig() if CLIENT_ID and CLIENT_SECRET: config.sh_client_id = CLIENT_ID config.sh_client_secret = CLIENT_SECRET if config.sh_client_id == '' or config.sh_client_secret == '': print( "Warning! To use Sentinel Hub services, please provide the credentials (client ID and client secret)." ) # set coordinates, bounding box and a resolution (Naklo) TODO -> use input coordinates # [Longitude (x1), Latitude (y1) ... ] # betsiboka_coords_wgs84 = [14.2864, 46.2335, 14.3741, 46.2912] # Naklo # betsiboka_coords_wgs84 = [14.3964, 46.2369, 14.4555, 46.2744] # Sencur bbox = get_bounding_box(lat, lng, 5000) payload['bbox'] = [[bbox[0], bbox[3]], [bbox[2], bbox[1]]] #if 1 == 1: # return payload resolution = 20 betsiboka_bbox = BBox(bbox=bbox, crs=CRS.WGS84) betsiboka_size = bbox_to_dimensions(betsiboka_bbox, resolution=resolution) print(f'Image shape at {resolution} m resolution: {betsiboka_size} pixels') # get snow and vegetation data weatherData = weatherDragons(bbox) payload['weather'] = weatherData vegData = snowyVegetation(betsiboka_bbox, betsiboka_size, config) payload['vegetation'] = vegData return payload
def load_sh(): config = SHConfig() for credentials in ['sh_client_id', 'sh_client_secret', 'instance_id']: if config.__getattribute__(credentials) == '': try: if credentials.startswith('sh_'): env = credentials else: env = 'sh_' + credentials setattr(config, credentials, os.environ.__getitem__(env.upper())) print( f'SentinelHub credential: {credentials} was set from env as {os.environ.__getitem__(env.upper())}' ) config.save() except KeyError: print( f'SentinelHub credential as environmental variable {env.upper()} was not found. ' f'Set credential {credentials} manualy') else: print( f'Actual SentinelHub credential: {credentials} is {config.__getattribute__(credentials)}' ) return SHConfig()
def get_images(coords, folder): INSTANCE_ID = 'f4531504-b71f-4dc9-a931-73be0f9b97d0' if INSTANCE_ID: config = SHConfig() config.instance_id = INSTANCE_ID else: config = None '''for i in range(int(len(coords)/2)): # area of interest mine_bbox = BBox(bbox=coords[i], crs=CRS.WGS84) wcs_true_color_request = WcsRequest( data_folder = folder, data_source = DataSource.SENTINEL2_L2A, layer='TRUE-COLOR-S2-L2A', bbox=mine_bbox, resx = 15, resy = 15, maxcc = 0.1, config=config ) wcs_true_color_img = wcs_true_color_request.get_data(save_data=True)''' for i in range(10000): # area of interest mine_bbox = BBox(bbox=coords[i], crs=CRS.WGS84) wms_true_color_request = WmsRequest( data_folder=folder, data_source=DataSource.SENTINEL2_L2A, layer='TRUE-COLOR-S2-L2A', bbox=mine_bbox, width=700, height=700, maxcc=0.1, config=config) wms_true_color_img = wms_true_color_request.get_data(save_data=True)
def test_s3_filesystem(aws_session_token): folder_name = "my_folder" s3_url = f"s3://test-eo-bucket/{folder_name}" filesystem = get_filesystem(s3_url) assert isinstance(filesystem, S3FS) assert filesystem.dir_path == folder_name custom_config = SHConfig() custom_config.aws_access_key_id = "fake-key" custom_config.aws_secret_access_key = "fake-secret" custom_config.aws_session_token = aws_session_token filesystem1 = load_s3_filesystem(s3_url, strict=False, config=custom_config) filesystem2 = get_filesystem(s3_url, config=custom_config) for filesystem in [filesystem1, filesystem2]: assert isinstance(filesystem, S3FS) assert filesystem.aws_access_key_id == custom_config.aws_access_key_id assert filesystem.aws_secret_access_key == custom_config.aws_secret_access_key assert filesystem.aws_session_token == aws_session_token
def set_sh_config(config: BaseConfig) -> SHConfig: """ Set AWS and SH credentials in SHConfig file to allow usage of download and io tasks """ sh_config = SHConfig() sh_config.aws_access_key_id = config.aws_access_key_id sh_config.aws_secret_access_key = config.aws_secret_access_key if all(key in config.__annotations__.keys() for key in ['sh_client_id', 'sh_client_secret']): sh_config.sh_client_id = config.sh_client_id sh_config.sh_client_secret = config.sh_client_secret sh_config.save() return sh_config
def test_get_aws_credentials(mocked_copy): fake_credentials = Credentials(access_key="my-aws-access-key", secret_key="my-aws-secret-key") mocked_copy.return_value.get_credentials.return_value = fake_credentials config = get_aws_credentials("xyz") assert config.aws_access_key_id == fake_credentials.access_key assert config.aws_secret_access_key == fake_credentials.secret_key default_config = SHConfig() config = get_aws_credentials("default", config=default_config) assert config.aws_access_key_id != default_config.aws_access_key_id assert config.aws_secret_access_key != default_config.aws_secret_access_key
def test_get_available_timestamps_with_missing_data_collection_service_url( self): collection = DataCollection.SENTINEL2_L1C.define_from( "COLLECTION_WITHOUT_URL", service_url=None) timestamps = get_available_timestamps( bbox=self.bbox, config=SHConfig(), data_collection=collection, time_difference=self.time_difference, time_interval=self.time_interval, maxcc=self.maxcc, ) assert len(timestamps) == 4 assert all(timestamp.tzinfo is not None for timestamp in timestamps)
def __init__(self, feature, reproject=True, clip=False, config=None): """ :param feature: A vector feature into which to import data :type feature: (FeatureType, str) :param reproject: Should the geometries be transformed to coordinate reference system of the requested bbox? :type reproject: bool, default = True :param clip: Should the geometries be clipped to the requested bbox, or should be geometries kept as they are? :type clip: bool, default = False :param config: A configuration object with credentials :type config: SHConfig """ self.feature = self.parse_feature( feature, allowed_feature_types=FeatureTypeSet.VECTOR_TYPES) self.config = config or SHConfig() self.reproject = reproject self.clip = clip
def request(self, scene, bbox, shape): x, y = shape evalscript = '''//VERSION=3 function setup() { return { input: ["POLAR"], output: { id:"default", bands: 1, sampleType: SampleType.FLOAT32} } } function evaluatePixel(samples) { return [samples.POLAR] }'''.replace('POLAR', scene.polar) request = SentinelHubRequest( evalscript=evalscript, input_data=[{ "type": "S1GRD", "dataFilter": { "timeRange": { "from": scene.from_time.strftime('%Y-%m-%dT%H:%M:%SZ'), "to": scene.to_time.strftime('%Y-%m-%dT%H:%M:%SZ') }, "acquisitionMode": "IW", "polarization": "DV", "orbitDirection ": scene.orbit_path }, "processing": { "backCoeff": "GAMMA0_ELLIPSOID", "orthorectify": "true" } }], responses=[ SentinelHubRequest.output_response( 'default', MimeType.TIFF, ) ], bbox=BBox(bbox, self.aoi.crs.to_epsg()), size=(x, y), config=SHConfig()) array = request.get_data(max_threads=min(32, os.cpu_count() + 4))[0] if array is not None: return array else: return None
def get_aws_credentials(aws_profile: str, config: Optional[SHConfig] = None) -> SHConfig: """Collects credentials from AWS profile and adds them to an instance of SHConfig. :param aws_profile: A name of AWS profile :param config: If existing config object is given credentials will be added to its copy, otherwise a new config object will be created. :return: A config object with AWS credentials that have been loaded from AWS profile. """ config = config.copy() if config else SHConfig() aws_session = Session(profile_name=aws_profile) aws_credentials = aws_session.get_credentials() config.aws_access_key_id = aws_credentials.access_key or "" config.aws_secret_access_key = aws_credentials.secret_key or "" config.aws_session_token = aws_credentials.token or "" return config
def get_available_timestamps( bbox: BBox, data_collection: DataCollection, *, time_interval: Optional[Tuple[dt.datetime, dt.datetime]] = None, time_difference: dt.timedelta = dt.timedelta(seconds=-1), maxcc: Optional[float] = None, config: Optional[SHConfig] = None, ) -> List[dt.datetime]: """Helper function to search for all available timestamps, based on query parameters. :param bbox: A bounding box of the search area. :param data_collection: A data collection for which to find available timestamps. :param time_interval: A time interval from which to provide the timestamps. :param time_difference: Minimum allowed time difference, used when filtering dates. :param maxcc: Maximum cloud coverage filter from interval [0, 1], default is None. :param config: A configuration object. :return: A list of timestamps of available observations. """ query = None if maxcc is not None and data_collection.has_cloud_coverage: if isinstance(maxcc, (int, float)) and (maxcc < 0 or maxcc > 1): raise ValueError( 'Maximum cloud coverage "maxcc" parameter should be a float on an interval [0, 1]' ) query = {"eo:cloud_cover": {"lte": int(maxcc * 100)}} fields = {"include": ["properties.datetime"], "exclude": []} if data_collection.service_url: config = config.copy() if config else SHConfig() config.sh_base_url = data_collection.service_url catalog = SentinelHubCatalog(config=config) search_iterator = catalog.search(collection=data_collection, bbox=bbox, time=time_interval, query=query, fields=fields) all_timestamps = search_iterator.get_timestamps() return filter_times(all_timestamps, time_difference)
def test_bad_credentials(self): evalscript = """ //VERSION=3 function setup() { return { input: ["B02", "B03", "B04"], sampleType: "UINT16", output: { bands: 3 } }; } function evaluatePixel(sample) { return [2.5 * sample.B04, 2.5 * sample.B03, 2.5 * sample.B02]; } """ request_params = dict( evalscript=evalscript, input_data=[ SentinelHubRequest.input_data( data_source=DataSource.SENTINEL2_L1C, time_interval=('2017-12-15T07:12:03', '2017-12-15T07:12:04'), ) ], responses=[SentinelHubRequest.output_response('default', 'tiff')], bbox=BBox(bbox=[46.16, -16.15, 46.51, -15.58], crs=CRS.WGS84), size=(512, 856)) bad_credentials_config = SHConfig() bad_credentials_config.sh_client_secret = 'test-wrong-credentials' request = SentinelHubRequest(**request_params, config=bad_credentials_config) with self.assertRaises(CustomOAuth2Error): request.get_data() missing_credentials_config = SHConfig() missing_credentials_config.sh_client_secret = '' request = SentinelHubRequest(**request_params, config=missing_credentials_config) with self.assertRaises(ValueError): request.get_data()
def load_s3_filesystem(path: str, strict: bool = False, config: Optional[SHConfig] = None, aws_profile: Optional[str] = None) -> S3FS: """Loads AWS s3 filesystem from a path. :param path: A path to a folder on s3 bucket that will be the base folder in this filesystem :type path: str :param strict: If `True` the filesystem will be making additional checks to the s3. Default is `False`. :type strict: bool :param config: A configuration object with AWS credentials. By default is set to None and in this case the default configuration will be taken. :type config: SHConfig or None :param aws_profile: A name of AWS profile. If given, AWS credentials will be taken from there. :return: A S3 filesystem object :rtype: fs_s3fs.S3FS """ if not is_s3_path(path): raise ValueError( f"AWS path has to start with s3:// but found '{path}'.") config = config or SHConfig() if aws_profile: config = get_aws_credentials(aws_profile, config=config) path_chunks = path.split("/", 3)[2:] bucket_name = path_chunks[0] dir_path = path_chunks[1] if len(path_chunks) > 1 else "/" return S3FS( bucket_name=bucket_name, dir_path=dir_path, aws_access_key_id=config.aws_access_key_id or None, aws_secret_access_key=config.aws_secret_access_key or None, aws_session_token=config.aws_session_token or None, strict=strict, )
def plot_image(image, factor, clip_range=None, **kwargs): """ Utility function for plotting RGB images. """ fig, ax = plt.subplots(nrows=1, ncols=1, figsize=(5, 5)) if clip_range is not None: ax.imshow(np.clip(image * factor, *clip_range), **kwargs) else: ax.imshow(image * factor, **kwargs) ax.set_xticks([]) ax.set_yticks([]) fig.savefig('plot2.png') config = SHConfig() config.sh_client_id = 'dd0d9906-9ccb-470a-b80e-1ff099f833cc' config.sh_client_secret = 'CZ!T.hY-PFj272{v{)J}~9%BK*d2ph,8,5v0V5By' @app.route('/test', methods=['POST']) @cross_origin() def getVars(): if request.method == 'POST': jsonData = request.get_json() longitude = float(jsonData['longitude']) latitude = float(jsonData['latitude']) distance = float(jsonData['distance']) coords = [(longitude - (distance / 2)), (latitude - (distance / 2)), (longitude + (distance / 2)), (latitude + (distance / 2))]