def test_aws_session_class_session(): """AWSSession works""" boto3 = pytest.importorskip("boto3") sesh = AWSSession(session=boto3.session.Session(aws_access_key_id='foo', aws_secret_access_key='bar')) assert sesh._session assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo' assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
def activate_rio_env(aws=None, defaults=True, **kwargs): """ Inject activated rasterio.Env into current thread. This de-activates previously setup environment. :param aws: Dictionary of options for rasterio.session.AWSSession OR False in which case session won't be setup OR None -- session = rasterio.session.AWSSession() :param defaults: Supply False to not inject COG defaults :param **kwargs: Passed on to rasterio.Env(..) constructor """ env_old = getattr(_local, 'env', None) if env_old is not None: env_old.__exit__(None, None, None) _local.env = None if aws is None: session = AWSSession() elif aws is False: session = None else: session = AWSSession(**aws) opts = dict( GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR' ) if defaults else {} opts.update(**kwargs) env = rasterio.Env(session=session, **opts) env.__enter__() _local.env = env return get_rio_env()
def test_requester_pays(): """GDAL is configured with requester pays""" sesh = AWSSession(aws_access_key_id='foo', aws_secret_access_key='bar', requester_pays=True) assert sesh._session assert sesh.get_credential_options()['AWS_REQUEST_PAYER'] == 'requester'
def _process_file_stream(src_stream, on_file_cbk, gdal_opts=None, region_name=None, timer=None): from rasterio.path import parse_path session = AWSSession(session=_session(region_name)) if timer is not None: def proc(url, userdata): t0 = timer() with rasterio.DatasetReader(parse_path(url), sharing=False) as f: on_file_cbk(f, userdata, t0=t0) else: def proc(url, userdata): with rasterio.DatasetReader(parse_path(url), sharing=False) as f: on_file_cbk(f, userdata) with rasterio.Env(session=session, **gdal_opts): for userdata, url in src_stream: try: proc(url, userdata) except Exception as e: print('Error when reading: {}\n...({})'.format( url, str(e)), file=sys.stderr)
def _warmup(): session = _session(region_name=self._region_name) with rasterio.Env(session=AWSSession(session=session), **self._gdal_opts): if action: action() return session.get_credentials()
def test_aws_session_class_profile(tmpdir, monkeypatch): """Confirm that profile_name kwarg works.""" pytest.importorskip("boto3") credentials_file = tmpdir.join('credentials') credentials_file.write("[testing]\n" "aws_access_key_id = foo\n" "aws_secret_access_key = bar\n" "aws_session_token = baz") monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', str(credentials_file)) monkeypatch.setenv('AWS_SESSION_TOKEN', 'ignore_me') sesh = AWSSession(profile_name='testing') assert sesh._session assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo' assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar' assert sesh.get_credential_options()['AWS_SESSION_TOKEN'] == 'baz' monkeypatch.undo()
def get_subset(fkey, features): import numpy as np # import pandas as pd import rasterio from rasterio.mask import mask from rasterio.session import AWSSession s3host_ = S3HOST.replace('http://', '') s3host_ = s3host_.replace('https://', '') session = connection('session') with rasterio.Env(AWSSession(session), AWS_S3_ENDPOINT=s3host_, AWS_HTTPS='NO', AWS_VIRTUAL_HOSTING=False) as env: with rasterio.open('/vsis3/{}/{}'.format(BUCKET, fkey)) as src: out_image, out_transform = mask(src, features, crop=True, pad=False, all_touched=False) # print('--out_image.shape: ', out_image.shape) w_5 = np.percentile(out_image, 5.0) w_95 = np.percentile(out_image, 95.0) # print('--w_5, w_95: ', w_5, w_95) chip_set = np.clip(255 * (out_image - w_5) / (w_95 - w_5), 0, 255).astype(np.uint16) return chip_set, out_transform
def __getitem__(self, i): match = self.overlap.iloc[i] s = boto3.Session(profile_name=self.aws_profile) with rio.Env(AWSSession(s)): mask = np.squeeze(rio.open(match.path_mask).read()) dataAcquired = False attempts_remaining = MAX_RETRY_ATTEMPTS while (not dataAcquired) and (attempts_remaining > 0): try: data = rio.open(match.path).read() dataAcquired = True except rio.errors.RasterioError as e: print( f"Rasterio Read Failure (retrying, {attempts_remaining} attempts remain). Path: {match.path}" ) attempts_remaining = attempts_remaining - 1 sleep(RETRY_LENGTH_SECONDS) if not dataAcquired: raise Exception( "Rasterio Read Failure, all attempts exhausted. (Path: {match.path})." ) if self.joint_transform: augmented = self.joint_transform(image=data, mask=mask) data = augmented['image'] mask = augmented['mask'] return (torch.FloatTensor(data), torch.from_numpy(mask).double(), match.name)
def main_group( ctx, verbose, quiet, aws_profile, aws_no_sign_requests, aws_requester_pays, gdal_version, ): """Rasterio command line interface. """ verbosity = verbose - quiet configure_logging(verbosity) ctx.obj = {} ctx.obj["verbosity"] = verbosity ctx.obj["aws_profile"] = aws_profile envopts = {"CPL_DEBUG": (verbosity > 2)} if aws_profile or aws_no_sign_requests or aws_requester_pays: ctx.obj["env"] = rasterio.Env( session=AWSSession( profile_name=aws_profile, aws_unsigned=aws_no_sign_requests, requester_pays=aws_requester_pays, ), **envopts) else: ctx.obj["env"] = rasterio.Env(**envopts)
def activate_rio_env(aws=None, cloud_defaults=False, **kwargs): """ Inject activated rasterio.Env into current thread. This de-activates previously setup environment. :param aws: Dictionary of options for rasterio.session.AWSSession OR 'auto' -- session = rasterio.session.AWSSession() :param cloud_defaults: When True inject settings for reading COGs :param **kwargs: Passed on to rasterio.Env(..) constructor """ session = DummySession() if aws is not None: if not (aws == "auto" or isinstance(aws, dict)): raise ValueError( 'Only support: None|"auto"|{..} for `aws` parameter') aws = {} if aws == "auto" else dict(**aws) region_name = aws.get("region_name", "auto") if region_name == "auto": from datacube.utils.aws import auto_find_region try: aws["region_name"] = auto_find_region() except ValueError as e: # only treat it as error if it was requested by user if "region_name" in aws: raise e session = AWSSession(**aws) opts = (dict( GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR", GDAL_HTTP_MAX_RETRY="10", GDAL_HTTP_RETRY_DELAY="0.5", ) if cloud_defaults else {}) opts.update(**kwargs) state = _state() if state.env is not None: state.env.__exit__(None, None, None) env = rasterio.Env(session=session, **opts) env.__enter__() state.env = env state.epoch = -1 return get_rio_env()
def __getitem__(self, i): match = self.overlap.iloc[i] s = boto3.Session(profile_name=self.aws_profile) with rio.Env(AWSSession(s)): mask = np.squeeze(rio.open(match.path_mask).read()) data = rio.open(match.path).read() if self.joint_transform: augmented = self.joint_transform(image=data, mask=mask) data = augmented['image'] mask = augmented['mask'] return (torch.FloatTensor(data), torch.from_numpy(mask).double(), match.name)
def test_nested_credentials(monkeypatch): """Check that rasterio.open() doesn't wipe out surrounding credentials""" @ensure_env_credentialled def fake_opener(path): return getenv() with rasterio.Env(session=AWSSession(aws_access_key_id='foo', aws_secret_access_key='bar')): assert getenv()['AWS_ACCESS_KEY_ID'] == 'foo' assert getenv()['AWS_SECRET_ACCESS_KEY'] == 'bar' monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'lol') monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'wut') gdalenv = fake_opener('s3://foo/bar') assert gdalenv['AWS_ACCESS_KEY_ID'] == 'foo' assert gdalenv['AWS_SECRET_ACCESS_KEY'] == 'bar'
def download_raster_aws(services, path, dst_path, requester_pays=False): from rasterio.session import AWSSession aws_session = AWSSession(services.session, requester_pays=requester_pays) with rasterio.Env(aws_session, AWS_SESSION_TOKEN=""): with rasterio.open(path) as src: profile = src.profile.copy() profile.update({ 'blockxsize': 2048, 'blockysize': 2048, 'tiled': True }) arr = src.read(1) with rasterio.open(dst_path, 'w', **profile) as dataset: dataset.write(arr, 1) return dst_path
def activate_rio_env(aws=None, defaults=True, **kwargs): """Inject activated rasterio.Env into current thread. This de-activates previously setup environment. :param aws: Dictionary of options for rasterio.session.AWSSession OR False in which case session won't be setup OR None -- session = rasterio.session.AWSSession() :param defaults: Supply False to not inject COG defaults :param **kwargs: Passed on to rasterio.Env(..) constructor """ env_old = getattr(_local, "env", None) if env_old is not None: env_old.__exit__(None, None, None) _local.env = None if aws is False: session = None else: aws = {} if aws is None else dict(**aws) region_name = aws.get("region_name", "auto") if region_name == "auto": from odc.aws import auto_find_region try: aws["region_name"] = auto_find_region() except Exception as e: # only treat it as error if it was requested by user if "region_name" in aws: raise e session = AWSSession(**aws) opts = dict(GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR") if defaults else {} opts.update(**kwargs) env = rasterio.Env(session=session, **opts) env.__enter__() _local.env = env return get_rio_env()
def getSubset(fkey, features, lut, fixscale=False): try: flut = [float(l) for l in lut] logging.debug(flut) except: logging.error("Error converting LUT to float values") flut = [5.0, 95.0] BUCKET = 'DIAS' access_key = 'anystring' secret_key = 'anystring' session = boto3.Session(aws_access_key_id=access_key, aws_secret_access_key=secret_key) with rasterio.Env(AWSSession(session), AWS_S3_ENDPOINT='data.cloudferro.com', AWS_HTTPS='NO', AWS_VIRTUAL_HOSTING=False) as env: with rasterio.open(f"/vsis3/{BUCKET}/{fkey}") as src: out_image, out_transform = mask(src, features, crop=True, pad=False, all_touched=False) # print(out_image.shape) if not fixscale: w_low = np.percentile(out_image, flut[0]) w_high = np.percentile(out_image, flut[1]) else: w_low = flut[0] w_high = flut[1] if w_low < w_high: chip_set = np.clip( 255 * (out_image - w_low) / (w_high - w_low), 0, 255).astype(np.uint8) else: # Degenerate chips will be zero filled chip_set = np.clip(255 * (out_image - w_low), 0, 255).astype(np.uint8) return chip_set, out_transform
def activate_rio_env(aws=None, cloud_defaults=False, **kwargs): """ Inject activated rasterio.Env into current thread. This de-activates previously setup environment. :param aws: Dictionary of options for rasterio.session.AWSSession OR 'auto' -- session = rasterio.session.AWSSession() :param cloud_defaults: When True inject settings for reading COGs :param **kwargs: Passed on to rasterio.Env(..) constructor """ session = DummySession() if aws is not None: if not (aws == 'auto' or isinstance(aws, dict)): raise ValueError( 'Only support: None|"auto"|{..} for `aws` parameter') aws = {} if aws == 'auto' else dict(**aws) region_name = aws.get('region_name', 'auto') if region_name == 'auto': from datacube.utils.aws import auto_find_region try: aws['region_name'] = auto_find_region() except ValueError as e: # only treat it as error if it was requested by user if 'region_name' in aws: raise e session = AWSSession(**aws) opts = dict( GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR') if cloud_defaults else {} opts.update(**kwargs) deactivate_rio_env() env = rasterio.Env(session=session, **opts) env.__enter__() _local.env = env return get_rio_env()
from rio_tiler_mosaic.methods import defaults from rio_tiler_mosaic.mosaic import mosaic_tiler from cogeo_mosaic import version as mosaic_version from cogeo_mosaic.backends import MosaicBackend from cogeo_mosaic.backends.utils import get_hash from cogeo_mosaic.mosaic import MosaicJSON from cogeo_mosaic_tiler import custom_cmaps, custom_methods from cogeo_mosaic_tiler.ogc import wmts_template from cogeo_mosaic_tiler.utils import _aws_head_object, _get_layer_names, _postprocess cmap.register("custom_above", custom_cmaps.above_cmap) session = boto3_session() s3_client = session.client("s3") aws_session = AWSSession(session=session) PIXSEL_METHODS = { "first": defaults.FirstMethod, "highest": defaults.HighestMethod, "lowest": defaults.LowestMethod, "mean": defaults.MeanMethod, "median": defaults.MedianMethod, "stdev": defaults.StdevMethod, "bdix_stdev": custom_methods.bidx_stddev, } app = API(name="cogeo-mosaic-tiler") params = dict(payload_compression_method="gzip", binary_b64encode=True) if os.environ.get("CORS"): params["cors"] = True
region_name='eu-west-1') s3 = session.client('s3') keyName = f"{BUCKET_FOLDER_INPUT}/{start_str}_{end_str}.zip" upload(io.BytesIO(resp.content), keyName) zip = s3.get_object(Bucket=BUCKET_FOLDER, Key=keyName)['Body'].read() with ZipFile(io.BytesIO(zip)) as theZip: fileNames = theZip.namelist() for fileName in fileNames: if fileName.endswith('_B04.jp2'): upload( theZip.read(fileName), f"{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}/band4.jp2") if fileName.endswith('_B08.jp2'): upload( theZip.read(fileName), f"{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}/band8.jp2") url1 = f"s3://{BUCKET_FOLDER}/{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}/band4.jp2" url2 = f"s3://{BUCKET_FOLDER}/{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}/band8.jp2" with rasterio.env.Env(AWSSession(session)): with rasterio.open(url1) as band4, rasterio.open(url2) as band8: result = calculate_ndvi(band4, band8) print(result.min(), result.max()) upload(result.tobytes(), f"{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}.csv")
def test_aws_session_class(): """AWSSession works""" sesh = AWSSession(aws_access_key_id='foo', aws_secret_access_key='bar') assert sesh._session assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo' assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
def test_aws_session_class_unsigned(): """AWSSession works""" pytest.importorskip("boto3") sesh = AWSSession(aws_unsigned=True) assert sesh._session is None assert sesh.get_credential_options()['AWS_NO_SIGN_REQUEST'] == 'YES'
def test_aws_session_class_unsigned(): """AWSSession works""" pytest.importorskip("boto3") sesh = AWSSession(aws_unsigned=True) assert sesh._session assert sesh.get_credential_options()['AWS_NO_SIGN_REQUEST'] == 'YES'
array_to_image, get_colormap, expression, linear_rescale, _chunks, ) from rio_color.operations import parse_operations from rio_color.utils import scale_dtype, to_math_type from cogeo_tiler.ogc import wmts_template from lambda_proxy.proxy import API app = API(name="cogeo-tiler") aws_session = AWSSession(session=boto3_session()) def _postprocess( tile: numpy.ndarray, mask: numpy.ndarray, rescale: str = None, color_formula: str = None, ) -> numpy.ndarray: """Post-process tile data.""" if rescale: rescale_arr = list(map(float, rescale.split(","))) rescale_arr = list(_chunks(rescale_arr, 2)) if len(rescale_arr) != tile.shape[0]: rescale_arr = ((rescale_arr[0]), ) * tile.shape[0]
import sys from concurrent import futures from functools import partial import boto3 import numpy as np import rasterio from PIL import Image, ImageDraw from sat_giffer import settings from rasterio import transform from rasterio.session import AWSSession from rasterio.vrt import WarpedVRT from rasterio.warp import calculate_default_transform, Resampling session = rasterio.Env( AWSSession(aws_access_key_id=settings.AWS_KEY, aws_secret_access_key=settings.AWS_SECRET)) if 'test' not in \ sys.argv[0] else None MAX_WORKERS = 2 def get_cropped_data_from_bucket(band, key, bounds, vrt_params, out_crs): """ Recovered the data for a given band for a given scene :param band: Number of the band of interest :param key: Tile location on AWS :param bounds: bounding box of the area of interest :param vrt_params: meta dictionary for resulting fle :param out_crs: output coordinate system :return: the cropped data from the band for the bounds """ f = key + 'B0%s.jp2' % band
def __init__(self, session=None, aws_unsigned=False, profile_name=None, session_class=AWSSession, **options): """Create a new GDAL/AWS environment. Note: this class is a context manager. GDAL isn't configured until the context is entered via `with rasterio.Env():` Parameters ---------- session : optional A Session object. aws_unsigned : bool, optional Do not sign cloud requests. profile_name : str, optional A shared credentials profile name, as per boto3. session_class : Session, optional A sub-class of Session. **options : optional A mapping of GDAL configuration options, e.g., `CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`. Returns ------- Env Notes ----- We raise EnvError if the GDAL config options AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS credentials are handled exclusively by boto3. Examples -------- >>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True): ... with rasterio.open("https://example.com/a.tif") as src: ... print(src.profile) For access to secured cloud resources, a Rasterio Session or a foreign session object may be passed to the constructor. >>> import boto3 >>> from rasterio.session import AWSSession >>> boto3_session = boto3.Session(...) >>> with Env(AWSSession(boto3_session)): ... with rasterio.open("s3://mybucket/a.tif") as src: ... print(src.profile) """ aws_access_key_id = options.pop('aws_access_key_id', None) # Before 1.0, Rasterio only supported AWS. We will special # case AWS in 1.0.x. TODO: warn deprecation in 1.1. if aws_access_key_id: warnings.warn( "Passing abstract session keyword arguments is deprecated. " "Pass a Rasterio AWSSession object instead.", RasterioDeprecationWarning) aws_secret_access_key = options.pop('aws_secret_access_key', None) aws_session_token = options.pop('aws_session_token', None) region_name = options.pop('region_name', None) if ('AWS_ACCESS_KEY_ID' in options or 'AWS_SECRET_ACCESS_KEY' in options): raise EnvError( "GDAL's AWS config options can not be directly set. " "AWS credentials are handled exclusively by boto3.") if session: # Passing a session via keyword argument is the canonical # way to configure access to secured cloud resources. if not isinstance(session, Session): warnings.warn( "Passing a boto3 session is deprecated. Pass a Rasterio " "AWSSession object instead.", RasterioDeprecationWarning) session = AWSSession(session=session) self.session = session elif aws_access_key_id or profile_name or aws_unsigned: self.session = AWSSession( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, region_name=region_name, profile_name=profile_name, aws_unsigned=aws_unsigned) elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ: self.session = AWSSession() else: self.session = DummySession() self.options = options.copy() self.context_options = {}
def test_aws_session_class_endpoint(): """Confirm that endpoint_url kwarg works.""" pytest.importorskip("boto3") sesh = AWSSession(endpoint_url="example.com") assert sesh.get_credential_options()['AWS_S3_ENDPOINT'] == 'example.com'
def test_aws_session_class_unsigned(): """AWSSession works""" pytest.importorskip("boto3") sesh = AWSSession(aws_unsigned=True, region_name='us-mountain-1') assert sesh.get_credential_options()['AWS_NO_SIGN_REQUEST'] == 'YES' assert sesh.get_credential_options()['AWS_REGION'] == 'us-mountain-1'
def test_no_credentialization_if_unsigned(monkeypatch): """Don't get credentials if we're not signing, see #1984""" sesh = AWSSession(aws_unsigned=True) assert sesh._creds is None
def test_no_sign_request(monkeypatch): """If AWS_NO_SIGN_REQUEST is set do not default to aws_unsigned=False""" monkeypatch.setenv("AWS_NO_SIGN_REQUEST", "YES") assert AWSSession().unsigned