def test_aws_session_class_session():
"""AWSSession works"""
boto3 = pytest.importorskip("boto3")
sesh = AWSSession(session=boto3.session.Session(aws_access_key_id='foo', aws_secret_access_key='bar'))
assert sesh._session
assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo'
assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
def test_aws_session_class_session():
"""AWSSession works"""
boto3 = pytest.importorskip("boto3")
sesh = AWSSession(session=boto3.session.Session(aws_access_key_id='foo', aws_secret_access_key='bar'))
assert sesh._session
assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo'
assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
def activate_rio_env(aws=None, defaults=True, **kwargs):
""" Inject activated rasterio.Env into current thread.
This de-activates previously setup environment.
:param aws: Dictionary of options for rasterio.session.AWSSession
OR False in which case session won't be setup
OR None -- session = rasterio.session.AWSSession()
:param defaults: Supply False to not inject COG defaults
:param **kwargs: Passed on to rasterio.Env(..) constructor
"""
env_old = getattr(_local, 'env', None)
if env_old is not None:
env_old.__exit__(None, None, None)
_local.env = None
if aws is None:
session = AWSSession()
elif aws is False:
session = None
else:
session = AWSSession(**aws)
opts = dict(
GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR'
) if defaults else {}
opts.update(**kwargs)
env = rasterio.Env(session=session, **opts)
env.__enter__()
_local.env = env
return get_rio_env()
def test_requester_pays():
"""GDAL is configured with requester pays"""
sesh = AWSSession(aws_access_key_id='foo',
aws_secret_access_key='bar',
requester_pays=True)
assert sesh._session
assert sesh.get_credential_options()['AWS_REQUEST_PAYER'] == 'requester'
def _process_file_stream(src_stream,
on_file_cbk,
gdal_opts=None,
region_name=None,
timer=None):
from rasterio.path import parse_path
session = AWSSession(session=_session(region_name))
if timer is not None:
def proc(url, userdata):
t0 = timer()
with rasterio.DatasetReader(parse_path(url),
sharing=False) as f:
on_file_cbk(f, userdata, t0=t0)
else:
def proc(url, userdata):
with rasterio.DatasetReader(parse_path(url),
sharing=False) as f:
on_file_cbk(f, userdata)
with rasterio.Env(session=session, **gdal_opts):
for userdata, url in src_stream:
try:
proc(url, userdata)
except Exception as e:
print('Error when reading: {}\n...({})'.format(
url, str(e)),
file=sys.stderr)
def _warmup():
session = _session(region_name=self._region_name)
with rasterio.Env(session=AWSSession(session=session),
**self._gdal_opts):
if action:
action()
return session.get_credentials()
def test_aws_session_class_profile(tmpdir, monkeypatch):
"""Confirm that profile_name kwarg works."""
pytest.importorskip("boto3")
credentials_file = tmpdir.join('credentials')
credentials_file.write("[testing]\n"
"aws_access_key_id = foo\n"
"aws_secret_access_key = bar\n"
"aws_session_token = baz")
monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', str(credentials_file))
monkeypatch.setenv('AWS_SESSION_TOKEN', 'ignore_me')
sesh = AWSSession(profile_name='testing')
assert sesh._session
assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo'
assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
assert sesh.get_credential_options()['AWS_SESSION_TOKEN'] == 'baz'
monkeypatch.undo()
def test_aws_session_class_profile(tmpdir, monkeypatch):
"""Confirm that profile_name kwarg works."""
pytest.importorskip("boto3")
credentials_file = tmpdir.join('credentials')
credentials_file.write("[testing]\n"
"aws_access_key_id = foo\n"
"aws_secret_access_key = bar\n"
"aws_session_token = baz")
monkeypatch.setenv('AWS_SHARED_CREDENTIALS_FILE', str(credentials_file))
monkeypatch.setenv('AWS_SESSION_TOKEN', 'ignore_me')
sesh = AWSSession(profile_name='testing')
assert sesh._session
assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo'
assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
assert sesh.get_credential_options()['AWS_SESSION_TOKEN'] == 'baz'
monkeypatch.undo()
def get_subset(fkey, features):
import numpy as np
# import pandas as pd
import rasterio
from rasterio.mask import mask
from rasterio.session import AWSSession
s3host_ = S3HOST.replace('http://', '')
s3host_ = s3host_.replace('https://', '')
session = connection('session')
with rasterio.Env(AWSSession(session),
AWS_S3_ENDPOINT=s3host_,
AWS_HTTPS='NO',
AWS_VIRTUAL_HOSTING=False) as env:
with rasterio.open('/vsis3/{}/{}'.format(BUCKET, fkey)) as src:
out_image, out_transform = mask(src,
features,
crop=True,
pad=False,
all_touched=False)
# print('--out_image.shape: ', out_image.shape)
w_5 = np.percentile(out_image, 5.0)
w_95 = np.percentile(out_image, 95.0)
# print('--w_5, w_95: ', w_5, w_95)
chip_set = np.clip(255 * (out_image - w_5) / (w_95 - w_5), 0,
255).astype(np.uint16)
return chip_set, out_transform
def __getitem__(self, i):
match = self.overlap.iloc[i]
s = boto3.Session(profile_name=self.aws_profile)
with rio.Env(AWSSession(s)):
mask = np.squeeze(rio.open(match.path_mask).read())
dataAcquired = False
attempts_remaining = MAX_RETRY_ATTEMPTS
while (not dataAcquired) and (attempts_remaining > 0):
try:
data = rio.open(match.path).read()
dataAcquired = True
except rio.errors.RasterioError as e:
print(
f"Rasterio Read Failure (retrying, {attempts_remaining} attempts remain). Path: {match.path}"
)
attempts_remaining = attempts_remaining - 1
sleep(RETRY_LENGTH_SECONDS)
if not dataAcquired:
raise Exception(
"Rasterio Read Failure, all attempts exhausted. (Path: {match.path})."
)
if self.joint_transform:
augmented = self.joint_transform(image=data, mask=mask)
data = augmented['image']
mask = augmented['mask']
return (torch.FloatTensor(data), torch.from_numpy(mask).double(),
match.name)
def main_group(
ctx,
verbose,
quiet,
aws_profile,
aws_no_sign_requests,
aws_requester_pays,
gdal_version,
):
"""Rasterio command line interface.
"""
verbosity = verbose - quiet
configure_logging(verbosity)
ctx.obj = {}
ctx.obj["verbosity"] = verbosity
ctx.obj["aws_profile"] = aws_profile
envopts = {"CPL_DEBUG": (verbosity > 2)}
if aws_profile or aws_no_sign_requests or aws_requester_pays:
ctx.obj["env"] = rasterio.Env(
session=AWSSession(
profile_name=aws_profile,
aws_unsigned=aws_no_sign_requests,
requester_pays=aws_requester_pays,
), **envopts)
else:
ctx.obj["env"] = rasterio.Env(**envopts)
def activate_rio_env(aws=None, cloud_defaults=False, **kwargs):
""" Inject activated rasterio.Env into current thread.
This de-activates previously setup environment.
:param aws: Dictionary of options for rasterio.session.AWSSession
OR 'auto' -- session = rasterio.session.AWSSession()
:param cloud_defaults: When True inject settings for reading COGs
:param **kwargs: Passed on to rasterio.Env(..) constructor
"""
session = DummySession()
if aws is not None:
if not (aws == "auto" or isinstance(aws, dict)):
raise ValueError(
'Only support: None|"auto"|{..} for `aws` parameter')
aws = {} if aws == "auto" else dict(**aws)
region_name = aws.get("region_name", "auto")
if region_name == "auto":
from datacube.utils.aws import auto_find_region
try:
aws["region_name"] = auto_find_region()
except ValueError as e:
# only treat it as error if it was requested by user
if "region_name" in aws:
raise e
session = AWSSession(**aws)
opts = (dict(
GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR",
GDAL_HTTP_MAX_RETRY="10",
GDAL_HTTP_RETRY_DELAY="0.5",
) if cloud_defaults else {})
opts.update(**kwargs)
state = _state()
if state.env is not None:
state.env.__exit__(None, None, None)
env = rasterio.Env(session=session, **opts)
env.__enter__()
state.env = env
state.epoch = -1
return get_rio_env()
def __getitem__(self, i):
match = self.overlap.iloc[i]
s = boto3.Session(profile_name=self.aws_profile)
with rio.Env(AWSSession(s)):
mask = np.squeeze(rio.open(match.path_mask).read())
data = rio.open(match.path).read()
if self.joint_transform:
augmented = self.joint_transform(image=data, mask=mask)
data = augmented['image']
mask = augmented['mask']
return (torch.FloatTensor(data), torch.from_numpy(mask).double(),
match.name)
def test_nested_credentials(monkeypatch):
"""Check that rasterio.open() doesn't wipe out surrounding credentials"""
@ensure_env_credentialled
def fake_opener(path):
return getenv()
with rasterio.Env(session=AWSSession(aws_access_key_id='foo',
aws_secret_access_key='bar')):
assert getenv()['AWS_ACCESS_KEY_ID'] == 'foo'
assert getenv()['AWS_SECRET_ACCESS_KEY'] == 'bar'
monkeypatch.setenv('AWS_ACCESS_KEY_ID', 'lol')
monkeypatch.setenv('AWS_SECRET_ACCESS_KEY', 'wut')
gdalenv = fake_opener('s3://foo/bar')
assert gdalenv['AWS_ACCESS_KEY_ID'] == 'foo'
assert gdalenv['AWS_SECRET_ACCESS_KEY'] == 'bar'
def download_raster_aws(services, path, dst_path, requester_pays=False):
from rasterio.session import AWSSession
aws_session = AWSSession(services.session, requester_pays=requester_pays)
with rasterio.Env(aws_session, AWS_SESSION_TOKEN=""):
with rasterio.open(path) as src:
profile = src.profile.copy()
profile.update({
'blockxsize': 2048,
'blockysize': 2048,
'tiled': True
})
arr = src.read(1)
with rasterio.open(dst_path, 'w', **profile) as dataset:
dataset.write(arr, 1)
return dst_path
def activate_rio_env(aws=None, defaults=True, **kwargs):
"""Inject activated rasterio.Env into current thread.
This de-activates previously setup environment.
:param aws: Dictionary of options for rasterio.session.AWSSession
OR False in which case session won't be setup
OR None -- session = rasterio.session.AWSSession()
:param defaults: Supply False to not inject COG defaults
:param **kwargs: Passed on to rasterio.Env(..) constructor
"""
env_old = getattr(_local, "env", None)
if env_old is not None:
env_old.__exit__(None, None, None)
_local.env = None
if aws is False:
session = None
else:
aws = {} if aws is None else dict(**aws)
region_name = aws.get("region_name", "auto")
if region_name == "auto":
from odc.aws import auto_find_region
try:
aws["region_name"] = auto_find_region()
except Exception as e:
# only treat it as error if it was requested by user
if "region_name" in aws:
raise e
session = AWSSession(**aws)
opts = dict(GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR") if defaults else {}
opts.update(**kwargs)
env = rasterio.Env(session=session, **opts)
env.__enter__()
_local.env = env
return get_rio_env()
def getSubset(fkey, features, lut, fixscale=False):
try:
flut = [float(l) for l in lut]
logging.debug(flut)
except:
logging.error("Error converting LUT to float values")
flut = [5.0, 95.0]
BUCKET = 'DIAS'
access_key = 'anystring'
secret_key = 'anystring'
session = boto3.Session(aws_access_key_id=access_key,
aws_secret_access_key=secret_key)
with rasterio.Env(AWSSession(session),
AWS_S3_ENDPOINT='data.cloudferro.com',
AWS_HTTPS='NO',
AWS_VIRTUAL_HOSTING=False) as env:
with rasterio.open(f"/vsis3/{BUCKET}/{fkey}") as src:
out_image, out_transform = mask(src,
features,
crop=True,
pad=False,
all_touched=False)
# print(out_image.shape)
if not fixscale:
w_low = np.percentile(out_image, flut[0])
w_high = np.percentile(out_image, flut[1])
else:
w_low = flut[0]
w_high = flut[1]
if w_low < w_high:
chip_set = np.clip(
255 * (out_image - w_low) / (w_high - w_low), 0,
255).astype(np.uint8)
else:
# Degenerate chips will be zero filled
chip_set = np.clip(255 * (out_image - w_low), 0,
255).astype(np.uint8)
return chip_set, out_transform
def activate_rio_env(aws=None, cloud_defaults=False, **kwargs):
""" Inject activated rasterio.Env into current thread.
This de-activates previously setup environment.
:param aws: Dictionary of options for rasterio.session.AWSSession
OR 'auto' -- session = rasterio.session.AWSSession()
:param cloud_defaults: When True inject settings for reading COGs
:param **kwargs: Passed on to rasterio.Env(..) constructor
"""
session = DummySession()
if aws is not None:
if not (aws == 'auto' or isinstance(aws, dict)):
raise ValueError(
'Only support: None|"auto"|{..} for `aws` parameter')
aws = {} if aws == 'auto' else dict(**aws)
region_name = aws.get('region_name', 'auto')
if region_name == 'auto':
from datacube.utils.aws import auto_find_region
try:
aws['region_name'] = auto_find_region()
except ValueError as e:
# only treat it as error if it was requested by user
if 'region_name' in aws:
raise e
session = AWSSession(**aws)
opts = dict(
GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR') if cloud_defaults else {}
opts.update(**kwargs)
deactivate_rio_env()
env = rasterio.Env(session=session, **opts)
env.__enter__()
_local.env = env
return get_rio_env()
from rio_tiler_mosaic.methods import defaults
from rio_tiler_mosaic.mosaic import mosaic_tiler
from cogeo_mosaic import version as mosaic_version
from cogeo_mosaic.backends import MosaicBackend
from cogeo_mosaic.backends.utils import get_hash
from cogeo_mosaic.mosaic import MosaicJSON
from cogeo_mosaic_tiler import custom_cmaps, custom_methods
from cogeo_mosaic_tiler.ogc import wmts_template
from cogeo_mosaic_tiler.utils import _aws_head_object, _get_layer_names, _postprocess
cmap.register("custom_above", custom_cmaps.above_cmap)
session = boto3_session()
s3_client = session.client("s3")
aws_session = AWSSession(session=session)
PIXSEL_METHODS = {
"first": defaults.FirstMethod,
"highest": defaults.HighestMethod,
"lowest": defaults.LowestMethod,
"mean": defaults.MeanMethod,
"median": defaults.MedianMethod,
"stdev": defaults.StdevMethod,
"bdix_stdev": custom_methods.bidx_stddev,
}
app = API(name="cogeo-mosaic-tiler")
params = dict(payload_compression_method="gzip", binary_b64encode=True)
if os.environ.get("CORS"):
params["cors"] = True
def test_requester_pays():
"""GDAL is configured with requester pays"""
sesh = AWSSession(aws_access_key_id='foo', aws_secret_access_key='bar', requester_pays=True)
assert sesh._session
assert sesh.get_credential_options()['AWS_REQUEST_PAYER'] == 'requester'
region_name='eu-west-1')
s3 = session.client('s3')
keyName = f"{BUCKET_FOLDER_INPUT}/{start_str}_{end_str}.zip"
upload(io.BytesIO(resp.content), keyName)
zip = s3.get_object(Bucket=BUCKET_FOLDER, Key=keyName)['Body'].read()
with ZipFile(io.BytesIO(zip)) as theZip:
fileNames = theZip.namelist()
for fileName in fileNames:
if fileName.endswith('_B04.jp2'):
upload(
theZip.read(fileName),
f"{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}/band4.jp2")
if fileName.endswith('_B08.jp2'):
upload(
theZip.read(fileName),
f"{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}/band8.jp2")
url1 = f"s3://{BUCKET_FOLDER}/{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}/band4.jp2"
url2 = f"s3://{BUCKET_FOLDER}/{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}/band8.jp2"
with rasterio.env.Env(AWSSession(session)):
with rasterio.open(url1) as band4, rasterio.open(url2) as band8:
result = calculate_ndvi(band4, band8)
print(result.min(), result.max())
upload(result.tobytes(),
f"{BUCKET_FOLDER_OUTPUT}/{start_str}_{end_str}.csv")
def test_aws_session_class():
"""AWSSession works"""
sesh = AWSSession(aws_access_key_id='foo', aws_secret_access_key='bar')
assert sesh._session
assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo'
assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
def test_aws_session_class_unsigned():
"""AWSSession works"""
pytest.importorskip("boto3")
sesh = AWSSession(aws_unsigned=True)
assert sesh._session is None
assert sesh.get_credential_options()['AWS_NO_SIGN_REQUEST'] == 'YES'
def test_aws_session_class():
"""AWSSession works"""
sesh = AWSSession(aws_access_key_id='foo', aws_secret_access_key='bar')
assert sesh._session
assert sesh.get_credential_options()['AWS_ACCESS_KEY_ID'] == 'foo'
assert sesh.get_credential_options()['AWS_SECRET_ACCESS_KEY'] == 'bar'
def test_aws_session_class_unsigned():
"""AWSSession works"""
pytest.importorskip("boto3")
sesh = AWSSession(aws_unsigned=True)
assert sesh._session
assert sesh.get_credential_options()['AWS_NO_SIGN_REQUEST'] == 'YES'
array_to_image,
get_colormap,
expression,
linear_rescale,
_chunks,
)
from rio_color.operations import parse_operations
from rio_color.utils import scale_dtype, to_math_type
from cogeo_tiler.ogc import wmts_template
from lambda_proxy.proxy import API
app = API(name="cogeo-tiler")
aws_session = AWSSession(session=boto3_session())
def _postprocess(
tile: numpy.ndarray,
mask: numpy.ndarray,
rescale: str = None,
color_formula: str = None,
) -> numpy.ndarray:
"""Post-process tile data."""
if rescale:
rescale_arr = list(map(float, rescale.split(",")))
rescale_arr = list(_chunks(rescale_arr, 2))
if len(rescale_arr) != tile.shape[0]:
rescale_arr = ((rescale_arr[0]), ) * tile.shape[0]
import sys
from concurrent import futures
from functools import partial
import boto3
import numpy as np
import rasterio
from PIL import Image, ImageDraw
from sat_giffer import settings
from rasterio import transform
from rasterio.session import AWSSession
from rasterio.vrt import WarpedVRT
from rasterio.warp import calculate_default_transform, Resampling
session = rasterio.Env(
AWSSession(aws_access_key_id=settings.AWS_KEY, aws_secret_access_key=settings.AWS_SECRET)) if 'test' not in \
sys.argv[0] else None
MAX_WORKERS = 2
def get_cropped_data_from_bucket(band, key, bounds, vrt_params, out_crs):
"""
Recovered the data for a given band for a given scene
:param band: Number of the band of interest
:param key: Tile location on AWS
:param bounds: bounding box of the area of interest
:param vrt_params: meta dictionary for resulting fle
:param out_crs: output coordinate system
:return: the cropped data from the band for the bounds
"""
f = key + 'B0%s.jp2' % band
def __init__(self,
session=None,
aws_unsigned=False,
profile_name=None,
session_class=AWSSession,
**options):
"""Create a new GDAL/AWS environment.
Note: this class is a context manager. GDAL isn't configured
until the context is entered via `with rasterio.Env():`
Parameters
----------
session : optional
A Session object.
aws_unsigned : bool, optional
Do not sign cloud requests.
profile_name : str, optional
A shared credentials profile name, as per boto3.
session_class : Session, optional
A sub-class of Session.
**options : optional
A mapping of GDAL configuration options, e.g.,
`CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`.
Returns
-------
Env
Notes
-----
We raise EnvError if the GDAL config options
AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS
credentials are handled exclusively by boto3.
Examples
--------
>>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True):
... with rasterio.open("https://example.com/a.tif") as src:
... print(src.profile)
For access to secured cloud resources, a Rasterio Session or a
foreign session object may be passed to the constructor.
>>> import boto3
>>> from rasterio.session import AWSSession
>>> boto3_session = boto3.Session(...)
>>> with Env(AWSSession(boto3_session)):
... with rasterio.open("s3://mybucket/a.tif") as src:
... print(src.profile)
"""
aws_access_key_id = options.pop('aws_access_key_id', None)
# Before 1.0, Rasterio only supported AWS. We will special
# case AWS in 1.0.x. TODO: warn deprecation in 1.1.
if aws_access_key_id:
warnings.warn(
"Passing abstract session keyword arguments is deprecated. "
"Pass a Rasterio AWSSession object instead.",
RasterioDeprecationWarning)
aws_secret_access_key = options.pop('aws_secret_access_key', None)
aws_session_token = options.pop('aws_session_token', None)
region_name = options.pop('region_name', None)
if ('AWS_ACCESS_KEY_ID' in options
or 'AWS_SECRET_ACCESS_KEY' in options):
raise EnvError(
"GDAL's AWS config options can not be directly set. "
"AWS credentials are handled exclusively by boto3.")
if session:
# Passing a session via keyword argument is the canonical
# way to configure access to secured cloud resources.
if not isinstance(session, Session):
warnings.warn(
"Passing a boto3 session is deprecated. Pass a Rasterio "
"AWSSession object instead.", RasterioDeprecationWarning)
session = AWSSession(session=session)
self.session = session
elif aws_access_key_id or profile_name or aws_unsigned:
self.session = AWSSession(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
region_name=region_name,
profile_name=profile_name,
aws_unsigned=aws_unsigned)
elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ:
self.session = AWSSession()
else:
self.session = DummySession()
self.options = options.copy()
self.context_options = {}
def test_aws_session_class_endpoint():
"""Confirm that endpoint_url kwarg works."""
pytest.importorskip("boto3")
sesh = AWSSession(endpoint_url="example.com")
assert sesh.get_credential_options()['AWS_S3_ENDPOINT'] == 'example.com'
def test_aws_session_class_unsigned():
"""AWSSession works"""
pytest.importorskip("boto3")
sesh = AWSSession(aws_unsigned=True, region_name='us-mountain-1')
assert sesh.get_credential_options()['AWS_NO_SIGN_REQUEST'] == 'YES'
assert sesh.get_credential_options()['AWS_REGION'] == 'us-mountain-1'
def test_no_credentialization_if_unsigned(monkeypatch):
"""Don't get credentials if we're not signing, see #1984"""
sesh = AWSSession(aws_unsigned=True)
assert sesh._creds is None
def test_no_sign_request(monkeypatch):
"""If AWS_NO_SIGN_REQUEST is set do not default to aws_unsigned=False"""
monkeypatch.setenv("AWS_NO_SIGN_REQUEST", "YES")
assert AWSSession().unsigned
