def activate_rio_env(aws=None, cloud_defaults=False, **kwargs):
""" Inject activated rasterio.Env into current thread.
This de-activates previously setup environment.
:param aws: Dictionary of options for rasterio.session.AWSSession
OR 'auto' -- session = rasterio.session.AWSSession()
:param cloud_defaults: When True inject settings for reading COGs
:param **kwargs: Passed on to rasterio.Env(..) constructor
"""
session = DummySession()
if aws is not None:
if not (aws == "auto" or isinstance(aws, dict)):
raise ValueError(
'Only support: None|"auto"|{..} for `aws` parameter')
aws = {} if aws == "auto" else dict(**aws)
region_name = aws.get("region_name", "auto")
if region_name == "auto":
from datacube.utils.aws import auto_find_region
try:
aws["region_name"] = auto_find_region()
except ValueError as e:
# only treat it as error if it was requested by user
if "region_name" in aws:
raise e
session = AWSSession(**aws)
opts = (dict(
GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR",
GDAL_HTTP_MAX_RETRY="10",
GDAL_HTTP_RETRY_DELAY="0.5",
) if cloud_defaults else {})
opts.update(**kwargs)
state = _state()
if state.env is not None:
state.env.__exit__(None, None, None)
env = rasterio.Env(session=session, **opts)
env.__enter__()
state.env = env
state.epoch = -1
return get_rio_env()
def wrapper(*args, **kwds):
if local._env:
env_ctor = Env
else:
env_ctor = Env.from_defaults
if isinstance(args[0], str):
session_cls = Session.cls_from_path(args[0])
if local._env and session_cls.hascreds(getenv()):
session_cls = DummySession
session = session_cls()
else:
session = DummySession()
with env_ctor(session=session):
return f(*args, **kwds)
def activate_rio_env(aws=None, cloud_defaults=False, **kwargs):
""" Inject activated rasterio.Env into current thread.
This de-activates previously setup environment.
:param aws: Dictionary of options for rasterio.session.AWSSession
OR 'auto' -- session = rasterio.session.AWSSession()
:param cloud_defaults: When True inject settings for reading COGs
:param **kwargs: Passed on to rasterio.Env(..) constructor
"""
session = DummySession()
if aws is not None:
if not (aws == 'auto' or isinstance(aws, dict)):
raise ValueError(
'Only support: None|"auto"|{..} for `aws` parameter')
aws = {} if aws == 'auto' else dict(**aws)
region_name = aws.get('region_name', 'auto')
if region_name == 'auto':
from datacube.utils.aws import auto_find_region
try:
aws['region_name'] = auto_find_region()
except ValueError as e:
# only treat it as error if it was requested by user
if 'region_name' in aws:
raise e
session = AWSSession(**aws)
opts = dict(
GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR') if cloud_defaults else {}
opts.update(**kwargs)
deactivate_rio_env()
env = rasterio.Env(session=session, **opts)
env.__enter__()
_local.env = env
return get_rio_env()
def wrapper(*args, **kwds):
if local._env:
env_ctor = Env
else:
env_ctor = Env.from_defaults
fp_arg = kwds.get("fp", None) or args[0]
if isinstance(fp_arg, str):
session_cls = Session.cls_from_path(fp_arg)
if local._env and session_cls.hascreds(getenv()):
session_cls = DummySession
session = session_cls()
else:
session = DummySession()
with env_ctor(session=session):
return f(*args, **kwds)
def test_dummy_session():
"""DummySession works"""
sesh = DummySession()
assert sesh._session is None
assert sesh.get_credential_options() == {}
class Env(object):
"""Abstraction for GDAL and AWS configuration
The GDAL library is stateful: it has a registry of format drivers,
an error stack, and dozens of configuration options.
Rasterio's approach to working with GDAL is to wrap all the state
up using a Python context manager (see PEP 343,
https://www.python.org/dev/peps/pep-0343/). When the context is
entered GDAL drivers are registered, error handlers are
configured, and configuration options are set. When the context
is exited, drivers are removed from the registry and other
configurations are removed.
Example:
with rasterio.Env(GDAL_CACHEMAX=128000000) as env:
# All drivers are registered, GDAL's raster block cache
# size is set to 128 MB.
# Commence processing...
...
# End of processing.
# At this point, configuration options are set to their
# previous (possible unset) values.
A boto3 session or boto3 session constructor arguments
`aws_access_key_id`, `aws_secret_access_key`, `aws_session_token`
may be passed to Env's constructor. In the latter case, a session
will be created as soon as needed. AWS credentials are configured
for GDAL as needed.
"""
@classmethod
def default_options(cls):
"""Default configuration options
Parameters
----------
None
Returns
-------
dict
"""
return {'GTIFF_IMPLICIT_JPEG_OVR': False, "RASTERIO_ENV": True}
def __init__(self,
session=None,
aws_unsigned=False,
profile_name=None,
session_class=Session.aws_or_dummy,
**options):
"""Create a new GDAL/AWS environment.
Note: this class is a context manager. GDAL isn't configured
until the context is entered via `with rasterio.Env():`
Parameters
----------
session : optional
A Session object.
aws_unsigned : bool, optional
Do not sign cloud requests.
profile_name : str, optional
A shared credentials profile name, as per boto3.
session_class : Session, optional
A sub-class of Session.
**options : optional
A mapping of GDAL configuration options, e.g.,
`CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`.
Returns
-------
Env
Notes
-----
We raise EnvError if the GDAL config options
AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS
credentials are handled exclusively by boto3.
Examples
--------
>>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True):
... with rasterio.open("https://example.com/a.tif") as src:
... print(src.profile)
For access to secured cloud resources, a Rasterio Session or a
foreign session object may be passed to the constructor.
>>> import boto3
>>> from rasterio.session import AWSSession
>>> boto3_session = boto3.Session(...)
>>> with Env(AWSSession(boto3_session)):
... with rasterio.open("s3://mybucket/a.tif") as src:
... print(src.profile)
"""
aws_access_key_id = options.pop('aws_access_key_id', None)
# Before 1.0, Rasterio only supported AWS. We will special
# case AWS in 1.0.x. TODO: warn deprecation in 1.1.
if aws_access_key_id:
warnings.warn(
"Passing abstract session keyword arguments is deprecated. "
"Pass a Rasterio AWSSession object instead.",
RasterioDeprecationWarning)
aws_secret_access_key = options.pop('aws_secret_access_key', None)
aws_session_token = options.pop('aws_session_token', None)
region_name = options.pop('region_name', None)
if ('AWS_ACCESS_KEY_ID' in options
or 'AWS_SECRET_ACCESS_KEY' in options):
raise EnvError(
"GDAL's AWS config options can not be directly set. "
"AWS credentials are handled exclusively by boto3.")
if session:
# Passing a session via keyword argument is the canonical
# way to configure access to secured cloud resources.
if not isinstance(session, Session):
warnings.warn(
"Passing a boto3 session is deprecated. Pass a Rasterio "
"AWSSession object instead.", RasterioDeprecationWarning)
session = Session.aws_or_dummy(session=session)
self.session = session
elif aws_access_key_id or profile_name or aws_unsigned:
self.session = Session.aws_or_dummy(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
region_name=region_name,
profile_name=profile_name,
aws_unsigned=aws_unsigned)
elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ:
self.session = Session.from_environ()
else:
self.session = DummySession()
self.options = options.copy()
self.context_options = {}
@classmethod
def from_defaults(cls, *args, **kwargs):
"""Create an environment with default config options
Parameters
----------
args : optional
Positional arguments for Env()
kwargs : optional
Keyword arguments for Env()
Returns
-------
Env
Notes
-----
The items in kwargs will be overlaid on the default values.
"""
options = Env.default_options()
options.update(**kwargs)
return Env(*args, **options)
def credentialize(self):
"""Get credentials and configure GDAL
Note well: this method is a no-op if the GDAL environment
already has credentials, unless session is not None.
Returns
-------
None
"""
cred_opts = self.session.get_credential_options()
self.options.update(**cred_opts)
setenv(**cred_opts)
def drivers(self):
"""Return a mapping of registered drivers."""
return local._env.drivers()
def _dump_open_datasets(self):
"""Writes descriptions of open datasets to stderr
For debugging and testing purposes.
"""
return local._env._dump_open_datasets()
def __enter__(self):
log.debug("Entering env context: %r", self)
if local._env is None:
log.debug("Starting outermost env")
self._has_parent_env = False
# See note directly above where _discovered_options is globally
# defined. This MUST happen before calling 'defenv()'.
local._discovered_options = {}
# Don't want to reinstate the "RASTERIO_ENV" option.
probe_env = {k for k in self.options.keys() if k != "RASTERIO_ENV"}
for key in probe_env:
val = get_gdal_config(key, normalize=False)
if val is not None:
local._discovered_options[key] = val
defenv(**self.options)
self.context_options = {}
else:
self._has_parent_env = True
self.context_options = getenv()
setenv(**self.options)
self.credentialize()
log.debug("Entered env context: %r", self)
return self
def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
log.debug("Exiting env context: %r", self)
delenv()
if self._has_parent_env:
defenv()
setenv(**self.context_options)
else:
log.debug("Exiting outermost env")
# See note directly above where _discovered_options is globally
# defined.
while local._discovered_options:
key, val = local._discovered_options.popitem()
set_gdal_config(key, val, normalize=False)
local._discovered_options = None
log.debug("Exited env context: %r", self)
def __init__(self,
session=None,
aws_unsigned=False,
profile_name=None,
session_class=Session.aws_or_dummy,
**options):
"""Create a new GDAL/AWS environment.
Note: this class is a context manager. GDAL isn't configured
until the context is entered via `with rasterio.Env():`
Parameters
----------
session : optional
A Session object.
aws_unsigned : bool, optional
Do not sign cloud requests.
profile_name : str, optional
A shared credentials profile name, as per boto3.
session_class : Session, optional
A sub-class of Session.
**options : optional
A mapping of GDAL configuration options, e.g.,
`CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`.
Returns
-------
Env
Notes
-----
We raise EnvError if the GDAL config options
AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS
credentials are handled exclusively by boto3.
Examples
--------
>>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True):
... with rasterio.open("https://example.com/a.tif") as src:
... print(src.profile)
For access to secured cloud resources, a Rasterio Session or a
foreign session object may be passed to the constructor.
>>> import boto3
>>> from rasterio.session import AWSSession
>>> boto3_session = boto3.Session(...)
>>> with Env(AWSSession(boto3_session)):
... with rasterio.open("s3://mybucket/a.tif") as src:
... print(src.profile)
"""
aws_access_key_id = options.pop('aws_access_key_id', None)
# Before 1.0, Rasterio only supported AWS. We will special
# case AWS in 1.0.x. TODO: warn deprecation in 1.1.
if aws_access_key_id:
warnings.warn(
"Passing abstract session keyword arguments is deprecated. "
"Pass a Rasterio AWSSession object instead.",
RasterioDeprecationWarning)
aws_secret_access_key = options.pop('aws_secret_access_key', None)
aws_session_token = options.pop('aws_session_token', None)
region_name = options.pop('region_name', None)
if ('AWS_ACCESS_KEY_ID' in options
or 'AWS_SECRET_ACCESS_KEY' in options):
raise EnvError(
"GDAL's AWS config options can not be directly set. "
"AWS credentials are handled exclusively by boto3.")
if session:
# Passing a session via keyword argument is the canonical
# way to configure access to secured cloud resources.
if not isinstance(session, Session):
warnings.warn(
"Passing a boto3 session is deprecated. Pass a Rasterio "
"AWSSession object instead.", RasterioDeprecationWarning)
session = Session.aws_or_dummy(session=session)
self.session = session
elif aws_access_key_id or profile_name or aws_unsigned:
self.session = Session.aws_or_dummy(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
region_name=region_name,
profile_name=profile_name,
aws_unsigned=aws_unsigned)
elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ:
self.session = Session.from_environ()
else:
self.session = DummySession()
self.options = options.copy()
self.context_options = {}
def test_dummy_session():
"""DummySession works"""
sesh = DummySession()
assert sesh._session is None
assert sesh.get_credential_options() == {}
class Env(object):
"""Abstraction for GDAL and AWS configuration
The GDAL library is stateful: it has a registry of format drivers,
an error stack, and dozens of configuration options.
Rasterio's approach to working with GDAL is to wrap all the state
up using a Python context manager (see PEP 343,
https://www.python.org/dev/peps/pep-0343/). When the context is
entered GDAL drivers are registered, error handlers are
configured, and configuration options are set. When the context
is exited, drivers are removed from the registry and other
configurations are removed.
Example:
with rasterio.Env(GDAL_CACHEMAX=512) as env:
# All drivers are registered, GDAL's raster block cache
# size is set to 512MB.
# Commence processing...
...
# End of processing.
# At this point, configuration options are set to their
# previous (possible unset) values.
A boto3 session or boto3 session constructor arguments
`aws_access_key_id`, `aws_secret_access_key`, `aws_session_token`
may be passed to Env's constructor. In the latter case, a session
will be created as soon as needed. AWS credentials are configured
for GDAL as needed.
"""
@classmethod
def default_options(cls):
"""Default configuration options
Parameters
----------
None
Returns
-------
dict
"""
return {
'CHECK_WITH_INVERT_PROJ': True,
'GTIFF_IMPLICIT_JPEG_OVR': False,
"RASTERIO_ENV": True
}
def __init__(self, session=None, aws_unsigned=False, profile_name=None,
session_class=AWSSession, **options):
"""Create a new GDAL/AWS environment.
Note: this class is a context manager. GDAL isn't configured
until the context is entered via `with rasterio.Env():`
Parameters
----------
session : optional
A Session object.
aws_unsigned : bool, optional
Do not sign cloud requests.
profile_name : str, optional
A shared credentials profile name, as per boto3.
session_class : Session, optional
A sub-class of Session.
**options : optional
A mapping of GDAL configuration options, e.g.,
`CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`.
Returns
-------
Env
Notes
-----
We raise EnvError if the GDAL config options
AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS
credentials are handled exclusively by boto3.
Examples
--------
>>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True):
... with rasterio.open("https://example.com/a.tif") as src:
... print(src.profile)
For access to secured cloud resources, a Rasterio Session or a
foreign session object may be passed to the constructor.
>>> import boto3
>>> from rasterio.session import AWSSession
>>> boto3_session = boto3.Session(...)
>>> with Env(AWSSession(boto3_session)):
... with rasterio.open("s3://mybucket/a.tif") as src:
... print(src.profile)
"""
aws_access_key_id = options.pop('aws_access_key_id', None)
# Before 1.0, Rasterio only supported AWS. We will special
# case AWS in 1.0.x. TODO: warn deprecation in 1.1.
if aws_access_key_id:
warnings.warn(
"Passing abstract session keyword arguments is deprecated. "
"Pass a Rasterio AWSSession object instead.",
RasterioDeprecationWarning
)
aws_secret_access_key = options.pop('aws_secret_access_key', None)
aws_session_token = options.pop('aws_session_token', None)
region_name = options.pop('region_name', None)
if ('AWS_ACCESS_KEY_ID' in options or
'AWS_SECRET_ACCESS_KEY' in options):
raise EnvError(
"GDAL's AWS config options can not be directly set. "
"AWS credentials are handled exclusively by boto3.")
if session:
# Passing a session via keyword argument is the canonical
# way to configure access to secured cloud resources.
if not isinstance(session, Session):
warnings.warn(
"Passing a boto3 session is deprecated. Pass a Rasterio "
"AWSSession object instead.",
RasterioDeprecationWarning
)
session = AWSSession(session=session)
self.session = session
elif aws_access_key_id or profile_name or aws_unsigned:
self.session = AWSSession(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
region_name=region_name,
profile_name=profile_name,
aws_unsigned=aws_unsigned)
elif 'AWS_ACCESS_KEY_ID' in os.environ:
self.session = AWSSession()
else:
self.session = DummySession()
self.options = options.copy()
self.context_options = {}
@classmethod
def from_defaults(cls, *args, **kwargs):
"""Create an environment with default config options
Parameters
----------
args : optional
Positional arguments for Env()
kwargs : optional
Keyword arguments for Env()
Returns
-------
Env
Notes
-----
The items in kwargs will be overlaid on the default values.
"""
options = Env.default_options()
options.update(**kwargs)
return Env(*args, **options)
@property
def is_credentialized(self):
"""Test for existence of cloud credentials
Returns
-------
bool
"""
return hascreds() # bool(self.session)
def credentialize(self):
"""Get credentials and configure GDAL
Note well: this method is a no-op if the GDAL environment
already has credentials, unless session is not None.
Returns
-------
None
"""
if hascreds():
pass
else:
cred_opts = self.session.get_credential_options()
self.options.update(**cred_opts)
setenv(**cred_opts)
def drivers(self):
"""Return a mapping of registered drivers."""
return local._env.drivers()
def __enter__(self):
log.debug("Entering env context: %r", self)
if local._env is None:
log.debug("Starting outermost env")
self._has_parent_env = False
# See note directly above where _discovered_options is globally
# defined. This MUST happen before calling 'defenv()'.
local._discovered_options = {}
# Don't want to reinstate the "RASTERIO_ENV" option.
probe_env = {k for k in self.options.keys() if k != "RASTERIO_ENV"}
for key in probe_env:
val = get_gdal_config(key, normalize=False)
if val is not None:
local._discovered_options[key] = val
defenv(**self.options)
self.context_options = {}
else:
self._has_parent_env = True
self.context_options = getenv()
setenv(**self.options)
self.credentialize()
log.debug("Entered env context: %r", self)
return self
def __exit__(self, exc_type=None, exc_val=None, exc_tb=None):
log.debug("Exiting env context: %r", self)
delenv()
if self._has_parent_env:
defenv()
setenv(**self.context_options)
else:
log.debug("Exiting outermost env")
# See note directly above where _discovered_options is globally
# defined.
while local._discovered_options:
key, val = local._discovered_options.popitem()
set_gdal_config(key, val, normalize=False)
local._discovered_options = None
log.debug("Exited env context: %r", self)
def __init__(self, session=None, aws_unsigned=False, profile_name=None,
session_class=AWSSession, **options):
"""Create a new GDAL/AWS environment.
Note: this class is a context manager. GDAL isn't configured
until the context is entered via `with rasterio.Env():`
Parameters
----------
session : optional
A Session object.
aws_unsigned : bool, optional
Do not sign cloud requests.
profile_name : str, optional
A shared credentials profile name, as per boto3.
session_class : Session, optional
A sub-class of Session.
**options : optional
A mapping of GDAL configuration options, e.g.,
`CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`.
Returns
-------
Env
Notes
-----
We raise EnvError if the GDAL config options
AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS
credentials are handled exclusively by boto3.
Examples
--------
>>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True):
... with rasterio.open("https://example.com/a.tif") as src:
... print(src.profile)
For access to secured cloud resources, a Rasterio Session or a
foreign session object may be passed to the constructor.
>>> import boto3
>>> from rasterio.session import AWSSession
>>> boto3_session = boto3.Session(...)
>>> with Env(AWSSession(boto3_session)):
... with rasterio.open("s3://mybucket/a.tif") as src:
... print(src.profile)
"""
aws_access_key_id = options.pop('aws_access_key_id', None)
# Before 1.0, Rasterio only supported AWS. We will special
# case AWS in 1.0.x. TODO: warn deprecation in 1.1.
if aws_access_key_id:
warnings.warn(
"Passing abstract session keyword arguments is deprecated. "
"Pass a Rasterio AWSSession object instead.",
RasterioDeprecationWarning
)
aws_secret_access_key = options.pop('aws_secret_access_key', None)
aws_session_token = options.pop('aws_session_token', None)
region_name = options.pop('region_name', None)
if ('AWS_ACCESS_KEY_ID' in options or
'AWS_SECRET_ACCESS_KEY' in options):
raise EnvError(
"GDAL's AWS config options can not be directly set. "
"AWS credentials are handled exclusively by boto3.")
if session:
# Passing a session via keyword argument is the canonical
# way to configure access to secured cloud resources.
if not isinstance(session, Session):
warnings.warn(
"Passing a boto3 session is deprecated. Pass a Rasterio "
"AWSSession object instead.",
RasterioDeprecationWarning
)
session = AWSSession(session=session)
self.session = session
elif aws_access_key_id or profile_name or aws_unsigned:
self.session = AWSSession(
aws_access_key_id=aws_access_key_id,
aws_secret_access_key=aws_secret_access_key,
aws_session_token=aws_session_token,
region_name=region_name,
profile_name=profile_name,
aws_unsigned=aws_unsigned)
elif 'AWS_ACCESS_KEY_ID' in os.environ:
self.session = AWSSession()
else:
self.session = DummySession()
self.options = options.copy()
self.context_options = {}
