def activate_rio_env(aws=None, cloud_defaults=False, **kwargs): """ Inject activated rasterio.Env into current thread. This de-activates previously setup environment. :param aws: Dictionary of options for rasterio.session.AWSSession OR 'auto' -- session = rasterio.session.AWSSession() :param cloud_defaults: When True inject settings for reading COGs :param **kwargs: Passed on to rasterio.Env(..) constructor """ session = DummySession() if aws is not None: if not (aws == "auto" or isinstance(aws, dict)): raise ValueError( 'Only support: None|"auto"|{..} for `aws` parameter') aws = {} if aws == "auto" else dict(**aws) region_name = aws.get("region_name", "auto") if region_name == "auto": from datacube.utils.aws import auto_find_region try: aws["region_name"] = auto_find_region() except ValueError as e: # only treat it as error if it was requested by user if "region_name" in aws: raise e session = AWSSession(**aws) opts = (dict( GDAL_DISABLE_READDIR_ON_OPEN="EMPTY_DIR", GDAL_HTTP_MAX_RETRY="10", GDAL_HTTP_RETRY_DELAY="0.5", ) if cloud_defaults else {}) opts.update(**kwargs) state = _state() if state.env is not None: state.env.__exit__(None, None, None) env = rasterio.Env(session=session, **opts) env.__enter__() state.env = env state.epoch = -1 return get_rio_env()
def wrapper(*args, **kwds): if local._env: env_ctor = Env else: env_ctor = Env.from_defaults if isinstance(args[0], str): session_cls = Session.cls_from_path(args[0]) if local._env and session_cls.hascreds(getenv()): session_cls = DummySession session = session_cls() else: session = DummySession() with env_ctor(session=session): return f(*args, **kwds)
def activate_rio_env(aws=None, cloud_defaults=False, **kwargs): """ Inject activated rasterio.Env into current thread. This de-activates previously setup environment. :param aws: Dictionary of options for rasterio.session.AWSSession OR 'auto' -- session = rasterio.session.AWSSession() :param cloud_defaults: When True inject settings for reading COGs :param **kwargs: Passed on to rasterio.Env(..) constructor """ session = DummySession() if aws is not None: if not (aws == 'auto' or isinstance(aws, dict)): raise ValueError( 'Only support: None|"auto"|{..} for `aws` parameter') aws = {} if aws == 'auto' else dict(**aws) region_name = aws.get('region_name', 'auto') if region_name == 'auto': from datacube.utils.aws import auto_find_region try: aws['region_name'] = auto_find_region() except ValueError as e: # only treat it as error if it was requested by user if 'region_name' in aws: raise e session = AWSSession(**aws) opts = dict( GDAL_DISABLE_READDIR_ON_OPEN='EMPTY_DIR') if cloud_defaults else {} opts.update(**kwargs) deactivate_rio_env() env = rasterio.Env(session=session, **opts) env.__enter__() _local.env = env return get_rio_env()
def wrapper(*args, **kwds): if local._env: env_ctor = Env else: env_ctor = Env.from_defaults fp_arg = kwds.get("fp", None) or args[0] if isinstance(fp_arg, str): session_cls = Session.cls_from_path(fp_arg) if local._env and session_cls.hascreds(getenv()): session_cls = DummySession session = session_cls() else: session = DummySession() with env_ctor(session=session): return f(*args, **kwds)
def test_dummy_session(): """DummySession works""" sesh = DummySession() assert sesh._session is None assert sesh.get_credential_options() == {}
class Env(object): """Abstraction for GDAL and AWS configuration The GDAL library is stateful: it has a registry of format drivers, an error stack, and dozens of configuration options. Rasterio's approach to working with GDAL is to wrap all the state up using a Python context manager (see PEP 343, https://www.python.org/dev/peps/pep-0343/). When the context is entered GDAL drivers are registered, error handlers are configured, and configuration options are set. When the context is exited, drivers are removed from the registry and other configurations are removed. Example: with rasterio.Env(GDAL_CACHEMAX=128000000) as env: # All drivers are registered, GDAL's raster block cache # size is set to 128 MB. # Commence processing... ... # End of processing. # At this point, configuration options are set to their # previous (possible unset) values. A boto3 session or boto3 session constructor arguments `aws_access_key_id`, `aws_secret_access_key`, `aws_session_token` may be passed to Env's constructor. In the latter case, a session will be created as soon as needed. AWS credentials are configured for GDAL as needed. """ @classmethod def default_options(cls): """Default configuration options Parameters ---------- None Returns ------- dict """ return {'GTIFF_IMPLICIT_JPEG_OVR': False, "RASTERIO_ENV": True} def __init__(self, session=None, aws_unsigned=False, profile_name=None, session_class=Session.aws_or_dummy, **options): """Create a new GDAL/AWS environment. Note: this class is a context manager. GDAL isn't configured until the context is entered via `with rasterio.Env():` Parameters ---------- session : optional A Session object. aws_unsigned : bool, optional Do not sign cloud requests. profile_name : str, optional A shared credentials profile name, as per boto3. session_class : Session, optional A sub-class of Session. **options : optional A mapping of GDAL configuration options, e.g., `CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`. Returns ------- Env Notes ----- We raise EnvError if the GDAL config options AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS credentials are handled exclusively by boto3. Examples -------- >>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True): ... with rasterio.open("https://example.com/a.tif") as src: ... print(src.profile) For access to secured cloud resources, a Rasterio Session or a foreign session object may be passed to the constructor. >>> import boto3 >>> from rasterio.session import AWSSession >>> boto3_session = boto3.Session(...) >>> with Env(AWSSession(boto3_session)): ... with rasterio.open("s3://mybucket/a.tif") as src: ... print(src.profile) """ aws_access_key_id = options.pop('aws_access_key_id', None) # Before 1.0, Rasterio only supported AWS. We will special # case AWS in 1.0.x. TODO: warn deprecation in 1.1. if aws_access_key_id: warnings.warn( "Passing abstract session keyword arguments is deprecated. " "Pass a Rasterio AWSSession object instead.", RasterioDeprecationWarning) aws_secret_access_key = options.pop('aws_secret_access_key', None) aws_session_token = options.pop('aws_session_token', None) region_name = options.pop('region_name', None) if ('AWS_ACCESS_KEY_ID' in options or 'AWS_SECRET_ACCESS_KEY' in options): raise EnvError( "GDAL's AWS config options can not be directly set. " "AWS credentials are handled exclusively by boto3.") if session: # Passing a session via keyword argument is the canonical # way to configure access to secured cloud resources. if not isinstance(session, Session): warnings.warn( "Passing a boto3 session is deprecated. Pass a Rasterio " "AWSSession object instead.", RasterioDeprecationWarning) session = Session.aws_or_dummy(session=session) self.session = session elif aws_access_key_id or profile_name or aws_unsigned: self.session = Session.aws_or_dummy( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, region_name=region_name, profile_name=profile_name, aws_unsigned=aws_unsigned) elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ: self.session = Session.from_environ() else: self.session = DummySession() self.options = options.copy() self.context_options = {} @classmethod def from_defaults(cls, *args, **kwargs): """Create an environment with default config options Parameters ---------- args : optional Positional arguments for Env() kwargs : optional Keyword arguments for Env() Returns ------- Env Notes ----- The items in kwargs will be overlaid on the default values. """ options = Env.default_options() options.update(**kwargs) return Env(*args, **options) def credentialize(self): """Get credentials and configure GDAL Note well: this method is a no-op if the GDAL environment already has credentials, unless session is not None. Returns ------- None """ cred_opts = self.session.get_credential_options() self.options.update(**cred_opts) setenv(**cred_opts) def drivers(self): """Return a mapping of registered drivers.""" return local._env.drivers() def _dump_open_datasets(self): """Writes descriptions of open datasets to stderr For debugging and testing purposes. """ return local._env._dump_open_datasets() def __enter__(self): log.debug("Entering env context: %r", self) if local._env is None: log.debug("Starting outermost env") self._has_parent_env = False # See note directly above where _discovered_options is globally # defined. This MUST happen before calling 'defenv()'. local._discovered_options = {} # Don't want to reinstate the "RASTERIO_ENV" option. probe_env = {k for k in self.options.keys() if k != "RASTERIO_ENV"} for key in probe_env: val = get_gdal_config(key, normalize=False) if val is not None: local._discovered_options[key] = val defenv(**self.options) self.context_options = {} else: self._has_parent_env = True self.context_options = getenv() setenv(**self.options) self.credentialize() log.debug("Entered env context: %r", self) return self def __exit__(self, exc_type=None, exc_val=None, exc_tb=None): log.debug("Exiting env context: %r", self) delenv() if self._has_parent_env: defenv() setenv(**self.context_options) else: log.debug("Exiting outermost env") # See note directly above where _discovered_options is globally # defined. while local._discovered_options: key, val = local._discovered_options.popitem() set_gdal_config(key, val, normalize=False) local._discovered_options = None log.debug("Exited env context: %r", self)
def __init__(self, session=None, aws_unsigned=False, profile_name=None, session_class=Session.aws_or_dummy, **options): """Create a new GDAL/AWS environment. Note: this class is a context manager. GDAL isn't configured until the context is entered via `with rasterio.Env():` Parameters ---------- session : optional A Session object. aws_unsigned : bool, optional Do not sign cloud requests. profile_name : str, optional A shared credentials profile name, as per boto3. session_class : Session, optional A sub-class of Session. **options : optional A mapping of GDAL configuration options, e.g., `CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`. Returns ------- Env Notes ----- We raise EnvError if the GDAL config options AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS credentials are handled exclusively by boto3. Examples -------- >>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True): ... with rasterio.open("https://example.com/a.tif") as src: ... print(src.profile) For access to secured cloud resources, a Rasterio Session or a foreign session object may be passed to the constructor. >>> import boto3 >>> from rasterio.session import AWSSession >>> boto3_session = boto3.Session(...) >>> with Env(AWSSession(boto3_session)): ... with rasterio.open("s3://mybucket/a.tif") as src: ... print(src.profile) """ aws_access_key_id = options.pop('aws_access_key_id', None) # Before 1.0, Rasterio only supported AWS. We will special # case AWS in 1.0.x. TODO: warn deprecation in 1.1. if aws_access_key_id: warnings.warn( "Passing abstract session keyword arguments is deprecated. " "Pass a Rasterio AWSSession object instead.", RasterioDeprecationWarning) aws_secret_access_key = options.pop('aws_secret_access_key', None) aws_session_token = options.pop('aws_session_token', None) region_name = options.pop('region_name', None) if ('AWS_ACCESS_KEY_ID' in options or 'AWS_SECRET_ACCESS_KEY' in options): raise EnvError( "GDAL's AWS config options can not be directly set. " "AWS credentials are handled exclusively by boto3.") if session: # Passing a session via keyword argument is the canonical # way to configure access to secured cloud resources. if not isinstance(session, Session): warnings.warn( "Passing a boto3 session is deprecated. Pass a Rasterio " "AWSSession object instead.", RasterioDeprecationWarning) session = Session.aws_or_dummy(session=session) self.session = session elif aws_access_key_id or profile_name or aws_unsigned: self.session = Session.aws_or_dummy( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, region_name=region_name, profile_name=profile_name, aws_unsigned=aws_unsigned) elif 'AWS_ACCESS_KEY_ID' in os.environ and 'AWS_SECRET_ACCESS_KEY' in os.environ: self.session = Session.from_environ() else: self.session = DummySession() self.options = options.copy() self.context_options = {}
class Env(object): """Abstraction for GDAL and AWS configuration The GDAL library is stateful: it has a registry of format drivers, an error stack, and dozens of configuration options. Rasterio's approach to working with GDAL is to wrap all the state up using a Python context manager (see PEP 343, https://www.python.org/dev/peps/pep-0343/). When the context is entered GDAL drivers are registered, error handlers are configured, and configuration options are set. When the context is exited, drivers are removed from the registry and other configurations are removed. Example: with rasterio.Env(GDAL_CACHEMAX=512) as env: # All drivers are registered, GDAL's raster block cache # size is set to 512MB. # Commence processing... ... # End of processing. # At this point, configuration options are set to their # previous (possible unset) values. A boto3 session or boto3 session constructor arguments `aws_access_key_id`, `aws_secret_access_key`, `aws_session_token` may be passed to Env's constructor. In the latter case, a session will be created as soon as needed. AWS credentials are configured for GDAL as needed. """ @classmethod def default_options(cls): """Default configuration options Parameters ---------- None Returns ------- dict """ return { 'CHECK_WITH_INVERT_PROJ': True, 'GTIFF_IMPLICIT_JPEG_OVR': False, "RASTERIO_ENV": True } def __init__(self, session=None, aws_unsigned=False, profile_name=None, session_class=AWSSession, **options): """Create a new GDAL/AWS environment. Note: this class is a context manager. GDAL isn't configured until the context is entered via `with rasterio.Env():` Parameters ---------- session : optional A Session object. aws_unsigned : bool, optional Do not sign cloud requests. profile_name : str, optional A shared credentials profile name, as per boto3. session_class : Session, optional A sub-class of Session. **options : optional A mapping of GDAL configuration options, e.g., `CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`. Returns ------- Env Notes ----- We raise EnvError if the GDAL config options AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS credentials are handled exclusively by boto3. Examples -------- >>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True): ... with rasterio.open("https://example.com/a.tif") as src: ... print(src.profile) For access to secured cloud resources, a Rasterio Session or a foreign session object may be passed to the constructor. >>> import boto3 >>> from rasterio.session import AWSSession >>> boto3_session = boto3.Session(...) >>> with Env(AWSSession(boto3_session)): ... with rasterio.open("s3://mybucket/a.tif") as src: ... print(src.profile) """ aws_access_key_id = options.pop('aws_access_key_id', None) # Before 1.0, Rasterio only supported AWS. We will special # case AWS in 1.0.x. TODO: warn deprecation in 1.1. if aws_access_key_id: warnings.warn( "Passing abstract session keyword arguments is deprecated. " "Pass a Rasterio AWSSession object instead.", RasterioDeprecationWarning ) aws_secret_access_key = options.pop('aws_secret_access_key', None) aws_session_token = options.pop('aws_session_token', None) region_name = options.pop('region_name', None) if ('AWS_ACCESS_KEY_ID' in options or 'AWS_SECRET_ACCESS_KEY' in options): raise EnvError( "GDAL's AWS config options can not be directly set. " "AWS credentials are handled exclusively by boto3.") if session: # Passing a session via keyword argument is the canonical # way to configure access to secured cloud resources. if not isinstance(session, Session): warnings.warn( "Passing a boto3 session is deprecated. Pass a Rasterio " "AWSSession object instead.", RasterioDeprecationWarning ) session = AWSSession(session=session) self.session = session elif aws_access_key_id or profile_name or aws_unsigned: self.session = AWSSession( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, region_name=region_name, profile_name=profile_name, aws_unsigned=aws_unsigned) elif 'AWS_ACCESS_KEY_ID' in os.environ: self.session = AWSSession() else: self.session = DummySession() self.options = options.copy() self.context_options = {} @classmethod def from_defaults(cls, *args, **kwargs): """Create an environment with default config options Parameters ---------- args : optional Positional arguments for Env() kwargs : optional Keyword arguments for Env() Returns ------- Env Notes ----- The items in kwargs will be overlaid on the default values. """ options = Env.default_options() options.update(**kwargs) return Env(*args, **options) @property def is_credentialized(self): """Test for existence of cloud credentials Returns ------- bool """ return hascreds() # bool(self.session) def credentialize(self): """Get credentials and configure GDAL Note well: this method is a no-op if the GDAL environment already has credentials, unless session is not None. Returns ------- None """ if hascreds(): pass else: cred_opts = self.session.get_credential_options() self.options.update(**cred_opts) setenv(**cred_opts) def drivers(self): """Return a mapping of registered drivers.""" return local._env.drivers() def __enter__(self): log.debug("Entering env context: %r", self) if local._env is None: log.debug("Starting outermost env") self._has_parent_env = False # See note directly above where _discovered_options is globally # defined. This MUST happen before calling 'defenv()'. local._discovered_options = {} # Don't want to reinstate the "RASTERIO_ENV" option. probe_env = {k for k in self.options.keys() if k != "RASTERIO_ENV"} for key in probe_env: val = get_gdal_config(key, normalize=False) if val is not None: local._discovered_options[key] = val defenv(**self.options) self.context_options = {} else: self._has_parent_env = True self.context_options = getenv() setenv(**self.options) self.credentialize() log.debug("Entered env context: %r", self) return self def __exit__(self, exc_type=None, exc_val=None, exc_tb=None): log.debug("Exiting env context: %r", self) delenv() if self._has_parent_env: defenv() setenv(**self.context_options) else: log.debug("Exiting outermost env") # See note directly above where _discovered_options is globally # defined. while local._discovered_options: key, val = local._discovered_options.popitem() set_gdal_config(key, val, normalize=False) local._discovered_options = None log.debug("Exited env context: %r", self)
def __init__(self, session=None, aws_unsigned=False, profile_name=None, session_class=AWSSession, **options): """Create a new GDAL/AWS environment. Note: this class is a context manager. GDAL isn't configured until the context is entered via `with rasterio.Env():` Parameters ---------- session : optional A Session object. aws_unsigned : bool, optional Do not sign cloud requests. profile_name : str, optional A shared credentials profile name, as per boto3. session_class : Session, optional A sub-class of Session. **options : optional A mapping of GDAL configuration options, e.g., `CPL_DEBUG=True, CHECK_WITH_INVERT_PROJ=False`. Returns ------- Env Notes ----- We raise EnvError if the GDAL config options AWS_ACCESS_KEY_ID or AWS_SECRET_ACCESS_KEY are given. AWS credentials are handled exclusively by boto3. Examples -------- >>> with Env(CPL_DEBUG=True, CPL_CURL_VERBOSE=True): ... with rasterio.open("https://example.com/a.tif") as src: ... print(src.profile) For access to secured cloud resources, a Rasterio Session or a foreign session object may be passed to the constructor. >>> import boto3 >>> from rasterio.session import AWSSession >>> boto3_session = boto3.Session(...) >>> with Env(AWSSession(boto3_session)): ... with rasterio.open("s3://mybucket/a.tif") as src: ... print(src.profile) """ aws_access_key_id = options.pop('aws_access_key_id', None) # Before 1.0, Rasterio only supported AWS. We will special # case AWS in 1.0.x. TODO: warn deprecation in 1.1. if aws_access_key_id: warnings.warn( "Passing abstract session keyword arguments is deprecated. " "Pass a Rasterio AWSSession object instead.", RasterioDeprecationWarning ) aws_secret_access_key = options.pop('aws_secret_access_key', None) aws_session_token = options.pop('aws_session_token', None) region_name = options.pop('region_name', None) if ('AWS_ACCESS_KEY_ID' in options or 'AWS_SECRET_ACCESS_KEY' in options): raise EnvError( "GDAL's AWS config options can not be directly set. " "AWS credentials are handled exclusively by boto3.") if session: # Passing a session via keyword argument is the canonical # way to configure access to secured cloud resources. if not isinstance(session, Session): warnings.warn( "Passing a boto3 session is deprecated. Pass a Rasterio " "AWSSession object instead.", RasterioDeprecationWarning ) session = AWSSession(session=session) self.session = session elif aws_access_key_id or profile_name or aws_unsigned: self.session = AWSSession( aws_access_key_id=aws_access_key_id, aws_secret_access_key=aws_secret_access_key, aws_session_token=aws_session_token, region_name=region_name, profile_name=profile_name, aws_unsigned=aws_unsigned) elif 'AWS_ACCESS_KEY_ID' in os.environ: self.session = AWSSession() else: self.session = DummySession() self.options = options.copy() self.context_options = {}