def patch_botocore(self): """Context manager that will patch botocore to use Localstack. Since boto3 relies on botocore to perform API calls, this method also effectively patches boto3. """ logger.debug("enter patch") try: factory = self patches = [] # Step 1: patch botocore Session to use Localstack. attr = {} @property def localstack_session(self): # Simlate the 'localstack_session' attr from Session class below. # Patch this into the botocore Session class. if "localstack_session" in self.__dict__: # We're patching this into the base botocore Session, # but we don't want to override things for the Session # subclass below. return self.__dict__["localstack_session"] return factory.localstack_session @localstack_session.setter def localstack_session(self, value): assert isinstance(self, Session) self.__dict__["localstack_session"] = value attr["localstack_session"] = localstack_session @property def _components(self): if isinstance(self, Session): try: return self.__dict__["_components"] except KeyError: raise AttributeError("_components") proxy_components = botocore.session.Session._proxy_components if self not in proxy_components: proxy_components[self] = botocore.session.ComponentLocator( ) self._register_components() return proxy_components[self] @_components.setter def _components(self, value): self.__dict__["_components"] = value @property def _internal_components(self): if isinstance(self, Session): try: return self.__dict__["_internal_components"] except KeyError: raise AttributeError("_internal_components") proxy_components = botocore.session.Session._proxy_components if self not in proxy_components: proxy_components[self] = DebugComponentLocator() self._register_components() return proxy_components[self] @_internal_components.setter def _internal_components(self, value): self.__dict__["_internal_components"] = value attr.update({ "_components": _components, "_internal_components": _internal_components, "_proxy_components": weakref.WeakKeyDictionary(), }) @property def _credentials(self): return self._proxy_credentials.get(self) @_credentials.setter def _credentials(self, value): self._proxy_credentials[self] = value attr.update({ "_credentials": _credentials, "_proxy_credentials": weakref.WeakKeyDictionary(), }) patches.append( mock.patch.multiple("botocore.session.Session", create=True, **attr)) patches.append( mock.patch.multiple( botocore.session.Session, _register_endpoint_resolver=utils.unbind( Session._register_endpoint_resolver), _register_credential_provider=utils.unbind( Session._register_credential_provider), create_client=utils.unbind(Session.create_client), )) # Step 2: Safety checks # Make absolutly sure we use Localstack and not AWS. _original_convert_to_request_dict = ( botocore.client.BaseClient._convert_to_request_dict) @functools.wraps(_original_convert_to_request_dict) def _convert_to_request_dict(self, *args, **kwargs): request_dict = _original_convert_to_request_dict( self, *args, **kwargs) assert factory.localstack_session.hostname in request_dict[ "url"] return request_dict patches.append( mock.patch( "botocore.client.BaseClient._convert_to_request_dict", _convert_to_request_dict, )) # Step 3: Patch existing clients # Patching botocore Session doesn't help with an existing # botocore Clients objects. They will have already been created with # endpoints aimed at AWS. We need to patch botocore.client.BaseClient # to temporarially act like a Localstack. original_init = botocore.client.BaseClient.__init__ @functools.wraps(original_init) def new_init(self, *args, **kwargs): # Every client created during the patch is a Localstack client. # Set this flag so that the proxy_client_attr() stuff below # won't break during original_init(). self._is_pytest_localstack = True original_init(self, *args, **kwargs) patches.append( mock.patch.multiple(botocore.client.BaseClient, __init__=new_init)) # Create a place to store proxy clients. patches.append( mock.patch( "botocore.client.BaseClient._proxy_clients", weakref.WeakKeyDictionary(), create=True, )) def new_getattribute(self, key): if key.startswith("__"): return object.__getattribute__(self, key) proxied_keys = [ "_cache", "_client_config", "_endpoint", "_exceptions_factory", "_exceptions", "exceptions", "_loader", "_request_signer", "_response_parser", "_serializer", "meta", ] __dict__ = object.__getattribute__(self, "__dict__") if (__dict__.get("_is_pytest_localstack", False) or key not in proxied_keys): # Don't proxy clients that are already Localstack clients return object.__getattribute__(self, key) if self not in botocore.client.BaseClient._proxy_clients: try: meta = __dict__["meta"] except KeyError: raise AttributeError("meta") proxy = factory.default_session.create_client( meta.service_model.service_name, # config=config, config=__dict__["_client_config"], ) botocore.client.BaseClient._proxy_clients[self] = proxy return object.__getattribute__( botocore.client.BaseClient._proxy_clients[self], key) patches.append( mock.patch( "botocore.client.BaseClient.__getattribute__", new_getattribute, create=True, )) with utils.nested(*patches): yield finally: logger.debug("exit patch")
def patch_botocore(self): """Context manager that will patch botocore to use Localstack. Since boto3 relies on botocore to perform API calls, this method also effectively patches boto3. """ # Q: Why is this method so complicated? # A: Because the most common usecase is something like this:: # # >>> import boto3 # >>> # >>> S3 = boto3.resource('s3') # >>> # >>> def do_stuff(): # >>> bucket = S3.Bucket('foobar') # >>> bucket.create() # ... # # The `S3` resource creates a botocore Client when the module # is loaded. It's hard to patch existing Client instances since # there isn't a good way to find them. # You must add a descriptor to the Client class # that overrides specific properties of the Client instances. # TODO: Could we use use `gc.get_referrers()` to find instances? logger.debug("enter patch") if boto3 is not None: preexisting_boto3_session = boto3.DEFAULT_SESSION try: factory = self patches = [] # Step 1: patch botocore Session to use Localstack. attr = {} @property def localstack_session(self): # Simlate the 'localstack_session' attr from Session class below. # Patch this into the botocore Session class. if "localstack_session" in self.__dict__: # We're patching this into the base botocore Session, # but we don't want to override things for the Session # subclass below. return self.__dict__["localstack_session"] return factory.localstack_session @localstack_session.setter def localstack_session(self, value): assert isinstance(self, Session) self.__dict__["localstack_session"] = value attr["localstack_session"] = localstack_session @property def _components(self): if isinstance(self, Session): try: return self.__dict__["_components"] except KeyError: raise AttributeError("_components") proxy_components = botocore.session.Session._proxy_components if self not in proxy_components: proxy_components[self] = botocore.session.ComponentLocator( ) self._register_components() return proxy_components[self] @_components.setter def _components(self, value): self.__dict__["_components"] = value @property def _internal_components(self): if isinstance(self, Session): try: return self.__dict__["_internal_components"] except KeyError: raise AttributeError("_internal_components") proxy_components = botocore.session.Session._proxy_components if self not in proxy_components: proxy_components[self] = DebugComponentLocator() self._register_components() return proxy_components[self] @_internal_components.setter def _internal_components(self, value): self.__dict__["_internal_components"] = value attr.update({ "_components": _components, "_internal_components": _internal_components, "_proxy_components": weakref.WeakKeyDictionary(), }) @property def _credentials(self): return self._proxy_credentials.get(self) @_credentials.setter def _credentials(self, value): self._proxy_credentials[self] = value attr.update({ "_credentials": _credentials, "_proxy_credentials": weakref.WeakKeyDictionary(), }) patches.append( mock.patch.multiple("botocore.session.Session", create=True, **attr)) patches.append( mock.patch.multiple( botocore.session.Session, _register_endpoint_resolver=utils.unbind( Session._register_endpoint_resolver), _register_credential_provider=utils.unbind( Session._register_credential_provider), create_client=utils.unbind(Session.create_client), )) # Step 2: Safety checks # Make absolutly sure we use Localstack and not AWS. _original_convert_to_request_dict = ( botocore.client.BaseClient._convert_to_request_dict) @functools.wraps(_original_convert_to_request_dict) def _convert_to_request_dict(self, *args, **kwargs): request_dict = _original_convert_to_request_dict( self, *args, **kwargs) assert any(( factory.localstack_session.hostname in request_dict["url"], socket.gethostname() in request_dict["url"], )) return request_dict patches.append( mock.patch( "botocore.client.BaseClient._convert_to_request_dict", _convert_to_request_dict, )) # Step 3: Patch existing clients # Patching botocore Session doesn't help with an existing # botocore Clients objects. They will have already been created with # endpoints aimed at AWS. We need to patch botocore.client.BaseClient # to temporarially act like a Localstack. original_init = botocore.client.BaseClient.__init__ @functools.wraps(original_init) def new_init(self, *args, **kwargs): # Every client created during the patch is a Localstack client. # Set this flag so that the proxy_client_attr() stuff below # won't break during original_init(). self._is_pytest_localstack = True original_init(self, *args, **kwargs) patches.append( mock.patch.multiple(botocore.client.BaseClient, __init__=new_init)) # Create a place to store proxy clients. patches.append( mock.patch( "botocore.client.BaseClient._proxy_clients", weakref.WeakKeyDictionary(), create=True, )) def new_getattribute(self, key): if key.startswith("__"): return object.__getattribute__(self, key) proxied_keys = [ "_cache", "_client_config", "_endpoint", "_exceptions_factory", "_exceptions", "exceptions", "_loader", "_request_signer", "_response_parser", "_serializer", "meta", ] __dict__ = object.__getattribute__(self, "__dict__") if (__dict__.get("_is_pytest_localstack", False) or key not in proxied_keys): # Don't proxy clients that are already Localstack clients return object.__getattribute__(self, key) if self not in botocore.client.BaseClient._proxy_clients: try: meta = __dict__["meta"] except KeyError: raise AttributeError("meta") proxy = factory.default_session.create_client( meta.service_model.service_name, # config=config, config=__dict__["_client_config"], ) botocore.client.BaseClient._proxy_clients[self] = proxy return object.__getattribute__( botocore.client.BaseClient._proxy_clients[self], key) patches.append( mock.patch( "botocore.client.BaseClient.__getattribute__", new_getattribute, create=True, )) # STS is sneaky and even after patching the endpoint it has a final custom check # to see whether it should override with the global endpoint url... patch that too patches.append( mock.patch( "botocore.args.ClientArgsCreator._should_set_global_sts_endpoint", lambda *args, **kwargs: False, )) with utils.nested(*patches): yield finally: logger.debug("exit patch") if boto3 is not None: boto3.DEFAULT_SESSION = preexisting_boto3_session