def __init__(self, name, access_key=None, secret_key=None, security_token=None, profile_name=None): self.host = None self.port = None self.host_header = None self.access_key = access_key self.secret_key = secret_key self.security_token = security_token self.profile_name = profile_name self.name = name self.acl_class = self.AclClassMap[self.name] self.canned_acls = self.CannedAclsMap[self.name] self._credential_expiry_time = None # Load shared credentials file if it exists shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials') self.shared_credentials = Config(do_load=False) if os.path.isfile(shared_path): self.shared_credentials.load_from_path(shared_path) self.get_credentials(access_key, secret_key, security_token, profile_name) self.configure_headers() self.configure_errors() # Allow config file to override default host and port. host_opt_name = '%s_host' % self.HostKeyMap[self.name] if config.has_option('Credentials', host_opt_name): self.host = config.get('Credentials', host_opt_name) port_opt_name = '%s_port' % self.HostKeyMap[self.name] if config.has_option('Credentials', port_opt_name): self.port = config.getint('Credentials', port_opt_name) host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name] if config.has_option('Credentials', host_header_opt_name): self.host_header = config.get('Credentials', host_header_opt_name)
def get_credentials(self, access_key=None, secret_key=None, security_token=None, profile_name=None): access_key_name, secret_key_name, security_token_name, \ profile_name_name = self.CredentialMap[self.name] # Load profile from shared environment variable if it was not # already passed in and the environment variable exists if profile_name is None and profile_name_name is not None and \ profile_name_name.upper() in os.environ: profile_name = os.environ[profile_name_name.upper()] shared = self.shared_credentials if access_key is not None: self.access_key = access_key txboto.log.debug("Using access key provided by client.") elif access_key_name.upper() in os.environ: self.access_key = os.environ[access_key_name.upper()] txboto.log.debug("Using access key found in environment variable.") elif profile_name is not None: if shared.has_option(profile_name, access_key_name): self.access_key = shared.get(profile_name, access_key_name) txboto.log.debug("Using access key found in shared credential " "file for profile %s." % profile_name) elif config.has_option("profile %s" % profile_name, access_key_name): self.access_key = config.get("profile %s" % profile_name, access_key_name) txboto.log.debug("Using access key found in config file: " "profile %s." % profile_name) else: raise ProfileNotFoundError('Profile "%s" not found!' % profile_name) elif shared.has_option('default', access_key_name): self.access_key = shared.get('default', access_key_name) txboto.log.debug("Using access key found in shared credential file.") elif config.has_option('Credentials', access_key_name): self.access_key = config.get('Credentials', access_key_name) txboto.log.debug("Using access key found in config file.") if secret_key is not None: self.secret_key = secret_key txboto.log.debug("Using secret key provided by client.") elif secret_key_name.upper() in os.environ: self.secret_key = os.environ[secret_key_name.upper()] txboto.log.debug("Using secret key found in environment variable.") elif profile_name is not None: if shared.has_option(profile_name, secret_key_name): self.secret_key = shared.get(profile_name, secret_key_name) txboto.log.debug("Using secret key found in shared credential " "file for profile %s." % profile_name) elif config.has_option("profile %s" % profile_name, secret_key_name): self.secret_key = config.get("profile %s" % profile_name, secret_key_name) txboto.log.debug("Using secret key found in config file: " "profile %s." % profile_name) else: raise ProfileNotFoundError('Profile "%s" not found!' % profile_name) elif shared.has_option('default', secret_key_name): self.secret_key = shared.get('default', secret_key_name) txboto.log.debug("Using secret key found in shared credential file.") elif config.has_option('Credentials', secret_key_name): self.secret_key = config.get('Credentials', secret_key_name) txboto.log.debug("Using secret key found in config file.") elif config.has_option('Credentials', 'keyring'): keyring_name = config.get('Credentials', 'keyring') try: import keyring except ImportError: txboto.log.error("The keyring module could not be imported. " "For keyring support, install the keyring " "module.") raise self.secret_key = keyring.get_password( keyring_name, self.access_key) txboto.log.debug("Using secret key found in keyring.") if security_token is not None: self.security_token = security_token txboto.log.debug("Using security token provided by client.") elif ((security_token_name is not None) and (access_key is None) and (secret_key is None)): # Only provide a token from the environment/config if the # caller did not specify a key and secret. Otherwise an # environment/config token could be paired with a # different set of credentials provided by the caller if security_token_name.upper() in os.environ: self.security_token = os.environ[security_token_name.upper()] txboto.log.debug("Using security token found in environment" " variable.") elif shared.has_option(profile_name or 'default', security_token_name): self.security_token = shared.get(profile_name or 'default', security_token_name) txboto.log.debug("Using security token found in shared " "credential file.") elif profile_name is not None: if config.has_option("profile %s" % profile_name, security_token_name): txboto.log.debug("config has option") self.security_token = config.get("profile %s" % profile_name, security_token_name) txboto.log.debug("Using security token found in config file: " "profile %s." % profile_name) elif config.has_option('Credentials', security_token_name): self.security_token = config.get('Credentials', security_token_name) txboto.log.debug("Using security token found in config file.") if ((self._access_key is None or self._secret_key is None) and self.MetadataServiceSupport[self.name]): self._populate_keys_from_metadata_server() self._secret_key = self._convert_key_to_str(self._secret_key)
def _mexe(self, request, override_num_retries=1, retry_handler=None): """ mexe - Multi-execute inside a loop, retrying multiple times to handle transient Internet errors by simply trying again. Also handles redirects. This code was inspired by the S3Utils classes posted to the txboto-users Google group by Larry Bates. Thanks! """ log.debug('Method: %s' % request.method) log.debug('Url: %s' % request.url) log.debug('Data: %s' % request.body) log.debug('Headers: %s' % request.headers) response = None body = None ex = None if override_num_retries is None: num_retries = config.getint('TxBoto', 'num_retries', self.num_retries) else: num_retries = override_num_retries i = 0 while i <= num_retries: # Use binary exponential backoff to desynchronize client requests. next_sleep = min(random.random() * (2 ** i), config.get('TxBoto', 'max_retry_delay', 60)) try: request.authorize(connection=self) log.debug('Final headers: %s' % request.headers) request.start_time = datetime.now() response = yield self.send_request(request) response_body = yield response.content() response.reason = code2status(response.code, 'N/A') log.debug('Response headers: %s' % response.headers) location = response.headers.getRawHeaders('location') if location: location = location[0] if callable(retry_handler): status = yield defer.maybeDeferred(retry_handler, response, response_body, i, next_sleep) if status: msg, i, next_sleep = status if msg: log.debug(msg) time.sleep(next_sleep) continue if response.code in [500, 502, 503, 504]: msg = 'Received %d response. ' % response.code msg += 'Retrying in %3.1f seconds' % next_sleep log.debug(msg) body = response_body if isinstance(body, bytes): body = body.decode('utf-8') elif response.code < 300 or response.code >= 400 or \ not location: # don't return connection to the pool if response contains # Connection:close header, because the connection has been # closed and default reconnect behavior may do something # different than new_http_connection. Also, it's probably # less efficient to try to reuse a closed connection. if self.request_hook is not None: yield defer.maybeDeferred( self.request_hook.handle_request_data, request, response) defer.returnValue((response, response_body,)) except PleaseRetryException as e: log.debug('encountered a retry exception: {}'.foramt(e)) response = e.response ex = e except self.http_exceptions as e: if isinstance(e, self.http_unretryable_exceptions): log.debug('encountered unretryable {} exception, re-raising' .format(e.__class__.__name__)) raise log.debug('encountered {} exception, reconnecting' .format(e.__class__.__name__)) ex = e time.sleep(next_sleep) i += 1 # If we made it here, it's because we have exhausted our retries # and stil haven't succeeded. So, if we have a response object, # use it to raise an exception. # Otherwise, raise the exception that must have already happened. if self.request_hook is not None: yield defer.maybeDeferred(self.request_hook.handle_request_data, request, response, error=True) if response: raise BotoServerError(response.status, response.reason, body) elif ex: raise ex else: msg = 'Please report this exception as a TxBoto Issue!' raise BotoClientError(msg)