def _populate_keys_from_metadata_server(self): # get_instance_metadata is imported here because of a circular # dependency. txboto.log.debug("Retrieving credentials from metadata server.") from txboto.utils import get_instance_metadata timeout = config.getfloat('TxBoto', 'metadata_service_timeout', 1.0) attempts = config.getint('TxBoto', 'metadata_service_num_attempts', 1) # The num_retries arg is actually the total number of attempts made, # so the config options is named *_num_attempts to make this more # clear to users. metadata = get_instance_metadata( timeout=timeout, num_retries=attempts, data='meta-data/iam/security-credentials/') if metadata: # I'm assuming there's only one role on the instance profile. security = list(metadata.values())[0] self._access_key = security['AccessKeyId'] self._secret_key = self._convert_key_to_str(security['SecretAccessKey']) self._security_token = security['Token'] expires_at = security['Expiration'] self._credential_expiry_time = datetime.strptime( expires_at, "%Y-%m-%dT%H:%M:%SZ") txboto.log.debug("Retrieved credentials will expire in %s at: %s", self._credential_expiry_time - datetime.now(), expires_at)
def __init__(self, name, access_key=None, secret_key=None, security_token=None, profile_name=None): self.host = None self.port = None self.host_header = None self.access_key = access_key self.secret_key = secret_key self.security_token = security_token self.profile_name = profile_name self.name = name self.acl_class = self.AclClassMap[self.name] self.canned_acls = self.CannedAclsMap[self.name] self._credential_expiry_time = None # Load shared credentials file if it exists shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials') self.shared_credentials = Config(do_load=False) if os.path.isfile(shared_path): self.shared_credentials.load_from_path(shared_path) self.get_credentials(access_key, secret_key, security_token, profile_name) self.configure_headers() self.configure_errors() # Allow config file to override default host and port. host_opt_name = '%s_host' % self.HostKeyMap[self.name] if config.has_option('Credentials', host_opt_name): self.host = config.get('Credentials', host_opt_name) port_opt_name = '%s_port' % self.HostKeyMap[self.name] if config.has_option('Credentials', port_opt_name): self.port = config.getint('Credentials', port_opt_name) host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name] if config.has_option('Credentials', host_header_opt_name): self.host_header = config.get('Credentials', host_header_opt_name)
def __init__(self, host, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, proxy_user=None, proxy_pass=None, debug=0, https_connection_factory=None, path='/', provider='aws', security_token=None, suppress_consec_slashes=True, validate_certs=True, profile_name=None, **kwargs): """ :type host: str :param host: The host to make the connection to :keyword str aws_access_key_id: Your AWS Access Key ID (provided by Amazon). If none is specified, the value in your ``AWS_ACCESS_KEY_ID`` environmental variable is used. :keyword str aws_secret_access_key: Your AWS Secret Access Key (provided by Amazon). If none is specified, the value in your ``AWS_SECRET_ACCESS_KEY`` environmental variable is used. :keyword str security_token: The security token associated with temporary credentials issued by STS. Optional unless using temporary credentials. If none is specified, the environment variable ``AWS_SECURITY_TOKEN`` is used if defined. :type is_secure: boolean :param is_secure: Whether the connection is over SSL :type https_connection_factory: list or tuple :param https_connection_factory: A pair of an HTTP connection factory and the exceptions to catch. The factory should have a similar interface to L{http_client.HTTPSConnection}. :param str proxy: Address/hostname for a proxy server :type proxy_port: int :param proxy_port: The port to use when connecting over a proxy :type proxy_user: str :param proxy_user: The username to connect with on the proxy :type proxy_pass: str :param proxy_pass: The password to use when connection over a proxy. :type port: int :param port: The port to use to connect :type suppress_consec_slashes: bool :param suppress_consec_slashes: If provided, controls whether consecutive slashes will be suppressed in key paths. :type validate_certs: bool :param validate_certs: Controls whether SSL certificates will be validated or not. Defaults to True. :type profile_name: str :param profile_name: Override usual Credentials section in config file to use a named set of keys instead. """ kw = dict(kwargs) kw.update({ 'aws_access_key_id': aws_access_key_id, 'aws_secret_access_key': aws_secret_access_key, 'is_secure': is_secure, 'host': host, 'port': port, 'proxy': proxy, 'proxy_port': proxy_port, 'proxy_user': proxy_user, 'proxy_pass': proxy_pass }) super(AWSAuthConnection, self).__init__(**kw) self.suppress_consec_slashes = suppress_consec_slashes self.num_retries = 6 self.path = path # if the value passed in for debug if not isinstance(debug, six.integer_types): debug = 0 self.debug = config.getint('TxBoto', 'debug', debug) if (sys.version_info[0], sys.version_info[1]) >= (2, 6): # If timeout isn't defined in txboto config file, use 70 second # default as recommended by # http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html self.timeout = config.getint('TxBoto', 'http_socket_timeout', 70) if isinstance(provider, Provider): # Allow overriding Provider self.provider = provider else: self._provider_type = provider self.provider = Provider(self._provider_type, aws_access_key_id, aws_secret_access_key, security_token, profile_name) # Allow config file to override default host, port, and host header. if self.provider.host: self.host = self.provider.host if self.provider.port: self.port = self.provider.port if self.provider.host_header: self.host_header = self.provider.host_header self._last_rs = None self._auth_handler = auth.get_auth_handler( host, config, self.provider, self._required_auth_capability()) if getattr(self, 'AuthServiceName', None) is not None: self.auth_service_name = self.AuthServiceName self.request_hook = None
def _mexe(self, request, override_num_retries=1, retry_handler=None): """ mexe - Multi-execute inside a loop, retrying multiple times to handle transient Internet errors by simply trying again. Also handles redirects. This code was inspired by the S3Utils classes posted to the txboto-users Google group by Larry Bates. Thanks! """ log.debug('Method: %s' % request.method) log.debug('Url: %s' % request.url) log.debug('Data: %s' % request.body) log.debug('Headers: %s' % request.headers) response = None body = None ex = None if override_num_retries is None: num_retries = config.getint('TxBoto', 'num_retries', self.num_retries) else: num_retries = override_num_retries i = 0 while i <= num_retries: # Use binary exponential backoff to desynchronize client requests. next_sleep = min(random.random() * (2 ** i), config.get('TxBoto', 'max_retry_delay', 60)) try: request.authorize(connection=self) log.debug('Final headers: %s' % request.headers) request.start_time = datetime.now() response = yield self.send_request(request) response_body = yield response.content() response.reason = code2status(response.code, 'N/A') log.debug('Response headers: %s' % response.headers) location = response.headers.getRawHeaders('location') if location: location = location[0] if callable(retry_handler): status = yield defer.maybeDeferred(retry_handler, response, response_body, i, next_sleep) if status: msg, i, next_sleep = status if msg: log.debug(msg) time.sleep(next_sleep) continue if response.code in [500, 502, 503, 504]: msg = 'Received %d response. ' % response.code msg += 'Retrying in %3.1f seconds' % next_sleep log.debug(msg) body = response_body if isinstance(body, bytes): body = body.decode('utf-8') elif response.code < 300 or response.code >= 400 or \ not location: # don't return connection to the pool if response contains # Connection:close header, because the connection has been # closed and default reconnect behavior may do something # different than new_http_connection. Also, it's probably # less efficient to try to reuse a closed connection. if self.request_hook is not None: yield defer.maybeDeferred( self.request_hook.handle_request_data, request, response) defer.returnValue((response, response_body,)) except PleaseRetryException as e: log.debug('encountered a retry exception: {}'.foramt(e)) response = e.response ex = e except self.http_exceptions as e: if isinstance(e, self.http_unretryable_exceptions): log.debug('encountered unretryable {} exception, re-raising' .format(e.__class__.__name__)) raise log.debug('encountered {} exception, reconnecting' .format(e.__class__.__name__)) ex = e time.sleep(next_sleep) i += 1 # If we made it here, it's because we have exhausted our retries # and stil haven't succeeded. So, if we have a response object, # use it to raise an exception. # Otherwise, raise the exception that must have already happened. if self.request_hook is not None: yield defer.maybeDeferred(self.request_hook.handle_request_data, request, response, error=True) if response: raise BotoServerError(response.status, response.reason, body) elif ex: raise ex else: msg = 'Please report this exception as a TxBoto Issue!' raise BotoClientError(msg)