def _populate_keys_from_metadata_server(self):
# get_instance_metadata is imported here because of a circular
# dependency.
txboto.log.debug("Retrieving credentials from metadata server.")
from txboto.utils import get_instance_metadata
timeout = config.getfloat('TxBoto', 'metadata_service_timeout', 1.0)
attempts = config.getint('TxBoto', 'metadata_service_num_attempts', 1)
# The num_retries arg is actually the total number of attempts made,
# so the config options is named *_num_attempts to make this more
# clear to users.
metadata = get_instance_metadata(
timeout=timeout, num_retries=attempts,
data='meta-data/iam/security-credentials/')
if metadata:
# I'm assuming there's only one role on the instance profile.
security = list(metadata.values())[0]
self._access_key = security['AccessKeyId']
self._secret_key = self._convert_key_to_str(security['SecretAccessKey'])
self._security_token = security['Token']
expires_at = security['Expiration']
self._credential_expiry_time = datetime.strptime(
expires_at, "%Y-%m-%dT%H:%M:%SZ")
txboto.log.debug("Retrieved credentials will expire in %s at: %s",
self._credential_expiry_time - datetime.now(),
expires_at)
def __init__(self, name, access_key=None, secret_key=None,
security_token=None, profile_name=None):
self.host = None
self.port = None
self.host_header = None
self.access_key = access_key
self.secret_key = secret_key
self.security_token = security_token
self.profile_name = profile_name
self.name = name
self.acl_class = self.AclClassMap[self.name]
self.canned_acls = self.CannedAclsMap[self.name]
self._credential_expiry_time = None
# Load shared credentials file if it exists
shared_path = os.path.join(expanduser('~'), '.' + name, 'credentials')
self.shared_credentials = Config(do_load=False)
if os.path.isfile(shared_path):
self.shared_credentials.load_from_path(shared_path)
self.get_credentials(access_key, secret_key, security_token, profile_name)
self.configure_headers()
self.configure_errors()
# Allow config file to override default host and port.
host_opt_name = '%s_host' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_opt_name):
self.host = config.get('Credentials', host_opt_name)
port_opt_name = '%s_port' % self.HostKeyMap[self.name]
if config.has_option('Credentials', port_opt_name):
self.port = config.getint('Credentials', port_opt_name)
host_header_opt_name = '%s_host_header' % self.HostKeyMap[self.name]
if config.has_option('Credentials', host_header_opt_name):
self.host_header = config.get('Credentials', host_header_opt_name)
def __init__(self, host, aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True, port=None, proxy=None, proxy_port=None,
proxy_user=None, proxy_pass=None, debug=0,
https_connection_factory=None, path='/',
provider='aws', security_token=None,
suppress_consec_slashes=True,
validate_certs=True, profile_name=None,
**kwargs):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:keyword str security_token: The security token associated with
temporary credentials issued by STS. Optional unless using
temporary credentials. If none is specified, the environment
variable ``AWS_SECURITY_TOKEN`` is used if defined.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch. The factory should have
a similar interface to L{http_client.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
:type suppress_consec_slashes: bool
:param suppress_consec_slashes: If provided, controls whether
consecutive slashes will be suppressed in key paths.
:type validate_certs: bool
:param validate_certs: Controls whether SSL certificates
will be validated or not. Defaults to True.
:type profile_name: str
:param profile_name: Override usual Credentials section in config
file to use a named set of keys instead.
"""
kw = dict(kwargs)
kw.update({
'aws_access_key_id': aws_access_key_id,
'aws_secret_access_key': aws_secret_access_key,
'is_secure': is_secure,
'host': host,
'port': port,
'proxy': proxy,
'proxy_port': proxy_port,
'proxy_user': proxy_user,
'proxy_pass': proxy_pass
})
super(AWSAuthConnection, self).__init__(**kw)
self.suppress_consec_slashes = suppress_consec_slashes
self.num_retries = 6
self.path = path
# if the value passed in for debug
if not isinstance(debug, six.integer_types):
debug = 0
self.debug = config.getint('TxBoto', 'debug', debug)
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
# If timeout isn't defined in txboto config file, use 70 second
# default as recommended by
# http://docs.aws.amazon.com/amazonswf/latest/apireference/API_PollForActivityTask.html
self.timeout = config.getint('TxBoto', 'http_socket_timeout', 70)
if isinstance(provider, Provider):
# Allow overriding Provider
self.provider = provider
else:
self._provider_type = provider
self.provider = Provider(self._provider_type,
aws_access_key_id,
aws_secret_access_key,
security_token,
profile_name)
# Allow config file to override default host, port, and host header.
if self.provider.host:
self.host = self.provider.host
if self.provider.port:
self.port = self.provider.port
if self.provider.host_header:
self.host_header = self.provider.host_header
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
if getattr(self, 'AuthServiceName', None) is not None:
self.auth_service_name = self.AuthServiceName
self.request_hook = None
def _mexe(self, request, override_num_retries=1,
retry_handler=None):
"""
mexe - Multi-execute inside a loop, retrying multiple times to handle
transient Internet errors by simply trying again.
Also handles redirects.
This code was inspired by the S3Utils classes posted to the txboto-users
Google group by Larry Bates. Thanks!
"""
log.debug('Method: %s' % request.method)
log.debug('Url: %s' % request.url)
log.debug('Data: %s' % request.body)
log.debug('Headers: %s' % request.headers)
response = None
body = None
ex = None
if override_num_retries is None:
num_retries = config.getint('TxBoto', 'num_retries', self.num_retries)
else:
num_retries = override_num_retries
i = 0
while i <= num_retries:
# Use binary exponential backoff to desynchronize client requests.
next_sleep = min(random.random() * (2 ** i),
config.get('TxBoto', 'max_retry_delay', 60))
try:
request.authorize(connection=self)
log.debug('Final headers: %s' % request.headers)
request.start_time = datetime.now()
response = yield self.send_request(request)
response_body = yield response.content()
response.reason = code2status(response.code, 'N/A')
log.debug('Response headers: %s' % response.headers)
location = response.headers.getRawHeaders('location')
if location:
location = location[0]
if callable(retry_handler):
status = yield defer.maybeDeferred(retry_handler, response,
response_body, i,
next_sleep)
if status:
msg, i, next_sleep = status
if msg:
log.debug(msg)
time.sleep(next_sleep)
continue
if response.code in [500, 502, 503, 504]:
msg = 'Received %d response. ' % response.code
msg += 'Retrying in %3.1f seconds' % next_sleep
log.debug(msg)
body = response_body
if isinstance(body, bytes):
body = body.decode('utf-8')
elif response.code < 300 or response.code >= 400 or \
not location:
# don't return connection to the pool if response contains
# Connection:close header, because the connection has been
# closed and default reconnect behavior may do something
# different than new_http_connection. Also, it's probably
# less efficient to try to reuse a closed connection.
if self.request_hook is not None:
yield defer.maybeDeferred(
self.request_hook.handle_request_data,
request, response)
defer.returnValue((response, response_body,))
except PleaseRetryException as e:
log.debug('encountered a retry exception: {}'.foramt(e))
response = e.response
ex = e
except self.http_exceptions as e:
if isinstance(e, self.http_unretryable_exceptions):
log.debug('encountered unretryable {} exception, re-raising'
.format(e.__class__.__name__))
raise
log.debug('encountered {} exception, reconnecting'
.format(e.__class__.__name__))
ex = e
time.sleep(next_sleep)
i += 1
# If we made it here, it's because we have exhausted our retries
# and stil haven't succeeded. So, if we have a response object,
# use it to raise an exception.
# Otherwise, raise the exception that must have already happened.
if self.request_hook is not None:
yield defer.maybeDeferred(self.request_hook.handle_request_data,
request, response, error=True)
if response:
raise BotoServerError(response.status, response.reason, body)
elif ex:
raise ex
else:
msg = 'Please report this exception as a TxBoto Issue!'
raise BotoClientError(msg)
