def test_throttled_call_timeout(self, mock_sleep):
"""Test throttle_call with timeout"""
mock_func = MagicMock()
boto_server_error = BotoServerError('error', None)
boto_server_error.error_code = "Throttling"
mock_func.side_effect = boto_server_error
self.assertRaises(BotoServerError, throttled_call, mock_func)
def test_throttled_call_error(self, mock_sleep):
"""Test throttle_call with error"""
mock_func = MagicMock()
boto_server_error = BotoServerError('error', None)
boto_server_error.error_code = "MyError"
mock_func.side_effect = boto_server_error
self.assertRaises(BotoServerError, throttled_call, mock_func)
self.assertEqual(1, mock_func.call_count)
def test_throttled_call_noerr(self, mock_sleep):
"""Test throttle_call with no error"""
mock_func = MagicMock()
boto_server_error = BotoServerError('error', None)
boto_server_error.error_code = "Throttling"
mock_func.side_effect = [boto_server_error, boto_server_error, True]
throttled_call(mock_func)
self.assertEqual(3, mock_func.call_count)
def __init__(self, message, original_exception=None, *args):
self.message = message
body = None
if original_exception:
body = original_exception.body
BotoServerError.__init__(self, status=400, reason='Bad Request', body=body, *args)
def boto_error_handler(request, location=None, template="{0}"):
try:
yield
except ClientError as err:
old_err = BotoServerError(
status=err.response.get('ResponseMetadata').get('HTTPStatusCode'),
reason=err.response.get('Error').get('Code')
)
old_err.message = err.response.get('Error').get('Message')
old_err.error_code = err.response.get('Error').get('Code')
BaseView.handle_error(err=old_err, request=request, location=location, template=template)
except BotoServerError as err:
BaseView.handle_error(err=err, request=request, location=location, template=template)
except socket.error as err:
BaseView.handle_error(err=BotoServerError(504, str(err)), request=request, location=location, template=template)
def test_check_access(self):
elb = mock.Mock()
botoerror = BotoServerError('Fail', 'Unit test')
botoerror.error_code = 'AccessDenied'
elb.set_listener_SSL_certificate = mock.Mock(
side_effect=botoerror)
actor = elb_actor.SetCert(
'Unit Test', {'name': 'unit-test',
'region': 'us-east-1',
'cert_name': 'unit-cert'}
)
# AccessDenied means check has failed.
with self.assertRaises(exceptions.UnrecoverableActorFailure):
yield actor._check_access(elb)
# Anything else means the check has passed.
botoerror.error_code = 'Cert Not Found'
yield actor._check_access(elb)
def my_retried_method(count_func):
count_func()
exception = BotoServerError("400", "error")
exception.code = 'Throttling'
raise exception
def test_botoservererror_basics(self):
bse = BotoServerError('400', 'Bad Request')
self.assertEqual(bse.status, '400')
self.assertEqual(bse.reason, 'Bad Request')
def my_retried_method(count_func):
count_func()
exception = BotoServerError("400", "error")
exception.code = 'Throttling'
raise exception
def test_getters(self):
body = "This is the body"
bse = BotoServerError('400', 'Bad Request', body=body)
self.assertEqual(bse.code, bse.error_code)
class AWSAuthConnection(object):
def __init__(self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path='/',
provider='aws'):
"""
:type host: str
:param host: The host to make the connection to
:keyword str aws_access_key_id: Your AWS Access Key ID (provided by
Amazon). If none is specified, the value in your
``AWS_ACCESS_KEY_ID`` environmental variable is used.
:keyword str aws_secret_access_key: Your AWS Secret Access Key
(provided by Amazon). If none is specified, the value in your
``AWS_SECRET_ACCESS_KEY`` environmental variable is used.
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:param str proxy: Address/hostname for a proxy server
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: str
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: str
:param proxy_pass: The password to use when connection over a proxy.
:type port: int
:param port: The port to use to connect
"""
self.num_retries = 5
# Override passed-in is_secure setting if value was defined in config.
if config.has_option('Boto', 'is_secure'):
is_secure = config.getboolean('Boto', 'is_secure')
self.is_secure = is_secure
# Whether or not to validate server certificates. At some point in the
# future, the default should be flipped to true.
self.https_validate_certificates = config.getbool(
'Boto', 'https_validate_certificates', False)
if self.https_validate_certificates and not HAVE_HTTPS_CONNECTION:
raise BotoClientError(
"SSL server certificate validation is enabled in boto "
"configuration, but Python dependencies required to "
"support this feature are not available. Certificate "
"validation is only supported when running under Python "
"2.6 or later.")
self.ca_certificates_file = config.get_value('Boto',
'ca_certificates_file',
DEFAULT_CA_CERTS_FILE)
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define subclasses of the above that are not retryable.
self.http_unretryable_exceptions = []
if HAVE_HTTPS_CONNECTION:
self.http_unretryable_exceptions.append(ssl.SSLError)
self.http_unretryable_exceptions.append(
https_connection.InvalidCertificateException)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR, )
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
# Timeout used to tell httplib how long to wait for socket timeouts.
# Default is to leave timeout unchanged, which will in turn result in
# the socket's default global timeout being used. To specify a
# timeout, set http_socket_timeout in Boto config. Regardless,
# timeouts will only be applied if Python is 2.6 or greater.
self.http_connection_kwargs = {}
if (sys.version_info[0], sys.version_info[1]) >= (2, 6):
if config.has_option('Boto', 'http_socket_timeout'):
timeout = config.getint('Boto', 'http_socket_timeout')
self.http_connection_kwargs['timeout'] = timeout
self.provider = Provider(provider, aws_access_key_id,
aws_secret_access_key)
# allow config file to override default host
if self.provider.host:
self.host = self.provider.host
self._pool = ConnectionPool()
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
self._auth_handler = auth.get_auth_handler(
host, config, self.provider, self._required_auth_capability())
def __repr__(self):
return '%s:%s' % (self.__class__.__name__, self.host)
def _required_auth_capability(self):
return []
def connection(self):
return self.get_http_connection(*self._connection)
connection = property(connection)
def aws_access_key_id(self):
return self.provider.access_key
aws_access_key_id = property(aws_access_key_id)
gs_access_key_id = aws_access_key_id
access_key = aws_access_key_id
def aws_secret_access_key(self):
return self.provider.secret_key
aws_secret_access_key = property(aws_secret_access_key)
gs_secret_access_key = aws_secret_access_key
secret_key = aws_secret_access_key
def get_path(self, path='/'):
pos = path.find('?')
if pos >= 0:
params = path[pos:]
path = path[:pos]
else:
params = None
if path[-1] == '/':
need_trailing = True
else:
need_trailing = False
path_elements = self.path.split('/')
path_elements.extend(path.split('/'))
path_elements = [p for p in path_elements if p]
path = '/' + '/'.join(path_elements)
if path[-1] != '/' and need_trailing:
path += '/'
if params:
path = path + params
return path
def server_name(self, port=None):
if not port:
port = self.port
if port == 80:
signature_host = self.host
else:
# This unfortunate little hack can be attributed to
# a difference in the 2.6 version of httplib. In old
# versions, it would append ":443" to the hostname sent
# in the Host header and so we needed to make sure we
# did the same when calculating the V2 signature. In 2.6
# (and higher!)
# it no longer does that. Hence, this kludge.
if ((ON_APP_ENGINE and sys.version[:3] == '2.5')
or sys.version[:3] in ('2.6', '2.7')) and port == 443:
signature_host = self.host
else:
signature_host = '%s:%d' % (self.host, port)
return signature_host
def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass):
self.proxy = proxy
self.proxy_port = proxy_port
self.proxy_user = proxy_user
self.proxy_pass = proxy_pass
if os.environ.has_key('http_proxy') and not self.proxy:
pattern = re.compile(
'(?:http://)?' \
'(?:(?P<user>\w+):(?P<pass>.*)@)?' \
'(?P<host>[\w\-\.]+)' \
'(?::(?P<port>\d+))?'
)
match = pattern.match(os.environ['http_proxy'])
if match:
self.proxy = match.group('host')
self.proxy_port = match.group('port')
self.proxy_user = match.group('user')
self.proxy_pass = match.group('pass')
else:
if not self.proxy:
self.proxy = config.get_value('Boto', 'proxy', None)
if not self.proxy_port:
self.proxy_port = config.get_value('Boto', 'proxy_port', None)
if not self.proxy_user:
self.proxy_user = config.get_value('Boto', 'proxy_user', None)
if not self.proxy_pass:
self.proxy_pass = config.get_value('Boto', 'proxy_pass', None)
if not self.proxy_port and self.proxy:
print "http_proxy environment variable does not specify " \
"a port, using default"
self.proxy_port = self.port
self.use_proxy = (self.proxy != None)
def get_http_connection(self, host, is_secure):
conn = self._pool.get_http_connection(host, is_secure)
if conn is not None:
return conn
else:
return self.new_http_connection(host, is_secure)
def new_http_connection(self, host, is_secure):
if self.use_proxy:
host = '%s:%d' % (self.proxy, int(self.proxy_port))
if host is None:
host = self.server_name()
if is_secure:
boto.log.debug('establishing HTTPS connection: host=%s, kwargs=%s',
host, self.http_connection_kwargs)
if self.use_proxy:
connection = self.proxy_ssl()
elif self.https_connection_factory:
connection = self.https_connection_factory(host)
elif self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
connection = https_connection.CertValidatingHTTPSConnection(
host,
ca_certs=self.ca_certificates_file,
**self.http_connection_kwargs)
else:
connection = httplib.HTTPSConnection(
host, **self.http_connection_kwargs)
else:
boto.log.debug('establishing HTTP connection: kwargs=%s' %
self.http_connection_kwargs)
connection = httplib.HTTPConnection(host,
**self.http_connection_kwargs)
if self.debug > 1:
connection.set_debuglevel(self.debug)
# self.connection must be maintained for backwards-compatibility
# however, it must be dynamically pulled from the connection pool
# set a private variable which will enable that
if host.split(':')[0] == self.host and is_secure == self.is_secure:
self._connection = (host, is_secure)
return connection
def put_http_connection(self, host, is_secure, connection):
self._pool.put_http_connection(host, is_secure, connection)
def proxy_ssl(self):
host = '%s:%d' % (self.host, self.port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((self.proxy, int(self.proxy_port)))
except:
raise
boto.log.debug("Proxy connection: CONNECT %s HTTP/1.0\r\n", host)
sock.sendall("CONNECT %s HTTP/1.0\r\n" % host)
sock.sendall("User-Agent: %s\r\n" % UserAgent)
if self.proxy_user and self.proxy_pass:
for k, v in self.get_proxy_auth_header().items():
sock.sendall("%s: %s\r\n" % (k, v))
sock.sendall("\r\n")
resp = httplib.HTTPResponse(sock, strict=True, debuglevel=self.debug)
resp.begin()
if resp.status != 200:
# Fake a socket error, use a code that make it obvious it hasn't
# been generated by the socket library
raise socket.error(
-71, "Error talking to HTTP proxy %s:%s: %s (%s)" %
(self.proxy, self.proxy_port, resp.status, resp.reason))
# We can safely close the response, it duped the original socket
resp.close()
h = httplib.HTTPConnection(host)
if self.https_validate_certificates and HAVE_HTTPS_CONNECTION:
boto.log.debug(
"wrapping ssl socket for proxied connection; "
"CA certificate file=%s", self.ca_certificates_file)
key_file = self.http_connection_kwargs.get('key_file', None)
cert_file = self.http_connection_kwargs.get('cert_file', None)
sslSock = ssl.wrap_socket(sock,
keyfile=key_file,
certfile=cert_file,
cert_reqs=ssl.CERT_REQUIRED,
ca_certs=self.ca_certificates_file)
cert = sslSock.getpeercert()
hostname = self.host.split(':', 0)[0]
if not https_connection.ValidateCertificateHostname(
cert, hostname):
raise https_connection.InvalidCertificateException(
hostname, cert, 'hostname mismatch')
else:
# Fallback for old Python without ssl.wrap_socket
if hasattr(httplib, 'ssl'):
sslSock = httplib.ssl.SSLSocket(sock)
else:
sslSock = socket.ssl(sock, None, None)
sslSock = httplib.FakeSocket(sock, sslSock)
# This is a bit unclean
h.sock = sslSock
return h
def prefix_proxy_to_path(self, path, host=None):
path = self.protocol + '://' + (host or self.server_name()) + path
return path
def get_proxy_auth_header(self):
auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass)
return {'Proxy-Authorization': 'Basic %s' % auth}
def _mexe(self, request, sender=None, override_num_retries=None):
"""
mexe - Multi-execute inside a loop, retrying multiple times to handle
transient Internet errors by simply trying again.
Also handles redirects.
This code was inspired by the S3Utils classes posted to the boto-users
Google group by Larry Bates. Thanks!
"""
boto.log.debug('Method: %s' % request.method)
boto.log.debug('Path: %s' % request.path)
boto.log.debug('Data: %s' % request.body)
boto.log.debug('Headers: %s' % request.headers)
boto.log.debug('Host: %s' % request.host)
response = None
body = None
e = None
if override_num_retries is None:
num_retries = config.getint('Boto', 'num_retries',
self.num_retries)
else:
num_retries = override_num_retries
i = 0
connection = self.get_http_connection(request.host, self.is_secure)
while i <= num_retries:
try:
# we now re-sign each request before it is retried
request.authorize(connection=self)
if callable(sender):
response = sender(connection, request.method, request.path,
request.body, request.headers)
else:
connection.request(request.method, request.path,
request.body, request.headers)
response = connection.getresponse()
location = response.getheader('location')
# -- gross hack --
# httplib gets confused with chunked responses to HEAD requests
# so I have to fake it out
if request.method == 'HEAD' and getattr(
response, 'chunked', False):
response.chunked = 0
if response.status == 500 or response.status == 503:
boto.log.debug(
'received %d response, retrying in %d seconds' %
(response.status, 2**i))
body = response.read()
elif response.status < 300 or response.status >= 400 or \
not location:
self.put_http_connection(request.host, self.is_secure,
connection)
return response
else:
scheme, request.host, request.path, params, query, fragment = \
urlparse.urlparse(location)
if query:
request.path += '?' + query
boto.log.debug('Redirecting: %s' % scheme + '://' +
request.host + request.path)
connection = self.get_http_connection(
request.host, scheme == 'https')
continue
except KeyboardInterrupt:
sys.exit('Keyboard Interrupt')
except self.http_exceptions, e:
for unretryable in self.http_unretryable_exceptions:
if isinstance(e, unretryable):
boto.log.debug(
'encountered unretryable %s exception, re-raising'
% e.__class__.__name__)
raise e
boto.log.debug('encountered %s exception, reconnecting' % \
e.__class__.__name__)
connection = self.new_http_connection(request.host,
self.is_secure)
time.sleep(2**i)
i += 1
# If we made it here, it's because we have exhausted our retries and stil haven't
# succeeded. So, if we have a response object, use it to raise an exception.
# Otherwise, raise the exception that must have already happened.
if response:
raise BotoServerError(response.status, response.reason, body)
elif e:
raise e
else:
raise BotoClientError(
'Please report this exception as a Boto Issue!')
def clean_json(resource_json, resources_map):
"""
Cleanup the a resource dict. For now, this just means replacing any Ref node
with the corresponding physical_resource_id.
Eventually, this is where we would add things like function parsing (fn::)
"""
if isinstance(resource_json, dict):
if 'Ref' in resource_json:
# Parse resource reference
resource = resources_map[resource_json['Ref']]
if hasattr(resource, 'physical_resource_id'):
return resource.physical_resource_id
else:
return resource
if "Fn::FindInMap" in resource_json:
map_name = resource_json["Fn::FindInMap"][0]
map_path = resource_json["Fn::FindInMap"][1:]
result = resources_map[map_name]
for path in map_path:
result = result[clean_json(path, resources_map)]
return result
if 'Fn::GetAtt' in resource_json:
resource = resources_map.get(resource_json['Fn::GetAtt'][0])
if resource is None:
return resource_json
try:
return resource.get_cfn_attribute(
resource_json['Fn::GetAtt'][1])
except NotImplementedError as n:
logger.warning(n.message.format(
resource_json['Fn::GetAtt'][0]))
except UnformattedGetAttTemplateException:
raise BotoServerError(
UnformattedGetAttTemplateException.status_code,
'Bad Request',
UnformattedGetAttTemplateException.description.format(
resource_json['Fn::GetAtt'][0],
resource_json['Fn::GetAtt'][1]))
if 'Fn::If' in resource_json:
condition_name, true_value, false_value = resource_json['Fn::If']
if resources_map[condition_name]:
return clean_json(true_value, resources_map)
else:
return clean_json(false_value, resources_map)
if 'Fn::Join' in resource_json:
join_list = []
for val in resource_json['Fn::Join'][1]:
cleaned_val = clean_json(val, resources_map)
join_list.append('{0}'.format(cleaned_val)
if cleaned_val else '{0}'.format(val))
return resource_json['Fn::Join'][0].join(join_list)
cleaned_json = {}
for key, value in resource_json.items():
cleaned_json[key] = clean_json(value, resources_map)
return cleaned_json
elif isinstance(resource_json, list):
return [clean_json(val, resources_map) for val in resource_json]
else:
return resource_json
def test_exception_message_with_no_error_in_body(self):
e = BotoServerError('400', 'Mocked error', body=no_error_body)
r = salt.utils.boto.BotoExecutionError(e)
self.assertTrue(isinstance(r, CommandExecutionError))
self.assertEqual(r.message, '400 Mocked error')
def se_exc(*args, **kwargs):
raise BotoServerError(None, None, None)
def delete_user(self, user_name):
try:
del self.users[user_name]
except KeyError:
raise BotoServerError(404, 'Not Found')
def test_exception_type_and_message(self):
e = BotoServerError('400', 'Mocked error', body=error_body)
r = salt.utils.boto.BotoExecutionError(e)
self.assertTrue(isinstance(r, CommandExecutionError))
self.assertEqual(r.message, '400 Mocked error: Error message')
def delete_access_key(self, access_key_id, user_name):
try:
user = self.users[user_name]
user.delete_access_key(access_key_id)
except KeyError:
raise BotoServerError(404, 'Not Found')
def delete_user_policy(self, user_name, policy_name):
try:
user = self.users[user_name]
user.delete_policy(policy_name)
except KeyError:
raise BotoServerError(404, 'Not Found')
def put_user_policy(self, user_name, policy_name, policy_json):
try:
user = self.users[user_name]
user.put_policy(policy_name, policy_json)
except KeyError:
raise BotoServerError(404, 'Not Found')
def list_role_policies(self, role_name):
role = self.get_role(role_name)
if role:
return role.policies.keys()
else:
raise BotoServerError(404, 'Not Found')
def delete_policy(self, policy_name):
if policy_name not in self.policies:
raise BotoServerError(404, 'Not Found')
del self.policies[policy_name]
def test_setup_create_error(self):
self.dynamo.describe_table.side_effect = [
BotoServerError(400, 'Kaboom')
]
self.assertRaises(Exception, self.tables.setup, ['revisions'])
class AWSAuthConnection:
def __init__(self,
host,
aws_access_key_id=None,
aws_secret_access_key=None,
is_secure=True,
port=None,
proxy=None,
proxy_port=None,
proxy_user=None,
proxy_pass=None,
debug=0,
https_connection_factory=None,
path='/'):
"""
:type host: string
:param host: The host to make the connection to
:type aws_access_key_id: string
:param aws_access_key_id: AWS Access Key ID (provided by Amazon)
:type aws_secret_access_key: string
:param aws_secret_access_key: Secret Access Key (provided by Amazon)
:type is_secure: boolean
:param is_secure: Whether the connection is over SSL
:type https_connection_factory: list or tuple
:param https_connection_factory: A pair of an HTTP connection
factory and the exceptions to catch.
The factory should have a similar
interface to L{httplib.HTTPSConnection}.
:type proxy:
:param proxy:
:type proxy_port: int
:param proxy_port: The port to use when connecting over a proxy
:type proxy_user: string
:param proxy_user: The username to connect with on the proxy
:type proxy_pass: string
:param proxy_pass: The password to use when connection over a proxy.
:type port: integer
:param port: The port to use to connect
"""
self.num_retries = 5
self.is_secure = is_secure
self.handle_proxy(proxy, proxy_port, proxy_user, proxy_pass)
# define exceptions from httplib that we want to catch and retry
self.http_exceptions = (httplib.HTTPException, socket.error,
socket.gaierror)
# define values in socket exceptions we don't want to catch
self.socket_exception_values = (errno.EINTR, )
if https_connection_factory is not None:
self.https_connection_factory = https_connection_factory[0]
self.http_exceptions += https_connection_factory[1]
else:
self.https_connection_factory = None
if (is_secure):
self.protocol = 'https'
else:
self.protocol = 'http'
self.host = host
self.path = path
if debug:
self.debug = debug
else:
self.debug = config.getint('Boto', 'debug', debug)
if port:
self.port = port
else:
self.port = PORTS_BY_SECURITY[is_secure]
if aws_access_key_id:
self.aws_access_key_id = aws_access_key_id
elif os.environ.has_key('AWS_ACCESS_KEY_ID'):
self.aws_access_key_id = os.environ['AWS_ACCESS_KEY_ID']
elif config.has_option('Credentials', 'aws_access_key_id'):
self.aws_access_key_id = config.get('Credentials',
'aws_access_key_id')
if aws_secret_access_key:
self.aws_secret_access_key = aws_secret_access_key
elif os.environ.has_key('AWS_SECRET_ACCESS_KEY'):
self.aws_secret_access_key = os.environ['AWS_SECRET_ACCESS_KEY']
elif config.has_option('Credentials', 'aws_secret_access_key'):
self.aws_secret_access_key = config.get('Credentials',
'aws_secret_access_key')
# initialize an HMAC for signatures, make copies with each request
self.hmac = hmac.new(self.aws_secret_access_key, digestmod=sha)
if sha256:
self.hmac_256 = hmac.new(self.aws_secret_access_key,
digestmod=sha256)
else:
self.hmac_256 = None
# cache up to 20 connections per host, up to 20 hosts
self._pool = ConnectionPool(20, 20)
self._connection = (self.server_name(), self.is_secure)
self._last_rs = None
def __repr__(self):
return '%s:%s' % (self.__class__.__name__, self.host)
def _cached_name(self, host, is_secure):
if host is None:
host = self.server_name()
cached_name = is_secure and 'https://' or 'http://'
cached_name += host
return cached_name
def connection(self):
return self.get_http_connection(*self._connection)
connection = property(connection)
def get_path(self, path='/'):
pos = path.find('?')
if pos >= 0:
params = path[pos:]
path = path[:pos]
else:
params = None
if path[-1] == '/':
need_trailing = True
else:
need_trailing = False
path_elements = self.path.split('/')
path_elements.extend(path.split('/'))
path_elements = [p for p in path_elements if p]
path = '/' + '/'.join(path_elements)
if path[-1] != '/' and need_trailing:
path += '/'
if params:
path = path + params
return path
def server_name(self, port=None):
if not port:
port = self.port
if port == 80:
signature_host = self.host
else:
# This unfortunate little hack can be attributed to
# a difference in the 2.6 version of httplib. In old
# versions, it would append ":443" to the hostname sent
# in the Host header and so we needed to make sure we
# did the same when calculating the V2 signature. In 2.6
# it no longer does that. Hence, this kludge.
if sys.version[:3] == "2.6" and port == 443:
signature_host = self.host
else:
signature_host = '%s:%d' % (self.host, port)
return signature_host
def handle_proxy(self, proxy, proxy_port, proxy_user, proxy_pass):
self.proxy = proxy
self.proxy_port = proxy_port
self.proxy_user = proxy_user
self.proxy_pass = proxy_pass
if os.environ.has_key('http_proxy') and not self.proxy:
pattern = re.compile(
'(?:http://)?' \
'(?:(?P<user>\w+):(?P<pass>.*)@)?' \
'(?P<host>[\w\-\.]+)' \
'(?::(?P<port>\d+))?'
)
match = pattern.match(os.environ['http_proxy'])
if match:
self.proxy = match.group('host')
self.proxy_port = match.group('port')
self.proxy_user = match.group('user')
self.proxy_pass = match.group('pass')
else:
if not self.proxy:
self.proxy = config.get_value('Boto', 'proxy', None)
if not self.proxy_port:
self.proxy_port = config.get_value('Boto', 'proxy_port', None)
if not self.proxy_user:
self.proxy_user = config.get_value('Boto', 'proxy_user', None)
if not self.proxy_pass:
self.proxy_pass = config.get_value('Boto', 'proxy_pass', None)
if not self.proxy_port and self.proxy:
print "http_proxy environment variable does not specify " \
"a port, using default"
self.proxy_port = self.port
self.use_proxy = (self.proxy != None)
def get_http_connection(self, host, is_secure):
queue = self._pool[self._cached_name(host, is_secure)]
try:
return queue.get_nowait()
except Queue.Empty:
return self.new_http_connection(host, is_secure)
def new_http_connection(self, host, is_secure):
if self.use_proxy:
host = '%s:%d' % (self.proxy, int(self.proxy_port))
if host is None:
host = self.server_name()
boto.log.debug('establishing HTTP connection')
if is_secure:
if self.use_proxy:
connection = self.proxy_ssl()
elif self.https_connection_factory:
connection = self.https_connection_factory(host)
else:
connection = httplib.HTTPSConnection(host)
else:
connection = httplib.HTTPConnection(host)
if self.debug > 1:
connection.set_debuglevel(self.debug)
# self.connection must be maintained for backwards-compatibility
# however, it must be dynamically pulled from the connection pool
# set a private variable which will enable that
if host.split(':')[0] == self.host and is_secure == self.is_secure:
self._connection = (host, is_secure)
return connection
def put_http_connection(self, host, is_secure, connection):
try:
self._pool[self._cached_name(host,
is_secure)].put_nowait(connection)
except Queue.Full:
# gracefully fail in case of pool overflow
connection.close()
def proxy_ssl(self):
host = '%s:%d' % (self.host, self.port)
sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
try:
sock.connect((self.proxy, int(self.proxy_port)))
except:
raise
sock.sendall("CONNECT %s HTTP/1.0\r\n" % host)
sock.sendall("User-Agent: %s\r\n" % UserAgent)
if self.proxy_user and self.proxy_pass:
for k, v in self.get_proxy_auth_header().items():
sock.sendall("%s: %s\r\n" % (k, v))
sock.sendall("\r\n")
resp = httplib.HTTPResponse(sock, strict=True)
resp.begin()
if resp.status != 200:
# Fake a socket error, use a code that make it obvious it hasn't
# been generated by the socket library
raise socket.error(
-71, "Error talking to HTTP proxy %s:%s: %s (%s)" %
(self.proxy, self.proxy_port, resp.status, resp.reason))
# We can safely close the response, it duped the original socket
resp.close()
h = httplib.HTTPConnection(host)
# Wrap the socket in an SSL socket
if hasattr(httplib, 'ssl'):
sslSock = httplib.ssl.SSLSocket(sock)
else: # Old Python, no ssl module
sslSock = socket.ssl(sock, None, None)
sslSock = httplib.FakeSocket(sock, sslSock)
# This is a bit unclean
h.sock = sslSock
return h
def prefix_proxy_to_path(self, path, host=None):
path = self.protocol + '://' + (host or self.server_name()) + path
return path
def get_proxy_auth_header(self):
auth = base64.encodestring(self.proxy_user + ':' + self.proxy_pass)
return {'Proxy-Authorization': 'Basic %s' % auth}
def _mexe(self, method, path, data, headers, host=None, sender=None):
"""
mexe - Multi-execute inside a loop, retrying multiple times to handle
transient Internet errors by simply trying again.
Also handles redirects.
This code was inspired by the S3Utils classes posted to the boto-users
Google group by Larry Bates. Thanks!
"""
boto.log.debug('Method: %s' % method)
boto.log.debug('Path: %s' % path)
boto.log.debug('Data: %s' % data)
boto.log.debug('Headers: %s' % headers)
boto.log.debug('Host: %s' % host)
response = None
body = None
e = None
num_retries = config.getint('Boto', 'num_retries', self.num_retries)
i = 0
connection = self.get_http_connection(host, self.is_secure)
while i <= num_retries:
try:
if callable(sender):
response = sender(connection, method, path, data, headers)
else:
connection.request(method, path, data, headers)
response = connection.getresponse()
location = response.getheader('location')
# -- gross hack --
# httplib gets confused with chunked responses to HEAD requests
# so I have to fake it out
if method == 'HEAD' and getattr(response, 'chunked', False):
response.chunked = 0
if response.status == 500 or response.status == 503:
boto.log.debug(
'received %d response, retrying in %d seconds' %
(response.status, 2**i))
body = response.read()
elif response.status == 408:
body = response.read()
print '-------------------------'
print ' 4 0 8 '
print 'path=%s' % path
print body
print '-------------------------'
elif response.status < 300 or response.status >= 400 or \
not location:
self.put_http_connection(host, self.is_secure, connection)
return response
else:
scheme, host, path, params, query, fragment = \
urlparse.urlparse(location)
if query:
path += '?' + query
boto.log.debug('Redirecting: %s' % scheme + '://' + host +
path)
connection = self.get_http_connection(
host, scheme == 'https')
continue
except KeyboardInterrupt:
sys.exit('Keyboard Interrupt')
except self.http_exceptions, e:
boto.log.debug('encountered %s exception, reconnecting' % \
e.__class__.__name__)
connection = self.new_http_connection(host, self.is_secure)
time.sleep(2**i)
i += 1
# If we made it here, it's because we have exhausted our retries and stil haven't
# succeeded. So, if we have a response object, use it to raise an exception.
# Otherwise, raise the exception that must have already happened.
if response:
raise BotoServerError(response.status, response.reason, body)
elif e:
raise e
else:
raise BotoClientError(
'Please report this exception as a Boto Issue!')
'encountered unretryable %s exception, re-raising'
% e.__class__.__name__)
raise
boto.log.debug('encountered %s exception, reconnecting' % \
e.__class__.__name__)
connection = self.new_http_connection(request.host,
request.port,
self.is_secure)
time.sleep(next_sleep)
i += 1
# If we made it here, it's because we have exhausted our retries
# and stil haven't succeeded. So, if we have a response object,
# use it to raise an exception.
# Otherwise, raise the exception that must have already happened.
if response:
raise BotoServerError(response.status, response.reason, body)
elif e:
raise
else:
msg = 'Please report this exception as a Boto Issue!'
raise BotoClientError(msg)
def build_base_http_request(self,
method,
path,
auth_path,
params=None,
headers=None,
data='',
host=None):
path = self.get_path(path)
def test_get_conn_error_raises_command_execution_error(self):
with patch('boto.{0}.connect_to_region'.format(service),
side_effect=BotoServerError(400, 'Mocked error', body=error_body)):
with self.assertRaises(BotoServerError):
salt.utils.boto.get_connection(service)
def test_purchase_item_email_boto_failure(self, error_logger):
cart = Order.get_cart_for_user(user=self.user)
CertificateItem.add_to_order(cart, self.course_id, self.cost, 'honor')
with patch('shoppingcart.models.send_mail', side_effect=BotoServerError("status", "reason")):
cart.purchase()
self.assertTrue(error_logger.called)
def test_exception_message_with_no_error_in_body(self):
e = BotoServerError('400', 'Mocked error', body=no_error_body)
r = salt.utils.boto.get_error(e)
expected = {'aws': {'reason': 'Mocked error', 'status': '400'},
'message': 'Mocked error'}
self.assertEqual(r, expected)
def _mexe(self, request, sender=None, override_num_retries=None,
retry_handler=None):
"""
mexe - Multi-execute inside a loop, retrying multiple times to handle
transient Internet errors by simply trying again.
Also handles redirects.
This code was inspired by the S3Utils classes posted to the boto-users
Google group by Larry Bates. Thanks!
"""
boto.log.debug('Method: %s' % request.method)
boto.log.debug('Path: %s' % request.path)
boto.log.debug('Data: %s' % request.body)
boto.log.debug('Headers: %s' % request.headers)
boto.log.debug('Host: %s' % request.host)
boto.log.debug('Port: %s' % request.port)
boto.log.debug('Params: %s' % request.params)
response = None
body = None
ex = None
if override_num_retries is None:
num_retries = config.getint('Boto', 'num_retries', self.num_retries)
else:
num_retries = override_num_retries
i = 0
connection = self.get_http_connection(request.host, request.port,
self.is_secure)
# Convert body to bytes if needed
if not isinstance(request.body, bytes) and hasattr(request.body,
'encode'):
request.body = request.body.encode('utf-8')
while i <= num_retries:
# Use binary exponential backoff to desynchronize client requests.
next_sleep = min(random.random() * (2 ** i),
boto.config.get('Boto', 'max_retry_delay', 60))
try:
# we now re-sign each request before it is retried
boto.log.debug('Token: %s' % self.provider.security_token)
request.authorize(connection=self)
# Only force header for non-s3 connections, because s3 uses
# an older signing method + bucket resource URLs that include
# the port info. All others should be now be up to date and
# not include the port.
if 's3' not in self._required_auth_capability():
if not getattr(self, 'anon', False):
if not request.headers.get('Host'):
self.set_host_header(request)
boto.log.debug('Final headers: %s' % request.headers)
request.start_time = datetime.now()
if callable(sender):
response = sender(connection, request.method, request.path,
request.body, request.headers)
else:
connection.request(request.method, request.path,
request.body, request.headers)
response = connection.getresponse()
boto.log.debug('Response headers: %s' % response.getheaders())
location = response.getheader('location')
# -- gross hack --
# http_client gets confused with chunked responses to HEAD requests
# so I have to fake it out
if request.method == 'HEAD' and getattr(response,
'chunked', False):
response.chunked = 0
if callable(retry_handler):
status = retry_handler(response, i, next_sleep)
if status:
msg, i, next_sleep = status
if msg:
boto.log.debug(msg)
time.sleep(next_sleep)
continue
if response.status in [500, 502, 503, 504]:
msg = 'Received %d response. ' % response.status
msg += 'Retrying in %3.1f seconds' % next_sleep
boto.log.debug(msg)
body = response.read()
if isinstance(body, bytes):
body = body.decode('utf-8')
elif response.status < 300 or response.status >= 400 or \
not location:
# don't return connection to the pool if response contains
# Connection:close header, because the connection has been
# closed and default reconnect behavior may do something
# different than new_http_connection. Also, it's probably
# less efficient to try to reuse a closed connection.
conn_header_value = response.getheader('connection')
if conn_header_value == 'close':
connection.close()
else:
self.put_http_connection(request.host, request.port,
self.is_secure, connection)
if self.request_hook is not None:
self.request_hook.handle_request_data(request, response)
return response
else:
scheme, request.host, request.path, \
params, query, fragment = urlparse(location)
if query:
request.path += '?' + query
# urlparse can return both host and port in netloc, so if
# that's the case we need to split them up properly
if ':' in request.host:
request.host, request.port = request.host.split(':', 1)
msg = 'Redirecting: %s' % scheme + '://'
msg += request.host + request.path
boto.log.debug(msg)
connection = self.get_http_connection(request.host,
request.port,
scheme == 'https')
response = None
continue
except PleaseRetryException as e:
boto.log.debug('encountered a retry exception: %s' % e)
connection = self.new_http_connection(request.host, request.port,
self.is_secure)
response = e.response
ex = e
except self.http_exceptions as e:
for unretryable in self.http_unretryable_exceptions:
if isinstance(e, unretryable):
boto.log.debug(
'encountered unretryable %s exception, re-raising' %
e.__class__.__name__)
raise
boto.log.debug('encountered %s exception, reconnecting' %
e.__class__.__name__)
connection = self.new_http_connection(request.host, request.port,
self.is_secure)
ex = e
time.sleep(next_sleep)
i += 1
# If we made it here, it's because we have exhausted our retries
# and stil haven't succeeded. So, if we have a response object,
# use it to raise an exception.
# Otherwise, raise the exception that must have already happened.
if self.request_hook is not None:
self.request_hook.handle_request_data(request, response, error=True)
if response:
raise BotoServerError(response.status, response.reason, body)
elif ex:
raise ex
else:
msg = 'Please report this exception as a Boto Issue!'
raise BotoClientError(msg)
def test_get_latest_value_returns_default_value(self, kms, ec2_api, cfn_mock):
cfn_mock.return_value.get_stack_parameters_dict.side_effect = CfnSphereBotoError(BotoServerError("500", "foo"))
resolver = ParameterResolver()
with self.assertRaises(CfnSphereException):
resolver.get_latest_value('my-key', '|keepOrUse|default-value', 'my-stack')
def my_retried_method(count_func):
count_func()
exception = BotoServerError("400", "error")
exception.code = 'ValidationError'
raise exception
def test_message_not_xml(self):
body = 'This is not XML'
bse = BotoServerError('400', 'Bad Request', body=body)
self.assertEqual(bse.error_message, 'This is not XML')
def create_server_error(code):
err = BotoServerError('', '', '')
err.error_code = code
return err
def _credentials_expired(self, response):
if response.status != 401:
return False
error = BotoServerError('', '', body=response.read())
return error.error_code == 'InvalidAccessKeyId'
def my_retried_method(count_func):
count_func()
exception = BotoServerError("400", "error")
exception.code = 'ValidationError'
raise exception
def _mexe(self, request, sender=None, override_num_retries=None,
retry_handler=None):
"""
mexe - Multi-execute inside a loop, retrying multiple times to handle
transient Internet errors by simply trying again.
Also handles redirects.
This code was inspired by the S3Utils classes posted to the boto-users
Google group by Larry Bates. Thanks!
"""
boto.log.debug('Method: %s' % request.method)
boto.log.debug('Path: %s' % request.path)
boto.log.debug('Data: %s' % request.body)
boto.log.debug('Headers: %s' % request.headers)
boto.log.debug('Host: %s' % request.host)
boto.log.debug('Params: %s' % request.params)
response = None
body = None
ex = None
if override_num_retries is None:
num_retries = config.getint('Boto', 'num_retries', self.num_retries)
else:
num_retries = override_num_retries
i = 0
connection = self.get_http_connection(request.host, self.is_secure)
while i <= num_retries:
# Use binary exponential backoff to desynchronize client requests.
next_sleep = random.random() * (2 ** i)
try:
# we now re-sign each request before it is retried
boto.log.debug('Token: %s' % self.provider.security_token)
request.authorize(connection=self)
if isinstance(sender, collections.Callable):
response = sender(connection, request.method, request.path,
request.body, request.headers)
else:
connection.request(request.method, request.path,
request.body, request.headers)
response = connection.getresponse()
location = response.getheader('location')
# -- gross hack --
# httplib gets confused with chunked responses to HEAD requests
# so I have to fake it out
if request.method == 'HEAD' and getattr(response,
'chunked', False):
response.chunked = 0
if isinstance(retry_handler, collections.Callable):
status = retry_handler(response, i, next_sleep)
if status:
msg, i, next_sleep = status
if msg:
boto.log.debug(msg)
time.sleep(next_sleep)
continue
if response.status == 500 or response.status == 503:
msg = 'Received %d response. ' % response.status
msg += 'Retrying in %3.1f seconds' % next_sleep
boto.log.debug(msg)
body = response.read()
elif response.status < 300 or response.status >= 400 or \
not location:
self.put_http_connection(request.host, self.is_secure,
connection)
return response
else:
scheme, request.host, request.path, \
params, query, fragment = urllib.parse.urlparse(location)
if query:
request.path += '?' + query
msg = 'Redirecting: %s' % scheme + '://'
msg += request.host + request.path
boto.log.debug(msg)
connection = self.get_http_connection(request.host,
scheme == 'https')
response = None
continue
except PleaseRetryException as e:
boto.log.debug('encountered a retry exception: %s' % e)
connection = self.new_http_connection(request.host,
self.is_secure)
response = e.response
except self.http_exceptions as e:
ex = e # e will be unset after leaving the except: block
for unretryable in self.http_unretryable_exceptions:
if isinstance(e, unretryable):
boto.log.debug(
'encountered unretryable %s exception, re-raising' %
e.__class__.__name__)
raise e
boto.log.debug('encountered %s exception, reconnecting' % \
e.__class__.__name__)
connection = self.new_http_connection(request.host,
self.is_secure)
time.sleep(next_sleep)
i += 1
# If we made it here, it's because we have exhausted our retries
# and stil haven't succeeded. So, if we have a response object,
# use it to raise an exception.
# Otherwise, raise the exception that must have already happened.
if response:
raise BotoServerError(response.status, response.reason, body)
elif ex:
raise ex
else:
msg = 'Please report this exception as a Boto Issue!'
raise BotoClientError(msg)
def testRecoveryFromBotoServerError(self, createAdapterMock, repoMock,
metricStreamerMock, multiprocessingMock):
"""
This test verifies recovery and continuation for normal
operation for metric collector after encoutering BotoServerError
for getData call
"""
exception = BotoServerError(500, "Fake BotoServerError")
# Configure multiprocessing
def mapAsync(fn, tasks):
class _(object):
def wait(self):
map(fn, tasks)
return _()
multiprocessingMock.Pool.return_value.map_async.side_effect = mapAsync
multiprocessingMock.Pipe.side_effect = multiprocessing.Pipe
multiprocessingMock.Manager = (
Mock(return_value=(
Mock(JoinableQueue=(
Mock(side_effect=multiprocessing.JoinableQueue))))))
metricPollInterval = 5
resultsOfGetCloudwatchMetricsPendingDataCollection = [
[_makeFreshMetricMockInstance(metricPollInterval, 1)],
[_makeFreshMetricMockInstance(metricPollInterval, 2)],
[_makeFreshMetricMockInstance(metricPollInterval, 3)],
KeyboardInterrupt("Fake KeyboardInterrupt to interrupt run-loop")
]
repoMock.getCloudwatchMetricsPendingDataCollection.side_effect = (
resultsOfGetCloudwatchMetricsPendingDataCollection)
repoMock.retryOnTransientErrors.side_effect = lambda f: f
# Configure the metric_collector.adapters module mock
now = datetime.datetime.utcnow()
mockResults = [exception,
([[now, 1]],
now + datetime.timedelta(seconds=metricPollInterval)),
([[now, 2]],
now + datetime.timedelta(seconds=metricPollInterval))]
adapterInstanceMock = Mock(
spec_set=_CloudwatchDatasourceAdapter)
adapterInstanceMock.getMetricData.side_effect = mockResults
adapterInstanceMock.getMetricResourceStatus.return_value = "status"
createAdapterMock.return_value = adapterInstanceMock
# Now, run MetricCollector and check results
resultOfRunCollector = dict()
def runCollector():
try:
collector = metric_collector.MetricCollector()
resultOfRunCollector["returnCode"] = collector.run()
except:
resultOfRunCollector["exception"] = sys.exc_info()[1]
# We run it in a thread in order to detect if MetricCollector.run fails to
# return and to make sure that the test script will finish (in case run
# doesn't)
thread = threading.Thread(target=runCollector)
thread.setDaemon(True)
thread.start()
thread.join(5)
self.assertFalse(thread.isAlive())
self.assertIn("exception", resultOfRunCollector)
self.assertIsInstance(resultOfRunCollector["exception"], KeyboardInterrupt)
self.assertNotIn("returnCode", resultOfRunCollector)
# Verify that metric collector didn't call setStatus on the first metric
# as the result of ClosedConnection
self.assertFalse(repoMock.setMetricStatus.called)
self.assertEqual(
metricStreamerMock.return_value.streamMetricData.call_count,
len(mockResults) -1)
def put_role_policy(self, role_name, policy_name, policy_json):
role = self.get_role(role_name)
if role:
role.put_policy(policy_name, policy_json)
else:
raise BotoServerError(404, 'Not Found')
def testMe():
accumulator["value"] += 1
raise BotoServerError(
400,
"",
json.dumps(dict(Error=(dict(Code="InvalidAction")))))
def __new__(cls, *args, **kw):
error = BotoServerError(*args, **kw)
newclass = globals().get(error.error_code, ResponseError)
obj = newclass.__new__(newclass, *args, **kw)
obj.__dict__.update(error.__dict__)
return obj
