def make_request(self, action, body):
headers = {
'X-Amz-Target': '%s.%s' % (self.TargetPrefix, action),
'Host': self.region.endpoint,
'Content-Type': 'application/x-amz-json-1.1',
'Content-Length': str(len(body)),
}
http_request = self.build_base_http_request(method='POST',
path='/',
auth_path='/',
params={},
headers=headers,
data=body)
response = self._mexe(http_request,
sender=None,
override_num_retries=10)
response_body = response.read()
boto.log.debug(response_body)
if response.status == 200:
if response_body:
return json.loads(response_body)
else:
json_body = json.loads(response_body)
fault_name = json_body.get('__type', None)
exception_class = self._faults.get(fault_name, self.ResponseError)
raise exception_class(response.status,
response.reason,
body=json_body)
def _make_request(self, action, verb, path, params):
params['ContentType'] = 'JSON'
response = self.make_request(action=action, verb='POST',
path='/', params=params)
body = response.read().decode('utf-8')
boto.log.debug(body)
if response.status == 200:
return json.loads(body)
else:
json_body = json.loads(body)
fault_name = json_body.get('Error', {}).get('Code', None)
exception_class = self._faults.get(fault_name, self.ResponseError)
raise exception_class(response.status, response.reason,
body=json_body)
def _json_iterload(fd):
"""Lazily load newline-separated JSON objects from a file-like object."""
buffer = ""
eof = False
while not eof:
try:
# Add a line to the buffer
buffer += fd.next()
except StopIteration:
# We can't let that exception bubble up, otherwise the last
# object in the file will never be decoded.
eof = True
try:
# Try to decode a JSON object.
json_object = json.loads(buffer.strip())
# Success: clear the buffer (everything was decoded).
buffer = ""
except ValueError:
if eof and buffer.strip():
# No more lines to load and the buffer contains something other
# than whitespace: the file is, in fact, malformed.
raise
# We couldn't decode a complete JSON object: load more lines.
continue
yield json_object
def test_delete_stack(self):
self.set_http_response(status_code=200)
api_response = self.service_connection.delete_stack("stack_name")
self.assertEqual(api_response, json.loads(self.default_body().decode("utf-8")))
self.assert_request_parameters(
{"Action": "DeleteStack", "ContentType": "JSON", "StackName": "stack_name", "Version": "2010-05-15"}
)
def decode(self, value):
try:
value = base64.b64decode(value)
value = json.loads(value)
except Exception as exc:
raise SQSDecodeError('Unable to decode message %s' % exc, self)
return value
def __init__(self, response, doc_service, sdf):
self.response = response
self.doc_service = doc_service
self.sdf = sdf
try:
self.content = json.loads(response.content)
except:
boto.log.error(
'Error indexing documents.\nResponse Content:\n{}\n\n'
'SDF:\n{}'.format(response.content, self.sdf))
raise boto.exception.BotoServerError(self.response.status_code,
'',
body=response.content)
self.status = self.content['status']
if self.status == 'error':
self.errors = [
e.get('message') for e in self.content.get('errors', [])
]
else:
self.errors = []
self.adds = self.content['adds']
self.deletes = self.content['deletes']
self._check_num_ops('add', self.adds)
self._check_num_ops('delete', self.deletes)
def test_decoding_full_doc(self):
'''Simple List decoding that had caused some errors'''
dynamizer = types.Dynamizer()
doc = '{"__type__":{"S":"Story"},"company_tickers":{"SS":["NASDAQ-TSLA","NYSE-F","NYSE-GM"]},"modified_at":{"N":"1452525162"},"created_at":{"N":"1452525162"},"version":{"N":"1"},"categories":{"SS":["AUTOMTVE","LTRTR","MANUFCTU","PN","PRHYPE","TAXE","TJ","TL"]},"provider_categories":{"L":[{"S":"F"},{"S":"GM"},{"S":"TSLA"}]},"received_at":{"S":"2016-01-11T11:26:31Z"}}'
output_doc = {
'provider_categories': ['F', 'GM', 'TSLA'],
'__type__':
'Story',
'company_tickers':
set(['NASDAQ-TSLA', 'NYSE-GM', 'NYSE-F']),
'modified_at':
Decimal('1452525162'),
'version':
Decimal('1'),
'received_at':
'2016-01-11T11:26:31Z',
'created_at':
Decimal('1452525162'),
'categories':
set([
'LTRTR', 'TAXE', 'MANUFCTU', 'TL', 'TJ', 'AUTOMTVE', 'PRHYPE',
'PN'
])
}
self.assertEqual(json.loads(doc, object_hook=dynamizer.decode),
output_doc)
def set_topic_attributes(self, topic, attr_name, attr_value):
"""
Get attributes of a Topic
:type topic: string
:param topic: The ARN of the topic.
:type attr_name: string
:param attr_name: The name of the attribute you want to set.
Only a subset of the topic's attributes are mutable.
Valid values: Policy | DisplayName
:type attr_value: string
:param attr_value: The new value for the attribute.
"""
params = {'ContentType': 'JSON',
'TopicArn': topic,
'AttributeName': attr_name,
'AttributeValue': attr_value}
response = self.make_request('SetTopicAttributes', params, '/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def get_all_subscriptions_by_topic(self, topic, next_token=None):
"""
Get list of all subscriptions to a specific topic.
:type topic: string
:param topic: The ARN of the topic for which you wish to
find subscriptions.
:type next_token: string
:param next_token: Token returned by the previous call to
this method.
"""
params = {'ContentType': 'JSON',
'TopicArn': topic}
if next_token:
params['NextToken'] = next_token
response = self.make_request('ListSubscriptionsByTopic', params,
'/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def confirm_subscription(self, topic, token,
authenticate_on_unsubscribe=False):
"""
Get properties of a Topic
:type topic: string
:param topic: The ARN of the new topic.
:type token: string
:param token: Short-lived token sent to and endpoint during
the Subscribe operation.
:type authenticate_on_unsubscribe: bool
:param authenticate_on_unsubscribe: Optional parameter indicating
that you wish to disable
unauthenticated unsubscription
of the subscription.
"""
params = {'ContentType': 'JSON',
'TopicArn': topic,
'Token': token}
if authenticate_on_unsubscribe:
params['AuthenticateOnUnsubscribe'] = 'true'
response = self.make_request('ConfirmSubscription', params, '/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def _do_request(self, call, params, method):
"""
Do a request via ``self.make_request`` and parse the JSON response.
:type call: string
:param call: Call name, e.g. ``CreateStack``
:type params: dict
:param params: Dictionary of call parameters
:type method: string
:param method: HTTP method to use
:rtype: dict
:return: Parsed JSON response data
"""
params['Version'] = self.version
logging.debug('action = ' + call)
logging.debug('params: ' + json.dumps(params))
response = self.make_request(call, params, self.path, method)
body = response.read().decode('utf-8')
logging.debug('response: ' + body)
if response.status == 200:
body = json.loads(body)
return body
else:
logging.error('%s %s' % (response.status, response.reason))
logging.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body=body)
def _do_request(self, call, params, path, method):
"""
Do a request via ``self.make_request`` and parse the JSON response.
:type call: string
:param call: Call name, e.g. ``CreateStack``
:type params: dict
:param params: Dictionary of call parameters
:type path: string
:param path: Server path
:type method: string
:param method: HTTP method to use
:rtype: dict
:return: Parsed JSON response data
"""
response = self.make_request(call, params, path, method)
body = response.read().decode('utf-8')
if response.status == 200:
body = json.loads(body)
return body
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body=body)
def make_request(self, action, body='', object_hook=None):
"""
:raises: ``DynamoDBExpiredTokenError`` if the security token expires.
"""
headers = {
'X-Amz-Target':
'%s_%s.%s' % (self.ServiceName, self.Version, action),
'Host': self.region.endpoint,
'Content-Type': 'application/x-amz-json-1.0',
'Content-Length': str(len(body))
}
http_request = self.build_base_http_request('POST', '/', '/', {},
headers, body, None)
start = time.time()
response = self._mexe(http_request,
sender=None,
override_num_retries=self.NumberRetries,
retry_handler=self._retry_handler)
elapsed = (time.time() - start) * 1000
request_id = response.getheader('x-amzn-RequestId')
boto.log.debug('RequestId: %s' % request_id)
boto.perflog.debug('%s: id=%s time=%sms', headers['X-Amz-Target'],
request_id, int(elapsed))
response_body = response.read().decode('utf-8')
boto.log.debug(response_body)
return json.loads(response_body, object_hook=object_hook)
def __init__(self, response, doc_service, sdf):
self.response = response
self.doc_service = doc_service
self.sdf = sdf
_body = response.content.decode("utf-8")
try:
self.content = json.loads(_body)
except:
boto.log.error("Error indexing documents.\nResponse Content:\n{0}" "\n\nSDF:\n{1}".format(_body, self.sdf))
raise boto.exception.BotoServerError(self.response.status_code, "", body=_body)
self.status = self.content["status"]
if self.status == "error":
self.errors = [e.get("message") for e in self.content.get("errors", [])]
for e in self.errors:
if "Illegal Unicode character" in e:
raise EncodingError("Illegal Unicode character in document")
elif e == "The Content-Length is too long":
raise ContentTooLongError("Content was too long")
else:
self.errors = []
self.adds = self.content["adds"]
self.deletes = self.content["deletes"]
self._check_num_ops("add", self.adds)
self._check_num_ops("delete", self.deletes)
def __getitem__(self, key):
if key not in self:
# allow dict to throw the KeyError
return super(LazyLoadMetadata, self).__getitem__(key)
# already loaded
val = super(LazyLoadMetadata, self).__getitem__(key)
if val is not None:
return val
if key in self._leaves:
resource = self._leaves[key]
val = boto.utils.retry_url(self._url + urllib.quote(resource,
safe="/:"),
num_retries=self._num_retries)
if val and val[0] == '{':
val = json.loads(val)
else:
p = val.find('\n')
if p > 0:
val = val.split('\n')
self[key] = val
elif key in self._dicts:
self[key] = LazyLoadMetadata(self._url + key + '/',
self._num_retries)
return super(LazyLoadMetadata, self).__getitem__(key)
def get_instance_identity(version='latest', url='http://169.254.169.254',
timeout=None, num_retries=5):
"""
Returns the instance identity as a nested Python dictionary.
"""
iid = {}
base_url = _build_instance_metadata_url(url, version,
'dynamic/instance-identity/')
if timeout is not None:
original = socket.getdefaulttimeout()
socket.setdefaulttimeout(timeout)
try:
data = retry_url(base_url, num_retries=num_retries)
fields = data.split('\n')
for field in fields:
val = retry_url(base_url + '/' + field + '/')
if val[0] == '{':
val = json.loads(val)
if field:
iid[field] = val
return iid
except urllib.error.URLError as e:
return None
finally:
if timeout is not None:
socket.setdefaulttimeout(original)
def set_topic_attributes(self, topic, attr_name, attr_value):
"""
Get attributes of a Topic
:type topic: string
:param topic: The ARN of the topic.
:type attr_name: string
:param attr_name: The name of the attribute you want to set.
Only a subset of the topic's attributes are mutable.
Valid values: Policy | DisplayName
:type attr_value: string
:param attr_value: The new value for the attribute.
"""
params = {
'ContentType': 'JSON',
'TopicArn': topic,
'AttributeName': attr_name,
'AttributeValue': attr_value
}
response = self.make_request('SetTopicAttributes', params, '/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def add_permission(self, topic, label, account_ids, actions):
"""
Adds a statement to a topic's access control policy, granting
access for the specified AWS accounts to the specified actions.
:type topic: string
:param topic: The ARN of the topic.
:type label: string
:param label: A unique identifier for the new policy statement.
:type account_ids: list of strings
:param account_ids: The AWS account ids of the users who will be
give access to the specified actions.
:type actions: list of strings
:param actions: The actions you want to allow for each of the
specified principal(s).
"""
params = {"ContentType": "JSON", "TopicArn": topic, "Label": label}
self.build_list_params(params, account_ids, "AWSAccountId.member")
self.build_list_params(params, actions, "ActionName.member")
response = self.make_request("AddPermission", params, "/", "GET")
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error("%s %s" % (response.status, response.reason))
boto.log.error("%s" % body)
raise self.ResponseError(response.status, response.reason, body)
def __init__(self, response, doc_service, sdf, signed_request=False):
self.response = response
self.doc_service = doc_service
self.sdf = sdf
self.signed_request = signed_request
if self.signed_request:
self.content = response
else:
_body = response.content.decode('utf-8')
try:
self.content = json.loads(_body)
except:
boto.log.error('Error indexing documents.\nResponse Content:\n{0}'
'\n\nSDF:\n{1}'.format(_body, self.sdf))
raise boto.exception.BotoServerError(self.response.status_code, '',
body=_body)
self.status = self.content['status']
if self.status == 'error':
self.errors = [e.get('message') for e in self.content.get('errors',
[])]
for e in self.errors:
if "Illegal Unicode character" in e:
raise EncodingError("Illegal Unicode character in document")
elif e == "The Content-Length is too long":
raise ContentTooLongError("Content was too long")
else:
self.errors = []
self.adds = self.content['adds']
self.deletes = self.content['deletes']
self._check_num_ops('add', self.adds)
self._check_num_ops('delete', self.deletes)
def __call__(self, query):
"""Make a call to CloudSearch
:type query: :class:`boto.cloudsearch2.search.Query`
:param query: A group of search criteria
:rtype: :class:`boto.cloudsearch2.search.SearchResults`
:return: search results
"""
api_version = '2013-01-01'
if self.domain:
api_version = self.domain.layer1.APIVersion
url = "http://%s/%s/search" % (self.endpoint, api_version)
params = query.to_params()
r2 = self.domain.layer1.make_dummy_request_for_cloudsearch2(
action=None,
params=params,
path='/' + api_version + '/search',
verb='GET',
host=self.endpoint)
r = self.session.get(url, params=params, headers=r2.headers)
_body = r.content.decode('utf-8')
try:
data = json.loads(_body)
except ValueError:
if r.status_code == 403:
msg = ''
import re
g = re.search('<html><body><h1>403 Forbidden</h1>([^<]+)<',
_body)
try:
msg = ': %s' % (g.groups()[0].strip())
except AttributeError:
pass
raise SearchServiceException(
'Authentication error from Amazon%s' % msg)
raise SearchServiceException(
"Got non-json response from Amazon. %s" % _body, query)
if '__type' in data and data['__type'] == '#AccessDenied':
message = 'Access Denied'
if 'message' in data:
message = data['message']
raise SearchServiceException(message, '#AccessDenied')
if 'messages' in data and 'error' in data:
for m in data['messages']:
if m['severity'] == 'fatal':
raise SearchServiceException(
"Error processing search %s "
"=> %s" % (params, m['message']), query)
elif 'error' in data:
raise SearchServiceException(
"Unknown error processing search %s" % json.dumps(data), query)
data['query'] = query
data['search_service'] = self
return SearchResults(**data)
def __init__(self, response, doc_service, sdf):
self.response = response
self.doc_service = doc_service
self.sdf = sdf
_body = response.content.decode('utf-8')
try:
self.content = json.loads(_body)
except:
boto.log.error('Error indexing documents.\nResponse Content:\n{0}'
'\n\nSDF:\n{1}'.format(_body, self.sdf))
raise boto.exception.BotoServerError(self.response.status_code,
'',
body=_body)
self.status = self.content['status']
if self.status == 'error':
self.errors = [
e.get('message') for e in self.content.get('errors', [])
]
for e in self.errors:
if "Illegal Unicode character" in e:
raise EncodingError(
"Illegal Unicode character in document")
elif e == "The Content-Length is too long":
raise ContentTooLongError("Content was too long")
else:
self.errors = []
self.adds = self.content['adds']
self.deletes = self.content['deletes']
self._check_num_ops('add', self.adds)
self._check_num_ops('delete', self.deletes)
def subscribe(self, topic, protocol, endpoint):
"""
Subscribe to a Topic.
:type topic: string
:param topic: The ARN of the new topic.
:type protocol: string
:param protocol: The protocol used to communicate with
the subscriber. Current choices are:
email|email-json|http|https|sqs
:type endpoint: string
:param endpoint: The location of the endpoint for
the subscriber.
* For email, this would be a valid email address
* For email-json, this would be a valid email address
* For http, this would be a URL beginning with http
* For https, this would be a URL beginning with https
* For sqs, this would be the ARN of an SQS Queue
"""
params = {
'ContentType': 'JSON',
'TopicArn': topic,
'Protocol': protocol,
'Endpoint': endpoint
}
response = self.make_request('Subscribe', params, '/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def decode(self, value):
try:
value = base64.b64decode(value)
value = json.loads(value)
except:
raise SQSDecodeError("Unable to decode message", self)
return value
def confirm_subscription(self,
topic,
token,
authenticate_on_unsubscribe=False):
"""
Get properties of a Topic
:type topic: string
:param topic: The ARN of the new topic.
:type token: string
:param token: Short-lived token sent to and endpoint during
the Subscribe operation.
:type authenticate_on_unsubscribe: bool
:param authenticate_on_unsubscribe: Optional parameter indicating
that you wish to disable
unauthenticated unsubscription
of the subscription.
"""
params = {'ContentType': 'JSON', 'TopicArn': topic, 'Token': token}
if authenticate_on_unsubscribe:
params['AuthenticateOnUnsubscribe'] = 'true'
response = self.make_request('ConfirmSubscription', params, '/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def _json_iterload(fd):
"""Lazily load newline-separated JSON objects from a file-like object."""
buffer = ""
eof = False
while not eof:
try:
# Add a line to the buffer
buffer += fd.next()
except StopIteration:
# We can't let that exception bubble up, otherwise the last
# object in the file will never be decoded.
eof = True
try:
# Try to decode a JSON object.
json_object = json.loads(buffer.strip())
# Success: clear the buffer (everything was decoded).
buffer = ""
except ValueError:
if eof and buffer.strip():
# No more lines to load and the buffer contains something other
# than whitespace: the file is, in fact, malformed.
raise
# We couldn't decode a complete JSON object: load more lines.
continue
yield json_object
def get_all_subscriptions_by_topic(self, topic, next_token=None):
"""
Get list of all subscriptions to a specific topic.
:type topic: string
:param topic: The ARN of the topic for which you wish to
find subscriptions.
:type next_token: string
:param next_token: Token returned by the previous call to
this method.
"""
params = {'ContentType': 'JSON', 'TopicArn': topic}
if next_token:
params['NextToken'] = next_token
response = self.make_request('ListSubscriptionsByTopic', params, '/',
'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def __init__(self, response, doc_service, sdf):
self.response = response
self.doc_service = doc_service
self.sdf = sdf
try:
if hasattr(response.content, 'decode') and not hasattr(response.content, 'encode'):
# decode the python3 byte literal
rc = response.content.decode('utf-8')
else:
rc = response.content
self.content = json.loads(rc)
except:
boto.log.error('Error indexing documents.\nResponse Content:\n{0}\n\n'
'SDF:\n{1}'.format(response.content, self.sdf))
raise boto.exception.BotoServerError(self.response.status_code, '',
body=response.content)
self.status = self.content['status']
if self.status == 'error':
self.errors = [e.get('message') for e in self.content.get('errors',
[])]
for e in self.errors:
if "Illegal Unicode character" in e:
raise EncodingError("Illegal Unicode character in document")
elif e == "The Content-Length is too long":
raise ContentTooLongError("Content was too long")
else:
self.errors = []
self.adds = self.content['adds']
self.deletes = self.content['deletes']
self._check_num_ops('add', self.adds)
self._check_num_ops('delete', self.deletes)
def publish(self, topic, message, subject=None):
"""
Get properties of a Topic
:type topic: string
:param topic: The ARN of the new topic.
:type message: string
:param message: The message you want to send to the topic.
Messages must be UTF-8 encoded strings and
be at most 4KB in size.
:type subject: string
:param subject: Optional parameter to be used as the "Subject"
line of the email notifications.
"""
params = {'ContentType': 'JSON', 'TopicArn': topic, 'Message': message}
if subject:
params['Subject'] = subject
response = self.make_request('Publish', params, '/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def get_instance_identity(version='latest',
url='http://169.254.169.254',
timeout=None,
num_retries=5):
"""
Returns the instance identity as a nested Python dictionary.
"""
iid = {}
base_url = _build_instance_metadata_url(url, version,
'dynamic/instance-identity/')
if timeout is not None:
original = socket.getdefaulttimeout()
socket.setdefaulttimeout(timeout)
try:
data = retry_url(base_url, num_retries=num_retries)
fields = data.split('\n')
for field in fields:
val = retry_url(base_url + '/' + field + '/')
if val[0] == '{':
val = json.loads(val)
if field:
iid[field] = val
return iid
except urllib.error.URLError as e:
return None
finally:
if timeout is not None:
socket.setdefaulttimeout(original)
def add_permission(self, topic, label, account_ids, actions):
"""
Adds a statement to a topic's access control policy, granting
access for the specified AWS accounts to the specified actions.
:type topic: string
:param topic: The ARN of the topic.
:type label: string
:param label: A unique identifier for the new policy statement.
:type account_ids: list of strings
:param account_ids: The AWS account ids of the users who will be
give access to the specified actions.
:type actions: list of strings
:param actions: The actions you want to allow for each of the
specified principal(s).
"""
params = {'ContentType': 'JSON', 'TopicArn': topic, 'Label': label}
self.build_list_params(params, account_ids, 'AWSAccountId.member')
self.build_list_params(params, actions, 'ActionName.member')
response = self.make_request('AddPermission', params, '/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def set_topic_attributes(self, topic, attr_name, attr_value):
"""
Get attributes of a Topic
:type topic: string
:param topic: The ARN of the topic.
:type attr_name: string
:param attr_name: The name of the attribute you want to set.
Only a subset of the topic's attributes are mutable.
Valid values: Policy | DisplayName
:type attr_value: string
:param attr_value: The new value for the attribute.
"""
params = {"ContentType": "JSON", "TopicArn": topic, "AttributeName": attr_name, "AttributeValue": attr_value}
response = self.make_request("SetTopicAttributes", params, "/", "GET")
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error("%s %s" % (response.status, response.reason))
boto.log.error("%s" % body)
raise self.ResponseError(response.status, response.reason, body)
def __call__(self, query):
"""Make a call to CloudSearch
:type query: :class:`boto.cloudsearch.search.Query`
:param query: A group of search criteria
:rtype: :class:`boto.cloudsearch.search.SearchResults`
:return: search results
"""
url = "http://%s/2011-02-01/search" % (self.endpoint)
params = query.to_params()
r = requests.get(url, params=params)
try:
data = json.loads(r.content)
except ValueError, e:
if r.status_code == 403:
msg = ''
import re
g = re.search('<html><body><h1>403 Forbidden</h1>([^<]+)<', r.content)
try:
msg = ': %s' % (g.groups()[0].strip())
except AttributeError:
pass
raise SearchServiceException('Authentication error from Amazon%s' % msg)
raise SearchServiceException("Got non-json response from Amazon")
def publish(self, topic, message, subject=None):
"""
Get properties of a Topic
:type topic: string
:param topic: The ARN of the new topic.
:type message: string
:param message: The message you want to send to the topic.
Messages must be UTF-8 encoded strings and
be at most 4KB in size.
:type subject: string
:param subject: Optional parameter to be used as the "Subject"
line of the email notifications.
"""
params = {"ContentType": "JSON", "TopicArn": topic, "Message": message}
if subject:
params["Subject"] = subject
response = self.make_request("Publish", params, "/", "GET")
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error("%s %s" % (response.status, response.reason))
boto.log.error("%s" % body)
raise self.ResponseError(response.status, response.reason, body)
def __call__(self, query):
"""Make a call to CloudSearch
:type query: :class:`boto.cloudsearch.search.Query`
:param query: A group of search criteria
:rtype: :class:`boto.cloudsearch.search.SearchResults`
:return: search results
"""
url = "http://%s/2011-02-01/search" % (self.endpoint)
params = query.to_params()
r = requests.get(url, params=params)
try:
data = json.loads(r.content)
except json.JSONDecodeError, e:
if r.status_code == 403:
msg = ''
import re
g = re.search('<html><body><h1>403 Forbidden</h1>([^<]+)<',
r.content)
try:
msg = ': %s' % (g.groups()[0].strip())
except AttributeError:
pass
raise SearchServiceException(
'Authentication error from Amazon%s' % msg)
raise SearchServiceException("Got non-json response from Amazon")
def confirm_subscription(self, topic, token, authenticate_on_unsubscribe=False):
"""
Get properties of a Topic
:type topic: string
:param topic: The ARN of the new topic.
:type token: string
:param token: Short-lived token sent to and endpoint during
the Subscribe operation.
:type authenticate_on_unsubscribe: bool
:param authenticate_on_unsubscribe: Optional parameter indicating
that you wish to disable
unauthenticated unsubscription
of the subscription.
"""
params = {"ContentType": "JSON", "TopicArn": topic, "Token": token}
if authenticate_on_unsubscribe:
params["AuthenticateOnUnsubscribe"] = "true"
response = self.make_request("ConfirmSubscription", params, "/", "GET")
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error("%s %s" % (response.status, response.reason))
boto.log.error("%s" % body)
raise self.ResponseError(response.status, response.reason, body)
def _do_request(self, call, params, path, method):
"""
Do a request via ``self.make_request`` and parse the JSON response.
:type call: string
:param call: Call name, e.g. ``CreateStack``
:type params: dict
:param params: Dictionary of call parameters
:type path: string
:param path: Server path
:type method: string
:param method: HTTP method to use
:rtype: dict
:return: Parsed JSON response data
"""
response = self.make_request(call, params, path, method)
body = response.read().decode('utf-8')
if response.status == 200:
body = json.loads(body)
return body
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body=body)
def decode(self, value):
try:
value = base64.b64decode(value.encode('utf-8')).decode('utf-8')
value = json.loads(value)
except:
raise SQSDecodeError('Unable to decode message', self)
return value
def __call__(self, query):
"""Make a call to CloudSearch
:type query: :class:`boto.cloudsearch.search.Query`
:param query: A group of search criteria
:rtype: :class:`boto.cloudsearch.search.SearchResults`
:return: search results
"""
url = "http://%s/2011-02-01/search" % (self.endpoint)
params = query.to_params()
r = requests.get(url, params=params)
data = json.loads(r.content)
data['query'] = query
data['search_service'] = self
if 'messages' in data and 'error' in data:
for m in data['messages']:
if m['severity'] == 'fatal':
raise SearchServiceException("Error processing search %s "
"=> %s" % (params, m['message']), query)
elif 'error' in data:
raise SearchServiceException("Unknown error processing search %s"
% (params), query)
return SearchResults(**data)
def subscribe(self, topic, protocol, endpoint):
"""
Subscribe to a Topic.
:type topic: string
:param topic: The ARN of the new topic.
:type protocol: string
:param protocol: The protocol used to communicate with
the subscriber. Current choices are:
email|email-json|http|https|sqs
:type endpoint: string
:param endpoint: The location of the endpoint for
the subscriber.
* For email, this would be a valid email address
* For email-json, this would be a valid email address
* For http, this would be a URL beginning with http
* For https, this would be a URL beginning with https
* For sqs, this would be the ARN of an SQS Queue
"""
params = {"ContentType": "JSON", "TopicArn": topic, "Protocol": protocol, "Endpoint": endpoint}
response = self.make_request("Subscribe", params, "/", "GET")
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error("%s %s" % (response.status, response.reason))
boto.log.error("%s" % body)
raise self.ResponseError(response.status, response.reason, body)
def update_stack(self, stack_name, template_body=None, template_url=None,
parameters=[], notification_arns=[], disable_rollback=False,
timeout_in_minutes=None, capabilities=None, tags=None):
"""
Updates a CloudFormation Stack as specified by the template.
:type stack_name: string
:param stack_name: The name of the Stack, must be unique amoung running
Stacks.
:type template_body: string
:param template_body: The template body (JSON string)
:type template_url: string
:param template_url: An S3 URL of a stored template JSON document. If
both the template_body and template_url are
specified, the template_body takes precedence.
:type parameters: list of tuples
:param parameters: A list of (key, value) pairs for template input
parameters.
:type notification_arns: list of strings
:param notification_arns: A list of SNS topics to send Stack event
notifications to.
:type disable_rollback: bool
:param disable_rollback: Indicates whether or not to rollback on
failure.
:type timeout_in_minutes: int
:param timeout_in_minutes: Maximum amount of time to let the Stack
spend creating itself. If this timeout is exceeded,
the Stack will enter the CREATE_FAILED state
:type capabilities: list
:param capabilities: The list of capabilities you want to allow in
the stack. Currently, the only valid capability is
'CAPABILITY_IAM'.
:type tags: dict
:param tags: A dictionary of (key, value) pairs of tags to
associate with this stack.
:rtype: string
:return: The unique Stack ID.
"""
params = self._build_create_or_update_params(stack_name,
template_body, template_url, parameters, notification_arns,
disable_rollback, timeout_in_minutes, capabilities, tags)
response = self.make_request('UpdateStack', params, '/', 'POST')
body = response.read()
if response.status == 200:
body = json.loads(body)
return body['UpdateStackResponse']['UpdateStackResult']['StackId']
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def update_stack(self, stack_name, template_body=None, template_url=None,
parameters=[], notification_arns=[], disable_rollback=False,
timeout_in_minutes=None, capabilities=None, tags=None):
"""
Updates a CloudFormation Stack as specified by the template.
:type stack_name: string
:param stack_name: The name of the Stack, must be unique amoung running
Stacks.
:type template_body: string
:param template_body: The template body (JSON string)
:type template_url: string
:param template_url: An S3 URL of a stored template JSON document. If
both the template_body and template_url are
specified, the template_body takes precedence.
:type parameters: list of tuples
:param parameters: A list of (key, value) pairs for template input
parameters.
:type notification_arns: list of strings
:param notification_arns: A list of SNS topics to send Stack event
notifications to.
:type disable_rollback: bool
:param disable_rollback: Indicates whether or not to rollback on
failure.
:type timeout_in_minutes: int
:param timeout_in_minutes: Maximum amount of time to let the Stack
spend creating itself. If this timeout is exceeded,
the Stack will enter the CREATE_FAILED state
:type capabilities: list
:param capabilities: The list of capabilities you want to allow in
the stack. Currently, the only valid capability is
'CAPABILITY_IAM'.
:type tags: dict
:param tags: A dictionary of (key, value) pairs of tags to
associate with this stack.
:rtype: string
:return: The unique Stack ID.
"""
params = self._build_create_or_update_params(stack_name,
template_body, template_url, parameters, notification_arns,
disable_rollback, timeout_in_minutes, capabilities, tags)
response = self.make_request('UpdateStack', params, '/', 'POST')
body = response.read()
if response.status == 200:
body = json.loads(body)
return body['UpdateStackResponse']['UpdateStackResult']['StackId']
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def subscribe_sqs_queue(self, topic, queue):
"""
Subscribe an SQS queue to a topic.
This is convenience method that handles most of the complexity involved
in using an SQS queue as an endpoint for an SNS topic. To achieve this
the following operations are performed:
* The correct ARN is constructed for the SQS queue and that ARN is
then subscribed to the topic.
* A JSON policy document is contructed that grants permission to
the SNS topic to send messages to the SQS queue.
* This JSON policy is then associated with the SQS queue using
the queue's set_attribute method. If the queue already has
a policy associated with it, this process will add a Statement to
that policy. If no policy exists, a new policy will be created.
:type topic: string
:param topic: The ARN of the new topic.
:type queue: A boto Queue object
:param queue: The queue you wish to subscribe to the SNS Topic.
"""
t = queue.id.split('/')
q_arn = queue.arn
sid = hashlib.md5(topic + q_arn).hexdigest()
sid_exists = False
resp = self.subscribe(topic, 'sqs', q_arn)
attr = queue.get_attributes('Policy')
if 'Policy' in attr:
policy = json.loads(attr['Policy'])
else:
policy = {}
if 'Version' not in policy:
policy['Version'] = '2008-10-17'
if 'Statement' not in policy:
policy['Statement'] = []
# See if a Statement with the Sid exists already.
for s in policy['Statement']:
if s['Sid'] == sid:
sid_exists = True
if not sid_exists:
statement = {
'Action': 'SQS:SendMessage',
'Effect': 'Allow',
'Principal': {
'AWS': '*'
},
'Resource': q_arn,
'Sid': sid,
'Condition': {
'StringLike': {
'aws:SourceArn': topic
}
}
}
policy['Statement'].append(statement)
queue.set_attribute('Policy', json.dumps(policy))
return resp
def __init__(self, status, reason, body=None, *args):
super(BotoServerError, self).__init__(status, reason, body, *args)
self.status = status
self.reason = reason
self.body = body or ''
self.request_id = None
self.error_code = None
self._error_message = None
self.message = ''
self.box_usage = None
if isinstance(self.body, bytes):
try:
self.body = self.body.decode('utf-8')
except UnicodeDecodeError:
boto.log.debug('Unable to decode body from bytes!')
# Attempt to parse the error response. If body isn't present,
# then just ignore the error response.
if self.body:
# Check if it looks like a ``dict``.
if hasattr(self.body, 'items'):
# It's not a string, so trying to parse it will fail.
# But since it's data, we can work with that.
self.request_id = self.body.get('RequestId', None)
if 'Error' in self.body:
# XML-style
error = self.body.get('Error', {})
self.error_code = error.get('Code', None)
self.message = error.get('Message', None)
else:
# JSON-style.
self.message = self.body.get('message', None)
else:
try:
h = handler.XmlHandlerWrapper(self, self)
h.parseString(self.body)
except (TypeError, xml.sax.SAXParseException) as pe:
# What if it's JSON? Let's try that.
try:
parsed = json.loads(self.body)
if 'RequestId' in parsed:
self.request_id = parsed['RequestId']
if 'Error' in parsed:
if 'Code' in parsed['Error']:
self.error_code = parsed['Error']['Code']
if 'Message' in parsed['Error']:
self.message = parsed['Error']['Message']
except (TypeError, ValueError):
# Remove unparsable message body so we don't include garbage
# in exception. But first, save self.body in self.error_message
# because occasionally we get error messages from Eucalyptus
# that are just text strings that we want to preserve.
self.message = self.body
self.body = None
def __init__(self, status, reason, body=None, *args):
super(BotoServerError, self).__init__(status, reason, body, *args)
self.status = status
self.reason = reason
self.body = body or ''
self.request_id = None
self.error_code = None
self._error_message = None
self.message = ''
self.box_usage = None
if isinstance(self.body, bytes):
try:
self.body = self.body.decode('utf-8')
except UnicodeDecodeError:
boto.log.debug('Unable to decode body from bytes!')
# Attempt to parse the error response. If body isn't present,
# then just ignore the error response.
if self.body:
# Check if it looks like a ``dict``.
if hasattr(self.body, 'items'):
# It's not a string, so trying to parse it will fail.
# But since it's data, we can work with that.
self.request_id = self.body.get('RequestId', None)
if 'Error' in self.body:
# XML-style
error = self.body.get('Error', {})
self.error_code = error.get('Code', None)
self.message = error.get('Message', None)
else:
# JSON-style.
self.message = self.body.get('message', None)
else:
try:
h = handler.XmlHandlerWrapper(self, self)
h.parseString(self.body)
except (TypeError, xml.sax.SAXParseException):
# What if it's JSON? Let's try that.
try:
parsed = json.loads(self.body)
if 'RequestId' in parsed:
self.request_id = parsed['RequestId']
if 'Error' in parsed:
if 'Code' in parsed['Error']:
self.error_code = parsed['Error']['Code']
if 'Message' in parsed['Error']:
self.message = parsed['Error']['Message']
except (TypeError, ValueError):
# Remove unparsable message body so we don't include garbage
# in exception. But first, save self.body in self.error_message
# because occasionally we get error messages from Eucalyptus
# that are just text strings that we want to preserve.
self.message = self.body
self.body = None
def _get_response(self, action, params, path='/', verb='GET'):
params['ContentType'] = 'JSON'
response = self.make_request(action, params, path, verb)
body = response.read().decode('utf-8')
boto.log.debug(body)
if response.status == 200:
return json.loads(body)
else:
raise self.ResponseError(response.status, response.reason, body)
def test_create_role_default(self):
self.set_http_response(status_code=200)
self.service_connection.create_role('a_name')
self.assert_request_parameters(
{'Action': 'CreateRole',
'RoleName': 'a_name'},
ignore_params_values=['Version', 'AssumeRolePolicyDocument'])
self.assertDictEqual(json.loads(self.actual_request.params["AssumeRolePolicyDocument"]), {"Statement": [{"Action": ["sts:AssumeRole"], "Effect": "Allow", "Principal": {"Service": ["ec2.amazonaws.com"]}}]})
def get_template(self, stack_name_or_id):
params = {"ContentType": "JSON", "StackName": stack_name_or_id}
response = self.make_request("GetTemplate", params, "/", "GET")
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error("%s %s" % (response.status, response.reason))
boto.log.error("%s" % body)
raise self.ResponseError(response.status, response.reason, body)
def test_delete_stack(self):
self.set_http_response(status_code=200)
api_response = self.service_connection.delete_stack('stack_name')
self.assertEqual(api_response, json.loads(self.default_body().decode('utf-8')))
self.assert_request_parameters({
'Action': 'DeleteStack',
'ContentType': 'JSON',
'StackName': 'stack_name',
'Version': '2010-05-15',
})
def test_binary_input(self):
"""
This test ensures that binary is base64 encoded when it is sent to
the service.
"""
self.set_http_response(status_code=200)
data = b'\x00\x01\x02\x03\x04\x05'
self.service_connection.encrypt(key_id='foo', plaintext=data)
body = json.loads(self.actual_request.body)
self.assertEqual(body['Plaintext'], 'AAECAwQF')
def get_template(self, stack_name_or_id):
params = {'ContentType': "JSON", 'StackName': stack_name_or_id}
response = self.make_request('GetTemplate', params, '/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def test_delete_stack(self):
self.set_http_response(status_code=200)
api_response = self.service_connection.delete_stack('stack_name')
self.assertEqual(api_response, json.loads(self.default_body().decode('utf-8')))
self.assert_request_parameters({
'Action': 'DeleteStack',
'ContentType': 'JSON',
'StackName': 'stack_name',
'Version': '2010-05-15',
})
def describe_stack_resource(self, stack_name_or_id, logical_resource_id):
params = {"ContentType": "JSON", "StackName": stack_name_or_id, "LogicalResourceId": logical_resource_id}
response = self.make_request("DescribeStackResource", params, "/", "GET")
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error("%s %s" % (response.status, response.reason))
boto.log.error("%s" % body)
raise self.ResponseError(response.status, response.reason, body)
def test_put_record_binary(self):
self.set_http_response(status_code=200)
self.service_connection.put_record('stream-name',
b'\x00\x01\x02\x03\x04\x05', 'partition-key')
body = json.loads(self.actual_request.body)
self.assertEqual(body['Data'], 'AAECAwQF')
target = self.actual_request.headers['X-Amz-Target']
self.assertTrue('PutRecord' in target)
def test_put_record_string(self):
self.set_http_response(status_code=200)
self.service_connection.put_record('stream-name', 'data',
'partition-key')
body = json.loads(self.actual_request.body)
self.assertEqual(body['Data'], 'ZGF0YQ==')
target = self.actual_request.headers['X-Amz-Target']
self.assertTrue('PutRecord' in target)
def test_binary_input(self):
"""
This test ensures that binary is base64 encoded when it is sent to
the service.
"""
self.set_http_response(status_code=200)
data = b'\x00\x01\x02\x03\x04\x05'
self.service_connection.encrypt(key_id='foo', plaintext=data)
body = json.loads(self.actual_request.body.decode('utf-8'))
self.assertEqual(body['Plaintext'], 'AAECAwQF')
def get_template(self, stack_name_or_id):
params = {'ContentType': "JSON", 'StackName': stack_name_or_id}
response = self.make_request('GetTemplate', params, '/', 'GET')
body = response.read()
if response.status == 200:
return json.loads(body)
else:
boto.log.error('%s %s' % (response.status, response.reason))
boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)