def _make_request(self, action, params, path='/', verb='GET'):
params['ContentType'] = 'JSON'
response = self.make_request(action=action,
verb=verb,
path=path,
params=params)
body = response.read().decode('utf-8')
fcu_boto.log.debug(body)
if response.status == 200:
return json.loads(body)
else:
fcu_boto.log.error('%s %s' % (response.status, response.reason))
fcu_boto.log.error('%s' % body)
raise self.ResponseError(response.status, response.reason, body)
def endElement(self, name, value, connection):
if name == 'AlarmName':
self.name = value
elif name == 'HistoryData':
self.data = json.loads(value)
elif name == 'HistoryItemType':
self.tem_type = value
elif name == 'HistorySummary':
self.summary = value
elif name == 'Timestamp':
try:
self.timestamp = datetime.strptime(value,
'%Y-%m-%dT%H:%M:%S.%fZ')
except ValueError:
self.timestamp = datetime.strptime(value, '%Y-%m-%dT%H:%M:%SZ')
def __init__(self, expected_responses, response):
self.status = response.status
self.body = response.read()
self.code = None
try:
body = json.loads(self.body)
self.code = body["code"]
msg = 'Expected %s, got ' % expected_responses
msg += '(%d, code=%s, message=%s)' % (response.status,
self.code,
body["message"])
except Exception:
msg = 'Expected %s, got (%d, %s)' % (expected_responses,
response.status,
self.body)
super(UnexpectedHTTPResponseError, self).__init__(msg)
def load_from_sdb(self, domain_name, item_name):
from fcu_boto.compat import json
sdb = fcu_boto.connect_sdb()
domain = sdb.lookup(domain_name)
item = domain.get_item(item_name)
for section in item.keys():
if not self.has_section(section):
self.add_section(section)
d = json.loads(item[section])
for attr_name in d.keys():
attr_value = d[attr_name]
if attr_value is None:
attr_value = 'None'
if isinstance(attr_value, bool):
self.setbool(section, attr_name, attr_value)
else:
self.set(section, attr_name, attr_value)
def __call__(self, query):
"""Make a call to CloudSearch
:type query: :class:`fcu_boto.cloudsearch.search.Query`
:param query: A group of search criteria
:rtype: :class:`fcu_boto.cloudsearch.search.SearchResults`
:return: search results
"""
url = "http://%s/2011-02-01/search" % (self.endpoint)
params = query.to_params()
r = requests.get(url, params=params)
body = r.content.decode('utf-8')
try:
data = json.loads(body)
except ValueError as e:
if r.status_code == 403:
msg = ''
import re
g = re.search('<html><body><h1>403 Forbidden</h1>([^<]+)<',
body)
try:
msg = ': %s' % (g.groups()[0].strip())
except AttributeError:
pass
raise SearchServiceException(
'Authentication error from Amazon%s' % msg)
raise SearchServiceException(
"Got non-json response from Amazon. %s" % body, query)
if 'messages' in data and 'error' in data:
for m in data['messages']:
if m['severity'] == 'fatal':
raise SearchServiceException(
"Error processing search %s "
"=> %s" % (params, m['message']), query)
elif 'error' in data:
raise SearchServiceException(
"Unknown error processing search %s" % json.dumps(data), query)
data['query'] = query
data['search_service'] = self
return SearchResults(**data)
def _retry_handler(self, response, i, next_sleep):
status = None
if response.status == 400:
response_body = response.read().decode('utf-8')
fcu_boto.log.debug(response_body)
data = json.loads(response_body)
if self.ThruputError in data.get('__type'):
self.throughput_exceeded_events += 1
msg = "%s, retry attempt %s" % (self.ThruputError, i)
next_sleep = self._exponential_time(i)
i += 1
status = (msg, i, next_sleep)
if i == self.NumberRetries:
# If this was our last retry attempt, raise
# a specific error saying that the throughput
# was exceeded.
raise dynamodb_exceptions.DynamoDBThroughputExceededError(
response.status, response.reason, data)
elif self.SessionExpiredError in data.get('__type'):
msg = 'Renewing Session Token'
self._get_session_token()
status = (msg, i + self.num_retries - 1, 0)
elif self.ConditionalCheckFailedError in data.get('__type'):
raise dynamodb_exceptions.DynamoDBConditionalCheckFailedError(
response.status, response.reason, data)
elif self.ValidationError in data.get('__type'):
raise dynamodb_exceptions.DynamoDBValidationError(
response.status, response.reason, data)
else:
raise self.ResponseError(response.status, response.reason,
data)
expected_crc32 = response.getheader('x-amz-crc32')
if self._validate_checksums and expected_crc32 is not None:
fcu_boto.log.debug('Validating crc32 checksum for body: %s',
response.read().decode('utf-8'))
actual_crc32 = crc32(response.read()) & 0xffffffff
expected_crc32 = int(expected_crc32)
if actual_crc32 != expected_crc32:
msg = ("The calculated checksum %s did not match the expected "
"checksum %s" % (actual_crc32, expected_crc32))
status = (msg, i + 1, self._exponential_time(i))
return status
def make_request(self,
verb,
resource,
headers=None,
data='',
expected_status=None,
params=None):
if headers is None:
headers = {}
response = AWSAuthConnection.make_request(self,
verb,
resource,
headers=headers,
data=data,
params=params)
body = json.loads(response.read().decode('utf-8'))
if response.status == expected_status:
return body
else:
raise JSONResponseError(response.status, response.reason, body)
def make_request(self,
verb,
resource,
headers=None,
data='',
expected_status=None,
params=None):
if headers is None:
headers = {}
response = AWSAuthConnection.make_request(self,
verb,
resource,
headers=headers,
data=data,
params=params)
body = json.loads(response.read().decode('utf-8'))
if response.status == expected_status:
return body
else:
error_type = response.getheader('x-amzn-ErrorType').split(':')[0]
error_class = self._faults.get(error_type, self.ResponseError)
raise error_class(response.status, response.reason, body)
def __init__(self, response, doc_service, sdf):
self.response = response
self.doc_service = doc_service
self.sdf = sdf
_body = response.content.decode('utf-8')
try:
self.content = json.loads(_body)
except:
fcu_boto.log.error(
'Error indexing documents.\nResponse Content:\n{0}\n\n'
'SDF:\n{1}'.format(_body, self.sdf))
raise fcu_boto.exception.BotoServerError(self.response.status_code,
'',
body=_body)
self.status = self.content['status']
if self.status == 'error':
self.errors = [
e.get('message') for e in self.content.get('errors', [])
]
for e in self.errors:
if "Illegal Unicode character" in e:
raise EncodingError(
"Illegal Unicode character in document")
elif e == "The Content-Length is too long":
raise ContentTooLongError("Content was too long")
if 'adds' not in self.content or 'deletes' not in self.content:
raise SearchServiceException("Error indexing documents"
" => %s" %
self.content.get('message', ''))
else:
self.errors = []
self.adds = self.content['adds']
self.deletes = self.content['deletes']
self._check_num_ops('add', self.adds)
self._check_num_ops('delete', self.deletes)
def get_instance_identity(version='latest',
url='http://169.254.169.254',
timeout=None,
num_retries=5):
"""
Returns the instance identity as a nested Python dictionary.
"""
iid = {}
base_url = _build_instance_metadata_url(url, version,
'dynamic/instance-identity/')
try:
data = retry_url(base_url, num_retries=num_retries, timeout=timeout)
fields = data.split('\n')
for field in fields:
val = retry_url(base_url + '/' + field + '/',
num_retries=num_retries,
timeout=timeout)
if val[0] == '{':
val = json.loads(val)
if field:
iid[field] = val
return iid
except urllib.error.URLError:
return None
def __getitem__(self, key):
if key not in self:
# allow dict to throw the KeyError
return super(LazyLoadMetadata, self).__getitem__(key)
# already loaded
val = super(LazyLoadMetadata, self).__getitem__(key)
if val is not None:
return val
if key in self._leaves:
resource = self._leaves[key]
last_exception = None
for i in range(0, self._num_retries):
try:
val = fcu_boto.utils.retry_url(
self._url + urllib.parse.quote(resource, safe="/:"),
num_retries=self._num_retries,
timeout=self._timeout)
if val and val[0] == '{':
val = json.loads(val)
break
else:
p = val.find('\n')
if p > 0:
val = val.split('\n')
break
except JSONDecodeError as e:
fcu_boto.log.debug("encountered '%s' exception: %s" %
(e.__class__.__name__, e))
fcu_boto.log.debug('corrupted JSON data found: %s' % val)
last_exception = e
except Exception as e:
fcu_boto.log.debug("encountered unretryable" +
" '%s' exception, re-raising" %
(e.__class__.__name__))
last_exception = e
raise
fcu_boto.log.error("Caught exception reading meta data" +
" for the '%s' try" % (i + 1))
if i + 1 != self._num_retries:
next_sleep = min(
random.random() * 2**i,
fcu_boto.config.get('Boto', 'max_retry_delay', 60))
time.sleep(next_sleep)
else:
fcu_boto.log.error('Unable to read meta data, giving up')
fcu_boto.log.error(
"encountered '%s' exception: %s" %
(last_exception.__class__.__name__, last_exception))
raise last_exception
self[key] = val
elif key in self._dicts:
self[key] = LazyLoadMetadata(self._url + key + '/',
self._num_retries)
return super(LazyLoadMetadata, self).__getitem__(key)
def _update_options(self, options):
if options:
self.update(json.loads(options))