def __init__(self, accessKey, secretKey, endpoint, result_format, signature_version, api_version): ''' Constructor ''' self._accessKey = accessKey self._secretKey = secretKey self._endpoint = endpoint self._result_format = result_format self._signature_version = signature_version self._api_version = api_version self._signer = AWSSignature(accessKey, secretKey, endpoint)
class AWSQueryClient(object): ''' Client implementing AWS/Query protocol ''' _default_opener = urllib2.build_opener(CaValidationHttpsHandler()) # for handler in _default_opener.handlers: # if not isinstance(handler, urllib2.UnknownHandler): # handler._debuglevel = 1 def __init__(self, accessKey, secretKey, endpoint, result_format, signature_version, api_version): ''' Constructor ''' self._accessKey = accessKey self._secretKey = secretKey self._endpoint = endpoint self._result_format = result_format self._signature_version = signature_version self._api_version = api_version self._signer = AWSSignature(accessKey, secretKey, endpoint) def call(self, params, format_ = None, method = HTTP_POST): if format_ is None: format_ = self._result_format if method == HTTP_GET: url = self._construct_get_url(params.get_dict()) headers={'Accept' : 'application/' + format_, 'Accept-Charset' : ServiceDefault.CHAR_CODEC, 'User-Agent' : ServiceDefault.USER_AGENT} result = self._request_with_retry(HTTP_GET, url, headers) elif method == HTTP_POST: url = self._endpoint if self._endpoint.endswith(u'/') else self._endpoint + u'/' headers = {'Accept' : 'application/' + format_, 'Accept-Charset' : ServiceDefault.CHAR_CODEC, 'User-Agent' : ServiceDefault.USER_AGENT, 'Content-Type' : 'application/x-www-form-urlencoded', 'Expect' : '100-continue'} post_data = self._construct_post_data(params.get_dict()) result = self._request_with_retry(HTTP_POST, url, headers, post_data) return result def _construct_get_url(self, inParams, verb=HTTP_GET): ''' Construct AWS Query Get url ''' params = dict(inParams) params[u'SignatureVersion'] = self._signature_version params[u'Version'] = self._api_version params[u'AWSAccessKeyId'] = self._accessKey params[u'Timestamp'] = datetime.utcnow().isoformat() params[u'SignatureMethod'] = u'HmacSHA256' params[u'ContentType'] = u'JSON' params[u'Signature'] = self._signer.sign(verb, params) host = self._endpoint if self._endpoint.endswith(u'/') else self._endpoint + u'/' return misc.to_bytes(host) + '?' + '&'.join\ (urllib.quote(misc.to_bytes(k)) + '=' + urllib.quote(misc.to_bytes(v)) \ for k, v in params.iteritems()) def _construct_post_data(self, inParams, verb=HTTP_POST): data = dict(inParams) data[u'SignatureVersion'] = self._signature_version data[u'Version'] = self._api_version data[u'AWSAccessKeyId'] = self._accessKey data[u'Timestamp'] = datetime.utcnow().isoformat() data[u'SignatureMethod'] = u'HmacSHA256' data[u'ContentType'] = u'JSON' data[u'Signature'] = self._signer.sign(verb, data) return misc.to_bytes('&'.join(urllib.quote(misc.to_bytes(k)) + '=' \ + urllib.quote(misc.to_bytes(v)) \ for k, v in data.iteritems())) def _urlopen_withretry(self, request_or_url, max_tries = 5, http_error_extractor = _extractAwsErrorMessage, opener = _default_opener): """ Exponentially retries up to max_tries to open request_or_url. Raises an IOError on failure http_error_extractor is a function that takes a urllib2.HTTPError and returns a tuple of (AWS error code, message) """ durations = _exponential_backoff(max_tries) for index, length in enumerate(durations): if length > 0: log.debug(u'Sleeping for %f seconds before retrying', length) time.sleep(length) try: return opener.open(request_or_url) except urllib2.HTTPError as ex: http_code = ex.code aws_code, message = http_error_extractor(ex) if http_code < 500 and aws_code != AwsErrorCode.Throttling: raise AwsServiceException(message, aws_code, http_code) elif AwsErrorCode.Throttling == aws_code or http_code == 503: _extend_backoff(durations) if index + 1 < len(durations): prompt.info(u'Error {0}:{1}. Wait {2} seconds before retry.'.\ format(aws_code, message, durations[index + 1])) log.error(message + u' ' + aws_code) last_message = message except urllib2.URLError as url_ex: log.error(u'URLError: %s', url_ex.reason) raise else: raise AwsServiceException(last_message, aws_code, http_code) def _request_with_retry(self, verb, url, headers, data, max_tries = 5): durations = _exponential_backoff(max_tries) for index, length in enumerate(durations): if length > 0: log.debug(u'Sleeping for %f seconds before retrying', length) time.sleep(length) try: if verb == HTTP_GET: response = requests.get(url, headers = headers, verify=True) elif verb == HTTP_POST: response = requests.post(url, data = data, headers = headers, verify=True) else: raise AttributeError(u'Not supported HTTP action "{0}".'.format(verb)) # check status code if response.status_code != 200: http_code = response.status_code try: aws_code = response.json[u'Error'][u'Code'] message = response.json[u'Error'][u'Message'] except TypeError as ex: raise AttributeError(u'HTTP {0}: {1}'.format(http_code, response.text)) if http_code < 500 and aws_code != AwsErrorCode.Throttling: raise AwsServiceException(message, aws_code, http_code) elif AwsErrorCode.Throttling == aws_code or http_code == 503: prompt.info(u'Request is throttled.') _extend_backoff(durations) if index + 1 < len(durations): prompt.info(u'Error {0}:{1}. Wait {2} seconds before retry.'.\ format(aws_code, message, durations[index + 1])) log.error(message + u' ' + aws_code) last_message = message else: if response.json is None: raise ValueError(u'Cannot parse response form {0}.'.format(response.url)) return response except requests.exceptions.SSLError as ex: log.error(u'SSL Error: %s', ex) raise except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError, requests.exceptions.Timeout) as ex: log.error(ex) last_message = ex else: if aws_code is None: aws_code = u'' if http_code is None: http_code = u'' raise AwsServiceException(last_message, aws_code, http_code)
class AWSQueryClient(object): ''' Client implementing AWS/Query protocol ''' _default_opener = urllib.request.build_opener(CaValidationHttpsHandler()) # for handler in _default_opener.handlers: # if not isinstance(handler, urllib2.UnknownHandler): # handler._debuglevel = 1 def __init__(self, accessKey, secretKey, endpoint, result_format, signature_version, api_version): ''' Constructor ''' self._accessKey = accessKey self._secretKey = secretKey self._endpoint = endpoint self._result_format = result_format self._signature_version = signature_version self._api_version = api_version self._signer = AWSSignature(accessKey, secretKey, endpoint) def call(self, params, format_ = None, method = HTTP_POST): if format_ is None: format_ = self._result_format if method == HTTP_GET: url = self._construct_get_url(params.get_dict()) headers={'Accept' : 'application/' + format_, 'Accept-Charset' : ServiceDefault.CHAR_CODEC, 'User-Agent' : ServiceDefault.USER_AGENT} result = self._request_with_retry(HTTP_GET, url, headers) elif method == HTTP_POST: url = self._endpoint if self._endpoint.endswith('/') else self._endpoint + '/' headers = {'Accept' : 'application/' + format_, 'Accept-Charset' : ServiceDefault.CHAR_CODEC, 'User-Agent' : ServiceDefault.USER_AGENT, 'Content-Type' : 'application/x-www-form-urlencoded', 'Expect' : '100-continue'} post_data = self._construct_post_data(params.get_dict()) result = self._request_with_retry(HTTP_POST, url, headers, post_data) return result def _construct_get_url(self, inParams, verb=HTTP_GET): ''' Construct AWS Query Get url ''' params = dict(inParams) params['SignatureVersion'] = self._signature_version params['Version'] = self._api_version params['AWSAccessKeyId'] = self._accessKey params['Timestamp'] = datetime.utcnow().isoformat() params['SignatureMethod'] = 'HmacSHA256' params['ContentType'] = 'JSON' params['Signature'] = self._signer.sign(verb, params) host = self._endpoint if self._endpoint.endswith('/') else self._endpoint + '/' return misc.to_bytes(host) + '?' + '&'.join\ (urllib.parse.quote(misc.to_bytes(k)) + '=' + urllib.parse.quote(misc.to_bytes(v)) \ for k, v in params.items()) def _construct_post_data(self, inParams, verb=HTTP_POST): data = dict(inParams) data['SignatureVersion'] = self._signature_version data['Version'] = self._api_version data['AWSAccessKeyId'] = self._accessKey data['Timestamp'] = datetime.utcnow().isoformat() data['SignatureMethod'] = 'HmacSHA256' data['ContentType'] = 'JSON' data['Signature'] = self._signer.sign(verb, data) return misc.to_bytes('&'.join(urllib.parse.quote(misc.to_bytes(k)) + '=' \ + urllib.parse.quote(misc.to_bytes(v)) \ for k, v in data.items())) def _urlopen_withretry(self, request_or_url, max_tries = 5, http_error_extractor = _extractAwsErrorMessage, opener = _default_opener): """ Exponentially retries up to max_tries to open request_or_url. Raises an IOError on failure http_error_extractor is a function that takes a urllib2.HTTPError and returns a tuple of (AWS error code, message) """ durations = _exponential_backoff(max_tries) for index, length in enumerate(durations): if length > 0: log.debug('Sleeping for %f seconds before retrying', length) time.sleep(length) try: return opener.open(request_or_url) except urllib.error.HTTPError as ex: http_code = ex.code aws_code, message = http_error_extractor(ex) if http_code < 500 and aws_code != AwsErrorCode.Throttling: raise AwsServiceException(message, aws_code, http_code) elif AwsErrorCode.Throttling == aws_code or http_code == 503: _extend_backoff(durations) if index + 1 < len(durations): prompt.info('Error {0}:{1}. Wait {2} seconds before retry.'.\ format(aws_code, message, durations[index + 1])) log.error(message + ' ' + aws_code) last_message = message except urllib.error.URLError as url_ex: log.error('URLError: %s', url_ex.reason) raise else: raise AwsServiceException(last_message, aws_code, http_code) def _request_with_retry(self, verb, url, headers, data, max_tries = 5): durations = _exponential_backoff(max_tries) for index, length in enumerate(durations): if length > 0: log.debug('Sleeping for %f seconds before retrying', length) time.sleep(length) try: if verb == HTTP_GET: response = requests.get(url, headers = headers, verify=True) elif verb == HTTP_POST: response = requests.post(url, data = data, headers = headers, verify=True) else: raise AttributeError('Not supported HTTP action "{0}".'.format(verb)) # check status code if response.status_code != 200: http_code = response.status_code try: aws_code = response.json['Error']['Code'] message = response.json['Error']['Message'] except TypeError as ex: raise AttributeError('HTTP {0}: {1}'.format(http_code, response.text)) if http_code < 500 and aws_code != AwsErrorCode.Throttling: raise AwsServiceException(message, aws_code, http_code) elif AwsErrorCode.Throttling == aws_code or http_code == 503: prompt.info('Request is throttled.') _extend_backoff(durations) if index + 1 < len(durations): prompt.info('Error {0}:{1}. Wait {2} seconds before retry.'.\ format(aws_code, message, durations[index + 1])) log.error(message + ' ' + aws_code) last_message = message else: if response.json is None: raise ValueError('Cannot parse response form {0}.'.format(response.url)) return response except requests.exceptions.SSLError as ex: log.error('SSL Error: %s', ex) raise except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError, requests.exceptions.Timeout) as ex: log.error(ex) last_message = ex else: if aws_code is None: aws_code = '' if http_code is None: http_code = '' raise AwsServiceException(last_message, aws_code, http_code)
class AWSQueryClient(object): ''' Client implementing AWS/Query protocol ''' def __init__(self, accessKey, secretKey, endpoint, region, service_name, result_format, signature_version, api_version): ''' Constructor ''' self._accessKey = accessKey self._secretKey = secretKey self._endpoint = endpoint self._result_format = result_format self._signature_version = signature_version self._api_version = api_version self._signer = AWSSignature(accessKey, secretKey, endpoint, region, service_name, api_version, signature_version) def call(self, params, format_ = None, method = HTTP_POST): if format_ is None: format_ = self._result_format if method == HTTP_GET: raw_headers={'Accept' : 'application/' + format_, 'Accept-Charset' : ServiceDefault.CHAR_CODEC, 'User-Agent' : ServiceDefault.USER_AGENT} url, headers = self._signer.construct_get_url(params.get_dict(), raw_headers) result = self._request_with_retry(HTTP_GET, url, headers) elif method == HTTP_POST: url = self._endpoint if self._endpoint.endswith('/') else self._endpoint + '/' raw_headers = {'Accept' : 'application/' + format_, # 'Accept-Charset' : ServiceDefault.CHAR_CODEC, 'User-Agent' : ServiceDefault.USER_AGENT, 'Content-Type' : 'application/x-www-form-urlencoded', 'Expect' : '100-continue'} post_data, headers = self._signer.construct_post_data(params.get_dict(), raw_headers) result = self._request_with_retry(HTTP_POST, url, headers, post_data) return result def _request_with_retry(self, verb, url, headers, data, max_tries = 5): aws_code = None http_code = None durations = _exponential_backoff(max_tries) for index, length in enumerate(durations): if length > 0: log.debug('Sleeping for %f seconds before retrying', length) time.sleep(length) try: if verb == HTTP_GET: response = requests.get(url, headers = headers, verify=True) elif verb == HTTP_POST: response = requests.post(url, data = data, headers = headers, verify=True) else: raise AttributeError('Not supported HTTP action "{0}".'.format(verb)) # check status code if response.status_code != 200: http_code = response.status_code try: aws_code = response.json()['Error']['Code'] message = response.json()['Error']['Message'] except TypeError as ex: raise AttributeError('HTTP {0}: {1}'.format(http_code, response.text)) if http_code < 500 and aws_code != AwsErrorCode.Throttling: raise AwsServiceException(message, aws_code, http_code) elif AwsErrorCode.Throttling == aws_code or http_code == 503: prompt.info('Request is throttled.') _extend_backoff(durations) if index + 1 < len(durations): prompt.info('Error {0}:{1}. Wait {2} seconds before retry.'.\ format(aws_code, message, durations[index + 1])) log.error(message + ' ' + aws_code) last_message = message else: if response.json() is None: raise ValueError('Cannot parse response form {0}.'.format(response.url)) return response except requests.exceptions.SSLError as ex: log.error('SSL Error: %s', ex) raise except (requests.exceptions.HTTPError, requests.exceptions.ConnectionError, requests.exceptions.Timeout) as ex: log.error(ex) last_message = ex else: if aws_code is None: aws_code = '' if http_code is None: http_code = '' raise AwsServiceException(last_message, aws_code, http_code)
class AWSQueryClient(object): """ Client implementing AWS/Query protocol """ _default_opener = urllib.request.build_opener(CaValidationHttpsHandler()) # for handler in _default_opener.handlers: # if not isinstance(handler, urllib2.UnknownHandler): # handler._debuglevel = 1 def __init__(self, accessKey, secretKey, endpoint, result_format, signature_version, api_version): """ Constructor """ self._accessKey = accessKey self._secretKey = secretKey self._endpoint = endpoint self._result_format = result_format self._signature_version = signature_version self._api_version = api_version self._signer = AWSSignature(accessKey, secretKey, endpoint) def call(self, params, format_=None, method=HTTP_POST): if format_ is None: format_ = self._result_format if method == HTTP_GET: url = self._construct_get_url(params.get_dict()) headers = { "Accept": "application/" + format_, "Accept-Charset": ServiceDefault.CHAR_CODEC, "User-Agent": ServiceDefault.USER_AGENT, } result = self._request_with_retry(HTTP_GET, url, headers) elif method == HTTP_POST: url = self._endpoint if self._endpoint.endswith("/") else self._endpoint + "/" headers = { "Accept": "application/" + format_, "Accept-Charset": ServiceDefault.CHAR_CODEC, "User-Agent": ServiceDefault.USER_AGENT, "Content-Type": "application/x-www-form-urlencoded", "Expect": "100-continue", } post_data = self._construct_post_data(params.get_dict()) result = self._request_with_retry(HTTP_POST, url, headers, post_data) return result def _construct_get_url(self, inParams, verb=HTTP_GET): """ Construct AWS Query Get url """ params = dict(inParams) params["SignatureVersion"] = self._signature_version params["Version"] = self._api_version params["AWSAccessKeyId"] = self._accessKey params["Timestamp"] = datetime.utcnow().isoformat() params["SignatureMethod"] = "HmacSHA256" params["ContentType"] = "JSON" params["Signature"] = self._signer.sign(verb, params) host = self._endpoint if self._endpoint.endswith("/") else self._endpoint + "/" return ( misc.to_bytes(host) + "?" + "&".join( urllib.parse.quote(misc.to_bytes(k)) + "=" + urllib.parse.quote(misc.to_bytes(v)) for k, v in params.items() ) ) def _construct_post_data(self, inParams, verb=HTTP_POST): data = dict(inParams) data["SignatureVersion"] = self._signature_version data["Version"] = self._api_version data["AWSAccessKeyId"] = self._accessKey data["Timestamp"] = datetime.utcnow().isoformat() data["SignatureMethod"] = "HmacSHA256" data["ContentType"] = "JSON" data["Signature"] = self._signer.sign(verb, data) return misc.to_bytes( "&".join( urllib.parse.quote(misc.to_bytes(k)) + "=" + urllib.parse.quote(misc.to_bytes(v)) for k, v in data.items() ) ) def _urlopen_withretry( self, request_or_url, max_tries=5, http_error_extractor=_extractAwsErrorMessage, opener=_default_opener ): """ Exponentially retries up to max_tries to open request_or_url. Raises an IOError on failure http_error_extractor is a function that takes a urllib2.HTTPError and returns a tuple of (AWS error code, message) """ durations = _exponential_backoff(max_tries) for index, length in enumerate(durations): if length > 0: log.debug("Sleeping for %f seconds before retrying", length) time.sleep(length) try: return opener.open(request_or_url) except urllib.error.HTTPError as ex: http_code = ex.code aws_code, message = http_error_extractor(ex) if http_code < 500 and aws_code != AwsErrorCode.Throttling: raise AwsServiceException(message, aws_code, http_code) elif AwsErrorCode.Throttling == aws_code or http_code == 503: _extend_backoff(durations) if index + 1 < len(durations): prompt.info( "Error {0}:{1}. Wait {2} seconds before retry.".format(aws_code, message, durations[index + 1]) ) log.error(message + " " + aws_code) last_message = message except urllib.error.URLError as url_ex: log.error("URLError: %s", url_ex.reason) raise else: raise AwsServiceException(last_message, aws_code, http_code) def _request_with_retry(self, verb, url, headers, data, max_tries=5): durations = _exponential_backoff(max_tries) for index, length in enumerate(durations): if length > 0: log.debug("Sleeping for %f seconds before retrying", length) time.sleep(length) try: if verb == HTTP_GET: response = requests.get(url, headers=headers, verify=True) elif verb == HTTP_POST: response = requests.post(url, data=data, headers=headers, verify=True) else: raise AttributeError('Not supported HTTP action "{0}".'.format(verb)) # check status code if response.status_code != 200: http_code = response.status_code aws_code = response.json["Error"]["Code"] message = response.json["Error"]["Message"] if http_code < 500 and aws_code != AwsErrorCode.Throttling: raise AwsServiceException(message, aws_code, http_code) elif AwsErrorCode.Throttling == aws_code or http_code == 503: _extend_backoff(durations) if index + 1 < len(durations): prompt.info( "Error {0}:{1}. Wait {2} seconds before retry.".format( aws_code, message, durations[index + 1] ) ) log.error(message + " " + aws_code) last_message = message else: return response except requests.exceptions.SSLError as ex: log.error("SSL Error: %s", ex) raise except ( requests.exceptions.HTTPError, requests.exceptions.ConnectionError, requests.exceptions.Timeout, ) as ex: last_message = ex else: if aws_code is None: aws_code = "" if http_code is None: http_code = "" raise AwsServiceException(last_message, aws_code, http_code)