class AsyncDynamoDB(AWSAuthConnection): """The main class for asynchronous connections to DynamoDB. The user should maintain one instance of this class (though more than one is ok), parametrized with the user's access key and secret key. Make calls with make_request or the helper methods, and AsyncDynamoDB will maintain session tokens in the background. """ DefaultHost = 'dynamodb.us-east-1.amazonaws.com' """The default DynamoDB API endpoint to connect to.""" ServiceName = 'DynamoDB' """The name of the Service""" Version = '20111205' """DynamoDB API version.""" ExpiredSessionError = 'ExpiredTokenException' """The error response returned when session token has expired""" UnrecognizedClientException = 'UnrecognizedClientException' """Another error response that is possible with a bad session token""" ProvisionedThroughputExceededException = 'ProvisionedThroughputExceededException' """Provisioned throughput for requests to table exceeded.""" LimitExceededException = 'LimitExceededException' """Limit for subscriber requests exceeded.""" ConditionalCheckFailedException = 'ConditionalCheckFailedException' def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, host=None, debug=0, session_token=None, authenticate_requests=True, validate_cert=True, max_sts_attempts=3): if not host: host = self.DefaultHost self.validate_cert = validate_cert self.authenticate_requests = authenticate_requests AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_access_key, is_secure, port, proxy, proxy_port, debug=debug, security_token=session_token) self.pending_requests = deque() self.sts = AsyncAwsSts(aws_access_key_id, aws_secret_access_key) assert (isinstance(max_sts_attempts, int) and max_sts_attempts >= 0) self.max_sts_attempts = max_sts_attempts def _init_session_token_cb(self, error=None): if error: logging.warn("Unable to get session token: %s" % error) def _required_auth_capability(self): return ['hmac-v3-http'] def _update_session_token(self, callback, attempts=0, bypass_lock=False): """Begins the logic to get a new session token. Performs checks to ensure that only one request goes out at a time and that backoff is respected, so it can be called repeatedly with no ill effects. Set bypass_lock to True to override this behavior. """ if self.provider.security_token == PENDING_SESSION_TOKEN_UPDATE and not bypass_lock: return self.provider.security_token = PENDING_SESSION_TOKEN_UPDATE # invalidate the current security token return self.sts.get_session_token( functools.partial(self._update_session_token_cb, callback=callback, attempts=attempts)) def _update_session_token_cb(self, creds, provider='aws', callback=None, error=None, attempts=0): """Callback to use with `async_aws_sts`. The 'provider' arg is a bit misleading, it is a relic from boto and should probably be left to its default. This will take the new Credentials obj from `async_aws_sts.get_session_token()` and use it to update self.provider, and then will clear the deque of pending requests. A callback is optional. If provided, it must be callable without any arguments. """ def raise_error(): # get out of locked state self.provider.security_token = None if callable(callback): return callback(error=error) else: logging.error(error) raise error if error: if isinstance(error, InvalidClientTokenIdError): # no need to retry if error is due to bad tokens raise_error() else: if attempts > self.max_sts_attempts: raise_error() else: seconds_to_wait = (0.1 * (2**attempts)) logging.warning( "Got error[ %s ] getting session token, retrying in %.02f seconds" % (error, seconds_to_wait)) ioloop.IOLoop.current().add_timeout( time.time() + seconds_to_wait, functools.partial(self._update_session_token, attempts=attempts + 1, callback=callback, bypass_lock=True)) return else: self.provider = Provider(provider, creds.access_key, creds.secret_key, creds.session_token) # force the correct auth, with the new provider self._auth_handler = HmacAuthV3HTTPHandler(self.host, None, self.provider) while self.pending_requests: request = self.pending_requests.pop() request() if callable(callback): return callback() def make_request(self, action, body='', callback=None, object_hook=None): """Make an asynchronous HTTP request to DynamoDB. Callback should operate on the decoded json response (with object hook applied, of course). It should also accept an error argument, which will be a boto.exception.DynamoDBResponseError. If there is not a valid session token, this method will ensure that a new one is fetched and cache the request when it is retrieved. """ this_request = functools.partial(self.make_request, action=action, body=body, callback=callback, object_hook=object_hook) if self.authenticate_requests and self.provider.security_token in [ None, PENDING_SESSION_TOKEN_UPDATE ]: # we will not be able to complete this request because we do not have a valid session token. # queue it and try to get a new one. _update_session_token will ensure that only one request # for a session token goes out at a time self.pending_requests.appendleft(this_request) def cb_for_update(error=None): # create a callback to handle errors getting session token # callback here is assumed to take a json response, and an instance of DynamoDBResponseError if error: raise DynamoDBResponseError(error.status, error.reason, body={'message': error.body}) else: return self._update_session_token(cb_for_update) return headers = { 'X-Amz-Target': '%s_%s.%s' % (self.ServiceName, self.Version, action), 'Content-Type': 'application/x-amz-json-1.0', 'Content-Length': str(len(body)) } request = HTTPRequest('https://%s' % self.host, method='POST', headers=headers, body=body, validate_cert=self.validate_cert) request.path = '/' # Important! set the path variable for signing by boto. '/' is the path for all dynamodb requests if self.authenticate_requests: self._auth_handler.add_auth( request) # add signature to headers of the request http_client = httpclient.AsyncHTTPClient() http_client.fetch( request, functools.partial(self._finish_make_request, callback=callback, orig_request=this_request, token_used=self.provider.security_token, object_hook=object_hook)) def _finish_make_request(self, response, callback, orig_request, token_used, object_hook=None): """Check for errors and decode the json response (in the tornado response body), then pass on to orig callback. This method also contains some of the logic to handle reacquiring session tokens. """ if not response.body: assert response.error, 'How can there be no response body and no error? Response: %s' % response raise DynamoDBResponseError(response.error.code, response.error.message, None) json_response = json.loads(response.body, object_hook=object_hook) if response.error: aws_error_type = None try: # The error code should be in the __type field of the json response, and should be a string # in the form 'namespace.version#errorcode'. If the field doesn't exist or is in some other form, # just treat this as an unknown error type. aws_error_type = json_response.get('__type').split('#')[1] except: aws_error_type = None if aws_error_type == self.ExpiredSessionError or aws_error_type == self.UnrecognizedClientException: if self.provider.security_token == token_used: # The token that we used has expired, wipe it out. self.provider.security_token = None # make_request will handle logic to get a new token if needed, and queue until it is fetched return orig_request() if aws_error_type == AsyncDynamoDB.ProvisionedThroughputExceededException: raise DBProvisioningExceededError(json_response['message']) if aws_error_type == AsyncDynamoDB.LimitExceededException: raise DBLimitExceededError(json_response['message']) if aws_error_type == AsyncDynamoDB.ConditionalCheckFailedException: raise DBConditionalCheckFailedError(json_response['message']) raise DynamoDBResponseError(response.error.code, response.error.message, json_response) return callback(json_response)
class AsyncDynamoDB(AWSAuthConnection): """ The main class for asynchronous connections to DynamoDB. The user should maintain one instance of this class (though more than one is ok), parametrized with the user's access key and secret key. Make calls with make_request or the helper methods, and AsyncDynamoDB will maintain session tokens in the background. As in Boto Layer1: "This is the lowest-level interface to DynamoDB. Methods at this layer map directly to API requests and parameters to the methods are either simple, scalar values or they are the Python equivalent of the JSON input as defined in the DynamoDB Developer's Guide. All responses are direct decoding of the JSON response bodies to Python data structures via the json or simplejson modules." """ DefaultHost = 'dynamodb.us-east-1.amazonaws.com' """The default DynamoDB API endpoint to connect to.""" ServiceName = 'DynamoDB' """The name of the Service""" Version = '20111205' """DynamoDB API version.""" ThruputError = "ProvisionedThroughputExceededException" """The error response returned when provisioned throughput is exceeded""" ExpiredSessionError = 'com.amazon.coral.service#ExpiredTokenException' """The error response returned when session token has expired""" UnrecognizedClientException = 'com.amazon.coral.service#UnrecognizedClientException' '''Another error response that is possible with a bad session token''' def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, host=None, debug=0, session_token=None, authenticate_requests=True, validate_cert=True, max_sts_attempts=3): if not host: host = self.DefaultHost self.validate_cert = validate_cert self.authenticate_requests = authenticate_requests AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_access_key, is_secure, port, proxy, proxy_port, debug=debug, security_token=session_token) self.http_client = AsyncHTTPClient() self.pending_requests = deque() self.sts = AsyncAwsSts(aws_access_key_id, aws_secret_access_key) assert (isinstance(max_sts_attempts, int) and max_sts_attempts >= 0) self.max_sts_attempts = max_sts_attempts def _init_session_token_cb(self, error=None): if error: logging.warn("Unable to get session token: %s" % error) def _required_auth_capability(self): return ['hmac-v3-http'] def _update_session_token(self, callback, attempts=0, bypass_lock=False): ''' Begins the logic to get a new session token. Performs checks to ensure that only one request goes out at a time and that backoff is respected, so it can be called repeatedly with no ill effects. Set bypass_lock to True to override this behavior. ''' if self.provider.security_token == PENDING_SESSION_TOKEN_UPDATE and not bypass_lock: return self.provider.security_token = PENDING_SESSION_TOKEN_UPDATE # invalidate the current security token return self.sts.get_session_token( functools.partial(self._update_session_token_cb, callback=callback, attempts=attempts)) def _update_session_token_cb(self, creds, provider='aws', callback=None, error=None, attempts=0): ''' Callback to use with `async_aws_sts`. The 'provider' arg is a bit misleading, it is a relic from boto and should probably be left to its default. This will take the new Credentials obj from `async_aws_sts.get_session_token()` and use it to update self.provider, and then will clear the deque of pending requests. A callback is optional. If provided, it must be callable without any arguments, but also accept an optional error argument that will be an instance of BotoServerError. ''' def raise_error(): # get out of locked state self.provider.security_token = None if callable(callback): return callback(error=error) else: logging.error(error) raise error if error: if isinstance(error, InvalidClientTokenIdError): # no need to retry if error is due to bad tokens raise_error() else: if attempts > self.max_sts_attempts: raise_error() else: seconds_to_wait = (0.1 * (2**attempts)) logging.warning( "Got error[ %s ] getting session token, retrying in %.02f seconds" % (error, seconds_to_wait)) IOLoop.instance().add_timeout( time.time() + seconds_to_wait, functools.partial(self._update_session_token, attempts=attempts + 1, callback=callback, bypass_lock=True)) return else: self.provider = Provider(provider, creds.access_key, creds.secret_key, creds.session_token) # force the correct auth, with the new provider self._auth_handler = HmacAuthV3HTTPHandler(self.host, None, self.provider) while self.pending_requests: request = self.pending_requests.pop() request() if callable(callback): return callback() def make_request(self, action, body='', callback=None, object_hook=None): ''' Make an asynchronous HTTP request to DynamoDB. Callback should operate on the decoded json response (with object hook applied, of course). It should also accept an error argument, which will be a boto.exception.DynamoDBResponseError. If there is not a valid session token, this method will ensure that a new one is fetched and cache the request when it is retrieved. ''' this_request = functools.partial(self.make_request, action=action, body=body, callback=callback, object_hook=object_hook) if self.authenticate_requests and self.provider.security_token in [ None, PENDING_SESSION_TOKEN_UPDATE ]: # we will not be able to complete this request because we do not have a valid session token. # queue it and try to get a new one. _update_session_token will ensure that only one request # for a session token goes out at a time self.pending_requests.appendleft(this_request) def cb_for_update(error=None): # create a callback to handle errors getting session token # callback here is assumed to take a json response, and an instance of DynamoDBResponseError if error: return callback({}, error=DynamoDBResponseError( error.status, error.reason, error.body)) else: return self._update_session_token(cb_for_update) return headers = { 'X-Amz-Target': '%s_%s.%s' % (self.ServiceName, self.Version, action), 'Content-Type': 'application/x-amz-json-1.0', 'Content-Length': str(len(body)) } request = HTTPRequest('https://%s' % self.host, method='POST', headers=headers, body=body, validate_cert=self.validate_cert) request.path = '/' # Important! set the path variable for signing by boto. '/' is the path for all dynamodb requests if self.authenticate_requests: self._auth_handler.add_auth( request) # add signature to headers of the request self.http_client.fetch(request, functools.partial( self._finish_make_request, callback=callback, orig_request=this_request, token_used=self.provider.security_token, object_hook=object_hook)) # bam! def _finish_make_request(self, response, callback, orig_request, token_used, object_hook=None): ''' Check for errors and decode the json response (in the tornado response body), then pass on to orig callback. This method also contains some of the logic to handle reacquiring session tokens. ''' json_response = json.loads(response.body, object_hook=object_hook) if response.error: if any((token_error in json_response.get('__type', []) \ for token_error in (self.ExpiredSessionError, self.UnrecognizedClientException))): if self.provider.security_token == token_used: # the token that we used has expired. wipe it out self.provider.security_token = None return orig_request( ) # make_request will handle logic to get a new token if needed, and queue until it is fetched else: # because some errors are benign, include the response when an error is passed return callback(json_response, error=DynamoDBResponseError( response.error.code, response.error.message, json_response)) return callback(json_response, error=None) def get_item(self, table_name, key, callback, attributes_to_get=None, consistent_read=False, object_hook=None): ''' Return a set of attributes for an item that matches the supplied key. The callback should operate on a dict representing the decoded response from DynamoDB (using the object_hook, if supplied) :type table_name: str :param table_name: The name of the table to delete. :type key: dict :param key: A Python version of the Key data structure defined by DynamoDB. :type attributes_to_get: list :param attributes_to_get: A list of attribute names. If supplied, only the specified attribute names will be returned. Otherwise, all attributes will be returned. :type consistent_read: bool :param consistent_read: If True, a consistent read request is issued. Otherwise, an eventually consistent request is issued. ''' data = {'TableName': table_name, 'Key': key} if attributes_to_get: data['AttributesToGet'] = attributes_to_get if consistent_read: data['ConsistentRead'] = True return self.make_request('GetItem', body=json.dumps(data), callback=callback, object_hook=object_hook) def batch_get_item(self, request_items, callback, object_hook=None): """ Return a set of attributes for a multiple items in multiple tables using their primary keys. The callback should operate on a dict representing the decoded response from DynamoDB (using the object_hook, if supplied) :type request_items: dict :param request_items: A Python version of the RequestItems data structure defined by DynamoDB. """ data = {'RequestItems': request_items} json_input = json.dumps(data) self.make_request('BatchGetItem', json_input, callback, object_hook=object_hook) def put_item(self, table_name, item, callback, expected=None, return_values=None, object_hook=None): ''' Create a new item or replace an old item with a new item (including all attributes). If an item already exists in the specified table with the same primary key, the new item will completely replace the old item. You can perform a conditional put by specifying an expected rule. The callback should operate on a dict representing the decoded response from DynamoDB (using the object_hook, if supplied) :type table_name: str :param table_name: The name of the table to delete. :type item: dict :param item: A Python version of the Item data structure defined by DynamoDB. :type expected: dict :param expected: A Python version of the Expected data structure defined by DynamoDB. :type return_values: str :param return_values: Controls the return of attribute name-value pairs before then were changed. Possible values are: None or 'ALL_OLD'. If 'ALL_OLD' is specified and the item is overwritten, the content of the old item is returned. ''' data = {'TableName': table_name, 'Item': item} if expected: data['Expected'] = expected if return_values: data['ReturnValues'] = return_values json_input = json.dumps(data) return self.make_request('PutItem', json_input, callback=callback, object_hook=object_hook) def query(self, table_name, hash_key_value, callback, range_key_conditions=None, attributes_to_get=None, limit=None, consistent_read=False, scan_index_forward=True, exclusive_start_key=None, object_hook=None): ''' Perform a query of DynamoDB. This version is currently punting and expecting you to provide a full and correct JSON body which is passed as is to DynamoDB. The callback should operate on a dict representing the decoded response from DynamoDB (using the object_hook, if supplied) :type table_name: str :param table_name: The name of the table to delete. :type hash_key_value: dict :param key: A DynamoDB-style HashKeyValue. :type range_key_conditions: dict :param range_key_conditions: A Python version of the RangeKeyConditions data structure. :type attributes_to_get: list :param attributes_to_get: A list of attribute names. If supplied, only the specified attribute names will be returned. Otherwise, all attributes will be returned. :type limit: int :param limit: The maximum number of items to return. :type consistent_read: bool :param consistent_read: If True, a consistent read request is issued. Otherwise, an eventually consistent request is issued. :type scan_index_forward: bool :param scan_index_forward: Specified forward or backward traversal of the index. Default is forward (True). :type exclusive_start_key: list or tuple :param exclusive_start_key: Primary key of the item from which to continue an earlier query. This would be provided as the LastEvaluatedKey in that query. ''' data = {'TableName': table_name, 'HashKeyValue': hash_key_value} if range_key_conditions: data['RangeKeyCondition'] = range_key_conditions if attributes_to_get: data['AttributesToGet'] = attributes_to_get if limit: data['Limit'] = limit if consistent_read: data['ConsistentRead'] = True if scan_index_forward: data['ScanIndexForward'] = True else: data['ScanIndexForward'] = False if exclusive_start_key: data['ExclusiveStartKey'] = exclusive_start_key json_input = json.dumps(data) return self.make_request('Query', body=json_input, callback=callback, object_hook=object_hook)
class AsyncDynamoDB(AWSAuthConnection): """ The main class for asynchronous connections to DynamoDB. The user should maintain one instance of this class (though more than one is ok), parametrized with the user's access key and secret key. Make calls with make_request or the helper methods, and AsyncDynamoDB will maintain session tokens in the background. As in Boto Layer1: "This is the lowest-level interface to DynamoDB. Methods at this layer map directly to API requests and parameters to the methods are either simple, scalar values or they are the Python equivalent of the JSON input as defined in the DynamoDB Developer's Guide. All responses are direct decoding of the JSON response bodies to Python data structures via the json or simplejson modules." """ DefaultHost = 'dynamodb.us-east-1.amazonaws.com' """The default DynamoDB API endpoint to connect to.""" ServiceName = 'DynamoDB' """The name of the Service""" Version = '20120810' """DynamoDB API version.""" ThruputError = "ProvisionedThroughputExceededException" """The error response returned when provisioned throughput is exceeded""" ExpiredSessionError = 'com.amazon.coral.service#ExpiredTokenException' """The error response returned when session token has expired""" UnrecognizedClientException = 'com.amazon.coral.service#UnrecognizedClientException' '''Another error response that is possible with a bad session token''' def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, host=None, debug=0, session_token=None, endpoint=None, authenticate_requests=True, validate_cert=True, max_sts_attempts=3, ioloop=None): if not host: host = self.DefaultHost if endpoint is not None: self.url = endpoint parse_url = urlparse(self.url) self.host = parse_url.hostname self.port = parse_url.port self.protocol = parse_url.scheme else: self.protocol = 'https' if is_secure else 'http' self.host = host self.port = port url = '{0}://{1}'.format(self.protocol, self.host) if self.port: url += ':{}'.format(self.port) self.url = url self.validate_cert = validate_cert self.authenticate_requests = authenticate_requests AWSAuthConnection.__init__(self, self.host, aws_access_key_id, aws_secret_access_key, is_secure, self.port, proxy, proxy_port, debug=debug, security_token=session_token, validate_certs=self.validate_cert) self.ioloop = ioloop or IOLoop.instance() self.http_client = AsyncHTTPClient(io_loop=self.ioloop) self.pending_requests = deque() self.sts = AsyncAwsSts(aws_access_key_id, aws_secret_access_key, is_secure, self.port, proxy, proxy_port) assert (isinstance(max_sts_attempts, int) and max_sts_attempts >= 0) self.max_sts_attempts = max_sts_attempts def _init_session_token_cb(self, error=None): if error: logging.warn("Unable to get session token: %s" % error) def _required_auth_capability(self): return ['hmac-v4'] def _update_session_token(self, callback, attempts=0, bypass_lock=False): ''' Begins the logic to get a new session token. Performs checks to ensure that only one request goes out at a time and that backoff is respected, so it can be called repeatedly with no ill effects. Set bypass_lock to True to override this behavior. ''' if self.provider.security_token == PENDING_SESSION_TOKEN_UPDATE and not bypass_lock: return self.provider.security_token = PENDING_SESSION_TOKEN_UPDATE # invalidate the current security token return self.sts.get_session_token( functools.partial(self._update_session_token_cb, callback=callback, attempts=attempts)) def _update_session_token_cb(self, creds, provider='aws', callback=None, error=None, attempts=0): ''' Callback to use with `async_aws_sts`. The 'provider' arg is a bit misleading, it is a relic from boto and should probably be left to its default. This will take the new Credentials obj from `async_aws_sts.get_session_token()` and use it to update self.provider, and then will clear the deque of pending requests. A callback is optional. If provided, it must be callable without any arguments, but also accept an optional error argument that will be an instance of BotoServerError. ''' def raise_error(): # get out of locked state self.provider.security_token = None if callable(callback): return callback(error=error) else: logging.error(error) raise error if error: if isinstance(error, InvalidClientTokenIdError): # no need to retry if error is due to bad tokens raise_error() else: if attempts > self.max_sts_attempts: raise_error() else: seconds_to_wait = (0.1*(2**attempts)) logging.warning("Got error[ %s ] getting session token, retrying in %.02f seconds" % (error, seconds_to_wait)) self.ioloop.add_timeout(time.time() + seconds_to_wait, functools.partial(self._update_session_token, attempts=attempts+1, callback=callback, bypass_lock=True)) return else: self.provider = Provider(provider, creds.access_key, creds.secret_key, creds.session_token) # force the correct auth, with the new provider self._auth_handler = HmacAuthV4Handler(self.host, None, self.provider) while self.pending_requests: request = self.pending_requests.pop() request() if callable(callback): return callback() def make_request(self, action, body='', callback=None, object_hook=None): ''' Make an asynchronous HTTP request to DynamoDB. Callback should operate on the decoded json response (with object hook applied, of course). It should also accept an error argument, which will be a boto.exception.DynamoDBResponseError. If there is not a valid session token, this method will ensure that a new one is fetched and cache the request when it is retrieved. ''' this_request = functools.partial(self.make_request, action=action, body=body, callback=callback,object_hook=object_hook) if self.authenticate_requests and self.provider.security_token in [None, PENDING_SESSION_TOKEN_UPDATE]: # we will not be able to complete this request because we do not have a valid session token. # queue it and try to get a new one. _update_session_token will ensure that only one request # for a session token goes out at a time self.pending_requests.appendleft(this_request) def cb_for_update(error=None): # create a callback to handle errors getting session token # callback here is assumed to take a json response, and an instance of DynamoDBResponseError if error: return callback({}, error=DynamoDBResponseError(error.status, error.reason, error.body)) else: return self._update_session_token(cb_for_update) return headers = {'X-Amz-Target' : '%s_%s.%s' % (self.ServiceName, self.Version, action), 'Content-Type' : 'application/x-amz-json-1.0', 'Content-Length' : str(len(body))} request = HTTPRequest(self.url, method='POST', headers=headers, body=body, validate_cert=self.validate_cert) request.path = '/' # Important! set the path variable for signing by boto (<2.7). '/' is the path for all dynamodb requests request.auth_path = '/' # Important! set the auth_path variable for signing by boto(>2.7). '/' is the path for all dynamodb requests request.params = {} request.port = self.port request.protocol = self.protocol request.host = self.host if self.authenticate_requests: self._auth_handler.add_auth(request) # add signature to headers of the request self.http_client.fetch(request, functools.partial(self._finish_make_request, callback=callback, orig_request=this_request, token_used=self.provider.security_token, object_hook=object_hook)) # bam! def _finish_make_request(self, response, callback, orig_request, token_used, object_hook=None): ''' Check for errors and decode the json response (in the tornado response body), then pass on to orig callback. This method also contains some of the logic to handle reacquiring session tokens. ''' try: json_response = json.loads(response.body, object_hook=object_hook) except TypeError: json_response = None if json_response and response.error: # Normal error handling where we have a JSON response from AWS. if any((token_error in json_response.get('__type', []) \ for token_error in (self.ExpiredSessionError, self.UnrecognizedClientException))): if self.provider.security_token == token_used: # the token that we used has expired. wipe it out self.provider.security_token = None return orig_request() # make_request will handle logic to get a new token if needed, and queue until it is fetched else: # because some errors are benign, include the response when an error is passed return callback(json_response, error=DynamoDBResponseError(response.error.code, response.error.message, json_response)) if json_response is None: # We didn't get any JSON back, but we also didn't receive an error response. This can't be right. return callback(None, error=DynamoDBResponseError(response.code, response.body)) else: return callback(json_response, error=None) def get_item(self, table_name, key, callback, attributes_to_get=None, consistent_read=False, object_hook=None): ''' Return a set of attributes for an item that matches the supplied key. The callback should operate on a dict representing the decoded response from DynamoDB (using the object_hook, if supplied) :type table_name: str :param table_name: The name of the table to delete. :type key: dict :param key: A Python version of the Key data structure defined by DynamoDB. :type attributes_to_get: list :param attributes_to_get: A list of attribute names. If supplied, only the specified attribute names will be returned. Otherwise, all attributes will be returned. :type consistent_read: bool :param consistent_read: If True, a consistent read request is issued. Otherwise, an eventually consistent request is issued. ''' data = {'TableName': table_name, 'Key': key} if attributes_to_get: data['AttributesToGet'] = attributes_to_get if consistent_read: data['ConsistentRead'] = True return self.make_request('GetItem', body=json.dumps(data), callback=callback, object_hook=object_hook) def batch_get_item(self, request_items, callback): """ Return a set of attributes for a multiple items in multiple tables using their primary keys. The callback should operate on a dict representing the decoded response from DynamoDB (using the object_hook, if supplied) :type request_items: dict :param request_items: A Python version of the RequestItems data structure defined by DynamoDB. """ data = {'RequestItems' : request_items} json_input = json.dumps(data) self.make_request('BatchGetItem', json_input, callback) def put_item(self, table_name, item, callback, expected=None, return_values=None, object_hook=None): ''' Create a new item or replace an old item with a new item (including all attributes). If an item already exists in the specified table with the same primary key, the new item will completely replace the old item. You can perform a conditional put by specifying an expected rule. The callback should operate on a dict representing the decoded response from DynamoDB (using the object_hook, if supplied) :type table_name: str :param table_name: The name of the table to delete. :type item: dict :param item: A Python version of the Item data structure defined by DynamoDB. :type expected: dict :param expected: A Python version of the Expected data structure defined by DynamoDB. :type return_values: str :param return_values: Controls the return of attribute name-value pairs before then were changed. Possible values are: None or 'ALL_OLD'. If 'ALL_OLD' is specified and the item is overwritten, the content of the old item is returned. ''' data = {'TableName' : table_name, 'Item' : item} if expected: data['Expected'] = expected if return_values: data['ReturnValues'] = return_values json_input = json.dumps(data) return self.make_request('PutItem', json_input, callback=callback, object_hook=object_hook) def query(self, table_name, hash_key_value, callback, range_key_conditions=None, attributes_to_get=None, limit=None, consistent_read=False, scan_index_forward=True, exclusive_start_key=None, object_hook=None): ''' Perform a query of DynamoDB. This version is currently punting and expecting you to provide a full and correct JSON body which is passed as is to DynamoDB. The callback should operate on a dict representing the decoded response from DynamoDB (using the object_hook, if supplied) :type table_name: str :param table_name: The name of the table to delete. :type hash_key_value: dict :param key: A DynamoDB-style HashKeyValue. :type range_key_conditions: dict :param range_key_conditions: A Python version of the RangeKeyConditions data structure. :type attributes_to_get: list :param attributes_to_get: A list of attribute names. If supplied, only the specified attribute names will be returned. Otherwise, all attributes will be returned. :type limit: int :param limit: The maximum number of items to return. :type consistent_read: bool :param consistent_read: If True, a consistent read request is issued. Otherwise, an eventually consistent request is issued. :type scan_index_forward: bool :param scan_index_forward: Specified forward or backward traversal of the index. Default is forward (True). :type exclusive_start_key: list or tuple :param exclusive_start_key: Primary key of the item from which to continue an earlier query. This would be provided as the LastEvaluatedKey in that query. ''' data = {'TableName': table_name, 'HashKeyValue': hash_key_value} if range_key_conditions: data['RangeKeyCondition'] = range_key_conditions if attributes_to_get: data['AttributesToGet'] = attributes_to_get if limit: data['Limit'] = limit if consistent_read: data['ConsistentRead'] = True if scan_index_forward: data['ScanIndexForward'] = True else: data['ScanIndexForward'] = False if exclusive_start_key: data['ExclusiveStartKey'] = exclusive_start_key json_input = json.dumps(data) return self.make_request('Query', body=json_input, callback=callback, object_hook=object_hook)
class AsyncDynamoDB(AWSAuthConnection): """The main class for asynchronous connections to DynamoDB. The user should maintain one instance of this class (though more than one is ok), parametrized with the user's access key and secret key. Make calls with make_request or the helper methods, and AsyncDynamoDB will maintain session tokens in the background. """ DefaultHost = 'dynamodb.us-east-1.amazonaws.com' """The default DynamoDB API endpoint to connect to.""" ServiceName = 'DynamoDB' """The name of the Service""" Version = '20111205' """DynamoDB API version.""" ExpiredSessionError = 'ExpiredTokenException' """The error response returned when session token has expired""" UnrecognizedClientException = 'UnrecognizedClientException' """Another error response that is possible with a bad session token""" ProvisionedThroughputExceededException = 'ProvisionedThroughputExceededException' """Provisioned throughput for requests to table exceeded.""" LimitExceededException = 'LimitExceededException' """Limit for subscriber requests exceeded.""" ConditionalCheckFailedException = 'ConditionalCheckFailedException' def __init__(self, aws_access_key_id=None, aws_secret_access_key=None, is_secure=True, port=None, proxy=None, proxy_port=None, host=None, debug=0, session_token=None, authenticate_requests=True, validate_cert=True, max_sts_attempts=3): if not host: host = self.DefaultHost self.validate_cert = validate_cert self.authenticate_requests = authenticate_requests AWSAuthConnection.__init__(self, host, aws_access_key_id, aws_secret_access_key, is_secure, port, proxy, proxy_port, debug=debug, security_token=session_token) self.pending_requests = deque() self.sts = AsyncAwsSts(aws_access_key_id, aws_secret_access_key) assert (isinstance(max_sts_attempts, int) and max_sts_attempts >= 0) self.max_sts_attempts = max_sts_attempts def _init_session_token_cb(self, error=None): if error: logging.warn("Unable to get session token: %s" % error) def _required_auth_capability(self): return ['hmac-v3-http'] def _update_session_token(self, callback, attempts=0, bypass_lock=False): """Begins the logic to get a new session token. Performs checks to ensure that only one request goes out at a time and that backoff is respected, so it can be called repeatedly with no ill effects. Set bypass_lock to True to override this behavior. """ if self.provider.security_token == PENDING_SESSION_TOKEN_UPDATE and not bypass_lock: return self.provider.security_token = PENDING_SESSION_TOKEN_UPDATE # invalidate the current security token return self.sts.get_session_token( functools.partial(self._update_session_token_cb, callback=callback, attempts=attempts)) def _update_session_token_cb(self, creds, provider='aws', callback=None, error=None, attempts=0): """Callback to use with `async_aws_sts`. The 'provider' arg is a bit misleading, it is a relic from boto and should probably be left to its default. This will take the new Credentials obj from `async_aws_sts.get_session_token()` and use it to update self.provider, and then will clear the deque of pending requests. A callback is optional. If provided, it must be callable without any arguments. """ def raise_error(): # get out of locked state self.provider.security_token = None if callable(callback): return callback(error=error) else: logging.error(error) raise error if error: if isinstance(error, InvalidClientTokenIdError): # no need to retry if error is due to bad tokens raise_error() else: if attempts > self.max_sts_attempts: raise_error() else: seconds_to_wait = (0.1 * (2 ** attempts)) logging.warning("Got error[ %s ] getting session token, retrying in %.02f seconds" % (error, seconds_to_wait)) ioloop.IOLoop.current().add_timeout(time.time() + seconds_to_wait, functools.partial(self._update_session_token, attempts=attempts + 1, callback=callback, bypass_lock=True)) return else: self.provider = Provider(provider, creds.access_key, creds.secret_key, creds.session_token) # force the correct auth, with the new provider self._auth_handler = HmacAuthV3HTTPHandler(self.host, None, self.provider) while self.pending_requests: request = self.pending_requests.pop() request() if callable(callback): return callback() def make_request(self, action, body='', callback=None, object_hook=None): """Make an asynchronous HTTP request to DynamoDB. Callback should operate on the decoded json response (with object hook applied, of course). It should also accept an error argument, which will be a boto.exception.DynamoDBResponseError. If there is not a valid session token, this method will ensure that a new one is fetched and cache the request when it is retrieved. """ this_request = functools.partial(self.make_request, action=action, body=body, callback=callback, object_hook=object_hook) if self.authenticate_requests and self.provider.security_token in [None, PENDING_SESSION_TOKEN_UPDATE]: # we will not be able to complete this request because we do not have a valid session token. # queue it and try to get a new one. _update_session_token will ensure that only one request # for a session token goes out at a time self.pending_requests.appendleft(this_request) def cb_for_update(error=None): # create a callback to handle errors getting session token # callback here is assumed to take a json response, and an instance of DynamoDBResponseError if error: raise DynamoDBResponseError(error.status, error.reason, body={'message': error.body}) else: return self._update_session_token(cb_for_update) return headers = {'X-Amz-Target' : '%s_%s.%s' % (self.ServiceName, self.Version, action), 'Content-Type' : 'application/x-amz-json-1.0', 'Content-Length' : str(len(body))} request = HTTPRequest('https://%s' % self.host, method='POST', headers=headers, body=body, validate_cert=self.validate_cert) request.path = '/' # Important! set the path variable for signing by boto. '/' is the path for all dynamodb requests if self.authenticate_requests: self._auth_handler.add_auth(request) # add signature to headers of the request http_client = httpclient.AsyncHTTPClient() http_client.fetch(request, functools.partial( self._finish_make_request, callback=callback, orig_request=this_request, token_used=self.provider.security_token, object_hook=object_hook)) def _finish_make_request(self, response, callback, orig_request, token_used, object_hook=None): """Check for errors and decode the json response (in the tornado response body), then pass on to orig callback. This method also contains some of the logic to handle reacquiring session tokens. """ if not response.body: assert response.error, 'How can there be no response body and no error? Response: %s' % response raise DynamoDBResponseError(response.error.code, response.error.message, None) json_response = json.loads(response.body, object_hook=object_hook) if response.error: aws_error_type = None try: # The error code should be in the __type field of the json response, and should be a string # in the form 'namespace.version#errorcode'. If the field doesn't exist or is in some other form, # just treat this as an unknown error type. aws_error_type = json_response.get('__type').split('#')[1] except: aws_error_type = None if aws_error_type == self.ExpiredSessionError or aws_error_type == self.UnrecognizedClientException: if self.provider.security_token == token_used: # The token that we used has expired, wipe it out. self.provider.security_token = None # make_request will handle logic to get a new token if needed, and queue until it is fetched return orig_request() if aws_error_type == AsyncDynamoDB.ProvisionedThroughputExceededException: raise DBProvisioningExceededError(json_response['message']) if aws_error_type == AsyncDynamoDB.LimitExceededException: raise DBLimitExceededError(json_response['message']) if aws_error_type == AsyncDynamoDB.ConditionalCheckFailedException: raise DBConditionalCheckFailedError(json_response['message']) raise DynamoDBResponseError(response.error.code, response.error.message, json_response) return callback(json_response)