def read_key(self, key, name=None, max_size=0):
"""
Provided ``key``, read field value at ``name`` or ``key.name`` if not
specified.
:param key: ``Key`` or ``PrimaryKey`` to read
:param name: override name field of key
:param max_size: if specified, check that the item is bellow a treshold
:return: field value at ``key``
:raises: :py:exc:`ddbmock.errors.ValidationException` if field does not exist, type does not match or is above ``max_size``
"""
if key is None:
return False
if name is None:
name = key.name
try:
field = self[name]
except KeyError:
raise ValidationException(u'Field {} not found'.format(name))
if max_size:
size = self.get_field_size(name)
if size > max_size:
raise ValidationException(
u'Field {} is over {} bytes limit. Got {}'.format(
name, max_size, size))
return key.read(field)
def update_item(self, key, actions, expected):
"""
Apply ``actions`` to item at ``key`` provided that it matches ``expected``.
This operation is atomic and blocks all other pending write operations.
:param key: Raw DynamoDB request hash and range key dict.
:param actions: Raw DynamoDB request actions.
:param expected: Raw DynamoDB request conditions.
:return: both deepcopies of :py:class:`ddbmock.database.item.Item` as it was (before, after) the update.
:raises: :py:exc:`ddbmock.errors.ConditionalCheckFailedException` if conditions are not met.
:raises: :py:exc:`ddbmock.errors.ValidationException` if ``actions`` attempted to modify the key or the resulting Item is biggere than :py:const:`config.MAX_ITEM_SIZE`
"""
key = Item(key)
hash_key = key.read_key(self.hash_key,
u'HashKeyElement',
max_size=config.MAX_HK_SIZE)
range_key = key.read_key(self.range_key,
u'RangeKeyElement',
max_size=config.MAX_RK_SIZE)
with self.write_lock:
# Need a deep copy as we will *modify* it
try:
old = self.store[hash_key, range_key]
new = copy.deepcopy(old)
old.assert_match_expected(expected)
except KeyError:
# Item was not in the DB yet
old = Item()
new = Item()
self.count += 1
# append the keys
new[self.hash_key.name] = key['HashKeyElement']
if self.range_key is not None:
new[self.range_key.name] = key['RangeKeyElement']
# Make sure we are not altering a key
if self.hash_key.name in actions:
raise ValidationException(
"UpdateItem can not alter the hash_key.")
if self.range_key is not None and self.range_key.name in actions:
raise ValidationException(
"UpdateItem can not alter the range_key.")
new.apply_actions(actions)
self.store[hash_key, range_key] = new
size = new.get_size()
if size > config.MAX_ITEM_SIZE:
self.store[hash_key, range_key] = old # roll back
raise ValidationException(
"Item size has exceeded the maximum allowed size of {}".
format(config.MAX_ITEM_SIZE))
return old, new
def get(self, key, fields):
"""
Get ``fields`` from :py:class:`ddbmock.database.item.Item` at ``key``.
:param key: Raw DynamoDB request key.
:param fields: Raw DynamoDB request array of field names to return. Empty to return all.
:return: reference to :py:class:`ddbmock.database.item.Item` at ``key`` or ``None`` when not found
:raises: :py:exc:`ddbmock.errors.ValidationException` if a ``range_key`` was provided while table has none.
"""
key = Item(key)
hash_key = key.read_key(self.hash_key, u'HashKeyElement')
range_key = key.read_key(self.range_key, u'RangeKeyElement')
if self.range_key is None and u'RangeKeyElement' in key:
raise ValidationException("Table {} has no range_key".format(
self.name))
try:
item = self.store[hash_key, range_key]
return item.filter(fields)
except KeyError:
# Item was not in the DB yet
return None
def scan(post, table):
if post[u'AttributesToGet'] and post[u'Count']:
raise ValidationException(
"Can not filter fields when only count is requested")
results = table.scan(
post[u'ScanFilter'],
post[u'AttributesToGet'],
post[u'ExclusiveStartKey'],
post[u'Limit'],
)
capacity = 0.5 * results.size.as_units()
push_write_throughput(table.name, capacity)
ret = {
"Count": len(results.items),
"ScannedCount": results.scanned,
"ConsumedCapacityUnits": capacity,
}
if results.last_key:
ret['LastEvaluatedKey'] = results.last_key
if not post[u'Count']:
ret[u'Items'] = results.items
return ret
def query(post, table):
if post[u'AttributesToGet'] and post[u'Count']:
raise ValidationException(
"Can filter fields when only count is requested")
base_capacity = 1 if post[u'ConsistentRead'] else 0.5
results = table.query(
post[u'HashKeyValue'],
post[u'RangeKeyCondition'],
post[u'AttributesToGet'],
post[u'ExclusiveStartKey'],
not post[u'ScanIndexForward'],
post[u'Limit'],
)
capacity = base_capacity * results.size.as_units()
push_write_throughput(table.name, capacity)
ret = {
"Count": len(results.items),
"ConsumedCapacityUnits": capacity,
}
if results.last_key is not None:
ret['LastEvaluatedKey'] = results.last_key
if not post[u'Count']:
ret[u'Items'] = results.items
return ret
def test_validation_exception_translation(self):
from ddbmock.router.boto import _ddbmock_exception_to_boto_exception
from boto.dynamodb.exceptions import DynamoDBValidationError
from ddbmock.errors import ValidationException
self.assertIsInstance(
_ddbmock_exception_to_boto_exception(
ValidationException('taratata')), DynamoDBValidationError)
def read(self, key):
"""
Parse a key as specified by DynamoDB API and return its value as long as
its typename matches :py:attr:`typename`
:param key: Raw DynamoDB request key.
:return: the value of the key
:raises: :py:exc:`ddbmock.errors.ValidationException` if field types does not match
"""
typename, value = key.items()[0]
if self.typename != typename:
raise ValidationException(
'Expected key type = {} for field {}. Got {}'.format(
self.typename, self.name, typename))
return value
def put(self, item, expected):
"""
Save ``item`` in the database provided that ``expected`` matches. Even
though DynamoDB ``UpdateItem`` operation only supports returning ``ALL_OLD``
or ``NONE``, this method returns both ``old`` and ``new`` values as the
throughput, computed in the view, takes the maximum of both size into
account.
This operation is atomic and blocks all other pending write operations.
:param item: Raw DynamoDB request item.
:param expected: Raw DynamoDB request conditions.
:return: both deepcopies of :py:class:`ddbmock.database.item.Item` as it was (before, after) the update or empty item if not found.
:raises: :py:exc:`ddbmock.errors.ConditionalCheckFailedException` if conditions are not met.
"""
item = Item(item)
if item.get_size() > config.MAX_ITEM_SIZE:
raise ValidationException(
"Item size has exceeded the maximum allowed size of {}".format(
config.MAX_ITEM_SIZE))
hash_key = item.read_key(self.hash_key, max_size=config.MAX_HK_SIZE)
range_key = item.read_key(self.range_key, max_size=config.MAX_RK_SIZE)
with self.write_lock:
try:
old = self.store[hash_key, range_key]
old.assert_match_expected(expected)
except KeyError:
# Item was not in the DB yet
self.count += 1
old = Item()
self.store[hash_key, range_key] = item
new = copy.deepcopy(item)
return old, new
def dynamodb_api_validate(action, post):
""" Find validator for ``action`` and run it on ``post``. If no validator
are found, return False.
:action: name of the route after translation to underscores
:post: data to validate
:return: False when no validator found, True on success
:raises: any onctuous exception
"""
try:
mod = import_module('.{}'.format(action), __name__)
schema = getattr(mod, 'post')
except (ImportError, AttributeError):
return post # Fixme: should log
# ignore the 'request_id' key but propagate it
schema['request_id'] = str
try:
validate = Schema(schema, required=True)
return validate(post)
except Invalid as e:
raise ValidationException(str(e))
def query(self, hash_key, rk_condition, fields, start, reverse, limit):
"""
Return ``fields`` of all items with provided ``hash_key`` whose ``range_key``
matches ``rk_condition``.
:param hash_key: Raw DynamoDB request hash_key.
:param rk_condition: Raw DynamoDB request ``range_key`` condition.
:param fields: Raw DynamoDB request array of field names to return. Empty to return all.
:param start: Raw DynamoDB request key of the first item to scan. Empty array to indicate first item.
:param reverse: Set to ``True`` to parse the range keys backward.
:param limit: Maximum number of items to return in this batch. Set to 0 or less for no maximum.
:return: Results(results, cumulated_size, last_key)
:raises: :py:exc:`ddbmock.errors.ValidationException` if ``start['HashKeyElement']`` is not ``hash_key``
"""
#FIXME: naive implementation
#FIXME: what if an item disappears during the operation ?
#TODO:
# - size limit
hash_value = self.hash_key.read(hash_key)
rk_name = self.range_key.name
size = ItemSize(0)
good_item_count = 0
results = []
lek = None
if start and start['HashKeyElement'] != hash_key:
raise ValidationException(
"'HashKeyElement' element of 'ExclusiveStartKey' must be the same as the hash_key. Expected {}, got {}"
.format(hash_key, start['HashKeyElement']))
try:
data = self.store[hash_value, None]
except KeyError:
# fix #9: return empty result set if first key does not exist
return Results(results, size, lek, -1)
keys = sorted(data.keys())
if reverse:
keys.reverse()
if start:
first_key = self.range_key.read(start['RangeKeyElement'])
index = keys.index(
first_key) + 1 # May raise ValueError but that's OK
keys = keys[index:]
for key in keys:
item = data[key]
if item.field_match(rk_name, rk_condition):
good_item_count += 1
size += item.get_size()
results.append(item.filter(fields))
if good_item_count == limit:
lek = {
u'HashKeyElement': hash_key,
u'RangeKeyElement': item[rk_name],
}
break
return Results(results, size, lek, -1)
def _apply_action(self, fieldname, action):
"""
Internal function. Applies a single action to a single field.
:param fieldname: Valid field name
:param action: Raw DynamoDB request action specification
:raises: :py:exc:`ddbmock.errors.ValidationException` whenever attempting an illegual action
"""
# Rewrite this function, it's disgustting code
if action[u'Action'] == u"PUT":
self[fieldname] = action[u'Value']
if action[u'Action'] == u"DELETE": # Starts to be anoying
if not fieldname in self:
return #shortcut
if u'Value' not in action:
del self[fieldname] # Nice and easy part
return
typename, value = _decode_field(action[u'Value'])
ftypename, fvalue = _decode_field(self[fieldname])
if len(ftypename) != 2:
raise ValidationException(
u"Can not DELETE elements from a non set type. Got {}".
format(ftypename))
if ftypename != typename:
raise ValidationException(
u"Expected type {t} for DELETE from type {t}. Got {}".
format(typename, t=ftypename))
# do the dirty work
data = set(fvalue).difference(value)
# if data empty => remove the key
if not data:
del self[fieldname]
else:
self[fieldname] = {ftypename: list(data)}
if action[u'Action'] == u"ADD": # Realy anoying to code :s
#FIXME: not perfect, action should be different if the item was new
typename, value = _decode_field(action[u'Value'])
if fieldname in self:
ftypename, fvalue = _decode_field(self[fieldname])
if ftypename == u"N":
data = Decimal(value) + Decimal(fvalue)
self[fieldname][u"N"] = unicode(data)
elif ftypename in [u"NS", u"SS", u"BS"]:
if ftypename != typename:
raise ValidationException(
u"Expected type {t} for ADD in type {t}. Got {}".
format(typename, t=ftypename))
data = set(fvalue).union(value)
self[fieldname][typename] = list(data)
else:
raise ValidationException(
u"Only N, NS, SS and BS types supports ADD operation. Got {}"
.format(ftypename))
else:
if typename not in [u"N", u"NS", u"SS"]:
raise ValidationException(
u"When performing ADD operation on new field, only Numbers or Sets are allowed. Got {} of type {}"
.format(value, typename))
self[fieldname] = action[u'Value']