def __init__(self, schema):
self.schema = schema
required_attrs = ('name', 'hash_key', 'read', 'write')
optional_attrs = ('range_key', )
for attr in required_attrs:
if not hasattr(self, attr):
raise MissingTableAttribute(
"Missing required Table attribute: {0}".format(attr))
for attr in optional_attrs:
if not hasattr(self, attr):
setattr(self, attr, None)
if self.hash_key not in self.schema.dynamorm_fields():
raise InvalidSchemaField(
"The hash key '{0}' does not exist in the schema".format(
self.hash_key))
if self.range_key and self.range_key not in self.schema.dynamorm_fields(
):
raise InvalidSchemaField(
"The range key '{0}' does not exist in the schema".format(
self.range_key))
def __init__(self):
for attr in self.REQUIRED_ATTRS:
if getattr(self, attr) is None:
raise MissingTableAttribute("Missing required Table attribute: {0}".format(attr))
if self.hash_key not in self.schema.dynamorm_fields():
raise InvalidSchemaField("The hash key '{0}' does not exist in the schema".format(self.hash_key))
if self.range_key and self.range_key not in self.schema.dynamorm_fields():
raise InvalidSchemaField("The range key '{0}' does not exist in the schema".format(self.range_key))
def get_batch(self,
keys,
consistent=False,
attrs=None,
batch_get_kwargs=None):
batch_get_kwargs = batch_get_kwargs or {}
batch_get_kwargs['Keys'] = []
for kwargs in keys:
for k, v in six.iteritems(kwargs):
if k not in self.schema.dynamorm_fields():
raise InvalidSchemaField(
"{0} does not exist in the schema fields".format(k))
batch_get_kwargs['Keys'].append(kwargs)
if consistent:
batch_get_kwargs['ConsistentRead'] = True
if attrs:
batch_get_kwargs['ProjectionExpression'] = attrs
while True:
response = self.resource.batch_get_item(
RequestItems={self.name: batch_get_kwargs})
for item in response['Responses'][self.name]:
yield item
try:
batch_get_kwargs = response['UnprocessedKeys'][self.name]
except KeyError:
# once our table is no longer listed in UnprocessedKeys we're done our while True loop
break
def query(self, query_kwargs=None, **kwargs):
assert len(kwargs) in (1, 2), "Query only takes 1 or 2 keyword arguments"
if query_kwargs is None:
query_kwargs = {}
while len(kwargs):
key, value = kwargs.popitem()
try:
key, op = key.split('__')
except ValueError:
op = 'eq'
if key not in (self.hash_key, self.range_key):
raise InvalidSchemaField("{0} is not our hash or range key".format(key))
key = Key(key)
op = getattr(key, op)
if 'KeyConditionExpression' in query_kwargs:
query_kwargs['KeyConditionExpression'] = query_kwargs['KeyConditionExpression'] & op(value)
else:
query_kwargs['KeyConditionExpression'] = op(value)
log.debug("Query: %s", query_kwargs)
return self.table.query(**query_kwargs)
def update(self, update_item_kwargs=None, conditions=None, **kwargs):
update_item_kwargs = update_item_kwargs or {}
conditions = conditions or {}
update_key = {}
update_fields = []
expr_names = {}
expr_vals = {}
UPDATE_FUNCTION_TEMPLATES = {
'append': '#uk_{0} = list_append(#uk_{0}, :uv_{0})',
'plus': '#uk_{0} = #uk_{0} + :uv_{0}',
'minus': '#uk_{0} = #uk_{0} - :uv_{0}',
'if_not_exists': '#uk_{0} = if_not_exists(#uk_{0}, :uv_{0})',
None: '#uk_{0} = :uv_{0}'
}
for key, value in six.iteritems(kwargs):
try:
key, function = key.split('__', 1)
except ValueError:
function = None
# make sure the field (key) exists
if key not in self.schema.dynamorm_fields():
raise InvalidSchemaField("{0} does not exist in the schema fields".format(key))
if key in (self.hash_key, self.range_key):
update_key[key] = value
else:
update_fields.append(UPDATE_FUNCTION_TEMPLATES[function].format(key))
expr_names['#uk_{0}'.format(key)] = key
expr_vals[':uv_{0}'.format(key)] = value
update_item_kwargs['Key'] = update_key
update_item_kwargs['UpdateExpression'] = 'SET {0}'.format(', '.join(update_fields))
update_item_kwargs['ExpressionAttributeNames'] = expr_names
update_item_kwargs['ExpressionAttributeValues'] = expr_vals
if isinstance(conditions, collections.Mapping):
condition_expression = Q(**conditions)
elif isinstance(conditions, collections.Iterable):
condition_expression = None
for condition in conditions:
try:
condition_expression = condition_expression & condition
except TypeError:
condition_expression = condition
else:
condition_expression = conditions
if condition_expression:
update_item_kwargs['ConditionExpression'] = condition_expression
try:
return self.table.update_item(**update_item_kwargs)
except botocore.exceptions.ClientError as exc:
if exc.response['Error']['Code'] == 'ConditionalCheckFailedException':
raise ConditionFailed(exc)
raise
def update(self, conditions=None, update_item_kwargs=None, **kwargs):
update_item_kwargs = update_item_kwargs or {}
conditions = conditions or {}
update_key = {}
update_fields = []
condition_fields = []
expr_names = {}
expr_vals = {}
for k, v in six.iteritems(kwargs):
if k not in self.schema.dynamorm_fields():
raise InvalidSchemaField("{0} does not exist in the schema fields".format(k))
if k in (self.hash_key, self.range_key):
update_key[k] = v
else:
update_fields.append('#u_{0} = :u_{0}'.format(k))
expr_names['#u_{0}'.format(k)] = k
expr_vals[':u_{0}'.format(k)] = v
for k, v in six.iteritems(conditions):
if k not in self.schema.dynamorm_fields():
raise InvalidSchemaField("{0} does not exist in the schema fields".format(k))
condition_fields.append('#c_{0} = :c_{0}'.format(k))
expr_names['#c_{0}'.format(k)] = k
expr_vals[':c_{0}'.format(k)] = v
update_item_kwargs['Key'] = update_key
update_item_kwargs['UpdateExpression'] = 'SET {0}'.format(', '.join(update_fields))
if condition_fields:
update_item_kwargs['ConditionExpression'] = ' AND '.join(condition_fields)
update_item_kwargs['ExpressionAttributeNames'] = expr_names
update_item_kwargs['ExpressionAttributeValues'] = expr_vals
try:
return self.table.update_item(**update_item_kwargs)
except botocore.exceptions.ClientError as exc:
if exc.response['Error']['Code'] == 'ConditionalCheckFailedException':
raise ConditionFailed(exc)
raise
def query(self, *args, **kwargs):
# copy query_kwargs, so that we don't mutate the original later on
query_kwargs = dict(
(k, v) for k, v in six.iteritems(kwargs.pop("query_kwargs", {})))
filter_kwargs = {}
if "IndexName" in query_kwargs:
attr_fields = self.index_attribute_fields(
index_name=query_kwargs["IndexName"])
else:
attr_fields = self.table_attribute_fields
while len(kwargs):
full_key, value = kwargs.popitem()
try:
key, op = full_key.split("__")
except ValueError:
key = full_key
op = "eq"
if key not in attr_fields:
filter_kwargs[full_key] = value
continue
key = Key(key)
key_expression = get_expression(key, op, value)
try:
query_kwargs["KeyConditionExpression"] = (
query_kwargs["KeyConditionExpression"] & key_expression)
except (KeyError, TypeError):
query_kwargs["KeyConditionExpression"] = key_expression
if "KeyConditionExpression" not in query_kwargs:
raise InvalidSchemaField(
"Primary key must be specified for queries")
filter_expression = Q(**filter_kwargs)
for arg in args:
try:
filter_expression = filter_expression & arg
except TypeError:
filter_expression = arg
if filter_expression:
query_kwargs["FilterExpression"] = filter_expression
log.debug("Query: %s", query_kwargs)
return self.table.query(**query_kwargs)
def query(self, *args, **kwargs):
query_kwargs = kwargs.pop('query_kwargs', {})
filter_kwargs = {}
if 'IndexName' in query_kwargs:
attr_fields = self.index_attribute_fields(
index_name=query_kwargs['IndexName'])
else:
attr_fields = self.table_attribute_fields
while len(kwargs):
full_key, value = kwargs.popitem()
try:
key, op = full_key.split('__')
except ValueError:
key = full_key
op = 'eq'
if key not in attr_fields:
filter_kwargs[full_key] = value
continue
key = Key(key)
key_expression = get_expression(key, op, value)
try:
if not query_kwargs['KeyConditionExpression'] == key_expression:
query_kwargs['KeyConditionExpression'] = query_kwargs[
'KeyConditionExpression'] & key_expression
except (KeyError, TypeError):
query_kwargs['KeyConditionExpression'] = key_expression
if 'KeyConditionExpression' not in query_kwargs:
raise InvalidSchemaField(
"Primary key must be specified for queries")
filter_expression = Q(**filter_kwargs)
for arg in args:
try:
filter_expression = filter_expression & arg
except TypeError:
filter_expression = arg
if filter_expression:
query_kwargs['FilterExpression'] = filter_expression
log.debug("Query: %s", query_kwargs)
return self.table.query(**query_kwargs)
def get(self, consistent=False, get_item_kwargs=None, **kwargs):
get_item_kwargs = get_item_kwargs or {}
for k, v in six.iteritems(kwargs):
if k not in self.schema.dynamorm_fields():
raise InvalidSchemaField("{0} does not exist in the schema fields".format(k))
get_item_kwargs['Key'] = kwargs
if consistent:
get_item_kwargs['ConsistentRead'] = True
response = self.table.get_item(**get_item_kwargs)
if 'Item' in response:
return response['Item']
def get(self, consistent=False, get_item_kwargs=None, **kwargs):
# copy get_item_kwargs, so that we don't mutate the original later on
get_item_kwargs = dict(
(k, v) for k, v in six.iteritems(get_item_kwargs or {}))
for k, v in six.iteritems(kwargs):
if k not in self.schema.dynamorm_fields():
raise InvalidSchemaField(
"{0} does not exist in the schema fields".format(k))
get_item_kwargs["Key"] = kwargs
if consistent:
get_item_kwargs["ConsistentRead"] = True
response = self.table.get_item(**get_item_kwargs)
if "Item" in response:
return response["Item"]
def get_batch(self,
keys,
consistent=False,
attrs=None,
batch_get_kwargs=None):
# copy batch_get_kwargs, so that we don't mutate the original later on
batch_get_kwargs = dict(
(k, v) for k, v in six.iteritems(batch_get_kwargs or {}))
batch_get_kwargs["Keys"] = []
for kwargs in keys:
for k, v in six.iteritems(kwargs):
if k not in self.schema.dynamorm_fields():
raise InvalidSchemaField(
"{0} does not exist in the schema fields".format(k))
batch_get_kwargs["Keys"].append(kwargs)
if consistent:
batch_get_kwargs["ConsistentRead"] = True
if attrs:
batch_get_kwargs["ProjectionExpression"] = attrs
while True:
response = self.resource.batch_get_item(
RequestItems={self.name: batch_get_kwargs})
for item in response["Responses"][self.name]:
yield item
try:
batch_get_kwargs = response["UnprocessedKeys"][self.name]
except KeyError:
# once our table is no longer listed in UnprocessedKeys we're done our while True loop
break
def update(self, update_item_kwargs=None, conditions=None, **kwargs):
# copy update_item_kwargs, so that we don't mutate the original later on
update_item_kwargs = dict(
(k, v) for k, v in six.iteritems(update_item_kwargs or {}))
conditions = conditions or {}
update_fields = []
expr_names = {}
expr_vals = {}
# First, pick out the keys for the update.
update_key = {
key: kwargs.pop(key)
for key in (self.hash_key, self.range_key) if key in kwargs
}
# Then, generate the keys and values for the update-expression.
for i, key in enumerate(kwargs):
key_parts = key.split("__")
top_level_key = key_parts[0]
# Make sure the top-level field (key) exists
# XXX TODO: Should we validate nested keys as well?
if top_level_key not in self.schema.dynamorm_fields():
raise InvalidSchemaField(
"{0} does not exist in the schema fields".format(key))
# Add the actual field expression, keys, and value.
(
field_expr,
field_expr_names,
field_expr_value,
) = self.get_update_expr_for_key(i, key_parts)
update_fields.append(field_expr)
expr_names.update(field_expr_names)
expr_vals[field_expr_value] = kwargs[key]
update_item_kwargs["Key"] = update_key
update_item_kwargs["UpdateExpression"] = "SET {0}".format(
", ".join(update_fields))
update_item_kwargs["ExpressionAttributeNames"] = expr_names
update_item_kwargs["ExpressionAttributeValues"] = expr_vals
if isinstance(conditions, Mapping):
condition_expression = Q(**conditions)
elif isinstance(conditions, Iterable):
condition_expression = None
for condition in conditions:
try:
condition_expression = condition_expression & condition
except TypeError:
condition_expression = condition
else:
condition_expression = conditions
if condition_expression:
update_item_kwargs["ConditionExpression"] = condition_expression
try:
return self.table.update_item(**update_item_kwargs)
except botocore.exceptions.ClientError as exc:
if exc.response["Error"][
"Code"] == "ConditionalCheckFailedException":
raise ConditionFailed(exc)
raise
def update(self, update_item_kwargs=None, conditions=None, **kwargs):
# copy update_item_kwargs, so that we don't mutate the original later on
update_item_kwargs = dict(
(k, v) for k, v in six.iteritems(update_item_kwargs or {}))
conditions = conditions or {}
update_key = {}
update_fields = []
expr_names = {}
expr_vals = {}
UPDATE_FUNCTION_TEMPLATES = {
"append": "#uk_{0} = list_append(#uk_{0}, :uv_{0})",
"plus": "#uk_{0} = #uk_{0} + :uv_{0}",
"minus": "#uk_{0} = #uk_{0} - :uv_{0}",
"if_not_exists": "#uk_{0} = if_not_exists(#uk_{0}, :uv_{0})",
None: "#uk_{0} = :uv_{0}",
}
for key, value in six.iteritems(kwargs):
try:
key, function = key.split("__", 1)
except ValueError:
function = None
# make sure the field (key) exists
if key not in self.schema.dynamorm_fields():
raise InvalidSchemaField(
"{0} does not exist in the schema fields".format(key))
if key in (self.hash_key, self.range_key):
update_key[key] = value
else:
update_fields.append(
UPDATE_FUNCTION_TEMPLATES[function].format(key))
expr_names["#uk_{0}".format(key)] = key
expr_vals[":uv_{0}".format(key)] = value
update_item_kwargs["Key"] = update_key
update_item_kwargs["UpdateExpression"] = "SET {0}".format(
", ".join(update_fields))
update_item_kwargs["ExpressionAttributeNames"] = expr_names
update_item_kwargs["ExpressionAttributeValues"] = expr_vals
if isinstance(conditions, collections.Mapping):
condition_expression = Q(**conditions)
elif isinstance(conditions, collections.Iterable):
condition_expression = None
for condition in conditions:
try:
condition_expression = condition_expression & condition
except TypeError:
condition_expression = condition
else:
condition_expression = conditions
if condition_expression:
update_item_kwargs["ConditionExpression"] = condition_expression
try:
return self.table.update_item(**update_item_kwargs)
except botocore.exceptions.ClientError as exc:
if exc.response["Error"][
"Code"] == "ConditionalCheckFailedException":
raise ConditionFailed(exc)
raise