def get_item(self):
name = self.body["TableName"]
self.dynamodb_backend.get_table(name)
key = self.body["Key"]
projection_expression = self.body.get("ProjectionExpression")
expression_attribute_names = self.body.get("ExpressionAttributeNames")
if expression_attribute_names == {}:
if projection_expression is None:
raise MockValidationException(
"ExpressionAttributeNames can only be specified when using expressions"
)
else:
raise MockValidationException(
"ExpressionAttributeNames must not be empty")
expression_attribute_names = expression_attribute_names or {}
projection_expression = self._adjust_projection_expression(
projection_expression, expression_attribute_names)
item = self.dynamodb_backend.get_item(name, key, projection_expression)
if item:
item_dict = item.describe_attrs(attributes=None)
return dynamo_json_dump(item_dict)
else:
# Item not found
return dynamo_json_dump({})
def restore_table_to_point_in_time(self):
body = self.body
target_table_name = body.get("TargetTableName")
source_table_name = body.get("SourceTableName")
restored_table = self.dynamodb_backend.restore_table_to_point_in_time(
target_table_name, source_table_name)
return dynamo_json_dump(restored_table.describe())
def restore_table_from_backup(self):
body = self.body
target_table_name = body.get("TargetTableName")
backup_arn = body.get("BackupArn")
restored_table = self.dynamodb_backend.restore_table_from_backup(
target_table_name, backup_arn)
return dynamo_json_dump(restored_table.describe())
def batch_write_item(self):
table_batches = self.body["RequestItems"]
for table_name, table_requests in table_batches.items():
for table_request in table_requests:
request_type = list(table_request.keys())[0]
request = list(table_request.values())[0]
if request_type == "PutRequest":
item = request["Item"]
self.dynamodb_backend.put_item(table_name, item)
elif request_type == "DeleteRequest":
keys = request["Key"]
self.dynamodb_backend.delete_item(table_name, keys)
response = {
"ConsumedCapacity": [{
"TableName": table_name,
"CapacityUnits": 1.0,
"Table": {
"CapacityUnits": 1.0
},
} for table_name, table_requests in table_batches.items()],
"ItemCollectionMetrics": {},
"UnprocessedItems": {},
}
return dynamo_json_dump(response)
def create_backup(self):
body = self.body
table_name = body.get("TableName")
backup_name = body.get("BackupName")
backup = self.dynamodb_backend.create_backup(table_name, backup_name)
response = {"BackupDetails": backup.details}
return dynamo_json_dump(response)
def transact_get_items(self):
transact_items = self.body["TransactItems"]
responses = list()
if len(transact_items) > TRANSACTION_MAX_ITEMS:
msg = "1 validation error detected: Value '["
err_list = list()
request_id = 268435456
for _ in transact_items:
request_id += 1
hex_request_id = format(request_id, "x")
err_list.append(
"com.amazonaws.dynamodb.v20120810.TransactGetItem@%s"
% hex_request_id
)
msg += ", ".join(err_list)
msg += (
"'] at 'transactItems' failed to satisfy constraint: "
"Member must have length less than or equal to %s"
% TRANSACTION_MAX_ITEMS
)
raise MockValidationException(msg)
ret_consumed_capacity = self.body.get("ReturnConsumedCapacity", "NONE")
consumed_capacity = dict()
for transact_item in transact_items:
table_name = transact_item["Get"]["TableName"]
key = transact_item["Get"]["Key"]
item = self.dynamodb_backend.get_item(table_name, key)
if not item:
responses.append({})
continue
item_describe = item.describe_attrs(False)
responses.append(item_describe)
table_capacity = consumed_capacity.get(table_name, {})
table_capacity["TableName"] = table_name
capacity_units = table_capacity.get("CapacityUnits", 0) + 2.0
table_capacity["CapacityUnits"] = capacity_units
read_capacity_units = table_capacity.get("ReadCapacityUnits", 0) + 2.0
table_capacity["ReadCapacityUnits"] = read_capacity_units
consumed_capacity[table_name] = table_capacity
if ret_consumed_capacity == "INDEXES":
table_capacity["Table"] = {
"CapacityUnits": capacity_units,
"ReadCapacityUnits": read_capacity_units,
}
result = dict()
result.update({"Responses": responses})
if ret_consumed_capacity != "NONE":
result.update({"ConsumedCapacity": [v for v in consumed_capacity.values()]})
return dynamo_json_dump(result)
def delete_item(self):
name = self.body["TableName"]
key = self.body["Key"]
return_values = self.body.get("ReturnValues", "NONE")
if return_values not in ("ALL_OLD", "NONE"):
raise MockValidationException("Return values set to invalid value")
try:
self.dynamodb_backend.get_table(name)
except ResourceNotFoundException:
raise ConditionalCheckFailed(
"A condition specified in the operation could not be evaluated."
)
# Attempt to parse simple ConditionExpressions into an Expected
# expression
condition_expression = self.body.get("ConditionExpression")
expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
item = self.dynamodb_backend.delete_item(
name,
key,
expression_attribute_names,
expression_attribute_values,
condition_expression,
)
if item and return_values == "ALL_OLD":
item_dict = item.to_json()
else:
item_dict = {"Attributes": {}}
item_dict["ConsumedCapacityUnits"] = 0.5
return dynamo_json_dump(item_dict)
def _wrapper(*args, **kwargs):
(handler, ) = args
expected_capacity = handler.body.get("ReturnConsumedCapacity",
"NONE")
if expected_capacity not in ["NONE", "TOTAL", "INDEXES"]:
type_ = "ValidationException"
message = "1 validation error detected: Value '{}' at 'returnConsumedCapacity' failed to satisfy constraint: Member must satisfy enum value set: [INDEXES, TOTAL, NONE]".format(
expected_capacity)
return (
400,
handler.response_headers,
dynamo_json_dump({
"__type": type_,
"message": message
}),
)
table_name = handler.body.get("TableName", "")
index_name = handler.body.get("IndexName", None)
response = f(*args, **kwargs)
if isinstance(response, str):
body = json.loads(response)
if expected_capacity == "TOTAL":
body["ConsumedCapacity"] = {
"TableName": table_name,
"CapacityUnits": val,
}
elif expected_capacity == "INDEXES":
body["ConsumedCapacity"] = {
"TableName": table_name,
"CapacityUnits": val,
"Table": {
"CapacityUnits": val
},
}
if index_name:
body["ConsumedCapacity"]["LocalSecondaryIndexes"] = {
index_name: {
"CapacityUnits": val
}
}
return dynamo_json_dump(body)
return response
def put_item(self):
name = self.body["TableName"]
item = self.body["Item"]
return_values = self.body.get("ReturnValues", "NONE")
if return_values not in ("ALL_OLD", "NONE"):
raise MockValidationException("Return values set to invalid value")
if put_has_empty_keys(item, self.dynamodb_backend.get_table(name)):
raise MockValidationException(
"One or more parameter values were invalid: An AttributeValue may not contain an empty string"
)
if put_has_empty_attrs(item, self.dynamodb_backend.get_table(name)):
raise MockValidationException(
"One or more parameter values were invalid: An number set may not be empty"
)
overwrite = "Expected" not in self.body
if not overwrite:
expected = self.body["Expected"]
else:
expected = None
if return_values == "ALL_OLD":
existing_item = self.dynamodb_backend.get_item(name, item)
if existing_item:
existing_attributes = existing_item.to_json()["Attributes"]
else:
existing_attributes = {}
# Attempt to parse simple ConditionExpressions into an Expected
# expression
condition_expression = self.body.get("ConditionExpression")
expression_attribute_names = self.body.get("ExpressionAttributeNames", {})
expression_attribute_values = self.body.get("ExpressionAttributeValues", {})
if condition_expression:
overwrite = False
result = self.dynamodb_backend.put_item(
name,
item,
expected,
condition_expression,
expression_attribute_names,
expression_attribute_values,
overwrite,
)
item_dict = result.to_json()
if return_values == "ALL_OLD":
item_dict["Attributes"] = existing_attributes
else:
item_dict.pop("Attributes", None)
return dynamo_json_dump(item_dict)
def list_tables(self):
body = self.body
limit = body.get("Limit", 100)
exclusive_start_table_name = body.get("ExclusiveStartTableName")
tables, last_eval = self.dynamodb_backend.list_tables(
limit, exclusive_start_table_name)
response = {"TableNames": tables}
if last_eval:
response["LastEvaluatedTableName"] = last_eval
return dynamo_json_dump(response)
def batch_get_item(self):
table_batches = self.body["RequestItems"]
results = {
"ConsumedCapacity": [],
"Responses": {},
"UnprocessedKeys": {}
}
# Validation: Can only request up to 100 items at the same time
# Scenario 1: We're requesting more than a 100 keys from a single table
for table_name, table_request in table_batches.items():
if len(table_request["Keys"]) > 100:
raise MockValidationException(
"1 validation error detected: Value at 'requestItems." +
table_name +
".member.keys' failed to satisfy constraint: Member must have length less than or equal to 100"
)
# Scenario 2: We're requesting more than a 100 keys across all tables
nr_of_keys_across_all_tables = sum(
[len(req["Keys"]) for _, req in table_batches.items()])
if nr_of_keys_across_all_tables > 100:
raise MockValidationException(
"Too many items requested for the BatchGetItem call")
for table_name, table_request in table_batches.items():
keys = table_request["Keys"]
if self._contains_duplicates(keys):
raise MockValidationException(
"Provided list of item keys contains duplicates")
attributes_to_get = table_request.get("AttributesToGet")
projection_expression = table_request.get("ProjectionExpression")
expression_attribute_names = table_request.get(
"ExpressionAttributeNames", {})
projection_expression = self._adjust_projection_expression(
projection_expression, expression_attribute_names)
results["Responses"][table_name] = []
for key in keys:
item = self.dynamodb_backend.get_item(table_name, key,
projection_expression)
if item:
item_describe = item.describe_attrs(attributes_to_get)
results["Responses"][table_name].append(
item_describe["Item"])
results["ConsumedCapacity"].append({
"CapacityUnits": len(keys),
"TableName": table_name
})
return dynamo_json_dump(results)
def get_empty_str_error():
er = "com.amazonaws.dynamodb.v20111205#ValidationException"
return (
400,
{
"server": "amazon.com"
},
dynamo_json_dump({
"__type":
er,
"message": ("One or more parameter values were "
"invalid: An AttributeValue may not "
"contain an empty string"),
}),
)
def update_table(self):
name = self.body["TableName"]
attr_definitions = self.body.get("AttributeDefinitions", None)
global_index = self.body.get("GlobalSecondaryIndexUpdates", None)
throughput = self.body.get("ProvisionedThroughput", None)
billing_mode = self.body.get("BillingMode", None)
stream_spec = self.body.get("StreamSpecification", None)
table = self.dynamodb_backend.update_table(
name=name,
attr_definitions=attr_definitions,
global_index=global_index,
throughput=throughput,
billing_mode=billing_mode,
stream_spec=stream_spec,
)
return dynamo_json_dump(table.describe())
def scan(self):
name = self.body["TableName"]
filters = {}
scan_filters = self.body.get("ScanFilter", {})
for attribute_name, scan_filter in scan_filters.items():
# Keys are attribute names. Values are tuples of (comparison,
# comparison_value)
comparison_operator = scan_filter["ComparisonOperator"]
comparison_values = scan_filter.get("AttributeValueList", [])
filters[attribute_name] = (comparison_operator, comparison_values)
filter_expression = self.body.get("FilterExpression")
expression_attribute_values = self.body.get(
"ExpressionAttributeValues", {})
expression_attribute_names = self.body.get("ExpressionAttributeNames",
{})
projection_expression = self.body.get("ProjectionExpression", "")
exclusive_start_key = self.body.get("ExclusiveStartKey")
limit = self.body.get("Limit")
index_name = self.body.get("IndexName")
try:
items, scanned_count, last_evaluated_key = self.dynamodb_backend.scan(
name,
filters,
limit,
exclusive_start_key,
filter_expression,
expression_attribute_names,
expression_attribute_values,
index_name,
projection_expression,
)
except ValueError as err:
raise MockValidationException(
"Bad Filter Expression: {0}".format(err))
result = {
"Count": len(items),
"Items": [item.attrs for item in items],
"ScannedCount": scanned_count,
}
if last_evaluated_key is not None:
result["LastEvaluatedKey"] = last_evaluated_key
return dynamo_json_dump(result)
def update_item(self):
name = self.body["TableName"]
key = self.body["Key"]
return_values = self.body.get("ReturnValues", "NONE")
update_expression = self.body.get("UpdateExpression", "").strip()
attribute_updates = self.body.get("AttributeUpdates")
if update_expression and attribute_updates:
raise MockValidationException(
"Can not use both expression and non-expression parameters in the same request: Non-expression parameters: {AttributeUpdates} Expression parameters: {UpdateExpression}"
)
# We need to copy the item in order to avoid it being modified by the update_item operation
existing_item = copy.deepcopy(self.dynamodb_backend.get_item(
name, key))
if existing_item:
existing_attributes = existing_item.to_json()["Attributes"]
else:
existing_attributes = {}
if return_values not in (
"NONE",
"ALL_OLD",
"ALL_NEW",
"UPDATED_OLD",
"UPDATED_NEW",
):
raise MockValidationException("Return values set to invalid value")
if "Expected" in self.body:
expected = self.body["Expected"]
else:
expected = None
# Attempt to parse simple ConditionExpressions into an Expected
# expression
condition_expression = self.body.get("ConditionExpression")
expression_attribute_names = self.body.get("ExpressionAttributeNames",
{})
expression_attribute_values = self.body.get(
"ExpressionAttributeValues", {})
item = self.dynamodb_backend.update_item(
name,
key,
update_expression=update_expression,
attribute_updates=attribute_updates,
expression_attribute_names=expression_attribute_names,
expression_attribute_values=expression_attribute_values,
expected=expected,
condition_expression=condition_expression,
)
item_dict = item.to_json()
item_dict["ConsumedCapacity"] = {
"TableName": name,
"CapacityUnits": 0.5
}
unchanged_attributes = {
k
for k in existing_attributes.keys()
if existing_attributes[k] == item_dict["Attributes"].get(k)
}
changed_attributes = (set(existing_attributes.keys()).union(
item_dict["Attributes"].keys()).difference(unchanged_attributes))
if return_values == "NONE":
item_dict["Attributes"] = {}
elif return_values == "ALL_OLD":
item_dict["Attributes"] = existing_attributes
elif return_values == "UPDATED_OLD":
item_dict["Attributes"] = {
k: v
for k, v in existing_attributes.items()
if k in changed_attributes
}
elif return_values == "UPDATED_NEW":
item_dict["Attributes"] = self._build_updated_new_attributes(
existing_attributes, item_dict["Attributes"])
return dynamo_json_dump(item_dict)
def query(self):
name = self.body["TableName"]
key_condition_expression = self.body.get("KeyConditionExpression")
projection_expression = self.body.get("ProjectionExpression")
expression_attribute_names = self.body.get("ExpressionAttributeNames",
{})
filter_expression = self.body.get("FilterExpression")
expression_attribute_values = self.body.get(
"ExpressionAttributeValues", {})
projection_expression = self._adjust_projection_expression(
projection_expression, expression_attribute_names)
filter_kwargs = {}
if key_condition_expression:
value_alias_map = self.body.get("ExpressionAttributeValues", {})
index_name = self.body.get("IndexName")
schema = self.dynamodb_backend.get_schema(table_name=name,
index_name=index_name)
reverse_attribute_lookup = dict((v, k) for k, v in self.body.get(
"ExpressionAttributeNames", {}).items())
if " and " in key_condition_expression.lower():
expressions = re.split(" AND ",
key_condition_expression,
maxsplit=1,
flags=re.IGNORECASE)
index_hash_key = [
key for key in schema if key["KeyType"] == "HASH"
][0]
hash_key_var = reverse_attribute_lookup.get(
index_hash_key["AttributeName"],
index_hash_key["AttributeName"])
hash_key_regex = r"(^|[\s(]){0}\b".format(hash_key_var)
i, hash_key_expression = next(
((i, e) for i, e in enumerate(expressions)
if re.search(hash_key_regex, e)),
(None, None),
)
if hash_key_expression is None:
raise MockValidationException(
"Query condition missed key schema element: {}".format(
hash_key_var))
hash_key_expression = hash_key_expression.strip("()")
expressions.pop(i)
# TODO implement more than one range expression and OR operators
range_key_expression = expressions[0].strip("()")
# Split expression, and account for all kinds of whitespacing around commas and brackets
range_key_expression_components = re.split(
r"\s*\(\s*|\s*,\s*|\s", range_key_expression)
# Skip whitespace
range_key_expression_components = [
c for c in range_key_expression_components if c
]
range_comparison = range_key_expression_components[1]
if " and " in range_key_expression.lower():
range_comparison = "BETWEEN"
# [range_key, between, x, and, y]
range_values = [
value_alias_map[range_key_expression_components[2]],
value_alias_map[range_key_expression_components[4]],
]
supplied_range_key = range_key_expression_components[0]
elif "begins_with" in range_key_expression:
range_comparison = "BEGINS_WITH"
# [begins_with, range_key, x]
range_values = [
value_alias_map[range_key_expression_components[-1]]
]
supplied_range_key = range_key_expression_components[1]
elif "begins_with" in range_key_expression.lower():
function_used = range_key_expression[
range_key_expression.lower(
).index("begins_with"):len("begins_with")]
raise MockValidationException(
"Invalid KeyConditionExpression: Invalid function name; function: {}"
.format(function_used))
else:
# [range_key, =, x]
range_values = [
value_alias_map[range_key_expression_components[2]]
]
supplied_range_key = range_key_expression_components[0]
supplied_range_key = expression_attribute_names.get(
supplied_range_key, supplied_range_key)
range_keys = [
k["AttributeName"] for k in schema
if k["KeyType"] == "RANGE"
]
if supplied_range_key not in range_keys:
raise MockValidationException(
"Query condition missed key schema element: {}".format(
range_keys[0]))
else:
hash_key_expression = key_condition_expression.strip("()")
range_comparison = None
range_values = []
if not re.search("[^<>]=", hash_key_expression):
raise MockValidationException(
"Query key condition not supported")
hash_key_value_alias = hash_key_expression.split("=")[1].strip()
# Temporary fix until we get proper KeyConditionExpression function
hash_key = value_alias_map.get(hash_key_value_alias,
{"S": hash_key_value_alias})
else:
# 'KeyConditions': {u'forum_name': {u'ComparisonOperator': u'EQ', u'AttributeValueList': [{u'S': u'the-key'}]}}
key_conditions = self.body.get("KeyConditions")
query_filters = self.body.get("QueryFilter")
if not (key_conditions or query_filters):
raise MockValidationException(
"Either KeyConditions or QueryFilter should be present")
if key_conditions:
(
hash_key_name,
range_key_name,
) = self.dynamodb_backend.get_table_keys_name(
name, key_conditions.keys())
for key, value in key_conditions.items():
if key not in (hash_key_name, range_key_name):
filter_kwargs[key] = value
if hash_key_name is None:
raise ResourceNotFoundException
hash_key = key_conditions[hash_key_name]["AttributeValueList"][
0]
if len(key_conditions) == 1:
range_comparison = None
range_values = []
else:
if range_key_name is None and not filter_kwargs:
raise MockValidationException("Validation Exception")
else:
range_condition = key_conditions.get(range_key_name)
if range_condition:
range_comparison = range_condition[
"ComparisonOperator"]
range_values = range_condition[
"AttributeValueList"]
else:
range_comparison = None
range_values = []
if query_filters:
filter_kwargs.update(query_filters)
index_name = self.body.get("IndexName")
exclusive_start_key = self.body.get("ExclusiveStartKey")
limit = self.body.get("Limit")
scan_index_forward = self.body.get("ScanIndexForward")
items, scanned_count, last_evaluated_key = self.dynamodb_backend.query(
name,
hash_key,
range_comparison,
range_values,
limit,
exclusive_start_key,
scan_index_forward,
projection_expression,
index_name=index_name,
expr_names=expression_attribute_names,
expr_values=expression_attribute_values,
filter_expression=filter_expression,
**filter_kwargs)
result = {
"Count": len(items),
"ScannedCount": scanned_count,
}
if self.body.get("Select", "").upper() != "COUNT":
result["Items"] = [item.attrs for item in items]
if last_evaluated_key is not None:
result["LastEvaluatedKey"] = last_evaluated_key
return dynamo_json_dump(result)
def list_backups(self):
body = self.body
table_name = body.get("TableName")
backups = self.dynamodb_backend.list_backups(table_name)
response = {"BackupSummaries": [backup.summary for backup in backups]}
return dynamo_json_dump(response)
def describe_backup(self):
body = self.body
backup_arn = body.get("BackupArn")
backup = self.dynamodb_backend.describe_backup(backup_arn)
response = {"BackupDescription": backup.description}
return dynamo_json_dump(response)
def describe_table(self):
name = self.body["TableName"]
table = self.dynamodb_backend.describe_table(name)
return dynamo_json_dump(table)
def create_table(self):
body = self.body
# get the table name
table_name = body["TableName"]
# check billing mode and get the throughput
if "BillingMode" in body.keys(
) and body["BillingMode"] == "PAY_PER_REQUEST":
if "ProvisionedThroughput" in body.keys():
raise MockValidationException(
"ProvisionedThroughput cannot be specified when BillingMode is PAY_PER_REQUEST"
)
throughput = None
billing_mode = "PAY_PER_REQUEST"
else: # Provisioned (default billing mode)
throughput = body.get("ProvisionedThroughput")
if throughput is None:
raise MockValidationException(
"One or more parameter values were invalid: ReadCapacityUnits and WriteCapacityUnits must both be specified when BillingMode is PROVISIONED"
)
billing_mode = "PROVISIONED"
# getting ServerSideEncryption details
sse_spec = body.get("SSESpecification")
# getting the schema
key_schema = body["KeySchema"]
# getting attribute definition
attr = body["AttributeDefinitions"]
# getting the indexes
global_indexes = body.get("GlobalSecondaryIndexes")
if global_indexes == []:
raise MockValidationException(
"One or more parameter values were invalid: List of GlobalSecondaryIndexes is empty"
)
global_indexes = global_indexes or []
local_secondary_indexes = body.get("LocalSecondaryIndexes")
if local_secondary_indexes == []:
raise MockValidationException(
"One or more parameter values were invalid: List of LocalSecondaryIndexes is empty"
)
local_secondary_indexes = local_secondary_indexes or []
# Verify AttributeDefinitions list all
expected_attrs = []
expected_attrs.extend([key["AttributeName"] for key in key_schema])
expected_attrs.extend(schema["AttributeName"]
for schema in itertools.chain(*list(
idx["KeySchema"]
for idx in local_secondary_indexes)))
expected_attrs.extend(schema["AttributeName"]
for schema in itertools.chain(*list(
idx["KeySchema"] for idx in global_indexes)))
expected_attrs = list(set(expected_attrs))
expected_attrs.sort()
actual_attrs = [item["AttributeName"] for item in attr]
actual_attrs.sort()
if actual_attrs != expected_attrs:
return self._throw_attr_error(
actual_attrs, expected_attrs, global_indexes
or local_secondary_indexes)
# get the stream specification
streams = body.get("StreamSpecification")
# Get any tags
tags = body.get("Tags", [])
table = self.dynamodb_backend.create_table(
table_name,
schema=key_schema,
throughput=throughput,
attr=attr,
global_indexes=global_indexes,
indexes=local_secondary_indexes,
streams=streams,
billing_mode=billing_mode,
sse_specification=sse_spec,
tags=tags,
)
return dynamo_json_dump(table.describe())
def transact_write_items(self):
transact_items = self.body["TransactItems"]
self.dynamodb_backend.transact_write_items(transact_items)
response = {"ConsumedCapacity": [], "ItemCollectionMetrics": {}}
return dynamo_json_dump(response)
def describe_endpoints(self):
response = {"Endpoints": self.dynamodb_backend.describe_endpoints()}
return dynamo_json_dump(response)
