def client_cycle_batch_items_check(materials_provider,
initial_actions,
initial_item,
table_name,
region_name=None):
kwargs = {}
if region_name is not None:
kwargs["region_name"] = region_name
client = boto3.client("dynamodb", **kwargs)
e_client = EncryptedClient(client=client,
materials_provider=materials_provider,
attribute_actions=initial_actions)
cycle_batch_item_check(
raw=client,
encrypted=e_client,
initial_actions=initial_actions,
initial_item=initial_item,
write_transformer=dict_to_ddb,
read_transformer=ddb_to_dict,
table_name=table_name,
)
raw_scan_result = client.scan(TableName=table_name, ConsistentRead=True)
e_scan_result = e_client.scan(TableName=table_name, ConsistentRead=True)
assert not raw_scan_result["Items"]
assert not e_scan_result["Items"]
def client_cycle_batch_items_check_scan_paginator(
materials_provider, initial_actions, initial_item, table_name, region_name=None
):
"""Helper function for testing the "scan" paginator.
Populate the specified table with encrypted items,
scan the table with raw client paginator to get encrypted items,
scan the table with encrypted client paginator to get decrypted items,
then verify that all items appear to have been encrypted correctly.
""" # noqa=D401
# pylint: disable=too-many-locals
kwargs = {}
if region_name is not None:
kwargs["region_name"] = region_name
client = boto3.client("dynamodb", **kwargs)
e_client = EncryptedClient(client=client, materials_provider=materials_provider, attribute_actions=initial_actions)
items_in_table = cycle_batch_item_check(
raw=client,
encrypted=e_client,
initial_actions=initial_actions,
initial_item=initial_item,
write_transformer=dict_to_ddb,
read_transformer=ddb_to_dict,
table_name=table_name,
delete_items=False,
)
try:
encrypted_items = []
raw_paginator = client.get_paginator("scan")
for page in raw_paginator.paginate(TableName=table_name, ConsistentRead=True):
encrypted_items.extend(page["Items"])
decrypted_items = []
encrypted_paginator = e_client.get_paginator("scan")
for page in encrypted_paginator.paginate(TableName=table_name, ConsistentRead=True):
decrypted_items.extend(page["Items"])
assert encrypted_items and decrypted_items
assert len(encrypted_items) == len(decrypted_items) == items_in_table
check_attribute_actions = initial_actions.copy()
check_attribute_actions.set_index_keys(*list(TEST_KEY.keys()))
check_many_encrypted_items(
actual=encrypted_items,
expected=decrypted_items,
attribute_actions=check_attribute_actions,
transformer=ddb_to_dict,
)
finally:
_cleanup_items(encrypted=e_client, write_transformer=dict_to_ddb, table_name=table_name)
raw_scan_result = client.scan(TableName=table_name, ConsistentRead=True)
e_scan_result = e_client.scan(TableName=table_name, ConsistentRead=True)
assert not raw_scan_result["Items"]
assert not e_scan_result["Items"]
def client_cycle_batch_items_check_paginators(materials_provider,
initial_actions,
initial_item,
table_name,
region_name=None):
kwargs = {}
if region_name is not None:
kwargs["region_name"] = region_name
client = boto3.client("dynamodb", **kwargs)
e_client = EncryptedClient(client=client,
materials_provider=materials_provider,
attribute_actions=initial_actions)
cycle_batch_item_check(
raw=client,
encrypted=e_client,
initial_actions=initial_actions,
initial_item=initial_item,
write_transformer=dict_to_ddb,
read_transformer=ddb_to_dict,
table_name=table_name,
delete_items=False,
)
encrypted_items = []
raw_paginator = client.get_paginator("scan")
for page in raw_paginator.paginate(TableName=table_name,
ConsistentRead=True):
encrypted_items.extend(page["Items"])
decrypted_items = []
encrypted_paginator = e_client.get_paginator("scan")
for page in encrypted_paginator.paginate(TableName=table_name,
ConsistentRead=True):
decrypted_items.extend(page["Items"])
print(encrypted_items)
print(decrypted_items)
check_attribute_actions = initial_actions.copy()
check_attribute_actions.set_index_keys(*list(TEST_KEY.keys()))
check_many_encrypted_items(
actual=encrypted_items,
expected=decrypted_items,
attribute_actions=check_attribute_actions,
transformer=ddb_to_dict,
)
_cleanup_items(encrypted=e_client,
write_transformer=dict_to_ddb,
table_name=table_name)
raw_scan_result = client.scan(TableName=table_name, ConsistentRead=True)
e_scan_result = e_client.scan(TableName=table_name, ConsistentRead=True)
assert not raw_scan_result["Items"]
assert not e_scan_result["Items"]
def client_cycle_single_item_check(materials_provider, initial_actions, initial_item, table_name, region_name=None):
check_attribute_actions = initial_actions.copy()
check_attribute_actions.set_index_keys(*list(TEST_KEY.keys()))
item = initial_item.copy()
item.update(TEST_KEY)
ddb_item = dict_to_ddb(item)
ddb_key = dict_to_ddb(TEST_KEY)
kwargs = {}
if region_name is not None:
kwargs["region_name"] = region_name
client = boto3.client("dynamodb", **kwargs)
e_client = EncryptedClient(client=client, materials_provider=materials_provider, attribute_actions=initial_actions)
_put_result = e_client.put_item(TableName=table_name, Item=ddb_item) # noqa
encrypted_result = client.get_item(TableName=table_name, Key=ddb_key, ConsistentRead=True)
check_encrypted_item(item, ddb_to_dict(encrypted_result["Item"]), check_attribute_actions)
decrypted_result = e_client.get_item(TableName=table_name, Key=ddb_key, ConsistentRead=True)
assert ddb_to_dict(decrypted_result["Item"]) == item
e_client.delete_item(TableName=table_name, Key=ddb_key)
del item
del check_attribute_actions
def client_batch_items_unprocessed_check(
materials_provider, initial_actions, initial_item, table_name, region_name=None
):
kwargs = {}
if region_name is not None:
kwargs["region_name"] = region_name
client = boto3.client("dynamodb", **kwargs)
with patch.object(client, "batch_write_item", return_requestitems_as_unprocessed):
e_client = EncryptedClient(
client=client, materials_provider=materials_provider, attribute_actions=initial_actions
)
batch_write_item_unprocessed_check(
encrypted=e_client, initial_item=initial_item, write_transformer=dict_to_ddb, table_name=table_name
)
def encrypt_item(table_name, aws_cmk_id):
"""Demonstrate use of EncryptedClient to transparently encrypt an item."""
index_key = {"partition_attribute": {"S": "is this"}, "sort_attribute": {"N": "55"}}
plaintext_item = {
"example": {"S": "data"},
"some numbers": {"N": "99"},
"and some binary": {"B": b"\x00\x01\x02"},
"leave me": {"S": "alone"}, # We want to ignore this attribute
}
# Collect all of the attributes that will be encrypted (used later).
encrypted_attributes = set(plaintext_item.keys())
encrypted_attributes.remove("leave me")
# Collect all of the attributes that will not be encrypted (used later).
unencrypted_attributes = set(index_key.keys())
unencrypted_attributes.add("leave me")
# Add the index pairs to the item.
plaintext_item.update(index_key)
# Create a normal client.
client = boto3.client("dynamodb")
# Create a crypto materials provider using the specified AWS KMS key.
aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id)
# Create attribute actions that tells the encrypted client to encrypt all attributes except one.
actions = AttributeActions(
default_action=CryptoAction.ENCRYPT_AND_SIGN, attribute_actions={"leave me": CryptoAction.DO_NOTHING}
)
# Use these objects to create an encrypted client.
encrypted_client = EncryptedClient(client=client, materials_provider=aws_kms_cmp, attribute_actions=actions)
# Put the item to the table, using the encrypted client to transparently encrypt it.
encrypted_client.put_item(TableName=table_name, Item=plaintext_item)
# Get the encrypted item using the standard client.
encrypted_item = client.get_item(TableName=table_name, Key=index_key)["Item"]
# Get the item using the encrypted client, transparently decyrpting it.
decrypted_item = encrypted_client.get_item(TableName=table_name, Key=index_key)["Item"]
# Verify that all of the attributes are different in the encrypted item
for name in encrypted_attributes:
assert encrypted_item[name] != plaintext_item[name]
assert decrypted_item[name] == plaintext_item[name]
# Verify that all of the attributes that should not be encrypted were not.
for name in unencrypted_attributes:
assert decrypted_item[name] == encrypted_item[name] == plaintext_item[name]
# Clean up the item
encrypted_client.delete_item(TableName=table_name, Key=index_key)
def _client_setup(materials_provider, table_name, table_index, ciphertext_item,
attribute_actions, prep):
prep() # Test scenario setup that needs to happen inside the test
cmp = materials_provider(
) # Some of the materials providers need to be constructed inside the test
client = fake_client(table_name, ciphertext_item)
table_info = TableInfo(name=table_name,
primary_index=TableIndex(
partition=table_index['partition'],
sort=table_index.get('sort', None)))
item_key = {
table_info.primary_index.partition:
ciphertext_item[table_info.primary_index.partition]
}
if table_info.primary_index.sort is not None:
item_key[table_info.primary_index.sort] = ciphertext_item[
table_info.primary_index.sort]
e_client = EncryptedClient(client=client,
materials_provider=cmp,
attribute_actions=attribute_actions,
auto_refresh_table_indexes=False)
e_client._table_info_cache._all_tables_info[table_name] = table_info
return e_client, dict_to_ddb(item_key)
def encrypt_item(table_name, cmk_mrk_arn_first_region,
cmk_mrk_arn_second_region):
"""Demonstrate use of Multi-Region Keys with DynamoDB Encryption Client.
This example encrypts an item with a Multi-Region Key in one region and decrypts it in another region. It
assumes that you have a Dynamo DB Global table in two regions, as well as a KMS
Multi-Region Key replicated to these regions.
"""
index_key = {
"partition_attribute": {
"S": "is this"
},
"sort_attribute": {
"N": "55"
}
}
plaintext_item = {
"example": {
"S": "data"
},
"some numbers": {
"N": "99"
},
"and some binary": {
"B": b"\x00\x01\x02"
},
"leave me": {
"S": "alone"
}, # We want to ignore this attribute
}
# Collect all of the attributes that will be encrypted (used later).
encrypted_attributes = set(plaintext_item.keys())
encrypted_attributes.remove("leave me")
# Collect all of the attributes that will not be encrypted (used later).
unencrypted_attributes = set(index_key.keys())
unencrypted_attributes.add("leave me")
# Add the index pairs to the item.
plaintext_item.update(index_key)
# Create attribute actions that tells the encrypted client to encrypt all attributes except one.
actions = AttributeActions(
default_action=CryptoAction.ENCRYPT_AND_SIGN,
attribute_actions={"leave me": CryptoAction.DO_NOTHING})
# Create a DDB client and KMS crypto materials provider in the first region using the specified AWS KMS key.
split_arn = cmk_mrk_arn_first_region.split(":")
encryption_region = split_arn[3]
ddb_client = boto3.client("dynamodb", region_name=encryption_region)
encryption_cmp = AwsKmsCryptographicMaterialsProvider(
key_id=cmk_mrk_arn_first_region)
# Use these objects to create an encrypted client.
encryption_client = EncryptedClient(client=ddb_client,
materials_provider=encryption_cmp,
attribute_actions=actions)
# Put the item to the table, using the encrypted client to transparently encrypt it.
encryption_client.put_item(TableName=table_name, Item=plaintext_item)
# Create a DDB client and KMS crypto materials provider in the second region
split_arn = cmk_mrk_arn_second_region.split(":")
decryption_region = split_arn[3]
decryption_cmp = AwsKmsCryptographicMaterialsProvider(
key_id=cmk_mrk_arn_second_region)
ddb_client = boto3.client("dynamodb", region_name=decryption_region)
# Use these objects to create an encrypted client.
decryption_client = EncryptedClient(client=ddb_client,
materials_provider=decryption_cmp,
attribute_actions=actions)
# DDB Global Table replication takes some time. Sleep for a moment to give the item a chance to replicate to the
# second region
time.sleep(1)
# Get the item from the second region, transparently decrypting it. This allows you to avoid a cross-region KMS
# call to the first region if your application is running in the second region
decrypted_item = decryption_client.get_item(TableName=table_name,
Key=index_key)["Item"]
# Verify that the decryption successfully retrieved the original plaintext
for name in encrypted_attributes:
assert plaintext_item[name] == decrypted_item[name]
# Clean up the item
encryption_client.delete_item(TableName=table_name, Key=index_key)
def encrypt_batch_items(table_name, aws_cmk_id):
"""Demonstrate use of EncryptedClient to transparently encrypt multiple items in a batch request."""
index_keys = [
{
"partition_attribute": {
"S": "is this"
},
"sort_attribute": {
"N": "55"
}
},
{
"partition_attribute": {
"S": "is this"
},
"sort_attribute": {
"N": "56"
}
},
{
"partition_attribute": {
"S": "is this"
},
"sort_attribute": {
"N": "57"
}
},
{
"partition_attribute": {
"S": "another"
},
"sort_attribute": {
"N": "55"
}
},
]
plaintext_additional_attributes = {
"example": {
"S": "data"
},
"some numbers": {
"N": "99"
},
"and some binary": {
"B": b"\x00\x01\x02"
},
"leave me": {
"S": "alone"
}, # We want to ignore this attribute
}
plaintext_items = []
for key in index_keys:
_attributes = key.copy()
_attributes.update(plaintext_additional_attributes)
plaintext_items.append(_attributes)
# Collect all of the attributes that will be encrypted (used later).
encrypted_attributes = set(plaintext_additional_attributes.keys())
encrypted_attributes.remove("leave me")
# Collect all of the attributes that will not be encrypted (used later).
unencrypted_attributes = set(index_keys[0].keys())
unencrypted_attributes.add("leave me")
# Create a normal client.
client = boto3.client("dynamodb")
# Create a crypto materials provider using the specified AWS KMS key.
aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id)
# Create attribute actions that tells the encrypted client to encrypt all attributes except one.
actions = AttributeActions(
default_action=CryptoAction.ENCRYPT_AND_SIGN,
attribute_actions={"leave me": CryptoAction.DO_NOTHING})
# Use these objects to create an encrypted client.
encrypted_client = EncryptedClient(client=client,
materials_provider=aws_kms_cmp,
attribute_actions=actions)
# Put the items to the table, using the encrypted client to transparently encrypt them.
encrypted_client.batch_write_item(RequestItems={
table_name: [{
"PutRequest": {
"Item": item
}
} for item in plaintext_items]
})
# Get the encrypted item using the standard client.
encrypted_items = client.batch_get_item(
RequestItems={table_name: {
"Keys": index_keys
}})["Responses"][table_name]
# Get the item using the encrypted client, transparently decyrpting it.
decrypted_items = encrypted_client.batch_get_item(
RequestItems={table_name: {
"Keys": index_keys
}})["Responses"][table_name]
def _select_index_from_item(item):
"""Find the index keys that match this item."""
for index in index_keys:
if all(item[key] == value for key, value in index.items()):
return index
raise Exception("Index key not found in item.")
def _select_item_from_index(index, all_items):
"""Find the item that matches these index keys."""
for item in all_items:
if all(item[key] == value for key, value in index.items()):
return item
raise Exception("Index key not found in item.")
for encrypted_item in encrypted_items:
key = _select_index_from_item(encrypted_item)
plaintext_item = _select_item_from_index(key, plaintext_items)
decrypted_item = _select_item_from_index(key, decrypted_items)
# Verify that all of the attributes are different in the encrypted item
for name in encrypted_attributes:
assert encrypted_item[name] != plaintext_item[name]
assert decrypted_item[name] == plaintext_item[name]
# Verify that all of the attributes that should not be encrypted were not.
for name in unencrypted_attributes:
assert decrypted_item[name] == encrypted_item[
name] == plaintext_item[name]
# Clean up the item
encrypted_client.batch_write_item(RequestItems={
table_name: [{
"DeleteRequest": {
"Key": key
}
} for key in index_keys]
})