def __attrs_post_init__(self):
"""Prepare table info is it was not set and set up translation methods."""
if self._table_info is None:
self._table_info = TableInfo(name=self._table.name)
if self._auto_refresh_table_indexes:
self._table_info.refresh_indexed_attributes(
self._table.meta.client)
# Clone the attribute actions before we modify them
self._attribute_actions = self._attribute_actions.copy()
self._attribute_actions.set_index_keys(
*self._table_info.protected_index_keys())
self._crypto_config = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
crypto_config_from_kwargs,
partial(crypto_config_from_table_info, self._materials_provider,
self._attribute_actions, self._table_info),
)
self.get_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
decrypt_get_item, decrypt_python_item, self._crypto_config,
self._table.get_item)
self.put_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
encrypt_put_item, encrypt_python_item, self._crypto_config,
self._table.put_item)
self.query = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
decrypt_multi_get, decrypt_python_item, self._crypto_config,
self._table.query)
self.scan = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
decrypt_multi_get, decrypt_python_item, self._crypto_config,
self._table.scan)
def table_info(self, table_name: str):
try:
return self._table_infos[table_name]
except KeyError:
_table_info = TableInfo(name=table_name)
if self._auto_refresh_table_indexes:
_table_info.refresh_indexed_attributes(self._client)
self._table_infos[table_name] = _table_info
return _table_info
def table_info(self, table_name):
"""Collect a TableInfo object for the specified table, creating and adding it to
the cache if not already present.
:param str table_name: Name of table
:returns: TableInfo describing the requested table
:rtype: TableInfo
"""
try:
return self._all_tables_info[table_name]
except KeyError:
_table_info = TableInfo(name=table_name)
if self._auto_refresh_table_indexes:
_table_info.refresh_indexed_attributes(self._client)
self._all_tables_info[table_name] = _table_info
return _table_info
def _item_check(materials_provider, table_name, table_index, ciphertext_item,
plaintext_item, attribute_actions, prep):
ciphertext_item = ddb_to_dict(ciphertext_item)
plaintext_item = ddb_to_dict(plaintext_item)
prep() # Test scenario setup that needs to happen inside the test
cmp = materials_provider(
) # Some of the materials providers need to be constructed inside the test
table = fake_table(ciphertext_item)
table_info = TableInfo(name=table_name,
primary_index=TableIndex(
partition=table_index['partition'],
sort=table_index.get('sort', None)))
item_key = {
table_info.primary_index.partition:
ciphertext_item[table_info.primary_index.partition]
}
if table_info.primary_index.sort is not None:
item_key[table_info.primary_index.sort] = ciphertext_item[
table_info.primary_index.sort]
e_table = EncryptedTable(table=table,
materials_provider=cmp,
table_info=table_info,
attribute_actions=attribute_actions,
auto_refresh_table_indexes=False)
decrypted_item = e_table.get_item(Key=item_key)['Item']
assert set(decrypted_item.keys()) == set(plaintext_item.keys())
for key in decrypted_item:
if key == 'version':
continue
assert decrypted_item[key] == plaintext_item[key]
def _resource_setup(materials_provider, table_name, table_index,
ciphertext_item, attribute_actions, prep):
prep() # Test scenario setup that needs to happen inside the test
cmp = materials_provider(
) # Some of the materials providers need to be constructed inside the test
resource = fake_resource(table_name, ciphertext_item)
table_info = TableInfo(
name=table_name,
primary_index=TableIndex(partition=table_index["partition"],
sort=table_index.get("sort", None)),
)
item_key = {
table_info.primary_index.partition:
ciphertext_item[table_info.primary_index.partition]
}
if table_info.primary_index.sort is not None:
item_key[table_info.primary_index.sort] = ciphertext_item[
table_info.primary_index.sort]
e_resource = EncryptedResource(resource=resource,
materials_provider=cmp,
attribute_actions=attribute_actions,
auto_refresh_table_indexes=False)
e_resource._table_info_cache._all_tables_info[table_name] = table_info
return e_resource, item_key
def _item_check(materials_provider, table_name, table_index, ciphertext_item,
plaintext_item, attribute_actions, prep):
ciphertext_item = ddb_to_dict(ciphertext_item)
plaintext_item = ddb_to_dict(plaintext_item)
metatable = None
try:
metatable = prep(
) # Test scenario setup that needs to happen inside the test
cmp = materials_provider(
) # Some of the materials providers need to be constructed inside the test
table = fake_table(ciphertext_item)
table_info = TableInfo(
name=table_name,
primary_index=TableIndex(partition=table_index["partition"],
sort=table_index.get("sort", None)),
)
item_key = {
table_info.primary_index.partition:
ciphertext_item[table_info.primary_index.partition]
}
if table_info.primary_index.sort is not None:
item_key[table_info.primary_index.sort] = ciphertext_item[
table_info.primary_index.sort]
e_table = EncryptedTable(
table=table,
materials_provider=cmp,
table_info=table_info,
attribute_actions=attribute_actions,
auto_refresh_table_indexes=False,
)
decrypted_item = e_table.get_item(Key=item_key)["Item"]
assert set(decrypted_item.keys()) == set(plaintext_item.keys())
for key in decrypted_item:
if key == "version":
continue
assert decrypted_item[key] == plaintext_item[key]
finally:
if metatable:
metatable.delete()
metatable.wait_until_not_exists()
def __init__(
self,
table: ServiceResource,
key_store: KeyStore,
table_info: Optional[TableInfo] = None,
attribute_actions: Optional[AttributeActions] = None,
auto_refresh_table_indexes: Optional[bool] = True,
) -> None:
if attribute_actions is None:
attribute_actions = AttributeActions()
if table_info is None:
table_info = TableInfo(name=table.name)
if auto_refresh_table_indexes:
table_info.refresh_indexed_attributes(table.meta.client)
self._table = table
self._key_store = key_store
self._table_info = table_info
self._attribute_actions = attribute_actions
def _client_setup(materials_provider, table_name, table_index, ciphertext_item,
attribute_actions, prep):
prep() # Test scenario setup that needs to happen inside the test
cmp = materials_provider(
) # Some of the materials providers need to be constructed inside the test
client = fake_client(table_name, ciphertext_item)
table_info = TableInfo(name=table_name,
primary_index=TableIndex(
partition=table_index['partition'],
sort=table_index.get('sort', None)))
item_key = {
table_info.primary_index.partition:
ciphertext_item[table_info.primary_index.partition]
}
if table_info.primary_index.sort is not None:
item_key[table_info.primary_index.sort] = ciphertext_item[
table_info.primary_index.sort]
e_client = EncryptedClient(client=client,
materials_provider=cmp,
attribute_actions=attribute_actions,
auto_refresh_table_indexes=False)
e_client._table_info_cache._all_tables_info[table_name] = table_info
return e_client, dict_to_ddb(item_key)
def encrypt_item(table_name, aws_cmk_id):
"""Demonstrate use of EncryptedTable to transparently encrypt an item."""
index_key = {
'partition_attribute': 'is this',
'sort_attribute': 55
}
plaintext_item = {
'example': 'data',
'some numbers': 99,
'and some binary': Binary(b'\x00\x01\x02'),
'leave me': 'alone' # We want to ignore this attribute
}
# Collect all of the attributes that will be encrypted (used later).
encrypted_attributes = set(plaintext_item.keys())
encrypted_attributes.remove('leave me')
# Collect all of the attributes that will not be encrypted (used later).
unencrypted_attributes = set(index_key.keys())
unencrypted_attributes.add('leave me')
# Add the index pairs to the item.
plaintext_item.update(index_key)
# Create a normal table resource.
table = boto3.resource('dynamodb').Table(table_name)
# Use the TableInfo helper to collect information about the indexes.
table_info = TableInfo(name=table_name)
table_info.refresh_indexed_attributes(table.meta.client)
# Create a crypto materials provider using the specified AWS KMS key.
aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id)
encryption_context = EncryptionContext(
table_name=table_name,
partition_key_name=table_info.primary_index.partition,
sort_key_name=table_info.primary_index.sort,
# The only attributes that are used by the AWS KMS cryptographic materials providers
# are the primary index attributes.
# These attributes need to be in the form of a DynamoDB JSON structure, so first
# convert the standard dictionary.
attributes=dict_to_ddb(index_key)
)
# Create attribute actions that tells the encrypted table to encrypt all attributes,
# only sign the primary index attributes, and ignore the one identified attribute to
# ignore.
actions = AttributeActions(
default_action=CryptoAction.ENCRYPT_AND_SIGN,
attribute_actions={'leave me': CryptoAction.DO_NOTHING}
)
actions.set_index_keys(*table_info.protected_index_keys())
# Build the crypto config to use for this item.
# When using the higher-level helpers, this is handled for you.
crypto_config = CryptoConfig(
materials_provider=aws_kms_cmp,
encryption_context=encryption_context,
attribute_actions=actions
)
# Encrypt the plaintext item directly
encrypted_item = encrypt_python_item(plaintext_item, crypto_config)
# You could now put the encrypted item to DynamoDB just as you would any other item.
# table.put_item(Item=encrypted_item)
# We will skip this for the purposes of this example.
# Decrypt the encrypted item directly
decrypted_item = decrypt_python_item(encrypted_item, crypto_config)
# Verify that all of the attributes are different in the encrypted item
for name in encrypted_attributes:
assert encrypted_item[name] != plaintext_item[name]
assert decrypted_item[name] == plaintext_item[name]
# Verify that all of the attributes that should not be encrypted were not.
for name in unencrypted_attributes:
assert decrypted_item[name] == encrypted_item[name] == plaintext_item[name]
def _generate(materials_provider, table_data, ciphertext_file, metastore_info):
# pylint: disable=too-many-locals
client = boto3.client("dynamodb", region_name="us-west-2")
data_table_output = defaultdict(list)
metastore_output = defaultdict(list)
metatable = _create_meta_table(client, metastore_info)
for table_name in table_data:
table = None
try:
table_index = table_data[table_name]["index"]
table_index_types = table_data[table_name]["index_types"]
table_items = table_data[table_name]["items"]
_create_data_table(client, table_name, table_index,
table_index_types)
table = boto3.resource("dynamodb",
region_name="us-west-2").Table(table_name)
table.wait_until_exists()
cmp = materials_provider()
table_info = TableInfo(
name=table_name,
primary_index=TableIndex(partition=table_index["partition"],
sort=table_index.get("sort", None)),
)
for plaintext_item in table_items:
source_item = plaintext_item["item"]
item_key = {
table_info.primary_index.partition:
source_item[table_info.primary_index.partition]
}
if table_info.primary_index.sort is not None:
item_key[table_info.primary_index.sort] = source_item[
table_info.primary_index.sort]
attribute_actions = plaintext_item["action"]
e_table = EncryptedTable(
table=table,
materials_provider=cmp,
table_info=table_info,
attribute_actions=attribute_actions,
auto_refresh_table_indexes=False,
)
e_table.put_item(Item=ddb_to_dict(source_item))
retrieved_item = table.get_item(Key=ddb_to_dict(item_key))
parsed_item = dict_to_ddb(retrieved_item["Item"])
data_table_output[table_name].append(ddb_to_json(parsed_item))
finally:
if table:
table.delete()
with open(ciphertext_file, "w", encoding="utf-8") as outfile:
json.dump(data_table_output, outfile, indent=4)
if metatable:
# Assume exactly one entry in metastore table
wrapping_key = dict_to_ddb(metatable.scan()["Items"][0])
metastore_output[metastore_info["table_name"]].append(
ddb_to_json(wrapping_key))
metastore_ciphertext_file = _filename_from_uri(
metastore_info["ciphertext"])
with open(metastore_ciphertext_file, "w", encoding="utf-8") as outfile:
json.dump(metastore_output, outfile, indent=4)
metatable.delete()
def test_tableinfo_refresh_indexes_with_lsis(
table_with_local_seconary_indexes):
client = boto3.client("dynamodb", region_name="us-west-2")
table = TableInfo(name=TEST_TABLE_NAME)
table.refresh_indexed_attributes(client)
def test_tableinfo_refresh_indexes_no_secondary_indexes(example_table):
client = boto3.client("dynamodb", region_name="us-west-2")
table = TableInfo(name=TEST_TABLE_NAME)
table.refresh_indexed_attributes(client)
def test_tableinfo_refresh_indexes_with_gsis(
table_with_global_seconary_indexes):
client = boto3.client('dynamodb', region_name='us-west-2')
table = TableInfo(name=TEST_TABLE_NAME)
table.refresh_indexed_attributes(client)
def test_tableinfo_refresh_indexes_with_gsis(
table_with_global_secondary_indexes):
client = boto3.client("dynamodb", region_name=TEST_REGION_NAME)
table = TableInfo(name=TEST_TABLE_NAME)
table.refresh_indexed_attributes(client)
class EncryptedTable(object):
# pylint: disable=too-few-public-methods,too-many-instance-attributes
"""High-level helper class to provide a familiar interface to encrypted tables.
>>> import boto3
>>> from dynamodb_encryption_sdk.encrypted.table import EncryptedTable
>>> from dynamodb_encryption_sdk.material_providers.aws_kms import AwsKmsCryptographicMaterialsProvider
>>> table = boto3.resource('dynamodb').Table('my_table')
>>> aws_kms_cmp = AwsKmsCryptographicMaterialsProvider('alias/MyKmsAlias')
>>> encrypted_table = EncryptedTable(
... table=table,
... materials_provider=aws_kms_cmp
... )
.. note::
This class provides a superset of the boto3 DynamoDB Table API, so should work as
a drop-in replacement once configured.
https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#table
If you want to provide per-request cryptographic details, the ``put_item``, ``get_item``,
``query``, and ``scan`` methods will also accept a ``crypto_config`` parameter, defining
a custom :class:`CryptoConfig` instance for this request.
.. warning::
We do not currently support the ``update_item`` method.
:param table: Pre-configured boto3 DynamoDB Table object
:type table: boto3.resources.base.ServiceResource
:param CryptographicMaterialsProvider materials_provider: Cryptographic materials provider
to use
:param TableInfo table_info: Information about the target DynamoDB table
:param AttributeActions attribute_actions: Table-level configuration of how to encrypt/sign
attributes
:param bool auto_refresh_table_indexes: Should we attempt to refresh information about table indexes?
Requires ``dynamodb:DescribeTable`` permissions on each table. (default: True)
"""
_table = attr.ib(validator=attr.validators.instance_of(ServiceResource))
_materials_provider = attr.ib(
validator=attr.validators.instance_of(CryptographicMaterialsProvider))
_table_info = attr.ib(validator=attr.validators.optional(
attr.validators.instance_of(TableInfo)),
default=None)
_attribute_actions = attr.ib(
validator=attr.validators.instance_of(AttributeActions),
default=attr.Factory(AttributeActions))
_auto_refresh_table_indexes = attr.ib(
validator=attr.validators.instance_of(bool), default=True)
def __init__(
self,
table, # type: ServiceResource
materials_provider, # type: CryptographicMaterialsProvider
table_info=None, # type: Optional[TableInfo]
attribute_actions=None, # type: Optional[AttributeActions]
auto_refresh_table_indexes=True, # type: Optional[bool]
): # noqa=D107
# type: (...) -> None
# Workaround pending resolution of attrs/mypy interaction.
# https://github.com/python/mypy/issues/2088
# https://github.com/python-attrs/attrs/issues/215
if attribute_actions is None:
attribute_actions = AttributeActions()
self._table = table
self._materials_provider = materials_provider
self._table_info = table_info
self._attribute_actions = attribute_actions
self._auto_refresh_table_indexes = auto_refresh_table_indexes
attr.validate(self)
self.__attrs_post_init__()
def __attrs_post_init__(self):
"""Prepare table info is it was not set and set up translation methods."""
if self._table_info is None:
self._table_info = TableInfo(name=self._table.name)
if self._auto_refresh_table_indexes:
self._table_info.refresh_indexed_attributes(
self._table.meta.client)
# Clone the attribute actions before we modify them
self._attribute_actions = self._attribute_actions.copy()
self._attribute_actions.set_index_keys(
*self._table_info.protected_index_keys())
self._crypto_config = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
crypto_config_from_kwargs,
partial(crypto_config_from_table_info, self._materials_provider,
self._attribute_actions, self._table_info),
)
self.get_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
decrypt_get_item, decrypt_python_item, self._crypto_config,
self._table.get_item)
self.put_item = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
encrypt_put_item, encrypt_python_item, self._crypto_config,
self._table.put_item)
self.query = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
decrypt_multi_get, decrypt_python_item, self._crypto_config,
self._table.query)
self.scan = partial( # attrs confuses pylint: disable=attribute-defined-outside-init
decrypt_multi_get, decrypt_python_item, self._crypto_config,
self._table.scan)
def __getattr__(self, name):
"""Catch any method/attribute lookups that are not defined in this class and try
to find them on the provided bridge object.
:param str name: Attribute name
:returns: Result of asking the provided table object for that attribute name
:raises AttributeError: if attribute is not found on provided bridge object
"""
return getattr(self._table, name)
def update_item(self, **kwargs):
"""Update item is not yet supported."""
raise NotImplementedError('"update_item" is not yet implemented')
def batch_writer(self, overwrite_by_pkeys=None):
"""Create a batch writer object.
https://boto3.readthedocs.io/en/latest/reference/services/dynamodb.html#DynamoDB.Table.batch_writer
:type overwrite_by_pkeys: list(string)
:param overwrite_by_pkeys: De-duplicate request items in buffer if match new request
item on specified primary keys. i.e ``["partition_key1", "sort_key2", "sort_key3"]``
"""
encrypted_client = EncryptedClient(
client=self._table.meta.client,
materials_provider=self._materials_provider,
attribute_actions=self._attribute_actions,
auto_refresh_table_indexes=self._auto_refresh_table_indexes,
expect_standard_dictionaries=True,
)
return BatchWriter(table_name=self._table.name,
client=encrypted_client,
overwrite_by_pkeys=overwrite_by_pkeys)
