def test_caching_provider_decryption_materials_cache_expired():
"""Test decryption expiration behavior for CachingMostRecentProvider.
When using a CachingMostRecentProvider and the cache is expired on decryption, we retrieve materials
from the provider store again.
Note that this test only runs for CachingMostRecentProvider, as MostRecentProvider does not use TTL on decryption.
"""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
context = MagicMock(material_description=0)
test1 = provider.decryption_materials(context)
assert test1 is sentinel.material_0_decryption
assert len(provider._cache._cache) == 1
expected_calls = [("version_from_material_description", 0),
("get_or_create_provider", name, 0)]
assert store.provider_calls == expected_calls
test2 = provider.decryption_materials(context)
assert test2 is sentinel.material_0_decryption
assert len(provider._cache._cache) == 1
# With the cache expired, we should see another call to get_or_create_provider
expected_calls.append(("version_from_material_description", 0))
expected_calls.append(("get_or_create_provider", name, 0))
assert store.provider_calls == expected_calls
def test_encryption_materials_cache_expired():
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
test1 = provider.encryption_materials(sentinel.encryption_context_1)
assert test1 is sentinel.material_0_encryption
assert provider._version == 0
assert len(provider._cache._cache) == 1
# On the first call, we expect calls to each of the provider's APIs
expected_calls = [
("max_version", name),
("get_or_create_provider", name, 0),
("version_from_material_description", 0),
]
assert store.provider_calls == expected_calls
test2 = provider.encryption_materials(sentinel.encryption_context_1)
assert test2 is sentinel.material_0_encryption
assert provider._version == 0
assert len(provider._cache._cache) == 1
# On the second call, we don't call get_or_create because max_version matches the version in the cache.
expected_calls.append(("max_version", name))
expected_calls.append(("version_from_material_description", 0))
assert store.provider_calls == expected_calls
def test_decryption_materials_cache_use():
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=10.0)
context = MagicMock(material_description=0)
test1 = provider.decryption_materials(context)
assert test1 is sentinel.material_0_decryption
assert len(provider._cache._cache) == 1
expected_calls = [("version_from_material_description", 0),
("get_or_create_provider", name, 0)]
assert store.provider_calls == expected_calls
test2 = provider.decryption_materials(context)
assert test2 is sentinel.material_0_decryption
assert len(provider._cache._cache) == 1
expected_calls.append(("version_from_material_description", 0))
assert store.provider_calls == expected_calls
def test_encryption_materials_cache_use():
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=10.0)
test1 = provider.encryption_materials(sentinel.encryption_context_1)
assert test1 is sentinel.material_0_encryption
assert provider._version == 0
assert len(provider._cache._cache) == 1
expected_calls = [
("max_version", name),
("get_or_create_provider", name, 0),
("version_from_material_description", 0),
]
assert store.provider_calls == expected_calls
test2 = provider.encryption_materials(sentinel.encryption_context_1)
assert test2 is sentinel.material_0_encryption
assert provider._version == 0
assert len(provider._cache._cache) == 1
assert store.provider_calls == expected_calls
def check_metastore_cache_use_encrypt(metastore, table_name, log_capture):
try:
table = boto3.resource("dynamodb").Table(table_name)
except NoRegionError:
table = boto3.resource("dynamodb",
region_name=TEST_REGION_NAME).Table(table_name)
most_recent_provider = CachingMostRecentProvider(provider_store=metastore,
material_name="test",
version_ttl=600.0)
e_table = EncryptedTable(table=table,
materials_provider=most_recent_provider)
item = diverse_item()
item.update(TEST_KEY)
e_table.put_item(Item=item)
e_table.put_item(Item=item)
e_table.put_item(Item=item)
e_table.put_item(Item=item)
try:
primary_puts = _count_puts(log_capture.records, e_table.name)
metastore_puts = _count_puts(log_capture.records,
metastore._table.name)
assert primary_puts == 4
assert metastore_puts == 1
e_table.get_item(Key=TEST_KEY)
e_table.get_item(Key=TEST_KEY)
e_table.get_item(Key=TEST_KEY)
primary_gets = _count_gets(log_capture.records, e_table.name)
metastore_gets = _count_gets(log_capture.records,
metastore._table.name)
metastore_puts = _count_puts(log_capture.records,
metastore._table.name)
assert primary_gets == 3
assert metastore_gets == 0
assert metastore_puts == 1
most_recent_provider.refresh()
e_table.get_item(Key=TEST_KEY)
e_table.get_item(Key=TEST_KEY)
e_table.get_item(Key=TEST_KEY)
primary_gets = _count_gets(log_capture.records, e_table.name)
metastore_gets = _count_gets(log_capture.records,
metastore._table.name)
assert primary_gets == 6
assert metastore_gets == 1
finally:
e_table.delete_item(Key=TEST_KEY)
def test_ttl_action_not_in_cache():
"""Test that when a version is not in the cache, ttl_action returns TtlActions.EXPIRED."""
store = MagicMock(__class__=ProviderStore)
provider = CachingMostRecentProvider(provider_store=store,
material_name="my material",
version_ttl=10.0)
assert provider._last_updated is None
ttl_action = provider._ttl_action(0, "decrypt")
assert ttl_action is TtlActions.EXPIRED
def test_ttl_action_first_encrypt():
"""Test that when _last_updated has never been set, ttl_action returns TtlActions.EXPIRED."""
store = MagicMock(__class__=ProviderStore)
provider = CachingMostRecentProvider(provider_store=store,
material_name="my material",
version_ttl=10.0)
assert provider._last_updated is None
ttl_action = provider._ttl_action(0, "encrypt")
assert ttl_action is TtlActions.EXPIRED
def test_decryption_materials_cache_expired_cache_removed():
"""Tests that when the cache is expired on decrypt, we evict the entry from the cache."""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
provider._cache = MagicMock()
provider._cache.get.return_value = (0.0, MagicMock())
provider.encryption_materials(sentinel.encryption_context_1)
provider._cache.evict.assert_called_once()
def test_ttl_action_live():
"""Test that when a version is within the ttl, ttl_action returns TtlActions.LIVE."""
version = 0
store = MagicMock(__class__=ProviderStore)
provider = CachingMostRecentProvider(provider_store=store,
material_name="my material",
version_ttl=10.0)
provider._cache.put(version, (time.time(), "value"))
assert provider._last_updated is None
ttl_action = provider._ttl_action(version, "decrypt")
assert ttl_action is TtlActions.LIVE
def test_ttl_action_first_encrypt_previous_decrypt():
"""Test that on the first call to encrypt, ttl_action returns TtlActions.EXPIRED."""
version = 0
store = MagicMock(__class__=ProviderStore)
provider = CachingMostRecentProvider(provider_store=store,
material_name="my material",
version_ttl=10.0)
provider._cache.put(version, "bar")
assert provider._last_updated is None
ttl_action = provider._ttl_action(version, "encrypt")
assert ttl_action is TtlActions.EXPIRED
def test_ttl_action_expired():
"""Test that when a version is expired and not in the grace period, ttl_action returns TtlActions.EXPIRED."""
version = 0
store = MagicMock(__class__=ProviderStore)
provider = CachingMostRecentProvider(provider_store=store,
material_name="my material",
version_ttl=0.0)
provider._grace_period = 0.0
provider._cache.put(version, (time.time(), "value"))
assert provider._last_updated is None
ttl_action = provider._ttl_action(version, "decrypt")
assert ttl_action is TtlActions.EXPIRED
def test_failed_lock_acquisition():
store = MagicMock(__class__=ProviderStore)
provider = CachingMostRecentProvider(provider_store=store,
material_name="my material",
version_ttl=10.0)
provider._version = 9
provider._cache.put(provider._version, (time.time(), sentinel.nine))
with provider._lock:
test = provider._get_most_recent_version(
ttl_action=TtlActions.GRACE_PERIOD)
assert test is sentinel.nine
assert not store.mock_calls
def test_get_provider_with_grace_period_grace_period_lock_acquired():
"""Test for _get_provider_with_grace_period when entry is in grace period.
When the entry is in grace_period and we acquire the lock, we should go to the provider store
"""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
provider._get_provider_with_grace_period(sentinel.version,
TtlActions.GRACE_PERIOD)
assert len(provider._cache._cache) == 1
expected_calls = [("get_or_create_provider", name, sentinel.version)]
assert store.provider_calls == expected_calls
def _build_most_recent_cmp(scenario, keys):
table = boto3.resource("dynamodb", region_name="us-west-2").Table(scenario["metastore"]["table_name"])
meta_cmp, _, _ = _build_cmp(scenario["metastore"], keys)
metastore = MetaStore(table=table, materials_provider=meta_cmp())
most_recent_cmp = CachingMostRecentProvider(
provider_store=metastore, material_name=scenario["material_name"], version_ttl=600.0
)
return most_recent_cmp
def test_get_most_recent_version_expired():
"""Test for _get_most_recent_version when entry is expired.
When the entry is expired, we should check the cache before going to the provider store.
"""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
provider._cache = MagicMock()
provider._cache.get.return_value = (sentinel.timestamp, sentinel.provider)
test1 = provider._get_most_recent_version(TtlActions.EXPIRED)
assert test1 == sentinel.provider
expected_calls = []
assert store.provider_calls == expected_calls
def test_get_most_recent_version_grace_period_lock_acquired():
"""Test for _get_most_recent_version when entry is in grace period.
When the entry is in grace_period and we acquire the lock, we should go to the provider store
"""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
provider._get_most_recent_version(TtlActions.GRACE_PERIOD)
assert len(provider._cache._cache) == 1
expected_calls = [
("max_version", name),
("get_or_create_provider", name, 0),
("version_from_material_description", 0),
]
assert store.provider_calls == expected_calls
def test_get_most_recent_version_grace_period_lock_not_acquired():
"""Test for _get_most_recent_version when entry is in grace period.
When the entry is in grace_period and we do not acquire the lock, we should not go to the provider store
"""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
provider._cache = MagicMock()
provider._cache.get.return_value = (sentinel.timestamp, sentinel.provider)
provider._lock = MagicMock()
provider._lock.acquire.return_value = False
test = provider._get_most_recent_version(TtlActions.GRACE_PERIOD)
assert test == sentinel.provider
expected_calls = []
assert store.provider_calls == expected_calls
def test_constructor():
"""Tests that when the cache is expired on encrypt, we evict the entry from the cache."""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=1.0,
cache_size=42)
assert provider._provider_store == store
assert provider._material_name == name
assert provider._version_ttl == 1.0
assert provider._cache.capacity == 42
def test_encryption_materials_cache_in_grace_period_acquire_lock():
"""Test encryption grace period behavior.
When the TTL is GRACE_PERIOD and we successfully acquire the lock for retrieving new materials,
we call to the provider store for new materials.
"""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
provider._grace_period = 10.0
test1 = provider.encryption_materials(sentinel.encryption_context_1)
assert test1 is sentinel.material_0_encryption
assert provider._version == 0
assert len(provider._cache._cache) == 1
# On the first call, we expect calls to each of the provider's APIs
expected_calls = [
("max_version", name),
("get_or_create_provider", name, 0),
("version_from_material_description", 0),
]
assert store.provider_calls == expected_calls
provider._lock = MagicMock()
provider._lock.acquire.return_value = True
test2 = provider.encryption_materials(sentinel.encryption_context_1)
assert test2 is sentinel.material_0_encryption
assert provider._version == 0
assert len(provider._cache._cache) == 1
# On the second call, we acquired the lock so we should have tried to retrieve new materials (note no extra call
# to get_or_create_provider, because the version has not changed)
expected_calls.append(("max_version", name))
expected_calls.append(("version_from_material_description", 0))
assert store.provider_calls == expected_calls
def test_encryption_materials_cache_in_grace_period_fail_to_acquire_lock():
"""Test encryption grace period behavior.
When the TTL is GRACE_PERIOD and we fail to acquire the lock for retrieving new materials,
we use the materials from the cache.
"""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
provider._grace_period = 10.0
test1 = provider.encryption_materials(sentinel.encryption_context_1)
assert test1 is sentinel.material_0_encryption
assert provider._version == 0
assert len(provider._cache._cache) == 1
# On the first call, we expect calls to each of the provider's APIs
expected_calls = [
("max_version", name),
("get_or_create_provider", name, 0),
("version_from_material_description", 0),
]
assert store.provider_calls == expected_calls
# Now that the cache is populated, pretend the lock cannot be acquired; grace_period should allow the cached value
provider._lock = MagicMock()
provider._lock.acquire.return_value = False
test2 = provider.encryption_materials(sentinel.encryption_context_1)
assert test2 is sentinel.material_0_encryption
assert provider._version == 0
assert len(provider._cache._cache) == 1
# On the second call, we expect no additional calls because we are in our grace period.
assert store.provider_calls == expected_calls
def test_caching_provider_decryption_materials_cache_in_grace_period_fail_to_acquire_lock(
):
"""Test decryption grace period behavior for CachingMostRecentProvider.
When using a CachingMostRecentProvider and the cache is in grace period on decryption and we fail to
acquire the lock, we use materials from the cache.
Note that this test only runs for CachingMostRecentProvider, as MostRecentProvider does not use TTL on decryption.
"""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
provider._grace_period = 10.0
context = MagicMock(material_description=0)
test1 = provider.decryption_materials(context)
assert test1 is sentinel.material_0_decryption
assert len(provider._cache._cache) == 1
expected_calls = [("version_from_material_description", 0),
("get_or_create_provider", name, 0)]
assert store.provider_calls == expected_calls
# Now that the cache is populated, pretend the lock cannot be acquired; grace_period should allow the cached value
provider._lock = MagicMock()
provider._lock.acquire.return_value = False
test2 = provider.decryption_materials(context)
assert test2 is sentinel.material_0_decryption
assert len(provider._cache._cache) == 1
# Since we used the cache value, we should not see another call to get_or_create_provider
expected_calls.append(("version_from_material_description", 0))
assert store.provider_calls == expected_calls
def test_caching_provider_decryption_materials_cache_in_grace_period_acquire_lock(
):
"""Test decryption grace period behavior for CachingMostRecentProvider.
When using a CachingMostRecentProvider and the cache is in grace period on decryption and we
successfully acquire the lock, we retrieve new materials.
Note that this test only runs for CachingMostRecentProvider, as MostRecentProvider does not use TTL on decryption.
"""
store = MockProviderStore()
name = "material"
provider = CachingMostRecentProvider(provider_store=store,
material_name=name,
version_ttl=0.0)
provider._grace_period = 10.0
context = MagicMock(material_description=0)
test1 = provider.decryption_materials(context)
assert test1 is sentinel.material_0_decryption
assert len(provider._cache._cache) == 1
expected_calls = [("version_from_material_description", 0),
("get_or_create_provider", name, 0)]
assert store.provider_calls == expected_calls
provider._lock = MagicMock()
provider._lock.acquire.return_value = True
test2 = provider.decryption_materials(context)
assert test2 is sentinel.material_0_decryption
assert len(provider._cache._cache) == 1
# Since we successfully acquired the lock we should have made a new call to the provider store
expected_calls.append(("version_from_material_description", 0))
expected_calls.append(("get_or_create_provider", name, 0))
assert store.provider_calls == expected_calls
def encrypt_item(table_name, aws_cmk_id, meta_table_name, material_name):
"""Demonstrate use of EncryptedTable to transparently encrypt an item."""
index_key = {"partition_attribute": "is this", "sort_attribute": 55}
plaintext_item = {
"example": "data",
"some numbers": 99,
"and some binary": Binary(b"\x00\x01\x02"),
"leave me": "alone", # We want to ignore this attribute
}
# Collect all of the attributes that will be encrypted (used later).
encrypted_attributes = set(plaintext_item.keys())
encrypted_attributes.remove("leave me")
# Collect all of the attributes that will not be encrypted (used later).
unencrypted_attributes = set(index_key.keys())
unencrypted_attributes.add("leave me")
# Add the index pairs to the item.
plaintext_item.update(index_key)
# Create a normal table resource for the meta store.
meta_table = boto3.resource("dynamodb").Table(meta_table_name) # generated code confuse pylint: disable=no-member
# Create a crypto materials provider for the meta store using the specified AWS KMS key.
aws_kms_cmp = AwsKmsCryptographicMaterialsProvider(key_id=aws_cmk_id)
# Create a meta store using the AWS KMS crypto materials provider.
meta_store = MetaStore(table=meta_table, materials_provider=aws_kms_cmp)
# Create a most recent provider using the meta store.
most_recent_cmp = CachingMostRecentProvider(
provider_store=meta_store,
material_name=material_name,
version_ttl=
600.0, # Check for a new material version every five minutes.
)
# Create a normal table resource.
table = boto3.resource("dynamodb").Table(table_name) # generated code confuse pylint: disable=no-member
# Create attribute actions that tells the encrypted table to encrypt all attributes except one.
actions = AttributeActions(
default_action=CryptoAction.ENCRYPT_AND_SIGN,
attribute_actions={"leave me": CryptoAction.DO_NOTHING})
# Use these objects to create an encrypted table resource.
encrypted_table = EncryptedTable(table=table,
materials_provider=most_recent_cmp,
attribute_actions=actions)
# Put the item to the table, using the encrypted table resource to transparently encrypt it.
encrypted_table.put_item(Item=plaintext_item)
# Get the encrypted item using the standard table resource.
encrypted_item = table.get_item(Key=index_key)["Item"]
# Get the item using the encrypted table resource, transparently decyrpting it.
decrypted_item = encrypted_table.get_item(Key=index_key)["Item"]
# Verify that all of the attributes are different in the encrypted item
for name in encrypted_attributes:
assert encrypted_item[name] != plaintext_item[name]
assert decrypted_item[name] == plaintext_item[name]
# Verify that all of the attributes that should not be encrypted were not.
for name in unencrypted_attributes:
assert decrypted_item[name] == encrypted_item[name] == plaintext_item[
name]
# Clean up the item
encrypted_table.delete_item(Key=index_key)