class Cookie(Model):
"""
A "Cookie" is a representation of a data object's iteratively enriched metadata.
"""
def __init__(self, identifier: str):
from cookiemonster.common.collections import EnrichmentCollection
self.identifier = identifier
self.enrichments = EnrichmentCollection()
def enrich(self, enrichment: Enrichment):
"""
Enrich this cookie.
:param enrichment: the enrichment
"""
self.enrichments.add(enrichment)
def setUp(self):
self.enrichment_collection = EnrichmentCollection()
interesting_modifications = [
DataObjectModification(IrodsMetadata({_METADATA_KEY: {0}})),
DataObjectModification(IrodsMetadata({_METADATA_KEY: {1}}))
]
encoder = DataObjectModificationJSONEncoder()
self.interesting_modifications_as_metadata = [
Metadata(encoder.default(modification)) for modification in interesting_modifications
]
interesting_data_objects = [
DataObject("/path-1", metadata=IrodsMetadata({_METADATA_KEY: {2}})),
DataObject("/path-2", metadata=IrodsMetadata({_METADATA_KEY: {3}}))
]
encoder = DataObjectJSONEncoder()
self.interesting_data_objects_as_metadata = [
Metadata(encoder.default(data_object)) for data_object in interesting_data_objects
]
class TestWasCreationObserved(unittest.TestCase):
"""
Tests for `was_creation_observed`.
"""
def setUp(self):
self.enrichment_collection = EnrichmentCollection()
def test_no_enrichments(self):
self.assertFalse(was_creation_observed(self.enrichment_collection))
def test_unrelated_enrichment(self):
self.enrichment_collection.add(Enrichment("other", datetime(1, 1, 1), Metadata()))
self.assertFalse(was_creation_observed(self.enrichment_collection))
def test_no_creation_observed(self):
modified_replicas = DataObjectReplicaCollection(
{DataObjectReplica(IRODS_FIRST_REPLICA_TO_BE_CREATED_VALUE + 1, "")})
unrelated_modification = DataObjectModification(IrodsMetadata({"other": {"value"}}), modified_replicas)
unrelated_modification_as_dict = DataObjectModificationJSONEncoder().default(unrelated_modification)
self.enrichment_collection.add([
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(1, 1, 1), UNINTERESTING_DATA_OBJECT_MODIFICATION_AS_METADATA),
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(2, 2, 2), Metadata(unrelated_modification_as_dict)),
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(3, 3, 3), UNINTERESTING_DATA_OBJECT_MODIFICATION_AS_METADATA),
])
self.assertFalse(was_creation_observed(self.enrichment_collection))
def test_creation_observed(self):
modified_replicas = DataObjectReplicaCollection(
{DataObjectReplica(IRODS_FIRST_REPLICA_TO_BE_CREATED_VALUE, "")})
modification = DataObjectModification(IrodsMetadata({"other": {"value"}}), modified_replicas)
modification_as_dict = DataObjectModificationJSONEncoder().default(modification)
self.enrichment_collection.add([
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(1, 1, 1), UNINTERESTING_DATA_OBJECT_MODIFICATION_AS_METADATA),
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(2, 2, 2), Metadata(modification_as_dict)),
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(3, 3, 3), UNINTERESTING_DATA_OBJECT_MODIFICATION_AS_METADATA),
])
self.assertTrue(was_creation_observed(self.enrichment_collection))
class TestExtractLatestMetadataKeyValueKnownInIrods(unittest.TestCase):
"""
Tests for `extract_latest_metadata_key_value_known_in_irods`.
"""
def setUp(self):
self.enrichment_collection = EnrichmentCollection()
interesting_modifications = [
DataObjectModification(IrodsMetadata({_METADATA_KEY: {0}})),
DataObjectModification(IrodsMetadata({_METADATA_KEY: {1}}))
]
encoder = DataObjectModificationJSONEncoder()
self.interesting_modifications_as_metadata = [
Metadata(encoder.default(modification)) for modification in interesting_modifications
]
interesting_data_objects = [
DataObject("/path-1", metadata=IrodsMetadata({_METADATA_KEY: {2}})),
DataObject("/path-2", metadata=IrodsMetadata({_METADATA_KEY: {3}}))
]
encoder = DataObjectJSONEncoder()
self.interesting_data_objects_as_metadata = [
Metadata(encoder.default(data_object)) for data_object in interesting_data_objects
]
def test_no_enrichments(self):
self.assertIsNone(extract_latest_metadata_key_value_known_in_irods(self.enrichment_collection, _METADATA_KEY))
def test_no_irods_related_enrichments(self):
self.enrichment_collection.add(Enrichment("other", datetime(1, 1, 1), Metadata()))
self.assertIsNone(extract_latest_metadata_key_value_known_in_irods(self.enrichment_collection, _METADATA_KEY))
def test_not_in_irods_update_enrichment(self):
enrichment = Enrichment(
IRODS_UPDATE_ENRICHMENT, datetime(1, 1, 1), Metadata(UNINTERESTING_DATA_OBJECT_MODIFICATION_AS_METADATA))
self.enrichment_collection.add(enrichment)
self.assertIsNone(extract_latest_metadata_key_value_known_in_irods(self.enrichment_collection, _METADATA_KEY))
def test_not_in_irods_enrichment(self):
enrichment = Enrichment(IRODS_ENRICHMENT, datetime(1, 1, 1), UNINTERESTING_DATA_OBJECT_AS_METADATA)
self.enrichment_collection.add(enrichment)
self.assertIsNone(extract_latest_metadata_key_value_known_in_irods(self.enrichment_collection, _METADATA_KEY))
def test_not_in_irods_enrichment_or_irods_update(self):
enrichments = [
Enrichment(IRODS_ENRICHMENT, datetime(1, 1, 1), UNINTERESTING_DATA_OBJECT_AS_METADATA),
Enrichment(
IRODS_UPDATE_ENRICHMENT, datetime(2, 2, 2),
Metadata(UNINTERESTING_DATA_OBJECT_MODIFICATION_AS_METADATA))
]
self.enrichment_collection.add(enrichments)
self.assertIsNone(extract_latest_metadata_key_value_known_in_irods(self.enrichment_collection, _METADATA_KEY))
def test_in_irods_update_enrichments(self):
enrichments = [
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(1, 1, 1), self.interesting_modifications_as_metadata[0]),
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(2, 2, 2), self.interesting_modifications_as_metadata[1]),
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(3, 3, 3), UNINTERESTING_DATA_OBJECT_MODIFICATION_AS_METADATA),
]
self.enrichment_collection.add(enrichments)
value = extract_latest_metadata_key_value_known_in_irods(self.enrichment_collection, _METADATA_KEY)
self.assertEqual(len(list(value)), 1)
self.assertIn(1, value)
def test_in_irods_enrichments(self):
enrichments = [
Enrichment(IRODS_ENRICHMENT, datetime(1, 1, 1), self.interesting_data_objects_as_metadata[0]),
Enrichment(IRODS_ENRICHMENT, datetime(2, 2, 2), self.interesting_data_objects_as_metadata[1]),
Enrichment(IRODS_ENRICHMENT, datetime(3, 3, 3), UNINTERESTING_DATA_OBJECT_AS_METADATA),
]
self.enrichment_collection.add(enrichments)
value = extract_latest_metadata_key_value_known_in_irods(self.enrichment_collection, _METADATA_KEY)
self.assertEqual(len(list(value)), 1)
self.assertIn(3, value)
def test_in_irods_enrichments_and_irods_update_enrichments_when_most_recent_from_irods_enrichment(self):
enrichments = [
Enrichment(IRODS_ENRICHMENT, datetime(1, 1, 1), self.interesting_data_objects_as_metadata[0]),
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(2, 2, 2), self.interesting_modifications_as_metadata[0]),
Enrichment(IRODS_ENRICHMENT, datetime(3, 3, 3), UNINTERESTING_DATA_OBJECT_AS_METADATA),
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(4, 4, 4), self.interesting_modifications_as_metadata[1]),
Enrichment(IRODS_ENRICHMENT, datetime(5, 5, 5), self.interesting_data_objects_as_metadata[1])
]
self.enrichment_collection.add(enrichments)
value = extract_latest_metadata_key_value_known_in_irods(self.enrichment_collection, _METADATA_KEY)
self.assertEqual(len(list(value)), 1)
self.assertIn(3, value)
def test_in_irods_enrichments_and_irods_update_enrichments_when_most_recent_from_irods_update_enrichment(self):
enrichments = [
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(1, 1, 1), self.interesting_modifications_as_metadata[0]),
Enrichment(IRODS_ENRICHMENT, datetime(2, 2, 2), self.interesting_data_objects_as_metadata[0]),
Enrichment(IRODS_ENRICHMENT, datetime(3, 3, 3), UNINTERESTING_DATA_OBJECT_AS_METADATA),
Enrichment(IRODS_ENRICHMENT, datetime(4, 4, 4), self.interesting_data_objects_as_metadata[1]),
Enrichment(IRODS_UPDATE_ENRICHMENT, datetime(5, 5, 5), self.interesting_modifications_as_metadata[1])
]
self.enrichment_collection.add(enrichments)
value = extract_latest_metadata_key_value_known_in_irods(self.enrichment_collection, _METADATA_KEY)
self.assertEqual(len(list(value)), 1)
self.assertIn(1, value)
def setUp(self):
self.enrichment_collection = EnrichmentCollection()
class TestEnrichmentCollection(unittest.TestCase):
"""
Tests for `EnrichmentCollection`.
"""
def setUp(self):
self.enrichments = EnrichmentCollection()
def test_enrichment_no_diff(self):
self.enrichments.add(Enrichment('source', datetime(1, 1, 1), Metadata()))
self.enrichments.add(Enrichment('source', datetime(2, 2, 2), Metadata()))
diffs = get_enrichment_changes_from_source(self.enrichments, 'source')
self.assertEqual(len(diffs), 0)
def test_enrichment_diff(self):
self.enrichments.add(Enrichment('source', datetime(1, 1, 1), Metadata({'foo': 123, 'bar': 456, 'quux': 789})))
self.enrichments.add(Enrichment('source', datetime(2, 2, 2), Metadata({'xyz': 123, 'bar': 456, 'quux': 999})))
diffs = get_enrichment_changes_from_source(self.enrichments, 'source')
self.assertEqual(len(diffs), 1)
diff = diffs[0]
self.assertIsInstance(diff, EnrichmentDiff)
self.assertEqual(diff.source, 'source')
self.assertEqual(diff.timestamp, datetime(2, 2, 2))
self.assertTrue(diff.is_different())
self.assertEqual(diff.additions, Metadata({'xyz': 123, 'quux': 999}))
self.assertEqual(diff.deletions, Metadata({'foo': 123, 'quux': 789}))
def test_enrichment_diff_by_key(self):
self.enrichments.add(Enrichment('source', datetime(1, 1, 1), Metadata({'foo': 123, 'bar': 456, 'quux': 789})))
self.enrichments.add(Enrichment('source', datetime(2, 2, 2), Metadata({'xyz': 123, 'bar': 456, 'quux': 999})))
diffs = get_enrichment_changes_from_source(self.enrichments, 'source', 'foo')
self.assertEqual(len(diffs), 1)
diff = diffs[0]
self.assertEqual(diff.additions, Metadata())
self.assertEqual(diff.deletions, Metadata({'foo': 123}))
def test_enrichment_diff_by_keys(self):
self.enrichments.add(Enrichment('source', datetime(1, 1, 1), Metadata({'foo': 123, 'bar': 456, 'quux': 789})))
self.enrichments.add(Enrichment('source', datetime(2, 2, 2), Metadata({'xyz': 123, 'bar': 456, 'quux': 999})))
diffs = get_enrichment_changes_from_source(self.enrichments, 'source', ['bar', 'quux'])
self.assertEqual(len(diffs), 1)
diff = diffs[0]
self.assertEqual(diff.additions, Metadata({'quux': 999}))
self.assertEqual(diff.deletions, Metadata({'quux': 789}))
def test_enrichment_diff_from_timestamp(self):
self.enrichments.add(Enrichment('source', datetime(1, 1, 1), Metadata()))
self.enrichments.add(Enrichment('source', datetime(2, 2, 2), Metadata({'foo': 123})))
self.enrichments.add(Enrichment('source', datetime(3, 3, 3), Metadata({'bar': 123})))
all_diffs = get_enrichment_changes_from_source(self.enrichments, 'source')
self.assertEqual(len(all_diffs), 2)
since_diffs = get_enrichment_changes_from_source(self.enrichments, 'source', since=datetime(3, 3, 3))
self.assertEqual(len(since_diffs), 1)
diff = since_diffs[0]
self.assertEqual(diff.timestamp, datetime(3, 3, 3))
self.assertEqual(diff.additions, Metadata({'bar': 123}))
self.assertEqual(diff.deletions, Metadata({'foo': 123}))
def __init__(self, identifier: str):
from cookiemonster.common.collections import EnrichmentCollection
self.identifier = identifier
self.enrichments = EnrichmentCollection()
class TestEnrichmentCollection(unittest.TestCase):
"""
Tests for `EnrichmentCollection`.
"""
assert _ENRICHMENT_1.timestamp < _ENRICHMENT_2.timestamp < _ENRICHMENT_3.timestamp
def setUp(self):
self.enrichments = EnrichmentCollection()
def test_eq_with_none(self):
self.assertNotEqual(_ENRICHMENT_1, None)
def test_eq_with_other_object_type(self):
self.assertNotEqual(_ENRICHMENT_1, object())
def test_eq_with_different_enrichment(self):
self.assertNotEqual(_ENRICHMENT_1, _ENRICHMENT_2)
def test_eq_with_same_enrichment(self):
self.assertEqual(_ENRICHMENT_1, deepcopy(_ENRICHMENT_1))
def test_eq_with_self(self):
self.assertEqual(_ENRICHMENT_1, _ENRICHMENT_1)
def test_init_with_enrichments(self):
enrichments = EnrichmentCollection([_ENRICHMENT_2, _ENRICHMENT_3, _ENRICHMENT_1])
self.assertSequenceEqual(enrichments, [_ENRICHMENT_1, _ENRICHMENT_2, _ENRICHMENT_3])
def test_add_with_no_previous(self):
self.enrichments.add(_ENRICHMENT_1)
self.assertSequenceEqual(self.enrichments, [_ENRICHMENT_1])
def test_add_with_older_previous(self):
self.enrichments.add(_ENRICHMENT_1)
self.enrichments.add(_ENRICHMENT_2)
self.assertSequenceEqual(self.enrichments, [_ENRICHMENT_1, _ENRICHMENT_2])
def test_add_with_newer_previous(self):
self.enrichments.add(_ENRICHMENT_2)
self.enrichments.add(_ENRICHMENT_1)
self.assertSequenceEqual(self.enrichments, [_ENRICHMENT_1, _ENRICHMENT_2])
def test_add_with_older_and_newer_previous(self):
self.enrichments.add(_ENRICHMENT_1)
self.enrichments.add(_ENRICHMENT_3)
self.enrichments.add(_ENRICHMENT_2)
self.assertSequenceEqual(self.enrichments, [_ENRICHMENT_1, _ENRICHMENT_2, _ENRICHMENT_3])
def test_add_multiple(self):
self.enrichments.add([_ENRICHMENT_3, _ENRICHMENT_1, _ENRICHMENT_2])
self.assertSequenceEqual(self.enrichments, [_ENRICHMENT_1, _ENRICHMENT_2, _ENRICHMENT_3])
def test_get_most_recent_from_source_when_none_from_source(self):
self.assertIsNone(self.enrichments.get_most_recent_from_source("other_source"))
def test_get_most_recent_from_source_when_multiple_from_source(self):
assert _ENRICHMENT_1.source == _ENRICHMENT_2.source == _ENRICHMENT_3.source
self.enrichments.add([_ENRICHMENT_1, _ENRICHMENT_2, _ENRICHMENT_3])
self.assertEqual(self.enrichments.get_most_recent_from_source(_ENRICHMENT_1.source), _ENRICHMENT_3)
def test_get_all_since_enrichment_from_source_when_no_enrichments(self):
enrichments = self.enrichments.get_all_since_enrichment_from_source(_ENRICHMENT_1.source)
self.assertEqual(len(enrichments), 0)
def test_get_all_since_enrichment_from_source_when_no_enrichments_from_source(self):
self.enrichments.add(_ENRICHMENT_1)
enrichments = self.enrichments.get_all_since_enrichment_from_source("other_source")
self.assertCountEqual(enrichments, [_ENRICHMENT_1])
def test_get_all_since_enrichment_from_source_when_only_enrichment_from_source(self):
assert _ENRICHMENT_1.source == _ENRICHMENT_2.source
self.enrichments.add([_ENRICHMENT_1, _ENRICHMENT_2])
enrichments = self.enrichments.get_all_since_enrichment_from_source(_ENRICHMENT_1.source)
self.assertEqual(len(enrichments), 0)
def test_get_all_since_enrichment_from_source_when_no_after_enrichment_from_source(self):
assert _ENRICHMENT_1.source == _ENRICHMENT_2.source
self.enrichments.add([_ENRICHMENT_1, Enrichment("other_source", _ENRICHMENT_2.timestamp, Metadata())])
enrichments = self.enrichments.get_all_since_enrichment_from_source("other_source")
self.assertEqual(len(enrichments), 0)
def test_get_all_since_enrichment_from_source_when_one_enrichment_from_source_and_enrichments_afterwards(self):
delta = timedelta(seconds=1)
assert _ENRICHMENT_2.timestamp - _ENRICHMENT_1.timestamp > delta
self.enrichments.add([
_ENRICHMENT_1,
Enrichment("other_source", _ENRICHMENT_1.timestamp + delta, Metadata()),
_ENRICHMENT_2,
_ENRICHMENT_3
])
enrichments = self.enrichments.get_all_since_enrichment_from_source("other_source")
self.assertCountEqual(enrichments, [_ENRICHMENT_2, _ENRICHMENT_3])
def test_get_all_since_enrichment_from_source_when_multiple_enrichment_from_source_and_enrichments_afterwards(self):
delta = timedelta(seconds=1)
assert _ENRICHMENT_2.timestamp - _ENRICHMENT_1.timestamp > delta
assert _ENRICHMENT_3.timestamp - _ENRICHMENT_2.timestamp > delta
source = "other_source"
self.enrichments.add([
_ENRICHMENT_1,
Enrichment(source, _ENRICHMENT_1.timestamp + delta, Metadata()),
_ENRICHMENT_2,
Enrichment(source, _ENRICHMENT_2.timestamp + delta, Metadata()),
_ENRICHMENT_3
])
enrichments = self.enrichments.get_all_since_enrichment_from_source(source)
self.assertCountEqual(enrichments, [_ENRICHMENT_3])
