def verify_datastore_entities(self):
# Verify 'SubmissionType' entities
assertCountEqual(self, [
self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('SubmissionType', 'Attacks')),
self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('SubmissionType',
'TargetedAttacks')),
self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('SubmissionType', 'Defenses')),
], self.datastore_client.query_fetch(kind='SubmissionType'))
# Verify 'Submission' entities
expected_submission_entities = []
for key_prefix, submission_entries in [
(('SubmissionType', 'Attacks'), self.submissions.attacks),
(('SubmissionType', 'TargetedAttacks'),
self.submissions.targeted_attacks),
(('SubmissionType', 'Defenses'), self.submissions.defenses)]:
for k, v in submission_entries.items():
entity = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey(
*(key_prefix + ('Submission', k))))
entity['submission_path'] = v.path
entity.update(v.participant_id)
expected_submission_entities.append(entity)
assertCountEqual(self, expected_submission_entities,
self.datastore_client.query_fetch(kind='Submission'))
def verify_datastore_entities(self):
# Verify 'DatasetBatch' entities
expected_batch_entities = []
for batch_id, batch in self.dataset_batches.data.items():
entity = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('DatasetBatch', batch_id))
entity['epsilon'] = batch['epsilon']
expected_batch_entities.append(entity)
assertCountEqual(
self, expected_batch_entities,
self.datastore_client.query_fetch(kind='DatasetBatch'))
# Verify 'DatasetImage' entities
expected_image_entities = []
for batch_id, batch in self.dataset_batches.data.items():
for image_id, image in batch['images'].items():
entity = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('DatasetBatch',
batch_id,
'DatasetImage',
image_id))
entity.update(image)
expected_image_entities.append(entity)
assertCountEqual(
self, expected_image_entities,
self.datastore_client.query_fetch(kind='DatasetImage'))
def test_write_to_datastore(self):
self.reset_work_pieces()
self.work_pieces.work["w1"] = self.work1
self.work_pieces.work["w2"] = self.work2
self.work_pieces.write_all_to_datastore()
# verify content of the datastore
parent_key = fake_cloud_client.FakeDatastoreKey(
work_data.KIND_WORK_TYPE, TEST_WORK_TYPE_ENTITY_ID)
assertCountEqual(
self,
[fake_cloud_client.make_entity(parent_key)],
self.datastore_client.query_fetch(kind=work_data.KIND_WORK_TYPE),
)
entity1 = fake_cloud_client.make_entity(
fake_cloud_client.FakeDatastoreKey(work_data.KIND_WORK,
"w1",
parent=parent_key))
entity1.update(self.work1)
entity2 = fake_cloud_client.make_entity(
fake_cloud_client.FakeDatastoreKey(work_data.KIND_WORK,
"w2",
parent=parent_key))
entity2.update(self.work2)
assertCountEqual(
self,
[entity1, entity2],
self.datastore_client.query_fetch(kind=work_data.KIND_WORK),
)
def test_write_single_batch_images_to_datastore(self):
# add 2 batches and 3 images, write only one batch to datastore
self.image_batches.add_batch('batch1',
batch_properties={
'k1': 'v1',
'k2': 'v2'
})
self.image_batches.add_batch('batch2', batch_properties={'k3': 'v3'})
self.image_batches.add_image('batch1',
'img1',
image_properties={'k4': 'v4'})
self.image_batches.add_image('batch1',
'img2',
image_properties={'k5': 'v5'})
self.image_batches.add_image('batch2',
'img3',
image_properties={'k6': 'v6'})
self.image_batches.write_single_batch_images_to_datastore('batch2')
# verify batches
# write_single_batch_images_to_datastore writes only images, so no batches
assertCountEqual(self, [],
self.datastore_client.query_fetch(kind='Batch'))
# verify images
img_entity3 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('Batch', 'batch2', 'Image',
'img3'))
img_entity3.update({'k6': 'v6'})
assertCountEqual(self, [img_entity3],
self.datastore_client.query_fetch(kind='Image'))
def test_deep_copy(self):
entity1 = fake_cloud_client.make_entity(('abc', '1'))
entity1['k1'] = ['v1']
self.assertEqual(entity1.key,
fake_cloud_client.FakeDatastoreKey('abc', '1'))
self.assertEqual(dict(entity1), {'k1': ['v1']})
entity2 = copy.deepcopy(entity1)
entity2['k1'].append('v2')
entity2['k3'] = 'v3'
self.assertIsInstance(entity2, fake_cloud_client.FakeDatastoreEntity)
self.assertEqual(entity1.key,
fake_cloud_client.FakeDatastoreKey('abc', '1'))
self.assertEqual(dict(entity1), {'k1': ['v1']})
self.assertEqual(entity2.key,
fake_cloud_client.FakeDatastoreKey('abc', '1'))
self.assertEqual(dict(entity2), {'k1': ['v1', 'v2'], 'k3': 'v3'})
def test_deep_copy(self):
entity1 = fake_cloud_client.make_entity(("abc", "1"))
entity1["k1"] = ["v1"]
self.assertEqual(entity1.key,
fake_cloud_client.FakeDatastoreKey("abc", "1"))
self.assertEqual(dict(entity1), {"k1": ["v1"]})
entity2 = copy.deepcopy(entity1)
entity2["k1"].append("v2")
entity2["k3"] = "v3"
self.assertIsInstance(entity2, fake_cloud_client.FakeDatastoreEntity)
self.assertEqual(entity1.key,
fake_cloud_client.FakeDatastoreKey("abc", "1"))
self.assertEqual(dict(entity1), {"k1": ["v1"]})
self.assertEqual(entity2.key,
fake_cloud_client.FakeDatastoreKey("abc", "1"))
self.assertEqual(dict(entity2), {"k1": ["v1", "v2"], "k3": "v3"})
def test_equality(self):
key1a = fake_cloud_client.FakeDatastoreKey('abc', '1')
key1b = fake_cloud_client.FakeDatastoreKey('abc', '1')
key2a = fake_cloud_client.FakeDatastoreKey('def', 'xyz', parent=key1a)
key2b = fake_cloud_client.FakeDatastoreKey('def', 'xyz', parent=key1a)
# key equal to self
self.assertTrue(key1a == key1a)
self.assertFalse(key1a != key1a)
# key equal to the same key
self.assertTrue(key1a == key1b)
self.assertFalse(key1a != key1b)
self.assertTrue(key2a == key2b)
self.assertFalse(key2a != key2b)
# key different from other key
self.assertFalse(key1a == key2a)
self.assertTrue(key1a != key2a)
# key not equal to tuple
self.assertTrue(key1a != key1a.flat_path)
self.assertFalse(key1a == key1a.flat_path)
def test_write_to_datastore(self):
# add 2 batches and 3 images, write everything to datastore
self.image_batches.add_batch('batch1',
batch_properties={
'k1': 'v1',
'k2': 'v2'
})
self.image_batches.add_batch('batch2', batch_properties={'k3': 'v3'})
self.image_batches.add_image('batch1',
'img1',
image_properties={'k4': 'v4'})
self.image_batches.add_image('batch1',
'img2',
image_properties={'k5': 'v5'})
self.image_batches.add_image('batch2',
'img3',
image_properties={'k6': 'v6'})
self.image_batches.write_to_datastore()
# verify batches
batch_entity1 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('Batch', 'batch1'))
batch_entity1.update({'k1': 'v1', 'k2': 'v2'})
batch_entity2 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('Batch', 'batch2'))
batch_entity2.update({'k3': 'v3'})
assertCountEqual(self, [batch_entity1, batch_entity2],
self.datastore_client.query_fetch(kind='Batch'))
# verify images
img_entity1 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('Batch', 'batch2', 'Image',
'img1'))
img_entity1.update({'k4': 'v4'})
img_entity2 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('Batch', 'batch2', 'Image',
'img2'))
img_entity2.update({'k5': 'v5'})
img_entity3 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey('Batch', 'batch2', 'Image',
'img3'))
img_entity3.update({'k6': 'v6'})
def test_write_to_datastore(self):
# add 2 batches and 3 images, write everything to datastore
self.image_batches.add_batch(
"batch1", batch_properties={"k1": "v1", "k2": "v2"}
)
self.image_batches.add_batch("batch2", batch_properties={"k3": "v3"})
self.image_batches.add_image("batch1", "img1", image_properties={"k4": "v4"})
self.image_batches.add_image("batch1", "img2", image_properties={"k5": "v5"})
self.image_batches.add_image("batch2", "img3", image_properties={"k6": "v6"})
self.image_batches.write_to_datastore()
# verify batches
batch_entity1 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey("Batch", "batch1")
)
batch_entity1.update({"k1": "v1", "k2": "v2"})
batch_entity2 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey("Batch", "batch2")
)
batch_entity2.update({"k3": "v3"})
assertCountEqual(
self,
[batch_entity1, batch_entity2],
self.datastore_client.query_fetch(kind="Batch"),
)
# verify images
img_entity1 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey("Batch", "batch2", "Image", "img1")
)
img_entity1.update({"k4": "v4"})
img_entity2 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey("Batch", "batch2", "Image", "img2")
)
img_entity2.update({"k5": "v5"})
img_entity3 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey("Batch", "batch2", "Image", "img3")
)
img_entity3.update({"k6": "v6"})
def test_write_single_batch_images_to_datastore(self):
# add 2 batches and 3 images, write only one batch to datastore
self.image_batches.add_batch(
"batch1", batch_properties={"k1": "v1", "k2": "v2"}
)
self.image_batches.add_batch("batch2", batch_properties={"k3": "v3"})
self.image_batches.add_image("batch1", "img1", image_properties={"k4": "v4"})
self.image_batches.add_image("batch1", "img2", image_properties={"k5": "v5"})
self.image_batches.add_image("batch2", "img3", image_properties={"k6": "v6"})
self.image_batches.write_single_batch_images_to_datastore("batch2")
# verify batches
# write_single_batch_images_to_datastore writes only images, so no batches
assertCountEqual(self, [], self.datastore_client.query_fetch(kind="Batch"))
# verify images
img_entity3 = self.datastore_client.entity(
fake_cloud_client.FakeDatastoreKey("Batch", "batch2", "Image", "img3")
)
img_entity3.update({"k6": "v6"})
assertCountEqual(
self, [img_entity3], self.datastore_client.query_fetch(kind="Image")
)
def test_key(self):
entity = fake_cloud_client.make_entity(('abc', '1'))
self.assertEqual(entity.key,
fake_cloud_client.FakeDatastoreKey('abc', '1'))
def test_flat_path(self):
key1 = fake_cloud_client.FakeDatastoreKey('abc', '1')
self.assertTupleEqual(('abc', '1'), key1.flat_path)
key2 = fake_cloud_client.FakeDatastoreKey('def', 'xyz', parent=key1)
self.assertTupleEqual(('abc', '1', 'def', 'xyz'), key2.flat_path)
def test_key(self):
entity = fake_cloud_client.make_entity(("abc", "1"))
self.assertEqual(entity.key,
fake_cloud_client.FakeDatastoreKey("abc", "1"))
def test_flat_path(self):
key1 = fake_cloud_client.FakeDatastoreKey("abc", "1")
self.assertTupleEqual(("abc", "1"), key1.flat_path)
key2 = fake_cloud_client.FakeDatastoreKey("def", "xyz", parent=key1)
self.assertTupleEqual(("abc", "1", "def", "xyz"), key2.flat_path)
