def setUp(self): self.datastore_client = fake_cloud_client.FakeDatastoreClient() self.image_batches = image_batches.ImageBatchesBase( datastore_client=self.datastore_client, entity_kind_batches="Batch", entity_kind_images="Image", )
def test_query_fetch_all(self): entity1 = fake_cloud_client.make_entity(('abc', '1')) entity1['k1'] = 'v1' entity2 = fake_cloud_client.make_entity(('abc', '1', 'def', '2')) entity2['k2'] = 'v2' client = fake_cloud_client.FakeDatastoreClient([entity1, entity2]) assertCountEqual(self, [entity1, entity2], client.query_fetch())
def test_query_fetch_data_filter(self): entity1 = fake_cloud_client.make_entity(("abc", "1")) entity1["k1"] = "v1" entity2 = fake_cloud_client.make_entity(("abc", "2")) entity2["k1"] = "v2" entity2["k2"] = "v2" entity3 = fake_cloud_client.make_entity(("abc", "3")) entity3["k2"] = "v3" client = fake_cloud_client.FakeDatastoreClient( [entity1, entity2, entity3]) assertCountEqual(self, [entity1], client.query_fetch(filters=[("k1", "=", "v1")])) assertCountEqual(self, [entity2], client.query_fetch(filters=[("k1", ">", "v1")])) assertCountEqual(self, [entity1, entity2], client.query_fetch(filters=[("k1", ">=", "v1")])) assertCountEqual(self, [entity2], client.query_fetch(filters=[("k2", "<", "v3")])) assertCountEqual(self, [entity2, entity3], client.query_fetch(filters=[("k2", "<=", "v3")])) assertCountEqual( self, [entity2], client.query_fetch(filters=[("k1", ">=", "v1"), ("k2", "<=", "v3")]), )
def setUp(self): # prepare dataset batches and submissions storage_blobs = [ 'dataset/dev/img1.png', 'dataset/dev/img2.png', 'dataset/dev/img3.png', 'dataset/dev/img4.png', 'dataset/dev/img5.png', 'dataset/dev_dataset.csv', ROUND_NAME + '/submissions/nontargeted/1.zip', ROUND_NAME + '/submissions/nontargeted/baseline_nt.zip', ROUND_NAME + '/submissions/targeted/1.zip', ROUND_NAME + '/submissions/targeted/2.zip', ROUND_NAME + '/submissions/defense/3.zip', ROUND_NAME + '/submissions/defense/baseline_adv_train.zip', ] self.storage_client = fake_cloud_client.FakeStorageClient( storage_blobs) self.datastore_client = fake_cloud_client.FakeDatastoreClient() self.dataset_batches = image_batches.DatasetBatches( datastore_client=self.datastore_client, storage_client=self.storage_client, dataset_name='dev') self.dataset_batches.init_from_storage_write_to_datastore(batch_size=3) self.submissions = submissions.CompetitionSubmissions( datastore_client=self.datastore_client, storage_client=self.storage_client, round_name=ROUND_NAME) self.submissions.init_from_storage_write_to_datastore()
def setUp(self): self._client = fake_cloud_client.FakeDatastoreClient() self._key1 = self._client.key("abc", "def") self._key2 = self._client.key("qwe", "rty", parent=self._key1) self._key3 = self._client.key("123", "456") self._entity1 = self._client.entity(self._key1) self._entity1["k1"] = "v1" self._entity2 = self._client.entity(self._key2) self._entity2["k2"] = "v2" self._entity2["k3"] = "v3" self._entity3 = self._client.entity(self._key3) self._entity3["k4"] = "v4" self._entity3["k5"] = "v5" self._entity3["k6"] = "v6" self._client.put(self._entity1) self._client.put(self._entity2) self._client.put(self._entity3) # verify datastore content assertCountEqual(self, [self._key1, self._key2, self._key3], self._client.entities.keys()) self.assertDictEqual({"k1": "v1"}, dict(self._client.entities[self._key1])) self.assertDictEqual({ "k2": "v2", "k3": "v3" }, dict(self._client.entities[self._key2])) self.assertDictEqual( { "k4": "v4", "k5": "v5", "k6": "v6" }, dict(self._client.entities[self._key3]), )
def test_query_fetch_all(self): entity1 = fake_cloud_client.make_entity(("abc", "1")) entity1["k1"] = "v1" entity2 = fake_cloud_client.make_entity(("abc", "1", "def", "2")) entity2["k2"] = "v2" client = fake_cloud_client.FakeDatastoreClient([entity1, entity2]) assertCountEqual(self, [entity1, entity2], client.query_fetch())
def setUp(self): self.storage_client = fake_cloud_client.FakeStorageClient() self.datastore_client = fake_cloud_client.FakeDatastoreClient() self.submissions = submissions.CompetitionSubmissions( datastore_client=self.datastore_client, storage_client=self.storage_client, round_name=ROUND_NAME, ) # we only need list of submissin ids in CompetitionSubmissions for this test self.submissions._defenses = { "SUBD000": {}, "SUBD001": {}, } self.adv_batches = image_batches.AversarialBatches( datastore_client=self.datastore_client) self.adv_batches._data = { "ADVBATCH000": { "dataset_batch_id": "BATCH000", "images": {}, "submission_id": "SUBA000", }, "ADVBATCH001": { "dataset_batch_id": "BATCH000", "images": {}, "submission_id": "SUBA001", }, "ADVBATCH002": { "dataset_batch_id": "BATCH000", "images": {}, "submission_id": "SUBT000", }, }
def setUp(self): self._client = fake_cloud_client.FakeDatastoreClient() self._key1 = self._client.key('abc', 'def') self._key2 = self._client.key('qwe', 'rty', parent=self._key1) self._key3 = self._client.key('123', '456') self._entity1 = self._client.entity(self._key1) self._entity1['k1'] = 'v1' self._entity2 = self._client.entity(self._key2) self._entity2['k2'] = 'v2' self._entity2['k3'] = 'v3' self._entity3 = self._client.entity(self._key3) self._entity3['k4'] = 'v4' self._entity3['k5'] = 'v5' self._entity3['k6'] = 'v6' self._client.put(self._entity1) self._client.put(self._entity2) self._client.put(self._entity3) # verify datastore content assertCountEqual(self, [self._key1, self._key2, self._key3], self._client.entities.keys()) self.assertDictEqual({'k1': 'v1'}, dict(self._client.entities[self._key1])) self.assertDictEqual({ 'k2': 'v2', 'k3': 'v3' }, dict(self._client.entities[self._key2])) self.assertDictEqual({ 'k4': 'v4', 'k5': 'v5', 'k6': 'v6' }, dict(self._client.entities[self._key3]))
def setUp(self): self.storage_client = fake_cloud_client.FakeStorageClient() self.datastore_client = fake_cloud_client.FakeDatastoreClient() self.submissions = submissions.CompetitionSubmissions( datastore_client=self.datastore_client, storage_client=self.storage_client, round_name=ROUND_NAME) # we only need list of submissin ids in CompetitionSubmissions for this test self.submissions._defenses = { 'SUBD000': {}, 'SUBD001': {}, } self.adv_batches = image_batches.AversarialBatches( datastore_client=self.datastore_client) self.adv_batches._data = { 'ADVBATCH000': {'dataset_batch_id': 'BATCH000', 'images': {}, 'submission_id': 'SUBA000'}, 'ADVBATCH001': {'dataset_batch_id': 'BATCH000', 'images': {}, 'submission_id': 'SUBA001'}, 'ADVBATCH002': {'dataset_batch_id': 'BATCH000', 'images': {}, 'submission_id': 'SUBT000'}, }
def setUp(self): self._client = fake_cloud_client.FakeDatastoreClient() self._key1 = self._client.key("abc", "def") self._key2 = self._client.key("qwe", "rty", parent=self._key1) self._entity1 = self._client.entity(self._key1) self._entity1["k1"] = "v1" self._entity2 = self._client.entity(self._key2) self._entity2["k2"] = "v2" self._entity2["k3"] = "v3"
def setUp(self): self._client = fake_cloud_client.FakeDatastoreClient() self._key1 = self._client.key('abc', 'def') self._key2 = self._client.key('qwe', 'rty', parent=self._key1) self._entity1 = self._client.entity(self._key1) self._entity1['k1'] = 'v1' self._entity2 = self._client.entity(self._key2) self._entity2['k2'] = 'v2' self._entity2['k3'] = 'v3'
def test_query_fetch_ancestor_filter(self): entity1 = fake_cloud_client.make_entity(("abc", "1", "def", "2")) entity1["k1"] = "v1" entity2 = fake_cloud_client.make_entity(("xyz", "3", "qwe", "4")) entity2["k2"] = "v2" client = fake_cloud_client.FakeDatastoreClient([entity1, entity2]) assertCountEqual(self, [entity1], client.query_fetch(ancestor=client.key("abc", "1"))) assertCountEqual(self, [entity2], client.query_fetch(ancestor=client.key("xyz", "3")))
def test_query_fetch_ancestor_filter(self): entity1 = fake_cloud_client.make_entity(('abc', '1', 'def', '2')) entity1['k1'] = 'v1' entity2 = fake_cloud_client.make_entity(('xyz', '3', 'qwe', '4')) entity2['k2'] = 'v2' client = fake_cloud_client.FakeDatastoreClient([entity1, entity2]) assertCountEqual(self, [entity1], client.query_fetch(ancestor=client.key('abc', '1'))) assertCountEqual(self, [entity2], client.query_fetch(ancestor=client.key('xyz', '3')))
def setUp(self): self.datastore_client = fake_cloud_client.FakeDatastoreClient() self.work1 = {'submission_id': 's1', 'output_adversarial_batch_id': 'o1', 'claimed_worker_id': 'worker9999', 'claimed_worker_start_time': -1, 'is_completed': True} self.work2 = {'submission_id': 's2', 'output_adversarial_batch_id': 'o2', 'claimed_worker_id': None, 'claimed_worker_start_time': None, 'is_completed': False}
def setUp(self): storage_blobs = [ ROUND_NAME + '/submissions/nontargeted/1.zip', ROUND_NAME + '/submissions/nontargeted/baseline_nt.zip', ROUND_NAME + '/submissions/targeted/1.zip', ROUND_NAME + '/submissions/targeted/2.zip', ROUND_NAME + '/submissions/defense/3.zip', ROUND_NAME + '/submissions/defense/baseline_adv_train.zip', ] self.storage_client = fake_cloud_client.FakeStorageClient(storage_blobs) self.datastore_client = fake_cloud_client.FakeDatastoreClient() self.submissions = submissions.CompetitionSubmissions( datastore_client=self.datastore_client, storage_client=self.storage_client, round_name=ROUND_NAME)
def setUp(self): storage_blobs = [ 'dataset/dev/img1.png', 'dataset/dev/img2.png', 'dataset/dev/img3.png', 'dataset/dev/img4.png', 'dataset/dev/img5.png', 'dataset/dev_dataset.csv', ] self.storage_client = fake_cloud_client.FakeStorageClient( storage_blobs) self.datastore_client = fake_cloud_client.FakeDatastoreClient() self.dataset_batches = image_batches.DatasetBatches( datastore_client=self.datastore_client, storage_client=self.storage_client, dataset_name='dev')
def setUp(self): self.datastore_client = fake_cloud_client.FakeDatastoreClient() self.work1 = { "submission_id": "s1", "output_adversarial_batch_id": "o1", "claimed_worker_id": "worker9999", "claimed_worker_start_time": -1, "is_completed": True, } self.work2 = { "submission_id": "s2", "output_adversarial_batch_id": "o2", "claimed_worker_id": None, "claimed_worker_start_time": None, "is_completed": False, }
def test_overwrite_values(self): client = fake_cloud_client.FakeDatastoreClient() key1 = client.key("abc", "def") entity1 = client.entity(key1) entity1["k1"] = "v1" entity2 = client.entity(key1) entity2["k1"] = "v2" entity2["k2"] = "v3" with client.no_transact_batch() as batch: batch.put(entity1) assertCountEqual(self, [key1], client.entities.keys()) self.assertEqual(key1, client.entities[key1].key) self.assertDictEqual({"k1": "v1"}, dict(client.entities[key1])) with client.no_transact_batch() as batch: batch.put(entity2) assertCountEqual(self, [key1], client.entities.keys()) self.assertEqual(key1, client.entities[key1].key) self.assertDictEqual({ "k1": "v2", "k2": "v3" }, dict(client.entities[key1]))
def test_overwrite_values(self): client = fake_cloud_client.FakeDatastoreClient() key1 = client.key('abc', 'def') entity1 = client.entity(key1) entity1['k1'] = 'v1' entity2 = client.entity(key1) entity2['k1'] = 'v2' entity2['k2'] = 'v3' with client.no_transact_batch() as batch: batch.put(entity1) assertCountEqual(self, [key1], client.entities.keys()) self.assertEqual(key1, client.entities[key1].key) self.assertDictEqual({'k1': 'v1'}, dict(client.entities[key1])) with client.no_transact_batch() as batch: batch.put(entity2) assertCountEqual(self, [key1], client.entities.keys()) self.assertEqual(key1, client.entities[key1].key) self.assertDictEqual({ 'k1': 'v2', 'k2': 'v3' }, dict(client.entities[key1]))
def test_query_fetch_data_filter(self): entity1 = fake_cloud_client.make_entity(('abc', '1')) entity1['k1'] = 'v1' entity2 = fake_cloud_client.make_entity(('abc', '2')) entity2['k1'] = 'v2' entity2['k2'] = 'v2' entity3 = fake_cloud_client.make_entity(('abc', '3')) entity3['k2'] = 'v3' client = fake_cloud_client.FakeDatastoreClient( [entity1, entity2, entity3]) assertCountEqual(self, [entity1], client.query_fetch(filters=[('k1', '=', 'v1')])) assertCountEqual(self, [entity2], client.query_fetch(filters=[('k1', '>', 'v1')])) assertCountEqual(self, [entity1, entity2], client.query_fetch(filters=[('k1', '>=', 'v1')])) assertCountEqual(self, [entity2], client.query_fetch(filters=[('k2', '<', 'v3')])) assertCountEqual(self, [entity2, entity3], client.query_fetch(filters=[('k2', '<=', 'v3')])) assertCountEqual( self, [entity2], client.query_fetch(filters=[('k1', '>=', 'v1'), ('k2', '<=', 'v3')]))
def setUp(self): self.datastore_client = fake_cloud_client.FakeDatastoreClient()