def test_analyze_one_classification_result(self):
self.storage_client = fake_cloud_client.FakeStorageClient(
{'filename':
'a1.png,1\na2.png,4\na3.png,1\na4.png,1\na5.png,2\na6.png,9'})
adv_batch = {
'dataset_batch_id': 'BATCH000',
'images': {'a' + str(i):
{'clean_image_id': 'c' + str(i)} for i in range(1, 6)}
}
dataset_batches = image_batches.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name='final')
dataset_batches._data = {
'BATCH000': {'images': {'c' + str(i): {'dataset_image_id': str(i)}
for i in range(1, 6)}},
}
(count_correctly_classified, count_errors,
count_hit_target_class, num_images) = (
classification_results.analyze_one_classification_result(
self.storage_client, 'filename', adv_batch, dataset_batches,
FakeDatasetMeta()))
self.assertEqual(3, count_correctly_classified)
self.assertEqual(2, count_errors)
self.assertEqual(1, count_hit_target_class)
self.assertEqual(5, num_images)
def setUp(self):
self.storage_client = fake_cloud_client.FakeStorageClient()
self.datastore_client = fake_cloud_client.FakeDatastoreClient()
self.submissions = submissions.CompetitionSubmissions(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=ROUND_NAME)
# we only need list of submissin ids in CompetitionSubmissions for this test
self.submissions._defenses = {
'SUBD000': {},
'SUBD001': {},
}
self.adv_batches = image_batches.AversarialBatches(
datastore_client=self.datastore_client)
self.adv_batches._data = {
'ADVBATCH000': {'dataset_batch_id': 'BATCH000',
'images': {},
'submission_id': 'SUBA000'},
'ADVBATCH001': {'dataset_batch_id': 'BATCH000',
'images': {},
'submission_id': 'SUBA001'},
'ADVBATCH002': {'dataset_batch_id': 'BATCH000',
'images': {},
'submission_id': 'SUBT000'},
}
def setUp(self):
self.storage_client = fake_cloud_client.FakeStorageClient()
self.datastore_client = fake_cloud_client.FakeDatastoreClient()
self.submissions = submissions.CompetitionSubmissions(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=ROUND_NAME,
)
# we only need list of submissin ids in CompetitionSubmissions for this test
self.submissions._defenses = {
"SUBD000": {},
"SUBD001": {},
}
self.adv_batches = image_batches.AversarialBatches(
datastore_client=self.datastore_client)
self.adv_batches._data = {
"ADVBATCH000": {
"dataset_batch_id": "BATCH000",
"images": {},
"submission_id": "SUBA000",
},
"ADVBATCH001": {
"dataset_batch_id": "BATCH000",
"images": {},
"submission_id": "SUBA001",
},
"ADVBATCH002": {
"dataset_batch_id": "BATCH000",
"images": {},
"submission_id": "SUBT000",
},
}
def setUp(self):
# prepare dataset batches and submissions
storage_blobs = [
'dataset/dev/img1.png',
'dataset/dev/img2.png',
'dataset/dev/img3.png',
'dataset/dev/img4.png',
'dataset/dev/img5.png',
'dataset/dev_dataset.csv',
ROUND_NAME + '/submissions/nontargeted/1.zip',
ROUND_NAME + '/submissions/nontargeted/baseline_nt.zip',
ROUND_NAME + '/submissions/targeted/1.zip',
ROUND_NAME + '/submissions/targeted/2.zip',
ROUND_NAME + '/submissions/defense/3.zip',
ROUND_NAME + '/submissions/defense/baseline_adv_train.zip',
]
self.storage_client = fake_cloud_client.FakeStorageClient(
storage_blobs)
self.datastore_client = fake_cloud_client.FakeDatastoreClient()
self.dataset_batches = image_batches.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name='dev')
self.dataset_batches.init_from_storage_write_to_datastore(batch_size=3)
self.submissions = submissions.CompetitionSubmissions(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=ROUND_NAME)
self.submissions.init_from_storage_write_to_datastore()
def test_get_blob(self):
client = fake_cloud_client.FakeStorageClient({
'some_blob': 'some_content',
'blob2': 'another_content'
})
self.assertIsNone(client.get_blob('blob3'))
buf = BytesIO()
client.get_blob('some_blob').download_to_file(buf)
self.assertEqual(six_b('some_content'), buf.getvalue())
def test_get_blob(self):
client = fake_cloud_client.FakeStorageClient({
"some_blob": "some_content",
"blob2": "another_content"
})
self.assertIsNone(client.get_blob("blob3"))
buf = BytesIO()
client.get_blob("some_blob").download_to_file(buf)
self.assertEqual(six_b("some_content"), buf.getvalue())
def test_list_blobs(self):
all_blobs = [
'some_blob', 'dataset/dev_dataset.csv', 'dataset/dev/img1.png',
'dataset/dev/img2.png'
]
client = fake_cloud_client.FakeStorageClient(all_blobs)
assertCountEqual(self, all_blobs, client.list_blobs())
assertCountEqual(self, [
'dataset/dev_dataset.csv', 'dataset/dev/img1.png',
'dataset/dev/img2.png'
], client.list_blobs('dataset/dev'))
assertCountEqual(self,
['dataset/dev/img1.png', 'dataset/dev/img2.png'],
client.list_blobs('dataset/dev/'))
def setUp(self):
storage_blobs = [
ROUND_NAME + '/submissions/nontargeted/1.zip',
ROUND_NAME + '/submissions/nontargeted/baseline_nt.zip',
ROUND_NAME + '/submissions/targeted/1.zip',
ROUND_NAME + '/submissions/targeted/2.zip',
ROUND_NAME + '/submissions/defense/3.zip',
ROUND_NAME + '/submissions/defense/baseline_adv_train.zip',
]
self.storage_client = fake_cloud_client.FakeStorageClient(storage_blobs)
self.datastore_client = fake_cloud_client.FakeDatastoreClient()
self.submissions = submissions.CompetitionSubmissions(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=ROUND_NAME)
def setUp(self):
storage_blobs = [
'dataset/dev/img1.png',
'dataset/dev/img2.png',
'dataset/dev/img3.png',
'dataset/dev/img4.png',
'dataset/dev/img5.png',
'dataset/dev_dataset.csv',
]
self.storage_client = fake_cloud_client.FakeStorageClient(
storage_blobs)
self.datastore_client = fake_cloud_client.FakeDatastoreClient()
self.dataset_batches = image_batches.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name='dev')
def test_analyze_one_classification_result(self):
self.storage_client = fake_cloud_client.FakeStorageClient({
"filename":
"a1.png,1\na2.png,4\na3.png,1\na4.png,1\na5.png,2\na6.png,9"
})
adv_batch = {
"dataset_batch_id": "BATCH000",
"images": {
"a" + str(i): {
"clean_image_id": "c" + str(i)
}
for i in range(1, 6)
},
}
dataset_batches = image_batches.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name="final",
)
dataset_batches._data = {
"BATCH000": {
"images": {
"c" + str(i): {
"dataset_image_id": str(i)
}
for i in range(1, 6)
}
},
}
(
count_correctly_classified,
count_errors,
count_hit_target_class,
num_images,
) = classification_results.analyze_one_classification_result(
self.storage_client,
"filename",
adv_batch,
dataset_batches,
FakeDatasetMeta(),
)
self.assertEqual(3, count_correctly_classified)
self.assertEqual(2, count_errors)
self.assertEqual(1, count_hit_target_class)
self.assertEqual(5, num_images)
def test_list_blobs(self):
all_blobs = [
"some_blob",
"dataset/dev_dataset.csv",
"dataset/dev/img1.png",
"dataset/dev/img2.png",
]
client = fake_cloud_client.FakeStorageClient(all_blobs)
assertCountEqual(self, all_blobs, client.list_blobs())
assertCountEqual(
self,
[
"dataset/dev_dataset.csv", "dataset/dev/img1.png",
"dataset/dev/img2.png"
],
client.list_blobs("dataset/dev"),
)
assertCountEqual(
self,
["dataset/dev/img1.png", "dataset/dev/img2.png"],
client.list_blobs("dataset/dev/"),
)
def test_read_classification_results(self):
self.storage_client = fake_cloud_client.FakeStorageClient(
{'filename': 'img1.png,123\nimg2.jpg,456'})
results = classification_results.read_classification_results(
self.storage_client, 'filename')
self.assertDictEqual({'img1': 123, 'img2': 456}, results)
def test_read_classification_results(self):
self.storage_client = fake_cloud_client.FakeStorageClient(
{"filename": "img1.png,123\nimg2.jpg,456"})
results = classification_results.read_classification_results(
self.storage_client, "filename")
self.assertDictEqual({"img1": 123, "img2": 456}, results)
