def test_analyze_one_classification_result(self):
self.storage_client = fake_cloud_client.FakeStorageClient(
{'filename':
'a1.png,1\na2.png,4\na3.png,1\na4.png,1\na5.png,2\na6.png,9'})
adv_batch = {
'dataset_batch_id': 'BATCH000',
'images': {'a' + str(i):
{'clean_image_id': 'c' + str(i)} for i in range(1, 6)}
}
dataset_batches = image_batches.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name='final')
dataset_batches._data = {
'BATCH000': {'images': {'c' + str(i): {'dataset_image_id': str(i)}
for i in range(1, 6)}},
}
(count_correctly_classified, count_errors,
count_hit_target_class, num_images) = (
classification_results.analyze_one_classification_result(
self.storage_client, 'filename', adv_batch, dataset_batches,
FakeDatasetMeta()))
self.assertEqual(3, count_correctly_classified)
self.assertEqual(2, count_errors)
self.assertEqual(1, count_hit_target_class)
self.assertEqual(5, num_images)
def setUp(self):
# prepare dataset batches and submissions
storage_blobs = [
'dataset/dev/img1.png',
'dataset/dev/img2.png',
'dataset/dev/img3.png',
'dataset/dev/img4.png',
'dataset/dev/img5.png',
'dataset/dev_dataset.csv',
ROUND_NAME + '/submissions/nontargeted/1.zip',
ROUND_NAME + '/submissions/nontargeted/baseline_nt.zip',
ROUND_NAME + '/submissions/targeted/1.zip',
ROUND_NAME + '/submissions/targeted/2.zip',
ROUND_NAME + '/submissions/defense/3.zip',
ROUND_NAME + '/submissions/defense/baseline_adv_train.zip',
]
self.storage_client = fake_cloud_client.FakeStorageClient(
storage_blobs)
self.datastore_client = fake_cloud_client.FakeDatastoreClient()
self.dataset_batches = image_batches.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name='dev')
self.dataset_batches.init_from_storage_write_to_datastore(batch_size=3)
self.submissions = submissions.CompetitionSubmissions(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=ROUND_NAME)
self.submissions.init_from_storage_write_to_datastore()
def test_init_from_datastore(self):
self.dataset_batches.init_from_storage_write_to_datastore(batch_size=3)
self.dataset_batches = image_batches.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name='dev')
self.dataset_batches.init_from_datastore()
self.verify_dataset_batches()
def setUp(self):
storage_blobs = [
'dataset/dev/img1.png',
'dataset/dev/img2.png',
'dataset/dev/img3.png',
'dataset/dev/img4.png',
'dataset/dev/img5.png',
'dataset/dev_dataset.csv',
]
self.storage_client = fake_cloud_client.FakeStorageClient(
storage_blobs)
self.datastore_client = fake_cloud_client.FakeDatastoreClient()
self.dataset_batches = image_batches.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name='dev')
def test_analyze_one_classification_result(self):
self.storage_client = fake_cloud_client.FakeStorageClient({
"filename":
"a1.png,1\na2.png,4\na3.png,1\na4.png,1\na5.png,2\na6.png,9"
})
adv_batch = {
"dataset_batch_id": "BATCH000",
"images": {
"a" + str(i): {
"clean_image_id": "c" + str(i)
}
for i in range(1, 6)
},
}
dataset_batches = image_batches.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name="final",
)
dataset_batches._data = {
"BATCH000": {
"images": {
"c" + str(i): {
"dataset_image_id": str(i)
}
for i in range(1, 6)
}
},
}
(
count_correctly_classified,
count_errors,
count_hit_target_class,
num_images,
) = classification_results.analyze_one_classification_result(
self.storage_client,
"filename",
adv_batch,
dataset_batches,
FakeDatasetMeta(),
)
self.assertEqual(3, count_correctly_classified)
self.assertEqual(2, count_errors)
self.assertEqual(1, count_hit_target_class)
self.assertEqual(5, num_images)
