def prepare_defenses(self):
"""Prepares all data needed for evaluation of defenses."""
print_header('PREPARING DEFENSE DATA')
# verify that defense data not written yet
if not self.ask_when_work_is_populated(self.defense_work):
return
self.defense_work = eval_lib.DefenseWorkPieces(
datastore_client=self.datastore_client)
# load results of attacks
self.submissions.init_from_datastore()
self.dataset_batches.init_from_datastore()
self.adv_batches.init_from_datastore()
self.attack_work.read_all_from_datastore()
# populate classification results
print_header('Initializing classification batches')
self.class_batches.init_from_adversarial_batches_write_to_datastore(
self.submissions, self.adv_batches)
if self.verbose:
print(self.class_batches)
# populate work pieces
print_header('Preparing defense work pieces')
self.defense_work.init_from_class_batches(
self.class_batches.data, num_shards=self.num_defense_shards)
self.defense_work.write_all_to_datastore()
if self.verbose:
print(self.defense_work)
def __init__(self, worker_id, storage_client, datastore_client,
storage_bucket, round_name, dataset_name,
blacklisted_submissions='', num_defense_shards=None):
"""Initializes EvaluationWorker.
Args:
worker_id: ID of the worker
storage_client: instance of eval_lib.CompetitionStorageClient
datastore_client: instance of eval_lib.CompetitionDatastoreClient
storage_bucket: name of the Google Cloud Storage bucket where all
competition data is stored
round_name: name of the competition round
dataset_name: name of the dataset to use, typically 'dev' of 'final'
blacklisted_submissions: optional list of blacklisted submissions which
are excluded from evaluation
num_defense_shards: optional number of shards to use for evaluation of
defenses
"""
self.worker_id = int(worker_id)
self.storage_client = storage_client
self.datastore_client = datastore_client
self.storage_bucket = storage_bucket
self.round_name = round_name
self.dataset_name = dataset_name
self.blacklisted_submissions = [s.strip()
for s in blacklisted_submissions.split(',')]
if num_defense_shards:
self.num_defense_shards = int(num_defense_shards)
else:
self.num_defense_shards = None
logging.info('Number of defense shards: %s', str(self.num_defense_shards))
# init client classes
self.submissions = eval_lib.CompetitionSubmissions(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=self.round_name)
self.dataset_batches = eval_lib.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name=self.dataset_name)
self.adv_batches = eval_lib.AversarialBatches(
datastore_client=self.datastore_client)
self.attack_work = eval_lib.AttackWorkPieces(
datastore_client=self.datastore_client)
self.defense_work = eval_lib.DefenseWorkPieces(
datastore_client=self.datastore_client)
self.class_batches = eval_lib.ClassificationBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=self.round_name)
# whether data was initialized
self.attacks_data_initialized = False
self.defenses_data_initialized = False
# dataset metadata
self.dataset_meta = None
def __init__(self,
storage_client,
datastore_client,
round_name,
dataset_name,
blacklisted_submissions='',
results_dir='',
num_defense_shards=None,
verbose=False,
batch_size=DEFAULT_BATCH_SIZE,
max_dataset_num_images=None):
"""Initializes EvaluationMaster.
Args:
storage_client: instance of eval_lib.CompetitionStorageClient
datastore_client: instance of eval_lib.CompetitionDatastoreClient
round_name: name of the current round
dataset_name: name of the dataset, 'dev' or 'final'
blacklisted_submissions: optional list of blacklisted submissions which
should not be evaluated
results_dir: local directory where results and logs should be written
num_defense_shards: optional number of defense shards
verbose: whether output should be verbose on not. If True, then methods
of this class will print some additional information which is useful
for debugging.
batch_size: batch size to use
max_dataset_num_images: maximum number of images from the dataset to use
or None if entire dataset should be used.
"""
self.storage_client = storage_client
self.datastore_client = datastore_client
self.round_name = round_name
self.dataset_name = dataset_name
self.results_dir = results_dir
if num_defense_shards:
self.num_defense_shards = int(num_defense_shards)
else:
self.num_defense_shards = None
self.verbose = verbose
self.blacklisted_submissions = [
s.strip() for s in blacklisted_submissions.split(',')
]
self.batch_size = batch_size
self.max_dataset_num_images = max_dataset_num_images
# init client classes
self.submissions = eval_lib.CompetitionSubmissions(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=self.round_name)
self.dataset_batches = eval_lib.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name=self.dataset_name)
self.adv_batches = eval_lib.AversarialBatches(
datastore_client=self.datastore_client)
self.class_batches = eval_lib.ClassificationBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=self.round_name)
self.attack_work = eval_lib.AttackWorkPieces(
datastore_client=self.datastore_client)
self.defense_work = eval_lib.DefenseWorkPieces(
datastore_client=self.datastore_client)