def prepare_attacks(self):
"""Prepares all data needed for evaluation of attacks."""
print_header('PREPARING ATTACKS DATA')
# verify that attacks data not written yet
if not self.ask_when_work_is_populated(self.attack_work):
return
self.attack_work = eval_lib.AttackWorkPieces(
datastore_client=self.datastore_client)
# prepare submissions
print_header('Initializing submissions')
self.submissions.init_from_storage_write_to_datastore()
if self.verbose:
print(self.submissions)
# prepare dataset batches
print_header('Initializing dataset batches')
self.dataset_batches.init_from_storage_write_to_datastore(
batch_size=self.batch_size,
allowed_epsilon=ALLOWED_EPS,
skip_image_ids=[],
max_num_images=self.max_dataset_num_images)
if self.verbose:
print(self.dataset_batches)
# prepare adversarial batches
print_header('Initializing adversarial batches')
self.adv_batches.init_from_dataset_and_submissions_write_to_datastore(
dataset_batches=self.dataset_batches,
attack_submission_ids=self.submissions.get_all_attack_ids())
if self.verbose:
print(self.adv_batches)
# prepare work pieces
print_header('Preparing attack work pieces')
self.attack_work.init_from_adversarial_batches(self.adv_batches.data)
self.attack_work.write_all_to_datastore()
if self.verbose:
print(self.attack_work)
def __init__(self, worker_id, storage_client, datastore_client,
storage_bucket, round_name, dataset_name,
blacklisted_submissions='', num_defense_shards=None):
"""Initializes EvaluationWorker.
Args:
worker_id: ID of the worker
storage_client: instance of eval_lib.CompetitionStorageClient
datastore_client: instance of eval_lib.CompetitionDatastoreClient
storage_bucket: name of the Google Cloud Storage bucket where all
competition data is stored
round_name: name of the competition round
dataset_name: name of the dataset to use, typically 'dev' of 'final'
blacklisted_submissions: optional list of blacklisted submissions which
are excluded from evaluation
num_defense_shards: optional number of shards to use for evaluation of
defenses
"""
self.worker_id = int(worker_id)
self.storage_client = storage_client
self.datastore_client = datastore_client
self.storage_bucket = storage_bucket
self.round_name = round_name
self.dataset_name = dataset_name
self.blacklisted_submissions = [s.strip()
for s in blacklisted_submissions.split(',')]
if num_defense_shards:
self.num_defense_shards = int(num_defense_shards)
else:
self.num_defense_shards = None
logging.info('Number of defense shards: %s', str(self.num_defense_shards))
# init client classes
self.submissions = eval_lib.CompetitionSubmissions(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=self.round_name)
self.dataset_batches = eval_lib.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name=self.dataset_name)
self.adv_batches = eval_lib.AversarialBatches(
datastore_client=self.datastore_client)
self.attack_work = eval_lib.AttackWorkPieces(
datastore_client=self.datastore_client)
self.defense_work = eval_lib.DefenseWorkPieces(
datastore_client=self.datastore_client)
self.class_batches = eval_lib.ClassificationBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=self.round_name)
# whether data was initialized
self.attacks_data_initialized = False
self.defenses_data_initialized = False
# dataset metadata
self.dataset_meta = None
def __init__(self,
storage_client,
datastore_client,
round_name,
dataset_name,
blacklisted_submissions='',
results_dir='',
num_defense_shards=None,
verbose=False,
batch_size=DEFAULT_BATCH_SIZE,
max_dataset_num_images=None):
"""Initializes EvaluationMaster.
Args:
storage_client: instance of eval_lib.CompetitionStorageClient
datastore_client: instance of eval_lib.CompetitionDatastoreClient
round_name: name of the current round
dataset_name: name of the dataset, 'dev' or 'final'
blacklisted_submissions: optional list of blacklisted submissions which
should not be evaluated
results_dir: local directory where results and logs should be written
num_defense_shards: optional number of defense shards
verbose: whether output should be verbose on not. If True, then methods
of this class will print some additional information which is useful
for debugging.
batch_size: batch size to use
max_dataset_num_images: maximum number of images from the dataset to use
or None if entire dataset should be used.
"""
self.storage_client = storage_client
self.datastore_client = datastore_client
self.round_name = round_name
self.dataset_name = dataset_name
self.results_dir = results_dir
if num_defense_shards:
self.num_defense_shards = int(num_defense_shards)
else:
self.num_defense_shards = None
self.verbose = verbose
self.blacklisted_submissions = [
s.strip() for s in blacklisted_submissions.split(',')
]
self.batch_size = batch_size
self.max_dataset_num_images = max_dataset_num_images
# init client classes
self.submissions = eval_lib.CompetitionSubmissions(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=self.round_name)
self.dataset_batches = eval_lib.DatasetBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
dataset_name=self.dataset_name)
self.adv_batches = eval_lib.AversarialBatches(
datastore_client=self.datastore_client)
self.class_batches = eval_lib.ClassificationBatches(
datastore_client=self.datastore_client,
storage_client=self.storage_client,
round_name=self.round_name)
self.attack_work = eval_lib.AttackWorkPieces(
datastore_client=self.datastore_client)
self.defense_work = eval_lib.DefenseWorkPieces(
datastore_client=self.datastore_client)