print("MODEL CONFIGS:", args.model_configs)
print("DATA CONFIGS:", args.data_configs)
print("ATTACK CONFIGS:", args.attack_configs)
print("OUTPUT ROOT:", args.output_root)
print("DEBUGGING MODE:", args.debug)
print('----------------------------\n')
# parse configurations (into a dictionary) from json file
model_configs = load_from_json(args.model_configs)
data_configs = load_from_json(args.data_configs)
attack_configs = load_from_json(args.attack_configs)
# load the targeted model
model_file = os.path.join(model_configs.get("dir"),
model_configs.get("um_file"))
target = load_lenet(file=model_file, wrap=True)
# load the benign samples
data_file = os.path.join(data_configs.get('dir'),
data_configs.get('bs_file'))
data_bs = np.load(data_file)
# load the corresponding true labels
label_file = os.path.join(data_configs.get('dir'),
data_configs.get('label_file'))
labels = np.load(label_file)
# generate adversarial examples for a small subset
data_bs = data_bs[:100]
labels = labels[:100]
generate_ae(model=target,
def evaluate(trans_configs, model_configs,
data_configs, save=True, output_dir=None):
"""
Apply transformation(s) on images.
:param trans_configs: dictionary. The collection of the parameterized transformations to test.
in the form of
{ configsx: {
param: value,
}
}
The key of a configuration is 'configs'x, where 'x' is the id of corresponding weak defense.
:param model_configs: dictionary. Defines model related information.
Such as, location, the undefended model, the file format, etc.
:param data_configs: dictionary. Defines data related information.
Such as, location, the file for the true labels, the file for the benign samples,
the files for the adversarial examples, etc.
:param save: boolean. Save the transformed sample or not.
:param output_dir: path or str. The location to store the transformed samples.
It cannot be None when save is True.
:return:
"""
# Load the baseline defense (PGD-ADT model)
baseline = load_lenet(file=model_configs.get('pgd_trained'), trans_configs=None,
use_logits=False, wrap=False)
# get the undefended model (UM)
file = os.path.join(model_configs.get('dir'), model_configs.get('um_file'))
undefended = load_lenet(file=file,
trans_configs=trans_configs.get('configs0'),
wrap=True)
print(">>> um:", type(undefended))
# load weak defenses into a pool
pool, _ = load_pool(trans_configs=trans_configs,
model_configs=model_configs,
active_list=True,
wrap=True)
# create an AVEP ensemble from the WD pool
wds = list(pool.values())
print(">>> wds:", type(wds), type(wds[0]))
ensemble = Ensemble(classifiers=wds, strategy=ENSEMBLE_STRATEGY.AVEP.value)
# load the benign samples
bs_file = os.path.join(data_configs.get('dir'), data_configs.get('bs_file'))
x_bs = np.load(bs_file)
img_rows, img_cols = x_bs.shape[1], x_bs.shape[2]
# load the corresponding true labels, take just the first 1000
label_file = os.path.join(data_configs.get('dir'), data_configs.get('label_file'))
labels = np.load(label_file)
labels = labels[:1000]
# get indices of benign samples that are correctly classified by the targeted model
print(">>> Evaluating UM on [{}], it may take a while...".format(bs_file))
pred_bs = undefended.predict(x_bs)
corrections = get_corrections(y_pred=pred_bs, y_true=labels)
if save:
if output_dir is None:
raise ValueError("Cannot save to a none path.")
# save with a random name
f = os.path.join(output_dir, "minerva_AE-results.txt")
out_file = open(f, 'w')
# Evaluate AEs.
ae_list = data_configs.get('ae_files')
for _ in range(len(ae_list)):
results = {}
ae_file = os.path.join(data_configs.get('dir'), ae_list[_])
print(ae_list[_])
print(ae_file)
x_adv = np.load(ae_file)
# evaluate the undefended model on the AE
print(">>> Evaluating UM on [{}], it may take a while...".format(ae_file))
pred_adv_um = undefended.predict(x_adv)
err_um = error_rate(y_pred=pred_adv_um, y_true=labels, correct_on_bs=corrections)
# track the result
results['UM'] = err_um
# evaluate the ensemble on the AE
print(">>> Evaluating ensemble on [{}], it may take a while...".format(ae_file))
pred_adv_ens = ensemble.predict(x_adv)
err_ens = error_rate(y_pred=pred_adv_ens, y_true=labels, correct_on_bs=corrections)
# track the result
results['Ensemble'] = err_ens
# evaluate the baseline on the AE
print(">>> Evaluating baseline model on [{}], it may take a while...".format(ae_file))
pred_adv_bl = baseline.predict(x_adv)
err_bl = error_rate(y_pred=pred_adv_bl, y_true=labels, correct_on_bs=corrections)
# track the result
results['PGD-ADT'] = err_bl
out_file.write(">>> Evaluations on [{}]:\n{}\n".format(ae_file, results))
def evaluate(trans_configs,
trans_configs2,
model_configs,
data_configs,
save=True,
output_dir='../results'):
"""
Apply transformation(s) on images.
:param trans_configs: dictionary. The collection of the parameterized transformations to test.
in the form of
{ configsx: {
param: value,
}
}
The key of a configuration is 'configs'x, where 'x' is the id of corresponding weak defense.
:param model_configs: dictionary. Defines model related information.
Such as, location, the undefended model, the file format, etc.
:param data_configs: dictionary. Defines data related information.
Such as, location, the file for the true labels, the file for the benign samples,
the files for the adversarial examples, etc.
:param save: boolean. Save the transformed sample or not.
:param output_dir: path or str. The location to store the transformed samples.
It cannot be None when save is True.
:return:
"""
# Load the baseline defense (PGD-ADT model)
baseline = load_lenet(file=model_configs.get('pgd_trained'),
trans_configs=None,
use_logits=False,
wrap=False)
# get the undefended model (UM)
cnn_configs = model_configs.get('cnn')
file = os.path.join(cnn_configs.get('dir'), cnn_configs.get('um_file'))
undefended = load_lenet(file=file,
trans_configs=trans_configs.get('configs0'),
wrap=True)
print(">>> um:", type(undefended))
# load weak defenses into a pool
cnn_pool, _ = load_pool(trans_configs=trans_configs,
model_configs=cnn_configs,
active_list=True,
wrap=True)
# create an AVEP ensemble from the WD pool
cnns = list(cnn_pool.values())
# load SVM weak defenses into a pool
# tiny pool: 3 weak defenses
svm_configs = model_configs.get('svm')
svm_pool, _ = load_pool(trans_configs=trans_configs2,
model_configs=svm_configs,
active_list=True,
wrap=True)
svms = list(svm_pool.values())
wds = cnns
wds.extend(svms)
ensemble = Ensemble(classifiers=wds, strategy=ENSEMBLE_STRATEGY.AVEP.value)
# load the benign samples
bs_file = os.path.join(data_configs.get('dir'),
data_configs.get('bs_file'))
x_bs = np.load(bs_file)
img_rows, img_cols = x_bs.shape[1], x_bs.shape[2]
# load the corresponding true labels, take just the first 1000
label_file = os.path.join(data_configs.get('dir'),
data_configs.get('label_file'))
labels = np.load(label_file)
labels = labels[:1000]
# get indices of benign samples that are correctly classified by the targeted model
print(">>> Evaluating UM on [{}], it may take a while...".format(bs_file))
pred_bs = undefended.predict(x_bs)
corrections = get_corrections(y_pred=pred_bs, y_true=labels)
if save:
if output_dir is None:
raise ValueError("Cannot save to a none path.")
# save with a random name
f = os.path.join(output_dir, "minerva_AE_rand_eval_results.txt")
out_file = open(f, 'a')
out_file.write(
'--------------------------------------NEW RANDOM TEST--------------------------------------\n'
)
out_file.write(
'| |\n'
)
out_file.write(
'| NEW TEST DATA WITH THE FOLLOWING {c} RANDOM CNN\'S AND {s} RANDOM SVM\'S \
|\n'.format(c=len(cnns), s=len(svms)))
out_file.write(
'CNNs: {c}\n'.format(c=list(trans_configs.get('active_wds'))))
out_file.write(
'SVMS: {s}\n'.format(s=list(trans_configs2.get('active_wds'))))
out_file.write('\n\n')
# Evaluate AEs.
ae_list = data_configs.get('ae_files')
start = time.time()
for _ in range(len(ae_list)):
ae_start = time.time()
results = {}
ae_file = os.path.join(data_configs.get('dir'), ae_list[_])
x_adv = np.load(ae_file)
# evaluate the undefended model on the AE
# print(">>> Evaluating UM on [{}], it may take a while...".format(ae_file))
# pred_adv_um = undefended.predict(x_adv)
# err_um = error_rate(y_pred=pred_adv_um, y_true=labels, correct_on_bs=corrections)
# # track the result
# results['UM'] = err_um
# evaluate the ensemble on the Hybrid
print(">>> Evaluating ensemble on [{}], it may take a while...".format(
ae_file))
pred_adv_ens = ensemble.predict(x_adv)
err_ens = error_rate(y_pred=pred_adv_ens,
y_true=labels,
correct_on_bs=corrections)
# track the result
results['Ensemble'] = err_ens
ae_end = time.time()
ae_final = ae_end - ae_start
# evaluate the baseline on the AE
# print(">>> Evaluating baseline model on [{}], it may take a while...".format(ae_file))
# pred_adv_bl = baseline.predict(x_adv)
# err_bl = error_rate(y_pred=pred_adv_bl, y_true=labels, correct_on_bs=corrections)
# # track the result
# results['PGD-ADT'] = err_bl
out_file.write(">>> Evaluations on [{}]:\n{}\n".format(
ae_file, results))
out_file.write('AE test took {t} seconds\n'.format(t=str(ae_final)))
end = time.time()
final = end - start
out_file.write('Full test suite took {t} seconds\n'.format(t=str(final)))
out_file.close()