def main(args):
assert args.dataset in ['mnist', 'cifar', 'svhn'], \
"Dataset parameter must be either 'mnist', 'cifar' or 'svhn'"
assert args.attack in ['fgsm', 'bim-a', 'bim-b', 'jsma', 'cw-l2', 'all'], \
"Attack parameter must be either 'fgsm', 'bim-a', 'bim-b', " \
"'jsma' or 'cw-l2'"
assert args.characteristic in ['kd', 'bu', 'lid', 'km', 'all'], \
"Characteristic(s) to use 'kd', 'bu', 'lid', 'km', 'all'"
model_file = os.path.join(PATH_DATA, "model_%s.h5" % args.dataset)
assert os.path.isfile(model_file), \
'model file not found... must first train model using train_model.py.'
adv_file = os.path.join(PATH_DATA, "Adv_%s_%s.npy" % (args.dataset, args.attack))
assert os.path.isfile(adv_file), \
'adversarial sample file not found... must first craft adversarial ' \
'samples using craft_adv_samples.py'
print('Loading the data and model...')
# Load the model
model = load_model(model_file)
# Load the dataset
X_train, Y_train, X_test, Y_test = get_data(args.dataset)
# Check attack type, select adversarial and noisy samples accordingly
print('Loading noisy and adversarial samples...')
if args.attack == 'all':
# TODO: implement 'all' option
# X_test_adv = ...
# X_test_noisy = ...
raise NotImplementedError("'All' types detector not yet implemented.")
else:
# Load adversarial samples
X_test_adv = np.load(adv_file)
print("X_test_adv: ", X_test_adv.shape)
# as there are some parameters to tune for noisy example, so put the generation
# step here instead of the adversarial step which can take many hours
noisy_file = os.path.join(PATH_DATA, 'Noisy_%s_%s.npy' % (args.dataset, args.attack))
if os.path.isfile(noisy_file):
X_test_noisy = np.load(noisy_file)
else:
# Craft an equal number of noisy samples
print('Crafting %s noisy samples. ' % args.dataset)
X_test_noisy = get_noisy_samples(X_test, X_test_adv, args.dataset, args.attack)
np.save(noisy_file, X_test_noisy)
# Check model accuracies on each sample type
for s_type, dataset in zip(['normal', 'noisy', 'adversarial'],
[X_test, X_test_noisy, X_test_adv]):
_, acc = model.evaluate(dataset, Y_test, batch_size=args.batch_size,
verbose=0)
print("Model accuracy on the %s test set: %0.2f%%" %
(s_type, 100 * acc))
# Compute and display average perturbation sizes
if not s_type == 'normal':
l2_diff = np.linalg.norm(
dataset.reshape((len(X_test), -1)) -
X_test.reshape((len(X_test), -1)),
axis=1
).mean()
print("Average L-2 perturbation size of the %s test set: %0.2f" %
(s_type, l2_diff))
# Refine the normal, noisy and adversarial sets to only include samples for
# which the original version was correctly classified by the model
preds_test = model.predict_classes(X_test, verbose=0,
batch_size=args.batch_size)
inds_correct = np.where(preds_test == Y_test.argmax(axis=1))[0]
print("Number of correctly predict images: %s" % (len(inds_correct)))
X_test = X_test[inds_correct]
X_test_noisy = X_test_noisy[inds_correct]
X_test_adv = X_test_adv[inds_correct]
print("X_test: ", X_test.shape)
print("X_test_noisy: ", X_test_noisy.shape)
print("X_test_adv: ", X_test_adv.shape)
if args.characteristic == 'kd':
# extract kernel density
characteristics, labels = get_kd(model, X_train, Y_train, X_test, X_test_noisy, X_test_adv)
print("KD: [characteristic shape: ", characteristics.shape, ", label shape: ", labels.shape)
# save to file
bandwidth = BANDWIDTHS[args.dataset]
file_name = os.path.join(PATH_DATA, 'kd_%s_%s_%.4f.npy' % (args.dataset, args.attack, bandwidth))
data = np.concatenate((characteristics, labels), axis=1)
np.save(file_name, data)
elif args.characteristic == 'bu':
# extract Bayesian uncertainty
characteristics, labels = get_bu(model, X_test, X_test_noisy, X_test_adv)
print("BU: [characteristic shape: ", characteristics.shape, ", label shape: ", labels.shape)
# save to file
file_name = os.path.join(PATH_DATA, 'bu_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((characteristics, labels), axis=1)
np.save(file_name, data)
elif args.characteristic == 'lid':
# extract local intrinsic dimensionality
characteristics, labels = get_lid(model, X_test, X_test_noisy, X_test_adv,
args.k_nearest, args.batch_size, args.dataset)
print("LID: [characteristic shape: ", characteristics.shape, ", label shape: ", labels.shape)
# save to file
file_name = os.path.join(PATH_DATA, 'lid_%s_%s.npy' % (args.dataset, args.attack))
# file_name = os.path.join('../data_grid_search/lid_large_batch/', 'lid_%s_%s_%s.npy' %
# (args.dataset, args.attack, args.k_nearest))
data = np.concatenate((characteristics, labels), axis=1)
np.save(file_name, data)
elif args.characteristic == 'km':
# extract k means distance
characteristics, labels = get_kmeans(model, X_test, X_test_noisy, X_test_adv,
args.k_nearest, args.batch_size, args.dataset)
print("K-Mean: [characteristic shape: ", characteristics.shape, ", label shape: ", labels.shape)
# save to file
file_name = os.path.join(PATH_DATA, 'km_pca_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((characteristics, labels), axis=1)
np.save(file_name, data)
elif args.characteristic == 'all':
# extract kernel density
characteristics, labels = get_kd(model, X_train, Y_train, X_test, X_test_noisy, X_test_adv)
file_name = os.path.join(PATH_DATA, 'kd_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((characteristics, labels), axis=1)
np.save(file_name, data)
# extract Bayesian uncertainty
characteristics, labels = get_bu(model, X_test, X_test_noisy, X_test_adv)
file_name = os.path.join(PATH_DATA, 'bu_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((characteristics, labels), axis=1)
np.save(file_name, data)
# extract local intrinsic dimensionality
characteristics, labels = get_lid(model, X_test, X_test_noisy, X_test_adv,
args.k_nearest, args.batch_size, args.dataset)
file_name = os.path.join(PATH_DATA, 'lid_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((characteristics, labels), axis=1)
np.save(file_name, data)
def main(args):
assert args.dataset in ['mnist', 'cifar', 'svhn'], \
"Dataset parameter must be either 'mnist', 'cifar' or 'svhn'"
assert args.attack in ['fgsm', 'bim-a', 'bim-b', 'jsma', 'cw-l2', 'all'], \
"Attack parameter must be either 'fgsm', 'bim-a', 'bim-b', " \
"'jsma' or 'cw-l2'"
assert args.artifact in ['kd', 'bu', 'lid', 'km', 'all'], \
"Artifact(s) to use 'kd', 'bu', 'lid', 'km', 'all'"
model_file = os.path.join(PATH_DATA, "model_%s.h5" % args.dataset)
assert os.path.isfile(model_file), \
'model file not found... must first train model using train_model.py.'
adv_file = os.path.join(PATH_DATA, "Adv_%s_%s.npy" % (args.dataset, args.attack))
assert os.path.isfile(adv_file), \
'adversarial sample file not found... must first craft adversarial ' \
'samples using craft_adv_samples.py'
print('Loading the data and model...')
# Load the model
model = load_model(model_file)
# Load the dataset
X_train, Y_train, X_test, Y_test = get_data(args.dataset)
# Check attack type, select adversarial and noisy samples accordingly
print('Loading noisy and adversarial samples...')
if args.attack == 'all':
# TODO: implement 'all' option
# X_test_adv = ...
# X_test_noisy = ...
raise NotImplementedError("'All' types detector not yet implemented.")
else:
# Load adversarial samples
X_test_adv = np.load(adv_file)
print("X_test_adv: ", X_test_adv.shape)
# as there are some parameters to tune for noisy example, so put the generation
# step here instead of the adversarial step which can take many hours
noisy_file = os.path.join(PATH_DATA, 'Noisy_%s_%s.npy' % (args.dataset, args.attack))
if os.path.isfile(noisy_file):
X_test_noisy = np.load(noisy_file)
else:
# Craft an equal number of noisy samples
print('Crafting %s noisy samples. ' % args.dataset)
X_test_noisy = get_noisy_samples(X_test, X_test_adv, args.dataset, args.attack)
np.save(noisy_file, X_test_noisy)
# Check model accuracies on each sample type
for s_type, dataset in zip(['normal', 'noisy', 'adversarial'],
[X_test, X_test_noisy, X_test_adv]):
_, acc = model.evaluate(dataset, Y_test, batch_size=args.batch_size,
verbose=0)
print("Model accuracy on the %s test set: %0.2f%%" %
(s_type, 100 * acc))
# Compute and display average perturbation sizes
if not s_type == 'normal':
l2_diff = np.linalg.norm(
dataset.reshape((len(X_test), -1)) -
X_test.reshape((len(X_test), -1)),
axis=1
).mean()
print("Average L-2 perturbation size of the %s test set: %0.2f" %
(s_type, l2_diff))
# Refine the normal, noisy and adversarial sets to only include samples for
# which the original version was correctly classified by the model
preds_test = model.predict_classes(X_test, verbose=0,
batch_size=args.batch_size)
inds_correct = np.where(preds_test == Y_test.argmax(axis=1))[0]
print("Number of correctly predict images: %s" % (len(inds_correct)))
X_test = X_test[inds_correct]
X_test_noisy = X_test_noisy[inds_correct]
X_test_adv = X_test_adv[inds_correct]
print("X_test: ", X_test.shape)
print("X_test_noisy: ", X_test_noisy.shape)
print("X_test_adv: ", X_test_adv.shape)
if args.artifact == 'kd':
# extract kernel density
artifacts, labels = get_kd(model, X_train, Y_train, X_test, X_test_noisy, X_test_adv)
print("KD: [artifact shape: ", artifacts.shape, ", label shape: ", labels.shape)
# save to file
bandwidth = BANDWIDTHS[args.dataset]
file_name = os.path.join(PATH_DATA, 'kd_%s_%s_%.4f.npy' % (args.dataset, args.attack, bandwidth))
data = np.concatenate((artifacts, labels), axis=1)
np.save(file_name, data)
elif args.artifact == 'bu':
# extract Bayesian uncertainty
artifacts, labels = get_bu(model, X_test, X_test_noisy, X_test_adv)
print("BU: [artifact shape: ", artifacts.shape, ", label shape: ", labels.shape)
# save to file
file_name = os.path.join(PATH_DATA, 'bu_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((artifacts, labels), axis=1)
np.save(file_name, data)
elif args.artifact == 'lid':
# extract local intrinsic dimensionality
np.save("/tmp/xtest", X_test)
np.save("/tmp/xtest_noisy", X_test_noisy)
np.save("/tmp/xtest_adv", X_test_adv)
print(args)
artifacts, labels = get_lid(model, X_test, X_test_noisy, X_test_adv,
args.k_nearest, args.batch_size, args.dataset)
print("LID: [artifact shape: ", artifacts.shape, ", label shape: ", labels.shape)
exit(0)
# save to file
# file_name = os.path.join(PATH_DATA, 'lid_%s_%s.npy' % (args.dataset, args.attack))
file_name = os.path.join('../data_grid_search/lid_large_batch/', 'lid_%s_%s_%s.npy' %
(args.dataset, args.attack, args.k_nearest))
data = np.concatenate((artifacts, labels), axis=1)
np.save(file_name, data)
elif args.artifact == 'km':
# extract k means distance
artifacts, labels = get_kmeans(model, X_test, X_test_noisy, X_test_adv,
args.k_nearest, args.batch_size, args.dataset)
print("K-Mean: [artifact shape: ", artifacts.shape, ", label shape: ", labels.shape)
# save to file
file_name = os.path.join(PATH_DATA, 'km_pca_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((artifacts, labels), axis=1)
np.save(file_name, data)
elif args.artifact == 'all':
# extract kernel density
artifacts, labels = get_kd(model, X_train, Y_train, X_test, X_test_noisy, X_test_adv)
file_name = os.path.join(PATH_DATA, 'kd_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((artifacts, labels), axis=1)
np.save(file_name, data)
# extract Bayesian uncertainty
artifacts, labels = get_bu(model, X_test, X_test_noisy, X_test_adv)
file_name = os.path.join(PATH_DATA, 'bu_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((artifacts, labels), axis=1)
np.save(file_name, data)
# extract local intrinsic dimensionality
artifacts, labels = get_lid(model, X_test, X_test_noisy, X_test_adv,
args.k_nearest, args.batch_size, args.dataset)
file_name = os.path.join(PATH_DATA, 'lid_%s_%s.npy' % (args.dataset, args.attack))
data = np.concatenate((artifacts, labels), axis=1)
np.save(file_name, data)