def test_Adversarial(option, repo, filename):
if option=='Adversarial_Szegedy':
adv_object = Adversarial_Szegedy(repo=repo, filename=filename, nb_class=100, data_name='cifar100')
#adv_object.generate()
adv_object.save('./adv/Szegedy', 'cifar100_v0')
if option=='Adversarial_Goodfellow':
adv_object = Adversarial_Goodfellow(repo=repo, filename=filename, nb_class=100, data_name='cifar100')
adv_object.generate()
adv_object.save('./adv/Goodfellow', 'cifar100_v0')
def test_Adversarial(option):
if option == 'Adversarial_Szegedy':
adv_object = Adversarial_Szegedy()
#adv_object.generate()
adv_object.save('./adv/Szegedy', 'cifar10_v0')
if option == 'Adversarial_Goodfellow':
adv_object = Adversarial_Goodfellow()
#adv_object.generate()
adv_object.save('./adv/Goodfellow', 'cifar10_v0')
if option == 'Adversarial_DeepFool':
adv_object = Adversarial_DeepFool()
adv_object.generate()
def build_confusion_matrix(option_adv):
repo = "./weights/ensemble/CNN/MNIST"
nb_class = 10
filenames = os.listdir(repo)
model, dataset = build_model_MNIST()
(X_train, Y_train), (X_test, Y_test) = dataset
model.load_weights(os.path.join(repo, filenames[0]))
if option_adv == 'goodfellow':
adv_object = Adversarial_Goodfellow(epsilon=1.,
model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=True)
z, x, y = adv_object.load('./adv/MNIST/train', 'goodfellow')
if option_adv == 'szegedy':
adv_object = Adversarial_Szegedy(model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=True)
z, x, y = adv_object.load('./adv/MNIST/train', 'szegedy')
confusion_matrix = np.zeros((nb_class, nb_class))
prediction = np.argmax(model.predict(z), axis=1)
for i in range(len(y)):
confusion_matrix[prediction[i], y[i]] += 1.
dico_class = dict([(i, []) for i in range(nb_class)])
for i in range(nb_class):
confusion_matrix[i] /= sum(confusion_matrix[i])
label_sort = np.argsort(confusion_matrix[i])[::-1]
confidence_sort = np.sort(confusion_matrix[i])[::-1]
cumsum_confidence = np.cumsum(confidence_sort)
tmp = cumsum_confidence - 0.8 * np.ones((nb_class, ))
index = np.argmin((tmp)**2)
if tmp[index] < 0:
index += 1
dico_class[i] = np.sort(label_sort[:index + 1])
return dico_class
def test_classification(adv_option, fisher_ensemble=False):
ensemble = []
repo = "./weights/ensemble/CNN/MNIST/ensemble"
filenames = os.listdir(repo)
model, dataset = build_model_MNIST()
(X_train, Y_train), (X_test, Y_test) = dataset
model.load_weights(os.path.join(repo, filenames[0]))
fisher = None
if fisher_ensemble:
fisher = Fisher(model)
fisher.load('./weights/ensemble/CNN', 'fisher_mnist')
filenames = filenames[1:]
for filename in filenames:
if fisher_ensemble:
net = fisher.fisher_sample()
else:
net, _ = build_model_MNIST()
net.load_weights(os.path.join(repo, filename))
ensemble.append(net)
if not (adv_option in ['szegedy', 'goodfellow', 'deepfool']):
raise NotImplementedError()
if adv_option == 'szegedy':
adv_object = Adversarial_Szegedy(model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=True)
true_x, adv_x, adv_y = adv_object.load('./adv/MNIST/train', 'szegedy')
if adv_option == 'goodfellow':
adv_object = Adversarial_Goodfellow(epsilon=1.,
model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=False)
true_x, adv_x, adv_y = adv_object.load('./adv/MNIST/test',
'goodfellow')
if adv_option == 'deepfool':
adv_object = Adversarial_DeepFool(model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=False)
true_x, adv_x, adv_y = adv_object.load('./adv/MNIST/test', 'deepfool')
prediction = model.predict(adv_x)
prediction_ensemble = [net.predict(adv_x) for net in ensemble[1:]]
predict_ensemble = np.mean(prediction_ensemble, axis=0)
prediction_true_ensemble = [net.predict(true_x) for net in ensemble[1:]]
predict_true = np.mean(prediction_true_ensemble, axis=0)
i = 3
print(np.linalg.norm(predict_ensemble[i] - predict_true[i]))
return
distance = prediction - predict_ensemble
dico = {}
dico['ADVERSARIAL'] = []
print('ADVERSARIAL')
for i in range(len(distance)):
tmp = np.linalg.norm(distance[i])
print(tmp, np.min(distance[i]), np.max(distance[i]),
np.max(prediction[i]), np.max(predict_ensemble[i]), 0)
dico['ADVERSARIAL'].append(tmp)
_, (x_test, y_test) = dataset
n = len(y_test)
x_true = x_test[np.random.permutation(n)][:len(adv_x)]
prediction = model.predict(x_true)
prediction_ensemble = [net.predict(x_true) for net in ensemble[1:]]
predict_ensemble = np.mean(prediction_ensemble, axis=0)
distance = prediction - predict_ensemble
dico['GROUNDTRUTH'] = []
print('GROUNDTRUTH')
for i in range(len(distance)):
tmp = np.linalg.norm(distance[i])
print(tmp, np.min(distance[i]), np.max(distance[i]),
np.max(prediction[i]), np.max(predict_ensemble[i]), 1)
dico['GROUNDTRUTH'].append(tmp)
import pickle as pkl
from contextlib import closing
if fisher_ensemble:
filename = os.path.join('./adv/MNIST/test',
adv_option + '_density_adv')
else:
filename = os.path.join('./adv/MNIST/test', adv_option + '_density')
with closing(open(filename, 'wb')) as f:
pkl.dump(dico, f, protocol=pkl.HIGHEST_PROTOCOL)
def test_committee(repo, option_adv):
dico_C_U = build_confusion_matrix(option_adv)
dico_name = generate_filenames(dico_C_U, option_adv)
with closing(open('./MNIST_dico_ICLR_' + option_adv, 'rb')) as f:
dico_name = pkl.load(f)
#pkl.dump(dico_name, f, protocol=pkl.HIGHEST_PROTOCOL)
#return
nb_class = 10
filename = 'ensemble_MNIST0'
ensemble = []
label_table = []
model, dataset = model, dataset = build_model_MNIST()
model.load_weights(os.path.join(repo, filename))
ensemble.append(model)
label_table.append(range(nb_class))
for key in dico_name.keys():
f_0, c, f_1, u = dico_name[key]
net_c, _ = build_model_MNIST(num_classes=len(c))
net_u, _ = build_model_MNIST(num_classes=len(u))
net_c.load_weights(os.path.join(repo, f_0))
net_u.load_weights(os.path.join(repo, f_1))
label_table.append(c)
label_table.append(u)
ensemble.append(net_c)
ensemble.append(net_u)
# step 1 load adv examples
adv_object = Adversarial_Szegedy(model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=True)
#adv_object.save('./adv/MNIST/test', 'goodfellow')
print('ADVERSARIAL')
adv_x, _ = adv_object.load('./adv/MNIST/test', 'szegedy')
dico = dict([('ADVERSARIAL', []), ('GROUNDTRUTH', [])])
# step 1
predict_ensemble = np.array([
predict(net, adv_x, table)
for net, table in zip(ensemble, label_table)
])
confidency_ensemble = np.array(
[np.max(net.predict(adv_x), axis=1) for net in ensemble])
for i in range(len(adv_x)):
occurences = [
len(np.where(predict_ensemble[:, i] == j)) for j in range(nb_class)
]
winning = max(occurences) == nb_class + 1
if winning:
print('WINNING')
# pick only the classifier with the right label
label = np.argmax(occurences)
networks_index = np.where(predict_ensemble[:, i] == label)
else:
networks_index = range(nb_class + 1)
result = np.max(confidency_ensemble[networks_index, i])
dico['ADVERSARIAL'].append(result)
print(result, 0)
print('GROUNDTRUTH')
_, (x_test, y_test) = dataset
n = len(y_test)
x_true = x_test[np.random.permutation(n)][:len(adv_x)]
# step 1
predict_ensemble = np.array([
predict(net, x_true, table)
for net, table in zip(ensemble, label_table)
])
confidency_ensemble = np.array(
[np.max(net.predict(x_true), axis=1) for net in ensemble])
for i in range(len(x_true)):
occurences = [
len(np.where(predict_ensemble[:, i] == j)) for j in range(nb_class)
]
winning = max(occurences) == nb_class + 1
if winning:
print('WINNING')
# pick only the classifier with the right label
label = np.argmax(occurences)
networks_index = np.where(predict_ensemble[:, i] == label)
else:
networks_index = range(nb_class + 1)
result = np.max(confidency_ensemble[networks_index, i])
dico['GROUNDTRUTH'].append(result)
print(result, 1)
#import pdb; pdb.set_trace()
filename = os.path.join('./adv/MNIST/test',
option_adv + '_density_committee')
with closing(open(filename, 'wb')) as f:
pkl.dump(dico, f, protocol=pkl.HIGHEST_PROTOCOL)
def test_ensemble_robustness_adv():
repo = "./weights/ensemble/CNN/MNIST"
filenames = os.listdir(repo)
model, dataset = build_model_MNIST()
(X_train, Y_train), (X_test, Y_test) = dataset
model.load_weights(os.path.join(repo, filenames[0]))
adv_object = Adversarial_Goodfellow(epsilon=0.03,
model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=True)
#adv_object.save('./adv/MNIST/train', 'goodfellow')
#z, x, y = adv_object.load('./adv/MNIST/train', 'goodfellow')
z, x, y = adv_object.generate()
im_00 = z[0].reshape((28, 28))
im_01 = x[0].reshape((28, 28))
adv_object = Adversarial_Szegedy(0.5,
model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=True)
#adv_object.save('./adv/MNIST/train', 'szegedy')
#z, x,y = adv_object.load('./adv/MNIST/train', 'szegedy')
z, x, y = adv_object.generate()
im_10 = z[0].reshape((28, 28))
im_11 = x[0].reshape((28, 28))
"""
adv_object = Adversarial_DeepFool(model=model, dataset=dataset,
n_channels=1, img_nrows=28, img_ncols=28, use_train=True)
adv_object.generate()
#adv_object.save('./adv/MNIST/test', 'deepfool')
#z, x, y = adv_object.load('./adv/MNIST/test', 'deepfool')
print('deepfool')
#print(len(x))
#import pdb; pdb.set_trace()
"""
import matplotlib.pyplot as plt
plt.subplot(2, 2, 1)
plt.imshow(im_00)
plt.subplot(2, 2, 2)
plt.imshow(im_01)
plt.subplot(2, 2, 3)
plt.imshow(im_10)
plt.subplot(2, 2, 4)
plt.imshow(im_11)
conf_0 = np.max(model.predict(im_10.reshape((1, 28, 28, 1))))
conf_1 = np.max(model.predict(im_11.reshape((1, 28, 28, 1))))
print(conf_0, conf_1)