def test_classification(adv_option, fisher_ensemble=False):
ensemble = []
repo = "./weights/ensemble/CNN/MNIST/ensemble"
filenames = os.listdir(repo)
model, dataset = build_model_MNIST()
(X_train, Y_train), (X_test, Y_test) = dataset
model.load_weights(os.path.join(repo, filenames[0]))
fisher = None
if fisher_ensemble:
fisher = Fisher(model)
fisher.load('./weights/ensemble/CNN', 'fisher_mnist')
filenames = filenames[1:]
for filename in filenames:
if fisher_ensemble:
net = fisher.fisher_sample()
else:
net, _ = build_model_MNIST()
net.load_weights(os.path.join(repo, filename))
ensemble.append(net)
if not (adv_option in ['szegedy', 'goodfellow', 'deepfool']):
raise NotImplementedError()
if adv_option == 'szegedy':
adv_object = Adversarial_Szegedy(model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=True)
true_x, adv_x, adv_y = adv_object.load('./adv/MNIST/train', 'szegedy')
if adv_option == 'goodfellow':
adv_object = Adversarial_Goodfellow(epsilon=1.,
model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=False)
true_x, adv_x, adv_y = adv_object.load('./adv/MNIST/test',
'goodfellow')
if adv_option == 'deepfool':
adv_object = Adversarial_DeepFool(model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=False)
true_x, adv_x, adv_y = adv_object.load('./adv/MNIST/test', 'deepfool')
prediction = model.predict(adv_x)
prediction_ensemble = [net.predict(adv_x) for net in ensemble[1:]]
predict_ensemble = np.mean(prediction_ensemble, axis=0)
prediction_true_ensemble = [net.predict(true_x) for net in ensemble[1:]]
predict_true = np.mean(prediction_true_ensemble, axis=0)
i = 3
print(np.linalg.norm(predict_ensemble[i] - predict_true[i]))
return
distance = prediction - predict_ensemble
dico = {}
dico['ADVERSARIAL'] = []
print('ADVERSARIAL')
for i in range(len(distance)):
tmp = np.linalg.norm(distance[i])
print(tmp, np.min(distance[i]), np.max(distance[i]),
np.max(prediction[i]), np.max(predict_ensemble[i]), 0)
dico['ADVERSARIAL'].append(tmp)
_, (x_test, y_test) = dataset
n = len(y_test)
x_true = x_test[np.random.permutation(n)][:len(adv_x)]
prediction = model.predict(x_true)
prediction_ensemble = [net.predict(x_true) for net in ensemble[1:]]
predict_ensemble = np.mean(prediction_ensemble, axis=0)
distance = prediction - predict_ensemble
dico['GROUNDTRUTH'] = []
print('GROUNDTRUTH')
for i in range(len(distance)):
tmp = np.linalg.norm(distance[i])
print(tmp, np.min(distance[i]), np.max(distance[i]),
np.max(prediction[i]), np.max(predict_ensemble[i]), 1)
dico['GROUNDTRUTH'].append(tmp)
import pickle as pkl
from contextlib import closing
if fisher_ensemble:
filename = os.path.join('./adv/MNIST/test',
adv_option + '_density_adv')
else:
filename = os.path.join('./adv/MNIST/test', adv_option + '_density')
with closing(open(filename, 'wb')) as f:
pkl.dump(dico, f, protocol=pkl.HIGHEST_PROTOCOL)
def test_committee(repo, option_adv):
dico_C_U = build_confusion_matrix(option_adv)
dico_name = generate_filenames(dico_C_U, option_adv)
with closing(open('./MNIST_dico_ICLR_' + option_adv, 'rb')) as f:
dico_name = pkl.load(f)
#pkl.dump(dico_name, f, protocol=pkl.HIGHEST_PROTOCOL)
#return
nb_class = 10
filename = 'ensemble_MNIST0'
ensemble = []
label_table = []
model, dataset = model, dataset = build_model_MNIST()
model.load_weights(os.path.join(repo, filename))
ensemble.append(model)
label_table.append(range(nb_class))
for key in dico_name.keys():
f_0, c, f_1, u = dico_name[key]
net_c, _ = build_model_MNIST(num_classes=len(c))
net_u, _ = build_model_MNIST(num_classes=len(u))
net_c.load_weights(os.path.join(repo, f_0))
net_u.load_weights(os.path.join(repo, f_1))
label_table.append(c)
label_table.append(u)
ensemble.append(net_c)
ensemble.append(net_u)
# step 1 load adv examples
adv_object = Adversarial_Szegedy(model=model,
dataset=dataset,
n_channels=1,
img_nrows=28,
img_ncols=28,
use_train=True)
#adv_object.save('./adv/MNIST/test', 'goodfellow')
print('ADVERSARIAL')
adv_x, _ = adv_object.load('./adv/MNIST/test', 'szegedy')
dico = dict([('ADVERSARIAL', []), ('GROUNDTRUTH', [])])
# step 1
predict_ensemble = np.array([
predict(net, adv_x, table)
for net, table in zip(ensemble, label_table)
])
confidency_ensemble = np.array(
[np.max(net.predict(adv_x), axis=1) for net in ensemble])
for i in range(len(adv_x)):
occurences = [
len(np.where(predict_ensemble[:, i] == j)) for j in range(nb_class)
]
winning = max(occurences) == nb_class + 1
if winning:
print('WINNING')
# pick only the classifier with the right label
label = np.argmax(occurences)
networks_index = np.where(predict_ensemble[:, i] == label)
else:
networks_index = range(nb_class + 1)
result = np.max(confidency_ensemble[networks_index, i])
dico['ADVERSARIAL'].append(result)
print(result, 0)
print('GROUNDTRUTH')
_, (x_test, y_test) = dataset
n = len(y_test)
x_true = x_test[np.random.permutation(n)][:len(adv_x)]
# step 1
predict_ensemble = np.array([
predict(net, x_true, table)
for net, table in zip(ensemble, label_table)
])
confidency_ensemble = np.array(
[np.max(net.predict(x_true), axis=1) for net in ensemble])
for i in range(len(x_true)):
occurences = [
len(np.where(predict_ensemble[:, i] == j)) for j in range(nb_class)
]
winning = max(occurences) == nb_class + 1
if winning:
print('WINNING')
# pick only the classifier with the right label
label = np.argmax(occurences)
networks_index = np.where(predict_ensemble[:, i] == label)
else:
networks_index = range(nb_class + 1)
result = np.max(confidency_ensemble[networks_index, i])
dico['GROUNDTRUTH'].append(result)
print(result, 1)
#import pdb; pdb.set_trace()
filename = os.path.join('./adv/MNIST/test',
option_adv + '_density_committee')
with closing(open(filename, 'wb')) as f:
pkl.dump(dico, f, protocol=pkl.HIGHEST_PROTOCOL)