def test_classification(adv_option, fisher_ensemble=False): ensemble = [] repo = "./weights/ensemble/CNN/MNIST/ensemble" filenames = os.listdir(repo) model, dataset = build_model_MNIST() (X_train, Y_train), (X_test, Y_test) = dataset model.load_weights(os.path.join(repo, filenames[0])) fisher = None if fisher_ensemble: fisher = Fisher(model) fisher.load('./weights/ensemble/CNN', 'fisher_mnist') filenames = filenames[1:] for filename in filenames: if fisher_ensemble: net = fisher.fisher_sample() else: net, _ = build_model_MNIST() net.load_weights(os.path.join(repo, filename)) ensemble.append(net) if not (adv_option in ['szegedy', 'goodfellow', 'deepfool']): raise NotImplementedError() if adv_option == 'szegedy': adv_object = Adversarial_Szegedy(model=model, dataset=dataset, n_channels=1, img_nrows=28, img_ncols=28, use_train=True) true_x, adv_x, adv_y = adv_object.load('./adv/MNIST/train', 'szegedy') if adv_option == 'goodfellow': adv_object = Adversarial_Goodfellow(epsilon=1., model=model, dataset=dataset, n_channels=1, img_nrows=28, img_ncols=28, use_train=False) true_x, adv_x, adv_y = adv_object.load('./adv/MNIST/test', 'goodfellow') if adv_option == 'deepfool': adv_object = Adversarial_DeepFool(model=model, dataset=dataset, n_channels=1, img_nrows=28, img_ncols=28, use_train=False) true_x, adv_x, adv_y = adv_object.load('./adv/MNIST/test', 'deepfool') prediction = model.predict(adv_x) prediction_ensemble = [net.predict(adv_x) for net in ensemble[1:]] predict_ensemble = np.mean(prediction_ensemble, axis=0) prediction_true_ensemble = [net.predict(true_x) for net in ensemble[1:]] predict_true = np.mean(prediction_true_ensemble, axis=0) i = 3 print(np.linalg.norm(predict_ensemble[i] - predict_true[i])) return distance = prediction - predict_ensemble dico = {} dico['ADVERSARIAL'] = [] print('ADVERSARIAL') for i in range(len(distance)): tmp = np.linalg.norm(distance[i]) print(tmp, np.min(distance[i]), np.max(distance[i]), np.max(prediction[i]), np.max(predict_ensemble[i]), 0) dico['ADVERSARIAL'].append(tmp) _, (x_test, y_test) = dataset n = len(y_test) x_true = x_test[np.random.permutation(n)][:len(adv_x)] prediction = model.predict(x_true) prediction_ensemble = [net.predict(x_true) for net in ensemble[1:]] predict_ensemble = np.mean(prediction_ensemble, axis=0) distance = prediction - predict_ensemble dico['GROUNDTRUTH'] = [] print('GROUNDTRUTH') for i in range(len(distance)): tmp = np.linalg.norm(distance[i]) print(tmp, np.min(distance[i]), np.max(distance[i]), np.max(prediction[i]), np.max(predict_ensemble[i]), 1) dico['GROUNDTRUTH'].append(tmp) import pickle as pkl from contextlib import closing if fisher_ensemble: filename = os.path.join('./adv/MNIST/test', adv_option + '_density_adv') else: filename = os.path.join('./adv/MNIST/test', adv_option + '_density') with closing(open(filename, 'wb')) as f: pkl.dump(dico, f, protocol=pkl.HIGHEST_PROTOCOL)
def test_committee(repo, option_adv): dico_C_U = build_confusion_matrix(option_adv) dico_name = generate_filenames(dico_C_U, option_adv) with closing(open('./MNIST_dico_ICLR_' + option_adv, 'rb')) as f: dico_name = pkl.load(f) #pkl.dump(dico_name, f, protocol=pkl.HIGHEST_PROTOCOL) #return nb_class = 10 filename = 'ensemble_MNIST0' ensemble = [] label_table = [] model, dataset = model, dataset = build_model_MNIST() model.load_weights(os.path.join(repo, filename)) ensemble.append(model) label_table.append(range(nb_class)) for key in dico_name.keys(): f_0, c, f_1, u = dico_name[key] net_c, _ = build_model_MNIST(num_classes=len(c)) net_u, _ = build_model_MNIST(num_classes=len(u)) net_c.load_weights(os.path.join(repo, f_0)) net_u.load_weights(os.path.join(repo, f_1)) label_table.append(c) label_table.append(u) ensemble.append(net_c) ensemble.append(net_u) # step 1 load adv examples adv_object = Adversarial_Szegedy(model=model, dataset=dataset, n_channels=1, img_nrows=28, img_ncols=28, use_train=True) #adv_object.save('./adv/MNIST/test', 'goodfellow') print('ADVERSARIAL') adv_x, _ = adv_object.load('./adv/MNIST/test', 'szegedy') dico = dict([('ADVERSARIAL', []), ('GROUNDTRUTH', [])]) # step 1 predict_ensemble = np.array([ predict(net, adv_x, table) for net, table in zip(ensemble, label_table) ]) confidency_ensemble = np.array( [np.max(net.predict(adv_x), axis=1) for net in ensemble]) for i in range(len(adv_x)): occurences = [ len(np.where(predict_ensemble[:, i] == j)) for j in range(nb_class) ] winning = max(occurences) == nb_class + 1 if winning: print('WINNING') # pick only the classifier with the right label label = np.argmax(occurences) networks_index = np.where(predict_ensemble[:, i] == label) else: networks_index = range(nb_class + 1) result = np.max(confidency_ensemble[networks_index, i]) dico['ADVERSARIAL'].append(result) print(result, 0) print('GROUNDTRUTH') _, (x_test, y_test) = dataset n = len(y_test) x_true = x_test[np.random.permutation(n)][:len(adv_x)] # step 1 predict_ensemble = np.array([ predict(net, x_true, table) for net, table in zip(ensemble, label_table) ]) confidency_ensemble = np.array( [np.max(net.predict(x_true), axis=1) for net in ensemble]) for i in range(len(x_true)): occurences = [ len(np.where(predict_ensemble[:, i] == j)) for j in range(nb_class) ] winning = max(occurences) == nb_class + 1 if winning: print('WINNING') # pick only the classifier with the right label label = np.argmax(occurences) networks_index = np.where(predict_ensemble[:, i] == label) else: networks_index = range(nb_class + 1) result = np.max(confidency_ensemble[networks_index, i]) dico['GROUNDTRUTH'].append(result) print(result, 1) #import pdb; pdb.set_trace() filename = os.path.join('./adv/MNIST/test', option_adv + '_density_committee') with closing(open(filename, 'wb')) as f: pkl.dump(dico, f, protocol=pkl.HIGHEST_PROTOCOL)