(batch_size, img_size, img_size, img_chan), name='x_fixed') env.adv_eps = tf.placeholder(tf.float32, (), name='adv_eps') env.adv_epochs = tf.placeholder(tf.int32, (), name='adv_epochs') env.adv_y = tf.placeholder(tf.int32, (), name='adv_y') env.x_fgsm = fgm(model, env.x, epochs=env.adv_epochs, eps=env.adv_eps) env.x_deepfool = deepfool(model, env.x, epochs=env.adv_epochs, batch=True) env.x_jsma = jsma(model, env.x, env.adv_y, eps=env.adv_eps, epochs=env.adv_epochs) env.cw_train_op, env.x_cw, env.cw_noise = cw(model, env.x_fixed, y=env.adv_y, eps=env.adv_eps, optimizer=optimizer) print('\nInitializing graph') sess = tf.InteractiveSession() sess.run(tf.global_variables_initializer()) sess.run(tf.local_variables_initializer()) def evaluate(sess, env, X_data, y_data, batch_size=128): """ Evaluate TF model by running env.loss and env.acc. """ print('\nEvaluating')
env.train_op = optimizer.minimize(env.loss, var_list=vs) env.saver = tf.train.Saver() # Note here that the shape has to be fixed during the graph construction # since the internal variable depends upon the shape. env.x_fixed = tf.placeholder(tf.float32, (batch_size, img_size, img_size, img_chan), name='x_fixed') env.adv_eps = tf.placeholder(tf.float32, (), name='adv_eps') env.adv_y = tf.placeholder(tf.int32, (), name='adv_y') optimizer = tf.train.AdamOptimizer(learning_rate=0.1) env.adv_train_op, env.xadv, env.noise = cw(model, env.x_fixed, ord_='inf', y=env.adv_y, eps=env.adv_eps, optimizer=optimizer) print('\nInitializing graph') env.sess = tf.InteractiveSession() env.sess.run(tf.global_variables_initializer()) env.sess.run(tf.local_variables_initializer()) def evaluate(env, X_data, y_data, batch_size=128): """ Evaluate TF model by running env.loss and env.acc. """ print('\nEvaluating')
from attacks import fgsm_it, cw from utils import stats, stats_cari, heat, heat_cari from mnist_cnn import build_network, loss_func, evaluation from tensorflow.examples.tutorials.mnist import input_data as mnist_data from keras.datasets import cifar10 from tqdm import tqdm ITERS = 100 ALPHA = 0.01 EPSI = 0.1 CW_ITERS = 100 BIN_STEPS = 20 NORM = 'l2' #l0/l2/linf mnist = mnist_data.read_data_sets('MNIST_data', one_hot=True) if len(sys.argv) < 2 or sys.argv[1] == "fgsm_it": perturbed_accuracy = fgsm_it(mnist.test.images[:1000], ITERS, EPSI, ALPHA, build_network, loss_func, evaluation) print([s[0] for s in perturbed_accuracy]) stats([s[1] for s in perturbed_accuracy], mnist.test.labels[:1000]) heat([s[1] for s in perturbed_accuracy], mnist.test.labels[:1000], "mnist_fgsm_") else: perturbed_norms, was = cw(mnist.test.images[:1000], mnist.test.labels[:1000], CW_ITERS, BIN_STEPS, build_network, loss_func, evaluation, NORM) stats_cari(perturbed_norms, was) heat_cari(perturbed_norms, was)
ITERS = 100 ALPHA = 0.01 EPSI = 0.1 CW_ITERS = 100 BIN_STEPS = 20 NORM = 'l2' #l0/l2/linf (train_features, train_labels), (test_features, test_labels) = cifar10.load_data() (test_features, test_labels) = agu(test_features, test_labels) train_features = train_features.astype('float32') test_features = test_features.astype('float32') train_features /= 255 test_features /= 255 test_labels = np_utils.to_categorical(test_labels, 10) if len(sys.argv) < 2 or sys.argv[1] == "fgsm_it": perturbed_accuracy = fgsm_it(test_features[:100], ITERS, EPSI, ALPHA, build_network, loss_func, evaluation, './tmp/original_cifar_model-8') print([s[0] for s in perturbed_accuracy]) stats([s[1] for s in perturbed_accuracy], test_labels[:1000]) heat([s[1] for s in perturbed_accuracy], test_labels[:1000], "cifar10_fgsm_") else: perturbed_norms, was = cw(test_features[:100], test_labels[:100], CW_ITERS, BIN_STEPS, build_network, loss_func, evaluation, NORM, './tmp/original_cifar_model-8') stats_cari(perturbed_norms, was)
def eval_adv_test(model, device, test_loader, attack, attack_params, results_dir): """ evaluate model by white-box attack """ model.eval() count = 0 if attack == 'pgd': restarts_matrices = [] for run in range(attack_params['num_restarts']): current_matrix_rows = [] count = 0 batch_num = 0 natural_accuracy = 0 for data, target, unsup in test_loader: batch_num = batch_num + 1 if batch_num > args.num_eval_batches: break data, target = data.to(device), target.to(device) count = count + len(target) X, y = Variable(data, requires_grad=True), Variable(target) # matrix_robust has batch_size*num_iterations dimensions indices_natural, matrix_robust = pgd( model, X, y, epsilon=attack_params['epsilon'], num_steps=attack_params['num_steps'], step_size=attack_params['step_size'], pretrain=args.pretrain, random_start=attack_params['random_start']) natural_accuracy = natural_accuracy + sum(indices_natural) current_matrix_rows.append(matrix_robust) logging.info('Completed restart: %g' % run) current_matrix = np.concatenate(current_matrix_rows, axis=0) logging.info("Shape of current_matrix %s" % str(current_matrix.shape)) restarts_matrices.append(current_matrix) final_matrix = np.asarray(restarts_matrices) if run == 0: final_success = np.ones([count, 1]) final_success = np.multiply( final_success, np.reshape(current_matrix[:, attack_params['num_steps'] - 1], [-1, 1])) logging.info('%d' % np.sum(final_success)) # logging.info('%d' % (np.sum(final_success))/count) logging.info("Shape of final matrix: %s" % str(final_matrix.shape)) # logging.info("Final accuracy %g:" % np.sum(final_success)/count) stats = { 'attack': 'pgd', 'count': count, 'epsilon': attack_params['epsilon'], 'num_steps': attack_params['num_steps'], 'step_size': attack_params['step_size'], 'natural_accuracy': float(natural_accuracy.item() / count), 'final_matrix': final_matrix.tolist(), 'robust_accuracy': float(np.sum(final_success) / count), 'restart_num': run } json_stats = { 'attack': 'pgd', 'natural_accuracy': float(natural_accuracy.item() / count), 'robust_accuracy': float(np.sum(final_success) / count) } with open(os.path.join(results_dir, 'stats.json'), 'w') as outfile: json.dump(json_stats, outfile) np.save(os.path.join(results_dir, 'stats' + str(run) + '.npy'), stats) np.save(os.path.join(results_dir, 'stats.npy'), stats) elif attack == 'cw': all_linf_distances = [] count = 0 for data, target, unsup in test_loader: logging.info('Batch: %g', count) count = count + 1 if count > args.num_eval_batches: break data, target = data.to(device), target.to(device) X, y = Variable(data, requires_grad=True), Variable(target) batch_linf_distances = cw( model, X, y, binary_search_steps=attack_params['binary_search_steps'], max_iterations=attack_params['max_iterations'], learning_rate=attack_params['learning_rate'], initial_const=attack_params['initial_const'], tau_decrease_factor=attack_params['tau_decrease_factor']) all_linf_distances.append(batch_linf_distances) np.savetxt(os.path.join(results_dir, 'CW_dist'), all_linf_distances) np.savetxt(os.path.join(results_dir, 'CW_dist'), all_linf_distances) else: raise ValueError("Attack not supported")