def main(model_name, adv_model_names, model_type): np.random.seed(0) assert keras.backend.backend() == "tensorflow" set_flags(32) config = tf.ConfigProto() config.gpu_options.allow_growth = True K.set_session(tf.Session(config=config)) flags.DEFINE_integer('NUM_EPOCHS', args.epochs, 'Number of epochs') flags.DEFINE_integer('type', args.type, 'model type') # Get MNIST test data X_train, Y_train, X_test, Y_test = load_data() data_gen = data_flow(X_train) x = K.placeholder(shape=(None, FLAGS.NUM_CHANNELS, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS)) y = K.placeholder(shape=(FLAGS.BATCH_SIZE, FLAGS.NUM_CLASSES)) eps = args.eps # if src_models is not None, we train on adversarial examples that come # from multiple models adv_models = [None] * len(adv_model_names) for i in range(len(adv_model_names)): adv_models[i] = load_model(adv_model_names[i]) model = model_select(type=model_type) x_advs = [None] * (len(adv_models) + 1) for i, m in enumerate(adv_models + [model]): x_noise = x + tf.random_uniform(shape=[FLAGS.BATCH_SIZE, FLAGS.NUM_CHANNELS, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS], minval= -args.eps, maxval=args.eps) x_noise = tf.clip_by_value(x_noise, 0., 1.) for _ in range(args.k): logits = m(x_noise) grad = gen_grad(x_noise, logits, y, loss='logloss') x_noise = K.stop_gradient(x_noise + args.eps / 4.0 * K.sign(grad)) x_noise = tf.clip_by_value(x_noise, x - args.eps, x + args.eps) x_noise = tf.clip_by_value(x_noise, 0., 1.) x_advs[i] = x_noise # Train an MNIST model tf_train(x, y, model, X_train, Y_train, data_gen, model_name, x_advs=x_advs) # Finally print the result! test_error = tf_test_error_rate(model, x, X_test, Y_test) with open(model_name + '_log.txt', 'a') as log: log.write('Test error: %.1f%%' % test_error) print('Test error: %.1f%%' % test_error) save_model(model, model_name) json_string = model.to_json() with open(model_name+'.json', 'w') as f: f.write(json_string)
def main(model_name, model_type): np.random.seed(0) assert keras.backend.backend() == "tensorflow" set_mnist_flags() flags.DEFINE_bool('NUM_EPOCHS', args.epochs, 'Number of epochs') # Get MNIST test data X_train, Y_train, X_test, Y_test = data_mnist() data_gen = data_gen_mnist(X_train) x = K.placeholder( (None, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS)) y = K.placeholder(shape=(None, FLAGS.NUM_CLASSES)) model = model_mnist(type=model_type) # Train an MNIST model tf_train(x, y, model, X_train, Y_train, data_gen) # Finally print the result! test_error = tf_test_error_rate(model, x, X_test, Y_test) print('Test error: %.1f%%' % test_error) save_model(model, model_name) json_string = model.to_json() with open(model_name + '.json', 'wr') as f: f.write(json_string)
def main(): np.random.seed(0) assert keras.backend.backend() == "tensorflow" set_model_flags(False) tf.reset_default_graph() g = tf.get_default_graph() x = tf.placeholder(tf.float32, shape=[None, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS] ) y = tf.placeholder(tf.float32, shape=[None, FLAGS.NUM_CLASSES] ) train_mode = tf.placeholder(tf.bool) adv_model = adv_models(FLAGS.TYPE) ata = dataset('../Defense_Model/tiny-imagenet-200/', normalize = False) sess, graph_dict = tf_train(g, x, y, data, adv_model, train_mode) #tf_train returns the sess and graph_dict #tf_test_error_rate also need to run the sess and use the feed_dict in the tf_train #graph_dict is the dictiorary that contains all the items that is necessary on the graph # Finally print the result! test_error = tf_test_error_rate(sess, graph_dict, data) print('Test error: %.1f%%' % test_error) sess.close() del(g)
def main(model_name, adv_model_names, model_type): np.random.seed(0) assert keras.backend.backend() == "tensorflow" # Get MNIST test data X_train, Y_train, X_test, Y_test = data_mnist() data_gen = data_gen_mnist(X_train) x = K.placeholder(shape=(None, 28, 28, 1)) y = K.placeholder(shape=(BATCH_SIZE, 10)) eps = args.eps norm = args.norm # if src_models is not None, we train on adversarial examples that come # from multiple models adv_models = [None] * len(adv_model_names) ens_str = '' for i in range(len(adv_model_names)): adv_models[i] = load_model(adv_model_names[i]) if len(adv_models) > 0: name = basename(adv_model_names[i]) model_index = name.replace('model', '') ens_str += model_index model = model_mnist(type=model_type) x_advs = [None] * (len(adv_models) + 1) for i, m in enumerate(adv_models + [model]): if args.iter == 0: logits = m(x) grad = gen_grad(x, logits, y, loss='training') x_advs[i] = symbolic_fgs(x, grad, eps=eps) elif args.iter == 1: x_advs[i] = iter_fgs(m, x, y, steps=40, alpha=0.01, eps=args.eps) # Train an MNIST model tf_train(x, y, model, X_train, Y_train, data_gen, x_advs=x_advs, benign=args.ben) # Finally print the result! test_error = tf_test_error_rate(model, x, X_test, Y_test) print('Test error: %.1f%%' % test_error) model_name += '_' + str(eps) + '_' + str(norm) + '_' + ens_str if args.iter == 1: model_name += 'iter' if args.ben == 0: model_name += '_nob' save_model(model, model_name) json_string = model.to_json() with open(model_name + '.json', 'wr') as f: f.write(json_string)
def main(model_name, adv_model_names, model_type): np.random.seed(0) assert keras.backend.backend() == "tensorflow" set_mnist_flags() flags.DEFINE_bool('NUM_EPOCHS', args.epochs, 'Number of epochs') # Get MNIST test data X_train, Y_train, X_test, Y_test = data_mnist() data_gen = data_gen_mnist(X_train) x = K.placeholder(shape=(None, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS)) y = K.placeholder(shape=(FLAGS.BATCH_SIZE, FLAGS.NUM_CLASSES)) eps = args.eps # if src_models is not None, we train on adversarial examples that come # from multiple models adv_models = [None] * len(adv_model_names) for i in range(len(adv_model_names)): adv_models[i] = load_model(adv_model_names[i]) model = model_mnist(type=model_type) x_advs = [None] * (len(adv_models) + 1) for i, m in enumerate(adv_models + [model]): logits = m(x) grad = gen_grad(x, logits, y, loss='training') x_advs[i] = symbolic_fgs(x, grad, eps=eps) # Train an MNIST model tf_train(x, y, model, X_train, Y_train, data_gen, x_advs=x_advs) # Finally print the result! test_error = tf_test_error_rate(model, x, X_test, Y_test) print('Test error: %.1f%%' % test_error) save_model(model, model_name) json_string = model.to_json() with open(model_name + '.json', 'wr') as f: f.write(json_string)
def main(args): np.random.seed(0) assert keras.backend.backend() == "tensorflow" with tf.device('/gpu:0'): x_train, y_train, x_test, y_test = data_mnist() if args.dataset == "mnist" \ else data( path=args.dataset, representation=args.representation, test_path=args.test_path, n=args.n ) data_gen = data_gen_mnist(x_train) x = K.placeholder((None, ) + args.x_dim) y = K.placeholder(shape=(None, 10)) model = model_mnist(type=args.type) tf_train(x, y, model, x_train, y_train, data_gen, None, None) _, _, test_error = tf_test_error_rate(model(x), x, x_test, y_test) print('Test error: %.1f%%' % test_error) accuracy = 100 - test_error with open(args.accuracy_path, 'w+') as f: f.write(accuracy) save_model(model, args.model) json_string = model.to_json() try: with open(args.model + '.json', 'w') as f: f.write(json_string) except Exception: print(json_string)
def main(attack, src_model_name, target_model_name): np.random.seed(0) tf.set_random_seed(0) dim = 28 * 28 * 1 x = K.placeholder((None, 28, 28, 1)) y = K.placeholder((None, 10)) _, _, X_test, Y_test = data_mnist() Y_test_uncat = np.argmax(Y_test, axis=1) # source model for crafting adversarial examples src_model = load_model(src_model_name) # model(s) to target target_model = load_model(target_model_name) # simply compute test error if attack == "test": _, _, err = tf_test_error_rate(src_model, x, X_test, Y_test) print('{}: {:.1f}'.format(basename(src_model_name), err)) _, _, err = tf_test_error_rate(target_model, x, X_test, Y_test) print('{}: {:.1f}'.format(basename(target_model_name), err)) return if args.targeted_flag == 1: targets = [] allowed_targets = list(range(10)) for i in range(len(Y_test)): allowed_targets.remove(Y_test_uncat[i]) targets.append(np.random.choice(allowed_targets)) allowed_targets = list(range(10)) targets = np.array(targets) print(targets) targets_cat = np_utils.to_categorical(targets, 10).astype(np.float32) Y_test = targets_cat logits = src_model(x) print('logits', logits) if args.loss_type == 'xent': loss, grad = gen_grad_ens(x, logits, y) assert grad is not None elif args.loss_type == 'cw': grad = gen_grad_cw(x, logits, y) if args.targeted_flag == 1: grad = -1.0 * grad for eps in eps_list: # FGSM and RAND+FGSM one-shot attack if attack in ["fgs", "rand_fgs"] and args.norm == 'linf': assert grad is not None adv_x = symbolic_fgs(x, grad, eps=eps) elif attack in ["fgs", "rand_fgs"] and args.norm == 'l2': adv_x = symbolic_fg(x, grad, eps=eps) # iterative FGSM if attack == "ifgs": l = 1000 X_test = X_test[0:l] Y_test = Y_test[0:l] adv_x = x # iteratively apply the FGSM with small step size for i in range(args.num_iter): adv_logits = src_model(adv_x) if args.loss_type == 'xent': loss, grad = gen_grad_ens(adv_x, adv_logits, y) elif args.loss_type == 'cw': grad = gen_grad_cw(adv_x, adv_logits, y) if args.targeted_flag == 1: grad = -1.0 * grad adv_x = symbolic_fgs(adv_x, grad, args.delta, True) r = adv_x - x r = K.clip(r, -eps, eps) adv_x = x + r adv_x = K.clip(adv_x, 0, 1) print('Generating adversarial samples') X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] avg_l2_perturb = np.mean(np.linalg.norm((X_adv - X_test).reshape(len(X_test), dim), axis=1)) # white-box attack l = len(X_adv) print('Carrying out white-box attack') preds_adv, orig, err = tf_test_error_rate(src_model, x, X_adv, Y_test[0:l]) if args.targeted_flag == 1: err = 100.0 - err print('{}->{}: {:.1f}'.format(src_model_name, src_model_name, err)) # black-box attack if target_model_name is not None: print('Carrying out black-box attack') preds, _, err = tf_test_error_rate(target_model, x, X_adv, Y_test) if args.targeted_flag == 1: err = 100.0 - err print('{}->{}: {:.1f}, {}, {} {}'.format(src_model_name, basename(target_model_name), err, avg_l2_perturb, eps, attack))
def evaluate_2(): test_error = tf_test_error_rate(model, x, X_test, Y_test) print('Test error: %.3f%%' % test_error)
def main(model_name, model_type): np.random.seed(0) assert keras.backend.backend() == "tensorflow" set_mnist_flags() flags.DEFINE_bool('NUM_EPOCHS', args.epochs, 'Number of epochs') # Get MNIST test data X_train, Y_train, X_test, Y_test = data_mnist() # Initialize substitute training set reserved for adversary X_sub = X_test[:150] Y_sub = np.argmax(Y_test[:150], axis=1) # Redefine test set as remaining samples unavailable to adversaries X_test = X_test[150:] Y_test = Y_test[150:] data_gen = data_gen_mnist(X_train) x = K.placeholder( (None, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS)) y = K.placeholder(shape=(None, FLAGS.NUM_CLASSES)) model = model_mnist(type=model_type) # model = cnn_model() prediction = model(x) # Train an MNIST model # tf_train(x, y, model, X_train, Y_train, data_gen) train_params = { 'nb_epochs': args.epochs, 'batch_size': FLAGS.BATCH_SIZE, 'learning_rate': 0.001 } def evaluate_1(): eval_params = {'batch_size': FLAGS.BATCH_SIZE} test_accuracy = model_eval(K.get_session(), x, y, prediction, X_test, Y_test, args=eval_params) print('Test accuracy of blackbox on legitimate test ' 'examples: {:.3f}'.format(test_accuracy)) model_train(K.get_session(), x, y, model, X_train, Y_train, data_gen, evaluate=evaluate_1, args=train_params) save_model(model, model_name) json_string = model.to_json() with open(model_name + '.json', 'wr') as f: f.write(json_string) # Finally print the result! test_error = tf_test_error_rate(model, x, X_test, Y_test) print('Test error: %.1f%%' % test_error)
def main(attack, src_model_name, target_model_names, data_train_dir, data_test_dir): np.random.seed(0) tf.set_random_seed(0) flags.DEFINE_integer('BATCH_SIZE', 32, 'Size of batches') set_gtsrb_flags() # Get MNIST test data _, _, X_test, Y_test = load_data(data_train_dir, data_test_dir) # One-hot encode image labels label_binarizer = LabelBinarizer() Y_test = label_binarizer.fit_transform(Y_test) x = tf.placeholder(tf.float32, (None, 32, 32, 1)) y = tf.placeholder(tf.int32, (None)) one_hot_y = tf.one_hot(y, 43) # source model for crafting adversarial examples src_model = load_model(src_model_name) # model(s) to target target_models = [None] * len(target_model_names) for i in range(len(target_model_names)): target_models[i] = load_model(target_model_names[i]) # simply compute test error if attack == "test": err = tf_test_error_rate(src_model, x, X_test, Y_test) print '{}: {:.3f}'.format(basename(src_model_name), err) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_test, Y_test) print '{}: {:.3f}'.format(basename(name), err) return eps = args.eps # take the random step in the RAND+FGSM if attack == "rand_fgs": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha logits = src_model(x) grad = gen_grad(x, logits, y) # FGSM and RAND+FGSM one-shot attack if attack in ["fgs", "rand_fgs"]: adv_x = symbolic_fgs(x, grad, eps=eps) # iterative FGSM if attack == "ifgs": adv_x = iter_fgs(src_model, x, y, steps=args.steps, eps=args.eps / args.steps) # Carlini & Wagner attack if attack == "CW": X_test = X_test[0:1000] Y_test = Y_test[0:1000] cli = CarliniLi(K.get_session(), src_model, targeted=False, confidence=args.kappa, eps=args.eps) X_adv = cli.attack(X_test, Y_test) r = np.clip(X_adv - X_test, -args.eps, args.eps) X_adv = X_test + r err = tf_test_error_rate(src_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(src_model_name), err) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(name), err) return if attack == "grad_ens": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha sub_model_ens = (sub_model_1, sub_model_2) sub_models = [None] * len(sub_model_ens) for i in range(len(sub_model_ens)): sub_models[i] = load_model(sub_model_ens[i]) adv_x = x for j in range(args.steps): for i, m in enumerate(sub_models + [src_model]): logits = m(adv_x) gradient = gen_grad(adv_x, logits, y) adv_x = symbolic_fgs(adv_x, gradient, eps=args.eps / args.steps, clipping=True) # compute the adversarial examples and evaluate X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] # white-box attack err = tf_test_error_rate(src_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(src_model_name), err) # black-box attack for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(name), err)
def main(attack, src_model_name, target_model_names): np.random.seed(0) tf.set_random_seed(0) flags.DEFINE_integer('BATCH_SIZE', 10, 'Size of batches') set_mnist_flags() x = K.placeholder( (None, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS)) y = K.placeholder((None, FLAGS.NUM_CLASSES)) _, _, X_test, Y_test = data_mnist() # source model for crafting adversarial examples src_model = load_model(src_model_name) # model(s) to target target_models = [None] * len(target_model_names) for i in range(len(target_model_names)): target_models[i] = load_model(target_model_names[i]) # simply compute test error if attack == "test": err = tf_test_error_rate(src_model, x, X_test, Y_test) print '{}: {:.1f}'.format(basename(src_model_name), err) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_test, Y_test) print '{}: {:.1f}'.format(basename(name), err) return eps = args.eps # take the random step in the RAND+FGSM if attack == "rand_fgs": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha logits = src_model(x) grad = gen_grad(x, logits, y) # FGSM and RAND+FGSM one-shot attack if attack in ["fgs", "rand_fgs"]: adv_x = symbolic_fgs(x, grad, eps=eps) # iterative FGSM if attack == "ifgs": adv_x = iter_fgs(src_model, x, y, steps=args.steps, eps=args.eps / args.steps) # Carlini & Wagner attack if attack == "CW": X_test = X_test[0:1000] Y_test = Y_test[0:1000] cli = CarliniLi(K.get_session(), src_model, targeted=False, confidence=args.kappa, eps=args.eps) X_adv = cli.attack(X_test, Y_test) r = np.clip(X_adv - X_test, -args.eps, args.eps) X_adv = X_test + r err = tf_test_error_rate(src_model, x, X_adv, Y_test) print '{}->{}: {:.1f}'.format(basename(src_model_name), basename(src_model_name), err) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.1f}'.format(basename(src_model_name), basename(name), err) return # compute the adversarial examples and evaluate X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] # white-box attack err = tf_test_error_rate(src_model, x, X_adv, Y_test) print '{}->{}: {:.1f}'.format(basename(src_model_name), basename(src_model_name), err) # black-box attack for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.1f}'.format(basename(src_model_name), basename(name), err)
def main(attack, src_model_name, target_model_names): np.random.seed(0) tf.set_random_seed(0) set_flags(20) config = tf.ConfigProto() config.gpu_options.allow_growth = True K.set_session(tf.Session(config=config)) x = K.placeholder( (None, FLAGS.NUM_CHANNELS, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS)) y = K.placeholder((None, FLAGS.NUM_CLASSES)) _, _, X_test, Y_test = load_data() # source model for crafting adversarial examples src_model = load_model(src_model_name) # model(s) to target target_models = [None] * len(target_model_names) for i in range(len(target_model_names)): target_models[i] = load_model(target_model_names[i]) # simply compute test error if attack == "test": err = tf_test_error_rate(src_model, x, X_test, Y_test) print('{}: {:.1f}'.format(basename(src_model_name), 100 - err)) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_test, Y_test) print('{}: {:.1f}'.format(basename(name), 100 - err)) return eps = args.eps # take the random step in the RAND+FGSM if attack == "rfgs": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha logits = src_model(x) grad = gen_grad(x, logits, y) # FGSM and RAND+FGSM one-shot attack if attack in ["fgs", "rfgs"]: adv_x = symbolic_fgs(x, grad, eps=eps) # iterative FGSM if attack == "pgd": adv_x = iter_fgs(src_model, x, y, steps=args.steps, eps=args.eps, alpha=args.eps / 10.0) if attack == 'mim': adv_x = momentum_fgs(src_model, x, y, eps=args.eps) print('start') # compute the adversarial examples and evaluate X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] print('-----done----') # white-box attack err = tf_test_error_rate(src_model, x, X_adv, Y_test) print('{}->{}: {:.1f}'.format(basename(src_model_name), basename(src_model_name), 100 - err)) # black-box attack for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print('{}->{}: {:.1f}'.format(basename(src_model_name), basename(name), 100 - err))
def main(attack, src_model_names, target_model_name): np.random.seed(0) tf.set_random_seed(0) flags.DEFINE_integer('BATCH_SIZE', 1, 'Size of batches') set_mnist_flags() dim = FLAGS.IMAGE_ROWS * FLAGS.IMAGE_COLS * FLAGS.NUM_CHANNELS x = K.placeholder( (None, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS)) y = K.placeholder((None, FLAGS.NUM_CLASSES)) _, _, X_test, Y_test = data_mnist() Y_test_uncat = np.argmax(Y_test, axis=1) # source model for crafting adversarial examples src_models = [None] * len(src_model_names) for i in range(len(src_model_names)): src_models[i] = load_model(src_model_names[i]) src_model_name_joint = '' for i in range(len(src_models)): src_model_name_joint += basename(src_model_names[i]) # model(s) to target if target_model_name is not None: target_model = load_model(target_model_name) # simply compute test error if attack == "test": for (name, src_model) in zip(src_model_names, src_models): _, _, err = tf_test_error_rate(src_model, x, X_test, Y_test) print '{}: {:.1f}'.format(basename(name), err) if target_model_name is not None: _, _, err = tf_test_error_rate(target_model, x, X_test, Y_test) print '{}: {:.1f}'.format(basename(target_model_name), err) return if args.targeted_flag == 1: pickle_name = attack + '_' + src_model_name_joint + '_' + '_' + args.loss_type + '_targets.p' if os.path.exists(pickle_name): targets = pickle.load(open(pickle_name, 'rb')) else: targets = [] allowed_targets = list(range(FLAGS.NUM_CLASSES)) for i in range(len(Y_test)): allowed_targets.remove(Y_test_uncat[i]) targets.append(np.random.choice(allowed_targets)) allowed_targets = list(range(FLAGS.NUM_CLASSES)) # targets = np.random.randint(10, size = BATCH_SIZE*BATCH_EVAL_NUM) targets = np.array(targets) print targets targets_cat = np_utils.to_categorical( targets, FLAGS.NUM_CLASSES).astype(np.float32) Y_test = targets_cat if SAVE_FLAG == True: pickle.dump(Y_test, open(pickle_name, 'wb')) # take the random step in the RAND+FGSM if attack == "rand_fgs": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha logits = [None] * len(src_model_names) for i in range(len(src_model_names)): curr_model = src_models[i] logits[i] = curr_model(x) if args.loss_type == 'xent': loss, grad = gen_grad_ens(x, logits, y) elif args.loss_type == 'cw': grad = gen_grad_cw(x, logits, y) if args.targeted_flag == 1: grad = -1.0 * grad for eps in eps_list: # FGSM and RAND+FGSM one-shot attack if attack in ["fgs", "rand_fgs"] and args.norm == 'linf': adv_x = symbolic_fgs(x, grad, eps=eps) elif attack in ["fgs", "rand_fgs"] and args.norm == 'l2': adv_x = symbolic_fg(x, grad, eps=eps) # iterative FGSM if attack == "ifgs": l = 1000 X_test = X_test[0:l] Y_test = Y_test[0:l] adv_x = x # iteratively apply the FGSM with small step size for i in range(args.num_iter): adv_logits = [None] * len(src_model_names) for i in range(len(src_model_names)): curr_model = src_models[i] adv_logits[i] = curr_model(adv_x) if args.loss_type == 'xent': loss, grad = gen_grad_ens(adv_x, adv_logits, y) elif args.loss_type == 'cw': grad = gen_grad_cw(adv_x, adv_logits, y) if args.targeted_flag == 1: grad = -1.0 * grad adv_x = symbolic_fgs(adv_x, grad, args.delta, True) r = adv_x - x r = K.clip(r, -eps, eps) adv_x = x + r adv_x = K.clip(adv_x, 0, 1) if attack == "CW_ens": l = 1000 pickle_name = attack + '_' + src_model_name_joint + '_' + str( args.eps) + '_adv.p' print(pickle_name) Y_test = Y_test[0:l] if os.path.exists(pickle_name) and attack == "CW_ens": print 'Loading adversarial samples' X_adv = pickle.load(open(pickle_name, 'rb')) for (name, src_model) in zip(src_model_names, src_models): preds_adv, _, err = tf_test_error_rate( src_model, x, X_adv, Y_test) print '{}->{}: {:.1f}'.format(src_model_name_joint, basename(name), err) preds_adv, _, err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.1f}'.format(src_model_name_joint, basename(target_model_name), err) return X_test = X_test[0:l] time1 = time() cli = CarliniLiEns(K.get_session(), src_models, targeted=False, confidence=args.kappa, eps=eps) X_adv = cli.attack(X_test, Y_test) r = np.clip(X_adv - X_test, -eps, eps) X_adv = X_test + r time2 = time() print("Run with Adam took {}s".format(time2 - time1)) if SAVE_FLAG == True: pickle.dump(X_adv, open(pickle_name, 'wb')) for (name, src_model) in zip(src_model_names, src_models): print('Carrying out white-box attack') pres, _, err = tf_test_error_rate(src_model, x, X_adv, Y_test) print '{}->{}: {:.1f}'.format(src_model_name_joint, basename(name), err) if target_model_name is not None: print('Carrying out black-box attack') preds, orig, err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.1f}'.format(src_model_name_joint, basename(target_model_name), err) return pickle_name = attack + '_' + src_model_name_joint + '_' + args.loss_type + '_' + str( eps) + '_adv.p' if args.targeted_flag == 1: pickle_name = attack + '_' + src_model_name_joint + '_' + args.loss_type + '_' + str( eps) + '_adv_t.p' if os.path.exists(pickle_name): print 'Loading adversarial samples' X_adv = pickle.load(open(pickle_name, 'rb')) else: print 'Generating adversarial samples' X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] if SAVE_FLAG == True: pickle.dump(X_adv, open(pickle_name, 'wb')) avg_l2_perturb = np.mean( np.linalg.norm((X_adv - X_test).reshape(len(X_test), dim), axis=1)) # white-box attack l = len(X_adv) print('Carrying out white-box attack') for (name, src_model) in zip(src_model_names, src_models): preds_adv, orig, err = tf_test_error_rate(src_model, x, X_adv, Y_test[0:l]) if args.targeted_flag == 1: err = 100.0 - err print '{}->{}: {:.1f}'.format(basename(name), basename(name), err) # black-box attack if target_model_name is not None: print('Carrying out black-box attack') preds, _, err = tf_test_error_rate(target_model, x, X_adv, Y_test) if args.targeted_flag == 1: err = 100.0 - err print '{}->{}: {:.1f}, {}, {} {}'.format( src_model_name_joint, basename(target_model_name), err, avg_l2_perturb, eps, attack)
def main(attack, src_model_name, target_model_names, data_train_dir, data_test_dir): np.random.seed(0) tf.set_random_seed(0) set_gtsrb_flags() # Get GTSRB test data _, _, _, _, X_test, Y_test = load_data(data_train_dir, data_test_dir) # display_leg_sample(X_test) # One-hot encode image labels label_binarizer = LabelBinarizer() Y_test = label_binarizer.fit_transform(Y_test) x = K.placeholder( (None, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS)) y = K.placeholder((None, FLAGS.NUM_CLASSES)) # one_hot_y = tf.one_hot(y, 43) # source model for crafting adversarial examples src_model = load_model(src_model_name) # model(s) to target target_models = [None] * len(target_model_names) for i in range(len(target_model_names)): target_models[i] = load_model(target_model_names[i]) # simply compute test error if attack == "test": err = tf_test_error_rate(src_model, x, X_test, Y_test) print '{}: {:.3f}'.format(basename(src_model_name), err) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_test, Y_test) print '{}: {:.3f}'.format(basename(name), err) return eps = args.eps # take the random step in the RAND+FGSM if attack == "rand_fgs": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha logits = src_model(x) grad = gen_grad(x, logits, y) # FGSM and RAND+FGSM one-shot attack if attack in ["fgs", "rand_fgs"]: adv_x = symbolic_fgs(x, grad, eps=eps) # iterative FGSM if attack == "ifgs": adv_x = iter_fgs(src_model, x, y, steps=args.steps, eps=args.eps / args.steps) # Carlini & Wagner attack if attack == "CW": X_test = X_test[0:200] Y_test = Y_test[0:200] cli = CarliniLi(K.get_session(), src_model, targeted=False, confidence=args.kappa, eps=args.eps) X_adv = cli.attack(X_test, Y_test) r = np.clip(X_adv - X_test, -args.eps, args.eps) X_adv = X_test + r np.save('Train_Carlini_200.npy', X_adv) np.save('Label_Carlini_200.npy', Y_test) err = tf_test_error_rate(src_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(src_model_name), err) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(name), err) display_leg_adv_sample(X_test, X_adv) return if attack == "cascade_ensemble": # X_test = np.clip( # X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), # 0.0, 1.0) # eps -= args.alpha sub_model_ens = (sub_model_2, sub_model_3) sub_models = [None] * len(sub_model_ens) for i in range(len(sub_model_ens)): sub_models[i] = load_model(sub_model_ens[i]) adv_x = x for j in range(args.steps): for i, m in enumerate(sub_models + [src_model]): logits = m(adv_x) gradient = gen_grad(adv_x, logits, y) adv_x = symbolic_fgs(adv_x, gradient, eps=args.eps / args.steps, clipping=True) if attack == "Iter_Casc": # X_test = np.clip( # X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), # 0.0, 1.0) # args.eps = args.eps - args.alpha sub_model_ens = (sub_model_1, sub_model_2, sub_model_3) sub_models = [None] * len(sub_model_ens) for i in range(len(sub_model_ens)): sub_models[i] = load_model(sub_model_ens[i]) x_advs = [None] * len(sub_models) errs = [None] * len(sub_models) adv_x = x eps_all = [] for i in range(args.steps): if i == 0: eps_all[0] = (1.0 / len(sub_models)) * args.eps else: for j in range(i): pre_sum = 0.0 pre_sum += eps_all[j] eps_all[i] = (args.eps - pre_sum) * (1.0 / len(sub_models)) # for i in range(args.steps): # if i == 0: # eps_0 = (1.0 / len(sub_models)) * args.eps # eps_all.append(eps_0) # elif i == 1: # eps_1 = (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models)) * args.eps # eps_all.append(eps_1) # elif i == 2: # eps_2 = (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models)) * args.eps # eps_all.append(eps_2) # elif i == 3: # eps_3 = (1 - (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models))) * ( # 1.0 / len(sub_models)) * args.eps # eps_all.append(eps_3) # elif i == 4: # eps_4 = (1 - ( # 1 - (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models))) * ( # 1.0 / len(sub_models))) * (1.0 / len(sub_models)) * args.eps # eps_all.append(eps_4) # elif i == 5: # eps_5 = (1 - (1 - ( # 1 - (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models))) * ( # 1.0 / len(sub_models)))) * (1.0 / len(sub_models)) * args.eps # eps_all.append(eps_5) # elif i == 6: # eps_6 = (1 - (1 - (1 - ( # 1 - (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models))) * ( # 1.0 / len(sub_models))))) * (1.0 / len(sub_models)) * args.eps # eps_all.append(eps_6) # # elif i == 7: # eps_7 = (1 - (1 - (1 - (1 - ( # 1 - (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models))) * ( # 1.0 / len(sub_models)))))) * (1.0 / len(sub_models)) * args.eps # eps_all.append(eps_7) # elif i == 8: # eps_8 = (1 - (1 - (1 - (1 - (1 - ( # 1 - (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models))) * ( # 1.0 / len(sub_models))))))) * (1.0 / len(sub_models)) * args.eps # eps_all.append(eps_8) # elif i == 9: # eps_9 = (1 - (1 - (1 - (1 - (1 - (1 - ( # 1 - (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models))) * ( # 1.0 / len(sub_models)))))))) * ( # 1.0 / len(sub_models)) * args.eps # eps_all.append(eps_9) # elif i == 10: # eps_10 = (1 - (1 - (1 - (1 - (1 - (1 - (1 - ( # 1 - (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models))) * ( # 1.0 / len(sub_models))))))))) * ( # 1.0 / len(sub_models)) * args.eps # eps_all.append(eps_10) # elif i == 11: # eps_11 = (1 - (1 - (1 - (1 - (1 - (1 - (1 - (1 - ( # 1 - (1 - (1 - 1.0 / len(sub_models)) * (1.0 / len(sub_models))) * (1.0 / len(sub_models))) * ( # 1.0 / len(sub_models)))))))))) * ( # 1.0 / len(sub_models)) * args.eps # eps_all.append(eps_11) for j in range(args.steps): print('iterative step is :', j) if j == 0: for i, m in enumerate(sub_models): logits = m(adv_x) gradient = gen_grad(adv_x, logits, y) adv_x_ = symbolic_fgs(adv_x, gradient, eps=eps_all[j], clipping=True) x_advs[i] = adv_x_ X_adv = batch_eval([x, y], [adv_x_], [X_test, Y_test])[0] err = tf_test_error_rate(m, x, X_adv, Y_test) errs[i] = err adv_x = x_advs[errs.index(min(errs))] else: t = errs.index(min(errs)) print('index of min value of errs:', t) logits = sub_models[t](adv_x) gradient = gen_grad(adv_x, logits, y) adv_x = symbolic_fgs(adv_x, gradient, eps=eps_all[j], clipping=True) for i, m in enumerate(sub_models): X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] err = tf_test_error_rate(m, x, X_adv, Y_test) errs[i] = err print('error rate of each substitute models_oldest: ', errs) print('\t') if min(errs) >= 99: success_rate = sum(errs) / len(sub_models) print('success rate is: {:.3f}'.format(success_rate)) break success_rate = sum(errs) / len(sub_models) print('success rate is: {:.3f}'.format(success_rate)) X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] np.save('results/iter_casc_0.2_leg_adv/X_adv_Iter_Casc_0.2.npy', X_adv) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(name), err) save_leg_adv_sample('results/iter_casc_0.2_leg_adv/', X_test, X_adv) # save adversarial example specified by user save_leg_adv_specified_by_user( 'results/iter_casc_0.2_leg_adv_label_4/', X_test, X_adv, Y_test) return if attack == "stack_paral": # X_test = np.clip( # X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), # 0.0, 1.0) # eps -= args.alpha sub_model_ens = (sub_model_1, sub_model_2, sub_model_3) sub_models = [None] * len(sub_model_ens) for i in range(len(sub_model_ens)): sub_models[i] = load_model(sub_model_ens[i]) errs = [None] * (len(sub_models) + 1) x_advs = [None] * len(sub_models) # print x_advs for i, m in enumerate(sub_models): # x = x + args.alpha * np.sign(np.random.randn(*x[0].shape)) logits = m(x) gradient = gen_grad(x, logits, y) adv_x = symbolic_fgs(x, gradient, eps=args.eps / 2, clipping=True) x_advs[i] = adv_x # print x_advs adv_x_sum = x_advs[0] for i in range(len(sub_models)): if i == 0: continue adv_x_sum = adv_x_sum + x_advs[i] adv_x_mean = adv_x_sum / (len(sub_models)) preds = src_model(adv_x_mean) grads = gen_grad(adv_x_mean, preds, y) adv_x = symbolic_fgs(adv_x_mean, grads, eps=args.eps, clipping=True) # compute the adversarial examples and evaluate for i, m in enumerate(sub_models + [src_model]): X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] err = tf_test_error_rate(m, x, X_adv, Y_test) errs[i] = err # compute success rate success_rate = sum(errs) / (len(sub_models) + 1) print('success rate is: {:.3f}'.format(success_rate)) # compute transfer rate for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(name), err) # save adversarial examples np.save('results/stack_paral_0.2_leg_adv/X_adv_stack_paral_0.2.npy', X_adv) # save_leg_adv_sample(X_test, X_adv) save_leg_adv_sample('results/stack_paral_0.2_leg_adv/', X_test, X_adv) # save adversarial example specified by user save_leg_adv_specified_by_user( 'results/stack_paral_0.2_leg_adv_label_4/', X_test, X_adv, Y_test) return if attack == "cascade_ensemble_2": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha sub_model_ens = (sub_model_1, sub_model_2) sub_models = [None] * len(sub_model_ens) for i in range(len(sub_model_ens)): sub_models[i] = load_model(sub_model_ens[i]) x_advs = [([None] * len(sub_models)) for i in range(args.steps)] # print x_advs x_adv = x for j in range(args.steps): for i, m in enumerate(sub_models): logits = m(x_adv) gradient = gen_grad(x_adv, logits, y) x_adv = symbolic_fgs(x_adv, gradient, eps=args.eps / args.steps, clipping=True) x_advs[j][i] = x_adv # print x_advs adv_x_sum = x_advs[0][0] for j in range(args.steps): for i in range(len(sub_models)): if j == 0 and i == 0: continue adv_x_sum = adv_x_sum + x_advs[j][i] adv_x_mean = adv_x_sum / (args.steps * len(sub_models)) preds = src_model(adv_x_mean) grads = gen_grad(adv_x_mean, preds, y) adv_x = symbolic_fgs(adv_x_mean, grads, eps=args.eps / args.steps, clipping=True) # compute the adversarial examples and evaluate X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] # white-box attack err = tf_test_error_rate(src_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(src_model_name), err) # black-box attack for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(name), err)
def main(attack, src_model_name, target_model_names): np.random.seed(0) tf.set_random_seed(0) flags.DEFINE_integer('BATCH_SIZE', 32, 'Size of batches') set_mnist_flags() x = K.placeholder( (None, FLAGS.IMAGE_ROWS, FLAGS.IMAGE_COLS, FLAGS.NUM_CHANNELS)) y = K.placeholder((None, FLAGS.NUM_CLASSES)) _, _, X_test, Y_test = data_mnist() # source model for crafting adversarial examples src_model = load_model(src_model_name) # model(s) to target target_models = [None] * len(target_model_names) for i in range(len(target_model_names)): target_models[i] = load_model(target_model_names[i]) # simply compute test error if attack == "test": err = tf_test_error_rate(src_model, x, X_test, Y_test) print '{}: {:.3f}'.format(basename(src_model_name), err) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_test, Y_test) print '{}: {:.3f}'.format(basename(name), err) return eps = args.eps # take the random step in the RAND+FGSM if attack == "rand_fgs": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha logits = src_model(x) grad = gen_grad(x, logits, y) # FGSM and RAND+FGSM one-shot attack if attack in ["fgs", "rand_fgs"]: adv_x = symbolic_fgs(x, grad, eps=eps) # iterative FGSM if attack == "ifgs": adv_x = iter_fgs(src_model, x, y, steps=args.steps, eps=args.eps / args.steps) # Carlini & Wagner attack if attack == "CW": X_test = X_test[0:1000] Y_test = Y_test[0:1000] cli = CarliniLi(K.get_session(), src_model, targeted=False, confidence=args.kappa, eps=args.eps) X_adv = cli.attack(X_test, Y_test) r = np.clip(X_adv - X_test, -args.eps, args.eps) X_adv = X_test + r err = tf_test_error_rate(src_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(src_model_name), err) for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(name), err) return if attack == "cascade_ensemble": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha sub_model_ens = (sub_model_1, sub_model_2, sub_model_3, sub_model_4, sub_model_5, sub_model_6, sub_model_7) sub_models = [None] * len(sub_model_ens) for i in range(len(sub_model_ens)): sub_models[i] = load_model(sub_model_ens[i]) adv_x = x for j in range(args.steps): for i, m in enumerate(sub_models + [src_model]): logits = m(adv_x) gradient = gen_grad(adv_x, logits, y) adv_x = symbolic_fgs(adv_x, gradient, eps=args.eps / args.steps, clipping=True) if attack == "parallel_ensemble": X_test = np.clip( X_test + args.alpha * np.sign(np.random.randn(*X_test.shape)), 0.0, 1.0) eps -= args.alpha sub_model_ens = (sub_model_1, sub_model_2, sub_model_3) sub_models = [None] * len(sub_model_ens) for i in range(len(sub_model_ens)): sub_models[i] = load_model(sub_model_ens[i]) x_advs = [([None] * len(sub_models)) for i in range(args.steps)] print x_advs x_adv = x for j in range(args.steps): for i, m in enumerate(sub_models): logits = m(x_adv) gradient = gen_grad(x_adv, logits, y) x_adv = symbolic_fgs(x_adv, gradient, eps=args.eps / args.steps, clipping=True) x_advs[j][i] = x_adv print x_advs adv_x_mean = x_advs[0][0] for j in range(args.steps): for i in range(len(sub_models)): if j == 0 and i == 0: continue adv_x_mean = adv_x_mean + x_advs[j][i] xadv = adv_x_mean / (args.steps * len(sub_models)) preds = src_model(xadv) grads = gen_grad(xadv, preds, y) adv_x = symbolic_fgs(xadv, grads, eps=args.eps, clipping=True) # compute the adversarial examples and evaluate X_adv = batch_eval([x, y], [adv_x], [X_test, Y_test])[0] # white-box attack err = tf_test_error_rate(src_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(src_model_name), err) # black-box attack for (name, target_model) in zip(target_model_names, target_models): err = tf_test_error_rate(target_model, x, X_adv, Y_test) print '{}->{}: {:.3f}'.format(basename(src_model_name), basename(name), err)