(batch_size, img_size, img_size, img_chan),
name='x_fixed')
env.adv_eps = tf.placeholder(tf.float32, (), name='adv_eps')
env.adv_epochs = tf.placeholder(tf.int32, (), name='adv_epochs')
env.adv_y = tf.placeholder(tf.int32, (), name='adv_y')
env.x_fgsm = fgm(model, env.x, epochs=env.adv_epochs, eps=env.adv_eps)
env.x_deepfool = deepfool(model, env.x, epochs=env.adv_epochs, batch=True)
env.x_jsma = jsma(model,
env.x,
env.adv_y,
eps=env.adv_eps,
epochs=env.adv_epochs)
env.cw_train_op, env.x_cw, env.cw_noise = cw(model,
env.x_fixed,
y=env.adv_y,
eps=env.adv_eps,
optimizer=optimizer)
print('\nInitializing graph')
sess = tf.InteractiveSession()
sess.run(tf.global_variables_initializer())
sess.run(tf.local_variables_initializer())
def evaluate(sess, env, X_data, y_data, batch_size=128):
"""
Evaluate TF model by running env.loss and env.acc.
"""
print('\nEvaluating')
env.train_op = optimizer.minimize(env.loss, var_list=vs)
env.saver = tf.train.Saver()
# Note here that the shape has to be fixed during the graph construction
# since the internal variable depends upon the shape.
env.x_fixed = tf.placeholder(tf.float32,
(batch_size, img_size, img_size, img_chan),
name='x_fixed')
env.adv_eps = tf.placeholder(tf.float32, (), name='adv_eps')
env.adv_y = tf.placeholder(tf.int32, (), name='adv_y')
optimizer = tf.train.AdamOptimizer(learning_rate=0.1)
env.adv_train_op, env.xadv, env.noise = cw(model,
env.x_fixed,
ord_='inf',
y=env.adv_y,
eps=env.adv_eps,
optimizer=optimizer)
print('\nInitializing graph')
env.sess = tf.InteractiveSession()
env.sess.run(tf.global_variables_initializer())
env.sess.run(tf.local_variables_initializer())
def evaluate(env, X_data, y_data, batch_size=128):
"""
Evaluate TF model by running env.loss and env.acc.
"""
print('\nEvaluating')
from attacks import fgsm_it, cw
from utils import stats, stats_cari, heat, heat_cari
from mnist_cnn import build_network, loss_func, evaluation
from tensorflow.examples.tutorials.mnist import input_data as mnist_data
from keras.datasets import cifar10
from tqdm import tqdm
ITERS = 100
ALPHA = 0.01
EPSI = 0.1
CW_ITERS = 100
BIN_STEPS = 20
NORM = 'l2' #l0/l2/linf
mnist = mnist_data.read_data_sets('MNIST_data', one_hot=True)
if len(sys.argv) < 2 or sys.argv[1] == "fgsm_it":
perturbed_accuracy = fgsm_it(mnist.test.images[:1000], ITERS, EPSI, ALPHA,
build_network, loss_func, evaluation)
print([s[0] for s in perturbed_accuracy])
stats([s[1] for s in perturbed_accuracy], mnist.test.labels[:1000])
heat([s[1] for s in perturbed_accuracy], mnist.test.labels[:1000],
"mnist_fgsm_")
else:
perturbed_norms, was = cw(mnist.test.images[:1000],
mnist.test.labels[:1000], CW_ITERS, BIN_STEPS,
build_network, loss_func, evaluation, NORM)
stats_cari(perturbed_norms, was)
heat_cari(perturbed_norms, was)
ITERS = 100
ALPHA = 0.01
EPSI = 0.1
CW_ITERS = 100
BIN_STEPS = 20
NORM = 'l2' #l0/l2/linf
(train_features, train_labels), (test_features,
test_labels) = cifar10.load_data()
(test_features, test_labels) = agu(test_features, test_labels)
train_features = train_features.astype('float32')
test_features = test_features.astype('float32')
train_features /= 255
test_features /= 255
test_labels = np_utils.to_categorical(test_labels, 10)
if len(sys.argv) < 2 or sys.argv[1] == "fgsm_it":
perturbed_accuracy = fgsm_it(test_features[:100], ITERS, EPSI, ALPHA,
build_network, loss_func, evaluation,
'./tmp/original_cifar_model-8')
print([s[0] for s in perturbed_accuracy])
stats([s[1] for s in perturbed_accuracy], test_labels[:1000])
heat([s[1] for s in perturbed_accuracy], test_labels[:1000],
"cifar10_fgsm_")
else:
perturbed_norms, was = cw(test_features[:100], test_labels[:100], CW_ITERS,
BIN_STEPS, build_network, loss_func, evaluation,
NORM, './tmp/original_cifar_model-8')
stats_cari(perturbed_norms, was)
def eval_adv_test(model, device, test_loader, attack, attack_params,
results_dir):
"""
evaluate model by white-box attack
"""
model.eval()
count = 0
if attack == 'pgd':
restarts_matrices = []
for run in range(attack_params['num_restarts']):
current_matrix_rows = []
count = 0
batch_num = 0
natural_accuracy = 0
for data, target, unsup in test_loader:
batch_num = batch_num + 1
if batch_num > args.num_eval_batches:
break
data, target = data.to(device), target.to(device)
count = count + len(target)
X, y = Variable(data, requires_grad=True), Variable(target)
# matrix_robust has batch_size*num_iterations dimensions
indices_natural, matrix_robust = pgd(
model,
X,
y,
epsilon=attack_params['epsilon'],
num_steps=attack_params['num_steps'],
step_size=attack_params['step_size'],
pretrain=args.pretrain,
random_start=attack_params['random_start'])
natural_accuracy = natural_accuracy + sum(indices_natural)
current_matrix_rows.append(matrix_robust)
logging.info('Completed restart: %g' % run)
current_matrix = np.concatenate(current_matrix_rows, axis=0)
logging.info("Shape of current_matrix %s" %
str(current_matrix.shape))
restarts_matrices.append(current_matrix)
final_matrix = np.asarray(restarts_matrices)
if run == 0:
final_success = np.ones([count, 1])
final_success = np.multiply(
final_success,
np.reshape(current_matrix[:, attack_params['num_steps'] - 1],
[-1, 1]))
logging.info('%d' % np.sum(final_success))
# logging.info('%d' % (np.sum(final_success))/count)
logging.info("Shape of final matrix: %s" % str(final_matrix.shape))
# logging.info("Final accuracy %g:" % np.sum(final_success)/count)
stats = {
'attack': 'pgd',
'count': count,
'epsilon': attack_params['epsilon'],
'num_steps': attack_params['num_steps'],
'step_size': attack_params['step_size'],
'natural_accuracy': float(natural_accuracy.item() / count),
'final_matrix': final_matrix.tolist(),
'robust_accuracy': float(np.sum(final_success) / count),
'restart_num': run
}
json_stats = {
'attack': 'pgd',
'natural_accuracy': float(natural_accuracy.item() / count),
'robust_accuracy': float(np.sum(final_success) / count)
}
with open(os.path.join(results_dir, 'stats.json'), 'w') as outfile:
json.dump(json_stats, outfile)
np.save(os.path.join(results_dir, 'stats' + str(run) + '.npy'),
stats)
np.save(os.path.join(results_dir, 'stats.npy'), stats)
elif attack == 'cw':
all_linf_distances = []
count = 0
for data, target, unsup in test_loader:
logging.info('Batch: %g', count)
count = count + 1
if count > args.num_eval_batches:
break
data, target = data.to(device), target.to(device)
X, y = Variable(data, requires_grad=True), Variable(target)
batch_linf_distances = cw(
model,
X,
y,
binary_search_steps=attack_params['binary_search_steps'],
max_iterations=attack_params['max_iterations'],
learning_rate=attack_params['learning_rate'],
initial_const=attack_params['initial_const'],
tau_decrease_factor=attack_params['tau_decrease_factor'])
all_linf_distances.append(batch_linf_distances)
np.savetxt(os.path.join(results_dir, 'CW_dist'),
all_linf_distances)
np.savetxt(os.path.join(results_dir, 'CW_dist'), all_linf_distances)
else:
raise ValueError("Attack not supported")
