def run_eval(checkpoint, train=False, num_eval_examples=None, data_dir=None):
mnist = input_data.read_data_sets('MNIST_data', one_hot=False)
model = Model(mode='eval')
saver = tf.train.Saver()
eval_batch_size = 64
total_corr = 0
cifar = cifar10_input.CIFAR10Data(data_dir)
if train:
x_nat = cifar.train_data.xs
y_nat = cifar.train_data.ys
else:
x_nat = cifar.eval_data.xs
y_nat = cifar.eval_data.ys
if num_eval_examples is None:
num_eval_examples = len(x_nat)
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
y_pred = [] # label accumulator
red_neck = []
loj_eet = []
with tf.Session() as sess:
# Restore the checkpoint
saver.restore(sess, checkpoint)
# Iterate over the samples batch-by-batch
for ibatch in tqdm(range(num_batches)):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
x_batch = x_nat[bstart:bend, :]
y_batch = y_nat[bstart:bend]
dict_clean = {model.x_input: x_batch,
model.y_input: y_batch}
cur_corr, y_pred_batch, neck, logitsez = sess.run([model.num_correct, model.y_pred, model.neck, model.pre_softmax],
feed_dict=dict_clean)
total_corr += cur_corr
y_pred.append(y_pred_batch)
red_neck.append(neck)
loj_eet.append(logitsez)
# print('ibatch: ', ibatch, '/', num_batches)
accuracy = total_corr / num_eval_examples
print('Accuracy: {:.2f}%'.format(100.0 * accuracy))
test_str = 'test'
if train:
test_str = 'train'
np.save(sys.argv[1] + test_str + '_red_neck.npy', np.concatenate(red_neck,
axis=0))
np.save(sys.argv[1] + test_str + '_logits.npy', np.concatenate(loj_eet,
axis=0))
def run_attack(checkpoint, x_adv, epsilon):
cifar = cifar10_input.CIFAR10Data(data_path)
model = Model(num_classes=10)
saver = tf.train.Saver()
num_eval_examples = 10000
eval_batch_size = 100
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
total_corr = 0
x_nat = cifar.eval_data.xs
l_inf = np.amax(np.abs(x_nat - x_adv))
if l_inf > epsilon + 0.0001:
print('maximum perturbation found: {}'.format(l_inf))
print('maximum perturbation allowed: {}'.format(epsilon))
return
y_pred = [] # label accumulator
with tf.Session() as sess:
# Restore the checkpoint
saver.restore(sess, checkpoint)
# Iterate over the samples batch-by-batch
for ibatch in range(num_batches):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
x_batch = x_adv[bstart:bend, :]
y_batch = cifar.eval_data.ys[bstart:bend]
y_batch = np.eye(10)[y_batch]
dict_adv = {
model.x_input: x_batch,
model.is_training: False,
model.y_input: y_batch
}
cur_corr, y_pred_batch = sess.run(
[model.num_correct, model.predictions], feed_dict=dict_adv)
total_corr += cur_corr
y_pred.append(y_pred_batch)
accuracy = total_corr / num_eval_examples
print('Accuracy: {:.2f}%'.format(100.0 * accuracy))
y_pred = np.concatenate(y_pred, axis=0)
np.save('pred.npy', y_pred)
print('Output saved at pred.npy')
def exp_over_grid_eval(model,
attack,
sess,
config,
attack_type,
eval_on_train=False):
num_eval_examples = config.eval.num_eval_examples
eval_batch_size = config.eval.batch_size
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.CIFAR10Data(config.data.data_path)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.CIFAR100Data(config.data.data_path)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.SVHNData(config.data.data_path)
else:
raise ValueError("Unknown dataset name.")
# Iterate over the samples batch-by-batch
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
n_grid = np.prod(config.attack.grid_granularity)
results = np.zeros([num_eval_examples, n_grid])
for ibatch in trange(num_batches):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
if eval_on_train:
x_batch = data_iterator.train_data.xs[bstart:bend, :]
y_batch = data_iterator.train_data.ys[bstart:bend]
else:
x_batch = data_iterator.eval_data.xs[bstart:bend, :]
y_batch = data_iterator.eval_data.ys[bstart:bend]
grid = product(*list(
np.linspace(-l, l, num=g) for l, g in zip(
config.attack.spatial_limits, config.attack.grid_granularity)))
n_batch = len(x_batch)
for i, (tx, ty, r) in enumerate(grid):
trans = np.stack(repeat([tx, ty, r], n_batch))
dict_nat = {
model.x_input: x_batch,
model.y_input: y_batch,
model.transform: trans,
model.is_training: False
}
results[bstart:bend, i] = sess.run(model.correct_prediction,
feed_dict=dict_nat)
return results
def run_attack(checkpoint, x_adv, epsilon):
cifar = cifar10_input.CIFAR10Data(data_path)
model = Model(mode='eval')
saver = tf.train.Saver()
num_eval_examples = adv_count
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
total_corr = 0
# another round of check
x_nat = cifar.eval_data.xs[:adv_count, :, :, :]
l_inf = np.amax(np.abs(x_nat - x_adv))
if l_inf > epsilon + 0.0001:
print('maximum perturbation found: {}'.format(l_inf))
print('maximum perturbation allowed: {}'.format(epsilon))
return
y_pred = [] # label accumulator
with tf.Session() as sess:
saver.restore(sess, checkpoint)
print("model restored")
for ibatch in range(num_batches):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
x_batch = x_adv[bstart:bend, :]
y_batch = cifar.eval_data.ys[bstart:bend]
print("bStart = %d, bEnd = %d, bSize = %d" % (bstart, bend, bend - bstart))
dict_adv = {model.x_input: x_batch, model.y_input: y_batch}
cur_corr, y_pred_batch = sess.run([model.num_correct, model.predictions],
feed_dict=dict_adv)
total_corr += cur_corr
y_pred.append(y_pred_batch)
accuracy = total_corr / num_eval_examples
print('Accuracy: {:.2f}%'.format(100.0 * accuracy))
y_pred = np.concatenate(y_pred, axis=0)
np.save('pred.npy', y_pred)
print('Output saved at pred.npy')
def exp_over_grid_eval_random_sample(model,
sess,
config,
eval_on_train=False,
num_eval_examples=100,
seed=1):
np.random.seed(seed)
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.CIFAR10Data(config.data.data_path)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.CIFAR100Data(config.data.data_path)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.SVHNData(config.data.data_path)
else:
raise ValueError("Unknown dataset name.")
if eval_on_train:
n = data_iterator.train_data.n
indices = np.random.choice(n, num_eval_examples)
x_batch = data_iterator.train_data.xs[indices, :]
y_batch = data_iterator.train_data.ys[indices]
else:
n = data_iterator.eval_data.n
indices = np.random.choice(n, num_eval_examples)
x_batch = data_iterator.eval_data.xs[indices, :]
y_batch = data_iterator.eval_data.ys[indices]
grid = product(*list(
np.linspace(-l, l, num=g) for l, g in zip(
config.attack.spatial_limits, config.attack.grid_granularity)))
n_grid = np.prod(config.attack.grid_granularity)
results = np.zeros([len(x_batch), n_grid])
n_batch = len(x_batch)
for i, (tx, ty, r) in enumerate(grid):
trans = np.stack(repeat([tx, ty, r], n_batch))
dict_nat = {
model.x_input: x_batch,
model.y_input: y_batch,
model.transform: trans,
model.is_training: False
}
results[:, i] = sess.run(model.correct_prediction, feed_dict=dict_nat)
return results
def get_model(theta_ratio):
img_size = 32
batch_size = config['eval_batch_size']
input_images = tf.placeholder(tf.float32,
shape=(batch_size, img_size, img_size, 3))
input_label = tf.placeholder(tf.int64, shape=(batch_size))
## MODEL to be attacked ##
model = Model_crop(config,
mode='eval',
input_images=input_images,
input_label=input_label,
theta_ratio=theta_ratio)
## training starts ###
FLAGS = tf.app.flags.FLAGS
tfconfig = tf.ConfigProto(
allow_soft_placement=True,
log_device_placement=True,
)
tfconfig.gpu_options.allow_growth = True
sess = tf.Session(config=tfconfig)
init = tf.global_variables_initializer()
sess.run(init)
saver = tf.train.Saver()
saver.restore(
sess,
'/home/hope-yao/Documents/adversarial_defense/cifar/ckpt/crop_4_20_adv/half_half/lr_config1_adv/checkpoint-25001'
)
import cifar10_input
if 1:
bstart = 0
bend = bstart + batch_size
data_path = config['data_path']
cifar = cifar10_input.CIFAR10Data(data_path)
x_batch_eval = cifar.eval_data.xs[bstart:bend, :]
x_batch_eval = np.asarray(x_batch_eval, 'float32') / 255.
y_batch_eval = cifar.eval_data.ys[bstart:bend]
nat_dict_eval = {input_images: x_batch_eval, input_label: y_batch_eval}
grad = sess.run(model.adv_grad, feed_dict=nat_dict_eval)
return grad, sess, model, input_images, input_label, nat_dict_eval
def evaluate(model, eval_attacks, sess, config, plot=False, summary_writer=None, eval_train=False, eval_validation=False, verbose=True):
num_eval_examples = config['num_eval_examples']
eval_batch_size = config['eval_batch_size']
dataset = config["data"]
assert dataset in ["mnist", "cifar10"]
if dataset == "mnist":
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=False)
if "model_type" in config and config["model_type"] == "linear":
x_train = mnist.train.images
y_train = mnist.train.labels
x_test = mnist.test.images
y_test = mnist.test.labels
pos_train = (y_train == 5) | (y_train == 7)
x_train = x_train[pos_train]
y_train = y_train[pos_train]
y_train = (y_train == 5).astype(np.int64)
pos_test = (y_test == 5) | (y_test == 7)
x_test = x_test[pos_test]
y_test = y_test[pos_test]
y_test = (y_test == 5).astype(np.int64)
from tensorflow.contrib.learn.python.learn.datasets.mnist import DataSet
from tensorflow.contrib.learn.python.learn.datasets import base
options = dict(dtype=tf.uint8, reshape=False, seed=None)
train = DataSet(x_train, y_train, **options)
test = DataSet(x_test, y_test, **options)
mnist = base.Datasets(train=train, validation=None, test=test)
else:
import cifar10_input
data_path = config["data_path"]
cifar = cifar10_input.CIFAR10Data(data_path)
np.random.seed(0)
tf.random.set_random_seed(0)
global_step = tf.contrib.framework.get_or_create_global_step()
# Iterate over the samples batch-by-batch
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
total_xent_nat = 0.
total_xent_advs = np.zeros(len(eval_attacks), dtype=np.float32)
total_corr_nat = 0.
total_corr_advs = [[] for _ in range(len(eval_attacks))]
l1_norms = [[] for _ in range(len(eval_attacks))]
l2_norms = [[] for _ in range(len(eval_attacks))]
linf_norms = [[] for _ in range(len(eval_attacks))]
for ibatch in range(num_batches):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
if eval_train:
if dataset == "mnist":
x_batch = mnist.train.images[bstart:bend, :].reshape(-1, 28, 28, 1)
y_batch = mnist.train.labels[bstart:bend]
else:
x_batch = cifar.train_data.xs[bstart:bend, :].astype(np.float32)
y_batch = cifar.train_data.ys[bstart:bend]
elif eval_validation:
assert dataset == "cifar10"
offset = len(cifar.eval_data.ys) - num_eval_examples
x_batch = cifar.eval_data.xs[offset+bstart:offset+bend, :].astype(np.float32)
y_batch = cifar.eval_data.ys[offset+bstart:offset+bend]
else:
if dataset == "mnist":
x_batch = mnist.test.images[bstart:bend, :].reshape(-1, 28, 28, 1)
y_batch = mnist.test.labels[bstart:bend]
else:
x_batch = cifar.eval_data.xs[bstart:bend, :].astype(np.float32)
y_batch = cifar.eval_data.ys[bstart:bend]
noop_trans = np.zeros([len(x_batch), 3])
dict_nat = {model.x_input: x_batch,
model.y_input: y_batch,
model.is_training: False,
model.transform: noop_trans}
cur_corr_nat, cur_xent_nat = sess.run(
[model.num_correct, model.xent],
feed_dict=dict_nat)
total_xent_nat += cur_xent_nat
total_corr_nat += cur_corr_nat
for i, attack in enumerate(eval_attacks):
x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess)
dict_adv = {model.x_input: x_batch_adv,
model.y_input: y_batch,
model.is_training: False,
model.transform: adv_trans if adv_trans is not None else np.zeros([len(x_batch), 3])}
cur_corr_adv, cur_xent_adv, cur_corr_pred, cur_adv_images = \
sess.run([model.num_correct, model.xent, model.correct_prediction, model.x_image],
feed_dict=dict_adv)
total_xent_advs[i] += cur_xent_adv
total_corr_advs[i].extend(cur_corr_pred)
l1_norms[i].extend(np.sum(np.abs(x_batch_adv - x_batch).reshape(len(x_batch), -1), axis=-1))
l2_norms[i].extend(np.linalg.norm((x_batch_adv - x_batch).reshape(len(x_batch), -1), axis=-1))
linf_norms[i].extend(np.max(np.abs(x_batch_adv - x_batch).reshape(len(x_batch), -1), axis=-1))
avg_xent_nat = total_xent_nat / num_eval_examples
acc_nat = total_corr_nat / num_eval_examples
avg_xent_advs = total_xent_advs / num_eval_examples
acc_advs = np.sum(total_corr_advs, axis=-1) / num_eval_examples
if len(eval_attacks) > 0:
tot_correct = np.bitwise_and.reduce(np.asarray(total_corr_advs), 0)
assert len(tot_correct) == num_eval_examples
any_acc = np.sum(tot_correct) / num_eval_examples
if verbose:
print('natural: {:.2f}%'.format(100 * acc_nat))
for i, attack in enumerate(eval_attacks):
t = attack.name
print('adversarial ({}):'.format(t))
print('\tacc: {:.2f}% '.format(100 * acc_advs[i]))
print("\tmean(l1)={:.1f}, min(l1)={:.1f}, max(l1)={:.1f}".format(
np.mean(l1_norms[i]), np.min(l1_norms[i]), np.max(l1_norms[i])))
print("\tmean(l2)={:.1f}, min(l2)={:.1f}, max(l2)={:.1f}".format(
np.mean(l2_norms[i]), np.min(l2_norms[i]), np.max(l2_norms[i])))
print("\tmean(linf)={:.1f}, min(linf)={:.1f}, max(linf)={:.1f}".format(
np.mean(linf_norms[i]), np.min(linf_norms[i]), np.max(linf_norms[i])))
print('avg nat loss: {:.2f}'.format(avg_xent_nat))
for i, attack in enumerate(eval_attacks):
t = attack.name
print('avg adv loss ({}): {:.2f}'.format(t, avg_xent_advs[i]))
if len(eval_attacks) > 0:
print("any attack: {:.2f}%".format(100 * any_acc))
if summary_writer:
values = [
tf.Summary.Value(tag='xent nat', simple_value=avg_xent_nat),
tf.Summary.Value(tag='accuracy nat', simple_value=acc_nat)
]
if len(eval_attacks) > 0:
values.append(tf.Summary.Value(tag='accuracy adv any', simple_value=any_acc))
for i, attack in enumerate(eval_attacks):
t = attack.name
adv_values = [
tf.Summary.Value(tag='xent adv eval ({})'.format(t), simple_value=avg_xent_advs[i]),
tf.Summary.Value(tag='xent adv ({})'.format(t), simple_value=avg_xent_advs[i]),
tf.Summary.Value(tag='accuracy adv eval ({})'.format(t), simple_value=acc_advs[i]),
tf.Summary.Value(tag='accuracy adv ({})'.format(t), simple_value=acc_advs[i])
]
values.extend(adv_values)
summary = tf.Summary(value=values)
summary_writer.add_summary(summary, global_step.eval(sess))
return acc_nat, total_corr_advs
# print("norm of lbd_hi*theta 3:", np.linalg.norm(x0+ np.array(lbd_hi*theta)))
# print("******")
return lbd_hi, nquery
model = get_model("cifar", softmax=True)
model.load_weights("data/lid_model_cifar.h5")
model_logits = get_model("cifar", softmax=False)
model_logits.load_weights("data/lid_model_cifar.h5")
sess = K.get_session()
x = tf.placeholder(tf.float32, (None, 32, 32, 3))
logits = model(x)
model = Model(model, model_logits, sess, [0.0, 1.0])
cifar = cifar10_input.CIFAR10Data("../cifar10_data")
train_img = cifar.eval_data.xs[1000:5000] / 255.0 - .5
train_lb = cifar.eval_data.ys[1000:5000]
test_img = cifar.eval_data.xs[:100] / 255.0 - .5
test_lb = cifar.eval_data.ys[:100]
attack = blackbox(model)
# =============================== pre-attack 1000 images for logistic training================
#labels = train_lb[:1000]
#images = train_img[:1000]
#count = 0
#pre_labs = []
#for i in range(1000):
# pre_lab = model.predict(images[i])
# pre_labs.append(pre_lab)
# if labels[i] == pre_lab:
def main(argv):
model_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if model_file is None:
print('No model found')
sys.exit()
cifar = cifar10_input.CIFAR10Data(FLAGS.dataset_dir)
nb_classes = 10
X_test = cifar.train_data.xs
Y_test = to_categorical(cifar.train_data.ys, nb_classes)
assert Y_test.shape[1] == 10.
print("train data shape", X_test.shape)
set_log_level(logging.DEBUG)
with tf.Session() as sess:
x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3))
y = tf.placeholder(tf.float32, shape=(None, 10))
from cleverhans.model_zoo.madry_lab_challenges.cifar10_model import make_wresnet
model = make_wresnet()
saver = tf.train.Saver()
# Restore the checkpoint
saver.restore(sess, model_file)
SCOPE = "cifar10_challenge"
model2 = make_wresnet(scope=SCOPE)
assert len(model.get_vars()) == len(model2.get_vars())
found = [False] * len(model2.get_vars())
for var1 in model.get_vars():
var1_found = False
var2_name = SCOPE + "/" + var1.name
for idx, var2 in enumerate(model2.get_vars()):
if var2.name == var2_name:
var1_found = True
found[idx] = True
sess.run(tf.assign(var2, var1))
break
assert var1_found, var1.name
assert all(found)
model = model2
saver = tf.train.Saver()
# Restore the checkpoint
#saver.restore(sess, model_file)
nb_samples = FLAGS.nb_samples
attack_params = {
'batch_size': FLAGS.batch_size,
'clip_min': 0.,
'clip_max': 255.
}
if FLAGS.attack_type == 'cwl2':
from cleverhans.attacks import CarliniWagnerL2
attacker = CarliniWagnerL2(model, sess=sess)
attack_params.update({
'binary_search_steps': 1,
'confidence': 0,
'max_iterations': 100,
'learning_rate': 0.1,
'initial_const': 10,
'batch_size': 10
})
else: # eps and eps_iter in range 0-255
attack_params.update({'eps': 16, 'ord': np.inf})
if FLAGS.attack_type == 'fgsm':
from cleverhans.attacks import FastGradientMethod
attacker = FastGradientMethod(model, sess=sess)
elif FLAGS.attack_type == 'pgd':
attack_params.update({
'eps': 8,
'eps_iter': .02,
'ord': np.inf,
'nb_iter': 10
})
from cleverhans.attacks import MadryEtAl
attacker = MadryEtAl(model, sess=sess)
elif FLAGS.attack_type == 'deepFool':
print("here")
attack_params.update({
'ord': np.inf,
'eps': 6.0,
'max_iter': 100
})
from CdeepFool_cleverhans import DeepFool
attacker = DeepFool(model, sess=sess)
eval_par = {'batch_size': FLAGS.batch_size}
if FLAGS.sweep:
max_eps = 16
epsilons = np.linspace(1, max_eps, max_eps)
for e in epsilons:
t1 = time.time()
attack_params.update({'eps': e})
x_adv = attacker.generate(x, **attack_params)
preds_adv = model.get_probs(x_adv)
x1 = sess.run(x_adv, feed_dict={x: X_test[0], y: Y_test[0]})
print(x1.shape)
l_inf = np.amax(np.abs(X_test[0] - x1))
print('perturbation found: {}'.format(l_inf))
acc = model_eval(sess,
x,
y,
preds_adv,
X_test[:nb_samples],
Y_test[:nb_samples],
args=eval_par)
print('Epsilon %.2f, accuracy on adversarial' % e,
'examples %0.4f\n' % acc)
t2 = time.time()
else:
t1 = time.time()
x_adv = attacker.generate(x, **attack_params)
preds_adv = model.get_probs(x_adv)
logits = model.get_logits(x)
#print (len(x_adv))
num_eval_examples = 1000
eval_batch_size = 100
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
x_adv_all = [] # adv accumulator
y_adv_all = []
y_true = []
print('Iterating over {} batches'.format(num_batches))
for ibatch in range(num_batches):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
print('batch size: {}'.format(bend - bstart))
x_batch = X_test[bstart:bend, :]
y_batch = Y_test[bstart:bend]
y_clean = np.argmax(sess.run(logits, feed_dict={x: x_batch}),
axis=1)
x_b_adv, pred = sess.run([x_adv, preds_adv],
feed_dict={
x: x_batch,
y: y_batch
})
y_b_adv = np.argmax(sess.run(logits, feed_dict={x: x_b_adv}),
axis=1)
count = 0
y_batch = np.argmax(y_batch, axis=1)
for i in range(eval_batch_size):
if (y_b_adv[i] != y_batch[i] and y_clean[i] == y_batch[i]):
l_inf = np.amax(np.abs(x_batch[i] - x_b_adv[i]))
print('perturbation found: {}'.format(l_inf))
#print (y_b_adv[i])
x_adv_all.append(x_b_adv[i])
y_adv_all.append(y_b_adv[i])
y_true.append(y_batch[i])
count += 1
#print (y_adv_all[0:20])
#print (y_true[0:20])
print("Totat adversariak cound in this batch", count)
#x_adv_all.extend(x_b_adv)
#y_adv_all.extend(y_b_adv)
x_adv_all = np.array(x_adv_all)
y_true = np.array(y_true)
y_adv_all = np.array(y_adv_all)
print('Adv Label', y_adv_all[0:20])
print('Ori Label', y_true[0:20])
#y_adv = np.squeeze(y_adv)
print(x_adv_all.shape)
print(y_adv_all.shape)
print(y_true.shape)
count = 0
for i in range(y_adv_all.shape[0]):
if y_true[i] != y_adv_all[i]:
count += 1
print("Total adversarial examples found", count)
pickle.dump(
(x_adv_all, y_true, y_adv_all),
open(
'/scratch/kiani/Projects/CIFAR data/Adversarial/deepFool/iter_100/deepFool_E6_train.p',
'wb'))
#from numpy import linalg as LA
#l_2 = LA.norm(X_test[0] - x1[0])
#l_inf = np.amax(np.abs(x - x_adv))
t2 = time.time()
print("Range of data should be 0-255 and actual is: ",
str(np.min(x_adv_all)) + " " + str(np.max(x_adv_all)))
image = ((x_adv_all[2])).astype(np.uint8)
img = Image.fromarray(image)
img.save("deepFool_attack.jpeg")
print("Took", t2 - t1, "seconds")
def evaluate_ch(model, config, sess, norm='l1', bound=None, verbose=True):
dataset = config['data']
num_eval_examples = config['num_eval_examples']
eval_batch_size = config['eval_batch_size']
if dataset == "mnist":
from tensorflow.examples.tutorials.mnist import input_data
mnist = input_data.read_data_sets('MNIST_data', one_hot=False)
X = mnist.test.images[0:num_eval_examples, :].reshape(-1, 28, 28, 1)
Y = mnist.test.labels[0:num_eval_examples]
x_image = tf.placeholder(tf.float32, shape=[None, 28, 28, 1])
else:
import cifar10_input
data_path = config["data_path"]
cifar = cifar10_input.CIFAR10Data(data_path)
X = cifar.eval_data.xs[0:num_eval_examples, :].astype(np.float32) / 255.0
Y = cifar.eval_data.ys[0:num_eval_examples]
x_image = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
assert norm == 'l1'
if norm=='l2':
attack = CarliniWagnerL2(model, sess)
params = {'batch_size': eval_batch_size, 'binary_search_steps': 9}
else:
attack = ElasticNetMethod(model, sess, clip_min=0.0, clip_max=1.0)
params = {'beta': 1e-2,
'decision_rule': 'L1',
'batch_size': eval_batch_size,
'learning_rate': 1e-2,
'max_iterations': 1000}
if verbose:
set_log_level(logging.DEBUG, name="cleverhans")
y = tf.placeholder(tf.int64, shape=[None, 10])
params['y'] = y
adv_x = attack.generate(x_image, **params)
preds_adv = model.get_predicted_class(adv_x)
preds_nat = model.get_predicted_class(x_image)
all_preds, all_preds_adv, all_adv_x = batch_eval(
sess, [x_image, y], [preds_nat, preds_adv, adv_x], [X, one_hot(Y, 10)], batch_size=eval_batch_size)
print('acc nat', np.mean(all_preds == Y))
print('acc adv', np.mean(all_preds_adv == Y))
if dataset == "cifar10":
X *= 255.0
all_adv_x *= 255.0
if norm == 'l2':
lps = np.sqrt(np.sum(np.square(all_adv_x - X), axis=(1,2,3)))
else:
lps = np.sum(np.abs(all_adv_x - X), axis=(1,2,3))
print('mean lp: ', np.mean(lps))
for b in [bound, bound/2.0, bound/4.0, bound/8.0]:
print('lp={}, acc={}'.format(b, np.mean((all_preds_adv == Y) | (lps > b))))
all_corr_adv = (all_preds_adv == Y)
all_corr_nat = (all_preds == Y)
return all_corr_nat, all_corr_adv, lps
if model_file is None:
print('No model found')
sys.exit()
class_count = 10
if _CIFAR100: class_count = 100
model = Model(mode='eval', class_count=class_count)
attack = LinfPGDAttack(model, config['epsilon'], config['num_steps'],
config['step_size'], config['random_start'],
config['loss_func'])
saver = tf.train.Saver()
data_path = config['data_path']
cifar = cifar100_input.CIFAR100Data(
data_path) if _CIFAR100 else cifar10_input.CIFAR10Data(data_path)
with tf.Session() as sess:
# Restore the checkpoint
saver.restore(sess, model_file)
# Iterate over the samples batch-by-batch
num_eval_examples = config['num_eval_examples']
eval_batch_size = config['eval_batch_size']
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
x_adv = [] # adv accumulator
print('Iterating over {} batches'.format(num_batches))
total_corr, total_count, clean_corr = 0.0, 0.0, 0.0
LEARNING_RATE = 1e-1
LR_DECAY_RATE = 1e-4 #5e-5
DECAY_STEPS = 1.0
adv_weight = 5000
ITER = 4000
else:
#LEARNING_RATE = 5e-2
#LR_DECAY_RATE = 1e-5 # 5e-5
DECAY_STEPS = 1.0
adv_weight = 128
ITER = 100
style_weight = 1
if data_set == "cifar10":
raw_cifar = cifar10_input.CIFAR10Data("cifar10_data")
def get_data(sess):
x_batch, y_batch = raw_cifar.train_data.get_next_batch(
batch_size=BATCH_SIZE, multiple_passes=True)
return x_batch, y_batch
elif data_set == "imagenet":
inet = imagenet(BATCH_SIZE, dataset="train")
def get_data(sess):
x_batch, y_batch = inet.get_next_batch(sess)
return x_batch, y_batch
ENCODER_WEIGHTS_PATH = 'vgg19_normalised.npz'
def train(config):
# seeding randomness
tf.set_random_seed(config.training.tf_random_seed)
np.random.seed(config.training.np_random_seed)
# Setting up training parameters
max_num_training_steps = config.training.max_num_training_steps
step_size_schedule = config.training.step_size_schedule
weight_decay = config.training.weight_decay
momentum = config.training.momentum
batch_size = config.training.batch_size
adversarial_training = config.training.adversarial_training
eval_during_training = config.training.eval_during_training
if eval_during_training:
num_eval_steps = config.training.num_eval_steps
# Setting up output parameters
num_output_steps = config.training.num_output_steps
num_summary_steps = config.training.num_summary_steps
num_checkpoint_steps = config.training.num_checkpoint_steps
# Setting up the data and the model
data_path = config.data.data_path
raw_cifar = cifar10_input.CIFAR10Data(data_path)
global_step = tf.contrib.framework.get_or_create_global_step()
model = resnet.Model(config.model)
# uncomment to get a list of trainable variables
# model_vars = tf.trainable_variables()
# slim.model_analyzer.analyze_vars(model_vars, print_info=True)
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
boundaries, values)
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
train_step = optimizer.minimize(total_loss, global_step=global_step)
# Set up adversary
attack = SpatialAttack(model, config.attack)
# Setting up the Tensorboard and checkpoint outputs
model_dir = config.model.output_dir
if eval_during_training:
eval_dir = os.path.join(model_dir, 'eval')
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
# We add accuracy and xent twice so we can easily make three types of
# comparisons in Tensorboard:
# - train vs eval (for a single run)
# - train of different runs
# - eval of different runs
saver = tf.train.Saver(max_to_keep=3)
tf.summary.scalar('accuracy_adv_train',
model.accuracy,
collections=['adv'])
tf.summary.scalar('accuracy_adv', model.accuracy, collections=['adv'])
tf.summary.scalar('xent_adv_train',
model.xent / batch_size,
collections=['adv'])
tf.summary.scalar('xent_adv', model.xent / batch_size, collections=['adv'])
tf.summary.image('images_adv_train', model.x_image, collections=['adv'])
adv_summaries = tf.summary.merge_all('adv')
tf.summary.scalar('accuracy_nat_train',
model.accuracy,
collections=['nat'])
tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat'])
tf.summary.scalar('xent_nat_train',
model.xent / batch_size,
collections=['nat'])
tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat'])
tf.summary.image('images_nat_train', model.x_image, collections=['nat'])
tf.summary.scalar('learning_rate', learning_rate, collections=['nat'])
nat_summaries = tf.summary.merge_all('nat')
os.environ["CUDA_VISIBLE_DEVICES"] = "1"
gpu_options = tf.GPUOptions(allow_growth=True)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
# initialize data augmentation
if config.training.data_augmentation:
cifar = cifar10_input.AugmentedCIFAR10Data(raw_cifar, sess)
else:
cifar = raw_cifar
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
if eval_during_training:
eval_summary_writer = tf.summary.FileWriter(eval_dir)
sess.run(tf.global_variables_initializer())
training_time = 0.0
# Main training loop
for ii in range(max_num_training_steps + 1):
x_batch, y_batch = cifar.train_data.get_next_batch(
batch_size, multiple_passes=True)
noop_trans = np.zeros([len(x_batch), 3])
# Compute Adversarial Perturbations
if adversarial_training:
start = timer()
x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess)
end = timer()
training_time += end - start
else:
x_batch_adv, adv_trans = x_batch, noop_trans
nat_dict = {
model.x_input: x_batch,
model.y_input: y_batch,
model.transform: noop_trans,
model.is_training: False
}
adv_dict = {
model.x_input: x_batch_adv,
model.y_input: y_batch,
model.transform: adv_trans,
model.is_training: False
}
# Output to stdout
if ii % num_output_steps == 0:
nat_acc = sess.run(model.accuracy, feed_dict=nat_dict)
adv_acc = sess.run(model.accuracy, feed_dict=adv_dict)
print('Step {}: ({})'.format(ii, datetime.now()))
print(' training nat accuracy {:.4}%'.format(nat_acc * 100))
print(' training adv accuracy {:.4}%'.format(adv_acc * 100))
if ii != 0:
print(' {} examples per second'.format(
num_output_steps * batch_size / training_time))
training_time = 0.0
# Tensorboard summaries
if ii % num_summary_steps == 0:
summary = sess.run(adv_summaries, feed_dict=adv_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
summary = sess.run(nat_summaries, feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
# Write a checkpoint
if ii % num_checkpoint_steps == 0:
saver.save(sess,
os.path.join(model_dir, 'checkpoint'),
global_step=global_step)
if eval_during_training and ii % num_eval_steps == 0:
evaluate(model, attack, sess, config, eval_summary_writer)
# Actual training step
start = timer()
if adversarial_training:
adv_dict[model.is_training] = True
sess.run(train_step, feed_dict=adv_dict)
else:
nat_dict[model.is_training] = True
sess.run(train_step, feed_dict=nat_dict)
end = timer()
training_time += end - start
def train(config='configs/fannyconfig.json',
save_root_path='/cluster/work/math/fanyang-broglil/CoreRepo',
experiment_json_fname='experiments.json',
local_json_dir_name='local_json_files',
worstofk=None,
attack_style=None,
attack_limits=None,
fo_epsilon=None,
fo_step_size=None,
fo_num_steps=None,
lambda_core=None,
num_ids=None,
group_size=None,
use_core=None,
seed=None,
save_in_local_json=True,
this_repo=None):
# reset default graph (needed for running locally with run_jobs_ray.py)
tf.reset_default_graph()
# get configs
config_dict = utilities.get_config(config)
config_dict_copy = copy.deepcopy(config_dict)
config = utilities.config_to_namedtuple(config_dict)
# seeding randomness
if seed == None:
seed = config.training.tf_random_seed
else:
config_dict_copy['training']['tf_random_seed'] = seed
tf.set_random_seed(seed)
np.random.seed(seed)
# Setting up training parameters
max_num_training_steps = config.training.max_num_training_steps
step_size_schedule = config.training.step_size_schedule
weight_decay = config.training.weight_decay
momentum = config.training.momentum
if group_size == None:
group_size = config.training.group_size
else:
config_dict_copy['training']['group_size'] = int(group_size)
if num_ids == None:
num_ids = config.training.num_ids
else:
config_dict_copy['training']['num_ids'] = int(num_ids)
if lambda_core == None:
lambda_core = config.training.lambda_
else:
config_dict_copy['training']['lambda_'] = float(lambda_core)
if use_core == None:
use_core = config.training.use_core
else:
config_dict_copy['training']['use_core'] = use_core
batch_size = config.training.batch_size
# number of groups with group size > 1
num_grouped_ids = batch_size - num_ids
# number of unique ids needs to be larger than half the desired batch size
# so that full batch can be filled up
assert num_ids >= batch_size / group_size
# currently, code is designed for groups of size 2
# assert batch_size % group_size == 0
adversarial_training = config.training.adversarial_training
eval_during_training = config.training.eval_during_training
if eval_during_training:
num_eval_steps = config.training.num_eval_steps
# Setting up output parameters
num_output_steps = config.training.num_output_steps
num_summary_steps = config.training.num_summary_steps
num_checkpoint_steps = config.training.num_checkpoint_steps
num_easyeval_steps = config.training.num_easyeval_steps
# Setting up the data and the model
data_path = config.data.data_path
if config.data.dataset_name == "cifar-10":
raw_iterator = cifar10_input.CIFAR10Data(data_path)
elif config.data.dataset_name == "cifar-100":
raw_iterator = cifar100_input.CIFAR100Data(data_path)
elif config.data.dataset_name == "svhn":
raw_iterator = svhn_input.SVHNData(data_path)
else:
raise ValueError("Unknown dataset name.")
global_step = tf.train.get_or_create_global_step()
model_family = config.model.model_family
if model_family == "resnet":
if config.attack.use_spatial and config.attack.spatial_method == 'fo':
diffable = True
else:
diffable = False
model = resnet.Model(config.model, num_ids, diffable)
elif model_family == "vgg":
if config.attack.use_spatial and config.attack.spatial_method == 'fo':
# TODO: add differentiable transformer to vgg.py
raise NotImplementedError
model = vgg.Model(config.model, num_ids)
# uncomment to get a list of trainable variables
# model_vars = tf.trainable_variables()
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
boundaries, values)
if use_core and lambda_core > 0:
total_loss = (model.mean_xent +
weight_decay * model.weight_decay_loss +
lambda_core * model.core_loss)
else:
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
train_step = optimizer.minimize(total_loss, global_step=global_step)
# Set up adversary
if worstofk == None:
worstofk = config.attack.random_tries
else:
config_dict_copy['attack']['random_tries'] = worstofk
if fo_epsilon == None:
fo_epsilon = config.attack.epsilon
else:
config_dict_copy['attack']['epsilon'] = fo_epsilon
if fo_step_size == None:
fo_step_size = config.attack.step_size
else:
config_dict_copy['attack']['step_size'] = fo_step_size
if fo_num_steps == None:
fo_num_steps = config.attack.num_steps
else:
config_dict_copy['attack']['num_steps'] = fo_num_steps
# @ Luzius: incorporate being able to choose multiple transformations
if attack_style == None:
attack_style = 'rotate'
# Training attack
# L-inf attack if use_spatial is False and use_linf is True
# spatial attack if use_spatial is True and use_linf is False
# spatial random attack if spatial_method is 'random'
# spatial PGD attack if spatial_method is 'fo'
attack = SpatialAttack(model, config.attack, config.attack.spatial_method,
worstofk, attack_limits, fo_epsilon, fo_step_size,
fo_num_steps)
# Different eval attacks
# Random attack
# L-inf attack if use_spatial is False and use_linf is True
# random (worst-of-1) spatial attack if use_spatial is True
# and use_linf is False
attack_eval_random = SpatialAttack(model, config.attack, 'random', 1,
attack_limits, fo_epsilon, fo_step_size,
fo_num_steps)
# First order attack
# L-inf attack if use_spatial is False and use_linf is True
# first-order spatial attack if use_spatial is True and use_linf is False
attack_eval_fo = SpatialAttack(model, config.attack, 'fo', 1,
attack_limits, fo_epsilon, fo_step_size,
fo_num_steps)
# Grid attack
# spatial attack if use_spatial is True and use_linf is False
# not executed for L-inf attacks
attack_eval_grid = SpatialAttack(model, config.attack, 'grid', None,
attack_limits)
# TODO(christina): add L-inf attack with random restarts
# ------------------START EXPERIMENT -------------------------
# Initialize the Repo
print("==> Creating repo..")
# Create repo object if it wasn't passed, comment out if repo has issues
if this_repo == None:
this_repo = exprepo.ExperimentRepo(
save_in_local_json=save_in_local_json,
json_filename=experiment_json_fname,
local_dir_name=local_json_dir_name,
root_dir=save_root_path)
# Create new experiment
if this_repo != None:
exp_id = this_repo.create_new_experiment(config.data.dataset_name,
model_family, worstofk,
attack_style, attack_limits,
lambda_core, num_grouped_ids,
group_size, config_dict_copy)
# Setting up the Tensorboard and checkpoint outputs
model_dir = '%s/logdir/%s' % (save_root_path, exp_id)
# We add accuracy and xent twice so we can easily make three types of
# comparisons in Tensorboard:
# - train vs eval (for a single run)
# - train of different runs
# - eval of different runs
saver = tf.train.Saver(max_to_keep=3)
tf.summary.scalar('accuracy_nat_train',
model.accuracy,
collections=['nat'])
tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat'])
tf.summary.scalar('xent_nat_train',
model.xent / batch_size,
collections=['nat'])
tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat'])
tf.summary.image('images_nat_train', model.x_image, collections=['nat'])
tf.summary.scalar('learning_rate', learning_rate, collections=['nat'])
nat_summaries = tf.summary.merge_all('nat')
# data augmentation used if config.training.data_augmentation_core is True
x_input_placeholder = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
flipped = tf.map_fn(lambda img: tf.image.random_flip_left_right(img),
x_input_placeholder)
with tf.Session() as sess:
# initialize standard data augmentation
if config.training.data_augmentation:
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.AugmentedCIFAR10Data(
raw_iterator, sess)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.AugmentedCIFAR100Data(
raw_iterator, sess)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.AugmentedSVHNData(
raw_iterator, sess)
else:
raise ValueError("Unknown dataset name.")
else:
data_iterator = raw_iterator
eval_dict = {
model.x_input: data_iterator.eval_data.xs,
model.y_input: data_iterator.eval_data.ys,
model.group: np.arange(0, batch_size, 1, dtype="int32"),
model.transform: np.zeros([data_iterator.eval_data.n, 3]),
model.is_training: False
}
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
# if eval_during_training:
eval_dir = os.path.join(model_dir, 'eval')
os.makedirs(eval_dir, exist_ok=True)
eval_summary_writer = tf.summary.FileWriter(eval_dir)
sess.run(tf.global_variables_initializer())
training_time = 0.0
run_time_without_eval = 0.0
run_time_adv_ex_creation = 0.0
run_time_train_step = 0.0
####################################
# Main training loop
####################################
start_time = time.time()
no_epochs_done = 0 # the same as epoch_count, need to merge
start_epoch = timer()
it_count = 0
epoch_count = 0
acc_sum = 0
for ii in range(max_num_training_steps + 1):
# original batch
x_batch, y_batch, epoch_done = data_iterator.train_data.get_next_batch(
num_ids, multiple_passes=True)
no_epochs_done += epoch_done
# noop trans
noop_trans = np.zeros([len(x_batch), 3])
# id_batch starts with IDs of original examples
id_batch = np.arange(0, num_ids, 1, dtype="int32")
if use_core:
# first num_id examples of batch are natural
x_batch_inp = x_batch
y_batch_inp = y_batch
trans_inp = noop_trans
id_batch_inp = id_batch
start = timer()
for _ in range(group_size - 1):
if config.training.data_augmentation_core:
raise NotImplementedError
# create rotated examples
x_batch_adv_i, trans_adv_i = attack.perturb(
x_batch, y_batch, sess)
# construct new batches including rotated examples
x_batch_inp = np.concatenate((x_batch_inp, x_batch_adv_i),
axis=0)
y_batch_inp = np.concatenate((y_batch_inp, y_batch),
axis=0)
trans_inp = np.concatenate((trans_inp, trans_adv_i),
axis=0)
id_batch_inp = np.concatenate((id_batch_inp, id_batch),
axis=0)
end = timer()
training_time += end - start
run_time_without_eval += end - start
run_time_adv_ex_creation += end - start
trans_adv = trans_inp[num_ids:, ...]
id_batch_adv = id_batch_inp[num_ids:]
y_batch_adv = y_batch_inp[num_ids:]
x_batch_adv = x_batch_inp[num_ids:, ...]
else:
if adversarial_training:
start = timer()
x_batch_inp, trans_inp = attack.perturb(
x_batch, y_batch, sess)
end = timer()
training_time += end - start
run_time_without_eval += end - start
run_time_adv_ex_creation += end - start
else:
x_batch_inp, trans_inp = x_batch, noop_trans
# for adversarial training and plain training, the following
# variables coincide
y_batch_inp = y_batch
y_batch_adv = y_batch
trans_adv = trans_inp
x_batch_adv = x_batch_inp
id_batch_inp = id_batch
id_batch_adv = id_batch
# feed_dict for training step
inp_dict = {
model.x_input: x_batch_inp,
model.y_input: y_batch_inp,
model.group: id_batch_inp,
model.transform: trans_inp,
model.is_training: False
}
# separate natural and adversarially transformed examples for eval
nat_dict = {
model.x_input: x_batch,
model.y_input: y_batch,
model.group: id_batch,
model.transform: noop_trans,
model.is_training: False
}
adv_dict = {
model.x_input: x_batch_adv,
model.y_input: y_batch_adv,
model.group: id_batch_adv,
model.transform: trans_adv,
model.is_training: False
}
########### Outputting/saving weights and evaluations ###########
acc_grid_te = -1.0
avg_xent_grid_te = -1.0
acc_fo_te = -1.0
avg_xent_fo_te = -1.0
saved_weights = 0
# Compute training accuracy on this minibatch
nat_acc_tr = 100 * sess.run(model.accuracy, feed_dict=nat_dict)
# Output to stdout
if epoch_done:
epoch_time = timer() - start_epoch
# Average
av_acc = acc_sum / it_count
# ToDo: Log this to file as well
# Training accuracy over epoch
print('Epoch {}: ({})'.format(epoch_count, datetime.now()))
print(' training natural accuracy {:.4}%'.format(av_acc))
print(' {:.4} seconds per epoch'.format(epoch_time))
# Accuracy on entire test set
nat_acc_te = 100 * sess.run(model.accuracy,
feed_dict=eval_dict)
print(
' test set natural accuracy {:.4}%'.format(nat_acc_te))
# Set loss sum, it count back to zero
acc_sum = nat_acc_tr
epoch_done = 0
epoch_count += 1
start_epoch = timer()
it_count = 1
else:
it_count += 1
acc_sum += nat_acc_tr
# Output to stdout
if ii % num_output_steps == 0:
# nat_acc_tr = 100 * sess.run(model.accuracy, feed_dict=nat_dict)
adv_acc_tr = 100 * sess.run(model.accuracy, feed_dict=adv_dict)
inp_acc_tr = 100 * sess.run(model.accuracy, feed_dict=inp_dict)
# print('Step {}: ({})'.format(ii, datetime.now()))
# print(' training nat accuracy {:.4}%'.format(nat_acc_tr))
# print(' training adv accuracy {:.4}%'.format(adv_acc_tr))
# print(' training inp accuracy {:.4}%'.format(inp_acc_tr))
if ii != 0:
# print(' {} examples per second'.format(
# num_output_steps * batch_size / training_time))
training_time = 0.0
# Tensorboard summaries and heavy checkpoints
if ii % num_summary_steps == 0:
summary = sess.run(nat_summaries, feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
# Write a checkpoint and eval if it's time
if ii % num_checkpoint_steps == 0 or ii == max_num_training_steps:
# Save checkpoint data (weights)
saver.save(sess,
os.path.join(model_dir, 'checkpoint'),
global_step=global_step)
saved_weights = 1
# Write evaluation meta data for checkpoint
if ii % num_easyeval_steps == 0 or ii == max_num_training_steps:
# Get training accuracies
nat_acc_tr = 100 * sess.run(model.accuracy, feed_dict=nat_dict)
adv_acc_tr = 100 * sess.run(model.accuracy, feed_dict=adv_dict)
inp_acc_tr = 100 * sess.run(model.accuracy, feed_dict=inp_dict)
# Evaluation on random and natural
[
acc_nat_te, acc_rand_adv_te, avg_xent_nat_te,
avg_xent_adv_te
] = evaluate(model, attack_eval_random, sess, config, 'random',
data_path, None)
# Evaluation on grid (only for spatial attacks)
if ((eval_during_training and ii % num_eval_steps == 0
and ii > 0 and config.attack.use_spatial) or
(eval_during_training and ii == max_num_training_steps
and config.attack.use_spatial)):
if config.attack.use_spatial and config.attack.spatial_method == 'fo':
# Evaluation on first-order PDG attack (too expensive to
# evaluate more frequently on whole dataset)
[_, acc_fo_te, _,
avg_xent_fo_te] = evaluate(model, attack_eval_fo,
sess, config, 'fo',
data_path, None)
# Evaluation on grid
[_, acc_grid_te, _, avg_xent_grid_te
] = evaluate(model, attack_eval_grid, sess, config,
"grid", data_path, eval_summary_writer)
chkpt_id = this_repo.create_training_checkpoint(
exp_id,
training_step=ii,
epoch=no_epochs_done,
train_acc_nat=nat_acc_tr,
train_acc_adv=adv_acc_tr,
train_acc_inp=inp_acc_tr,
test_acc_nat=acc_nat_te,
test_acc_adv=acc_rand_adv_te,
test_acc_fo=acc_fo_te,
test_acc_grid=acc_grid_te,
test_loss_nat=avg_xent_nat_te,
test_loss_adv=avg_xent_adv_te,
test_loss_fo=avg_xent_fo_te,
test_loss_grid=avg_xent_grid_te)
if saved_weights == 0:
# Save checkpoint data (weights)
saver.save(
sess,
os.path.join(model_dir,
'{}_checkpoint'.format(chkpt_id)))
# Actual training step
start = timer()
inp_dict[model.is_training] = True
sess.run(train_step, feed_dict=inp_dict)
end = timer()
training_time += end - start
run_time_without_eval += end - start
run_time_train_step += end - start
runtime = time.time() - start_time
this_repo.mark_experiment_as_completed(
exp_id,
train_acc_nat=nat_acc_tr,
train_acc_adv=adv_acc_tr,
train_acc_inp=inp_acc_tr,
test_acc_nat=acc_nat_te,
test_acc_adv=acc_rand_adv_te,
test_acc_fo=acc_fo_te,
test_acc_grid=acc_grid_te,
runtime=runtime,
runtime_wo_eval=run_time_without_eval,
runtime_train_step=run_time_train_step,
runtime_adv_ex_creation=run_time_adv_ex_creation)
return 0
def evaluate(model, attack, sess, config, attack_type, data_path,reg_model_path,
summary_writer=None, eval_on_train=False):
num_eval_examples = config.eval.num_eval_examples
eval_batch_size = config.eval.batch_size
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.CIFAR10Data(data_path)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.CIFAR100Data(data_path)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.SVHNData(data_path)
else:
raise ValueError("Unknown dataset name.")
global_step = tf.train.get_or_create_global_step()
# Iterate over the samples batch-by-batch
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
total_xent_nat = 0.
total_xent_adv = 0.
total_corr_nat = 0
total_corr_adv = 0
for ibatch in trange(num_batches):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
if eval_on_train:
x_batch = data_iterator.train_data.xs[bstart:bend, :]
y_batch = data_iterator.train_data.ys[bstart:bend]
else:
x_batch = data_iterator.eval_data.xs[bstart:bend, :]
y_batch = data_iterator.eval_data.ys[bstart:bend]
noop_trans = np.zeros([len(x_batch), 3])
if config.eval.adversarial_eval:
# print("PERTURBED!")
x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess)
else:
x_batch_adv, adv_trans = x_batch, noop_trans
ori_trans = adv_trans
# ori_trans_x = ori_trans[:0]
# ori_trans_y = ori_trans[:1]
# ori_rot = ori_trans[:2]
ori_trans_x, ori_trans_y, ori_rot = tf.unstack(tf.cast(adv_trans, tf.float32), axis=1)
adv_trans = tf.cast(adv_trans, tf.float32)
x_batch_adv_transformed = simple_transform(x_batch_adv, adv_trans, config.model.reg_pad_mode)
x_batch_adv_transformed = sess.run(x_batch_adv_transformed)
# utilities.show_image_batch(x_batch_adv_transformed, title='adv')
# print(ori_trans_x.shape)
# print(ori_trans_y.shape)
# print(ori_rot.shape)
# ones = tf.ones(shape=tf.shape(ori_trans_x))
# zeros = tf.zeros(shape=tf.shape(ori_trans_x))
# trans = tf.stack([ones, zeros, -ori_trans_x,
# zeros, ones, -ori_trans_y,
# zeros, zeros], axis=1)
# temp = tf.contrib.image.transform(x_batch_adv, trans, interpolation='BILINEAR')
# x_batch_adv_transformed = tf.contrib.image.rotate(temp, ori_rot, interpolation='BILINEAR')
# use regression model to turn back peturbed adversarial image (for evaluation)
g_temp = tf.Graph()
with tf.Session(graph = g_temp) as sess2:
print("sub session")
id_batch = np.arange(0, num_ids, 1, dtype="int32")
with g_temp.as_default():
# Restore the checkpoint of the regression model
metapath = reg_model_path + ".meta"
saver = tf.train.import_meta_graph(metapath)
saver.restore(sess2, reg_model_path)
# init.run()
# # load model_reg
model_reg = resnet_reg.Model(config.model, num_ids, diffable,
config.training.adversarial_ce)
graph=tf.get_default_graph()
# restore tensors
group = graph.get_tensor_by_name("input/group:0")
x_input = graph.get_tensor_by_name("input/Placeholder:0")
y_input = graph.get_tensor_by_name("input/Placeholder_1:0")
transform = graph.get_tensor_by_name("input/Placeholder_2:0")
is_training = graph.get_tensor_by_name("input/Placeholder_3:0")
prediction = graph.get_tensor_by_name("prediction/xw_plus_b:0")
# This is just a placeholder, since inference in reg model does not need label
y_batch_placeholder = np.zeros([len(x_batch), 3]).astype(np.float)
x_batch_adv = x_batch_adv.astype(np.float)
x_batch = x_batch.astype(np.float)
# the original x_batch_adv is not transformed yet, so we need to transform
# before we feed the batch into the model
regression_feed = {x_input: x_batch_adv_transformed,
y_input: y_batch_placeholder,
group: id_batch,
transform: noop_trans,
is_training: False}
predictions = sess2.run(prediction, feed_dict=regression_feed)
# we also feed the test batch for natural accuracy into the regression mode
# to check how much will it degrade the nat.acc
nat_reg_feed = {x_input: x_batch_adv,
y_input: y_batch_placeholder,
group: id_batch,
transform: noop_trans,
is_training: False
}
nat_predictions = sess2.run(prediction, feed_dict=nat_reg_feed)
# using the same style of transformation: for nat. should be noop_trans - nat_predictions
nat_attack = noop_trans - nat_predictions
# op = sess2.graph.get_operations()
# for m in op:
# print(m.name)
# print(transPara.shape)
# trans_x = transPara[:1]
# trans_y = transPara[:2]
# rot = transPara[:3]
trans_x, trans_y, rot = tf.unstack(predictions, axis=1)
ones = tf.ones(shape=tf.shape(trans_x))
zeros = tf.zeros(shape=tf.shape(trans_x))
print("check shapes")
print(predictions.shape)
print(ori_trans.shape)
print("original attack")
print("trans X: {}".format(sess.run(ori_trans_x)[0:4]))
print("trans Y: {}".format(sess.run(ori_trans_y)[0:4]))
print("rot angle: {}".format(sess.run(ori_rot)[0:4]))
print("\npredicted attack")
print("trans X: {}".format(sess2.run(trans_x)[0:4]))
print("trans Y: {}".format(sess2.run(trans_y)[0:4]))
print("rot angle: {}".format(sess2.run(rot)[0:4]))
remain_diff = ori_trans - predictions
diff_x, diff_y, diff_rot = tf.unstack(remain_diff, axis=1)
print("\nremain_diff")
print("trans X: {}".format(sess2.run(diff_x)[0:4]))
print("trans Y: {}".format(sess2.run(diff_y)[0:4]))
print("rot angle: {}".format(sess2.run(diff_rot)[0:4]))
x = x_batch_adv_transformed
# print(ones.shape)
trans_1 = tf.stack([ones, zeros, trans_x,
zeros, ones, trans_y,
zeros, zeros], axis=1)
x = tf.contrib.image.transform(x, trans_1, interpolation='BILINEAR')
x = tf.contrib.image.rotate(x, -rot*np.pi/180.0, interpolation='BILINEAR')
de_adv = sess2.run(x)
# utilities.show_image_batch(de_adv, title = 'de_adv')
dict_nat = {model.x_input: x_batch,
model.y_input: y_batch,
# model.transform: noop_trans,
model.transform: nat_attack,
model.is_training: False}
# we the transformation is done in the model part, Thus we use the adversarial_attack_para - prediction
# as the new model.transform, if it's perfectly predicted, then it should be close to zero
dict_adv = {model.x_input: x_batch_adv,
model.y_input: y_batch,
model.transform: remain_diff,
model.is_training: False}
# print("entered new session")
cur_corr_nat, cur_xent_nat = sess.run([model.num_correct, model.xent],
feed_dict = dict_nat)
cur_corr_adv, cur_xent_adv = sess.run([model.num_correct, model.xent],
feed_dict = dict_adv)
total_xent_nat += cur_xent_nat
total_xent_adv += cur_xent_adv
total_corr_nat += cur_corr_nat
total_corr_adv += cur_corr_adv
avg_xent_nat = total_xent_nat / num_eval_examples
avg_xent_adv = total_xent_adv / num_eval_examples
acc_nat = total_corr_nat / num_eval_examples
acc_adv = total_corr_adv / num_eval_examples
if summary_writer:
summary = tf.Summary(value=[
tf.Summary.Value(tag='xent_adv_eval', simple_value= avg_xent_adv),
tf.Summary.Value(tag='xent_nat_eval', simple_value= avg_xent_nat),
tf.Summary.Value(tag='xent_adv', simple_value= avg_xent_adv),
tf.Summary.Value(tag='xent_nat', simple_value= avg_xent_nat),
tf.Summary.Value(tag='accuracy_adv_eval', simple_value= acc_adv),
tf.Summary.Value(tag='accuracy_nat_eval', simple_value= acc_nat),
tf.Summary.Value(tag='accuracy_adv', simple_value= acc_adv),
tf.Summary.Value(tag='accuracy_nat', simple_value= acc_nat)])
summary_writer.add_summary(summary, global_step.eval(sess))
step = global_step.eval(sess)
print('Eval at step: {}'.format(step))
print(' Adversary: ', attack_type)
print(' natural: {:.2f}%'.format(100 * acc_nat))
print(' adversarial: {:.2f}%'.format(100 * acc_adv))
print(' avg nat xent: {:.4f}'.format(avg_xent_nat))
print(' avg adv xent: {:.4f}'.format(avg_xent_adv))
return [100 * acc_nat, 100 * acc_adv, avg_xent_nat, avg_xent_adv]
saver.restore(sess, model_file)
# Create attack class
attack_class = getattr(sys.modules[__name__], args.attack)
attack = attack_class(model, args)
# Create directory
if args.save_img:
tf.gfile.MakeDirs(args.save_dir)
# Print hyperparameters
for key, val in vars(args).items():
tf.logging.info('{}={}'.format(key, val))
# Load dataset
cifar = cifar10_input.CIFAR10Data(args.data_dir)
# Load indices
indices = np.load('../cifar10_data/indices_untargeted.npy')
count = 0
index = args.img_index_start
total_num_corrects = 0
total_queries = []
index_to_query = {}
while count < args.sample_size:
tf.logging.info("")
# Get image and label
initial_img = cifar.eval_data.xs[indices[index]]
def train(config='configs/cifar10_config_stn.json',
save_root_path='/cluster/work/math/fanyang-broglil/CoreRepo',
worstofk=None,
attack_style=None,
attack_limits=None,
lambda_core=None,
num_grouped_ids=None,
num_ids=None,
group_size=None,
use_core=None,
seed=None,
this_repo=None):
config_dict = utilities.get_config(config)
config_dict_copy = copy.deepcopy(config_dict)
# model_dir = config_dict['model']['output_dir']
# if not os.path.exists(model_dir):
# os.makedirs(model_dir)
# # keep the configuration file with the model for reproducibility
# with open(os.path.join(model_dir, 'config.json'), 'w') as f:
# json.dump(config_dict, f, sort_keys=True, indent=4)
config = utilities.config_to_namedtuple(config_dict)
# seeding randomness
if seed == None:
seed = config.training.tf_random_seed
else:
config_dict_copy['training']['tf_random_seed'] = seed
tf.set_random_seed(seed)
np.random.seed(seed)
# Setting up training parameters
max_num_epochs = config.training.max_num_epochs
step_size_schedule = config.training.step_size_schedule
weight_decay = config.training.weight_decay
momentum = config.training.momentum
num_ids = config.training.num_ids # number of IDs per minibatch
if group_size == None:
group_size = config.training.group_size
else:
config_dict_copy['training']['group_size'] = group_size
if num_grouped_ids == None:
num_grouped_ids = config.training.num_grouped_ids
else:
config_dict_copy['training']['num_grouped_ids'] = num_grouped_ids
if num_ids == None:
num_ids = config.training.num_ids
else:
config_dict_copy['training']['num_ids'] = num_ids
if lambda_core == None:
lambda_core = config.training.lambda_
else:
config_dict_copy['training']['lambda_'] = lambda_core
if use_core == None:
use_core = config.training.use_core
else:
config_dict_copy['training']['use_core'] = use_core
adversarial_training = config.training.adversarial_training
eval_during_training = config.training.eval_during_training
if eval_during_training:
num_eval_steps = config.training.num_eval_steps
# Setting up output parameters
num_summary_steps = config.training.num_summary_steps
num_checkpoint_steps = config.training.num_checkpoint_steps
num_easyeval_steps = config.training.num_easyeval_steps
# mini batch size per iteration
# ToDo: need to make this support variable number of num_grouped_ids
batch_size = num_ids + num_grouped_ids
# Setting up model and loss
model_family = config.model.model_family
with_transformer = config.model.transformer
translation_model = config.model.translation_model
if model_family == "resnet":
model = loc_net.Model(config.model)
else:
print("Model family does not exist")
exit()
if use_core:
total_loss = model.y_loss #model.mean_xent + weight_decay * model.weight_decay_loss + lambda_core * model.core_loss2
else:
total_loss = model.y_loss #model.mean_xent + weight_decay * model.weight_decay_loss
# Setting up the data and the model
data_path = config.data.data_path
if config.data.dataset_name == "cifar-10":
raw_cifar = cifar10_input.CIFAR10Data(data_path)
elif config.data.dataset_name == "cifar-100":
raw_cifar = cifar100_input.CIFAR100Data(data_path)
else:
raise ValueError("Unknown dataset name.")
# uncomment to get a list of trainable variables
# model_vars = tf.trainable_variables()
# slim.model_analyzer.analyze_vars(model_vars, print_info=True)
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
global_step = tf.train.get_or_create_global_step()
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
boundaries, values)
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
#optimizer = tf.train.AdamOptimizer(learning_rate=learning_rate,
# name="Adam")
train_step = optimizer.minimize(total_loss, global_step=global_step)
# Set up adversary
if worstofk == None:
worstofk = config.attack.random_tries
else:
config_dict_copy['attack']['random_tries'] = worstofk
# @ Luzius: incorporate being able to choose multiple transformations
if attack_style == None:
attack_style = 'rotate'
# Training attack
attack = SpatialAttack(model, config.attack, 'random', worstofk,
attack_limits)
# Different eval attacks
# Same attack as worstofk
# @ Luzius: currently the names are not clear/consistent since I wasn't sure if we actually want random or not since you originally had your attack like that but I feel like it should rather be worstofk?
# attack_eval_adv = SpatialAttack(model, config.attack, 'random', worstofk, attack_limits)
attack_eval_random = SpatialAttack(model, config.attack, 'random', 1,
attack_limits)
# Grid attack
attack_eval_grid = SpatialAttack(model, config.attack, 'grid', None,
attack_limits)
# ------------------START EXPERIMENT -------------------------
# Initialize the Repo
print("==> Creating repo..")
# Create repo object if it wasn't passed, comment out if repo has issues
if this_repo == None:
this_repo = exprepo.ExperimentRepo(root_dir=save_root_path)
# Create new experiment
if this_repo != None:
exp_id = this_repo.create_new_experiment('cifar-10', model_family,
worstofk, attack_style,
attack_limits, lambda_core,
num_grouped_ids, group_size,
config_dict_copy)
# Setting up the Tensorboard and checkpoint outputs
model_dir = '%s/logdir/%s' % (save_root_path, exp_id)
os.makedirs(model_dir, exist_ok=True)
# We add accuracy and xent twice so we can easily make three types of
# comparisons in Tensorboard:
# - train vs eval (for a single run)
# - train of different runs
# - eval of different runs
saver = tf.train.Saver(max_to_keep=3)
tf.summary.scalar('accuracy_nat_train',
model.accuracy,
collections=['nat'])
tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat'])
tf.summary.scalar('xent_nat_train',
model.xent / batch_size,
collections=['nat'])
tf.summary.scalar('xent_nat', model.xent / batch_size, collections=['nat'])
tf.summary.image('images_nat_train', model.x_image, collections=['nat'])
tf.summary.scalar('learning_rate', learning_rate, collections=['nat'])
nat_summaries = tf.summary.merge_all('nat')
#dataAugmentation
x_input_placeholder = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
flipped = tf.map_fn(lambda img: tf.image.random_flip_left_right(img),
x_input_placeholder)
tot_samp = raw_cifar.train_data.n
max_num_iterations = int(np.floor((tot_samp / num_ids) * max_num_epochs))
print("Total # of samples is: %d; This exp. will run %d iterations" %
(tot_samp, max_num_iterations))
# Compute the (epoch) gaps between summary, worstof1eval, checkpoints should happen
summary_gap = int(np.floor(max_num_epochs / num_summary_steps))
easyeval_gap = int(np.floor(max_num_epochs / num_easyeval_steps))
checkpoint_gap = int(np.floor(max_num_epochs / num_checkpoint_steps))
with tf.Session() as sess:
# initialize data augmentation
if config.training.data_augmentation:
if config.data.dataset_name == "cifar-10":
cifar = cifar10_input.AugmentedCIFAR10Data(raw_cifar, sess)
elif config.data.dataset_name == "cifar-100":
cifar = cifar100_input.AugmentedCIFAR100Data(raw_cifar, sess)
else:
raise ValueError("Unknown dataset name.")
else:
cifar = raw_cifar
cifar_eval_dict = {
model.x_input: cifar.eval_data.xs,
model.y_input: cifar.eval_data.ys,
model.group: np.arange(0, batch_size, 1, dtype="int32"),
model.transform: np.zeros([cifar.eval_data.n, 3]),
model.is_training: False
}
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
#if eval_during_training:
eval_dir = os.path.join(model_dir, 'eval')
os.makedirs(eval_dir, exist_ok=True)
eval_summary_writer = tf.summary.FileWriter(eval_dir)
sess.run(tf.global_variables_initializer())
training_time = 0.0
####################################
# Main training loop
####################################
# Initialize cache variables
start_time = time.time()
start_epoch = timer()
it_count = 0
epoch_count = 0
acc_sum = 0
it_summary = 0
it_easyeval = 0
it_ckpt = 0
adv_time = 0
train_time = 0
for ii in range(max_num_iterations + 1):
x_batch, y_batch, epoch_done = cifar.train_data.get_next_batch(
num_ids, multiple_passes=True)
noop_trans = np.zeros([len(x_batch), 3])
x_batch_nat = x_batch
y_batch_nat = y_batch
id_batch = np.arange(0, num_ids, 1, dtype="int32")
if use_core:
# Create rotated examples
start = timer()
ids = np.arange(0, num_grouped_ids, 1, dtype="int32")
for i in range(config.training.group_size):
if config.training.data_augmentation_core:
x_batch_core = sess.run(
flipped,
feed_dict={
x_input_placeholder:
x_batch[0:num_grouped_ids, :, :, :]
})
else:
x_batch_core = x_batch[0:num_grouped_ids, :, :, :]
x_batch_group, trans_group = attack.perturb(
x_batch_core, y_batch[0:num_grouped_ids], sess)
#construct new batches including rotated examples
x_batch_nat = np.concatenate((x_batch_nat, x_batch_group),
axis=0)
y_batch_nat = np.concatenate((y_batch_nat, y_batch),
axis=0)
noop_trans = np.concatenate((noop_trans, trans_group),
axis=0)
id_batch = np.concatenate((id_batch, ids), axis=0)
end = timer()
training_time += end - start
adv_time += end - start
else:
if adversarial_training:
start = timer()
x_batch_nat, noop_trans = attack.perturb(
x_batch, y_batch, sess)
end = timer()
adv_time += end - start
else:
x_batch_nat, noop_trans = x_batch, noop_trans
nat_dict = {
model.x_input: x_batch_nat,
model.y_input: y_batch_nat,
model.group: id_batch,
model.transform: noop_trans,
model.is_training: False
}
################# Outputting/saving weights and evaluations ###############
nat_acc = -1.0
acc_grid = -1.0
avg_xent_grid = -1.0
saved_weights = 0
# Compute training accuracy on this minibatch
train_nat_acc = sess.run(model.accuracy, feed_dict=nat_dict)
# Output to stdout
if epoch_done:
epoch_time = timer() - start_epoch
# Average
av_acc = acc_sum / it_count
# ToDo: Log this to file as well
# Training accuracy over epoch
print('Epoch {}: ({})'.format(epoch_count, datetime.now()))
print(' training natural accuracy {:.4}%'.format(av_acc *
100))
print(' {:.4} seconds per epoch'.format(epoch_time))
# Accuracy on entire test set
test_nat_acc = sess.run(model.accuracy,
feed_dict=cifar_eval_dict)
print(' test set natural accuracy {:.4}%'.format(
test_nat_acc * 100))
# print(' {:.4} seconds for test evaluation'.format(test_time))
print("example TIME")
print(adv_time)
print("train TIME")
print(train_time)
########### Things to do every xxx epochs #############
# Check if worstof1 eval should be run
if it_summary == summary_gap - 1 or epoch_count == max_num_epochs - 1:
summary = sess.run(nat_summaries, feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
it_summary = 0
else:
it_summary += 1
if it_easyeval == easyeval_gap - 1 or epoch_count == max_num_epochs - 1:
# Evaluation on adv and natural
[acc_nat, acc_adv, avg_xent_nat,
avg_xent_adv] = evaluate(model, attack_eval_random, sess,
config, "random", data_path,
None)
# Save in checkpoint
chkpt_id = this_repo.create_training_checkpoint(
exp_id,
training_step=ii,
epoch=epoch_count,
train_acc_nat=nat_acc,
test_acc_adv=acc_adv,
test_acc_nat=acc_nat,
test_loss_adv=avg_xent_adv,
test_loss_nat=avg_xent_nat)
it_easyeval = 0
else:
it_easyeval += 1
startt = timer()
if it_ckpt == checkpoint_gap - 1 or epoch_count == max_num_epochs - 1:
# Create checkpoint id if non-existent
if not chkpt_id:
chkpt_id = this_repo.create_training_checkpoint(
exp_id,
training_step=ii,
epoch=epoch_count,
train_acc_nat=train_nat_acc,
test_acc_nat=test_nat_acc)
# Save checkpoint data (weights)
saver.save(
sess,
os.path.join(model_dir,
'{}_checkpoint'.format(chkpt_id)))
print(' chkpt saving took {:.4}s '.format(timer() -
startt))
it_ckpt = 0
else:
it_ckpt += 1
# Set loss sum, it count back to zero
acc_sum = train_nat_acc
epoch_done = 0
epoch_count += 1
start_epoch = timer()
it_count = 1
else:
it_count += 1
acc_sum += train_nat_acc
# Actual training step
start = timer()
nat_dict[model.is_training] = True
sess.run(train_step, feed_dict=nat_dict)
training_time += timer() - start
train_time += timer() - start
runtime = time.time() - start_time
# Do all evaluations in the last step - on grid
[_, acc_grid, _,
avg_xent_grid] = evaluate(model, attack_eval_grid, sess, config,
"grid", data_path, eval_summary_writer)
this_repo.mark_experiment_as_completed(exp_id,
train_acc_nat=nat_acc,
test_acc_adv=acc_adv,
test_acc_nat=acc_nat,
test_acc_grid=acc_grid,
runtime=runtime)
return 0
def train(tf_seed, np_seed, train_steps, finetune_train_steps, out_steps,
summary_steps, checkpoint_steps, step_size_schedule, weight_decay,
momentum, train_batch_size, epsilon, replay_m, model_dir,
source_model_dir, dataset, beta, gamma, disc_update_steps,
adv_update_steps_per_iter, disc_layers, disc_base_channels,
steps_before_adv_opt, adv_encoder_type, enc_output_activation,
sep_opt_version, grad_image_ratio, final_grad_image_ratio,
num_grad_image_ratios, normalize_zero_mean, eval_adv_attack,
same_optimizer, only_fully_connected, finetuned_source_model_dir,
train_finetune_source_model, finetune_img_random_pert,
img_random_pert, only_finetune, finetune_whole_model, model_suffix,
**kwargs):
tf.set_random_seed(tf_seed)
np.random.seed(np_seed)
model_dir = model_dir + 'IGAM-%s_b%d_beta_%.3f_gamma_%.3f_disc_update_steps%d_l%dbc%d' % (
dataset, train_batch_size, beta, gamma, disc_update_steps, disc_layers,
disc_base_channels) # TODO Replace with not defaults
if img_random_pert:
model_dir = model_dir + '_imgpert'
if steps_before_adv_opt != 0:
model_dir = model_dir + '_advdelay%d' % (steps_before_adv_opt)
if train_steps != 80000:
model_dir = model_dir + '_%dsteps' % (train_steps)
if same_optimizer == False:
model_dir = model_dir + '_adamDopt'
if tf_seed != 451760341:
model_dir = model_dir + '_tf_seed%d' % (tf_seed)
if np_seed != 216105420:
model_dir = model_dir + '_np_seed%d' % (np_seed)
model_dir = model_dir + model_suffix
# Setting up the data and the model
data_path = get_path_dir(dataset=dataset, **kwargs)
if dataset == 'cifar10':
raw_data = cifar10_input.CIFAR10Data(data_path)
else:
raw_data = cifar100_input.CIFAR100Data(data_path)
global_step = tf.train.get_or_create_global_step()
increment_global_step_op = tf.assign(global_step, global_step + 1)
reset_global_step_op = tf.assign(global_step, 0)
source_model = ModelExtendedLogits(mode='train',
target_task_class_num=100,
train_batch_size=train_batch_size)
model = Model(mode='train',
dataset=dataset,
train_batch_size=train_batch_size,
normalize_zero_mean=normalize_zero_mean)
# Setting up the optimizers
boundaries = [int(sss[0]) for sss in step_size_schedule][1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
boundaries, values)
c_optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
finetune_optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
if same_optimizer:
d_optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
else:
print("Using ADAM opt for DISC model")
d_optimizer = tf.train.AdamOptimizer(learning_rate=0.001)
# Compute input gradient (saliency map)
input_grad = tf.gradients(model.target_softmax,
model.x_input,
name="gradients_ig")[0]
source_model_input_grad = tf.gradients(source_model.target_softmax,
source_model.x_input,
name="gradients_ig_source_model")[0]
# lp norm diff between input_grad & source_model_input_grad
input_grad_l2_norm_diff = tf.reduce_mean(
tf.reduce_sum(tf.pow(tf.subtract(input_grad, source_model_input_grad),
2.0),
keepdims=True))
# Setting up the discriminator model
labels_input_grad = tf.zeros(tf.shape(input_grad)[0], dtype=tf.int64)
labels_source_model_input_grad = tf.ones(tf.shape(input_grad)[0],
dtype=tf.int64)
disc_model = IgamConvDiscriminatorModel(
mode='train',
dataset=dataset,
train_batch_size=train_batch_size,
num_conv_layers=disc_layers,
base_num_channels=disc_base_channels,
normalize_zero_mean=normalize_zero_mean,
x_modelgrad_input_tensor=input_grad,
y_modelgrad_input_tensor=labels_input_grad,
x_source_modelgrad_input_tensor=source_model_input_grad,
y_source_modelgrad_input_tensor=labels_source_model_input_grad,
only_fully_connected=only_fully_connected)
t_vars = tf.trainable_variables()
C_vars = [var for var in t_vars if 'classifier' in var.name]
D_vars = [var for var in t_vars if 'discriminator' in var.name]
source_model_vars = [
var for var in t_vars
if ('discriminator' not in var.name and 'classifier' not in var.name
and 'target_task_logit' not in var.name)
]
source_model_target_logit_vars = [
var for var in t_vars if 'target_task_logit' in var.name
]
source_model_saver = tf.train.Saver(var_list=source_model_vars)
finetuned_source_model_vars = source_model_vars + source_model_target_logit_vars
finetuned_source_model_saver = tf.train.Saver(
var_list=finetuned_source_model_vars)
# Source model finetune optimization
source_model_finetune_loss = source_model.target_task_mean_xent + weight_decay * source_model.weight_decay_loss
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss - beta * disc_model.mean_xent + gamma * input_grad_l2_norm_diff
classification_c_loss = model.mean_xent + weight_decay * model.weight_decay_loss
adv_c_loss = -beta * disc_model.mean_xent
# Discriminator: Optimizating computation
# discriminator loss
total_d_loss = disc_model.mean_xent + weight_decay * disc_model.weight_decay_loss
# Finetune source_model
if finetune_whole_model:
finetune_min_step = finetune_optimizer.minimize(
source_model_finetune_loss, var_list=finetuned_source_model_vars)
else:
finetune_min_step = finetune_optimizer.minimize(
source_model_finetune_loss,
var_list=source_model_target_logit_vars)
# Train classifier
# classifier opt step
final_grads = c_optimizer.compute_gradients(total_loss, var_list=C_vars)
no_pert_grad = [(tf.zeros_like(v), v) if 'perturbation' in v.name else
(g, v) for g, v in final_grads]
c_min_step = c_optimizer.apply_gradients(no_pert_grad)
# c_min_step = c_optimizer.minimize(total_loss, var_list=C_vars)
classification_final_grads = c_optimizer.compute_gradients(
classification_c_loss, var_list=C_vars)
classification_no_pert_grad = [(tf.zeros_like(v),
v) if 'perturbation' in v.name else (g, v)
for g, v in classification_final_grads]
c_classification_min_step = c_optimizer.apply_gradients(
classification_no_pert_grad)
# discriminator opt step
d_min_step = d_optimizer.minimize(total_d_loss, var_list=D_vars)
# Loss gradients to the model params
logit_weights = tf.get_default_graph().get_tensor_by_name(
'classifier/logit/DW:0')
last_conv_weights = tf.get_default_graph().get_tensor_by_name(
'classifier/unit_3_4/sub2/conv2/DW:0')
first_conv_weights = tf.get_default_graph().get_tensor_by_name(
'classifier/input/init_conv/DW:0')
model_xent_logit_grad_norm = tf.norm(tf.gradients(model.mean_xent,
logit_weights)[0],
ord='euclidean')
disc_xent_logit_grad_norm = tf.norm(tf.gradients(disc_model.mean_xent,
logit_weights)[0],
ord='euclidean')
input_grad_l2_norm_diff_logit_grad_norm = tf.norm(tf.gradients(
input_grad_l2_norm_diff, logit_weights)[0],
ord='euclidean')
model_xent_last_conv_grad_norm = tf.norm(tf.gradients(
model.mean_xent, last_conv_weights)[0],
ord='euclidean')
disc_xent_last_conv_grad_norm = tf.norm(tf.gradients(
disc_model.mean_xent, last_conv_weights)[0],
ord='euclidean')
input_grad_l2_norm_diff_last_conv_grad_norm = tf.norm(tf.gradients(
input_grad_l2_norm_diff, last_conv_weights)[0],
ord='euclidean')
model_xent_first_conv_grad_norm = tf.norm(tf.gradients(
model.mean_xent, first_conv_weights)[0],
ord='euclidean')
disc_xent_first_conv_grad_norm = tf.norm(tf.gradients(
disc_model.mean_xent, first_conv_weights)[0],
ord='euclidean')
input_grad_l2_norm_diff_first_conv_grad_norm = tf.norm(tf.gradients(
input_grad_l2_norm_diff, first_conv_weights)[0],
ord='euclidean')
# Setting up the Tensorboard and checkpoint outputs
if not os.path.exists(model_dir):
os.makedirs(model_dir)
saver = tf.train.Saver(max_to_keep=1)
tf.summary.scalar('C accuracy', model.accuracy)
tf.summary.scalar('D accuracy', disc_model.accuracy)
tf.summary.scalar('C xent', model.xent / train_batch_size)
tf.summary.scalar('D xent', disc_model.xent / train_batch_size)
tf.summary.scalar('total C loss', total_loss / train_batch_size)
tf.summary.scalar('total D loss', total_d_loss / train_batch_size)
tf.summary.scalar('adv C loss', adv_c_loss / train_batch_size)
tf.summary.scalar('C cls xent loss', model.mean_xent)
tf.summary.scalar('D xent loss', disc_model.mean_xent)
# Loss gradients
tf.summary.scalar('model_xent_logit_grad_norm', model_xent_logit_grad_norm)
tf.summary.scalar('disc_xent_logit_grad_norm', disc_xent_logit_grad_norm)
tf.summary.scalar('input_grad_l2_norm_diff_logit_grad_norm',
input_grad_l2_norm_diff_logit_grad_norm)
tf.summary.scalar('model_xent_last_conv_grad_norm',
model_xent_last_conv_grad_norm)
tf.summary.scalar('disc_xent_last_conv_grad_norm',
disc_xent_last_conv_grad_norm)
tf.summary.scalar('input_grad_l2_norm_diff_last_conv_grad_norm',
input_grad_l2_norm_diff_last_conv_grad_norm)
tf.summary.scalar('model_xent_first_conv_grad_norm',
model_xent_first_conv_grad_norm)
tf.summary.scalar('disc_xent_first_conv_grad_norm',
disc_xent_first_conv_grad_norm)
tf.summary.scalar('input_grad_l2_norm_diff_first_conv_grad_norm',
input_grad_l2_norm_diff_first_conv_grad_norm)
merged_summaries = tf.summary.merge_all()
with tf.Session() as sess:
print(
'important params >>> \n model dir: %s \n dataset: %s \n training batch size: %d \n'
% (model_dir, dataset, train_batch_size))
# initialize data augmentation
if dataset == 'cifar10':
data = cifar10_input.AugmentedCIFAR10Data(raw_data, sess, model)
else:
data = cifar100_input.AugmentedCIFAR100Data(raw_data, sess, model)
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir + '/train',
sess.graph)
eval_summary_writer = tf.summary.FileWriter(model_dir + '/eval')
sess.run(tf.global_variables_initializer())
# Restore source model
source_model_file = tf.train.latest_checkpoint(source_model_dir)
source_model_saver.restore(sess, source_model_file)
# Finetune source model here
if train_finetune_source_model:
time_before_finetuning = datetime.now()
for ii in tqdm(range(finetune_train_steps)):
x_batch, y_batch = data.train_data.get_next_batch(
train_batch_size, multiple_passes=True)
if finetune_img_random_pert:
x_batch = x_batch + np.random.uniform(
-epsilon, epsilon, x_batch.shape)
x_batch = np.clip(x_batch, 0,
255) # ensure valid pixel range
nat_dict = {
source_model.x_input: x_batch,
source_model.y_input: y_batch
}
# Output to stdout
if ii % summary_steps == 0:
train_finetune_acc, train_finetune_loss = sess.run(
[
source_model.target_task_accuracy,
source_model_finetune_loss
],
feed_dict=nat_dict)
x_eval_batch, y_eval_batch = data.eval_data.get_next_batch(
train_batch_size, multiple_passes=True)
if img_random_pert:
x_eval_batch = x_eval_batch + np.random.uniform(
-epsilon, epsilon, x_eval_batch.shape)
x_eval_batch = np.clip(x_eval_batch, 0,
255) # ensure valid pixel range
eval_dict = {
source_model.x_input: x_eval_batch,
source_model.y_input: y_eval_batch
}
val_finetune_acc, val_finetune_loss = sess.run(
[
source_model.target_task_accuracy,
source_model_finetune_loss
],
feed_dict=eval_dict)
print('Source Model Finetune Step {}: ({})'.format(
ii, datetime.now()))
print(
' training nat accuracy {:.4}% -- validation nat accuracy {:.4}%'
.format(train_finetune_acc * 100,
val_finetune_acc * 100))
print(' training nat c loss: {}'.format(
train_finetune_loss))
print(' validation nat c loss: {}'.format(
val_finetune_loss))
sys.stdout.flush()
sess.run(finetune_min_step, feed_dict=nat_dict)
sess.run(increment_global_step_op)
time_after_finetuning = datetime.now()
finetuning_time = time_after_finetuning - time_before_finetuning
finetuning_time_file_path = os.path.join(model_dir,
'finetuning_time.txt')
with open(finetuning_time_file_path, "w") as f:
f.write("Total finetuning time: {}".format(
str(finetuning_time)))
print("Total finetuning time: {}".format(str(finetuning_time)))
finetuned_source_model_saver.save(sess,
os.path.join(
finetuned_source_model_dir,
'checkpoint'),
global_step=global_step)
if only_finetune:
return
else:
finetuned_source_model_file = tf.train.latest_checkpoint(
finetuned_source_model_dir)
finetuned_source_model_saver.restore(sess,
finetuned_source_model_file)
# reset global step to 0 before running main training loop
sess.run(reset_global_step_op)
time_before_training = datetime.now()
# Main training loop
for ii in tqdm(range(train_steps)):
x_batch, y_batch = data.train_data.get_next_batch(
train_batch_size, multiple_passes=True)
if img_random_pert:
x_batch = x_batch + np.random.uniform(-epsilon, epsilon,
x_batch.shape)
x_batch = np.clip(x_batch, 0, 255) # ensure valid pixel range
labels_source_modelgrad_disc = np.ones_like(y_batch,
dtype=np.int64)
# Sample randinit input grads
nat_dict = {
model.x_input: x_batch,
model.y_input: y_batch,
source_model.x_input: x_batch,
source_model.y_input: y_batch
}
# Output to stdout
if ii % summary_steps == 0:
train_acc, train_disc_acc, train_c_loss, train_d_loss, train_adv_c_loss, summary = sess.run(
[
model.accuracy, disc_model.accuracy, total_loss,
total_d_loss, adv_c_loss, merged_summaries
],
feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
x_eval_batch, y_eval_batch = data.eval_data.get_next_batch(
train_batch_size, multiple_passes=True)
if img_random_pert:
x_eval_batch = x_eval_batch + np.random.uniform(
-epsilon, epsilon, x_eval_batch.shape)
x_eval_batch = np.clip(x_eval_batch, 0,
255) # ensure valid pixel range
labels_source_modelgrad_disc = np.ones_like(y_eval_batch,
dtype=np.int64)
eval_dict = {
model.x_input: x_eval_batch,
model.y_input: y_eval_batch,
source_model.x_input: x_eval_batch,
source_model.y_input: y_eval_batch
}
val_acc, val_disc_acc, val_c_loss, val_d_loss, val_adv_c_loss, summary = sess.run(
[
model.accuracy, disc_model.accuracy, total_loss,
total_d_loss, adv_c_loss, merged_summaries
],
feed_dict=eval_dict)
eval_summary_writer.add_summary(summary,
global_step.eval(sess))
print('Step {}: ({})'.format(ii, datetime.now()))
print(
' training nat accuracy {:.4}% -- validation nat accuracy {:.4}%'
.format(train_acc * 100, val_acc * 100))
print(
' training nat disc accuracy {:.4}% -- validation nat disc accuracy {:.4}%'
.format(train_disc_acc * 100, val_disc_acc * 100))
print(
' training nat c loss: {}, d loss: {}, adv c loss: {}'
.format(train_c_loss, train_d_loss, train_adv_c_loss))
print(
' validation nat c loss: {}, d loss: {}, adv c loss: {}'
.format(val_c_loss, val_d_loss, val_adv_c_loss))
sys.stdout.flush()
# Tensorboard summaries
elif ii % out_steps == 0:
nat_acc, nat_disc_acc, nat_c_loss, nat_d_loss, nat_adv_c_loss = sess.run(
[
model.accuracy, disc_model.accuracy, total_loss,
total_d_loss, adv_c_loss
],
feed_dict=nat_dict)
print('Step {}: ({})'.format(ii, datetime.now()))
print(' training nat accuracy {:.4}%'.format(nat_acc * 100))
print(' training nat disc accuracy {:.4}%'.format(
nat_disc_acc * 100))
print(
' training nat c loss: {}, d loss: {}, adv c loss: {}'
.format(nat_c_loss, nat_d_loss, nat_adv_c_loss))
# Write a checkpoint
if (ii + 1) % checkpoint_steps == 0:
saver.save(sess,
os.path.join(model_dir, 'checkpoint'),
global_step=global_step)
# default mode
if sep_opt_version == 1:
if ii >= steps_before_adv_opt:
# Actual training step for Classifier
sess.run(c_min_step, feed_dict=nat_dict)
sess.run(increment_global_step_op)
if ii % disc_update_steps == 0:
# Actual training step for Discriminator
sess.run(d_min_step, feed_dict=nat_dict)
else:
# only train on classification loss
sess.run(c_classification_min_step, feed_dict=nat_dict)
sess.run(increment_global_step_op)
elif sep_opt_version == 2:
# Actual training step for Classifier
if ii >= steps_before_adv_opt:
if adv_update_steps_per_iter > 1:
sess.run(c_classification_min_step, feed_dict=nat_dict)
sess.run(increment_global_step_op)
for i in range(adv_update_steps_per_iter):
x_batch, y_batch = data.train_data.get_next_batch(
train_batch_size, multiple_passes=True)
if img_random_pert:
x_batch = x_batch + np.random.uniform(
-epsilon, epsilon, x_batch.shape)
x_batch = np.clip(
x_batch, 0,
255) # ensure valid pixel range
nat_dict = {
model.x_input: x_batch,
model.y_input: y_batch,
source_model.x_input: x_batch,
source_model.y_input: y_batch
}
sess.run(c_adv_min_step, feed_dict=nat_dict)
else:
sess.run(c_min_step, feed_dict=nat_dict)
sess.run(increment_global_step_op)
if ii % disc_update_steps == 0:
# Actual training step for Discriminator
sess.run(d_min_step, feed_dict=nat_dict)
else:
# only train on classification loss
sess.run(c_classification_min_step, feed_dict=nat_dict)
sess.run(increment_global_step_op)
elif sep_opt_version == 0:
if ii >= steps_before_adv_opt:
if ii % disc_update_steps == 0:
sess.run([c_min_step, d_min_step], feed_dict=nat_dict)
sess.run(increment_global_step_op)
else:
sess.run(c_min_step, feed_dict=nat_dict)
sess.run(increment_global_step_op)
else:
sess.run(c_classification_min_step, feed_dict=nat_dict)
sess.run(increment_global_step_op)
time_after_training = datetime.now()
training_time = time_after_training - time_before_training
training_time_file_path = os.path.join(model_dir, 'training_time.txt')
with open(training_time_file_path, "w") as f:
f.write("Total Training time: {}".format(str(training_time)))
print("Total Training time: {}".format(str(training_time)))
# full test evaluation
if dataset == 'cifar10':
raw_data = cifar10_input.CIFAR10Data(data_path)
else:
raw_data = cifar100_input.CIFAR100Data(data_path)
data_size = raw_data.eval_data.n
if data_size % train_batch_size == 0:
eval_steps = data_size // train_batch_size
else:
eval_steps = data_size // train_batch_size
# eval_steps = data_size // train_batch_size + 1
total_num_correct = 0
for ii in tqdm(range(eval_steps)):
x_eval_batch, y_eval_batch = raw_data.eval_data.get_next_batch(
train_batch_size, multiple_passes=False)
eval_dict = {
model.x_input: x_eval_batch,
model.y_input: y_eval_batch
}
num_correct = sess.run(model.num_correct, feed_dict=eval_dict)
total_num_correct += num_correct
eval_acc = total_num_correct / data_size
clean_eval_file_path = os.path.join(model_dir,
'full_clean_eval_acc.txt')
with open(clean_eval_file_path, "a+") as f:
f.write("Full clean eval_acc: {}%".format(eval_acc * 100))
print("Full clean eval_acc: {}%".format(eval_acc * 100))
devices = sess.list_devices()
for d in devices:
print("sess' device names:")
print(d.name)
return model_dir
if __name__ == '__main__':
with open('config.json') as config_file:
config = json.load(config_file)
if config['data_path'] == "cifar100_data":
mode = 'cifar_100'
elif config['data_path'] == "cifar10_data":
mode = 'cifar_10'
else:
print("mode should be either cifar_10 ro cifar_100")
exit(1)
if mode == 'cifar_10':
import cifar10_input
elif mode == 'cifar_100':
import cifar100_input
if mode == 'cifar_10':
cifar = cifar10_input.CIFAR10Data(config['data_path'])
elif mode == 'cifar_100':
cifar = cifar100_input.CIFAR100Data(config['data_path'])
if mode == 'cifar_10':
model = Model(mode='eval', class_count=10)
elif mode == 'cifar_100':
model = Model(mode='eval', class_count=100)
attack = LinfPGDAttack(model, config['epsilon'], config['num_steps'],
config['step_size'], config['random_start'],
config['loss_func'])
saver = tf.train.Saver()
with tf.Session() as sess:
saver.restore(
sess, tf.train.latest_checkpoint(config['pretrained_model_dir']))
# print('Results of Eval Data:')
# my_eval(config=config, cifar=cifar, model=model, attack=attack, sess=sess, source='eval_data')
def evaluate(model, attack, sess, config, summary_writer=None):
num_eval_examples = config.eval.num_eval_examples
eval_batch_size = config.eval.batch_size
data_path = config.data.data_path
model_dir = config.model.output_dir
# Setting up the Tensorboard and checkpoint outputs
if not os.path.exists(model_dir):
os.makedirs(model_dir)
cifar = cifar10_input.CIFAR10Data(data_path)
global_step = tf.contrib.framework.get_or_create_global_step()
# Iterate over the samples batch-by-batch
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
total_xent_nat = 0.
total_xent_adv = 0.
total_corr_nat = 0
total_corr_adv = 0
for ibatch in trange(num_batches):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
x_batch = cifar.eval_data.xs[bstart:bend, :]
y_batch = cifar.eval_data.ys[bstart:bend]
noop_trans = np.zeros([len(x_batch), 3])
if config.eval.adversarial_eval:
x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess)
else:
x_batch_adv, adv_trans = x_batch, noop_trans
dict_nat = {
model.x_input: x_batch,
model.y_input: y_batch,
model.transform: noop_trans,
model.is_training: False
}
dict_adv = {
model.x_input: x_batch_adv,
model.y_input: y_batch,
model.transform: adv_trans,
model.is_training: False
}
cur_corr_nat, cur_xent_nat = sess.run([model.num_correct, model.xent],
feed_dict=dict_nat)
cur_corr_adv, cur_xent_adv = sess.run([model.num_correct, model.xent],
feed_dict=dict_adv)
total_xent_nat += cur_xent_nat
total_xent_adv += cur_xent_adv
total_corr_nat += cur_corr_nat
total_corr_adv += cur_corr_adv
avg_xent_nat = total_xent_nat / num_eval_examples
avg_xent_adv = total_xent_adv / num_eval_examples
acc_nat = total_corr_nat / num_eval_examples
acc_adv = total_corr_adv / num_eval_examples
if summary_writer:
summary = tf.Summary(value=[
tf.Summary.Value(tag='xent_adv_eval', simple_value=avg_xent_adv),
tf.Summary.Value(tag='xent_nat_eval', simple_value=avg_xent_nat),
tf.Summary.Value(tag='xent_adv', simple_value=avg_xent_adv),
tf.Summary.Value(tag='xent_nat', simple_value=avg_xent_nat),
tf.Summary.Value(tag='accuracy_adv_eval', simple_value=acc_adv),
tf.Summary.Value(tag='accuracy_nat_eval', simple_value=acc_nat),
tf.Summary.Value(tag='accuracy_adv', simple_value=acc_adv),
tf.Summary.Value(tag='accuracy_nat', simple_value=acc_nat)
])
summary_writer.add_summary(summary, global_step.eval(sess))
step = global_step.eval(sess)
print('Eval at step: {}'.format(step))
print(' natural: {:.2f}%'.format(100 * acc_nat))
print(' adversarial: {:.2f}%'.format(100 * acc_adv))
print(' avg nat xent: {:.4f}'.format(avg_xent_nat))
print(' avg adv xent: {:.4f}'.format(avg_xent_adv))
result = {
'nat': '{:.2f}%'.format(100 * acc_nat),
'adv': '{:.2f}%'.format(100 * acc_adv)
}
with open('job_result.json', 'w') as result_file:
json.dump(result, result_file, sort_keys=True, indent=4)
def main(argv):
model_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if model_file is None:
print("No model found")
sys.exit()
cifar = cifar10_input.CIFAR10Data(FLAGS.dataset_dir)
nb_classes = 10
X_test = cifar.eval_data.xs
Y_test = to_categorical(cifar.eval_data.ys, nb_classes)
assert Y_test.shape[1] == 10.0
set_log_level(logging.DEBUG)
with tf.Session() as sess:
x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3))
y = tf.placeholder(tf.float32, shape=(None, 10))
from cleverhans.model_zoo.madry_lab_challenges.cifar10_model import make_wresnet
model = make_wresnet()
saver = tf.train.Saver()
# Restore the checkpoint
saver.restore(sess, model_file)
nb_samples = FLAGS.nb_samples
attack_params = {
"batch_size": FLAGS.batch_size,
"clip_min": 0.0,
"clip_max": 255.0,
}
if FLAGS.attack_type == "cwl2":
from cleverhans.attacks import CarliniWagnerL2
attacker = CarliniWagnerL2(model, sess=sess)
attack_params.update(
{
"binary_search_steps": 1,
"max_iterations": 100,
"learning_rate": 0.1,
"initial_const": 10,
"batch_size": 10,
}
)
else: # eps and eps_iter in range 0-255
attack_params.update({"eps": 8, "ord": np.inf})
if FLAGS.attack_type == "fgsm":
from cleverhans.attacks import FastGradientMethod
attacker = FastGradientMethod(model, sess=sess)
elif FLAGS.attack_type == "pgd":
attack_params.update({"eps_iter": 2, "nb_iter": 20})
from cleverhans.attacks import MadryEtAl
attacker = MadryEtAl(model, sess=sess)
eval_par = {"batch_size": FLAGS.batch_size}
if FLAGS.sweep:
max_eps = 16
epsilons = np.linspace(1, max_eps, max_eps)
for e in epsilons:
t1 = time.time()
attack_params.update({"eps": e})
x_adv = attacker.generate(x, **attack_params)
preds_adv = model.get_probs(x_adv)
acc = model_eval(
sess,
x,
y,
preds_adv,
X_test[:nb_samples],
Y_test[:nb_samples],
args=eval_par,
)
print(
"Epsilon %.2f, accuracy on adversarial" % e,
"examples %0.4f\n" % acc,
)
t2 = time.time()
else:
t1 = time.time()
x_adv = attacker.generate(x, **attack_params)
preds_adv = model.get_probs(x_adv)
acc = model_eval(
sess,
x,
y,
preds_adv,
X_test[:nb_samples],
Y_test[:nb_samples],
args=eval_par,
)
t2 = time.time()
print("Test accuracy on adversarial examples %0.4f\n" % acc)
print("Took", t2 - t1, "seconds")
print("after binary search: lbd_ih:", lbd_hi, "***")
# tmp_lb = self.model.predict(x0+ np.array(lbd_hi*theta),y0)
print("label after fine binary search:",
self.model.predict(x0 + np.array(lbd_hi * theta), y0))
print("again, label after fine binary search:",
self.model.predict(x0 + np.array(lbd_hi * theta), y0))
print("norm of lbd_hi*theta 3:",
np.linalg.norm(x0 + np.array(lbd_hi * theta)))
print("******")
return lbd_hi, nquery
# ============ test ====================
sess = tf.Session()
cifar = cifar10_input.CIFAR10Data(
"/content/drive/My Drive/STAT141C/cifar10_data")
orig_model = Model("/content/drive/My Drive/STAT141C/models/standard/",
tiny=False,
mode='eval',
sess=sess)
model = Model(model, sess, [0.0, 255.0])
attack = blackbox(model)
#xs = tf.placeholder(tf.float32, (1, 32, 32, 3))
image = cifar.eval_data.xs[:15]
label = cifar.eval_data.ys[:15]
new_img = image / 255.0
#print("original label is :", label[1])
def train(tf_seed, np_seed, train_steps, out_steps, summary_steps,
checkpoint_steps, step_size_schedule, weight_decay, momentum,
train_batch_size, epsilon, replay_m, model_dir, dataset,
poison_alpha, poison_config, **kwargs):
tf.compat.v1.set_random_seed(tf_seed)
np.random.seed(np_seed)
print('poison alpha = %f' % poison_alpha)
model_dir = model_dir + '%s_m%d_eps%.1f_b%d' % (
dataset, replay_m, epsilon, train_batch_size
) # TODO Replace with not defaults
# Setting up the data and the model
poison_config_dict = utilities.config_to_namedtuple(
utilities.get_config(poison_config))
print(poison_config_dict)
data_path = get_path_dir(dataset=dataset, **kwargs)
if dataset == 'cifar10':
raw_data = cifar10_input.CIFAR10Data(data_path)
elif dataset == 'cifar10_poisoned':
raw_data = dataset_input.CIFAR10Data(poison_config_dict, seed=np_seed)
else:
raw_data = cifar100_input.CIFAR100Data(data_path)
global_step = tf.compat.v1.train.get_or_create_global_step()
with tpu_strategy.scope():
model = Model(mode='train',
dataset=dataset,
train_batch_size=train_batch_size)
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule][1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.compat.v1.train.piecewise_constant(
tf.cast(global_step, tf.int32), boundaries, values)
optimizer = tf.compat.v1.train.MomentumOptimizer(learning_rate, momentum)
# Optimizing computation
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss
grads = optimizer.compute_gradients(total_loss)
# Compute new image
pert_grad = [g for g, v in grads if 'perturbation' in v.name]
sign_pert_grad = tf.sign(pert_grad[0])
new_pert = model.pert + epsilon * sign_pert_grad
clip_new_pert = tf.clip_by_value(new_pert, -epsilon, epsilon)
assigned = tf.compat.v1.assign(model.pert, clip_new_pert)
# Train
no_pert_grad = [(tf.zeros_like(v), v) if 'perturbation' in v.name else
(g, v) for g, v in grads]
with tf.control_dependencies([assigned]):
min_step = optimizer.apply_gradients(no_pert_grad,
global_step=global_step)
tf.compat.v1.initialize_variables([model.pert]) # TODO: Removed from TF
# Setting up the Tensorboard and checkpoint outputs
if not os.path.exists(model_dir):
os.makedirs(model_dir)
saver = tf.compat.v1.train.Saver(max_to_keep=1)
tf.compat.v1.summary.scalar('accuracy', model.accuracy)
tf.compat.v1.summary.scalar('xent', model.xent / train_batch_size)
tf.compat.v1.summary.scalar('total loss', total_loss / train_batch_size)
merged_summaries = tf.compat.v1.summary.merge_all()
gpu_options = tf.compat.v1.GPUOptions(per_process_gpu_memory_fraction=1.0)
with tf.compat.v1.Session(config=tf.compat.v1.ConfigProto(
gpu_options=gpu_options)) as sess:
print(
'\n\n********** free training for epsilon=%.1f using m_replay=%d **********\n\n'
% (epsilon, replay_m))
print(
'important params >>> \n model dir: %s \n dataset: %s \n training batch size: %d \n'
% (model_dir, dataset, train_batch_size))
if dataset == 'cifar100':
print(
'the ride for CIFAR100 is bumpy -- fasten your seatbelts! \n \
you will probably see the training and validation accuracy fluctuating a lot early in trainnig \n \
this is natural especially for large replay_m values because we see that mini-batch so many times.'
)
# initialize data augmentation
if dataset == 'cifar10':
data = cifar10_input.AugmentedCIFAR10Data(raw_data, sess, model)
elif dataset == 'cifar10_poisoned':
data = raw_data
else:
data = cifar100_input.AugmentedCIFAR100Data(raw_data, sess, model)
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.compat.v1.summary.FileWriter(
model_dir + '/train', sess.graph)
eval_summary_writer = tf.compat.v1.summary.FileWriter(model_dir +
'/eval')
sess.run(tf.compat.v1.global_variables_initializer())
# Main training loop
for ii in range(train_steps):
if ii % replay_m == 0:
x_batch, y_batch = data.train_data.get_next_batch(
train_batch_size, multiple_passes=True)
nat_dict = {model.x_input: x_batch, model.y_input: y_batch}
x_eval_batch, y_eval_batch = data.eval_data.get_next_batch(
train_batch_size, multiple_passes=True)
eval_dict = {
model.x_input: x_eval_batch,
model.y_input: y_eval_batch
}
# Output to stdout
if ii % summary_steps == 0:
train_acc, summary = sess.run(
[model.accuracy, merged_summaries], feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
val_acc, summary = sess.run([model.accuracy, merged_summaries],
feed_dict=eval_dict)
eval_summary_writer.add_summary(summary,
global_step.eval(sess))
print('Step {}: ({})'.format(ii, datetime.now()))
print(
' training nat accuracy {:.4}% -- validation nat accuracy {:.4}%'
.format(train_acc * 100, val_acc * 100))
sys.stdout.flush()
# Tensorboard summaries
elif ii % out_steps == 0:
nat_acc = sess.run(model.accuracy, feed_dict=nat_dict)
print('Step {}: ({})'.format(ii, datetime.now()))
print(' training nat accuracy {:.4}%'.format(nat_acc * 100))
# Write a checkpoint
if (ii + 1) % checkpoint_steps == 0:
saver.save(sess,
os.path.join(model_dir, 'checkpoint'),
global_step=global_step)
# Actual training step
sess.run(min_step, feed_dict=nat_dict)
def evaluate(model, attack, sess, config, attack_type, data_path,
summary_writer=None, eval_on_train=False):
num_eval_examples = config.eval.num_eval_examples
eval_batch_size = config.eval.batch_size
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.CIFAR10Data(data_path)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.CIFAR100Data(data_path)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.SVHNData(data_path)
else:
raise ValueError("Unknown dataset name.")
global_step = tf.train.get_or_create_global_step()
# Iterate over the samples batch-by-batch
num_batches = int(math.ceil(num_eval_examples / eval_batch_size))
total_xent_nat = 0.
total_xent_adv = 0.
total_corr_nat = 0
total_corr_adv = 0
for ibatch in trange(num_batches):
bstart = ibatch * eval_batch_size
bend = min(bstart + eval_batch_size, num_eval_examples)
if eval_on_train:
x_batch = data_iterator.train_data.xs[bstart:bend, :]
y_batch = data_iterator.train_data.ys[bstart:bend]
else:
x_batch = data_iterator.eval_data.xs[bstart:bend, :]
y_batch = data_iterator.eval_data.ys[bstart:bend]
noop_trans = np.zeros([len(x_batch), 3])
if config.eval.adversarial_eval:
x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess)
else:
x_batch_adv, adv_trans = x_batch, noop_trans
dict_nat = {model.x_input: x_batch,
model.y_input: y_batch,
model.transform: noop_trans,
model.is_training: False}
dict_adv = {model.x_input: x_batch_adv,
model.y_input: y_batch,
model.transform: adv_trans,
model.is_training: False}
cur_corr_nat, cur_xent_nat = sess.run([model.num_correct, model.xent],
feed_dict = dict_nat)
cur_corr_adv, cur_xent_adv = sess.run([model.num_correct, model.xent],
feed_dict = dict_adv)
total_xent_nat += cur_xent_nat
total_xent_adv += cur_xent_adv
total_corr_nat += cur_corr_nat
total_corr_adv += cur_corr_adv
avg_xent_nat = total_xent_nat / num_eval_examples
avg_xent_adv = total_xent_adv / num_eval_examples
acc_nat = total_corr_nat / num_eval_examples
acc_adv = total_corr_adv / num_eval_examples
if summary_writer:
summary = tf.Summary(value=[
tf.Summary.Value(tag='xent_adv_eval', simple_value= avg_xent_adv),
tf.Summary.Value(tag='xent_nat_eval', simple_value= avg_xent_nat),
tf.Summary.Value(tag='xent_adv', simple_value= avg_xent_adv),
tf.Summary.Value(tag='xent_nat', simple_value= avg_xent_nat),
tf.Summary.Value(tag='accuracy_adv_eval', simple_value= acc_adv),
tf.Summary.Value(tag='accuracy_nat_eval', simple_value= acc_nat),
tf.Summary.Value(tag='accuracy_adv', simple_value= acc_adv),
tf.Summary.Value(tag='accuracy_nat', simple_value= acc_nat)])
summary_writer.add_summary(summary, global_step.eval(sess))
step = global_step.eval(sess)
print('Eval at step: {}'.format(step))
print(' Adversary: ', attack_type)
print(' natural: {:.2f}%'.format(100 * acc_nat))
print(' adversarial: {:.2f}%'.format(100 * acc_adv))
print(' avg nat xent: {:.4f}'.format(avg_xent_nat))
print(' avg adv xent: {:.4f}'.format(avg_xent_adv))
return [100 * acc_nat, 100 * acc_adv, avg_xent_nat, avg_xent_adv]
def main(argv):
model_file = tf.train.latest_checkpoint(FLAGS.checkpoint_dir)
if model_file is None:
print('No model found')
sys.exit()
cifar = cifar10_input.CIFAR10Data(FLAGS.dataset_dir)
nb_classes = 10
X_test = cifar.eval_data.xs
Y_test = to_categorical(cifar.eval_data.ys, nb_classes)
assert Y_test.shape[1] == 10.
set_log_level(logging.DEBUG)
with tf.Session() as sess:
x = tf.placeholder(tf.float32, shape=(None, 32, 32, 3))
y = tf.placeholder(tf.float32, shape=(None, 10))
from madry_cifar10_model import make_madry_wresnet
model = make_madry_wresnet()
saver = tf.train.Saver()
# Restore the checkpoint
saver.restore(sess, model_file)
nb_samples = FLAGS.nb_samples
attack_params = {'batch_size': FLAGS.batch_size,
'clip_min': 0., 'clip_max': 255.}
if FLAGS.attack_type == 'cwl2':
from cleverhans.attacks import CarliniWagnerL2
attacker = CarliniWagnerL2(model, sess=sess)
attack_params.update({'binary_search_steps': 1,
'max_iterations': 100,
'learning_rate': 0.1,
'initial_const': 10,
'batch_size': 10
})
else: # eps and eps_iter in range 0-255
attack_params.update({'eps': 8, 'ord': np.inf})
if FLAGS.attack_type == 'fgsm':
from cleverhans.attacks import FastGradientMethod
attacker = FastGradientMethod(model, sess=sess)
elif FLAGS.attack_type == 'pgd':
attack_params.update({'eps_iter': 2, 'nb_iter': 20})
from cleverhans.attacks import MadryEtAl
attacker = MadryEtAl(model, sess=sess)
eval_par = {'batch_size': FLAGS.batch_size}
if FLAGS.sweep:
max_eps = 16
epsilons = np.linspace(1, max_eps, max_eps)
for e in epsilons:
t1 = time.time()
attack_params.update({'eps': e})
x_adv = attacker.generate(x, **attack_params)
preds_adv = model.get_probs(x_adv)
acc = model_eval(sess, x, y, preds_adv, X_test[
:nb_samples], Y_test[:nb_samples], args=eval_par)
print('Epsilon %.2f, accuracy on adversarial' % e,
'examples %0.4f\n' % acc)
t2 = time.time()
else:
t1 = time.time()
x_adv = attacker.generate(x, **attack_params)
preds_adv = model.get_probs(x_adv)
acc = model_eval(sess, x, y, preds_adv, X_test[
:nb_samples], Y_test[:nb_samples], args=eval_par)
t2 = time.time()
print('Test accuracy on adversarial examples %0.4f\n' % acc)
print("Took", t2 - t1, "seconds")
from eval_utils import *
os.environ['TF_CPP_MIN_LOG_LEVEL'] = '3' # or any {'0', '1', '2'}
tf.logging.set_verbosity(tf.logging.ERROR)
np.random.seed(123)
classifier = Model(mode='eval', var_scope='classifier')
classifier_vars = tf.get_collection(tf.GraphKeys.GLOBAL_VARIABLES,
scope='classifier')
classifier_saver = tf.train.Saver(var_list=classifier_vars)
classifier_checkpoint = 'models/naturally_trained_prefixed_classifier/checkpoint-70000'
factory = BaseDetectorFactory()
cifar = cifar10_input.CIFAR10Data('cifar10_data')
num_eval_examples = 10000 if len(sys.argv) <= 1 else int(sys.argv[1])
eval_data = cifar.eval_data
x_test = eval_data.xs.astype(np.float32)[:num_eval_examples]
y_test = eval_data.ys.astype(np.int32)[:num_eval_examples]
plt.figure(figsize=(3.5 * 1.5, 2 * 1.5))
with tf.Session() as sess:
classifier_saver.restore(sess, classifier_checkpoint)
factory.restore_base_detectors(sess)
base_detectors = factory.get_base_detectors()
bayes_classifier = BayesClassifier(base_detectors)
tf.set_random_seed(config['tf_random_seed'])
np.random.seed(config['np_random_seed'])
# Setting up training parameters
max_num_training_steps = config['max_num_training_steps']
num_output_steps = config['num_output_steps']
num_summary_steps = config['num_summary_steps']
num_checkpoint_steps = config['num_checkpoint_steps']
step_size_schedule = config['step_size_schedule']
weight_decay = config['weight_decay']
data_path = config['data_path']
momentum = config['momentum']
batch_size = config['training_batch_size']
# Setting up the data and the model
raw_cifar = cifar10_input.CIFAR10Data()
print(np.amax(raw_cifar.train_data.xs), np.amin(raw_cifar.train_data.xs))
global_step = tf.contrib.framework.get_or_create_global_step()
model = Model(mode='train')
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.train.piecewise_constant(
tf.cast(global_step, tf.int32),
boundaries,
values)
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss
train_step = tf.train.MomentumOptimizer(learning_rate, momentum).minimize(
total_loss,
def train(config):
# seeding randomness
tf.set_random_seed(config.training.tf_random_seed)
np.random.seed(config.training.np_random_seed)
# Setting up training parameters
max_num_training_steps = config.training.max_num_training_steps
step_size_schedule = config.training.step_size_schedule
weight_decay = config.training.weight_decay
momentum = config.training.momentum
batch_size = config.training.batch_size
group_size = config.training.group_size
adversarial_training = config.training.adversarial_training
eval_during_training = config.training.eval_during_training
if eval_during_training:
num_eval_steps = config.training.num_eval_steps
# Setting up output parameters
num_output_steps = config.training.num_output_steps
num_summary_steps = config.training.num_summary_steps
num_checkpoint_steps = config.training.num_checkpoint_steps
#adapting batch size
batch_size_group = batch_size * config.training.group_size
# Setting up the data and the model
data_path = config.data.data_path
raw_cifar = cifar10_input.CIFAR10Data(data_path)
global_step = tf.train.get_or_create_global_step()
model = resnet.Model(config.model)
# uncomment to get a list of trainable variables
# model_vars = tf.trainable_variables()
# slim.model_analyzer.analyze_vars(model_vars, print_info=True)
# Setting up the optimizer
boundaries = [int(sss[0]) for sss in step_size_schedule]
boundaries = boundaries[1:]
values = [sss[1] for sss in step_size_schedule]
learning_rate = tf.train.piecewise_constant(tf.cast(global_step, tf.int32),
boundaries, values)
total_loss = model.mean_xent + weight_decay * model.weight_decay_loss + model.core_loss
optimizer = tf.train.MomentumOptimizer(learning_rate, momentum)
train_step = optimizer.minimize(total_loss, global_step=global_step)
# Set up adversary
attack = SpatialAttack(model, config.attack)
attack_eval_random = SpatialAttack(model, config.eval_attack_random)
attack_eval_grid = SpatialAttack(model, config.eval_attack_grid)
# Setting up the Tensorboard and checkpoint outputs
model_dir = config.model.output_dir
if eval_during_training:
eval_dir = os.path.join(model_dir, 'eval')
if not os.path.exists(eval_dir):
os.makedirs(eval_dir)
# We add accuracy and xent twice so we can easily make three types of
# comparisons in Tensorboard:
# - train vs eval (for a single run)
# - train of different runs
# - eval of different runs
saver = tf.train.Saver(max_to_keep=3)
tf.summary.scalar('accuracy_adv_train',
model.accuracy,
collections=['adv'])
tf.summary.scalar('accuracy_adv', model.accuracy, collections=['adv'])
tf.summary.scalar('xent_adv_train',
model.xent / batch_size_group,
collections=['adv'])
tf.summary.scalar('xent_adv',
model.xent / batch_size_group,
collections=['adv'])
tf.summary.image('images_adv_train', model.x_image, collections=['adv'])
adv_summaries = tf.summary.merge_all('adv')
tf.summary.scalar('accuracy_nat_train',
model.accuracy,
collections=['nat'])
tf.summary.scalar('accuracy_nat', model.accuracy, collections=['nat'])
tf.summary.scalar('xent_nat_train',
model.xent / batch_size_group,
collections=['nat'])
tf.summary.scalar('xent_nat',
model.xent / batch_size_group,
collections=['nat'])
tf.summary.image('images_nat_train', model.x_image, collections=['nat'])
tf.summary.scalar('learning_rate', learning_rate, collections=['nat'])
nat_summaries = tf.summary.merge_all('nat')
#dataAugmentation
x_input_placeholder = tf.placeholder(tf.float32, shape=[None, 32, 32, 3])
flipped = tf.map_fn(lambda img: tf.image.random_flip_left_right(img),
x_input_placeholder)
with tf.Session() as sess:
# initialize data augmentation
if config.training.data_augmentation:
cifar = cifar10_input.AugmentedCIFAR10Data(raw_cifar, sess)
else:
cifar = raw_cifar
# Initialize the summary writer, global variables, and our time counter.
summary_writer = tf.summary.FileWriter(model_dir, sess.graph)
if eval_during_training:
eval_summary_writer = tf.summary.FileWriter(eval_dir)
sess.run(tf.global_variables_initializer())
training_time = 0.0
training_time_total = 0.0
adv_time = 0.0
eval_time = 0.0
core_time = 0.0
# Main training loop
for ii in range(max_num_training_steps + 1):
x_batch, y_batch = cifar.train_data.get_next_batch(
batch_size, multiple_passes=True)
noop_trans = np.zeros([len(x_batch), 3])
# Compute Adversarial Perturbations
if adversarial_training:
start = timer()
x_batch_adv, adv_trans = attack.perturb(x_batch, y_batch, sess)
end = timer()
adv_time += end - start
else:
x_batch_adv, adv_trans = x_batch, noop_trans
#Create rotatated examples
start = timer()
x_batch_nat = x_batch
y_batch_nat = y_batch
id_batch = np.arange(0, batch_size, 1, dtype="int32")
ids = np.arange(0, batch_size, 1, dtype="int32")
for i in range(config.training.group_size):
if config.training.data_augmentation_core:
x_batch_core = sess.run(
flipped, feed_dict={x_input_placeholder: x_batch})
else:
x_batch_core = x_batch
x_batch_group, trans_group = attack.perturb(
x_batch_core, y_batch, sess)
#construct new batches including rotateted examples
x_batch_adv = np.concatenate((x_batch_adv, x_batch_group),
axis=0)
x_batch_nat = np.concatenate((x_batch_nat, x_batch_group),
axis=0)
y_batch_nat = np.concatenate((y_batch_nat, y_batch), axis=0)
adv_trans = np.concatenate((adv_trans, trans_group), axis=0)
noop_trans = np.concatenate((noop_trans, trans_group), axis=0)
id_batch = np.concatenate((id_batch, ids), axis=0)
end = timer()
core_time += end - start
nat_dict = {
model.x_input: x_batch_nat,
model.y_input: y_batch_nat,
model.group: id_batch,
model.transform: noop_trans,
model.is_training: False
}
adv_dict = {
model.x_input: x_batch_adv,
model.y_input: y_batch_nat,
model.group: id_batch,
model.transform: adv_trans,
model.is_training: False
}
# Output to stdout
if ii % num_output_steps == 0:
nat_acc = sess.run(model.accuracy, feed_dict=nat_dict)
adv_acc = sess.run(model.accuracy, feed_dict=adv_dict)
print('Step {}: ({})'.format(ii, datetime.now()))
print(' training nat accuracy {:.4}%'.format(nat_acc * 100))
print(' training adv accuracy {:.4}%'.format(adv_acc * 100))
if ii != 0:
print(' {} examples per second'.format(
num_output_steps * batch_size_group / training_time))
training_time = 0.0
# Tensorboard summaries
if ii % num_summary_steps == 0:
summary = sess.run(adv_summaries, feed_dict=adv_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
summary = sess.run(nat_summaries, feed_dict=nat_dict)
summary_writer.add_summary(summary, global_step.eval(sess))
# Write a checkpoint
if ii % num_checkpoint_steps == 0:
saver.save(sess,
os.path.join(model_dir, 'checkpoint'),
global_step=global_step)
if eval_during_training and ii % num_eval_steps == 0:
start = timer()
evaluate(model, attack_eval_random, sess, config, "random",
eval_summary_writer)
evaluate(model, attack_eval_grid, sess, config, "grid",
eval_summary_writer)
end = timer()
eval_time += end - start
print(' {}seconds total training time'.format(
training_time_total))
print(' {}seconds total adv. example time'.format(adv_time))
print(
' {}seconds total core example time'.format(core_time))
print(' {}seconds total evalutation time'.format(eval_time))
# Actual training step
start = timer()
if adversarial_training:
adv_dict[model.is_training] = True
sess.run(train_step, feed_dict=adv_dict)
else:
nat_dict[model.is_training] = True
sess.run(train_step, feed_dict=nat_dict)
end = timer()
training_time += end - start
training_time_total += end - start
from model import Model
from pgd_attack import LinfPGDAttack
# from bpd_attack import LinfPGDAttack
# Global constants
with open('config_layer11.json') as config_file:
config = json.load(config_file)
num_eval_examples = int(config['num_eval_examples'])
eval_batch_size = config['eval_batch_size']
eval_on_cpu = config['eval_on_cpu']
data_path = config['data_path']
model_dir = config['new_model_dir']
# Set upd the data, hyperparameters, and the model
cifar = cifar10_input.CIFAR10Data(data_path)
if eval_on_cpu:
with tf.device("/cpu:0"):
model = Model(mode='eval')
attack = LinfPGDAttack(model, config['epsilon'], config['num_steps'],
config['step_size'], config['random_start'],
config['loss_func'])
else:
model = Model(mode='eval')
attack = LinfPGDAttack(model, config['epsilon'], config['num_steps'],
config['step_size'], config['random_start'],
config['loss_func'])
# global_step = tf.contrib.framework.get_or_create_global_step()
def get_correctly_classified_angles(model,
sess,
config,
eval_on_train=False,
num_eval_examples=200,
seed=1):
np.random.seed(seed)
if config.data.dataset_name == "cifar-10":
data_iterator = cifar10_input.CIFAR10Data(config.data.data_path)
elif config.data.dataset_name == "cifar-100":
data_iterator = cifar100_input.CIFAR100Data(config.data.data_path)
elif config.data.dataset_name == "svhn":
data_iterator = svhn_input.SVHNData(config.data.data_path)
if eval_on_train:
n = data_iterator.train_data.n
indices = np.random.choice(n, num_eval_examples)
x_batch = data_iterator.train_data.xs[indices, :]
y_batch = data_iterator.train_data.ys[indices]
else:
n = data_iterator.eval_data.n
indices = np.random.choice(n, num_eval_examples)
x_batch = data_iterator.eval_data.xs[indices, :]
y_batch = data_iterator.eval_data.ys[indices]
trans = np.zeros([len(x_batch), 3])
results = np.zeros([len(x_batch), 61])
predictions = np.zeros([len(x_batch), 61])
angles = np.arange(-30, 31)
for i, j in enumerate(range(-30, 31)):
trans[:, 2] = j
dict_nat = {
model.x_input: x_batch,
model.y_input: y_batch,
model.transform: trans,
model.is_training: False
}
results[:, i], predictions[:, i] = sess.run(
[model.correct_prediction, model.predictions], feed_dict=dict_nat)
cor_class_means = np.mean(results, axis=1)
# get images that fooled the model for all angles
indices_all_fooled = np.where(cor_class_means == 0)[0]
if len(indices_all_fooled) > 0:
fooled_tuple = get_images(indices_all_fooled, config, data_iterator,
model, sess, x_batch, y_batch, results,
predictions, angles)
else:
fooled_tuple = None
# get images that were correctly classified for all angles
indices_all_correct = np.where(cor_class_means == 1)[0]
if len(indices_all_correct) > 0:
correct_tuple = get_images(indices_all_correct, config, data_iterator,
model, sess, x_batch, y_batch, results,
predictions, angles)
else:
correct_tuple = None
# get images that were correctly classified for a large proportion of angles
indices_some_correct = np.where(
np.logical_and(cor_class_means >= .9, cor_class_means < 1))[0]
if len(indices_some_correct) > 0:
some_correct_tuple = get_images(indices_some_correct, config,
data_iterator, model, sess, x_batch,
y_batch, results, predictions, angles)
else:
some_correct_tuple = None
return results, predictions, fooled_tuple, correct_tuple, some_correct_tuple
