def main(data_name, method, dimZ, dimH, n_channel, batch_size, K_mc,
checkpoint, lbd):
# set up dataset specific stuff
from config import config
labels, n_iter, dimX, shape_high, ll = config(data_name, n_channel)
if data_name == 'mnist':
from mnist import load_mnist
if data_name == 'notmnist':
from notmnist import load_notmnist
# import functionalities
if method == 'onlinevi':
from bayesian_generator import generator_head, generator_shared, \
generator, construct_gen
from onlinevi import construct_optimizer, init_shared_prior, \
update_shared_prior, update_q_sigma
if method in ['ewc', 'noreg', 'laplace', 'si']:
from generator import generator_head, generator_shared, generator, construct_gen
if method in ['ewc', 'noreg']:
from vae_ewc import construct_optimizer, lowerbound
if method == 'ewc': from vae_ewc import update_ewc_loss, compute_fisher
if method == 'laplace':
from vae_laplace import construct_optimizer, lowerbound
from vae_laplace import update_laplace_loss, compute_fisher, init_fisher_accum
if method == 'si':
from vae_si import construct_optimizer, lowerbound, update_si_reg
# then define model
n_layers_shared = 2
batch_size_ph = tf.placeholder(tf.int32, shape=(), name='batch_size')
dec_shared = generator_shared(dimX, dimH, n_layers_shared, 'sigmoid',
'gen')
# initialise sessions
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
string = method
if method in ['ewc', 'laplace', 'si']:
string = string + '_lbd%.1f' % lbd
if method == 'onlinevi' and K_mc > 1:
string = string + '_K%d' % K_mc
path_name = data_name + '_%s/' % string
if not os.path.isdir('save/'):
os.mkdir('save/')
if not os.path.isdir('save/' + path_name):
os.mkdir('save/' + path_name)
print 'create path save/' + path_name
filename = 'save/' + path_name + 'checkpoint'
if checkpoint < 0:
print 'training from scratch'
old_var_list = init_variables(sess)
else:
load_params(sess, filename, checkpoint)
checkpoint += 1
# visualise the samples
N_gen = 10**2
path = 'figs/' + path_name
if not os.path.isdir('figs/'):
os.mkdir('figs/')
if not os.path.isdir(path):
os.mkdir(path)
print 'create path ' + path
X_ph = tf.placeholder(tf.float32, shape=(batch_size, dimX), name='x_ph')
# now start fitting
N_task = len(labels)
gen_ops = []
X_valid_list = []
X_test_list = []
eval_func_list = []
result_list = []
if method == 'onlinevi':
shared_prior_params = init_shared_prior()
if method in ['ewc', 'noreg']:
ewc_loss = 0.0
if method == 'laplace':
F_accum = init_fisher_accum()
laplace_loss = 0.0
if method == 'si':
old_params_shared = None
si_reg = None
n_layers_head = 2
n_layers_enc = n_layers_shared + n_layers_head - 1
for task in xrange(1, N_task + 1):
# first load data
if data_name == 'mnist':
X_train, X_test, _, _ = load_mnist(digits=labels[task - 1],
conv=False)
if data_name == 'notmnist':
X_train, X_test, _, _ = load_notmnist(data_path,
digits=labels[task - 1],
conv=False)
N_train = int(X_train.shape[0] * 0.9)
X_valid_list.append(X_train[N_train:])
X_train = X_train[:N_train]
X_test_list.append(X_test)
# define the head net and the generator ops
dec = generator(
generator_head(dimZ, dimH, n_layers_head, 'gen_%d' % task),
dec_shared)
enc = encoder(dimX, dimH, dimZ, n_layers_enc, 'enc_%d' % task)
gen_ops.append(construct_gen(dec, dimZ, sampling=False)(N_gen))
print 'construct eval function...'
eval_func_list.append(construct_eval_func(X_ph, enc, dec, ll, \
batch_size_ph, K=100, sample_W=False))
# then construct loss func and fit func
print 'construct fit function...'
if method == 'onlinevi':
fit = construct_optimizer(X_ph, enc, dec, ll, X_train.shape[0], batch_size_ph, \
shared_prior_params, task, K_mc)
if method in ['ewc', 'noreg']:
bound = lowerbound(X_ph, enc, dec, ll)
fit = construct_optimizer(X_ph, batch_size_ph, bound,
X_train.shape[0], ewc_loss)
if method == 'ewc':
fisher, var_list = compute_fisher(X_ph, batch_size_ph, bound,
X_train.shape[0])
if method == 'laplace':
bound = lowerbound(X_ph, enc, dec, ll)
fit = construct_optimizer(X_ph, batch_size_ph, bound,
X_train.shape[0], laplace_loss)
fisher, var_list = compute_fisher(X_ph, batch_size_ph, bound,
X_train.shape[0])
if method == 'si':
bound = lowerbound(X_ph, enc, dec, ll)
fit, shared_var_list = construct_optimizer(X_ph, batch_size_ph,
bound, X_train.shape[0],
si_reg,
old_params_shared, lbd)
if old_params_shared is None:
old_params_shared = sess.run(shared_var_list)
# initialise all the uninitialised stuff
old_var_list = init_variables(sess, old_var_list)
# start training for each task
if method == 'si':
new_params_shared, w_params_shared = fit(sess, X_train, n_iter, lr)
else:
fit(sess, X_train, n_iter, lr)
# plot samples
x_gen_list = sess.run(gen_ops, feed_dict={batch_size_ph: N_gen})
for i in xrange(len(x_gen_list)):
plot_images(x_gen_list[i], shape_high, path, \
data_name + '_gen_task%d_%d' % (task, i + 1))
x_list = [x_gen_list[i][:1] for i in xrange(len(x_gen_list))]
x_list = np.concatenate(x_list, 0)
tmp = np.zeros([10, dimX])
tmp[:task] = x_list
if task == 1:
x_gen_all = tmp
else:
x_gen_all = np.concatenate([x_gen_all, tmp], 0)
# print test-ll on all tasks
tmp_list = []
for i in xrange(len(eval_func_list)):
print 'task %d' % (i + 1),
test_ll = eval_func_list[i](sess, X_valid_list[i])
tmp_list.append(test_ll)
result_list.append(tmp_list)
# save param values
save_params(sess, filename, checkpoint)
checkpoint += 1
# update regularisers/priors
if method == 'ewc':
# update EWC loss
print 'update ewc loss...'
X_batch = X_train[np.random.permutation(range(
X_train.shape[0]))[:batch_size]]
ewc_loss = update_ewc_loss(sess, ewc_loss, var_list, fisher, lbd,
X_batch)
if method == 'laplace':
# update EWC loss
print 'update laplace loss...'
X_batch = X_train[np.random.permutation(range(
X_train.shape[0]))[:batch_size]]
laplace_loss, F_accum = update_laplace_loss(
sess, F_accum, var_list, fisher, lbd, X_batch)
if method == 'onlinevi':
# update prior
print 'update prior...'
shared_prior_params = update_shared_prior(sess,
shared_prior_params)
# reset the variance of q
update_q_sigma(sess)
if method == 'si':
# update regularisers/priors
print 'update SI big omega matrices...'
si_reg, _ = update_si_reg(sess, si_reg, new_params_shared,
old_params_shared, w_params_shared)
old_params_shared = new_params_shared
plot_images(x_gen_all, shape_high, path, data_name + '_gen_all')
for i in xrange(len(result_list)):
print result_list[i]
# save results
fname = 'results/' + data_name + '_%s.pkl' % string
import pickle
pickle.dump(result_list, open(fname, 'wb'))
print 'test-ll results saved in', fname
samples = pred_mean[:num_samples]
cache.append(samples)
# samples = model.gen_samples(i, num_samples).cpu().detach().numpy()
# samples = F.sigmoid(gen_samples(model, i, num_samples)).cpu().detach().numpy()
# for s in range(num_samples):
# ax[s][i].imshow(np.reshape(samples[s], [28,28]), cmap = 'gray')
# plt.savefig('./Gens/'+dataset_+'_reconstructed.png')
fig = plt.figure(figsize = [8,8])
show_images(torch.tensor(cache).permute(1,0,2).reshape(-1,28,28).unsqueeze(1))
plt.savefig('./Gens/'+dataset_+'recon.png')
plt.savefig('./Gens/'+dataset_+'recon.eps', format = 'eps')
print('Generating Samples')
for i in range(no_tasks):
task = i+1
tmp = np.zeros([10, 784])
model = get_model(get_till = task, dict_path = './' + paths[dataset_])
for j in range(task):
print(i,j)
# tmp[j] = model.gen_samples(j, 1).cpu().detach().numpy()
tmp[j] = F.sigmoid(gen_samples(model, j, 1)).cpu().detach().numpy()
if task == 1:
x_gen_all = tmp
else:
x_gen_all = np.concatenate([x_gen_all, tmp], 0)
plot_images(x_gen_all, [28,28], './Gens/', dataset_+'_gen_all')
def main(data_name, method, dimZ, dimH, n_channel, batch_size, K_mc, checkpoint, lbd):
# set up dataset specific stuff
from config import config
labels, n_iter, dimX, shape_high, ll = config(data_name, n_channel)
if data_name == 'mnist':
from mnist import load_mnist
if data_name == 'notmnist':
from notmnist import load_notmnist
# import functionalities
if method == 'onlinevi':
from bayesian_generator import generator_head, generator_shared, \
generator, construct_gen
if method in ['ewc', 'noreg', 'si', 'laplace']:
from generator import generator_head, generator_shared, generator, construct_gen
# then define model
n_layers_shared = 2
batch_size_ph = tf.placeholder(tf.int32, shape=(), name='batch_size')
dec_shared = generator_shared(dimX, dimH, n_layers_shared, 'sigmoid', 'gen')
# initialise sessions
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
string = method
if method in ['ewc', 'laplace', 'si']:
string = string + '_lbd%.1f' % lbd
if method == 'onlinevi' and K_mc > 1:
string = string + '_K%d' % K_mc
path_name = data_name + '_%s/' % string
assert os.path.isdir('save/'+path_name)
filename = 'save/' + path_name + 'checkpoint'
# visualise the samples
N_gen = 10**2
X_ph = tf.placeholder(tf.float32, shape=(batch_size, dimX), name = 'x_ph')
# now start fitting
N_task = len(labels)
gen_ops = []
X_valid_list = []
X_test_list = []
eval_func_list = []
result_list = []
n_layers_head = 2
n_layers_enc = n_layers_shared + n_layers_head - 1
for task in xrange(1, N_task+1):
# first load data
# first load data
if data_name == 'mnist':
X_train, X_test, _, _ = load_mnist(digits = labels[task-1], conv = False)
if data_name == 'notmnist':
X_train, X_test, _, _ = load_notmnist(data_path, digits = labels[task-1], conv = False)
N_train = int(X_train.shape[0] * 0.9)
X_valid_list.append(X_train[N_train:])
X_train = X_train[:N_train]
X_test_list.append(X_test)
# define the head net and the generator ops
dec = generator(generator_head(dimZ, dimH, n_layers_head, 'gen_%d' % task), dec_shared)
enc = encoder(dimX, dimH, dimZ, n_layers_enc, 'enc_%d' % task)
gen_ops.append(construct_gen(dec, dimZ, sampling=False)(N_gen))
eval_func_list.append(construct_eval_func(X_ph, enc, dec, ll, batch_size_ph,
K = 5000, sample_W = False))
# then load the trained model
load_params(sess, filename, checkpoint=task-1, init_all = False)
# plot samples
x_gen_list = sess.run(gen_ops, feed_dict={batch_size_ph: N_gen})
x_list = []
for i in xrange(len(x_gen_list)):
ind = np.random.randint(len(x_gen_list[i]))
x_list.append(x_gen_list[i][ind:ind+1])
x_list = np.concatenate(x_list, 0)
tmp = np.zeros([10, dimX])
tmp[:task] = x_list
if task == 1:
x_gen_all = tmp
else:
x_gen_all = np.concatenate([x_gen_all, tmp], 0)
# print test-ll on all tasks
tmp_list = []
for i in xrange(len(eval_func_list)):
print 'task %d' % (i+1),
test_ll = eval_func_list[i](sess, X_test_list[i])
tmp_list.append(test_ll)
result_list.append(tmp_list)
#x_gen_all = 1.0 - x_gen_all
if not os.path.isdir('figs/visualisation/'):
os.mkdir('figs/visualisation/')
print 'create path figs/visualisation/'
plot_images(x_gen_all, shape_high, 'figs/visualisation/', data_name+'_gen_all_'+method)
for i in xrange(len(result_list)):
print result_list[i]
# save results
fname = 'results/' + data_name + '_%s.pkl' % string
import pickle
pickle.dump(result_list, open(fname, 'wb'))
print 'test-ll results saved in', fname
def test_attacks(batch_size=128,
source_samples=10,
model_path=os.path.join("models", "mnist"),
targeted=True):
"""
Test many attacks on MNIST with deep Bayes classifier.
:param batch_size: size of training batches
:param source_samples: number of test inputs to attack
:param model_path: path to the model file
:param targeted: should we run a targeted attack? or untargeted?
:return: an AccuracyReport object
"""
# Object used to keep track of (and return) key accuracies
report = AccuracyReport()
# Set TF random seed to improve reproducibility
tf.set_random_seed(1234)
# Create TF session
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session()
print("Created TensorFlow session.")
set_log_level(logging.DEBUG)
# Get MNIST test data
from cleverhans.utils_mnist import data_mnist
X_train, Y_train, X_test, Y_test = data_mnist(train_start=0,
train_end=60000,
test_start=0,
test_end=10000)
img_rows, img_cols, channels = X_train[0].shape
nb_classes = Y_train.shape[1]
# Define input TF placeholder
batch_size = min(batch_size, source_samples)
x = tf.placeholder(tf.float32,
shape=(batch_size, img_rows, img_cols, channels))
y = tf.placeholder(tf.float32, shape=(batch_size, nb_classes))
# Define TF model graph
model_name = str(sys.argv[1])
if model_name == 'bayes':
from load_bayes_classifier import BayesModel
conv = True
checkpoint = 0 #int(sys.argv[1])
K = int(sys.argv[3])
use_mean = True
model = BayesModel(sess,
'mnist',
conv,
K,
checkpoint=checkpoint,
attack_snapshot=False,
use_mean=use_mean)
if use_mean:
model_name = 'bayes_mean_mlp'
else:
model_name = 'bayes_K%d' % K
if model_name == 'cnn':
from load_cnn_classifier import CNNModel
model = CNNModel(sess, 'mnist')
if model_name == 'wgan':
from load_wgan_classifier import WGANModel
conv = True
checkpoint = 0 #int(sys.argv[1])
K = int(sys.argv[3])
T = int(sys.argv[4])
model = WGANModel(sess, 'mnist', conv, K, T, checkpoint=checkpoint)
model_name = 'wgan_K%d_T%d' % (K, T)
preds = model.predict(x, softmax=True) # output probabilities
print("Defined TensorFlow model graph.")
# Evaluate the accuracy of the MNIST model on legitimate test examples
eval_params = {'batch_size': batch_size}
accuracy = model_eval(sess, x, y, preds, X_test, Y_test, args=eval_params)
print('Test accuracy on legitimate test examples: {0}'.format(accuracy))
report.clean_train_clean_eval = accuracy
# Craft adversarial examples
nb_adv_per_sample = str(nb_classes - 1) if targeted else '1'
print('Crafting ' + str(source_samples) + ' * ' + nb_adv_per_sample +
' adversarial examples')
print("This could take some time ...")
# make adv inputs and labels for the attack if targeted
if targeted:
adv_inputs = np.array([[instance] * nb_classes
for instance in X_test[:source_samples]],
dtype=np.float32)
one_hot = np.zeros((nb_classes, nb_classes))
one_hot[np.arange(nb_classes), np.arange(nb_classes)] = 1
adv_inputs = adv_inputs.reshape(
(source_samples * nb_classes, img_rows, img_cols, 1))
adv_ys = np.array([one_hot] * source_samples,
dtype=np.float32).reshape(
(source_samples * nb_classes, nb_classes))
else:
adv_inputs = X_test[:source_samples]
adv_ys = Y_test[:source_samples]
# Instantiate an attack object
attack_method = str(sys.argv[2])
if attack_method == 'fgsm':
from cleverhans.attacks import FastGradientMethod
model_prob = lambda x: model.predict(x, softmax=True)
attack = FastGradientMethod(model_prob, sess=sess)
from attack_config import config_fgsm
attack_params = config_fgsm(targeted, adv_ys)
if attack_method == 'bim':
from cleverhans.attacks import BasicIterativeMethod
model_prob = lambda x: model.predict(x, softmax=True)
attack = BasicIterativeMethod(model_prob, sess=sess)
from attack_config import config_bim
attack_params = config_bim(targeted, adv_ys)
if attack_method == 'mim':
from cleverhans.attacks import MomentumIterativeMethod
model_prob = lambda x: model.predict(x, softmax=True)
attack = MomentumIterativeMethod(model_prob, sess=sess)
from attack_config import config_mim
attack_params = config_mim(targeted, adv_ys)
if attack_method == 'jsma':
from cleverhans.attacks import SaliencyMapMethod
model_prob = lambda x: model.predict(x, softmax=True)
attack = SaliencyMapMethod(model_prob, sess=sess)
from attack_config import config_jsma
attack_params = config_jsma(targeted, adv_ys)
if attack_method == 'vat':
from cleverhans.attacks import VirtualAdversarialMethod
model_logit = lambda x: model.predict(x, softmax=False)
attack = VirtualAdversarialMethod(model_logit, sess=sess)
from attack_config import config_vat
attack_params = config_vat(targeted, adv_ys)
if attack_method == 'cw':
from cleverhans.attacks import CarliniWagnerL2
model_logit = lambda x: model.predict(x, softmax=False)
attack = CarliniWagnerL2(model_logit, sess=sess)
from attack_config import config_cw
attack_params = config_cw(targeted, adv_ys)
if attack_method == 'elastic':
from cleverhans.attacks import ElasticNetMethod
model_logit = lambda x: model.predict(x, softmax=False)
attack = ElasticNetMethod(model_logit, sess=sess)
from attack_config import config_elastic
attack_params = config_elastic(targeted, adv_ys)
if attack_method == 'deepfool':
from cleverhans.attacks import DeepFool
model_logit = lambda x: model.predict(x, softmax=False)
attack = DeepFool(model_logit, sess=sess)
from attack_config import config_deepfool
attack_params = config_deepfool(targeted, adv_ys)
if attack_method == 'madry':
from cleverhans.attacks import MadryEtAl
model_prob = lambda x: model.predict(x, softmax=True)
attack = MadryEtAl(model_prob, sess=sess)
from attack_config import config_madry
attack_params = config_madry(targeted, adv_ys)
attack_params['batch_size'] = batch_size
print('batchsize', batch_size)
# perform the attack!
adv = []
n_batch = int(adv_inputs.shape[0] / batch_size)
for i in xrange(n_batch):
adv_batch = adv_inputs[i * batch_size:(i + 1) * batch_size]
adv.append(attack.generate_np(adv_batch, **attack_params))
adv = np.concatenate(adv, axis=0)
for _ in xrange(5):
y_adv = []
for i in xrange(n_batch):
adv_batch = adv[i * batch_size:(i + 1) * batch_size]
y_adv.append(sess.run(preds, {x: adv_batch}))
y_adv = np.concatenate(y_adv, axis=0)
print('--------------------------------------')
for i in xrange(10):
print(np.argmax(y_adv[i * 10:(i + 1) * 10], 1))
correct_pred = np.asarray(np.argmax(y_adv, 1) == np.argmax(adv_ys, 1),
dtype='f')
adv_accuracy = np.mean(correct_pred)
if not targeted:
# adv_accuracy, y_adv = model_eval(sess, x, y, preds, adv,
# adv_ys, args=eval_params,
# return_pred=True)
# else:
# adv_accuracy, y_adv = model_eval(sess, x, y, preds, adv,
# Y_test[:source_samples], args=eval_params,
# return_pred=True)
adv_accuracy = 1. - adv_accuracy
print('--------------------------------------')
print(np.argmax(adv_ys[:10], 1))
print(np.argmax(y_adv[:10], 1))
for i in xrange(5):
tmp = sess.run(preds, {x: adv[:100]})
print(np.argmax(tmp[:10], 1))
# Compute the number of adversarial examples that were successfully found
print('Avg. rate of successful adv. examples {0:.4f}'.format(adv_accuracy))
report.clean_train_adv_eval = 1. - adv_accuracy
# Compute the average distortion introduced by the algorithm
percent_perturbed = np.mean(
np.sum((adv - adv_inputs)**2, axis=(1, 2, 3))**.5)
print('Avg. L_2 norm of perturbations {0:.4f}'.format(percent_perturbed))
# Close TF session
sess.close()
# visualisation
vis_adv = True
if vis_adv:
N_vis = 100
sys.path.append('../../utils')
from visualisation import plot_images
if channels == 1:
shape = (img_rows, img_cols)
else:
shape = (img_rows, img_cols, channels)
path = 'figs/'
filename = model_name + '_' + attack_method
if targeted:
filename = filename + '_targeted'
else:
filename = filename + '_untargeted'
plot_images(adv_inputs[:N_vis], shape, path, filename + '_data')
plot_images(adv[:N_vis], shape, path, filename + '_adv')
save_result = True
if save_result:
path = 'results/'
filename = model_name + '_' + attack_method
if targeted:
filename = filename + '_targeted'
y_input = adv_ys
else:
filename = filename + '_untargeted'
y_input = Y_test[:source_samples]
results = [adv_inputs, y_input, adv, y_adv]
import pickle
pickle.dump(results, open(path + filename + '.pkl', 'w'))
print("results saved at %s.pkl" % filename)
return report