def main(data_name, method, dimZ, dimH, n_channel, batch_size, K_mc,
checkpoint, lbd):
# set up dataset specific stuff
from config import config
labels, n_iter, dimX, shape_high, ll = config(data_name, n_channel)
if data_name == 'mnist':
from mnist import load_mnist
if data_name == 'notmnist':
from notmnist import load_notmnist
# import functionalities
if method == 'onlinevi':
from bayesian_generator import generator_head, generator_shared, \
generator, construct_gen
from onlinevi import construct_optimizer, init_shared_prior, \
update_shared_prior, update_q_sigma
if method in ['ewc', 'noreg', 'laplace', 'si']:
from generator import generator_head, generator_shared, generator, construct_gen
if method in ['ewc', 'noreg']:
from vae_ewc import construct_optimizer, lowerbound
if method == 'ewc': from vae_ewc import update_ewc_loss, compute_fisher
if method == 'laplace':
from vae_laplace import construct_optimizer, lowerbound
from vae_laplace import update_laplace_loss, compute_fisher, init_fisher_accum
if method == 'si':
from vae_si import construct_optimizer, lowerbound, update_si_reg
# then define model
n_layers_shared = 2
batch_size_ph = tf.placeholder(tf.int32, shape=(), name='batch_size')
dec_shared = generator_shared(dimX, dimH, n_layers_shared, 'sigmoid',
'gen')
# initialise sessions
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
string = method
if method in ['ewc', 'laplace', 'si']:
string = string + '_lbd%.1f' % lbd
if method == 'onlinevi' and K_mc > 1:
string = string + '_K%d' % K_mc
path_name = data_name + '_%s/' % string
if not os.path.isdir('save/'):
os.mkdir('save/')
if not os.path.isdir('save/' + path_name):
os.mkdir('save/' + path_name)
print 'create path save/' + path_name
filename = 'save/' + path_name + 'checkpoint'
if checkpoint < 0:
print 'training from scratch'
old_var_list = init_variables(sess)
else:
load_params(sess, filename, checkpoint)
checkpoint += 1
# visualise the samples
N_gen = 10**2
path = 'figs/' + path_name
if not os.path.isdir('figs/'):
os.mkdir('figs/')
if not os.path.isdir(path):
os.mkdir(path)
print 'create path ' + path
X_ph = tf.placeholder(tf.float32, shape=(batch_size, dimX), name='x_ph')
# now start fitting
N_task = len(labels)
gen_ops = []
X_valid_list = []
X_test_list = []
eval_func_list = []
result_list = []
if method == 'onlinevi':
shared_prior_params = init_shared_prior()
if method in ['ewc', 'noreg']:
ewc_loss = 0.0
if method == 'laplace':
F_accum = init_fisher_accum()
laplace_loss = 0.0
if method == 'si':
old_params_shared = None
si_reg = None
n_layers_head = 2
n_layers_enc = n_layers_shared + n_layers_head - 1
for task in xrange(1, N_task + 1):
# first load data
if data_name == 'mnist':
X_train, X_test, _, _ = load_mnist(digits=labels[task - 1],
conv=False)
if data_name == 'notmnist':
X_train, X_test, _, _ = load_notmnist(data_path,
digits=labels[task - 1],
conv=False)
N_train = int(X_train.shape[0] * 0.9)
X_valid_list.append(X_train[N_train:])
X_train = X_train[:N_train]
X_test_list.append(X_test)
# define the head net and the generator ops
dec = generator(
generator_head(dimZ, dimH, n_layers_head, 'gen_%d' % task),
dec_shared)
enc = encoder(dimX, dimH, dimZ, n_layers_enc, 'enc_%d' % task)
gen_ops.append(construct_gen(dec, dimZ, sampling=False)(N_gen))
print 'construct eval function...'
eval_func_list.append(construct_eval_func(X_ph, enc, dec, ll, \
batch_size_ph, K=100, sample_W=False))
# then construct loss func and fit func
print 'construct fit function...'
if method == 'onlinevi':
fit = construct_optimizer(X_ph, enc, dec, ll, X_train.shape[0], batch_size_ph, \
shared_prior_params, task, K_mc)
if method in ['ewc', 'noreg']:
bound = lowerbound(X_ph, enc, dec, ll)
fit = construct_optimizer(X_ph, batch_size_ph, bound,
X_train.shape[0], ewc_loss)
if method == 'ewc':
fisher, var_list = compute_fisher(X_ph, batch_size_ph, bound,
X_train.shape[0])
if method == 'laplace':
bound = lowerbound(X_ph, enc, dec, ll)
fit = construct_optimizer(X_ph, batch_size_ph, bound,
X_train.shape[0], laplace_loss)
fisher, var_list = compute_fisher(X_ph, batch_size_ph, bound,
X_train.shape[0])
if method == 'si':
bound = lowerbound(X_ph, enc, dec, ll)
fit, shared_var_list = construct_optimizer(X_ph, batch_size_ph,
bound, X_train.shape[0],
si_reg,
old_params_shared, lbd)
if old_params_shared is None:
old_params_shared = sess.run(shared_var_list)
# initialise all the uninitialised stuff
old_var_list = init_variables(sess, old_var_list)
# start training for each task
if method == 'si':
new_params_shared, w_params_shared = fit(sess, X_train, n_iter, lr)
else:
fit(sess, X_train, n_iter, lr)
# plot samples
x_gen_list = sess.run(gen_ops, feed_dict={batch_size_ph: N_gen})
for i in xrange(len(x_gen_list)):
plot_images(x_gen_list[i], shape_high, path, \
data_name + '_gen_task%d_%d' % (task, i + 1))
x_list = [x_gen_list[i][:1] for i in xrange(len(x_gen_list))]
x_list = np.concatenate(x_list, 0)
tmp = np.zeros([10, dimX])
tmp[:task] = x_list
if task == 1:
x_gen_all = tmp
else:
x_gen_all = np.concatenate([x_gen_all, tmp], 0)
# print test-ll on all tasks
tmp_list = []
for i in xrange(len(eval_func_list)):
print 'task %d' % (i + 1),
test_ll = eval_func_list[i](sess, X_valid_list[i])
tmp_list.append(test_ll)
result_list.append(tmp_list)
# save param values
save_params(sess, filename, checkpoint)
checkpoint += 1
# update regularisers/priors
if method == 'ewc':
# update EWC loss
print 'update ewc loss...'
X_batch = X_train[np.random.permutation(range(
X_train.shape[0]))[:batch_size]]
ewc_loss = update_ewc_loss(sess, ewc_loss, var_list, fisher, lbd,
X_batch)
if method == 'laplace':
# update EWC loss
print 'update laplace loss...'
X_batch = X_train[np.random.permutation(range(
X_train.shape[0]))[:batch_size]]
laplace_loss, F_accum = update_laplace_loss(
sess, F_accum, var_list, fisher, lbd, X_batch)
if method == 'onlinevi':
# update prior
print 'update prior...'
shared_prior_params = update_shared_prior(sess,
shared_prior_params)
# reset the variance of q
update_q_sigma(sess)
if method == 'si':
# update regularisers/priors
print 'update SI big omega matrices...'
si_reg, _ = update_si_reg(sess, si_reg, new_params_shared,
old_params_shared, w_params_shared)
old_params_shared = new_params_shared
plot_images(x_gen_all, shape_high, path, data_name + '_gen_all')
for i in xrange(len(result_list)):
print result_list[i]
# save results
fname = 'results/' + data_name + '_%s.pkl' % string
import pickle
pickle.dump(result_list, open(fname, 'wb'))
print 'test-ll results saved in', fname
def main(data_name, vae_type, fea_layer, n_iter, batch_size, K, checkpoint,
data_path):
# load data
from import_data_cifar10 import load_data_cifar10
X_train, X_test, Y_train, Y_test = load_data_cifar10(data_path, conv=True)
dimY = Y_train.shape[1]
if vae_type == 'E':
from mlp_generator_cifar10_E import generator
if vae_type == 'F':
from mlp_generator_cifar10_F import generator
if vae_type == 'G':
from mlp_generator_cifar10_G import generator
from mlp_encoder_cifar10 import encoder_gaussian as encoder
#first build the feature extractor
input_shape = X_train[0].shape
sys.path.append('test_attacks/load/')
from vgg_cifar10 import cifar10vgg
cnn = cifar10vgg(path='test_attacks/load/vgg_model/', train=False)
if fea_layer == 'low':
N_layer = 16
if fea_layer == 'mid':
N_layer = 36
if fea_layer == 'high':
N_layer = len(cnn.model.layers) - 5
for layer in cnn.model.layers:
print(layer.__class__.__name__)
def feature_extractor(x):
out = cnn.normalize_production(x * 255.0)
for i in range(N_layer):
out = cnn.model.layers[i](out)
return out
print(fea_layer, N_layer, cnn.model.layers[N_layer-1].__class__.__name__, \
cnn.model.layers[N_layer-1].get_config())
# then define model
X_ph = tf.placeholder(tf.float32, shape=(batch_size, ) + input_shape)
Y_ph = tf.placeholder(tf.float32, shape=(batch_size, dimY))
dimZ = 128 #32
dimH = 1000
fea_op = feature_extractor(X_ph)
if len(fea_op.get_shape().as_list()) == 4:
fea_op = tf.reshape(fea_op, [batch_size, -1])
dimF = fea_op.get_shape().as_list()[-1]
dec = generator(dimF, dimH, dimZ, dimY, 'linear', 'gen')
n_layers_enc = 2
enc = encoder(dimF, dimH, dimZ, dimY, n_layers_enc, 'enc')
ll = 'l2'
identity = lambda x: x
fea_ph = tf.placeholder(tf.float32, shape=(batch_size, dimF))
fit, eval_acc = construct_optimizer(fea_ph, Y_ph, [identity, enc], dec, ll,
K, vae_type)
# initialise sessions
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
path_name = data_name + '_conv_vae_fea_%s_%s/' % (vae_type, fea_layer)
if not os.path.isdir('save/' + path_name):
os.mkdir('save/' + path_name)
print('create path save/' + path_name)
filename = 'save/' + path_name + 'checkpoint'
if checkpoint < 0:
print('training from scratch')
init_variables(sess)
else:
load_params(sess, filename, checkpoint)
checkpoint += 1
# set test phase
import keras.backend
keras.backend.set_session(sess)
cnnfile = 'test_attacks/load/vgg_model/cifar10vgg.h5'
cnn.model.load_weights(cnnfile)
print('load weight from', cnnfile)
keras.backend.set_learning_phase(0)
# extract features
def gen_feature(X):
F = []
for i in range(int(X.shape[0] / batch_size)):
batch = X[i * batch_size:(i + 1) * batch_size]
F.append(sess.run(fea_op, feed_dict={X_ph: batch}))
return np.concatenate(F, axis=0)
F_train = gen_feature(X_train)
F_test = gen_feature(X_test)
# now start fitting
beta = 1.0
n_iter_ = 10
for i in range(int(n_iter / n_iter_)):
fit(sess, F_train, Y_train, n_iter_, lr, beta)
# print training and test accuracy
eval_acc(sess, F_test, Y_test, 'test', beta)
# save param values
save_params(sess, filename, checkpoint, scope='vae')
checkpoint += 1
def main(data_name,
vae_type,
dimZ,
dimH,
n_iter,
batch_size,
K,
checkpoint,
categorical=False,
bin_num=128):
# load data
beta = 0.001 # here beta
from utils_spam import data_spam
X_train, Y_train, X_test, Y_test = data_spam(train_start=0,
train_end=295870,
test_start=0,
test_end=126082)
# if categorical:
# X_train_cat = categorize(X_train,bin_num)
# X_test_cat = categorize(X_test,bin_num)
# print(X_train_cat[0:10])
# print(X_test_cat[0:10])
# print('**********')
print(X_train[0])
print(Y_train[0])
dimY = Y_train.shape[1]
# if vae_type == 'A':
# from conv_generator_mnist_A import generator
# if vae_type == 'B':
# from conv_generator_mnist_B import generator
# if vae_type == 'C':
# from conv_generator_mnist_C import generator
# if vae_type == 'D':
# from conv_generator_mnist_D import generator
# if vae_type == 'E':
# from conv_generator_mnist_E import generator
if vae_type == 'F':
if categorical:
from models.mlp_generator_categorical_spam_F import generator
else:
from models.mlp_generator_spam_F import generator
# if vae_type == 'G':
# from models.conv_generator_mnist_G import generator
# from conv_encoder_mnist import encoder_gaussian as encoder
from models.mlp_encoder_spam import encoder_gaussian as encoder
input_shape = X_train[0].shape
dimX = input_shape[0]
print('input_shape:', input_shape)
print('dimX:', dimX)
# then define model
if categorical:
dec = generator(dimX,
dimH,
dimZ,
dimY,
'linear',
'gen',
bin_num=bin_num)
else:
dec = generator(dimX, dimH, dimZ, dimY, 'linear', 'gen')
n_layers_enc = 2
enc = encoder(dimX, dimH, dimZ, dimY, n_layers_enc, 'enc')
# define optimisers
X_ph = tf.placeholder(tf.float32,
shape=(batch_size, ) + input_shape) #TODO: ??
Y_ph = tf.placeholder(tf.float32, shape=(batch_size, dimY))
# if categorical:
# X_cat = tf.placeholder(tf.int32, shape=(batch_size, 25))
# else:
# X_cat = None
print('X_ph.shape:', X_ph.shape)
print('Y_ph.shape:', Y_ph.shape)
print('X_train.shape:', X_train.shape)
print('Y_train.shape:', Y_train.shape)
#ll = 'l2'
ll = 'xe'
identity = lambda x: x
fit, eval_acc = construct_optimizer(X_ph, Y_ph, [identity, enc], dec, ll,
K, vae_type, categorical)
# initialise sessions
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
if not os.path.isdir('save/'):
os.mkdir('save/')
print('create path save/')
path_name = data_name + '_conv_vae_%s' % (
vae_type + '_' + str(dimZ)) + '_beta_{}_ll_{}/'.format(beta, ll)
if not os.path.isdir('save/' + path_name):
os.mkdir('save/' + path_name)
print('create path save/' + path_name)
filename = 'save/' + path_name + 'checkpoint'
if checkpoint < 0:
print('training from scratch')
init_variables(sess)
else:
load_params(sess, filename, checkpoint)
checkpoint += 1
import keras.backend
keras.backend.set_session(sess)
keras.backend.set_learning_phase(0)
# now start fitting
n_iter_ = min(n_iter, 10)
for i in range(int(n_iter / n_iter_)):
fit(sess, X_train, Y_train, n_iter_, lr, beta)
# print training and test accuracy
eval_acc(sess, X_train, Y_train, 'train', beta)
eval_acc(sess, X_test, Y_test, 'test', beta)
save_params(sess, filename, checkpoint, scope='vae')
# save param values
save_params(sess, filename, checkpoint, scope='vae')
checkpoint += 1
def main(data_name, vae_type, dimZ, dimH, n_iter, batch_size, K, checkpoint):
dimY = 10
if vae_type == 'A':
from conv_generator_mnist_A import generator
if vae_type == 'B':
from conv_generator_mnist_B import generator
if vae_type == 'C':
from conv_generator_mnist_C import generator
if vae_type == 'D':
from conv_generator_mnist_D import generator
if vae_type == 'E':
from conv_generator_mnist_E import generator
if vae_type == 'F':
from conv_generator_mnist_F import generator
if vae_type == 'G':
from conv_generator_mnist_G import generator
from conv_encoder_mnist import encoder_gaussian as encoder
shape_high = (28, 28)
input_shape = (28, 28, 1)
n_channel = 64
# then define model
dec = generator(input_shape, dimH, dimZ, dimY, n_channel, 'sigmoid', 'gen')
enc, enc_conv, enc_mlp = encoder(input_shape, dimH, dimZ, dimY, n_channel,
'enc')
# define optimisers
X_ph = tf.placeholder(tf.float32, shape=(batch_size, ) + input_shape)
Y_ph = tf.placeholder(tf.float32, shape=(batch_size, dimY))
ll = 'l2'
fit, eval_acc = construct_optimizer(X_ph, Y_ph, [enc_conv, enc_mlp], dec,
ll, K, vae_type)
# load data
from utils_mnist import data_mnist
X_train, Y_train, X_test, Y_test = data_mnist(train_start=0,
train_end=60000,
test_start=0,
test_end=10000)
# initialise sessions
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
if not os.path.isdir('save/'):
os.mkdir('save/')
print('create path save/')
path_name = data_name + '_conv_vae_%s/' % (vae_type + '_' + str(dimZ))
if not os.path.isdir('save/' + path_name):
os.mkdir('save/' + path_name)
print('create path save/' + path_name)
filename = 'save/' + path_name + 'checkpoint'
if checkpoint < 0:
print('training from scratch')
init_variables(sess)
else:
load_params(sess, filename, checkpoint)
checkpoint += 1
# now start fitting
n_iter_ = min(n_iter, 20)
beta = 1.0
for i in range(int(n_iter / n_iter_)):
fit(sess, X_train, Y_train, n_iter_, lr, beta)
# print training and test accuracy
eval_acc(sess, X_test, Y_test, 'test')
# save param values
save_params(sess, filename, checkpoint)
checkpoint += 1
def main(data_name,
vae_type,
dimZ,
dimH,
n_iter,
batch_size,
K,
checkpoint,
categorical=True,
bin_num=2):
# load data
#beta = 0.1 # here beta
beta = 1 # here beta
from utils_pdf import data_pdf
X_train, Y_train, X_test, Y_test, X_attack, Y_attack = data_pdf(
train_start=0,
train_end=13190,
test_start=0,
test_end=6146,
attack_start=0,
attack_end=1600)
print(X_train[0])
print(Y_train[0])
if vae_type == 'X':
if categorical:
from models.mlp_generator_categorical_pdf import generator
else:
from models.mlp_generator_pdf import generator
from models.mlp_encoder_pdf import encoder_gaussian as encoder
input_shape = X_train[0].shape
dimX = input_shape[0]
print('input_shape:', input_shape)
print('dimX:', dimX)
# then define model
if categorical:
dec = generator(dimX, dimH, dimZ, 'linear', 'gen', bin_num=bin_num)
else:
dec = generator(dimX, dimH, dimZ, 'linear', 'gen')
n_layers_enc = 2
enc = encoder(dimX, dimH, dimZ, n_layers_enc, 'enc')
# define optimisers
X_ph = tf.placeholder(tf.float32, shape=(batch_size, ) + input_shape)
# if categorical:
# X_cat = tf.placeholder(tf.int32, shape=(batch_size, 25))
# else:
# X_cat = None
print('X_ph.shape:', X_ph.shape)
print('X_train.shape:', X_train.shape)
#ll = 'l2'
ll = 'xe'
identity = lambda x: x
fit, eval_acc = construct_optimizer(X_ph, [identity, enc], dec, ll, K,
vae_type, categorical)
# initialise sessions
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
sess = tf.Session(config=config)
if not os.path.isdir('save/'):
os.mkdir('save/')
print('create path save/')
path_name = data_name + '_conv_vae_%s' % (
vae_type + '_' + str(dimZ)) + '_beta_{}_ll_{}/'.format(beta, ll)
if not os.path.isdir('save/' + path_name):
os.mkdir('save/' + path_name)
print('create path save/' + path_name)
filename = 'save/' + path_name + 'checkpoint'
if checkpoint < 0:
print('training from scratch')
init_variables(sess)
else:
load_params(sess, filename, checkpoint)
checkpoint += 1
import keras.backend
keras.backend.set_session(sess)
keras.backend.set_learning_phase(0)
# now start fitting
n_iter_ = min(n_iter, 10)
for i in range(int(n_iter / n_iter_)):
fit(sess, X_train, n_iter_, lr, beta)
# print training and test accuracy
eval_acc(sess, X_train, 'train', beta)
eval_acc(sess, X_test, 'test', beta)
eval_acc(sess, X_attack, 'attack', beta)
save_params(sess, filename, checkpoint, scope='vae')
sys.stdout.flush()
# save param values
save_params(sess, filename, checkpoint, scope='vae')
checkpoint += 1