def view_params(identifier, epoch):
""" Visualise weight matrices in the GAN """
settings = json.load(
open('./experiments/settings/' + identifier + '.txt', 'r'))
parameters = model.load_parameters(identifier + '_' + str(epoch))
plotting.plot_parameters(parameters, identifier + '_' + str(epoch))
return True
def dis_D_model(settings, samples, para_path):
"""
Return the discrimination results of num_samples testing samples from a trained model described by settings dict
Note: this function is designed for ONE sample discrimination
"""
# if settings is a string, assume it's an identifier and load
if type(settings) == str:
settings = json.load(open('./experiments/settings/' + settings + '.txt', 'r'))
# num_samples = samples.shape[0]
samples = np.float32(samples)
samples = np.reshape(samples, [1, settings['seq_length'], settings['num_generated_features']])
# get the parameters, get other variables
parameters = model.load_parameters(para_path)
# create placeholder, T samples
T = tf.placeholder(tf.float32, [1, settings['seq_length'], settings['num_generated_features']])
# create the discriminator (GAN or CGAN)
# normal GAN
D_t, L_t = discriminator_o(T, settings['hidden_units_d'], reuse=False, parameters=parameters)
# D_t, L_t = model.discriminator(T, settings['hidden_units_d'], settings['seq_length'], num_samples, reuse=False,
# parameters=parameters, cond_dim=0, c=None, batch_mean=False)
config = tf.ConfigProto()
config.gpu_options.allow_growth = True
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
D_t, L_t = sess.run([D_t, L_t], feed_dict={T: samples})
tf.reset_default_graph()
return D_t, L_t
def dis_trained_model(settings, samples, para_path):
"""
Return the discrimination results of num_samples testing samples from a trained model described by settings dict
Note: this function is designed for ONE sample discrimination
"""
# if settings is a string, assume it's an identifier and load
if type(settings) == str:
settings = json.load(
open('./experiments/settings/' + settings + '.txt', 'r'))
num_samples = samples.shape[0]
samples = np.float32(samples)
num_variables = samples.shape[2]
# samples = np.reshape(samples, [1, settings['seq_length'], settings['num_generated_features']])
# get the parameters, get other variables
# parameters = model.load_parameters(settings['sub_id'] + '_' + str(settings['seq_length']) + '_' + str(epoch))
parameters = model.load_parameters(para_path)
# settings['sub_id'] + '_' + str(settings['seq_length']) + '_' + str(epoch)
# create placeholder, T samples
# T = tf.placeholder(tf.float32, [settings['batch_size'], settings['seq_length'], settings['num_generated_features']])
T = tf.placeholder(tf.float32,
[num_samples, settings['seq_length'], num_variables])
# create the discriminator (GAN)
# normal GAN
D_t, L_t = discriminator_o(T,
settings['hidden_units_d'],
reuse=False,
parameters=parameters)
# gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.5)
# config = tf.ConfigProto(gpu_options=gpu_options)
# config = tf.ConfigProto()
# config.gpu_options.allow_growth = True
# with tf.device('/gpu:1'):
gpu_options = tf.GPUOptions(per_process_gpu_memory_fraction=0.4)
with tf.Session(config=tf.ConfigProto(gpu_options=gpu_options)) as sess:
# with tf.Session(config=config) as sess:
sess.run(tf.global_variables_initializer())
D_t, L_t = sess.run([D_t, L_t], feed_dict={T: samples})
tf.reset_default_graph()
return D_t, L_t
'[Epoch %d Batch %d] loss %.2f, ppl %.2f, batch_latency %.4f'
% (epoch, i, cur_L, math.exp(cur_L), batch_latency))
total_L = 0.0
if args.export_model:
model.export('model')
return
val_L = eval(val_data)
print('[Epoch %d] time cost %.2fs, valid loss %.2f, valid ppl %.2f' %
(epoch, time.time() - start_time, val_L, math.exp(val_L)))
if val_L < best_val:
best_val = val_L
test_L = eval(test_data)
model.save_parameters(args.save)
print('test loss %.2f, test ppl %.2f' % (test_L, math.exp(test_L)))
else:
args.lr = args.lr * 0.25
trainer.set_learning_rate(args.lr)
if __name__ == '__main__':
train()
if not args.export_model:
model.load_parameters(args.save, context)
test_L = eval(test_data)
print('Best test loss %.2f, test ppl %.2f' %
(test_L, math.exp(test_L)))
def invert2(settings,
samples,
para_path,
g_tolerance=None,
e_tolerance=0.1,
n_iter=None,
max_iter=10000,
heuristic_sigma=None):
"""
Return the latent space points corresponding to a set of a samples (from gradient descent)
Note: this function is designed for ONE sample generation
"""
# num_samples = samples.shape[0]
# cast samples to float32
samples = np.float32(samples)
# get the model
# if settings is a string, assume it's an identifier and load
if type(settings) == str:
settings = json.load(
open('./experiments/settings/' + settings + '.txt', 'r'))
# get parameters
parameters = model.load_parameters(para_path)
Z = tf.compat.v1.get_variable(
name='Z',
shape=[1, settings['seq_length'], settings['latent_dim']],
initializer=tf.compat.v1.random_normal_initializer())
# create outputs
G_samples = generator_o(Z,
settings['hidden_units_g'],
settings['seq_length'],
1,
settings['num_generated_features'],
reuse=False,
parameters=parameters)
# generator_vars = ['hidden_units_g', 'seq_length', 'batch_size', 'num_generated_features', 'cond_dim', 'learn_scale']
# generator_settings = dict((k, settings[k]) for k in generator_vars)
# G_samples = model.generator(Z, **generator_settings, reuse=True)
# G_samples = model.generator(Z, settings['hidden_units_g'], settings['seq_length'], 1, settings['num_generated_features'], reuse=False, parameters=parameters)
fd = None
# define loss mmd-based loss
if heuristic_sigma is None:
heuristic_sigma = mmd.median_pairwise_distance_o(
samples) # this is noisy
print('heuristic_sigma:', heuristic_sigma)
samples = tf.compat.v1.reshape(
samples,
[1, settings['seq_length'], settings['num_generated_features']])
Kxx, Kxy, Kyy, wts = mmd._mix_rbf_kernel(G_samples,
samples,
sigmas=tf.constant(
value=heuristic_sigma,
shape=(1, 1)))
similarity_per_sample = tf.compat.v1.diag_part(Kxy)
reconstruction_error_per_sample = 1 - similarity_per_sample
# reconstruction_error_per_sample = tf.reduce_sum((tf.nn.l2_normalize(G_samples, dim=1) - tf.nn.l2_normalize(samples, dim=1))**2, axis=[1,2])
similarity = tf.compat.v1.reduce_mean(similarity_per_sample)
reconstruction_error = 1 - similarity
# updater
# solver = tf.compat.v1.train.AdamOptimizer().minimize(reconstruction_error_per_sample, var_list=[Z])
# solver = tf.train.RMSPropOptimizer(learning_rate=500).minimize(reconstruction_error, var_list=[Z])
solver = tf.compat.v1.train.RMSPropOptimizer(learning_rate=0.1).minimize(
reconstruction_error_per_sample, var_list=[Z])
# solver = tf.train.MomentumOptimizer(learning_rate=0.1, momentum=0.9).minimize(reconstruction_error_per_sample, var_list=[Z])
grad_Z = tf.compat.v1.gradients(reconstruction_error_per_sample, Z)[0]
grad_per_Z = tf.compat.v1.norm(grad_Z, axis=(1, 2))
grad_norm = tf.compat.v1.reduce_mean(grad_per_Z)
# solver = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(reconstruction_error, var_list=[Z])
print('Finding latent state corresponding to samples...')
sess = tf.compat.v1.Session()
sess.run(tf.compat.v1.global_variables_initializer())
with tf.compat.v1.Session() as sess:
# graph = tf.compat.v1.Graph()
# graphDef = graph.as_graph_def()
sess.run(tf.compat.v1.global_variables_initializer())
error = sess.run(reconstruction_error, feed_dict=fd)
g_n = sess.run(grad_norm, feed_dict=fd)
# print(g_n)
i = 0
if not n_iter is None:
while i < n_iter:
_ = sess.run(solver, feed_dict=fd)
error = sess.run(reconstruction_error, feed_dict=fd)
i += 1
else:
if not g_tolerance is None:
while g_n > g_tolerance:
_ = sess.run(solver, feed_dict=fd)
error, g_n = sess.run([reconstruction_error, grad_norm],
feed_dict=fd)
i += 1
print(error, g_n)
if i > max_iter:
break
else:
while np.abs(error) > e_tolerance:
_ = sess.run(solver, feed_dict=fd)
error = sess.run(reconstruction_error, feed_dict=fd)
i += 1
# print(error)
if i > max_iter:
break
Zs = sess.run(Z, feed_dict=fd)
Gs = sess.run(G_samples, feed_dict={Z: Zs})
error_per_sample = sess.run(reconstruction_error_per_sample,
feed_dict=fd)
print('Z found in', i, 'iterations with final reconstruction error of',
error)
tf.compat.v1.reset_default_graph()
return Gs, Zs, error_per_sample, heuristic_sigma
def invert2(settings,
samples,
para_path,
g_tolerance=None,
e_tolerance=0.1,
n_iter=None,
max_iter=10000,
heuristic_sigma=None):
"""
Return the latent space points corresponding to a set of a samples (from gradient descent)
Note: this function is designed for ONE sample generation
"""
# num_samples = samples.shape[0]
# cast samples to float32
samples = np.float32(samples)
# get the model
# if settings is a string, assume it's an identifier and load
if type(settings) == str:
settings = json.load(
open('./experiments/settings/' + settings + '.txt', 'r'))
# print('Inverting', 1, 'samples using model', settings['identifier'], 'at epoch', epoch,)
# if not g_tolerance is None:
# print('until gradient norm is below', g_tolerance)
# else:
# print('until error is below', e_tolerance)
# get parameters
parameters = model.load_parameters(para_path)
# # assertions
# assert samples.shape[2] == settings['num_generated_features']
# create VARIABLE Z
Z = tf.compat.v1.get_variable(
name='Z',
shape=[1, settings['seq_length'], settings['latent_dim']],
initializer=tf.compat.v1.random_normal_initializer())
# create outputs
G_samples = generator_o(Z,
settings['hidden_units_g'],
settings['seq_length'],
1,
settings['num_generated_features'],
reuse=False,
parameters=parameters)
# generator_vars = ['hidden_units_g', 'seq_length', 'batch_size', 'num_generated_features', 'cond_dim', 'learn_scale']
# generator_settings = dict((k, settings[k]) for k in generator_vars)
# G_samples = model.generator(Z, **generator_settings, reuse=True)
fd = None
# define loss mmd-based loss
if heuristic_sigma is None:
# heuristic_sigma = mmd.median_pairwise_distance_o(samples) # this is noisy
# heuristic_sigma = mmd.median_pairwise_distance_o(samples) + np.(0.0000001).astype(dtype=float32)
heuristic_sigma = mmd.median_pairwise_distance_o(samples) + np.array(
0.0000001, dtype=np.float32)
print('heuristic_sigma:', heuristic_sigma)
samples = tf.compat.v1.reshape(
samples,
[1, settings['seq_length'], settings['num_generated_features']])
# Kxx, Kxy, Kyy, wts = mmd._mix_rbf_kernel(G_samples, samples, sigmas=tf.constant(value=heuristic_sigma, shape=(1, 1)))
# base = 1.0
# sigma_list = [1, 2, 4, 8, 16]
# sigma_list = [sigma / base for sigma in sigma_list]
# Kxx, Kxy, Kyy, len_sigma_list = mmd._mix_rbf_kernel2(G_samples, samples, sigma_list)
# ---------------------
X = G_samples
Y = samples
sigmas = tf.constant(value=heuristic_sigma, shape=(1, 1))
wts = [1.0] * sigmas.get_shape()[0]
if len(X.shape) == 2:
# matrix
XX = tf.compat.v1.matmul(X, X, transpose_b=True)
XY = tf.compat.v1.matmul(X, Y, transpose_b=True)
YY = tf.compat.v1.matmul(Y, Y, transpose_b=True)
elif len(X.shape) == 3:
# tensor -- this is computing the Frobenius norm
XX = tf.compat.v1.tensordot(X, X, axes=[[1, 2], [1, 2]])
XY = tf.compat.v1.tensordot(X, Y, axes=[[1, 2], [1, 2]])
YY = tf.compat.v1.tensordot(Y, Y, axes=[[1, 2], [1, 2]])
else:
raise ValueError(X)
X_sqnorms = tf.compat.v1.diag_part(XX)
Y_sqnorms = tf.compat.v1.diag_part(YY)
r = lambda x: tf.compat.v1.expand_dims(x, 0)
c = lambda x: tf.compat.v1.expand_dims(x, 1)
K_XX, K_XY, K_YY = 0, 0, 0
for sigma, wt in zip(tf.compat.v1.unstack(sigmas, axis=0), wts):
gamma = 1 / (2 * sigma**2)
K_XX += wt * tf.compat.v1.exp(-gamma *
(-2 * XX + c(X_sqnorms) + r(X_sqnorms)))
K_XY += wt * tf.compat.v1.exp(-gamma *
(-2 * XY + c(X_sqnorms) + r(Y_sqnorms)))
K_YY += wt * tf.compat.v1.exp(-gamma *
(-2 * YY + c(Y_sqnorms) + r(Y_sqnorms)))
Kxx = K_XX
Kxy = K_XY
Kyy = K_YY
wts = tf.compat.v1.reduce_sum(wts)
# ---------------------
similarity_per_sample = tf.compat.v1.diag_part(Kxy)
reconstruction_error_per_sample = 1 - similarity_per_sample
# reconstruction_error_per_sample = tf.reduce_sum((tf.nn.l2_normalize(G_samples, dim=1) - tf.nn.l2_normalize(samples, dim=1))**2, axis=[1,2])
reconstruction_error = 1 - tf.compat.v1.reduce_mean(similarity_per_sample)
# updater
# solver = tf.compat.v1.train.AdamOptimizer().minimize(reconstruction_error_per_sample, var_list=[Z])
# solver = tf.train.RMSPropOptimizer(learning_rate=500).minimize(reconstruction_error, var_list=[Z])
solver = tf.compat.v1.train.RMSPropOptimizer(learning_rate=0).minimize(
reconstruction_error_per_sample, var_list=[Z])
# solver = tf.train.MomentumOptimizer(learning_rate=0.1, momentum=0.9).minimize(reconstruction_error_per_sample, var_list=[Z])
# grad_Z = tf.compat.v1.gradients(reconstruction_error_per_sample, Z)[0]
grad_Z = tf.compat.v1.gradients(reconstruction_error, Z)[0]
grad_per_Z = tf.compat.v1.norm(grad_Z, axis=(1, 2))
grad_norm = tf.compat.v1.reduce_mean(grad_per_Z)
# solver = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(reconstruction_error, var_list=[Z])
print('Finding latent state corresponding to samples...')
sess = tf.compat.v1.Session()
sess.run(tf.compat.v1.global_variables_initializer())
with tf.compat.v1.Session() as sess:
# graph = tf.compat.v1.Graph()
# graphDef = graph.as_graph_def()
sess.run(tf.compat.v1.global_variables_initializer())
error = sess.run(reconstruction_error, feed_dict=fd)
g_n = sess.run(grad_norm, feed_dict=fd)
# print(g_n)
i = 0
if not n_iter is None:
while i < n_iter:
_ = sess.run(solver, feed_dict=fd)
error = sess.run(reconstruction_error, feed_dict=fd)
i += 1
else:
if not g_tolerance is None:
while g_n > g_tolerance:
_ = sess.run(solver, feed_dict=fd)
error, g_n = sess.run([reconstruction_error, grad_norm],
feed_dict=fd)
i += 1
print(error, g_n)
if i > max_iter:
break
else:
while np.abs(error) > e_tolerance:
# while ((np.abs(error) > e_tolerance) or (math.isnan(error))):
solver.run(feed_dict=fd)
# error = sess.run(reconstruction_error, feed_dict=fd)
reconstruction_error_out, reconstruction_error_per_sample_out, similarity_per_sample_out, Kxy_out, G_samples_out, samples_out, Z_out, X_sqnorms_out, Y_sqnorms_out, XY_out, sigmas_out = sess.run(
[
reconstruction_error,
reconstruction_error_per_sample,
similarity_per_sample, Kxy, G_samples, samples, Z,
X_sqnorms, Y_sqnorms, XY, sigmas
],
feed_dict=fd)
if (math.isnan(reconstruction_error_out)):
print("nan")
# Zs, Gs, Kxy_s, error_per_sample, error = sess.run([Z, G_samples, Kxy, reconstruction_error_per_sample, reconstruction_error], feed_dict=fd)
# Zs, Gs, Kxy_s, error_per_sample, error, _ = sess.run([Z, G_samples, Kxy, reconstruction_error_per_sample, reconstruction_error, solver], feed_dict=fd)
i += 1
# print(error)
if i > max_iter:
break
Zs = sess.run(Z, feed_dict=fd)
Gs = sess.run(G_samples, feed_dict={Z: Zs})
error_per_sample = sess.run(reconstruction_error_per_sample,
feed_dict=fd)
print('Z found in', i, 'iterations with final reconstruction error of',
error)
tf.compat.v1.reset_default_graph()
return Gs, Zs, error_per_sample, heuristic_sigma
cur_L = total_L / args.log_interval
print('[Epoch %d Batch %d] loss %.2f, ppl %.2f'%(
epoch, i, cur_L, math.exp(cur_L)))
total_L = 0.0
if args.export_model:
model.export('model')
return
val_L = eval(val_data)
print('[Epoch %d] time cost %.2fs, valid loss %.2f, valid ppl %.2f'%(
epoch, time.time()-start_time, val_L, math.exp(val_L)))
if val_L < best_val:
best_val = val_L
test_L = eval(test_data)
model.save_parameters(args.save)
print('test loss %.2f, test ppl %.2f'%(test_L, math.exp(test_L)))
else:
args.lr = args.lr*0.25
trainer.set_learning_rate(args.lr)
if __name__ == '__main__':
train()
if not args.export_model:
model.load_parameters(args.save, context)
test_L = eval(test_data)
print('Best test loss %.2f, test ppl %.2f'%(test_L, math.exp(test_L)))
