def error_per_sample(identifier,
epoch,
samples,
n_rep=3,
n_iter=None,
g_tolerance=0.025,
use_min=True,
C_samples=None):
"""
Get (average over a few runs) of the reconstruction error per sample
"""
n_samples = samples.shape[0]
heuristic_sigma = np.float32(mmd.median_pairwise_distance(samples))
errors = np.zeros(shape=(n_samples, n_rep))
for rep in range(n_rep):
Z, rep_errors, sigma = model.invert(identifier,
epoch,
samples,
n_iter=n_iter,
heuristic_sigma=heuristic_sigma,
g_tolerance=g_tolerance,
C_samples=C_samples)
errors[:, rep] = rep_errors
# return min, or average?
if use_min:
errors = np.min(errors, axis=1)
else:
# use mean
errors = np.mean(errors, axis=1)
return errors
def view_interpolation(identifier, epoch, n_steps=6, input_samples=None, e_tolerance=0.01, sigma=3.29286853021):
"""
If samples: generate interpolation between real points
Else:
Sample two points in the latent space, view a linear interpolation between them.
"""
settings = json.load(open('./experiments/settings/' + identifier + '.txt', 'r'))
if input_samples is None:
# grab two trainng examples
data = np.load('./experiments/data/' + identifier + '.data.npy').item()
train = data['samples']['train']
input_samples = np.random.permutation(train)[:2]
# Z_sampleA, Z_sampleB = model.sample_Z(2, settings['seq_length'], settings['latent_dim'],
# settings['use_time'])
if sigma is None:
## gotta get a sigma somehow
sigma = mmd.median_pairwise_distance(train)
print('Calcualted heuristic sigma from training data:', sigma)
Zs, error, _ = model.invert(settings, epoch, input_samples, e_tolerance=e_tolerance)
Z_sampleA, Z_sampleB = Zs
Z_samples = plotting.interpolate(Z_sampleA, Z_sampleB, n_steps=n_steps)
samples = model.sample_trained_model(settings, epoch, Z_samples.shape[0], Z_samples)
# get distances from generated samples to target samples
d_A, d_B = [], []
for sample in samples:
d_A.append(sample_distance(sample, samples[0], sigma))
d_B.append(sample_distance(sample, samples[-1], sigma))
distances = pd.DataFrame({'dA': d_A, 'dB': d_B})
plotting.save_plot_interpolate(input_samples, samples, epoch, settings['identifier'] + '_epoch' + str(epoch), distances=distances, sigma=sigma)
return True
def invert(settings, epoch, samples, g_tolerance=None, e_tolerance=0.1,
n_iter=None, max_iter=10000, heuristic_sigma=None, C_samples=None):
"""
Return the latent space points corresponding to a set of a samples
( from gradient descent )
"""
# cast samples to float32
samples = np.float32(samples[:, :, :])
# get the model
if type(settings) == str:
settings = json.load(open('./experiments/settings/' + settings + '.txt', 'r'))
num_samples = samples.shape[0]
print('Inverting', num_samples, 'samples using model', settings['identifier'], 'at epoch', epoch,)
if not g_tolerance is None:
print('until gradient norm is below', g_tolerance)
else:
print('until error is below', e_tolerance)
# get parameters
parameters = load_parameters(settings['identifier'] + '_' + str(epoch))
# assertions
assert samples.shape[2] == settings['num_generated_features']
# create VARIABLE Z
Z = tf.get_variable(name='Z', shape=[num_samples, settings['seq_length'],
settings['latent_dim']],
initializer=tf.random_normal_initializer())
if C_samples is None:
# create outputs
G_samples = generator(Z, settings['hidden_units_g'], settings['seq_length'],
num_samples, settings['num_generated_features'],
reuse=False, parameters=parameters)
fd = None
else:
CG = tf.placeholder(tf.float32, [num_samples, settings['cond_dim']])
assert C_samples.shape[0] == samples.shape[0]
# CGAN
G_samples = generator(Z, settings['hidden_units_g'], settings['seq_length'],
num_samples, settings['num_generated_features'],
reuse=False, parameters=parameters, cond_dim=settings['cond_dim'], c=CG)
fd = {CG: C_samples}
# define loss
if heuristic_sigma is None:
heuristic_sigma = mmd.median_pairwise_distance(samples) # this is noisy
print('heuristic_sigma:', heuristic_sigma)
Kxx, Kxy, Kyy, wts = mmd._mix_rbf_kernel(G_samples, samples, sigmas=tf.constant(value=heuristic_sigma, shape=(1, 1)))
similarity_per_sample = tf.diag_part(Kxy)
reconstruction_error_per_sample = 1 - similarity_per_sample
#reconstruction_error_per_sample = tf.reduce_sum((tf.nn.l2_normalize(G_samples, dim=1) - tf.nn.l2_normalize(samples, dim=1))**2, axis=[1,2])
similarity = tf.reduce_mean(similarity_per_sample)
reconstruction_error = 1 - similarity
# updater
# solver = tf.train.AdamOptimizer().minimize(reconstruction_error_per_sample, var_list=[Z])
#solver = tf.train.RMSPropOptimizer(learning_rate=500).minimize(reconstruction_error, var_list=[Z])
solver = tf.train.RMSPropOptimizer(learning_rate=0.1).minimize(reconstruction_error_per_sample, var_list=[Z])
#solver = tf.train.MomentumOptimizer(learning_rate=0.1, momentum=0.9).minimize(reconstruction_error_per_sample, var_list=[Z])
grad_Z = tf.gradients(reconstruction_error_per_sample, Z)[0]
grad_per_Z = tf.norm(grad_Z, axis=(1, 2))
grad_norm = tf.reduce_mean(grad_per_Z)
#solver = tf.train.GradientDescentOptimizer(learning_rate=0.1).minimize(reconstruction_error, var_list=[Z])
print('Finding latent state corresponding to samples...')
with tf.Session() as sess:
sess.run(tf.global_variables_initializer())
error = sess.run(reconstruction_error, feed_dict=fd)
g_n = sess.run(grad_norm, feed_dict=fd)
print(g_n)
i = 0
if not n_iter is None:
while i < n_iter:
_ = sess.run(solver, feed_dict=fd)
error = sess.run(reconstruction_error, feed_dict=fd)
i += 1
else:
if not g_tolerance is None:
while g_n > g_tolerance:
_ = sess.run(solver, feed_dict=fd)
error, g_n = sess.run([reconstruction_error, grad_norm], feed_dict=fd)
i += 1
print(error, g_n)
if i > max_iter:
break
else:
while np.abs(error) > e_tolerance:
_ = sess.run(solver, feed_dict=fd)
error = sess.run(reconstruction_error, feed_dict=fd)
i += 1
print(error)
if i > max_iter:
break
Zs = sess.run(Z, feed_dict=fd)
error_per_sample = sess.run(reconstruction_error_per_sample, feed_dict=fd)
print('Z found in', i, 'iterations with final reconstruction error of', error)
tf.reset_default_graph()
return Zs, error_per_sample, heuristic_sigma
sigma=0,
dp=False)
G_sample = model.generator(Z, **generator_settings, reuse=True, c=CG)
print("Train Samples : ", samples['train'].shape)
print("Validation Samples : ", samples['vali'].shape)
print("Test Samples : ", samples['test'].shape)
print("Train Labels :", labels['train'])
# # --- evaluation --- #
# # # frequency to do visualisations
vis_freq = 10
eval_freq = 1
# # # get heuristic bandwidth for mmd kernel from evaluation samples
heuristic_sigma_training = median_pairwise_distance(samples['vali'])
best_mmd2_so_far = 1000
# # optimise sigma using that (that's t-hat)
n_samples_sigma = len(samples['vali'])
batch_multiplier = n_samples_sigma // batch_size
eval_size = batch_multiplier * batch_size
print(eval_size)
eval_eval_size = int(0.2 * eval_size)
eval_real_PH = tf.placeholder(
tf.float32, [eval_eval_size, seq_length, num_generated_features])
eval_sample_PH = tf.placeholder(
tf.float32, [eval_eval_size, seq_length, num_generated_features])
n_sigmas = 2
sigma = tf.get_variable(
name='sigma',
def model_memorisation(identifier, epoch, max_samples=2000, tstr=False):
"""
Compare samples from a model against training set and validation set in mmd
"""
if tstr:
print('Loading data from TSTR experiment (not sampling from model)')
# load pre-generated samples
synth_data = np.load('./experiments/tstr/' + identifier + '_' +
str(epoch) + '.data.npy').item()
model_samples = synth_data['samples']
synth_labels = synth_data['labels']
# load real data used in that experiment
real_data = np.load('./experiments/data/' + identifier +
'.data.npy').item()
real_samples = real_data['samples']
train = real_samples['train']
test = real_samples['test']
n_samples = test.shape[0]
if model_samples.shape[0] > n_samples:
model_samples = np.random.permutation(model_samples)[:n_samples]
print('Data loaded successfully!')
else:
if identifier == 'cristobal_eICU':
model_samples = pickle.load(open('REDACTED', 'rb'))
samples, labels = data_utils.eICU_task()
train = samples['train'].reshape(-1, 16, 4)
vali = samples['vali'].reshape(-1, 16, 4)
test = samples['test'].reshape(-1, 16, 4)
#train_targets = labels['train']
#vali_targets = labels['vali']
#test_targets = labels['test']
train, vali, test = data_utils.scale_data(train, vali, test)
n_samples = test.shape[0]
if n_samples > max_samples:
n_samples = max_samples
test = np.random.permutation(test)[:n_samples]
if model_samples.shape[0] > n_samples:
model_samples = np.random.permutation(
model_samples)[:n_samples]
elif identifier == 'cristobal_MNIST':
the_dir = 'REDACTED'
# pick a random one
which = np.random.choice(['NEW_OK_', '_r4', '_r5', '_r6', '_r7'])
model_samples, model_labels = pickle.load(
open(
the_dir +
'synth_mnist_minist_cdgan_1_2_100_multivar_14_nolr_rdim3_0_2_'
+ which + '_190.pk', 'rb'))
# get test and train...
# (generated with fixed seed...)
mnist_resized_dim = 14
samples, labels = data_utils.load_resized_mnist(mnist_resized_dim)
proportions = [0.6, 0.2, 0.2]
train, vali, test, labels_split = data_utils.split(
samples, labels=labels, random_seed=1, proportions=proportions)
np.random.seed()
train = train.reshape(-1, 14, 14)
test = test.reshape(-1, 14, 14)
vali = vali.reshape(-1, 14, 14)
n_samples = test.shape[0]
if n_samples > max_samples:
n_samples = max_samples
test = np.random.permutation(test)[:n_samples]
if model_samples.shape[0] > n_samples:
model_samples = np.random.permutation(
model_samples)[:n_samples]
else:
settings = json.load(
open('./experiments/settings/' + identifier + '.txt', 'r'))
# get the test, train sets
data = np.load('./experiments/data/' + identifier +
'.data.npy').item()
train = data['samples']['train']
test = data['samples']['test']
n_samples = test.shape[0]
if n_samples > max_samples:
n_samples = max_samples
test = np.random.permutation(test)[:n_samples]
model_samples = model.sample_trained_model(settings, epoch,
n_samples)
all_samples = np.vstack([train, test, model_samples])
heuristic_sigma = mmd.median_pairwise_distance(all_samples)
print('heuristic sigma:', heuristic_sigma)
pvalue, tstat, sigma, MMDXY, MMDXZ = MMD_3_Sample_Test(
model_samples,
test,
np.random.permutation(train)[:n_samples],
sigma=heuristic_sigma,
computeMMDs=False)
#pvalue, tstat, sigma, MMDXY, MMDXZ = MMD_3_Sample_Test(model_samples, np.random.permutation(train)[:n_samples], test, sigma=heuristic_sigma, computeMMDs=False)
# if pvalue < 0.05:
# print('At confidence level 0.05, we reject the null hypothesis that MMDXY <= MMDXZ, and conclude that the test data has a smaller MMD with the true data than the generated data')
# the function takes (X, Y, Z) as its first arguments, it's testing if MMDXY (i.e. MMD between model and train) is less than MMDXZ (MMd between model and test)
# else:
# print('We have failed to reject the null hypothesis that MMDXY <= MMDXZ, and cannot conclu#de that the test data has a smaller MMD with the true data than the generated data')
return pvalue, tstat, sigma
