def model_comparison(identifier_A, identifier_B, epoch_A=99, epoch_B=99):
"""
Compare two models using relative MMD test
"""
# make sure they used the same data
settings_A = json.load(open('./experiments/settings/' + identifier_A + '.txt', 'r'))
settings_B = json.load(open('./experiments/settings/' + identifier_B + '.txt', 'r'))
data_path = assert_same_data(settings_A, settings_B)
# now load the data
data = np.load(data_path + '.data.npy').item()['samples']['vali']
n_samples = data.shape[0]
A_samples = model.sample_trained_model(settings_A, epoch_A, n_samples)
B_samples = model.sample_trained_model(settings_B, epoch_B, n_samples)
# do the comparison
# TODO: support multiple signals
## some notes about this test:
## MMD_3_Sample_Test(X, Y, Z) tests the hypothesis that Px is closer to Pz than Py
## that is, test the null hypothesis H0:
## MMD(F, Px, Py) <= MMD(F, Px, Pz)
## versus the alternate hypothesis:
## MMD(F, Px, Py) > MMD(F, Px, Pz)
## at significance level that we select later (just the threshold on the p-value)
pvalue, tstat, sigma, MMDXY, MMDXZ = MMD_3_Sample_Test(data[:, :, 0], A_samples[:, :, 0], B_samples[:, :, 0], computeMMDs=True)
print(pvalue, tstat, sigma)
if pvalue < 0.05:
print('At confidence level 0.05, we reject the null hypothesis that MMDXY <= MMDXZ, and conclude that', identifier_B, 'has a smaller MMD with the true data than', identifier_A)
else:
print('We have failed to reject the null hypothesis that MMDXY <= MMDXZ, and cannot conclude that', identifier_B, 'has a smaller MMD with the true data than', identifier_A)
return pvalue, tstat, sigma, MMDXY, MMDXZ
def NIPS_toy_plot(identifier_rbf, epoch_rbf, identifier_sine, epoch_sine,
identifier_mnist, epoch_mnist):
"""
for each experiment:
- plot a bunch of train examples
- sample a bunch of generated examples
- plot all in separate PDFs so i can merge in illustrator
for sine and rbf, grey background
MNIST is just MNIST (square though)
"""
n_samples = 15
# settings
settings_rbf = json.load(
open('./experiments/settings/' + identifier_rbf + '.txt', 'r'))
settings_sine = json.load(
open('./experiments/settings/' + identifier_sine + '.txt', 'r'))
settings_mnist = json.load(
open('./experiments/settings/' + identifier_mnist + '.txt', 'r'))
# data
data_rbf = np.load('./experiments/data/' + identifier_rbf +
'.data.npy').item()
data_sine = np.load('./experiments/data/' + identifier_sine +
'.data.npy').item()
data_mnist = np.load('./experiments/data/' + identifier_mnist +
'.data.npy').item()
train_rbf = data_rbf['samples']['train']
train_sine = data_sine['samples']['train']
train_mnist = data_mnist['samples']['train']
# sample
samples_rbf = model.sample_trained_model(settings_rbf, epoch_rbf,
n_samples)
samples_sine = model.sample_trained_model(settings_sine, epoch_sine,
n_samples)
samples_mnist = model.sample_trained_model(settings_mnist, epoch_mnist,
n_samples)
# plot them all
index = 0
#for sample in np.random.permutation(train_rbf)[:n_samples]:
# plotting.nips_plot_rbf(sample, index, 'train')
# index += 1
#for sample in samples_rbf:
# plotting.nips_plot_rbf(sample, index, 'GAN')
# index += 1
#for sample in np.random.permutation(train_sine)[:n_samples]:
# plotting.nips_plot_sine(sample, index, 'train')
# index += 1
#for sample in samples_sine:
# plotting.nips_plot_sine(sample, index, 'GAN')
# index += 1
for sample in np.random.permutation(train_mnist)[:n_samples]:
plotting.nips_plot_mnist(sample, index, 'train')
index += 1
for sample in samples_mnist:
plotting.nips_plot_mnist(sample, index, 'GAN')
index += 1
return True
def generate_synthetic(identifier, epoch, n_train, predict_labels=False):
"""
- Load a CGAN pretrained model
- Load its corresponding test data (+ labels)
- Generate num_examples synthetic training data (+labels)
- Save to format easy for training classifier on (see Eval)
"""
settings = json.load(open('./experiments/settings/' + identifier + '.txt', 'r'))
if not settings['cond_dim'] > 0:
assert settings['predict_labels']
assert predict_labels
# get the test data
print('Loading test (real) data for', identifier)
data_dict = np.load('./experiments/data/' + identifier + '.data.npy').item()
test_data = data_dict['samples']['test']
test_labels = data_dict['labels']['test']
train_data = data_dict['samples']['train']
train_labels = data_dict['labels']['train']
print('Loaded', test_data.shape[0], 'test examples')
print('Sampling', n_train, 'train examples from the model')
if not predict_labels:
assert test_data.shape[0] == test_labels.shape[0]
if 'eICU' in settings['data']:
synth_labels = train_labels[np.random.choice(train_labels.shape[0], n_train), :]
else:
# this doesn't really work for eICU...
synth_labels = model.sample_C(n_train, settings['cond_dim'], settings['max_val'], settings['one_hot'])
synth_data = model.sample_trained_model(settings, epoch, n_train, Z_samples=None, cond_dim=settings['cond_dim'], C_samples=synth_labels)
else:
assert settings['predict_labels']
synth_data = model.sample_trained_model(settings, epoch, n_train, Z_samples=None, cond_dim=0)
# extract the labels
if 'eICU' in settings['data']:
n_labels = 7
synth_labels = synth_data[:, :, -n_labels:]
train_labels = train_data[:, :, -n_labels:]
test_labels = test_data[:, :, -n_labels:]
else:
n_labels = 6 # mnist
synth_labels, _ = mode(np.argmax(synth_data[:, :, -n_labels:], axis=2), axis=1)
train_labels, _ = mode(np.argmax(train_data[:, :, -n_labels:], axis=2), axis=1)
test_labels, _ = mode(np.argmax(test_data[:, :, -n_labels:], axis=2), axis=1)
synth_data = synth_data[:, :, :-n_labels]
train_data = train_data[:, :, :-n_labels]
test_data = test_data[:, :, :-n_labels]
# package up, save
exp_data = dict()
exp_data['test_data'] = test_data
exp_data['test_labels'] = test_labels
exp_data['train_data'] = train_data
exp_data['train_labels'] = train_labels
exp_data['synth_data'] = synth_data
exp_data['synth_labels'] = synth_labels
# save it all up
np.save('./experiments/tstr/' + identifier + '_' + str(epoch) + '.data.npy', exp_data)
return True
def view_digit(identifier, epoch, digit, n_samples=6):
"""
Generate a bunch of MNIST digits from a CGAN, view them
"""
settings = json.load(
open('./experiments/settings/' + identifier + '.txt', 'r'))
if settings['one_hot']:
assert settings['max_val'] == 1
assert digit <= settings['cond_dim']
C_samples = np.zeros(shape=(n_samples, settings['cond_dim']))
C_samples[:, digit] = 1
else:
assert settings['cond_dim'] == 1
assert digit <= settings['max_val']
C_samples = np.array([digit] * n_samples).reshape(-1, 1)
digit_samples = model.sample_trained_model(settings,
epoch,
n_samples,
Z_samples=None,
cond_dim=settings['cond_dim'],
C_samples=C_samples)
digit_samples = digit_samples.reshape(n_samples, -1, 1)
# visualise
plotting.save_mnist_plot_sample(digit_samples, digit,
identifier + '_' + str(epoch) + '_digit_',
n_samples)
return True
def view_interpolation(identifier, epoch, n_steps=6, input_samples=None, e_tolerance=0.01, sigma=3.29286853021):
"""
If samples: generate interpolation between real points
Else:
Sample two points in the latent space, view a linear interpolation between them.
"""
settings = json.load(open('./experiments/settings/' + identifier + '.txt', 'r'))
if input_samples is None:
# grab two trainng examples
data = np.load('./experiments/data/' + identifier + '.data.npy').item()
train = data['samples']['train']
input_samples = np.random.permutation(train)[:2]
# Z_sampleA, Z_sampleB = model.sample_Z(2, settings['seq_length'], settings['latent_dim'],
# settings['use_time'])
if sigma is None:
## gotta get a sigma somehow
sigma = mmd.median_pairwise_distance(train)
print('Calcualted heuristic sigma from training data:', sigma)
Zs, error, _ = model.invert(settings, epoch, input_samples, e_tolerance=e_tolerance)
Z_sampleA, Z_sampleB = Zs
Z_samples = plotting.interpolate(Z_sampleA, Z_sampleB, n_steps=n_steps)
samples = model.sample_trained_model(settings, epoch, Z_samples.shape[0], Z_samples)
# get distances from generated samples to target samples
d_A, d_B = [], []
for sample in samples:
d_A.append(sample_distance(sample, samples[0], sigma))
d_B.append(sample_distance(sample, samples[-1], sigma))
distances = pd.DataFrame({'dA': d_A, 'dB': d_B})
plotting.save_plot_interpolate(input_samples, samples, epoch, settings['identifier'] + '_epoch' + str(epoch), distances=distances, sigma=sigma)
return True
def view_latent_vary(identifier, epoch, n_steps=6):
settings = json.load(open('./experiments/settings/' + identifier + '.txt', 'r'))
Z_sample = model.sample_Z(1, settings['seq_length'], settings['latent_dim'],
settings['use_time'])[0]
samples_dim = []
for dim in range(settings['latent_dim']):
Z_samples_dim = plotting.vary_latent_dimension(Z_sample, dim, n_steps)
samples_dim.append(model.sample_trained_model(settings, epoch, Z_samples_dim.shape[0], Z_samples_dim))
plotting.save_plot_vary_dimension(samples_dim, epoch, settings['identifier'] + '_varydim', n_dim=settings['latent_dim'])
return True
def view_reconstruction(identifier, epoch, real_samples, tolerance=1):
"""
Given a set of real samples, find the "closest" latent space points
corresponding to them, generate samples from these, visualise!
"""
settings = json.load(open('./experiments/settings/' + identifier + '.txt', 'r'))
Zs, error, sigma = model.invert(settings, epoch, real_samples, tolerance=tolerance)
plotting.visualise_latent(Zs[0], identifier+'_' + str(epoch) + '_0')
plotting.visualise_latent(Zs[1], identifier+'_' + str(epoch) + '_1')
model_samples = model.sample_trained_model(settings, epoch, Zs.shape[0], Zs)
plotting.save_plot_reconstruct(real_samples, model_samples, settings['identifier'])
return True
def view_fixed(identifier, epoch, n_samples=6, dim=None):
""" What happens when we give the same point at each time step? """
settings = json.load(open('./experiments/settings/' + identifier + '.txt', 'r'))
Z_samples = model.sample_Z(n_samples, settings['seq_length'], settings['latent_dim'],
settings['use_time'])
# now, propagate forward the value at time 0 (which time doesn't matter)
for i in range(1, settings['seq_length']):
if dim is None:
Z_samples[:, i, :] = Z_samples[:, 0, :]
else:
Z_samples[:, i, dim] = Z_samples[:, 0, dim]
# now generate
samples = model.sample_trained_model(settings, epoch, n_samples, Z_samples)
# now visualise
plotting.save_plot_sample(samples, epoch, identifier + '_fixed', n_samples)
return True
def TSTR_eICU(identifier,
epoch,
generate=True,
vali=True,
CNN=False,
do_OR=False,
duplicate_synth=1,
reverse=False):
"""
"""
if vali:
test_set = 'vali'
else:
test_set = 'test'
data = np.load('./experiments/data/' + identifier + '.data.npy').item()
samples = data['samples']
train_X = samples['train']
test_X = samples[test_set]
labels = data['labels']
train_Y = labels['train']
test_Y = labels[test_set]
if generate:
# now sample from the model
synth_Y = np.tile(train_Y, [duplicate_synth, 1])
synth_X = model.sample_trained_model(identifier,
epoch,
num_samples=synth_Y.shape[0],
C_samples=synth_Y)
# for use in TRTS
synth_testX = model.sample_trained_model(identifier,
epoch,
num_samples=test_Y.shape[0],
C_samples=test_Y)
synth_data = {
'samples': synth_X,
'labels': synth_Y,
'test_samples': synth_testX,
'test_labels': test_Y
}
np.save(
'./experiments/tstr/' + identifier + '_' + str(epoch) +
'.data.npy', synth_data)
else:
print('Loading pre-generated data')
print('WARNING: not implemented for TRTS')
# get "train" data
exp_data = np.load('./experiments/tstr/' + identifier + '_' +
str(epoch) + '.data.npy').item()
synth_X = exp_data['samples']
synth_Y = exp_data['labels']
n_synth = synth_X.shape[0]
synth_X = synth_X.reshape(n_synth, -1)
# pdb.set_trace()
# # ALERT ALERT MODIFYING
# synth_X = 2*(synth_X > 0) - 1
orig_data = np.load('/cluster/home/hyland/eICU_task_data.npy').item()
if reverse:
which_setting = 'trts'
# visualise distribution of errors for train and test
print('Swapping synthetic test set in for real, to do TRTS!')
test_X = synth_testX
else:
print('Doing normal TSTR')
which_setting = 'tstr'
# # get test data
# test_X = data['test_X']
# test_Y = data['test_Y']
if not CNN:
model_choice = 'RF'
# if multivariate, reshape
if len(test_X.shape) == 3:
test_X = test_X.reshape(test_X.shape[0], -1)
if len(train_X.shape) == 3:
train_X = train_X.reshape(train_X.shape[0], -1)
if len(synth_X.shape) == 3:
synth_X = synth_X.reshape(synth_X.shape[0], -1)
else:
raise ValueError(CNN)
model_choice = 'CNN'
# we will select the best validation set epoch based on F1 score, take average across all the tasks
score_list = []
for label in range(synth_Y.shape[1]):
task = orig_data['Y_columns'][label]
if vali:
if not task in ['low_sao2', 'high_heartrate', 'low_respiration']:
print('Skipping task', task, 'because validation evaluation.')
continue
print('Evaluating on task:', task)
#print('(', np.mean(synth_Y[:, label]), 'positive in train, ', np.mean(test_Y[:, label]), 'in test)')
#m = RandomForestClassifier(n_estimators=50).fit(synth_X, synth_Y[:, label])
#m = SVC(gamma=0.001).fit(synth_X, synth_Y[:, label])
synth_classifier = RandomForestClassifier(n_estimators=100).fit(
synth_X, synth_Y[:, label])
synth_predY = synth_classifier.predict(test_X)
synth_predY_prob = synth_classifier.predict_proba(test_X)[:, 1]
real_classifier = RandomForestClassifier(n_estimators=100).fit(
train_X, train_Y[:, label])
real_predY = real_classifier.predict(test_X)
real_predY_prob = real_classifier.predict_proba(test_X)[:, 1]
#print('(predicted', np.mean(predict), 'positive labels)')
synth_prec, synth_recall, synth_f1, synth_support = precision_recall_fscore_support(
test_Y[:, label], synth_predY, average='weighted')
synth_accuracy = accuracy_score(test_Y[:, label], synth_predY)
synth_auprc = average_precision_score(test_Y[:, label],
synth_predY_prob)
synth_auroc = roc_auc_score(test_Y[:, label], synth_predY_prob)
synth_scores = [
synth_prec, synth_recall, synth_f1, synth_accuracy, synth_auprc,
synth_auroc
]
real_prec, real_recall, real_f1, real_support = precision_recall_fscore_support(
test_Y[:, label], real_predY, average='weighted')
real_accuracy = accuracy_score(test_Y[:, label], real_predY)
real_auprc = average_precision_score(test_Y[:, label], real_predY_prob)
real_auroc = roc_auc_score(test_Y[:, label], real_predY_prob)
real_scores = [
real_prec, real_recall, real_f1, real_accuracy, real_auprc,
real_auroc
]
all_scores = synth_scores + real_scores
if vali:
report_file = open(
'./experiments/tstr/vali.' + which_setting + '_report.v3.csv',
'a')
report_file.write('eICU_' + task + ',' + identifier + ',' +
model_choice + ',' + str(epoch) + ',' +
','.join(map(str, all_scores)) + '\n')
report_file.close()
else:
report_file = open(
'./experiments/tstr/' + which_setting + '_report.v3.csv', 'a')
report_file.write('eICU_' + task + ',' + identifier + ',' +
model_choice + ',' + str(epoch) + ',' +
','.join(map(str, all_scores)) + '\n')
report_file.close()
print(classification_report(test_Y[:, label], synth_predY))
print(classification_report(test_Y[:, label], real_predY))
if task in ['low_sao2', 'high_heartrate', 'low_respiration']:
score_list.append(synth_auprc + synth_auroc)
if do_OR:
raise NotImplementedError
# do the OR task
extreme_heartrate_test = test_Y[:, 1] + test_Y[:, 4]
extreme_respiration_test = test_Y[:, 2] + test_Y[:, 5]
extreme_systemicmean_test = test_Y[:, 3] + test_Y[:, 6]
Y_OR_test = np.vstack([
extreme_heartrate_test, extreme_respiration_test,
extreme_systemicmean_test
]).T
Y_OR_test = (Y_OR_test > 0) * 1
extreme_heartrate_synth = synth_Y[:, 1] + synth_Y[:, 4]
extreme_respiration_synth = synth_Y[:, 2] + synth_Y[:, 5]
extreme_systemicmean_synth = synth_Y[:, 3] + synth_Y[:, 6]
Y_OR_synth = np.vstack([
extreme_heartrate_synth, extreme_respiration_synth,
extreme_systemicmean_synth
]).T
Y_OR_synth = (Y_OR_synth > 0) * 1
OR_names = ['extreme heartrate', 'extreme respiration', 'extreme MAP']
OR_results = []
for label in range(Y_OR_synth.shape[1]):
print('task:', OR_names[label])
print('(', np.mean(Y_OR_synth[:, label]), 'positive in train, ',
np.mean(Y_OR_test[:, label]), 'in test)')
m = RandomForestClassifier(n_estimators=500).fit(
synth_X, Y_OR_synth[:, label])
predict = m.predict(X_test)
print('(predicted', np.mean(predict), 'positive labels)')
accuracy = accuracy_score(Y_OR_test[:, label], predict)
precision = sklearn.metrics.precision_score(
Y_OR_test[:, label], predict)
recall = sklearn.metrics.recall_score(Y_OR_test[:, label], predict)
print(accuracy, precision, recall)
OR_results.append([accuracy, precision, recall])
else:
OR_results = []
score_across_tasks = np.mean(np.array(score_list))
return score_across_tasks
def TSTR_mnist(identifier,
epoch,
generate=True,
duplicate_synth=1,
vali=True,
CNN=False,
reverse=False):
"""
Either load or generate synthetic training, real test data...
Load synthetic training, real test data, do multi-class SVM
(basically just this: http://scikit-learn.org/stable/auto_examples/classification/plot_digits_classification.html)
If reverse = True: do TRTS
"""
print('Running TSTR on', identifier, 'at epoch', epoch)
if vali:
test_set = 'vali'
else:
test_set = 'test'
if generate:
data = np.load('./experiments/data/' + identifier + '.data.npy').item()
samples = data['samples']
train_X = samples['train']
test_X = samples[test_set]
labels = data['labels']
train_Y = labels['train']
test_Y = labels[test_set]
# now sample from the model
synth_Y = np.tile(train_Y, [duplicate_synth, 1])
synth_X = model.sample_trained_model(identifier,
epoch,
num_samples=synth_Y.shape[0],
C_samples=synth_Y)
# for use in TRTS
synth_testX = model.sample_trained_model(identifier,
epoch,
num_samples=test_Y.shape[0],
C_samples=test_Y)
synth_data = {
'samples': synth_X,
'labels': synth_Y,
'test_samples': synth_testX,
'test_labels': test_Y
}
np.save(
'./experiments/tstr/' + identifier + '_' + str(epoch) +
'.data.npy', synth_data)
else:
print('Loading synthetic data from pre-sampled model')
exp_data = np.load('./experiments/tstr/' + identifier + '_' +
str(epoch) + '.data.npy').item()
test_X, test_Y = exp_data['test_data'], exp_data['test_labels']
train_X, train_Y = exp_data['train_data'], exp_data['train_labels']
synth_X, synth_Y = exp_data['synth_data'], exp_data['synth_labels']
if reverse:
which_setting = 'trts'
print('Swapping synthetic test set in for real, to do TRTS!')
test_X = synth_testX
else:
print('Doing normal TSTR')
which_setting = 'tstr'
# make classifier
if not CNN:
model_choice = 'RF'
# if multivariate, reshape
if len(test_X.shape) == 3:
test_X = test_X.reshape(test_X.shape[0], -1)
if len(train_X.shape) == 3:
train_X = train_X.reshape(train_X.shape[0], -1)
if len(synth_X.shape) == 3:
synth_X = synth_X.reshape(synth_X.shape[0], -1)
# if one hot, fix
if len(synth_Y.shape) > 1 and not synth_Y.shape[1] == 1:
synth_Y = np.argmax(synth_Y, axis=1)
train_Y = np.argmax(train_Y, axis=1)
test_Y = np.argmax(test_Y, axis=1)
# random forest
#synth_classifier = SVC(gamma=0.001)
#real_classifier = SVC(gamma=0.001)
synth_classifier = RandomForestClassifier(n_estimators=500)
real_classifier = RandomForestClassifier(n_estimators=500)
# fit
real_classifier.fit(train_X, train_Y)
synth_classifier.fit(synth_X, synth_Y)
# test on real
synth_predY = synth_classifier.predict(test_X)
real_predY = real_classifier.predict(test_X)
else:
model_choice = 'CNN'
synth_predY = train_CNN(synth_X, synth_Y, samples['vali'],
labels['vali'], test_X)
clear_session()
real_predY = train_CNN(train_X, train_Y, samples['vali'],
labels['vali'], test_X)
clear_session()
# CNN setting is all 'one-hot'
test_Y = np.argmax(test_Y, axis=1)
synth_predY = np.argmax(synth_predY, axis=1)
real_predY = np.argmax(real_predY, axis=1)
# report on results
synth_prec, synth_recall, synth_f1, synth_support = precision_recall_fscore_support(
test_Y, synth_predY, average='weighted')
synth_accuracy = accuracy_score(test_Y, synth_predY)
synth_auprc = 'NaN'
synth_auroc = 'NaN'
synth_scores = [
synth_prec, synth_recall, synth_f1, synth_accuracy, synth_auprc,
synth_auroc
]
real_prec, real_recall, real_f1, real_support = precision_recall_fscore_support(
test_Y, real_predY, average='weighted')
real_accuracy = accuracy_score(test_Y, real_predY)
real_auprc = 'NaN'
real_auroc = 'NaN'
real_scores = [
real_prec, real_recall, real_f1, real_accuracy, real_auprc, real_auroc
]
all_scores = synth_scores + real_scores
if vali:
report_file = open(
'./experiments/tstr/vali.' + which_setting + '_report.v3.csv', 'a')
report_file.write('mnist,' + identifier + ',' + model_choice + ',' +
str(epoch) + ',' + ','.join(map(str, all_scores)) +
'\n')
report_file.close()
else:
report_file = open(
'./experiments/tstr/' + which_setting + '_report.v3.csv', 'a')
report_file.write('mnist,' + identifier + ',' + model_choice + ',' +
str(epoch) + ',' + ','.join(map(str, all_scores)) +
'\n')
report_file.close()
# visualise results
try:
plotting.view_mnist_eval(identifier + '_' + str(epoch), train_X,
train_Y, synth_X, synth_Y, test_X, test_Y,
synth_predY, real_predY)
except ValueError:
print('PLOTTING ERROR')
pdb.set_trace()
print(classification_report(test_Y, synth_predY))
print(classification_report(test_Y, real_predY))
return synth_f1, real_f1
def model_memorisation(identifier, epoch, max_samples=2000, tstr=False):
"""
Compare samples from a model against training set and validation set in mmd
"""
if tstr:
print('Loading data from TSTR experiment (not sampling from model)')
# load pre-generated samples
synth_data = np.load('./experiments/tstr/' + identifier + '_' +
str(epoch) + '.data.npy').item()
model_samples = synth_data['samples']
synth_labels = synth_data['labels']
# load real data used in that experiment
real_data = np.load('./experiments/data/' + identifier +
'.data.npy').item()
real_samples = real_data['samples']
train = real_samples['train']
test = real_samples['test']
n_samples = test.shape[0]
if model_samples.shape[0] > n_samples:
model_samples = np.random.permutation(model_samples)[:n_samples]
print('Data loaded successfully!')
else:
if identifier == 'cristobal_eICU':
model_samples = pickle.load(open('REDACTED', 'rb'))
samples, labels = data_utils.eICU_task()
train = samples['train'].reshape(-1, 16, 4)
vali = samples['vali'].reshape(-1, 16, 4)
test = samples['test'].reshape(-1, 16, 4)
#train_targets = labels['train']
#vali_targets = labels['vali']
#test_targets = labels['test']
train, vali, test = data_utils.scale_data(train, vali, test)
n_samples = test.shape[0]
if n_samples > max_samples:
n_samples = max_samples
test = np.random.permutation(test)[:n_samples]
if model_samples.shape[0] > n_samples:
model_samples = np.random.permutation(
model_samples)[:n_samples]
elif identifier == 'cristobal_MNIST':
the_dir = 'REDACTED'
# pick a random one
which = np.random.choice(['NEW_OK_', '_r4', '_r5', '_r6', '_r7'])
model_samples, model_labels = pickle.load(
open(
the_dir +
'synth_mnist_minist_cdgan_1_2_100_multivar_14_nolr_rdim3_0_2_'
+ which + '_190.pk', 'rb'))
# get test and train...
# (generated with fixed seed...)
mnist_resized_dim = 14
samples, labels = data_utils.load_resized_mnist(mnist_resized_dim)
proportions = [0.6, 0.2, 0.2]
train, vali, test, labels_split = data_utils.split(
samples, labels=labels, random_seed=1, proportions=proportions)
np.random.seed()
train = train.reshape(-1, 14, 14)
test = test.reshape(-1, 14, 14)
vali = vali.reshape(-1, 14, 14)
n_samples = test.shape[0]
if n_samples > max_samples:
n_samples = max_samples
test = np.random.permutation(test)[:n_samples]
if model_samples.shape[0] > n_samples:
model_samples = np.random.permutation(
model_samples)[:n_samples]
else:
settings = json.load(
open('./experiments/settings/' + identifier + '.txt', 'r'))
# get the test, train sets
data = np.load('./experiments/data/' + identifier +
'.data.npy').item()
train = data['samples']['train']
test = data['samples']['test']
n_samples = test.shape[0]
if n_samples > max_samples:
n_samples = max_samples
test = np.random.permutation(test)[:n_samples]
model_samples = model.sample_trained_model(settings, epoch,
n_samples)
all_samples = np.vstack([train, test, model_samples])
heuristic_sigma = mmd.median_pairwise_distance(all_samples)
print('heuristic sigma:', heuristic_sigma)
pvalue, tstat, sigma, MMDXY, MMDXZ = MMD_3_Sample_Test(
model_samples,
test,
np.random.permutation(train)[:n_samples],
sigma=heuristic_sigma,
computeMMDs=False)
#pvalue, tstat, sigma, MMDXY, MMDXZ = MMD_3_Sample_Test(model_samples, np.random.permutation(train)[:n_samples], test, sigma=heuristic_sigma, computeMMDs=False)
# if pvalue < 0.05:
# print('At confidence level 0.05, we reject the null hypothesis that MMDXY <= MMDXZ, and conclude that the test data has a smaller MMD with the true data than the generated data')
# the function takes (X, Y, Z) as its first arguments, it's testing if MMDXY (i.e. MMD between model and train) is less than MMDXZ (MMd between model and test)
# else:
# print('We have failed to reject the null hypothesis that MMDXY <= MMDXZ, and cannot conclu#de that the test data has a smaller MMD with the true data than the generated data')
return pvalue, tstat, sigma
def get_reconstruction_errors(identifier,
epoch,
g_tolerance=0.05,
max_samples=1000,
rerun=False,
tstr=False):
"""
Get the reconstruction error of every point in the training set of a given
experiment.
"""
settings = json.load(
open('./experiments/settings/' + identifier + '.txt', 'r'))
if settings['data_load_from']:
data_dict = np.load('./experiments/data/' +
settings['data_load_from'] + '.data.npy').item()
else:
data_dict = np.load('./experiments/data/' + identifier +
'.data.npy').item()
samples = data_dict['samples']
train = samples['train']
vali = samples['vali']
test = samples['test']
labels = data_dict['labels']
train_labels, test_labels, synth_labels, vali_labels = None, None, None, None
try:
if rerun:
raise FileNotFoundError
errors = np.load('./experiments/eval/' + identifier + '_' +
str(epoch) + '_' + str(g_tolerance) +
'.reconstruction_errors.npy').item()
train_errors = errors['train']
test_errors = errors['test']
generated_errors = errors['generated']
noisy_errors = errors['noisy']
print('Loaded precomputed errors')
except FileNotFoundError:
if tstr:
synth_data = np.load('./experiments/tstr/' + identifier + '_' +
str(epoch) + '.data.npy').item()
generated = synth_data['samples']
synth_labels = synth_data['labels']
train_labels = labels['train']
test_labels = labels['test']
vali_labels = labels['vali']
else:
# generate new data
n_eval = 500
# generate "easy" samples from the distribution
generated = model.sample_trained_model(settings, epoch, n_eval)
# generate "hard' random samples, not from train/test distribution
# TODO: use original validation examples, add noise etc.
## random_samples = np.random.normal(size=generated.shape)
# random_samples -= np.mean(random_samples, axis=0)
# random_samples += np.mean(vali, axis=0)
# random_samples /= np.std(random_samples, axis=0)
# random_samples *= np.std(vali, axis=0)
# get all the errors
print('Getting reconstruction errors on train set')
if train.shape[0] > max_samples:
index_subset = np.random.permutation(train.shape[0])[:max_samples]
train = train[index_subset]
if train_labels is not None:
train_labels = train_labels[index_subset]
train_errors = error_per_sample(identifier,
epoch,
train,
n_rep=5,
g_tolerance=g_tolerance,
C_samples=train_labels)
print('Getting reconstruction errors on test set')
if test.shape[0] > max_samples:
index_subset = np.random.permutation(test.shape[0])[:max_samples]
test = test[index_subset]
if test_labels is not None:
test_labels = test_labels[index_subset]
test_errors = error_per_sample(identifier,
epoch,
test,
n_rep=5,
g_tolerance=g_tolerance,
C_samples=test_labels)
D_test, p_test = ks_2samp(train_errors, test_errors)
print('KS statistic and p-value for train v. test erors:', D_test,
p_test)
pdb.set_trace()
print('Getting reconstruction errors on generated set')
generated_errors = error_per_sample(identifier,
epoch,
generated,
n_rep=5,
g_tolerance=g_tolerance,
C_samples=synth_labels)
D_gen, p_gen = ks_2samp(generated_errors, train_errors)
print('KS statistic and p-value for train v. gen erors:', D_gen, p_gen)
D_gentest, p_gentest = ks_2samp(generated_errors, test_errors)
print('KS statistic and p-value for gen v. test erors:', D_gentest,
p_gentest)
# print('Getting reconstruction errors on noisy set')
# alpha = 0.5
# noisy_samples = alpha*vali + (1-alpha)*np.random.permutation(vali)
# noisy_errors = error_per_sample(identifier, epoch, noisy_samples, n_rep=5, g_tolerance=g_tolerance, C_samples=vali_labels)
noisy_errors = None
# save!
errors = {
'train': train_errors,
'test': test_errors,
'generated': generated_errors,
'noisy': noisy_errors
}
np.save(
'./experiments/eval/' + identifier + '_' + str(epoch) + '_' +
str(g_tolerance) + '.reconstruction_errors.npy', errors)
# do two-sample Kolomogorov-Smirnov test for equality
D_test, p_test = ks_2samp(train_errors, test_errors)
print('KS statistic and p-value for train v. test erors:', D_test, p_test)
D_gen, p_gen = ks_2samp(generated_errors, train_errors)
print('KS statistic and p-value for train v. gen erors:', D_gen, p_gen)
D_gentest, p_gentest = ks_2samp(generated_errors, test_errors)
print('KS statistic and p-value for gen v. test erors:', D_gentest,
p_gentest)
# visualise distribution of errors for train and test
plotting.reconstruction_errors(
identifier + '_' + str(epoch) + '_' + str(g_tolerance), train_errors,
test_errors, generated_errors, noisy_errors)
# visualise the "hardest" and "easiest" samples from train
ranking_train = np.argsort(train_errors)
easiest_train = ranking_train[:6]
hardest_train = ranking_train[-6:]
plotting.save_plot_sample(train[easiest_train],
epoch,
identifier + '_easytrain',
n_samples=6,
num_epochs=None,
ncol=2)
plotting.save_plot_sample(train[hardest_train],
epoch,
identifier + '_hardtrain',
n_samples=6,
num_epochs=None,
ncol=2)
# visualise the "hardest" and "easiest" samples from random
# ranking_random = np.argsort(noisy_errors)
# easiest_random = ranking_random[:6]
# hardest_random = ranking_random[-6:]
# plotting.save_plot_sample(random_samples[easiest_random], epoch, identifier + '_easyrandom', n_samples=6, num_epochs=None, ncol=2)
# plotting.save_plot_sample(random_samples[hardest_random], epoch, identifier + '_hardrandom', n_samples=6, num_epochs=None, ncol=2)
return True