def run_training(corruption_chance, perplexity, batch_size):
global train_data, test_data
corrupt = lambda x: 0 if np.random.uniform() <= corruption_chance else x
train_data = np.vectorize(corrupt)(train_data)
test_data = np.vectorize(corrupt)(test_data)
def hook(args):
print(args)
if np.isnan(args[2]):
raise Exception
if isinstance(args[0], PTSNE) and args[2] <= 0.0:
raise Exception
vae = VAE(
[n_input_dimensions],
get_gaussian_network_builder(vae_encoder_layers, n_latent_dimensions),
gaussian_prior_supplier,
gaussian_supplier,
get_bernoulli_network_builder(vae_decoder_layers, n_input_dimensions),
bernoulli_supplier)
ptsne = PTSNE(
[n_input_dimensions],
get_feed_forward_network_builder(vptsne_layers, batch_normalization=False),
perplexity=perplexity)
vptsne = VPTSNE(
vae,
get_feed_forward_network_builder(vptsne_layers, batch_normalization=False),
perplexity=perplexity)
ptsne.fit(train_data, n_iters=1500, batch_size=batch_size, hook_fn=hook)
vptsne.fit(train_data, n_iters=1500, n_vae_iters=10000, batch_size=batch_size, vae_batch_size=1000, hook_fn=hook)
knn_score = KNC(n_neighbors=1).fit(
ptsne.transform(train_data), train_labels).score(
ptsne.transform(test_data), test_labels)
knn_score_vptsne = KNC(n_neighbors=1).fit(
vptsne.transform(train_data), train_labels).score(
vptsne.transform(test_data), test_labels)
tw = trustworthiness(
test_data,
ptsne.transform(test_data),
n_neighbors=12)
tw_vptsne = trustworthiness(
test_data,
vptsne.transform(test_data),
n_neighbors=12)
train_data = np.copy(non_corrupted_train_data)
test_data = np.copy(non_corrupted_test_data)
return knn_score, tw, knn_score_vptsne, tw_vptsne
print(np.var(distances(mnist.test._images[:1000])))
n_input_dimensions = mnist.train._images.shape[1]
n_latent_dimensions = 3
vae_layer_definitions = [(256, tf.nn.relu), (128, tf.nn.relu),
(32, tf.nn.relu)]
vae_encoder_layers = LayerDefinition.from_array(vae_layer_definitions)
vae_decoder_layers = LayerDefinition.from_array(
reversed(vae_layer_definitions))
vae = VAE([n_input_dimensions],
get_gaussian_network_builder(vae_encoder_layers,
n_latent_dimensions),
gaussian_prior_supplier, gaussian_supplier,
get_bernoulli_network_builder(vae_decoder_layers,
n_input_dimensions),
bernoulli_supplier)
vae.fit(mnist.train._images, n_iters=14000, batch_size=1000, hook_fn=print)
from sklearn.neighbors import KNeighborsClassifier as KNC
print(
"1-NN, test set",
KNC(n_neighbors=1).fit(vae.transform(mnist.train._images),
mnist.train._labels).score(
vae.transform(mnist.test._images),
mnist.test._labels))
color_palette = np.random.rand(100, 3)
data, labels = hdata.load_all("CYTOMETRY_data")
print(data.shape, labels.shape)
indices = np.random.permutation(data.shape[0])
number_of_points = int(data.shape[0] * 0.01)
subset_indices = indices[:number_of_points]
n_input_dimensions = data.shape[1]
n_latent_dimensions = 5
vae = VAE([n_input_dimensions],
get_gaussian_network_builder(vae_encoder_layers,
n_latent_dimensions),
gaussian_prior_supplier, gaussian_supplier,
get_gaussian_network_builder(vae_decoder_layers,
n_input_dimensions,
constant_sigma=0.1), gaussian_supplier)
fit_params = {
"hook_fn": print,
"n_iters": 1500,
"batch_size": 5000,
"deterministic": True,
"fit_vae": True,
"n_vae_iters": 10000,
"vae_batch_size": 1000
}
vptsne = VPTSNE(vae,
vptsne.save_weights("models/mnist_vptsne_deterministic_%s.ckpt" % info,
"models/mnist_vae_deterministic_%s.ckpt" % info)
ptsne.save_weights("models/mnist_vptsne_nondeterministic_%s.ckpt" % info)
for f in vptsne_log_files:
f.close()
for f in ptsne_log_files:
f.close()
if __name__ == "__main__":
vae = VAE([n_input_dimensions],
get_gaussian_network_builder(vae_encoder_layers,
n_latent_dimensions),
gaussian_prior_supplier, gaussian_supplier,
get_bernoulli_network_builder(vae_decoder_layers,
n_input_dimensions),
bernoulli_supplier)
vae.fit(mnist_train_images, n_iters=10000, batch_size=1000, hook_fn=print)
for perplexity in [30]:
for batch_size in [5000, 800, 400, 200]:
for run_id in range(20):
run_training(vae, n_latent_dimensions, perplexity, batch_size,
run_id)
for batch_size in [5000, 800, 400, 200]:
for run_id in range(20, 40):
run_training(vae, n_latent_dimensions, perplexity, batch_size,
run_id)
def run_training(n_latent_dimensions, perplexity, batch_size, percent_missing):
data_points = mnist_train_images.shape[0]
indices = np.random.choice(data_points, int(data_points * (1 - percent_missing)), replace=False)
train_data = mnist_train_images[indices]
train_labels = mnist_train_labels[indices]
test_data = mnist_test_images
test_labels = mnist_test_labels
vae = VAE(
[n_input_dimensions],
get_gaussian_network_builder(vae_encoder_layers, n_latent_dimensions),
gaussian_prior_supplier,
gaussian_supplier,
get_bernoulli_network_builder(vae_decoder_layers, n_input_dimensions, output_hidden=False),
bernoulli_supplier)
vptsne = VPTSNE(
vae,
get_feed_forward_network_builder(vptsne_layers, batch_normalization=False),
perplexity=perplexity)
vptsne2 = VPTSNE(
vae,
get_feed_forward_network_builder(vptsne_layers, batch_normalization=False),
perplexity=perplexity)
ptsne = PTSNE(
[n_input_dimensions],
get_feed_forward_network_builder(vptsne_layers, batch_normalization=False),
perplexity=perplexity)
def hook(args):
print(args)
if np.isnan(args[2]):
raise Exception
if isinstance(args[0], PTSNE) and args[2] <= 0.0:
raise Exception
fit_params = {
"hook_fn": hook,
"n_iters": 1500,
"batch_size": batch_size,
"deterministic": True,
"fit_vae": True,
"n_vae_iters": 8000,
"vae_batch_size": np.min([indices.shape[0], 1000])}
fit_params_2 = {
"hook_fn": hook,
"n_iters": 1500,
"batch_size": batch_size,
"deterministic": False,
"fit_vae": False}
vptsne.fit(train_data, **fit_params)
vptsne_knn_score = KNC(n_neighbors=1).fit(vptsne.transform(train_data), train_labels).score(vptsne.transform(test_data), test_labels)
vptsne_trustworthiness = trustworthiness(test_data, vptsne.transform(test_data), n_neighbors=12)
print(vptsne_knn_score, vptsne_trustworthiness)
vptsne2.fit(train_data, **fit_params_2)
vptsne2_knn_score = KNC(n_neighbors=1).fit(vptsne2.transform(train_data), train_labels).score(vptsne2.transform(test_data), test_labels)
vptsne2_trustworthiness = trustworthiness(test_data, vptsne2.transform(test_data), n_neighbors=12)
print(vptsne2_knn_score, vptsne2_trustworthiness)
ptsne.fit(train_data, **fit_params)
ptsne_knn_score = KNC(n_neighbors=1).fit(ptsne.transform(train_data), train_labels).score(ptsne.transform(test_data), test_labels)
ptsne_trustworthiness = trustworthiness(test_data, ptsne.transform(test_data), n_neighbors=12)
print(ptsne_knn_score, ptsne_trustworthiness)
return vptsne_knn_score, ptsne_knn_score, vptsne_trustworthiness, ptsne_trustworthiness, vptsne2_knn_score, vptsne2_trustworthiness
hidden_1 = tf.reshape(hidden_0, [-1, *encoder_hidden_3_shape])
hidden_2 = relu_bn(
tf.layers.conv2d_transpose(hidden_1, 128 * 4, 2, [1, 1]), vae.training)
hidden_3 = relu_bn(
tf.layers.conv2d_transpose(hidden_2, 128 * 2, 2, [1, 1]), vae.training)
hidden_4 = relu_bn(tf.layers.conv2d_transpose(hidden_3, 128, 2, [1, 1]),
vae.training)
hidden_5 = tf.nn.sigmoid(tf.layers.conv2d(hidden_4, 1, 1, [1, 1]))
return {"output": z, "probs": hidden_5}
if True:
vae = VAE([784],
get_gaussian_network_builder(vae_encoder_layers,
n_latent_dimensions),
gaussian_prior_supplier, gaussian_supplier,
get_bernoulli_network_builder(vae_decoder_layers,
n_input_dimensions),
bernoulli_supplier)
else:
vae = VAE([28, 28, 1],
encoder_network_builder,
gaussian_prior_supplier,
gaussian_supplier,
decoder_network_builder,
bernoulli_supplier,
learning_rate=0.00001)
fit_params = {
"hook_fn": print,
"n_iters": 1,
activation=tf.nn.relu)
hidden_1 = tf.reshape(hidden_0, [-1, *encoder_hidden_3_shape])
hidden_2 = relu_bn(
tf.layers.conv2d_transpose(hidden_1, 128 * 4, 2, [2, 2]), training)
hidden_3 = relu_bn(
tf.layers.conv2d_transpose(hidden_2, 128 * 2, 2, [2, 2]), training)
hidden_4 = relu_bn(tf.layers.conv2d_transpose(hidden_3, 128, 2, [2, 2]),
training)
hidden_5 = tf.nn.sigmoid(tf.layers.conv2d(hidden_4, 3, 1, [1, 1]))
return {"output": z, "probs": hidden_5}
vae = VAE(input_shape,
encoder_network_builder,
gaussian_prior_supplier,
gaussian_supplier,
decoder_network_builder,
bernoulli_supplier,
learning_rate=0.00001)
#vae.load_weights("models/svhn_vae.ckpt")
vae.fit(svhn_train, n_epochs=200, batch_size=1000, hook_fn=print)
vae.save_weights("models/svhn_vae.ckpt")
#display_reconstructions(10)
vptsne_layers = LayerDefinition.from_array([(250, tf.nn.relu),
(2500, tf.nn.relu), (2, None)])
vptsne = VPTSNE(vae,
get_feed_forward_network_builder(vptsne_layers),
learning_rate=0.00001,