def run_training(corruption_chance, perplexity, batch_size): global train_data, test_data corrupt = lambda x: 0 if np.random.uniform() <= corruption_chance else x train_data = np.vectorize(corrupt)(train_data) test_data = np.vectorize(corrupt)(test_data) def hook(args): print(args) if np.isnan(args[2]): raise Exception if isinstance(args[0], PTSNE) and args[2] <= 0.0: raise Exception vae = VAE( [n_input_dimensions], get_gaussian_network_builder(vae_encoder_layers, n_latent_dimensions), gaussian_prior_supplier, gaussian_supplier, get_bernoulli_network_builder(vae_decoder_layers, n_input_dimensions), bernoulli_supplier) ptsne = PTSNE( [n_input_dimensions], get_feed_forward_network_builder(vptsne_layers, batch_normalization=False), perplexity=perplexity) vptsne = VPTSNE( vae, get_feed_forward_network_builder(vptsne_layers, batch_normalization=False), perplexity=perplexity) ptsne.fit(train_data, n_iters=1500, batch_size=batch_size, hook_fn=hook) vptsne.fit(train_data, n_iters=1500, n_vae_iters=10000, batch_size=batch_size, vae_batch_size=1000, hook_fn=hook) knn_score = KNC(n_neighbors=1).fit( ptsne.transform(train_data), train_labels).score( ptsne.transform(test_data), test_labels) knn_score_vptsne = KNC(n_neighbors=1).fit( vptsne.transform(train_data), train_labels).score( vptsne.transform(test_data), test_labels) tw = trustworthiness( test_data, ptsne.transform(test_data), n_neighbors=12) tw_vptsne = trustworthiness( test_data, vptsne.transform(test_data), n_neighbors=12) train_data = np.copy(non_corrupted_train_data) test_data = np.copy(non_corrupted_test_data) return knn_score, tw, knn_score_vptsne, tw_vptsne
print(np.var(distances(mnist.test._images[:1000]))) n_input_dimensions = mnist.train._images.shape[1] n_latent_dimensions = 3 vae_layer_definitions = [(256, tf.nn.relu), (128, tf.nn.relu), (32, tf.nn.relu)] vae_encoder_layers = LayerDefinition.from_array(vae_layer_definitions) vae_decoder_layers = LayerDefinition.from_array( reversed(vae_layer_definitions)) vae = VAE([n_input_dimensions], get_gaussian_network_builder(vae_encoder_layers, n_latent_dimensions), gaussian_prior_supplier, gaussian_supplier, get_bernoulli_network_builder(vae_decoder_layers, n_input_dimensions), bernoulli_supplier) vae.fit(mnist.train._images, n_iters=14000, batch_size=1000, hook_fn=print) from sklearn.neighbors import KNeighborsClassifier as KNC print( "1-NN, test set", KNC(n_neighbors=1).fit(vae.transform(mnist.train._images), mnist.train._labels).score( vae.transform(mnist.test._images), mnist.test._labels))
color_palette = np.random.rand(100, 3) data, labels = hdata.load_all("CYTOMETRY_data") print(data.shape, labels.shape) indices = np.random.permutation(data.shape[0]) number_of_points = int(data.shape[0] * 0.01) subset_indices = indices[:number_of_points] n_input_dimensions = data.shape[1] n_latent_dimensions = 5 vae = VAE([n_input_dimensions], get_gaussian_network_builder(vae_encoder_layers, n_latent_dimensions), gaussian_prior_supplier, gaussian_supplier, get_gaussian_network_builder(vae_decoder_layers, n_input_dimensions, constant_sigma=0.1), gaussian_supplier) fit_params = { "hook_fn": print, "n_iters": 1500, "batch_size": 5000, "deterministic": True, "fit_vae": True, "n_vae_iters": 10000, "vae_batch_size": 1000 } vptsne = VPTSNE(vae,
vptsne.save_weights("models/mnist_vptsne_deterministic_%s.ckpt" % info, "models/mnist_vae_deterministic_%s.ckpt" % info) ptsne.save_weights("models/mnist_vptsne_nondeterministic_%s.ckpt" % info) for f in vptsne_log_files: f.close() for f in ptsne_log_files: f.close() if __name__ == "__main__": vae = VAE([n_input_dimensions], get_gaussian_network_builder(vae_encoder_layers, n_latent_dimensions), gaussian_prior_supplier, gaussian_supplier, get_bernoulli_network_builder(vae_decoder_layers, n_input_dimensions), bernoulli_supplier) vae.fit(mnist_train_images, n_iters=10000, batch_size=1000, hook_fn=print) for perplexity in [30]: for batch_size in [5000, 800, 400, 200]: for run_id in range(20): run_training(vae, n_latent_dimensions, perplexity, batch_size, run_id) for batch_size in [5000, 800, 400, 200]: for run_id in range(20, 40): run_training(vae, n_latent_dimensions, perplexity, batch_size, run_id)
def run_training(n_latent_dimensions, perplexity, batch_size, percent_missing): data_points = mnist_train_images.shape[0] indices = np.random.choice(data_points, int(data_points * (1 - percent_missing)), replace=False) train_data = mnist_train_images[indices] train_labels = mnist_train_labels[indices] test_data = mnist_test_images test_labels = mnist_test_labels vae = VAE( [n_input_dimensions], get_gaussian_network_builder(vae_encoder_layers, n_latent_dimensions), gaussian_prior_supplier, gaussian_supplier, get_bernoulli_network_builder(vae_decoder_layers, n_input_dimensions, output_hidden=False), bernoulli_supplier) vptsne = VPTSNE( vae, get_feed_forward_network_builder(vptsne_layers, batch_normalization=False), perplexity=perplexity) vptsne2 = VPTSNE( vae, get_feed_forward_network_builder(vptsne_layers, batch_normalization=False), perplexity=perplexity) ptsne = PTSNE( [n_input_dimensions], get_feed_forward_network_builder(vptsne_layers, batch_normalization=False), perplexity=perplexity) def hook(args): print(args) if np.isnan(args[2]): raise Exception if isinstance(args[0], PTSNE) and args[2] <= 0.0: raise Exception fit_params = { "hook_fn": hook, "n_iters": 1500, "batch_size": batch_size, "deterministic": True, "fit_vae": True, "n_vae_iters": 8000, "vae_batch_size": np.min([indices.shape[0], 1000])} fit_params_2 = { "hook_fn": hook, "n_iters": 1500, "batch_size": batch_size, "deterministic": False, "fit_vae": False} vptsne.fit(train_data, **fit_params) vptsne_knn_score = KNC(n_neighbors=1).fit(vptsne.transform(train_data), train_labels).score(vptsne.transform(test_data), test_labels) vptsne_trustworthiness = trustworthiness(test_data, vptsne.transform(test_data), n_neighbors=12) print(vptsne_knn_score, vptsne_trustworthiness) vptsne2.fit(train_data, **fit_params_2) vptsne2_knn_score = KNC(n_neighbors=1).fit(vptsne2.transform(train_data), train_labels).score(vptsne2.transform(test_data), test_labels) vptsne2_trustworthiness = trustworthiness(test_data, vptsne2.transform(test_data), n_neighbors=12) print(vptsne2_knn_score, vptsne2_trustworthiness) ptsne.fit(train_data, **fit_params) ptsne_knn_score = KNC(n_neighbors=1).fit(ptsne.transform(train_data), train_labels).score(ptsne.transform(test_data), test_labels) ptsne_trustworthiness = trustworthiness(test_data, ptsne.transform(test_data), n_neighbors=12) print(ptsne_knn_score, ptsne_trustworthiness) return vptsne_knn_score, ptsne_knn_score, vptsne_trustworthiness, ptsne_trustworthiness, vptsne2_knn_score, vptsne2_trustworthiness
hidden_1 = tf.reshape(hidden_0, [-1, *encoder_hidden_3_shape]) hidden_2 = relu_bn( tf.layers.conv2d_transpose(hidden_1, 128 * 4, 2, [1, 1]), vae.training) hidden_3 = relu_bn( tf.layers.conv2d_transpose(hidden_2, 128 * 2, 2, [1, 1]), vae.training) hidden_4 = relu_bn(tf.layers.conv2d_transpose(hidden_3, 128, 2, [1, 1]), vae.training) hidden_5 = tf.nn.sigmoid(tf.layers.conv2d(hidden_4, 1, 1, [1, 1])) return {"output": z, "probs": hidden_5} if True: vae = VAE([784], get_gaussian_network_builder(vae_encoder_layers, n_latent_dimensions), gaussian_prior_supplier, gaussian_supplier, get_bernoulli_network_builder(vae_decoder_layers, n_input_dimensions), bernoulli_supplier) else: vae = VAE([28, 28, 1], encoder_network_builder, gaussian_prior_supplier, gaussian_supplier, decoder_network_builder, bernoulli_supplier, learning_rate=0.00001) fit_params = { "hook_fn": print, "n_iters": 1,
activation=tf.nn.relu) hidden_1 = tf.reshape(hidden_0, [-1, *encoder_hidden_3_shape]) hidden_2 = relu_bn( tf.layers.conv2d_transpose(hidden_1, 128 * 4, 2, [2, 2]), training) hidden_3 = relu_bn( tf.layers.conv2d_transpose(hidden_2, 128 * 2, 2, [2, 2]), training) hidden_4 = relu_bn(tf.layers.conv2d_transpose(hidden_3, 128, 2, [2, 2]), training) hidden_5 = tf.nn.sigmoid(tf.layers.conv2d(hidden_4, 3, 1, [1, 1])) return {"output": z, "probs": hidden_5} vae = VAE(input_shape, encoder_network_builder, gaussian_prior_supplier, gaussian_supplier, decoder_network_builder, bernoulli_supplier, learning_rate=0.00001) #vae.load_weights("models/svhn_vae.ckpt") vae.fit(svhn_train, n_epochs=200, batch_size=1000, hook_fn=print) vae.save_weights("models/svhn_vae.ckpt") #display_reconstructions(10) vptsne_layers = LayerDefinition.from_array([(250, tf.nn.relu), (2500, tf.nn.relu), (2, None)]) vptsne = VPTSNE(vae, get_feed_forward_network_builder(vptsne_layers), learning_rate=0.00001,