def test_split_data(self):
x_labeled, y_labeled, x_unlabeled, y_unlabeled = HyperspectralData(
).split_data(self.train_x, self.train_y)
self.assertEqual(x_labeled.shape[1], y_labeled.shape[0])
self.assertEqual(x_unlabeled.shape[1], y_unlabeled.shape[0])
# Labeled + unlabeled = total:
self.assertEqual(self.n_train,
x_labeled.shape[1] + x_unlabeled.shape[1])
class TestHyperspectralData(TestCase):
n_train = 200
n_valid = 100
n_test = 150
train_x, train_y, valid_x, valid_y, test_x, test_y = HyperspectralData(
).load_numpy(n_train=n_train, n_valid=n_valid, n_test=n_test)
def test_load_numpy(self):
# Loaded the number of examples that we request:
self.assertEqual(self.train_x.shape[1], self.n_train)
self.assertEqual(self.valid_x.shape[1], self.n_valid)
self.assertEqual(self.test_x.shape[1], self.n_test)
# Labels has the same size that the dataset:
self.assertEqual(self.train_x.shape[1], self.train_y.shape[0])
self.assertEqual(self.valid_x.shape[1], self.valid_y.shape[0])
self.assertEqual(self.test_x.shape[1], self.test_y.shape[0])
def test_split_data(self):
x_labeled, y_labeled, x_unlabeled, y_unlabeled = HyperspectralData(
).split_data(self.train_x, self.train_y)
self.assertEqual(x_labeled.shape[1], y_labeled.shape[0])
self.assertEqual(x_unlabeled.shape[1], y_unlabeled.shape[0])
# Labeled + unlabeled = total:
self.assertEqual(self.n_train,
x_labeled.shape[1] + x_unlabeled.shape[1])
def test_to_one_hot(self):
# In mineralogy are 100 classes.
n_classes = 100
y_one_hot = HyperspectralData().to_one_hot(self.train_y, n_classes)
# It must have a row for each class:
self.assertEqual(n_classes, y_one_hot.shape[0])
# It must have a column for each label:
self.assertEqual(self.train_y.shape[0], y_one_hot.shape[1])
# It must have a 1 in the row indicated by train_y:
for i in range(y_one_hot.shape[1]): # Iterate over the columns
if (self.train_y[i] == 0
): # If it doesn't have a label the entire column must be zero
for j in range(y_one_hot.shape[0]
): # Iterate over the rows of the column i
self.assertEqual(0., y_one_hot[j, i])
else:
self.assertEqual(1., y_one_hot[self.train_y[i], i])
def test_to_one_hot(self):
# In mineralogy are 100 classes.
n_classes = 100
y_one_hot = HyperspectralData().to_one_hot(self.train_y, n_classes)
# It must have a row for each class:
self.assertEqual(n_classes, y_one_hot.shape[0])
# It must have a column for each label:
self.assertEqual(self.train_y.shape[0], y_one_hot.shape[1])
# It must have a 1 in the row indicated by train_y:
for i in range(y_one_hot.shape[1]): # Iterate over the columns
if (self.train_y[i] == 0
): # If it doesn't have a label the entire column must be zero
for j in range(y_one_hot.shape[0]
): # Iterate over the rows of the column i
self.assertEqual(0., y_one_hot[j, i])
else:
self.assertEqual(1., y_one_hot[self.train_y[i], i])
from hyperspectralData import HyperspectralData
import numpy as np
n_unlabeled = 1000
n_files = 10
n_max = 5000
x_l, y_l, x_u, y_u, valid_x, valid_y, test_x, test_y = HyperspectralData(
).load_dataset_m2(n_unlabeled=n_unlabeled, n_files=n_files, n_max=n_max)
print("Shapes:")
print("Train x labeled:", x_l.shape)
print(" y labaled:", y_l.shape)
print("Train x unlabeled:", x_u.shape)
print(" y unlabaled:", y_u.shape)
print("Valid x:", valid_x.shape)
print(" y:", valid_y.shape)
print("Test x:", test_x.shape)
print(" y:", test_y.shape)
def print_distribuiton(labels):
total_per_class = np.sum(labels, axis=1)
for i in range(100):
if total_per_class[i] != 0:
print("Label:", i, "Total:", total_per_class[i])
print('')
print("Train:")
print_distribuiton(y_l)
print('')
print("Valid:")
def main(n_z, n_hidden, dataset, seed, comment, gfx=True):
# Initialize logdir
# ---------------------
# Setasouto:
# Create the directory to save the outputs files and log.
# ---------------------
import time
logdir = 'results/gpulearn_z_x_' + dataset + '_' + str(n_z) + '-' + str(
n_hidden) + '_' + comment + '_' + str(int(time.time())) + '/'
if not os.path.exists(logdir): os.makedirs(logdir)
print('Logdir:', logdir)
np.random.seed(seed)
gfx_freq = 1
weight_decay = 0
f_enc, f_dec = lambda x: x, lambda x: x
# Init data
if dataset == 'hyper':
# Hyperspectral images:
# Import 1 file of the dataset
# TODO: import more files: Edit hyperspectralData.py
from hyperspectralData import HyperspectralData
n_samples = 100000
train_x, train_y, valid_x, valid_y, test_x, test_y = HyperspectralData(
).load_numpy(n_samples)
# Dim input: How it has to be written like an image. We said that is:
dim_input = (67, 4)
n_x = train_x.shape[0] # Dimension of our data vector.
x = {'x': train_x.astype(np.float32)}
x_valid = {'x': valid_x.astype(np.float32)}
x_test = {'x': test_x.astype(np.float32)}
# Hyperparameters:
L_valid = 1
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
nonlinear = 'softplus'
type_px = 'bernoulli'
n_train = train_x.shape[1]
n_batch = 1000
colorImg = False
bernoulli_x = False
byteToFloat = False
weight_decay = float(n_batch) / n_train
# Write the hyperparameters used:
with open(logdir + 'AA_hyperparameters.txt', 'w') as file:
file.write("L_valid: " + str(L_valid) + '\n')
file.write("type_qz: " + type_qz + '\n')
file.write("type_pz: " + type_pz + '\n')
file.write("Nonlinear: " + nonlinear + '\n')
file.write("type_px: " + type_px + '\n')
file.write("n_train: " + str(n_train) + '\n')
file.write("n_batch: " + str(n_batch) + '\n')
file.write("colorImg: " + str(colorImg) + '\n')
file.write("bernoulli_x: " + str(bernoulli_x) + '\n')
file.write("byteToFloat: " + str(byteToFloat) + '\n')
file.close()
# Write the headers for the csv file output:
with open(logdir + 'AA_results.txt', 'w') as file:
# Like a csv file:
file.write("Step" + ',' + "TimeElapsed" + ',' +
"LowerboundMinibatch" + ',' + "LowerboundValid" + ',' +
"NumStepNotImproving" + '\n')
file.close()
# Construct model
from anglepy.models import GPUVAE_Z_X
updates = get_adam_optimizer(learning_rate=3e-4, weight_decay=weight_decay)
model = GPUVAE_Z_X(updates,
n_x,
n_hidden,
n_z,
n_hidden[::-1],
nonlinear,
nonlinear,
type_px,
type_qz=type_qz,
type_pz=type_pz,
prior_sd=100,
init_sd=1e-3)
# Some statistics for optimization
ll_valid_stats = [-1e99, 0]
# Progress hook
def hook(epoch, t, ll):
"""
Documented by SetaSouto, may contains errors.
:epoch: Number of the current step.
:t: Time elapsed from the beginning.
:ll: Loglikelihood.
"""
if epoch % 10 != 0: return
ll_valid, _ = model.est_loglik(x_valid,
n_samples=L_valid,
n_batch=n_batch,
byteToFloat=byteToFloat)
# Saves the value of our actual net.
ndict.savez(ndict.get_value(model.v), logdir + 'v')
ndict.savez(ndict.get_value(model.w), logdir + 'w')
# If the actual ll of the validset is the best:
if ll_valid > ll_valid_stats[0]:
ll_valid_stats[0] = ll_valid
# Reset the numbers of iterations without improving:
ll_valid_stats[1] = 0
ndict.savez(ndict.get_value(model.v), logdir + 'v_best')
ndict.savez(ndict.get_value(model.w), logdir + 'w_best')
else:
ll_valid_stats[1] += 1
# Stop when not improving validation set performance in 100 iterations
if ll_valid_stats[1] > 100:
print("Finished")
with open(logdir + 'hook.txt', 'a') as f:
print(f, "Finished")
exit()
# This will be showing the current results and write them in a file:
with open(logdir + 'AA_results.txt', 'a') as file:
# Like a csv file:
file.write(
str(epoch) + ',' + str(t) + ',' + str(ll) + ',' +
str(ll_valid) + ',' + str(ll_valid_stats[1]) + '\n')
file.close()
print("-------------------------")
print("Current results:")
print(" ")
print("Step:", epoch)
print("Time elapsed:", t)
print("Loglikelihood minibatch:", ll)
print("Loglikelihood validSet:", ll_valid)
print("N not improving:", ll_valid_stats[1])
# Graphics: Generate images from the math
if gfx and epoch % gfx_freq == 0:
# tail = '.png'
tail = '-' + str(epoch) + '.png'
v = {i: model.v[i].get_value() for i in model.v}
w = {i: model.w[i].get_value() for i in model.w}
if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset:
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'q_w0' + tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_w' + tail, 'PNG')
if 'out_unif' in w:
image = paramgraphics.mat_to_img(f_dec(
w['out_unif'].reshape((-1, 1))),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_unif' + tail, 'PNG')
if n_z == 2:
n_width = 10
import scipy.stats
z = {'z': np.zeros((2, n_width**2))}
for i in range(0, n_width):
for j in range(0, n_width):
z['z'][0, n_width * i + j] = scipy.stats.norm.ppf(
float(i) / n_width + 0.5 / n_width)
z['z'][1, n_width * i + j] = scipy.stats.norm.ppf(
float(j) / n_width + 0.5 / n_width)
x, _, _z = model.gen_xz({}, z, n_width**2)
if dataset == 'mnist':
x = 1 - _z['x']
image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input)
image.save(logdir + '2dmanifold' + tail, 'PNG')
else:
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = _z_confab['x']
image = paramgraphics.mat_to_img(f_dec(x_samples),
dim_input,
colorImg=colorImg)
image.save(logdir + 'samples' + tail, 'PNG')
# x_samples = _x['x']
# image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg)
# image.save(logdir+'samples2'+tail, 'PNG')
else:
# Model with preprocessing
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'q_w0' + tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_w' + tail, 'PNG')
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = f_dec(_z_confab['x'])
x_samples = np.minimum(np.maximum(x_samples, 0), 1)
image = paramgraphics.mat_to_img(x_samples,
dim_input,
colorImg=colorImg)
image.save(logdir + 'samples' + tail, 'PNG')
# Optimize
# SFO
dostep = epoch_vae_adam(model,
x,
n_batch=n_batch,
bernoulli_x=bernoulli_x,
byteToFloat=byteToFloat)
loop_va(dostep, hook)
pass
def main(n_z, n_hidden, dataset, seed, comment, gfx=True):
# Initialize logdir
#---------------------
# Setasouto:
# Create the directory to save the outputs files and log.
#---------------------
import time
logdir = 'results/gpulearn_z_x_' + dataset + '_' + str(n_z) + '-' + str(
n_hidden) + '_' + comment + '_' + str(int(time.time())) + '/'
if not os.path.exists(logdir): os.makedirs(logdir)
print('logdir:', logdir)
print('gpulearn_z_x', n_z, n_hidden, dataset, seed)
with open(logdir + 'hook.txt', 'a') as f:
print(f, 'learn_z_x', n_z, n_hidden, dataset, seed)
np.random.seed(seed)
gfx_freq = 1
weight_decay = 0
f_enc, f_dec = lambda x: x, lambda x: x
# Init data
if dataset == 'mnist':
import anglepy.data.mnist as mnist
# MNIST
size = 28
train_x, train_y, valid_x, valid_y, test_x, test_y = mnist.load_numpy(
size)
x = {'x': train_x.astype(np.float32)}
x_valid = {'x': valid_x.astype(np.float32)}
x_test = {'x': test_x.astype(np.float32)}
L_valid = 1
dim_input = (size, size)
n_x = size * size
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
nonlinear = 'softplus'
type_px = 'bernoulli'
n_train = 50000
n_batch = 1000
colorImg = False
bernoulli_x = True
byteToFloat = False
weight_decay = float(n_batch) / n_train
if dataset == 'mnist_binarized':
import anglepy.data.mnist_binarized as mnist_binarized
# MNIST
train_x, valid_x, test_x = mnist_binarized.load_numpy(28)
x = {'x': np.hstack((train_x, valid_x)).astype(np.float32)}
x_valid = {'x': test_x.astype(np.float32)}
L_valid = 1
dim_input = (28, 28)
n_x = 28 * 28
n_y = 10
type_qz = 'gaussianmarg'
type_pz = 'mog'
nonlinear = 'rectlin'
type_px = 'bernoulli'
n_train = 60000
n_batch = 1000
colorImg = False
bernoulli_x = False
byteToFloat = False
weight_decay = float(n_batch) / n_train
elif dataset == 'freyface':
# Frey's face
import anglepy.data.freyface as freyface
n_train = 1600
train_x = freyface.load_numpy()
np.random.shuffle(train_x)
x = {'x': train_x.T[:, 0:n_train]}
x_valid = {'x': train_x.T[:, n_train:]}
L_valid = 1
dim_input = (28, 20)
n_x = 20 * 28
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
type_px = 'bounded01'
nonlinear = 'tanh' #tanh works better with freyface #'softplus'
n_batch = 100
colorImg = False
bernoulli_x = False
byteToFloat = False
weight_decay = float(n_batch) / n_train
elif dataset == 'freyface_pca':
# Frey's face
import anglepy.data.freyface as freyface
n_train = 1600
train_x = freyface.load_numpy().T
np.random.shuffle(train_x.T)
f_enc, f_dec, _ = pp.PCA(train_x, 0.99)
train_x = f_enc(train_x)
x = {'x': train_x[:, 0:n_train].astype(np.float32)}
x_valid = {'x': train_x[:, n_train:].astype(np.float32)}
L_valid = 1
dim_input = (28, 20)
n_x = train_x.shape[0]
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
type_px = 'gaussian'
nonlinear = 'softplus'
n_batch = 100
colorImg = False
bernoulli_x = False
byteToFloat = False
elif dataset == 'freyface_bernoulli':
# Frey's face
import anglepy.data.freyface as freyface
n_train = 1600
train_x = freyface.load_numpy().T
np.random.shuffle(train_x.T)
x = {'x': train_x[:, 0:n_train].astype(np.float32)}
x_valid = {'x': train_x[:, n_train:].astype(np.float32)}
L_valid = 1
dim_input = (28, 20)
n_x = train_x.shape[0]
type_pz = 'gaussianmarg'
type_px = 'bernoulli'
nonlinear = 'softplus'
n_batch = 100
colorImg = False
bernoulli_x = False
byteToFloat = False
elif dataset == 'norb':
# small NORB dataset
import anglepy.data.norb as norb
size = 48
train_x, train_y, test_x, test_y = norb.load_resized(size,
binarize_y=True)
x = {'x': train_x.astype(np.float32)}
x_valid = {'x': test_x.astype(np.float32)}
L_valid = 1
n_x = train_x.shape[0]
dim_input = (size, size)
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
type_px = 'gaussian'
nonlinear = 'softplus'
n_batch = 900 #23400/900 = 27
colorImg = False
#binarize = False
byteToFloat = False
bernoulli_x = False
weight_decay = float(n_batch) / train_x.shape[1]
elif dataset == 'norb_pca':
# small NORB dataset
import anglepy.data.norb as norb
size = 48
train_x, train_y, test_x, test_y = norb.load_resized(size,
binarize_y=True)
f_enc, f_dec, _ = pp.PCA(train_x, 0.999)
#f_enc, f_dec, _ = pp.normalize_random(train_x)
train_x = f_enc(train_x)
test_x = f_enc(test_x)
x = {'x': train_x.astype(np.float32)}
x_valid = {'x': test_x.astype(np.float32)}
L_valid = 1
n_x = train_x.shape[0]
dim_input = (size, size)
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
type_px = 'gaussian'
nonlinear = 'softplus'
n_batch = 900 #23400/900 = 27
colorImg = False
#binarize = False
bernoulli_x = False
byteToFloat = False
weight_decay = float(n_batch) / train_x.shape[1]
elif dataset == 'norb_normalized':
# small NORB dataset
import anglepy.data.norb as norb
size = 48
train_x, train_y, test_x, test_y = norb.load_resized(size,
binarize_y=True)
#f_enc, f_dec, _ = pp.PCA(train_x, 0.99)
#f_enc, f_dec, _ = pp.normalize_random(train_x)
f_enc, f_dec, _ = pp.normalize(train_x)
train_x = f_enc(train_x)
test_x = f_enc(test_x)
x = {'x': train_x.astype(np.float32)}
x_valid = {'x': test_x.astype(np.float32)}
L_valid = 1
n_x = train_x.shape[0]
dim_input = (size, size)
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
type_px = 'gaussian'
nonlinear = 'softplus'
n_batch = 900 #23400/900 = 27
colorImg = False
#binarize = False
bernoulli_x = False
byteToFloat = False
weight_decay = float(n_batch) / train_x.shape[1]
elif dataset == 'svhn':
# SVHN dataset
import anglepy.data.svhn as svhn
size = 32
train_x, train_y, test_x, test_y = svhn.load_numpy(
False, binarize_y=True) #norb.load_resized(size, binarize_y=True)
extra_x, extra_y = svhn.load_numpy_extra(False, binarize_y=True)
x = {
'x': np.hstack((train_x, extra_x)),
'y': np.hstack((train_y, extra_y))
}
ndict.shuffleCols(x)
print('Performing PCA, can take a few minutes... ',
f_enc,
f_dec,
pca_params=pp.PCA(x['x'][:, :10000], cutoff=600, toFloat=True))
ndict.savez(pca_params, logdir + 'pca_params')
print('Done.')
n_y = 10
x = {'x': f_enc(x['x']).astype(np.float32)}
x_valid = {'x': f_enc(test_x).astype(np.float32)}
L_valid = 1
n_x = x['x'].shape[0]
dim_input = (size, size)
n_batch = 5000
colorImg = True
bernoulli_x = False
byteToFloat = False
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
type_px = 'gaussian'
nonlinear = 'softplus'
elif dataset == 'hyper':
# Hyperspectral images:
# Import 1 file of the dataset
# TODO: import more files: Edit hyperspectralData.py
#I added the hyperspectralData file in the anglepy library
from hyperspectralData import HyperspectralData
train_x, train_y, valid_x, valid_y, test_x, test_y = HyperspectralData(
).load_numpy(100000)
#Dim input: How it has to be written like an image. We said that is:
dim_input = (67, 4)
n_x = train_x.shape[0] #Dimension of our data vector.
x = {'x': train_x.astype(np.float32)}
x_valid = {'x': valid_x.astype(np.float32)}
x_test = {'x': test_x.astype(np.float32)}
L_valid = 1
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
nonlinear = 'softplus'
type_px = 'bernoulli'
n_train = train_x.shape[1]
n_batch = 1000
colorImg = False
bernoulli_x = False
byteToFloat = False
weight_decay = float(n_batch) / n_train
#Write the hyperparameters used:
with open(logdir + 'AA_hyperparameters.txt', 'w') as file:
file.write("L_valid: " + str(L_valid) + '\n')
file.write("type_qz: " + type_qz + '\n')
file.write("type_pz: " + type_pz + '\n')
file.write("Nonlinear: " + nonlinear + '\n')
file.write("type_px: " + type_px + '\n')
file.write("n_train: " + str(n_train) + '\n')
file.write("n_batch: " + str(n_batch) + '\n')
file.write("colorImg: " + str(colorImg) + '\n')
file.write("bernoulli_x: " + str(bernoulli_x) + '\n')
file.write("byteToFloat: " + str(byteToFloat) + '\n')
file.close()
# Write the headers for the csv file output:
with open(logdir + 'AA_results.txt', 'w') as file:
# Like a csv file:
file.write("Step" + ',' + "TimeElapsed" + ',' +
"LowerboundMinibatch" + ',' + "LowerboundValid" + ',' +
"NumStepNotImproving" + '\n')
file.close()
# Construct model
from anglepy.models import GPUVAE_Z_X
updates = get_adam_optimizer(learning_rate=3e-4, weight_decay=weight_decay)
model = GPUVAE_Z_X(updates,
n_x,
n_hidden,
n_z,
n_hidden[::-1],
nonlinear,
nonlinear,
type_px,
type_qz=type_qz,
type_pz=type_pz,
prior_sd=100,
init_sd=1e-3)
#---------------
# SetaSouto:
# The [::-1] is to reverse the list.
#---------------
if False:
#dir = '/Users/dpkingma/results/learn_z_x_mnist_binarized_50-(500, 500)_mog_1412689061/'
#dir = '/Users/dpkingma/results/learn_z_x_svhn_bernoulli_300-(1000, 1000)_l1l2_sharing_and_1000HU_1412676966/'
#dir = '/Users/dpkingma/results/learn_z_x_svhn_bernoulli_300-(1000, 1000)_l1l2_sharing_and_1000HU_1412695481/'
#dir = '/Users/dpkingma/results/learn_z_x_mnist_binarized_50-(500, 500)_mog_1412695455/'
#dir = '/Users/dpkingma/results/gpulearn_z_x_svhn_pca_300-(500, 500)__1413904756/'
dir = '/home/ubuntu/results/gpulearn_z_x_mnist_50-[500, 500]__1414259423/'
w = ndict.loadz(dir + 'w_best.ndict.tar.gz')
v = ndict.loadz(dir + 'v_best.ndict.tar.gz')
ndict.set_value(model.w, w)
ndict.set_value(model.v, v)
# Some statistics for optimization
ll_valid_stats = [-1e99, 0]
# Progress hook
def hook(epoch, t, ll):
'''
Documented by SetaSouto, may contains errors.
:epoch: Number of the current step.
:t: Time elapsed from the beginning.
:ll: Loglikelihood (?).
'''
if epoch % 10 != 0: return
ll_valid, _ = model.est_loglik(x_valid,
n_samples=L_valid,
n_batch=n_batch,
byteToFloat=byteToFloat)
# Log
ndict.savez(ndict.get_value(model.v), logdir + 'v')
ndict.savez(ndict.get_value(model.w), logdir + 'w')
if ll_valid > ll_valid_stats[0]:
ll_valid_stats[0] = ll_valid
ll_valid_stats[1] = 0
ndict.savez(ndict.get_value(model.v), logdir + 'v_best')
ndict.savez(ndict.get_value(model.w), logdir + 'w_best')
else:
ll_valid_stats[1] += 1
# Stop when not improving validation set performance in 100 iterations
if ll_valid_stats[1] > 100:
print("Finished")
with open(logdir + 'hook.txt', 'a') as f:
print(f, "Finished")
exit()
# This will be showing the current results and write them in a file:
with open(logdir + 'AA_results.txt', 'a') as file:
# Like a csv file:
file.write(
str(epoch) + ',' + str(t) + ',' + str(ll) + ',' +
str(ll_valid) + ',' + str(ll_valid_stats[1]) + '\n')
file.close()
print("-------------------------")
print("Current results:")
print(" ")
print("Step:", epoch)
print("Time elapsed:", t)
print("Loglikelihood minibatch:", ll)
print("Loglikelihood validSet:", ll_valid)
print("N not improving:", ll_valid_stats[1])
#print(epoch, t, ll, ll_valid, ll_valid_stats)
#This print the file where are written the stats.
#with open(logdir+'hook.txt', 'a') as f:
#print(f, epoch, t, ll, ll_valid, ll_valid_stats)
# Graphics
if gfx and epoch % gfx_freq == 0:
#tail = '.png'
tail = '-' + str(epoch) + '.png'
v = {i: model.v[i].get_value() for i in model.v}
w = {i: model.w[i].get_value() for i in model.w}
if 'pca' not in dataset and 'random' not in dataset and 'normalized' not in dataset:
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'q_w0' + tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_w' + tail, 'PNG')
if 'out_unif' in w:
image = paramgraphics.mat_to_img(f_dec(
w['out_unif'].reshape((-1, 1))),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_unif' + tail, 'PNG')
if n_z == 2:
n_width = 10
import scipy.stats
z = {'z': np.zeros((2, n_width**2))}
for i in range(0, n_width):
for j in range(0, n_width):
z['z'][0, n_width * i + j] = scipy.stats.norm.ppf(
float(i) / n_width + 0.5 / n_width)
z['z'][1, n_width * i + j] = scipy.stats.norm.ppf(
float(j) / n_width + 0.5 / n_width)
x, _, _z = model.gen_xz({}, z, n_width**2)
if dataset == 'mnist':
x = 1 - _z['x']
image = paramgraphics.mat_to_img(f_dec(_z['x']), dim_input)
image.save(logdir + '2dmanifold' + tail, 'PNG')
else:
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = _z_confab['x']
image = paramgraphics.mat_to_img(f_dec(x_samples),
dim_input,
colorImg=colorImg)
image.save(logdir + 'samples' + tail, 'PNG')
#x_samples = _x['x']
#image = paramgraphics.mat_to_img(x_samples, dim_input, colorImg=colorImg)
#image.save(logdir+'samples2'+tail, 'PNG')
else:
# Model with preprocessing
if 'w0' in v:
image = paramgraphics.mat_to_img(f_dec(v['w0'][:].T),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'q_w0' + tail, 'PNG')
image = paramgraphics.mat_to_img(f_dec(w['out_w'][:]),
dim_input,
True,
colorImg=colorImg)
image.save(logdir + 'out_w' + tail, 'PNG')
_x, _, _z_confab = model.gen_xz({}, {}, n_batch=144)
x_samples = f_dec(_z_confab['x'])
x_samples = np.minimum(np.maximum(x_samples, 0), 1)
image = paramgraphics.mat_to_img(x_samples,
dim_input,
colorImg=colorImg)
image.save(logdir + 'samples' + tail, 'PNG')
# Optimize
#SFO
dostep = epoch_vae_adam(model,
x,
n_batch=n_batch,
bernoulli_x=bernoulli_x,
byteToFloat=byteToFloat)
loop_va(dostep, hook)
pass
l1_model = VAE_Z_X(n_x=67 * 4,
n_hidden_q=n_h,
n_z=50,
n_hidden_p=n_h,
nonlinear_q='softplus',
nonlinear_p='softplus',
type_px='bernoulli',
type_qz='gaussianmarg',
type_pz='gaussianmarg',
prior_sd=1)
# Now we have to load the dataset that we wanna use.
from hyperspectralData import HyperspectralData
nsamples = 100
train_x, train_y, valid_x, valid_y, test_x, test_y = HyperspectralData(
).load_numpy(nsamples)
# Get the mean and the variance of the distribution learned to generate the z of the dataset.
def transform(v, _x):
# Receives the values of the net and the data
return l1_model.dist_qz['z'](*([_x] + list(v.values()) +
[np.ones((1, _x.shape[1]))]))
# Extract the parameters of the distributions Z (features):
# For example, from the train train_x:
x_mean, x_logvar = transform(l1_v, train_x)
# And now, the features are obtained by sampling the distributions:
features = np.random.normal(x_mean, np.exp(0.5 * x_logvar))
def main(n_passes, n_hidden, seed, alpha, n_minibatches, n_unlabeled,
n_classes):
"""
Learn a variational auto-encoder with generative model p(x,y,z)=p(y)p(z)p(x|y,z)
And where 'x' is always observed and 'y' is _sometimes_ observed (hence semi-supervised).
We're going to use q(y|x) as a classification model.
"""
# Create the directory for the log and outputs.
logdir = 'results/learn_yz_x_hyp' + '-' + str(int(time.time())) + '/'
if not os.path.exists(logdir): os.makedirs(logdir)
print("---------------")
print('Logdir:', logdir)
# Feed with the seed:
np.random.seed(seed)
# Load model for feature extraction
path = 'results/hyper_50-(500, 500)_longrun/'
# Load the parameters of the model that has been trained previously:
l1_v = ndict.loadz(path + 'v_best.ndict.tar.gz')
l1_w = ndict.loadz(path + 'w_best.ndict.tar.gz')
# Same hyperparameters that we use for training M1:
# Number of hidden nodes in the model:
n_h = (500, 500)
# Size of our feature vector:
n_x = 67 * 4
# Number of latent variables:
n_z = 50
nonlinear = 'softplus'
type_px = 'bernoulli'
type_qz = 'gaussianmarg'
type_pz = 'gaussianmarg'
# Create the M1:
from anglepy.models.VAE_Z_X import VAE_Z_X
l1_model = VAE_Z_X(n_x=n_x,
n_hidden_q=n_h,
n_z=n_z,
n_hidden_p=n_h,
nonlinear_q=nonlinear,
nonlinear_p=nonlinear,
type_px=type_px,
type_qz=type_qz,
type_pz=type_pz,
prior_sd=1)
# Load dataset:
from hyperspectralData import HyperspectralData
x_l, y_l, x_u, y_u, valid_x, valid_y, test_x, test_y = HyperspectralData(
).load_dataset_m2(n_unlabeled=n_unlabeled, n_classes=n_classes)
n_labeled = x_l.shape[1]
if n_labeled % n_minibatches != 0:
# We need to delete some samples
indexes_to_delete = np.random.choice(range(n_labeled),
size=(n_labeled % n_minibatches),
replace=False)
x_l = np.delete(x_l, indexes_to_delete, axis=1)
y_l = np.delete(y_l, indexes_to_delete, axis=1)
# Extract features
def transform(v, _x):
# Get the mean and the variance of the distribution learned to generate the z of the dataset.
return l1_model.dist_qz['z'](*([_x] + list(v.values()) +
[np.ones((1, _x.shape[1]))]))
# 3. Extract features
x_mean_u, x_logvar_u = transform(l1_v, x_u)
x_mean_l, x_logvar_l = transform(l1_v, x_l)
x_unlabeled = {'mean': x_mean_u, 'logvar': x_logvar_u, 'y': y_u}
x_labeled = {'mean': x_mean_l, 'logvar': x_logvar_l, 'y': y_l}
valid_x, _ = transform(l1_v, valid_x)
test_x, _ = transform(l1_v, test_x)
# Copied from learn_yz_x_ss:
n_x = l1_w[b'w0'].shape[1]
n_y = n_classes
type_pz = 'gaussianmarg'
type_px = 'gaussian'
nonlinear = 'softplus'
# Init VAE model p(x,y,z)
from anglepy.models.VAE_YZ_X import VAE_YZ_X
uniform_y = True
model = VAE_YZ_X(n_x,
n_y,
n_hidden,
n_z,
n_hidden,
nonlinear,
nonlinear,
type_px,
type_qz=type_qz,
type_pz=type_pz,
prior_sd=1,
uniform_y=uniform_y)
v, w = model.init_w(1e-3)
# Init q(y|x) model
from anglepy.models.MLP_Categorical import MLP_Categorical
n_units = [n_x] + list(n_hidden) + [n_y]
model_qy = MLP_Categorical(n_units=n_units,
prior_sd=1,
nonlinearity=nonlinear)
u = model_qy.init_w(1e-3)
write_headers(logdir)
# Progress hook
t0 = time.time()
def hook(step, u, v, w, ll):
print("---------------")
print("Current results:")
print("Step:", step)
print(" ")
# Get classification error of validation and test sets
def error(dataset_x, dataset_y):
_, _, _z = model_qy.gen_xz(u, {'x': dataset_x}, {})
n_examples = 20
max_row = dataset_y.shape[1]
example_rows = np.random.choice(max_row,
size=n_examples,
replace=False)
print(" Predictions:", np.argmax(_z['py'], axis=0)[example_rows])
print(" Real: ", np.argmax(dataset_y, axis=0)[example_rows])
return np.sum(
np.argmax(_z['py'], axis=0) != np.argmax(dataset_y, axis=0)
) / (0.0 + dataset_y.shape[1])
print("Validset:")
valid_error = error(valid_x, valid_y)
print("Testset:")
test_error = error(test_x, test_y)
# Save variables
ndict.savez(u, logdir + 'u')
ndict.savez(v, logdir + 'v')
ndict.savez(w, logdir + 'w')
time_elapsed = time.time() - t0
# This will be showing the current results and write them in a file:
with open(logdir + 'AA_results.txt', 'a') as file:
file.write(
str(step) + ',' + str(time_elapsed) + ',' + str(valid_error) +
',' + str(test_error) + '\n')
print("Time elapsed:", time_elapsed)
print("Validset error:", valid_error)
print("Testset error:", test_error)
print("LogLikelihood:", ll)
return valid_error
# Optimize
result = optim_vae_ss_adam(alpha,
model_qy,
model,
x_labeled,
x_unlabeled,
n_y,
u,
v,
w,
n_minibatches=n_minibatches,
n_passes=n_passes,
hook=hook)
return result