def points_to_imagevecs(points, decoder_params, decoder=gaussian_decoder):
decode = decoder(decoder_params)
vals = decode(points)
out = vals[0] if isinstance(vals, tuple) else vals
if not isinstance(vals, np.ndarray):
out = out.eval()
return out
def init_vae(group):
x, y = read_mfcc_data(group)
x, y = shuffle(x, y)
x_train, x_test, y_train, y_test = train_test_split(x,
y,
test_size=0.2,
random_state=True,
shuffle=True)
# reshape to (28, 28, 1) and normalize input images
image_shape = x_train.shape
x_train = np.reshape(
x_train, [-1, x_train.shape[1], x_train.shape[2], x_train.shape[3]])
x_test = np.reshape(
x_test, [-1, x_test.shape[1], x_test.shape[2], x_test.shape[3]])
x_train = x_train.astype('float32') / 255
x_test = x_test.astype('float32') / 255
lable_color_dict = {}
for p in list(set(y)):
lable_color_dict[p] = "#" + ''.join(
[random.choice('0123456789ABCDEF') for j in range(6)])
# network parameters
input_shape = (x_train.shape[1], x_train.shape[2], x_train.shape[3])
inputs = Input(shape=input_shape, name='encoder_input')
encoder_t, shape, z_log_var, z_mean = encoder(inputs, latent_dim, filters)
decoder_t = decoder(latent_dim, shape, filters, kernel_size)
vae_t = vae(inputs, encoder_t, decoder_t, image_shape, z_log_var, z_mean)
return vae_t, encoder_t, decoder_t, x, y, x_train, x_test, y_test, lable_color_dict, group
def __init__(self, nin, nhid, nout, hebb='h,z,r', ln=True):
super(PlasticPixelPredictor, self).__init__()
if params['rnn_type'] == 'GRU':
self.rnn = GRU(nin, nhid, hebb=set(hebb.split(',')), ln=True)
else:
self.rnn = RNN(nin,
nhid,
hebb=params['type'] == 'plastic',
ln=True)
self.out = nn.Linear(nhid, nout)
self.enc = vae.encoder()
self.dec = vae.decoder()
def __init__(self):
super(NETWORK, self).__init__()
# Notice that the vectors are row vectors, and the matrices are transposed wrt the usual order, following apparent pytorch conventions
# Each *column* of w targets a single output neuron
self.nhid = 300
self.w0 = Variable(0.01 * torch.randn(3, self.nhid).type(ttype),
requires_grad=True)
self.b0 = Variable(0.01 * torch.randn(self.nhid).type(ttype),
requires_grad=True)
self.w = Variable(
.01 * torch.eye(self.nhid, self.nhid).type(ttype),
requires_grad=True) # The matrix of fixed (baseline) weights
self.w1 = Variable(0.01 *
torch.randn(self.nhid, self.nhid).type(ttype),
requires_grad=True)
self.b1 = Variable(0.01 * torch.randn(self.nhid).type(ttype),
requires_grad=True)
self.alpha = Variable(
.01 * torch.randn(self.nhid, self.nhid).type(ttype),
requires_grad=True) # The matrix of plasticity coefficients
self.eta = Variable(
.01 * torch.ones(1).type(ttype), requires_grad=True
) # The weight decay term / "learning rate" of plasticity - trainable, but shared across all connections
self.pred_eta = Variable(
.01 * torch.randn(self.nhid, self.nhid**1).type(ttype),
requires_grad=True)
self.pred_eta_b = Variable(.01 * torch.randn(self.nhid).type(ttype),
requires_grad=True)
self.ln = torch.nn.LayerNorm(self.nhid).type(ttype)
self.enc = vae.encoder().type(ttype)
self.dec = vae.decoder().type(ttype)
self.eta_hat = 0
def main(args):
""" parameters """
RESULTS_DIR = ss.path.DATADIR + "vae/" + args.results_path
# network architecture
ADD_NOISE = args.add_noise
n_hidden = args.n_hidden
dim_img = IMAGE_SIZE_MNIST**2 # number of pixels for a MNIST image
dim_z = args.dim_z
# train
n_epochs = args.num_epochs
batch_size = args.batch_size
learn_rate = args.learn_rate
# Plot
PRR = args.PRR # Plot Reproduce Result
PRR_n_img_x = args.PRR_n_img_x # number of images along x-axis in a canvas
PRR_n_img_y = args.PRR_n_img_y # number of images along y-axis in a canvas
PRR_resize_factor = args.PRR_resize_factor # resize factor for each image in a canvas
PMLR = args.PMLR # Plot Manifold Learning Result
PMLR_n_img_x = args.PMLR_n_img_x # number of images along x-axis in a canvas
PMLR_n_img_y = args.PMLR_n_img_y # number of images along y-axis in a canvas
PMLR_resize_factor = args.PMLR_resize_factor # resize factor for each image in a canvas
PMLR_z_range = args.PMLR_z_range # range for random latent vector
PMLR_n_samples = args.PMLR_n_samples # number of labeled samples to plot a map from input data space to the latent space
""" prepare MNIST data """
train_total_data, train_size, _, _, test_data, test_labels = mnist_data.prepare_MNIST_data(
)
n_samples = train_size
""" build graph """
# input placeholders
# In denoising-autoencoder, x_hat == x + noise, otherwise x_hat == x
x_hat = tf.placeholder(tf.float32, shape=[None, dim_img], name='input_img')
x = tf.placeholder(tf.float32, shape=[None, dim_img], name='target_img')
# dropout
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
# input for PMLR
z_in = tf.placeholder(tf.float32,
shape=[None, dim_z],
name='latent_variable')
# network architecture
y, z, loss, neg_marginal_likelihood, KL_divergence = vae.autoencoder(
x_hat, x, dim_img, dim_z, n_hidden, keep_prob)
# optimization
train_op = tf.train.AdamOptimizer(learn_rate).minimize(loss)
""" training """
# Plot for reproduce performance
if PRR:
PRR = plot_utils.Plot_Reproduce_Performance(RESULTS_DIR, PRR_n_img_x,
PRR_n_img_y,
IMAGE_SIZE_MNIST,
IMAGE_SIZE_MNIST,
PRR_resize_factor)
x_PRR = test_data[0:PRR.n_tot_imgs, :]
x_PRR_img = x_PRR.reshape(PRR.n_tot_imgs, IMAGE_SIZE_MNIST,
IMAGE_SIZE_MNIST)
PRR.save_images(x_PRR_img, name='input.jpg')
if ADD_NOISE:
x_PRR = x_PRR * np.random.randint(2, size=x_PRR.shape)
x_PRR += np.random.randint(2, size=x_PRR.shape)
x_PRR_img = x_PRR.reshape(PRR.n_tot_imgs, IMAGE_SIZE_MNIST,
IMAGE_SIZE_MNIST)
PRR.save_images(x_PRR_img, name='input_noise.jpg')
# Plot for manifold learning result
if PMLR and dim_z == 2:
PMLR = plot_utils.Plot_Manifold_Learning_Result(
RESULTS_DIR, PMLR_n_img_x, PMLR_n_img_y, IMAGE_SIZE_MNIST,
IMAGE_SIZE_MNIST, PMLR_resize_factor, PMLR_z_range)
x_PMLR = test_data[0:PMLR_n_samples, :]
id_PMLR = test_labels[0:PMLR_n_samples, :]
if ADD_NOISE:
x_PMLR = x_PMLR * np.random.randint(2, size=x_PMLR.shape)
x_PMLR += np.random.randint(2, size=x_PMLR.shape)
decoded = vae.decoder(z_in, dim_img, n_hidden)
# train
total_batch = int(n_samples / batch_size)
min_tot_loss = 1e99
with tf.Session() as sess:
sess.run(tf.global_variables_initializer(), feed_dict={keep_prob: 0.9})
for epoch in range(n_epochs):
# Random shuffling
np.random.shuffle(train_total_data)
train_data_ = train_total_data[:, :-mnist_data.NUM_LABELS]
# Loop over all batches
for i in range(total_batch):
# Compute the offset of the current minibatch in the data.
offset = (i * batch_size) % (n_samples)
batch_xs_input = train_data_[offset:(offset + batch_size), :]
batch_xs_target = batch_xs_input
# add salt & pepper noise
if ADD_NOISE:
batch_xs_input = batch_xs_input * np.random.randint(
2, size=batch_xs_input.shape)
batch_xs_input += np.random.randint(
2, size=batch_xs_input.shape)
_, tot_loss, loss_likelihood, loss_divergence = sess.run(
(train_op, loss, neg_marginal_likelihood, KL_divergence),
feed_dict={
x_hat: batch_xs_input,
x: batch_xs_target,
keep_prob: 0.9
})
# print cost every epoch
print(
"epoch %d: L_tot %03.2f L_likelihood %03.2f L_divergence %03.2f"
% (epoch, tot_loss, loss_likelihood, loss_divergence))
# if minimum loss is updated or final epoch, plot results
if min_tot_loss > tot_loss or epoch + 1 == n_epochs:
min_tot_loss = tot_loss
# Plot for reproduce performance
if PRR:
y_PRR = sess.run(y, feed_dict={x_hat: x_PRR, keep_prob: 1})
y_PRR_img = y_PRR.reshape(PRR.n_tot_imgs, IMAGE_SIZE_MNIST,
IMAGE_SIZE_MNIST)
PRR.save_images(y_PRR_img,
name="/PRR_epoch_%02d" % (epoch) + ".jpg")
# Plot for manifold learning result
if PMLR and dim_z == 2:
y_PMLR = sess.run(decoded,
feed_dict={
z_in: PMLR.z,
keep_prob: 1
})
y_PMLR_img = y_PMLR.reshape(PMLR.n_tot_imgs,
IMAGE_SIZE_MNIST,
IMAGE_SIZE_MNIST)
PMLR.save_images(y_PMLR_img,
name="/PMLR_epoch_%02d" % (epoch) +
".jpg")
# plot distribution of labeled images
z_PMLR = sess.run(z,
feed_dict={
x_hat: x_PMLR,
keep_prob: 1
})
PMLR.save_scattered_image(z_PMLR,
id_PMLR,
name="/PMLR_map_epoch_%02d" %
(epoch) + ".jpg")
shape=[None, win_size * win_size],
name="input")
with tf.variable_scope("vae"):
#rec, rec_mean, rec_log_var, rec_sample = vae(input_layer,win_size**2,rec_hidden_units,
# latent_dim,vae_generative_units,vae_likelihood_std)
rec_mean, rec_log_var = encoder(input_layer, win_size**2, rec_hidden_units,
latent_dim)
with tf.variable_scope("rec_sample"):
standard_normal_sample = tf.random_normal(
[tf.shape(input_layer)[0], latent_dim])
rec_sample = rec_mean + 1 * standard_normal_sample * tf.sqrt(
tf.exp(rec_log_var))
rec = decoder(rec_sample, win_size**2, vae_generative_units, latent_dim)
#rec = tf.clip_by_value(rec,0.0,1.0)
# output_true shall have the original image for error calculations
output_true = tf.placeholder('float32', [None, win_size * win_size],
name="Truth")
with tf.variable_scope("recon_loss"):
# define our cost function
meansq = tf.reduce_mean(tf.square(rec - output_true))
meansq *= win_size * win_size
binarcs = -tf.reduce_mean(output_true * tf.log(rec + 10e-10) +
(1.0 - output_true) * tf.log(1.0 - rec + 10e-10))
vae_kl = 0.5 * tf.reduce_sum(
0.0 - rec_log_var - 1.0 + tf.exp(rec_log_var) +
tf.square(rec_mean - 0.0), 1)
keep_prob = tf.placeholder(tf.float32, name='keep_prob')
# input for PMLR
z_in = tf.placeholder(tf.float32,
shape=[None, dim_z],
name='latent_variable')
# network architecture
y, z, loss, neg_marginal_likelihood, KL_divergence, px_elem = vae.autoencoder(
x_hat, x, dim_img, dim_z, n_hidden, keep_prob)
# optimization
train_op = tf.train.AdamOptimizer(learn_rate).minimize(loss)
# latent space for PMLR
decoded = vae.decoder(z_in, dim_img, n_hidden)
saver = tf.train.Saver()
########## Generative Model Scores on Unlabeled/Training set ###########
## It needs to be done only "once"
batch_size = 50
import ipdb
model_path = "models/model_class_" + str(CLASS_NUM) + "_dim_" + str(
dim_z) + "_cifar.ckpt"
print model_path
with tf.Session() as session:
session.run(tf.global_variables_initializer(),
feed_dict={keep_prob: 0.9})
saver.restore(session, model_path)
z_VALS_ORIG, Px_val = session.run([z, px_elem],
theta, [win_size, win_size])[:, :, :, 0]
window = tf.clip_by_value(window, 0.0, 1.0)
with tf.variable_scope("vae"):
with tf.variable_scope("rsh"):
net = tf.reshape(window, [-1, win_size, win_size])
net = tf.reshape(net, [-1, win_size, win_size, 1])
rec_mean, rec_log_var = encoder(net, is_train, rec_hidden_units,
latent_dim)
with tf.variable_scope("rec_sample"):
standard_normal_sample = tf.random_normal(
[tf.shape(input_layer)[0], latent_dim])
rec_sample = rec_mean + 1 * standard_normal_sample * tf.sqrt(
tf.exp(rec_log_var))
rec = decoder(rec_sample, is_train, [0, 7, 7, 16], vae_generative_units,
latent_dim)
rec = tf.reshape(rec, [-1, win_size * win_size])
# output_true shall have the original image for error calculations
output_true = tf.placeholder('float32', [None, win_size * win_size],
name="Truth")
def gaussian_log_likelihood(x, mean, var, eps=1e-8):
# compute log P(x) for diagonal Guassian
# -1/2 log( (2pi)^k sig_1 * sig_2 * ... * sig_k ) - sum_i 1/2sig_i^2 (x_i - m_i)^2
bb = tf.square(x - mean)
bb /= (var + eps)
return -0.5 * tf.reduce_sum(tf.log(2. * np.pi * var + eps) + bb, axis=1)
with tf.variable_scope("loss_function"):
def __init__(self, nin, nhid, nout, hebb='h,z,r', ln=True):
self.rnn = GRU(nin, nhid, hebb=set(hebb.split(',')), ln=True)
self.out = nn.Linear(nhid, nout)
self.enc = vae.encoder()
self.dec = vae.decoder()
val_loss = sess.run(loss, feed_dict={x: valX})
_, loss_likelihood, loss_divergence = sess.run(
(train_op, neg_marginal_likelihood, KL_divergence),
feed_dict={x: trainX})
print("Train Loss \t" + str(train_loss))
print("Val Loss \t" + str(val_loss))
print("neg_ML \t" + str(loss_likelihood))
print("KL \t" + str(loss_divergence))
print("Training Completed")
for i in range(100):
z = tf.random_normal(np.array([1, dim_z]), 0, 1, dtype=tf.float32)
y = vae.decoder(z, dim_img, n_hidden)
img = np.reshape(((sess.run(y) * imgListStds) + imgListMeans) * 255,
origShape)
#img = np.reshape(sess.run(y)*255, origShape)
cv2.imwrite("genImages/image" + str(i) + ".png", img)
print("Generation Completed")
fobj = open(DIR + "_Record.pkl", "rb")
SizeDict = pickle.load(fobj)
KLHistory = []
fileSizes = []
for key in sorted(SizeDict.keys()):
