def __init__(self, args):
self.num_epochs = args.epochs
self.cuda = args.cuda
self.verbose = args.verbose
self.batch_size = args.batch_size
self.batch_size_val = args.batch_size_val
self.learning_rate = args.learning_rate
self.decay_epoch = args.decay_epoch
self.lr_decay = args.lr_decay
self.w_cat = args.w_categ
self.w_gauss = args.w_gauss
self.w_rec = args.w_rec
self.rec_type = args.rec_type
self.gaussian_size = args.gaussian_size
self.input_size = args.input_size
self.output_size = args.output_size
self.network = VAENet(self.input_size, self.gaussian_size,
self.output_size)
self.losses = LossFunctions()
self.metrics = Metrics()
if self.cuda:
self.network = self.network.cuda()
def __init__(self, args):
self.num_epochs = args.epochs
self.cuda = args.cuda
self.verbose = args.verbose
self.batch_size = args.batch_size
self.batch_size_val = args.batch_size_val
self.learning_rate = args.learning_rate
self.decay_epoch = args.decay_epoch
self.lr_decay = args.lr_decay
self.w_cat = args.w_categ
self.w_gauss = args.w_gauss
self.w_rec = args.w_rec
self.rec_type = args.rec_type
self.num_classes = args.num_classes
self.gaussian_size = args.gaussian_size
self.input_size = args.input_size
# gumbel
self.init_temp = args.init_temp
self.decay_temp = args.decay_temp
self.hard_gumbel = args.hard_gumbel
self.min_temp = args.min_temp
self.decay_temp_rate = args.decay_temp_rate
self.gumbel_temp = self.init_temp
self.network = GMVAENet(self.input_size, self.gaussian_size,
self.num_classes)
self.losses = LossFunctions()
self.metrics = Metrics()
if self.cuda:
self.network = self.network.cuda()
def __init__(self, d, lr, lambda_z_wu, do_classify, use_kl=True):
""" model architecture """
self.MLP_SIZES = [512, 256, 256, 128, 128]
self.Z_SIZES = [64, 32, 32, 32, 32]
self.L = L = len(self.MLP_SIZES)
self.do_classify = do_classify
""" flags for regularizers """
self.use_kl = use_kl
""" data and external toolkits """
self.d = d # dataset manager
self.ls = Layers()
self.lf = LossFunctions(self.ls, d, self.encoder)
""" placeholders defined outside"""
self.lr = lr
self.lambda_z_wu = lambda_z_wu
""" cache for mu and sigma """
self.e_mus, self.e_logsigmas = [0] * L, [
0
] * L # q(z_i+1 | z_i), bottom-up inference as Eq.7-9
self.p_mus, self.p_logsigmas = [0] * L, [
0
] * L # p(z_i | z_i+1), top-down prior as Eq.1-3
self.d_mus, self.d_logsigmas = [0] * L, [
0
] * L # q(z_i | .), bidirectional inference as Eq.17-19
def __init__(self, params):
self.batch_size = params.batch_size
self.batch_size_val = params.batch_size_val
self.initial_temperature = params.temperature
self.decay_temperature = params.decay_temperature
self.num_epochs = params.num_epochs
self.loss_type = params.loss_type
self.num_classes = params.num_classes
self.w_gauss = params.w_gaussian
self.w_categ = params.w_categorical
self.w_recon = params.w_reconstruction
self.decay_temp_rate = params.decay_temp_rate
self.gaussian_size = params.gaussian_size
self.min_temperature = params.min_temperature
self.temperature = params.temperature # current temperature
self.verbose = params.verbose
self.sess = tf.Session()
self.network = Networks(params)
self.losses = LossFunctions()
self.learning_rate = tf.placeholder(tf.float32, [])
self.lr = params.learning_rate
self.decay_epoch = params.decay_epoch
self.lr_decay = params.lr_decay
self.dataset = params.dataset
self.metrics = Metrics()
def __init__(self, d, lr, lambda_z_wu, read_attn, write_attn, do_classify,
do_reconst):
self.do_classify = do_classify
""" flags for each regularizor """
self.do_reconst = do_reconst
self.read_attn = read_attn
self.write_attn = write_attn
""" dataset information """
self.set_datainfo(d)
""" external toolkits """
self.ls = Layers()
self.lf = LossFunctions(self.ls, self.d, self.encoder)
self.ii = ImageInterface(_is_3d, self.read_attn, self.write_attn,
GLIMPSE_SIZE_READ, GLIMPSE_SIZE_WRITE, _h, _w,
_c)
# for refference from get_loss_kl_draw()
self.T = T
self.L = L
self.Z_SIZES = Z_SIZES
""" placeholders defined outside"""
self.lr = lr
self.lambda_z_wu = lambda_z_wu
"""sequence of canvases """
self.cs = [0] * T
""" initialization """
self.init_lstms()
self.init_time_zero()
""" workaround for variable_scope(reuse=True) """
self.DO_SHARE = None
def __init__(self, resource):
""" data and external toolkits """
self.d = resource.dh # dataset manager
self.ls = Layers()
self.lf = LossFunctions(self.ls, self.d, self.encoder)
""" placeholders defined outside"""
if c.DO_TRAIN:
self.lr = resource.ph['lr']
def __init__(self, d, lr, lambda_pi_usl, use_pi):
""" flags for each regularizor """
self.use_pi = use_pi
""" data and external toolkits """
self.d = d # dataset manager
self.ls = Layers()
self.lf = LossFunctions(self.ls, d, self.encoder)
""" placeholders defined outside"""
self.lr = lr
self.lambda_pi_usl = lambda_pi_usl
def __init__(self, params):
self.batch_size = params.batch_size
self.batch_size_val = params.batch_size_val
self.initial_temperature = params.temperature
self.decay_temperature = params.decay_temperature
self.num_epochs = params.num_epochs
self.loss_type = params.loss_type
self.num_classes = params.num_classes
self.w_gauss = params.w_gaussian
self.w_categ = params.w_categorical
self.w_recon = params.w_reconstruction
self.decay_temp_rate = params.decay_temp_rate
self.gaussian_size = params.gaussian_size
self.feature_size = params.feature_size
self.min_temperature = params.min_temperature
self.temperature = params.temperature # current temperature
self.verbose = params.verbose
self.sess = tf.Session()
self.network = CatVAENetwork(params)
self.losses = LossFunctions()
self.w_assign = params.w_assign
self.num_labeled = params.num_labeled
self.knn = params.knn
self.metric_loss = params.metric_loss
self.w_metric = tf.placeholder(tf.float32, [])
self.initial_w_metric = params.w_metric
self._w_metric = params.w_metric
self.anneal_metric_loss = params.anneal_w_metric
self.learning_rate = tf.placeholder(tf.float32, [])
self.lr = params.learning_rate
self.decay_epoch = params.decay_epoch
self.lr_decay = params.lr_decay
self.pretrain = params.pretrain
self.num_labeled_batch = params.num_labeled_batch
self.dataset = params.dataset
self.metric_margin = params.metric_margin
class PI(object):
def __init__(self, d, lr, lambda_pi_usl, use_pi):
""" flags for each regularizor """
self.use_pi = use_pi
""" data and external toolkits """
self.d = d # dataset manager
self.ls = Layers()
self.lf = LossFunctions(self.ls, d, self.encoder)
""" placeholders defined outside"""
self.lr = lr
self.lambda_pi_usl = lambda_pi_usl
def encoder(self, x, is_train=True, do_update_bn=True):
""" https://arxiv.org/pdf/1610.02242.pdf """
if is_train:
h = self.distort(x)
h = self.ls.get_corrupted(x, 0.15)
else:
h = x
scope = '1'
h = self.ls.conv2d(scope + '_1', h, 128, activation=self.ls.lrelu)
h = self.ls.conv2d(scope + '_2', h, 128, activation=self.ls.lrelu)
h = self.ls.conv2d(scope + '_3', h, 128, activation=self.ls.lrelu)
h = self.ls.max_pool(h)
if is_train: h = tf.nn.dropout(h, 0.5)
scope = '2'
h = self.ls.conv2d(scope + '_1', h, 256, activation=self.ls.lrelu)
h = self.ls.conv2d(scope + '_2', h, 256, activation=self.ls.lrelu)
h = self.ls.conv2d(scope + '_3', h, 256, activation=self.ls.lrelu)
h = self.ls.max_pool(h)
if is_train: h = tf.nn.dropout(h, 0.5)
scope = '3'
h = self.ls.conv2d(scope + '_1', h, 512, activation=self.ls.lrelu)
h = self.ls.conv2d(scope + '_2',
h,
256,
activation=self.ls.lrelu,
filter_size=(1, 1))
h = self.ls.conv2d(scope + '_3',
h,
128,
activation=self.ls.lrelu,
filter_size=(1, 1))
h = tf.reduce_mean(h, reduction_indices=[1,
2]) # Global average pooling
h = self.ls.dense(scope, h, self.d.l)
return h
def build_graph_train(self, x_l, y_l, x, is_supervised=True):
o = dict() # output
loss = 0
logit = self.encoder(x)
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
logit_l = self.encoder(
x_l, is_train=True,
do_update_bn=False) # for pyx and vat loss computation
""" Classification Loss """
o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_l, y_l)
loss += o['Ly']
""" PI Model Loss """
if self.use_pi:
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
_, _, o['Lp'] = self.lf.get_loss_pi(x, logit, is_train=True)
loss += self.lambda_pi_usl * o['Lp']
else:
o['Lp'] = tf.constant(0)
""" set losses """
o['loss'] = loss
self.o_train = o
""" set optimizer """
optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.5)
#self.op = optimizer.minimize(loss)
grads = optimizer.compute_gradients(loss)
for i, (g, v) in enumerate(grads):
if g is not None:
#g = tf.Print(g, [g], "g %s = "%(v))
grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients
else:
print('g is None:', v)
v = tf.Print(v, [v], "v = ", summarize=10000)
self.op = optimizer.apply_gradients(grads) # return train_op
def build_graph_test(self, x_l, y_l):
o = dict() # output
loss = 0
logit_l = self.encoder(
x_l, is_train=False,
do_update_bn=False) # for pyx and vat loss computation
""" classification loss """
o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_l, y_l)
loss += o['Ly']
""" set losses """
o['loss'] = loss
self.o_test = o
def distort(self, x):
_d = self.d
def _distort(a_image):
"""
bounding_boxes: A Tensor of type float32.
3-D with shape [batch, N, 4] describing the N bounding boxes associated with the image.
Bounding boxes are supplied and returned as [y_min, x_min, y_max, x_max]
"""
# shape: [1, 1, 4]
bounding_boxes = tf.constant([[[1 / 10, 1 / 10, 9 / 10, 9 / 10]]],
dtype=tf.float32)
begin, size, _ = tf.image.sample_distorted_bounding_box(
(_d.h, _d.w, _d.c),
bounding_boxes,
min_object_covered=(8.5 / 10.0),
aspect_ratio_range=[7.0 / 10.0, 10.0 / 7.0])
a_image = tf.slice(a_image, begin, size)
""" for the purpose of distorting not use tf.image.resize_image_with_crop_or_pad under """
a_image = tf.image.resize_images(a_image, [_d.h, _d.w])
""" due to the size of channel returned from tf.image.resize_images is not being given,
specify it manually. """
a_image = tf.reshape(a_image, [_d.h, _d.w, _d.c])
return a_image
""" process batch times in parallel """
return tf.map_fn(_distort, x)
class VAE:
def __init__(self, args):
self.num_epochs = args.epochs
self.cuda = args.cuda
self.verbose = args.verbose
self.batch_size = args.batch_size
self.batch_size_val = args.batch_size_val
self.learning_rate = args.learning_rate
self.decay_epoch = args.decay_epoch
self.lr_decay = args.lr_decay
self.w_cat = args.w_categ
self.w_gauss = args.w_gauss
self.w_rec = args.w_rec
self.rec_type = args.rec_type
self.gaussian_size = args.gaussian_size
self.input_size = args.input_size
self.output_size = args.output_size
self.network = VAENet(self.input_size, self.gaussian_size,
self.output_size)
self.losses = LossFunctions()
self.metrics = Metrics()
if self.cuda:
self.network = self.network.cuda()
# ===============================================================================
# This is where all the losses are computed; helper method for train_epoch.
def unlabeled_loss(self, chunk, out_net):
# obtain network variables
z, data_recon = out_net['gaussian'], out_net['x_rec']
mu, var = out_net['mean'], out_net['var']
# reconstruction loss
loss_rec = self.losses.reconstruction_loss(chunk, data_recon,
self.rec_type)
# gaussian loss
loss_gauss = self.losses.kl_divergence_v2(mu, var)
# total loss
loss_total = self.w_rec * loss_rec + self.w_gauss * loss_gauss
loss_dic = {
'total': loss_total,
'reconstruction': loss_rec,
'gaussian': loss_gauss,
}
return loss_dic
# ===============================================================================
# ===============================================================================
# Train the model for one epoch; helper method for train.
def train_epoch(self, optimizer, data_loader):
self.network.train()
total_loss = 0.
recon_loss = 0.
gauss_loss = 0.
accuracy = 0.
num_batches = 0.
for (path, chunk) in data_loader:
if self.cuda == 1:
path = path.cuda()
chunk = chunk.cuda()
optimizer.zero_grad()
# flatten data
path = path.view(path.size(0), -1)
chunk = chunk.view(chunk.size(0), -1)
# forward call
out_net = self.network(path)
unlab_loss_dic = self.unlabeled_loss(chunk, out_net)
total = unlab_loss_dic['total']
# accumulate values
total_loss += total.item()
recon_loss += unlab_loss_dic['reconstruction'].item()
gauss_loss += unlab_loss_dic['gaussian'].item()
# perform backpropagation
total.backward()
optimizer.step()
num_batches += 1.
# average per batch
total_loss /= num_batches
recon_loss /= num_batches
gauss_loss /= num_batches
return total_loss, recon_loss, gauss_loss
# ==============================================================================
# ==============================================================================
# Test the model. This method is essentially the same as train_epoch except that
# self.network.eval is called at the beginning of this method.
def test(self, data_loader):
self.network.eval()
total_loss = 0.
recon_loss = 0.
gauss_loss = 0.
accuracy = 0.
num_batches = 0.
with torch.no_grad():
for path, chunk in data_loader:
if self.cuda == 1:
path = path.cuda()
chunk = chunk.cuda()
# flatten data
path = path.view(path.size(0), -1)
chunk = chunk.view(chunk.size(0), -1)
# forward call
out_net = self.network(path)
unlab_loss_dic = self.unlabeled_loss(chunk, out_net)
# accumulate values
total_loss += unlab_loss_dic['total'].item()
recon_loss += unlab_loss_dic['reconstruction'].item()
gauss_loss += unlab_loss_dic['gaussian'].item()
num_batches += 1.
# average per batch
total_loss /= num_batches
recon_loss /= num_batches
gauss_loss /= num_batches
return total_loss, recon_loss, gauss_loss
# ==============================================================================
def train(self, train_loader, val_loader):
"""Train the model
Args:
train_loader: (DataLoader) corresponding loader containing the training data
val_loader: (DataLoader) corresponding loader containing the validation data
Returns:
output: (dict) contains the history of train/val loss
"""
optimizer = optim.Adam(self.network.parameters(),
lr=self.learning_rate)
train_total_losses = []
train_recon_losses = []
train_gauss_losses = []
valid_total_losses = []
valid_recon_losses = []
valid_gauss_losses = []
for epoch in range(1, self.num_epochs + 1):
train_total_loss, train_recon_loss, train_gauss_loss = self.train_epoch(
optimizer, train_loader)
valid_total_loss, valid_recon_loss, valid_gauss_loss = self.test(
val_loader)
print(
f"Epoch {epoch:5} / {self.num_epochs} | Total loss: {round(valid_total_loss, 2):7} | Recon loss: {round(valid_recon_loss, 2):7} | Gauss loss: {round(valid_gauss_loss, 2):7}"
)
train_total_losses.append(train_total_loss)
train_recon_losses.append(train_recon_loss)
train_gauss_losses.append(train_gauss_loss)
valid_total_losses.append(valid_total_loss)
valid_recon_losses.append(valid_recon_loss)
valid_gauss_losses.append(valid_gauss_loss)
return {
'train_total_losses': train_total_losses,
'train_recon_losses': train_recon_losses,
'train_gauss_losses': train_gauss_losses,
'valid_total_losses': valid_total_losses,
'valid_recon_losses': valid_recon_losses,
'valid_gauss_losses': valid_gauss_losses,
}
def latent_features(self, data_loader):
"""Obtain latent features learnt by the model
Args:
data_loader: (DataLoader) loader containing the data
return_labels: (boolean) whether to return true labels or not
Returns:
features: (array) array containing the features from the data
"""
self.network.eval()
N = len(data_loader.dataset)
features = np.zeros((N, self.gaussian_size))
start_ind = 0
with torch.no_grad():
for (data, ) in data_loader:
if self.cuda == 1:
data = data.cuda()
data = data.view(data.size(0), -1)
out = self.network(data)
latent_feat = out['mean']
end_ind = min(start_ind + data.size(0), N + 1)
features[start_ind:end_ind] = latent_feat.cpu().numpy()
start_ind += data.size(0)
return features
def reconstruct_data(self, data_loader, sample_size=-1):
self.network.eval()
# sample random data from loader
indices = np.random.randint(0,
len(data_loader.dataset),
size=sample_size)
test_random_loader = torch.utils.data.DataLoader(
data_loader.dataset,
batch_size=sample_size,
sampler=SubsetRandomSampler(indices))
# obtain values
it = iter(test_random_loader)
test_batch_data, _ = it.next()
original = test_batch_data.data.numpy()
if self.cuda:
test_batch_data = test_batch_data.cuda()
# obtain reconstructed data
out = self.network(test_batch_data)
reconstructed = out['x_rec']
return original, reconstructed.data.cpu().numpy()
def plot_latent_space(self, data_loader, save=False):
"""Plot the latent space learnt by the model
Args:
data: (array) corresponding array containing the data
labels: (array) corresponding array containing the labels
save: (bool) whether to save the latent space plot
Returns:
fig: (figure) plot of the latent space
"""
# obtain the latent features
features = self.latent_features(data_loader)
# plot only the first 2 dimensions
fig = plt.figure(figsize=(8, 6))
plt.scatter(features[:, 0],
features[:, 1],
c=labels,
marker='o',
edgecolor='none',
cmap=plt.cm.get_cmap('jet', 10),
s=10)
plt.colorbar()
if (save):
fig.savefig('latent_space.png')
return fig
def random_generation(self, num_chunks):
latent_vecs = torch.randn(num_chunks, 64).cuda()
chunks = self.network.generative.pxz(latent_vecs)
return chunks.cpu().detach().numpy()
class GMVAE:
def __init__(self, args):
self.num_epochs = args.epochs
self.cuda = args.cuda
self.verbose = args.verbose
self.batch_size = args.batch_size
self.batch_size_val = args.batch_size_val
self.learning_rate = args.learning_rate
self.decay_epoch = args.decay_epoch
self.lr_decay = args.lr_decay
self.w_cat = args.w_categ
self.w_gauss = args.w_gauss
self.w_rec = args.w_rec
self.rec_type = args.rec_type
self.num_classes = args.num_classes
self.gaussian_size = args.gaussian_size
self.input_size = args.input_size
# gumbel
self.init_temp = args.init_temp
self.decay_temp = args.decay_temp
self.hard_gumbel = args.hard_gumbel
self.min_temp = args.min_temp
self.decay_temp_rate = args.decay_temp_rate
self.gumbel_temp = self.init_temp
self.network = GMVAENet(self.input_size, self.gaussian_size,
self.num_classes)
self.losses = LossFunctions()
self.metrics = Metrics()
if self.cuda:
self.network = self.network.cuda()
def unlabeled_loss(self, data, out_net):
"""Method defining the loss functions derived from the variational lower bound
Args:
data: (array) corresponding array containing the input data
out_net: (dict) contains the graph operations or nodes of the network output
Returns:
loss_dic: (dict) contains the values of each loss function and predictions
"""
# obtain network variables
z, data_recon = out_net['gaussian'], out_net['x_rec']
logits, prob_cat = out_net['logits'], out_net['prob_cat']
y_mu, y_var = out_net['y_mean'], out_net['y_var']
mu, var = out_net['mean'], out_net['var']
# reconstruction loss
loss_rec = self.losses.reconstruction_loss(data, data_recon,
self.rec_type)
# gaussian loss
loss_gauss = self.losses.gaussian_loss(z, mu, var, y_mu, y_var)
# categorical loss
loss_cat = -self.losses.entropy(logits, prob_cat) - np.log(0.1)
# total loss
loss_total = self.w_rec * loss_rec + self.w_gauss * loss_gauss + self.w_cat * loss_cat
# obtain predictions
_, predicted_labels = torch.max(logits, dim=1)
loss_dic = {
'total': loss_total,
'predicted_labels': predicted_labels,
'reconstruction': loss_rec,
'gaussian': loss_gauss,
'categorical': loss_cat
}
return loss_dic
def train_epoch(self, optimizer, data_loader):
"""Train the model for one epoch
Args:
optimizer: (Optim) optimizer to use in backpropagation
data_loader: (DataLoader) corresponding loader containing the training data
Returns:
average of all loss values, accuracy, nmi
"""
self.network.train()
total_loss = 0.
recon_loss = 0.
cat_loss = 0.
gauss_loss = 0.
accuracy = 0.
nmi = 0.
num_batches = 0.
true_labels_list = []
predicted_labels_list = []
# iterate over the dataset
for (data, labels) in data_loader:
if self.cuda == 1:
data = data.cuda()
optimizer.zero_grad()
# flatten data
data = data.view(data.size(0), -1)
# forward call
out_net = self.network(data, self.gumbel_temp, self.hard_gumbel)
unlab_loss_dic = self.unlabeled_loss(data, out_net)
total = unlab_loss_dic['total']
# accumulate values
total_loss += total.item()
recon_loss += unlab_loss_dic['reconstruction'].item()
gauss_loss += unlab_loss_dic['gaussian'].item()
cat_loss += unlab_loss_dic['categorical'].item()
# perform backpropagation
total.backward()
optimizer.step()
# save predicted and true labels
predicted = unlab_loss_dic['predicted_labels']
true_labels_list.append(labels)
predicted_labels_list.append(predicted)
num_batches += 1.
# average per batch
total_loss /= num_batches
recon_loss /= num_batches
gauss_loss /= num_batches
cat_loss /= num_batches
# concat all true and predicted labels
true_labels = torch.cat(true_labels_list, dim=0).cpu().numpy()
predicted_labels = torch.cat(predicted_labels_list,
dim=0).cpu().numpy()
# compute metrics
accuracy = 100.0 * self.metrics.cluster_acc(predicted_labels,
true_labels)
nmi = 100.0 * self.metrics.nmi(predicted_labels, true_labels)
return total_loss, recon_loss, gauss_loss, cat_loss, accuracy, nmi
def test(self, data_loader, return_loss=False):
"""Test the model with new data
Args:
data_loader: (DataLoader) corresponding loader containing the test/validation data
return_loss: (boolean) whether to return the average loss values
Return:
accuracy and nmi for the given test data
"""
self.network.eval()
total_loss = 0.
recon_loss = 0.
cat_loss = 0.
gauss_loss = 0.
accuracy = 0.
nmi = 0.
num_batches = 0.
true_labels_list = []
predicted_labels_list = []
with torch.no_grad():
for data, labels in data_loader:
if self.cuda == 1:
data = data.cuda()
# flatten data
data = data.view(data.size(0), -1)
# forward call
out_net = self.network(data, self.gumbel_temp,
self.hard_gumbel)
unlab_loss_dic = self.unlabeled_loss(data, out_net)
# accumulate values
total_loss += unlab_loss_dic['total'].item()
recon_loss += unlab_loss_dic['reconstruction'].item()
gauss_loss += unlab_loss_dic['gaussian'].item()
cat_loss += unlab_loss_dic['categorical'].item()
# save predicted and true labels
predicted = unlab_loss_dic['predicted_labels']
true_labels_list.append(labels)
predicted_labels_list.append(predicted)
num_batches += 1.
# average per batch
if return_loss:
total_loss /= num_batches
recon_loss /= num_batches
gauss_loss /= num_batches
cat_loss /= num_batches
# concat all true and predicted labels
true_labels = torch.cat(true_labels_list, dim=0).cpu().numpy()
predicted_labels = torch.cat(predicted_labels_list,
dim=0).cpu().numpy()
# compute metrics
accuracy = 100.0 * self.metrics.cluster_acc(predicted_labels,
true_labels)
nmi = 100.0 * self.metrics.nmi(predicted_labels, true_labels)
if return_loss:
return total_loss, recon_loss, gauss_loss, cat_loss, accuracy, nmi
else:
return accuracy, nmi
def train(self, train_loader, val_loader):
"""Train the model
Args:
train_loader: (DataLoader) corresponding loader containing the training data
val_loader: (DataLoader) corresponding loader containing the validation data
Returns:
output: (dict) contains the history of train/val loss
"""
optimizer = optim.Adam(self.network.parameters(),
lr=self.learning_rate)
train_history_acc, val_history_acc = [], []
train_history_nmi, val_history_nmi = [], []
for epoch in range(1, self.num_epochs + 1):
train_loss, train_rec, train_gauss, train_cat, train_acc, train_nmi = self.train_epoch(
optimizer, train_loader)
val_loss, val_rec, val_gauss, val_cat, val_acc, val_nmi = self.test(
val_loader, True)
# if verbose then print specific information about training
if self.verbose == 1:
print("(Epoch %d / %d)" % (epoch, self.num_epochs))
print("Train - REC: %.5lf; Gauss: %.5lf; Cat: %.5lf;" % \
(train_rec, train_gauss, train_cat))
print("Valid - REC: %.5lf; Gauss: %.5lf; Cat: %.5lf;" % \
(val_rec, val_gauss, val_cat))
print("Accuracy=Train: %.5lf; Val: %.5lf NMI=Train: %.5lf; Val: %.5lf Total Loss=Train: %.5lf; Val: %.5lf" % \
(train_acc, val_acc, train_nmi, val_nmi, train_loss, val_loss))
else:
print('(Epoch %d / %d) Train_Loss: %.3lf; Val_Loss: %.3lf Train_ACC: %.3lf; Val_ACC: %.3lf Train_NMI: %.3lf; Val_NMI: %.3lf' % \
(epoch, self.num_epochs, train_loss, val_loss, train_acc, val_acc, train_nmi, val_nmi))
# decay gumbel temperature
if self.decay_temp == 1:
self.gumbel_temp = np.maximum(
self.init_temp * np.exp(-self.decay_temp_rate * epoch),
self.min_temp)
if self.verbose == 1:
print("Gumbel Temperature: %.3lf" % self.gumbel_temp)
train_history_acc.append(train_acc)
val_history_acc.append(val_acc)
train_history_nmi.append(train_nmi)
val_history_nmi.append(val_nmi)
return {
'train_history_nmi': train_history_nmi,
'val_history_nmi': val_history_nmi,
'train_history_acc': train_history_acc,
'val_history_acc': val_history_acc
}
def latent_features(self, data_loader, return_labels=False):
"""Obtain latent features learnt by the model
Args:
data_loader: (DataLoader) loader containing the data
return_labels: (boolean) whether to return true labels or not
Returns:
features: (array) array containing the features from the data
"""
self.network.eval()
N = len(data_loader.dataset)
features = np.zeros((N, self.gaussian_size))
if return_labels:
true_labels = np.zeros(N, dtype=np.int64)
start_ind = 0
with torch.no_grad():
for (data, labels) in data_loader:
if self.cuda == 1:
data = data.cuda()
# flatten data
data = data.view(data.size(0), -1)
out = self.network.inference(data, self.gumbel_temp,
self.hard_gumbel)
latent_feat = out['mean']
end_ind = min(start_ind + data.size(0), N + 1)
# return true labels
if return_labels:
true_labels[start_ind:end_ind] = labels.cpu().numpy()
features[start_ind:end_ind] = latent_feat.cpu().detach().numpy(
)
start_ind += data.size(0)
if return_labels:
return features, true_labels
return features
def reconstruct_data(self, data_loader, sample_size=-1):
"""Reconstruct Data
Args:
data_loader: (DataLoader) loader containing the data
sample_size: (int) size of random data to consider from data_loader
Returns:
reconstructed: (array) array containing the reconstructed data
"""
self.network.eval()
# sample random data from loader
indices = np.random.randint(0,
len(data_loader.dataset),
size=sample_size)
test_random_loader = torch.utils.data.DataLoader(
data_loader.dataset,
batch_size=sample_size,
sampler=SubsetRandomSampler(indices))
# obtain values
it = iter(test_random_loader)
test_batch_data, _ = it.next()
original = test_batch_data.data.numpy()
if self.cuda:
test_batch_data = test_batch_data.cuda()
# obtain reconstructed data
out = self.network(test_batch_data, self.gumbel_temp, self.hard_gumbel)
reconstructed = out['x_rec']
return original, reconstructed.data.cpu().numpy()
def plot_latent_space(self, data_loader, save=False):
"""Plot the latent space learnt by the model
Args:
data: (array) corresponding array containing the data
labels: (array) corresponding array containing the labels
save: (bool) whether to save the latent space plot
Returns:
fig: (figure) plot of the latent space
"""
# obtain the latent features
features = self.latent_features(data_loader)
# plot only the first 2 dimensions
fig = plt.figure(figsize=(8, 6))
plt.scatter(features[:, 0],
features[:, 1],
c=labels,
marker='o',
edgecolor='none',
cmap=plt.cm.get_cmap('jet', 10),
s=10)
plt.colorbar()
if (save):
fig.savefig('latent_space.png')
return fig
def random_generation(self, num_elements=1):
"""Random generation for each category
Args:
num_elements: (int) number of elements to generate
Returns:
generated data according to num_elements
"""
# categories for each element
arr = np.array([])
for i in range(self.num_classes):
arr = np.hstack([arr, np.ones(num_elements) * i])
indices = arr.astype(int).tolist()
categorical = F.one_hot(torch.tensor(indices),
self.num_classes).float()
if self.cuda:
categorical = categorical.cuda()
# infer the gaussian distribution according to the category
mean, var = self.network.generative.pzy(categorical)
# gaussian random sample by using the mean and variance
noise = torch.randn_like(var)
std = torch.sqrt(var)
gaussian = mean + noise * std
# generate new samples with the given gaussian
generated = self.network.generative.pxz(gaussian)
return generated.cpu().detach().numpy()
class LVAE(object):
def __init__(self, d, lr, lambda_z_wu, do_classify, use_kl=True):
""" model architecture """
self.MLP_SIZES = [512, 256, 256, 128, 128]
self.Z_SIZES = [64, 32, 32, 32, 32]
self.L = L = len(self.MLP_SIZES)
self.do_classify = do_classify
""" flags for regularizers """
self.use_kl = use_kl
""" data and external toolkits """
self.d = d # dataset manager
self.ls = Layers()
self.lf = LossFunctions(self.ls, d, self.encoder)
""" placeholders defined outside"""
self.lr = lr
self.lambda_z_wu = lambda_z_wu
""" cache for mu and sigma """
self.e_mus, self.e_logsigmas = [0] * L, [
0
] * L # q(z_i+1 | z_i), bottom-up inference as Eq.7-9
self.p_mus, self.p_logsigmas = [0] * L, [
0
] * L # p(z_i | z_i+1), top-down prior as Eq.1-3
self.d_mus, self.d_logsigmas = [0] * L, [
0
] * L # q(z_i | .), bidirectional inference as Eq.17-19
def encoder(self, x, is_train=True, do_update_bn=True):
h = x
for l in range(self.L):
scope = 'Encode_L' + str(l)
h = self.ls.dense(scope, h, self.MLP_SIZES[l])
h = self.ls.bn(scope, h, is_train, do_update_bn, name=scope)
h = tf.nn.elu(h)
""" prepare for bidirectional inference """
_, self.e_mus[l], self.e_logsigmas[l] = self.ls.vae_sampler(
scope, h, self.Z_SIZES[l], tf.nn.softplus) # Eq.13-15
#return h
return self.e_mus[-1]
def decoder(self, is_train=True, do_update_bn=True):
for l in range(self.L - 1, -1, -1):
scope = 'Decoder_L' + str(l)
if l == self.L - 1:
""" At the highest latent layer, mu & sigma are identical to those outputed from encoer.
And making actual z is not necessary for the highest layer."""
mu, logsigma = self.e_mus[l], self.e_logsigmas[l]
self.d_mus[l], self.d_logsigmas[l] = mu, logsigma
z = self.ls.sampler(self.d_mus[l], tf.exp(self.d_logsigmas[l]))
""" prior of z_L is set as standard Gaussian, N(0,I). """
self.p_mus[l], self.p_logsigmas[l] = tf.zeros(
(mu.get_shape())), tf.zeros((logsigma.get_shape()))
else:
""" prior is developed from z of the above layer """
_, self.p_mus[l], self.p_logsigmas[l] = self.ls.vae_sampler(
scope, z, self.Z_SIZES[l], tf.nn.softplus) # Eq.13-15
z, self.d_mus[l], self.d_logsigmas[
l] = self.ls.precision_weighted_sampler(
scope, (self.e_mus[l], tf.exp(self.e_logsigmas[l])),
(self.p_mus[l], tf.exp(
self.p_logsigmas[l]))) # Eq.17-19
""" go out to the input space """
_d = self.d
x = self.ls.dense('bottom', z, _d.img_size,
tf.nn.elu) # reconstructed input
if _d.is_3d: x = tf.reshape(x, (-1, _d.h, _d.w, _d.c))
return x
def build_graph_train(self, x_l, y_l, x):
o = dict() # output
loss = 0
logit = self.encoder(x)
x_reconst = self.decoder()
with tf.variable_scope(tf.get_variable_scope(), reuse=True):
logit_l = self.encoder(
x_l, is_train=True,
do_update_bn=False) # for pyx and vat loss computation
""" Classification Loss """
if self.do_classify:
o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_l, y_l)
loss += o['Ly']
""" for visualizationc """
o['z'], o['y'] = logit, y_l
""" p(x|z) Reconstruction Loss """
o['Lr'] = self.lf.get_loss_pxz(x_reconst, x, 'DiscretizedLogistic')
loss += o['Lr']
o['x'] = x
o['cs'] = x_reconst
""" VAE KL-Divergence Loss """
if self.use_kl:
o['KL1'], o['KL2'], o['Lz'] = self.lf.get_loss_kl(self,
_lambda=10.0)
loss += self.lambda_z_wu * o['Lz']
else:
o['KL1'], o['KL2'], o['Lz'] = tf.constant(0), tf.constant(
0), tf.constant(0)
""" set losses """
o['loss'] = loss
self.o_train = o
""" set optimizer """
optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.5)
#self.op = optimizer.minimize(loss)
grads = optimizer.compute_gradients(loss)
for i, (g, v) in enumerate(grads):
if g is not None:
#g = tf.Print(g, [g], "g %s = "%(v))
grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients
else:
print('g is None:', v)
v = tf.Print(v, [v], "v = ", summarize=10000)
#for v in tf.all_variables(): print("%s : %s" % (v.name,v.get_shape()))
self.op = optimizer.apply_gradients(grads) # return train_op
def build_graph_test(self, x_l, y_l):
o = dict() # output
loss = 0
logit_l = self.encoder(
x_l, is_train=False,
do_update_bn=False) # for pyx and vat loss computation
""" classification loss """
if self.do_classify:
o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_l, y_l)
loss += o['Ly']
""" for visualizationc """
o['z'], o['y'] = logit_l, y_l
""" set losses """
o['loss'] = loss
self.o_test = o
class GMVAE:
def __init__(self, params):
self.batch_size = params.batch_size
self.batch_size_val = params.batch_size_val
self.initial_temperature = params.temperature
self.decay_temperature = params.decay_temperature
self.num_epochs = params.num_epochs
self.loss_type = params.loss_type
self.num_classes = params.num_classes
self.w_gauss = params.w_gaussian
self.w_categ = params.w_categorical
self.w_recon = params.w_reconstruction
self.decay_temp_rate = params.decay_temp_rate
self.gaussian_size = params.gaussian_size
self.min_temperature = params.min_temperature
self.temperature = params.temperature # current temperature
self.verbose = params.verbose
self.sess = tf.Session()
self.network = Networks(params)
self.losses = LossFunctions()
self.learning_rate = tf.placeholder(tf.float32, [])
self.lr = params.learning_rate
self.decay_epoch = params.decay_epoch
self.lr_decay = params.lr_decay
self.dataset = params.dataset
self.metrics = Metrics()
def create_dataset(self, is_training, data, labels, batch_size):
"""Create dataset given input data
Args:
is_training: (bool) whether to use the train or test pipeline.
At training, we shuffle the data and have multiple epochs
data: (array) corresponding array containing the input data
labels: (array) corresponding array containing the labels of the input data
batch_size: (int) size of each batch to consider from the data
Returns:
output: (dict) contains what will be the input of the tensorflow graph
"""
num_samples = data.shape[0]
# create dataset object
if labels is None:
dataset = tf.data.Dataset.from_tensor_slices(data)
else:
dataset = tf.data.Dataset.from_tensor_slices((data, labels))
# shuffle data in training phase
if is_training:
dataset = dataset.shuffle(num_samples).repeat()
dataset = dataset.batch(batch_size)
dataset = dataset.prefetch(1)
# create reinitializable iterator from dataset
iterator = dataset.make_initializable_iterator()
if labels is None:
data = iterator.get_next()
else:
data, labels = iterator.get_next()
iterator_init = iterator.initializer
output = {
'data': data,
'labels': labels,
'iterator_init': iterator_init
}
return output
def unlabeled_loss(self, data, latent_spec, output_size, is_training=True):
"""Model function defining the loss functions derived from the variational lower bound
Args:
data: (array) corresponding array containing the input data
latent_spec: (dict) contains the graph operations or nodes of the latent variables
output_size: (int) size of the output layer
is_training: (bool) whether we are in training phase or not
Returns:
loss_dic: (dict) contains the values of each loss function and predictions
"""
gaussian, mean, var = latent_spec['gaussian'], latent_spec[
'mean'], latent_spec['var']
categorical, prob, log_prob = latent_spec['categorical'], latent_spec[
'prob_cat'], latent_spec['log_prob_cat']
_logits, features = latent_spec['logits'], latent_spec['features']
output, y_mean, y_var = latent_spec['output'], latent_spec[
'y_mean'], latent_spec['y_var']
# reconstruction loss
if self.loss_type == 'bce':
loss_rec = self.w_recon * self.losses.binary_cross_entropy(
data, output)
elif self.loss_type == 'mse':
loss_rec = self.w_recon * tf.losses.mean_squared_error(
data, output)
else:
raise "invalid loss function... try bce or mse..."
# gaussian loss
loss_gaussian = self.w_gauss * self.losses.labeled_loss(
gaussian, mean, var, y_mean, y_var)
# categorical loss
loss_categorical = self.w_categ * -self.losses.entropy(_logits, prob)
# obtain predictions
predicted_labels = tf.argmax(_logits, axis=1)
# total_loss
loss_total = loss_rec + loss_gaussian + loss_categorical
loss_dic = {
'total': loss_total,
'predicted_labels': predicted_labels,
'reconstruction': loss_rec,
'gaussian': loss_gaussian,
'categorical': loss_categorical
}
return loss_dic
def create_model(self, is_training, inputs, output_size):
"""Model function defining the graph operations.
Args:
is_training: (bool) whether we are in training phase or not
inputs: (dict) contains the inputs of the graph (features, labels...)
this can be `tf.placeholder` or outputs of `tf.data`
output_size: (int) size of the output layer
Returns:
model_spec: (dict) contains the graph operations or nodes needed for training / evaluation
"""
data, _labels = inputs['data'], inputs['labels']
latent_spec = self.network.encoder(data, self.num_classes, is_training)
out_logits, y_mean, y_var, output = self.network.decoder(
latent_spec['gaussian'], latent_spec['categorical'], output_size,
is_training)
latent_spec['output'] = out_logits
latent_spec['y_mean'] = y_mean
latent_spec['y_var'] = y_var
# unlabeled losses
unlabeled_loss_dic = self.unlabeled_loss(data, latent_spec,
output_size, is_training)
loss_total = unlabeled_loss_dic['total']
if is_training:
# use adam for optimization
optimizer = tf.train.AdamOptimizer(self.learning_rate)
# needed for batch normalization layer
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op = optimizer.minimize(loss_total)
# create model specification
model_spec = inputs
model_spec['variable_init_op'] = tf.global_variables_initializer()
# optimizers are only available in training phase
if is_training:
model_spec['train_op'] = train_op
else:
model_spec['output'] = output
model_spec['loss_total'] = loss_total
model_spec['loss_rec_ul'] = unlabeled_loss_dic['reconstruction']
model_spec['loss_gauss_ul'] = unlabeled_loss_dic['gaussian']
model_spec['loss_categ_ul'] = unlabeled_loss_dic['categorical']
model_spec['true_labels'] = _labels
model_spec['predicted'] = unlabeled_loss_dic['predicted_labels']
return model_spec
def evaluate_dataset(self, is_training, num_batches, model_spec):
"""Evaluate the model
Args:
is_training: (bool) whether we are training or not
num_batches: (int) number of batches to train/test
model_spec: (dict) contains the graph operations or nodes needed for evaluation
Returns:
(dic) average of loss functions and metrics for the given number of batches
"""
avg_accuracy = 0.0
avg_nmi = 0.0
avg_loss_cat = 0.0
avg_loss_total = 0.0
avg_loss_rec = 0.0
avg_loss_gauss = 0.0
list_predicted_labels = []
list_true_labels = []
# initialize dataset iteratior
self.sess.run(model_spec['iterator_init'])
if is_training:
train_optimizer = model_spec['train_op']
# training phase
for j in range(num_batches):
_, loss_total, loss_cat_ul, loss_rec_ul, loss_gauss_ul, true_labels, predicted_labels = self.sess.run(
[
train_optimizer, model_spec['loss_total'],
model_spec['loss_categ_ul'], model_spec['loss_rec_ul'],
model_spec['loss_gauss_ul'], model_spec['true_labels'],
model_spec['predicted']
],
feed_dict={
self.network.temperature: self.temperature,
self.learning_rate: self.lr
})
# save values
list_predicted_labels.append(predicted_labels)
list_true_labels.append(true_labels)
avg_loss_rec += loss_rec_ul
avg_loss_gauss += loss_gauss_ul
avg_loss_cat += loss_cat_ul
avg_loss_total += loss_total
else:
# validation phase
for j in range(num_batches):
# run the tensorflow flow graph
loss_rec_ul, loss_gauss_ul, loss_cat_ul, loss_total, true_labels, predicted_labels = self.sess.run(
[
model_spec['loss_rec_ul'], model_spec['loss_gauss_ul'],
model_spec['loss_categ_ul'], model_spec['loss_total'],
model_spec['true_labels'], model_spec['predicted']
],
feed_dict={
self.network.temperature: self.temperature,
self.learning_rate: self.lr
})
# save values
list_predicted_labels.append(predicted_labels)
list_true_labels.append(true_labels)
avg_loss_rec += loss_rec_ul
avg_loss_gauss += loss_gauss_ul
avg_loss_cat += loss_cat_ul
avg_loss_total += loss_total
# average values by the given number of batches
avg_loss_rec /= num_batches
avg_loss_gauss /= num_batches
avg_loss_cat /= num_batches
avg_loss_total /= num_batches
# average accuracy and nmi of all the data
predicted_labels = np.hstack(list_predicted_labels)
true_labels = np.hstack(list_true_labels)
avg_nmi = self.metrics.nmi(predicted_labels, true_labels)
avg_accuracy = self.metrics.cluster_acc(predicted_labels, true_labels)
return {
'loss_rec': avg_loss_rec,
'loss_gauss': avg_loss_gauss,
'loss_cat': avg_loss_cat,
'loss_total': avg_loss_total,
'accuracy': avg_accuracy,
'nmi': avg_nmi
}
def train(self, train_data, train_labels, val_data, val_labels):
"""Train the model
Args:
train_data: (array) corresponding array containing the training data
train_labels: (array) corresponding array containing the labels of the training data
val_data: (array) corresponding array containing the validation data
val_labels: (array) corresponding array containing the labels of the validation data
Returns:
output: (dict) contains the history of train/val loss
"""
train_history_loss, val_history_loss = [], []
train_history_acc, val_history_acc = [], []
train_history_nmi, val_history_nmi = [], []
# create training and validation dataset
train_dataset = self.create_dataset(True, train_data, train_labels,
self.batch_size)
val_dataset = self.create_dataset(False, val_data, val_labels,
self.batch_size_val)
self.output_size = train_data.shape[1]
# create train and validation models
train_model = self.create_model(True, train_dataset, self.output_size)
val_model = self.create_model(False, val_dataset, self.output_size)
# set number of batches
num_train_batches = int(
np.ceil(train_data.shape[0] / (1.0 * self.batch_size)))
num_val_batches = int(
np.ceil(val_data.shape[0] / (1.0 * self.batch_size_val)))
# initialize global variables
self.sess.run(train_model['variable_init_op'])
# training and validation phases
print('Training phase...')
for i in range(self.num_epochs):
# decay learning rate according to decay_epoch parameter
if self.decay_epoch > 0 and (i + 1) % self.decay_epoch == 0:
self.lr = self.lr * self.lr_decay
print('Decaying learning rate: %lf' % self.lr)
# evaluate train and validation datasets
train_loss = self.evaluate_dataset(True, num_train_batches,
train_model)
val_loss = self.evaluate_dataset(False, num_val_batches, val_model)
# get training results for printing
train_loss_rec = train_loss['loss_rec']
train_loss_gauss = train_loss['loss_gauss']
train_loss_cat = train_loss['loss_cat']
train_accuracy = train_loss['accuracy']
train_nmi = train_loss['nmi']
train_total_loss = train_loss['loss_total']
# get validation results for printing
val_loss_rec = val_loss['loss_rec']
val_loss_gauss = val_loss['loss_gauss']
val_loss_cat = val_loss['loss_cat']
val_accuracy = val_loss['accuracy']
val_nmi = val_loss['nmi']
val_total_loss = val_loss['loss_total']
# if verbose then print specific information about training
if self.verbose == 1:
print("(Epoch %d / %d)" % (i + 1, self.num_epochs))
print("Train - REC: %.5lf; Gauss: %.5lf; Cat: %.5lf;" % \
(train_loss_rec, train_loss_gauss, train_loss_cat))
print("Valid - REC: %.5lf; Gauss: %.5lf; Cat: %.5lf;" % \
(val_loss_rec, val_loss_gauss, val_loss_cat))
print("Accuracy=Train: %.5lf; Val: %.5lf NMI=Train: %.5lf; Val: %.5lf Total Loss=Train: %.5lf; Val: %.5lf" % \
(train_accuracy, val_accuracy, train_nmi, val_nmi, train_total_loss, val_total_loss))
else:
print("(Epoch %d / %d) Train Loss: %.5lf; Val Loss: %.5lf Train ACC: %.5lf; Val ACC: %.5lf Train NMI: %.5lf; Val NMI: %.5lf" % \
(i + 1, self.num_epochs, train_total_loss, val_total_loss, train_accuracy, val_accuracy, train_nmi, val_nmi))
# save loss and accuracy of each epoch
train_history_loss.append(train_total_loss)
val_history_loss.append(val_total_loss)
train_history_acc.append(train_accuracy)
val_history_acc.append(val_accuracy)
if self.decay_temperature == 1:
# decay temperature of gumbel-softmax
self.temperature = np.maximum(
self.initial_temperature * np.exp(-self.decay_temp_rate *
(i + 1)),
self.min_temperature)
if self.verbose == 1:
print("Gumbel Temperature: %.5lf" % self.temperature)
return {
'train_history_loss': train_history_loss,
'val_history_loss': val_history_loss,
'train_history_acc': train_history_acc,
'val_history_acc': val_history_acc
}
def test(self, test_data, test_labels, batch_size=-1):
"""Test the model with new data
Args:
test_data: (array) corresponding array containing the testing data
test_labels: (array) corresponding array containing the labels of the testing data
batch_size: (int) batch size used to run the model
Return:
accuracy for the given test data
"""
# if batch_size is not specified then use all data
if batch_size == -1:
batch_size = test_data.shape[0]
# create dataset
test_dataset = self.create_dataset(False, test_data, test_labels,
batch_size)
true_labels = test_dataset['labels']
# perform a forward call on the encoder to obtain predicted labels
latent = self.network.encoder(test_dataset['data'], self.num_classes)
logits = latent['logits']
predicted_labels = tf.argmax(logits, axis=1)
# initialize dataset iterator
self.sess.run(test_dataset['iterator_init'])
# calculate number of batches given batch size
num_batches = int(np.ceil(test_data.shape[0] / (1.0 * batch_size)))
# evaluate the model
list_predicted_labels = []
list_true_labels = []
for j in range(num_batches):
_predicted_labels, _true_labels = self.sess.run(
[predicted_labels, true_labels],
feed_dict={
self.network.temperature: self.temperature,
self.learning_rate: self.lr
})
# save values
list_predicted_labels.append(_predicted_labels)
list_true_labels.append(_true_labels)
# average accuracy and nmi of all the data
predicted_labels = np.hstack(list_predicted_labels)
true_labels = np.hstack(list_true_labels)
avg_nmi = self.metrics.nmi(predicted_labels, true_labels)
avg_accuracy = self.metrics.cluster_acc(predicted_labels, true_labels)
return avg_accuracy, avg_nmi
def latent_features(self, data, batch_size=-1):
"""Obtain latent features learnt by the model
Args:
data: (array) corresponding array containing the data
batch_size: (int) size of each batch to consider from the data
Returns:
features: (array) array containing the features from the data
"""
# if batch_size is not specified then use all data
if batch_size == -1:
batch_size = data.shape[0]
# create dataset
dataset = self.create_dataset(False, data, None, batch_size)
# we will use only the encoder network
latent = self.network.encoder(dataset['data'], self.num_classes)
encoder = latent['features']
# obtain the features from the input data
self.sess.run(dataset['iterator_init'])
num_batches = data.shape[0] // batch_size
features = np.zeros((data.shape[0], self.gaussian_size))
for j in range(num_batches):
features[j * batch_size:j * batch_size +
batch_size] = self.sess.run(encoder,
feed_dict={
self.network.temperature:
self.temperature,
self.learning_rate:
self.lr
})
return features
def reconstruct_data(self, data, batch_size=-1):
"""Reconstruct Data
Args:
data: (array) corresponding array containing the data
batch_size: (int) size of each batch to consider from the data
Returns:
reconstructed: (array) array containing the reconstructed data
"""
# if batch_size is not specified then use all data
if batch_size == -1:
batch_size = data.shape[0]
# create dataset
dataset = self.create_dataset(False, data, None, batch_size)
# reuse model used in training
model_spec = self.create_model(False, dataset, data.shape[1])
# obtain the reconstructed data
self.sess.run(model_spec['iterator_init'])
num_batches = data.shape[0] // batch_size
reconstructed = np.zeros(data.shape)
pos = 0
for j in range(num_batches):
reconstructed[pos:pos + batch_size] = self.sess.run(
model_spec['output'],
feed_dict={
self.network.temperature: self.temperature,
self.learning_rate: self.lr
})
pos += batch_size
return reconstructed
def plot_latent_space(self, data, labels, save=False):
"""Plot the latent space learnt by the model
Args:
data: (array) corresponding array containing the data
labels: (array) corresponding array containing the labels
save: (bool) whether to save the latent space plot
Returns:
fig: (figure) plot of the latent space
"""
# obtain the latent features
features = self.latent_features(data)
# plot only the first 2 dimensions
fig = plt.figure(figsize=(8, 6))
plt.scatter(features[:, 0],
features[:, 1],
c=labels,
marker='o',
edgecolor='none',
cmap=plt.cm.get_cmap('jet', 10),
s=10)
plt.colorbar()
if (save):
fig.savefig('latent_space.png')
return fig
def generate_data(self, num_elements=1, category=0):
"""Generate data for a specified category
Args:
num_elements: (int) number of elements to generate
category: (int) category from which we will generate data
Returns:
generated data according to num_elements
"""
indices = (np.ones(num_elements) * category).astype(int).tolist()
# category is specified with a one-hot array
categorical = tf.one_hot(indices, self.num_classes)
# infer the gaussian distribution according to the category
mean, var = self.network.gaussian_from_categorical(categorical)
# gaussian random sample by using the mean and variance
gaussian = tf.random_normal(tf.shape(mean), mean, tf.sqrt(var))
# generate new samples with the given gaussian
_, out = self.network.output_from_gaussian(gaussian, self.output_size)
return self.sess.run(out,
feed_dict={
self.network.temperature: self.temperature,
self.learning_rate: self.lr
})
def random_generation(self, num_elements=1):
"""Random generation for each category
Args:
num_elements: (int) number of elements to generate
Returns:
generated data according to num_elements
"""
# categories for each element
arr = np.array([])
for i in range(self.num_classes):
arr = np.hstack([arr, np.ones(num_elements) * i])
indices = arr.astype(int).tolist()
categorical = tf.one_hot(indices, self.num_classes)
# infer the gaussian distribution according to the category
mean, var = self.network.gaussian_from_categorical(categorical)
# gaussian random sample by using the mean and variance
gaussian = tf.random_normal(tf.shape(mean), mean, tf.sqrt(var))
# generate new samples with the given gaussian
_, out = self.network.output_from_gaussian(gaussian, self.output_size)
return self.sess.run(out,
feed_dict={
self.network.temperature: self.temperature,
self.learning_rate: self.lr
})
class ConvDRAW(object):
def __init__(self, d, lr, lambda_z_wu, read_attn, write_attn, do_classify,
do_reconst):
self.do_classify = do_classify
""" flags for each regularizor """
self.do_reconst = do_reconst
self.read_attn = read_attn
self.write_attn = write_attn
""" dataset information """
self.set_datainfo(d)
""" external toolkits """
self.ls = Layers()
self.lf = LossFunctions(self.ls, self.d, self.encoder)
self.ii = ImageInterface(_is_3d, self.read_attn, self.write_attn,
GLIMPSE_SIZE_READ, GLIMPSE_SIZE_WRITE, _h, _w,
_c)
# for refference from get_loss_kl_draw()
self.T = T
self.L = L
self.Z_SIZES = Z_SIZES
""" placeholders defined outside"""
self.lr = lr
self.lambda_z_wu = lambda_z_wu
"""sequence of canvases """
self.cs = [0] * T
""" initialization """
self.init_lstms()
self.init_time_zero()
""" workaround for variable_scope(reuse=True) """
self.DO_SHARE = None
def set_datainfo(self, d):
self.d = d # dataset manager
global _b, _h, _w, _c, _img_size, _is_3d
_b = d.batch_size
_h = d.h
_w = d.w
_c = d.c
_img_size = d.img_size
_is_3d = d.is_3d
def init_time_zero(self):
self.cs[0] = tf.zeros((_b, _h, _w, _c)) if _is_3d else tf.zeros(
(_b, _img_size))
self.h_dec[0][0] = tf.zeros((_b, RNN_SIZES[0]))
def init_lstms(self):
h_enc, e_mus, e_logsigmas = [[0] * L] * (T + 1), [[0] * L] * (
T + 1), [[0] * L] * (T + 1) # q(z_i+1 | z_i), bottom-up inference
h_dec, d_mus, d_logsigmas = [[0] * L] * (T + 1), [[0] * L] * (
T + 1), [[0] * L] * (T + 1) # q(z_i | .), bidirectional inference
p_mus, p_logsigmas = [[0] * L] * (T + 1), [[0] * L] * (
T + 1) # p(z_i | z_i+1), top-down prior
""" set-up LSTM cells """
e_cells, e_states = [None] * L, [None] * L
d_cells, d_states = [None] * L, [None] * L
for l in range(L):
e_cells[l] = tf.contrib.rnn.core_rnn_cell.LSTMCell(RNN_SIZES[l])
d_cells[l] = tf.contrib.rnn.core_rnn_cell.LSTMCell(RNN_SIZES[l])
e_states[l] = e_cells[l].zero_state(_b, tf.float32)
d_states[l] = d_cells[l].zero_state(_b, tf.float32)
""" set as standard Gaussian, N(0,I). """
d_mus[0][l], d_logsigmas[0][l] = tf.zeros(
(_b, Z_SIZES[l])), tf.zeros((_b, Z_SIZES[l]))
p_mus[0][l], p_logsigmas[0][l] = tf.zeros(
(_b, Z_SIZES[l])), tf.zeros((_b, Z_SIZES[l]))
self.h_enc, self.e_mus, self.e_logsigmas = h_enc, e_mus, e_logsigmas
self.h_dec, self.d_mus, self.d_logsigmas = h_dec, d_mus, d_logsigmas
self.p_mus, self.p_logsigmas = p_mus, p_logsigmas
self.e_cells, self.e_states = e_cells, e_states
self.d_cells, self.d_states = d_cells, d_states
self.z = [[0] * L] * (T + 1)
###########################################
""" LSTM cells """
###########################################
def lstm_encode(self, state, x, l, is_train):
scope = 'lstm_encode_' + str(l)
x = tf.reshape(x, (_b, -1))
if x.get_shape()[1] != RNN_SIZES[l]:
print(scope, ':', x.get_shape()[1:], '=>', RNN_SIZES[l])
x = self.ls.dense(scope, x, RNN_SIZES[l])
return self.e_cells[l](x, state)
def lstm_decode(self, state, x, l, is_train):
scope = 'lstm_decode_' + str(l)
x = tf.reshape(x, (_b, -1))
if x.get_shape()[1] != RNN_SIZES[l]:
print(scope, ':', x.get_shape()[1:], '=>', RNN_SIZES[l])
x = self.ls.dense(scope, x, RNN_SIZES[l])
return self.d_cells[l](x, state)
###########################################
""" Encoder """
###########################################
def encoder(self, x, t, is_train=True, do_update_bn=True):
for l in range(L):
scope = 'Encode_L' + str(l)
with tf.variable_scope(scope, reuse=self.DO_SHARE):
if l == 0:
x_hat = x - self.canvase_previous(t)
h_dec_lowest_prev = self.h_dec[
t - 1][0] if t == 0 else tf.zeros((_b, RNN_SIZES[0]))
input = self.ii.read(x, x_hat, h_dec_lowest_prev)
else:
input = self.h_enc[t][l - 1]
self.h_enc[t][l], self.e_states[l] = self.lstm_encode(
self.e_states[l], input, l, is_train)
input = self.ls.dense(scope, self.h_enc[t][l], Z_SIZES[l] * 2)
self.z[t][l], self.e_mus[t][l], self.e_logsigmas[t][
l] = self.ls.vae_sampler_w_feature_slice(
input, Z_SIZES[l])
""" classifier """
logit = self.ls.dense('top',
self.h_enc[t][-1],
self.d.l,
activation=tf.nn.elu)
return logit
###########################################
""" Decoder """
###########################################
def decoder(self, t, is_train=True, do_update_bn=True):
for l in range(L - 1, -1, -1):
scope = 'Decoder_L' + str(l)
with tf.variable_scope(scope, reuse=self.DO_SHARE):
if l == L - 1:
input = self.z[t][l]
else:
input = self.concat(self.z[t][l], self.h_dec[t][l + 1], l)
self.h_dec[t][l], self.d_states[l] = self.lstm_decode(
self.d_states[l], input, l, is_train)
""" go out to the input space """
if l == 0:
# [ToDo] replace bellow reconstructor with conv-lstm
if _is_3d:
o = self.canvase_previous(t) + self.ii.write(
self.h_dec[t][l])
#if t == T-1: # for MNIST
o = tf.nn.sigmoid(o)
self.cs[t] = o
else:
self.cs[t] = tf.nn.sigmoid(
self.canvase_previous(t) +
self.ii.write(self.h_dec[t][l]))
return self.cs[t]
""" set prior after building the decoder """
def prior(self, t):
for l in range(L - 1, -1, -1):
scope = 'Piror_L' + str(l)
""" preparation for p_* for t+1 and d_* for t with the output from lstm-decoder"""
if l != 0:
input = self.ls.dense(scope, self.h_dec[t][l],
Z_SIZES[l] * 2 + Z_SIZES[l - 1] * 2)
self.p_mus[t + 1][l], self.p_logsigmas[t + 1][l], self.d_mus[
t][l], self.d_logsigmas[t][l] = self.ls.split(
input, 1, [Z_SIZES[l]] * 2 + [Z_SIZES[l - 1]] * 2)
else:
""" no one uses d_* """
input = self.ls.dense(scope, self.h_dec[t][l], Z_SIZES[l] * 2)
self.p_mus[t +
1][l], self.p_logsigmas[t + 1][l] = self.ls.split(
input, 1, [Z_SIZES[l]] * 2)
""" setting p_mus[0][l] and p_logsigmas[0][l] """
if t == 0:
if l == L - 1:
""" has already been performed at init() """
pass
else:
""" by using only decoder's top-down path as prior since p(z) of t-1 does not exist """
self.p_mus[t][l], self.p_logsigmas[t][l] = self.d_mus[t][
l + 1], tf.exp(self.d_logsigmas[t][l +
1]) # Eq.19 at t=0
else:
if l == L - 1:
""" has already been performed at t-1 """
pass
else:
""" update p(z) of current t """
_, self.p_mus[t][l], self.p_logsigmas[t][
l] = self.ls.precision_weighted_sampler(
scope,
(self.p_mus[t][l], tf.exp(self.p_logsigmas[t][l])),
(self.d_mus[t][l + 1],
tf.exp(self.d_logsigmas[t][l + 1]))) # Eq.19
###########################################
""" Build Graph """
###########################################
def build_graph_train(self, x_l, y_l, x, is_supervised=True):
o = dict() # output
loss = 0
logit_ls = []
""" Build DRAW """
for t in range(T):
logit_ls.append(self.encoder(x, t))
x_reconst = self.decoder(t)
self.prior(t)
if t == 0:
self.DO_SHARE = DO_SHARE = True
self.ii.set_do_share(DO_SHARE)
self.ls.set_do_share(DO_SHARE)
""" p(x|z) Reconstruction Loss """
o['x'] = x
o['cs'] = self.cs
o['Lr'] = self.lf.get_loss_pxz(x_reconst, x, 'DiscretizedLogistic')
loss += o['Lr']
""" VAE KL-Divergence Loss """
o['KL1'], o['KL2'], o['Lz'] = self.lf.get_loss_kl_draw(self)
loss += self.lambda_z_wu * o['Lz']
""" set losses """
o['loss'] = loss
self.o_train = o
""" set optimizer """
optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.5)
grads = optimizer.compute_gradients(loss)
for i, (g, v) in enumerate(grads):
if g is not None:
#g = tf.Print(g, [g], "g %s = "%(v))
grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients
else:
print('g is None:', v)
v = tf.Print(v, [v], "v = ", summarize=10000)
self.op = optimizer.apply_gradients(grads) # return train_op
def build_graph_test(self, x_l, y_l, is_supervised=False):
o = dict() # output
loss = 0
logit_ls = []
""" Build DRAW """
for t in range(T):
logit_ls.append(
self.encoder(x_l, t, is_train=False, do_update_bn=False))
x_reconst = self.decoder(t)
self.prior(t)
if t == 0:
self.DO_SHARE = DO_SHARE = True
self.ii.set_do_share(DO_SHARE)
self.ls.set_do_share(DO_SHARE)
""" classification loss """
if is_supervised:
o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_ls[-1], y_l)
loss += o['Ly']
""" for visualizationc """
o['z'], o['y'] = logit_ls[-1], y_l
""" set losses """
o['loss'] = loss
self.o_test = o
###########################################
""" Utilities """
###########################################
def canvase_previous(self, t):
if _is_3d:
c_prev = tf.zeros((_b, _h, _w, _c)) if t == 0 else self.cs[t - 1]
else:
c_prev = tf.zeros((_b, _img_size)) if t == 0 else self.cs[t - 1]
return c_prev
def concat(self, x1, x2, l):
if False: # [ToDo]
x1 = tf.reshape(x1, (_b, IMAGE_SIZES[l][0], IMAGE_SIZES[l][1], -1))
x2 = tf.reshape(x2, (_b, IMAGE_SIZES[l][0], IMAGE_SIZES[l][1], -1))
return tf.concat([x1, x2], 3)
else:
x1 = tf.reshape(x1, (_b, -1))
x2 = tf.reshape(x2, (_b, -1))
return tf.concat([x1, x2], 1)
class SSVAE:
def __init__(self, params):
self.batch_size = params.batch_size
self.batch_size_val = params.batch_size_val
self.initial_temperature = params.temperature
self.decay_temperature = params.decay_temperature
self.num_epochs = params.num_epochs
self.loss_type = params.loss_type
self.num_classes = params.num_classes
self.w_gauss = params.w_gaussian
self.w_categ = params.w_categorical
self.w_recon = params.w_reconstruction
self.decay_temp_rate = params.decay_temp_rate
self.gaussian_size = params.gaussian_size
self.feature_size = params.feature_size
self.min_temperature = params.min_temperature
self.temperature = params.temperature # current temperature
self.verbose = params.verbose
self.sess = tf.Session()
self.network = CatVAENetwork(params)
self.losses = LossFunctions()
self.w_assign = params.w_assign
self.num_labeled = params.num_labeled
self.knn = params.knn
self.metric_loss = params.metric_loss
self.w_metric = tf.placeholder(tf.float32, [])
self.initial_w_metric = params.w_metric
self._w_metric = params.w_metric
self.anneal_metric_loss = params.anneal_w_metric
self.learning_rate = tf.placeholder(tf.float32, [])
self.lr = params.learning_rate
self.decay_epoch = params.decay_epoch
self.lr_decay = params.lr_decay
self.pretrain = params.pretrain
self.num_labeled_batch = params.num_labeled_batch
self.dataset = params.dataset
self.metric_margin = params.metric_margin
def create_dataset(self,
is_training,
data,
labels,
batch_size,
x_labeled=None,
y_labeled=None):
"""Create dataset given input data
Args:
is_training: (bool) whether to use the train or test pipeline.
At training, we shuffle the data and have multiple epochs
data: (array) corresponding array containing the input data
labels: (array) corresponding array containing the labels of the input data
batch_size: (int) size of each batch to consider from the data
x_labeled: (array) corresponding array containing the labeled input data
y_labeled: (array) corresponding array containing the labeles of the labeled input data
Returns:
output: (dict) contains what will be the input of the tensorflow graph
"""
num_samples = data.shape[0]
# create dataset object
if labels is None:
dataset = tf.data.Dataset.from_tensor_slices(data)
else:
dataset = tf.data.Dataset.from_tensor_slices((data, labels))
# shuffle data in training phase
if is_training:
dataset = dataset.shuffle(num_samples).repeat()
dataset = dataset.batch(batch_size)
dataset = dataset.prefetch(1)
# create reinitializable iterator from dataset
iterator = dataset.make_initializable_iterator()
labeled_data = False
if labels is None:
data = iterator.get_next()
else:
data, labels = iterator.get_next()
# append labeled data to each batch
if x_labeled is not None:
labeled_data = True
_data = tf.concat([data, x_labeled], 0)
_labels = tf.concat([labels, y_labeled], 0)
iterator_init = iterator.initializer
if labeled_data:
output = {
'data': _data,
'labels': _labels,
'iterator_init': iterator_init
}
output['labels_semisupervised'] = y_labeled
else:
output = {
'data': data,
'labels': labels,
'iterator_init': iterator_init
}
output['labels_semisupervised'] = None
return output
def create_model(self, is_training, inputs, output_size):
"""Model function defining the graph operations.
Args:
is_training: (bool) whether we are in training phase or not
inputs: (dict) contains the inputs of the graph (features, labels...)
this can be `tf.placeholder` or outputs of `tf.data`
output_size: (int) size of the output layer
Returns:
model_spec: (dict) contains the graph operations or nodes needed for training / evaluation
"""
data, _labels = inputs['data'], inputs['labels']
# create network and obtain latent vectors that will be used in loss functions
latent_spec = self.network.encoder(data, self.num_classes, is_training)
gaussian, mean, logVar = latent_spec['gaussian'], latent_spec[
'mean'], latent_spec['logVar']
categorical, prob, log_prob = latent_spec['categorical'], latent_spec[
'prob_cat'], latent_spec['log_prob_cat']
_logits, features = latent_spec['logits'], latent_spec['features']
output = self.network.decoder(gaussian, categorical, output_size,
is_training)
# reconstruction loss
if self.loss_type == 'bce':
loss_rec = self.losses.binary_cross_entropy(data, output)
elif self.loss_type == 'mse':
loss_rec = tf.losses.mean_squared_error(data, output)
else:
raise "invalid loss function... try bce or mse..."
# kl-divergence loss
loss_kl = self.losses.kl_gaussian(mean, logVar)
loss_kl_cat = self.losses.kl_categorical(prob, log_prob,
self.num_classes)
# auxiliary task to assign labels and regularize the feature space
if _labels is not None:
labeled_ss = inputs['labels_semisupervised']
if labeled_ss is not None:
# assignment loss only if labeled data is available (training phase)
predicted_labels = assign_labels_semisupervised(
features, labeled_ss, self.num_labeled_batch,
self.batch_size, self.num_classes, self.knn)
# use assigned labels and logits to calculate cross entropy loss
loss_assign = tf.losses.sparse_softmax_cross_entropy(
labels=predicted_labels, logits=_logits)
else:
# predict labels from logits or softmax(logits) (validation/testing phase)
loss_assign = tf.constant(0.)
predicted_labels = tf.argmax(prob, axis=1)
# calculate accuracy using the predicted and true labels
accuracy = tf.reduce_mean(
tf.cast(tf.equal(_labels, predicted_labels), tf.float32))
# metric embedding loss
if self.metric_loss == 'triplet':
loss_metric = tf.contrib.losses.metric_learning.triplet_semihard_loss(
predicted_labels, features, margin=self.metric_margin)
elif self.metric_loss == 'lifted':
loss_metric = tf.contrib.losses.metric_learning.lifted_struct_loss(
predicted_labels, features, margin=self.metric_margin)
else:
raise "invalid metric loss... currently we support triplet and lifted loss"
else:
accuracy = tf.constant(0.)
loss_assign = tf.constant(0.)
loss_metric = tf.constant(0.)
predicted_labels = tf.constant(0.)
# variational autoencoder loss
loss_vae = self.w_recon * loss_rec
loss_vae += self.w_gauss * loss_kl
loss_vae += self.w_categ * loss_kl_cat
# total loss
loss_total = loss_vae + self.w_assign * loss_assign + self.w_metric * loss_metric
if is_training:
# use adam for optimization
optimizer = tf.train.AdamOptimizer(self.learning_rate)
# needed for batch normalization layer
update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS)
with tf.control_dependencies(update_ops):
train_op_vae = optimizer.minimize(loss_vae)
train_op_tot = optimizer.minimize(loss_total)
# create model specification
model_spec = inputs
model_spec['variable_init_op'] = tf.global_variables_initializer()
model_spec['output'] = output
model_spec['features'] = features
model_spec['predicted_labels'] = predicted_labels
model_spec['true_labels'] = _labels
model_spec['loss_rec'] = loss_rec
model_spec['loss_kl'] = loss_kl
model_spec['loss_kl_cat'] = loss_kl_cat
model_spec['loss_total'] = loss_total
model_spec['loss_metric'] = loss_metric
model_spec['loss_assign'] = loss_assign
model_spec['accuracy'] = accuracy
# optimizers are only available in training phase
if is_training:
model_spec['train_op'] = train_op_tot
model_spec['train_vae'] = train_op_vae
return model_spec
def evaluate_dataset(self,
is_training,
num_batches,
model_spec,
labeled_data=None,
labeled_labels=None):
"""Evaluate the model
Args:
is_training: (bool) whether we are training or not
num_batches: (int) number of batches to train/test
model_spec: (dict) contains the graph operations or nodes needed for evaluation
labeled_data: (array) corresponding array containing the labeled input data
labeled_labels: (array) corresponding array containing the labeles of the labeled input data
Returns:
(dic) average of loss functions and metrics for the given number of batches
"""
avg_accuracy = 0.0
avg_nmi = 0.0
avg_loss_rec = 0.0
avg_loss_kl = 0.0
avg_loss_cat = 0.0
avg_loss_total = 0.0
avg_loss_metric = 0.0
avg_loss_assign = 0.0
# initialize dataset iteratior
self.sess.run(model_spec['iterator_init'])
if is_training:
# pretraining will train only the variational autoencoder losses
if self.pretrain < 1:
train_optimizer = model_spec['train_op']
else:
train_optimizer = model_spec['train_vae']
# training phase
for j in range(num_batches):
# select randomly subsets of labeled data according to the batch size
_x_labeled, _y_labeled, _, _ = create_semisupervised_dataset(
labeled_data, labeled_labels, self.num_classes,
self.num_labeled_batch)
# run the tensorflow flow graph
_, loss_rec, loss_kl, loss_metric, loss_assign, loss_cat, loss_total, accuracy = self.sess.run(
[
train_optimizer, model_spec['loss_rec'],
model_spec['loss_kl'], model_spec['loss_metric'],
model_spec['loss_assign'], model_spec['loss_kl_cat'],
model_spec['loss_total'], model_spec['accuracy']
],
feed_dict={
self.network.temperature: self.temperature,
self.w_metric: self._w_metric,
self.learning_rate: self.lr,
self.x_labeled: _x_labeled,
self.y_labeled: _y_labeled
})
# accumulate values
avg_accuracy += accuracy
avg_loss_rec += loss_rec
avg_loss_kl += loss_kl
avg_loss_cat += loss_cat
avg_loss_total += loss_total
avg_loss_metric += loss_metric
avg_loss_assign += loss_assign
else:
# validation phase
for j in range(num_batches):
# run the tensorflow flow graph
loss_rec, loss_kl, loss_metric, loss_assign, loss_cat, loss_total, accuracy = self.sess.run(
[
model_spec['loss_rec'], model_spec['loss_kl'],
model_spec['loss_metric'], model_spec['loss_assign'],
model_spec['loss_kl_cat'], model_spec['loss_total'],
model_spec['accuracy']
],
feed_dict={
self.network.temperature: self.temperature,
self.w_metric: self._w_metric,
self.learning_rate: self.lr
})
# accumulate values
avg_accuracy += accuracy
avg_loss_rec += loss_rec
avg_loss_kl += loss_kl
avg_loss_cat += loss_cat
avg_loss_total += loss_total
avg_loss_metric += loss_metric
avg_loss_assign += loss_assign
# average values by the given number of batches
avg_loss_rec /= num_batches
avg_loss_kl /= num_batches
avg_accuracy /= num_batches
avg_loss_cat /= num_batches
avg_loss_total /= num_batches
avg_loss_metric /= num_batches
avg_loss_assign /= num_batches
return {
'avg_loss_rec': avg_loss_rec,
'avg_loss_kl': avg_loss_kl,
'avg_loss_cat': avg_loss_cat,
'total_loss': avg_loss_total,
'avg_accuracy': avg_accuracy,
'avg_loss_metric': avg_loss_metric,
'avg_loss_assign': avg_loss_assign
}
def train(self, train_data, train_labels, val_data, val_labels,
labeled_data, labeled_labels):
"""Train the model
Args:
train_data: (array) corresponding array containing the training data
train_labels: (array) corresponding array containing the labels of the training data
val_data: (array) corresponding array containing the validation data
val_labels: (array) corresponding array containing the labels of the validation data
labeled_data: (array) corresponding array containing the labeled input data
labeled_labels: (array) corresponding array containing the labeles of the labeled input data
Returns:
output: (dict) contains the history of train/val loss
"""
train_history_loss, val_history_loss = [], []
train_history_acc, val_history_acc = [], []
train_history_nmi, val_history_nmi = [], []
# placeholders for the labeled data
self.x_labeled = tf.placeholder(
tf.float32, shape=[self.num_labeled_batch, labeled_data.shape[1]])
self.y_labeled = tf.placeholder(tf.int64,
shape=[self.num_labeled_batch])
# create training and validation dataset
train_dataset = self.create_dataset(
True, train_data, train_labels,
self.batch_size - self.num_labeled_batch, self.x_labeled,
self.y_labeled)
val_dataset = self.create_dataset(True, val_data, val_labels,
self.batch_size_val)
self.output_size = train_data.shape[1]
# create train and validation models
train_model = self.create_model(True, train_dataset, self.output_size)
val_model = self.create_model(False, val_dataset, self.output_size)
# set number of batches
num_train_batches = int(
np.ceil(train_data.shape[0] /
(1.0 * (self.batch_size - self.num_labeled_batch))))
num_val_batches = int(
np.ceil(val_data.shape[0] / (1.0 * self.batch_size_val)))
# initialize global variables
self.sess.run(train_model['variable_init_op'])
# training and validation phases
print('Training phase...')
for i in range(self.num_epochs):
# pretraining at each epoch
if self.pretrain > 0:
self.pretrain = self.pretrain - 1
# decay learning rate according to decay_epoch parameter
if self.decay_epoch > 0 and (i + 1) % self.decay_epoch == 0:
self.lr = self.lr * self.lr_decay
print('Decaying learning rate: %lf' % self.lr)
# evaluate train and validation datasets
train_loss = self.evaluate_dataset(True, num_train_batches,
train_model, labeled_data,
labeled_labels)
val_loss = self.evaluate_dataset(False, num_val_batches, val_model)
# get training results for printing
train_loss_rec = train_loss['avg_loss_rec']
train_loss_kl = train_loss['avg_loss_kl']
train_loss_cat = train_loss['avg_loss_cat']
train_loss_ass = train_loss['avg_loss_assign']
train_loss_met = train_loss['avg_loss_metric']
train_accuracy = train_loss['avg_accuracy']
train_total_loss = train_loss['total_loss']
# get validation results for printing
val_loss_rec = val_loss['avg_loss_rec']
val_loss_kl = val_loss['avg_loss_kl']
val_loss_cat = val_loss['avg_loss_cat']
val_loss_ass = val_loss['avg_loss_assign']
val_loss_met = val_loss['avg_loss_metric']
val_accuracy = val_loss['avg_accuracy']
val_total_loss = val_loss['total_loss']
# if verbose then print specific information about training
if self.verbose == 1:
print("(Epoch %d / %d) REC=Train: %.5lf; Val: %.5lf KL=Train: %.5lf; Val: %.5lf KL-Cat=Train: %.5lf; Val: %.5lf MET=Train: %.5lf; Val: %.5lf ASS=Train: %.5lf; Val: %.5lf ACC=Train %.5lf; Val %.5lf" % \
(i + 1, self.num_epochs, train_loss_rec, val_loss_rec, train_loss_kl, val_loss_kl, train_loss_cat, val_loss_cat, train_loss_met, val_loss_met, train_loss_ass, val_loss_ass, train_accuracy, val_accuracy))
else:
print("(Epoch %d / %d) Train Loss: %.5lf; Val Loss: %.5lf Train Accuracy: %.5lf; Val Accuracy: %.5lf" % \
(i + 1, self.num_epochs, train_total_loss, val_total_loss, train_accuracy, val_accuracy))
# save loss and accuracy of each epoch
train_history_loss.append(train_total_loss)
val_history_loss.append(val_total_loss)
train_history_acc.append(train_accuracy)
val_history_acc.append(val_accuracy)
if self.anneal_metric_loss == 1:
#anneal loss from initial_w_metric to 1 in the first 100 epochs
self._w_metric = np.minimum(
self.initial_w_metric * np.exp(0.06908 * (i + 1)), 1)
if self.verbose == 1:
print('Metric Weight: %.5lf' % self._w_metric)
if self.decay_temperature == 1:
# decay temperature of gumbel-softmax
self.temperature = np.maximum(
self.initial_temperature * np.exp(-self.decay_temp_rate *
(i + 1)),
self.min_temperature)
if self.verbose == 1:
print("Gumbel Temperature: %.5lf" % self.temperature)
return {
'train_history_loss': train_history_loss,
'val_history_loss': val_history_loss,
'train_history_acc': train_history_acc,
'val_history_acc': val_history_acc
}
def test(self, test_data, test_labels, batch_size=-1):
"""Test the model with new data
Args:
test_data: (array) corresponding array containing the testing data
test_labels: (array) corresponding array containing the labels of the testing data
batch_size: (int) batch size used to run the model
Return:
accuracy for the given test data
"""
# if batch_size is not specified then use all data
if batch_size == -1:
batch_size = test_data.shape[0]
# create dataset
test_dataset = self.create_dataset(False, test_data, test_labels,
batch_size)
true_labels = test_dataset['labels']
# perform a forward call on the encoder to obtain predicted labels
latent = self.network.encoder(test_dataset['data'], self.num_classes)
logits = latent['prob_cat']
predicted_labels = tf.argmax(logits, axis=1)
# calculate accuracy given the predicted and true labels
accuracy = tf.reduce_mean(
tf.cast(tf.equal(true_labels, predicted_labels), tf.float32))
# initialize dataset iterator
self.sess.run(test_dataset['iterator_init'])
# calculate number of batches given batch size
num_batches = int(np.ceil(test_data.shape[0] / (1.0 * batch_size)))
# evaluate the model
avg_accuracy = 0.0
for j in range(num_batches):
_accuracy = self.sess.run(accuracy,
feed_dict={
self.network.temperature:
self.temperature,
self.w_metric: self._w_metric,
self.learning_rate: self.lr
})
avg_accuracy += _accuracy
# average the accuracy
avg_accuracy /= num_batches
return avg_accuracy
def latent_features(self, data, batch_size=-1):
"""Obtain latent features learnt by the model
Args:
data: (array) corresponding array containing the data
batch_size: (int) size of each batch to consider from the data
Returns:
features: (array) array containing the features from the data
"""
# if batch_size is not specified then use all data
if batch_size == -1:
batch_size = data.shape[0]
# create dataset
dataset = self.create_dataset(False, data, None, batch_size)
# we will use only the encoder network
latent = self.network.encoder(dataset['data'], self.num_classes)
encoder = latent['features']
# obtain the features from the input data
self.sess.run(dataset['iterator_init'])
num_batches = data.shape[0] // batch_size
features = np.zeros((data.shape[0], self.feature_size))
for j in range(num_batches):
features[j * batch_size:j * batch_size +
batch_size] = self.sess.run(encoder,
feed_dict={
self.network.temperature:
self.temperature,
self.w_metric:
self._w_metric,
self.learning_rate:
self.lr
})
return features
def reconstruct_data(self, data, batch_size=-1):
"""Reconstruct Data
Args:
data: (array) corresponding array containing the data
batch_size: (int) size of each batch to consider from the data
Returns:
reconstructed: (array) array containing the reconstructed data
"""
# if batch_size is not specified then use all data
if batch_size == -1:
batch_size = data.shape[0]
# create dataset
dataset = self.create_dataset(False, data, None, batch_size)
# reuse model used in training
model_spec = self.create_model(False, dataset, data.shape[1])
# obtain the reconstructed data
self.sess.run(model_spec['iterator_init'])
num_batches = data.shape[0] // batch_size
reconstructed = np.zeros(data.shape)
pos = 0
for j in range(num_batches):
reconstructed[pos:pos + batch_size] = self.sess.run(
model_spec['output'],
feed_dict={
self.network.temperature: self.temperature,
self.w_metric: self._w_metric,
self.learning_rate: self.lr
})
pos += batch_size
return reconstructed
def plot_latent_space(self, data, labels, save=False):
"""Plot the latent space learnt by the model
Args:
data: (array) corresponding array containing the data
labels: (array) corresponding array containing the labels
save: (bool) whether to save the latent space plot
Returns:
fig: (figure) plot of the latent space
"""
# obtain the latent features
features = self.latent_features(data)
# plot only the first 2 dimensions
fig = plt.figure(figsize=(8, 6))
plt.scatter(features[:, 0],
features[:, 1],
c=labels,
marker='o',
edgecolor='none',
cmap=plt.cm.get_cmap('jet', 10),
s=10)
plt.colorbar()
if (save):
fig.savefig('latent_space.png')
return fig
def generate_data(self, num_elements=1, category=0):
"""Generate data for a specified category
Args:
num_elements: (int) number of elements to generate
category: (int) category from which we will generate data
Returns:
generated data according to num_elements
"""
# gaussian noise for each element
noise = tf.random_normal([num_elements, self.gaussian_size],
mean=0,
stddev=1,
dtype=tf.float32)
indices = (np.ones(num_elements) * category).astype(int).tolist()
# category is specified with a one-hot array
categorical = tf.one_hot(indices, self.num_classes)
# use the gaussian noise and category to generate data from the generator
out = self.network.decoder(noise, categorical, self.output_size)
return self.sess.run(out,
feed_dict={
self.network.temperature: self.temperature,
self.w_metric: self._w_metric,
self.learning_rate: self.lr
})
def random_generation(self, num_elements=1):
"""Random generation for each category
Args:
num_elements: (int) number of elements to generate
Returns:
generated data according to num_elements
"""
# gaussian noise for each element
noise = tf.random_normal(
[num_elements * self.num_classes, self.gaussian_size],
mean=0,
stddev=1,
dtype=tf.float32)
# categories for each element
arr = np.array([])
for i in range(self.num_classes):
arr = np.hstack([arr, np.ones(num_elements) * i])
indices = arr.astype(int).tolist()
categorical = tf.one_hot(indices, self.num_classes)
# use the gaussian noise and categories to generate data from the generator
out = self.network.decoder(noise, categorical, self.output_size)
return self.sess.run(out,
feed_dict={
self.network.temperature: self.temperature,
self.w_metric: self._w_metric,
self.learning_rate: self.lr
})
def style_generation(self, data):
"""Style transfer generation for each category given a predefined style
Args:
data: (array) corresponding array containing the input style
Returns:
generated data according to the style of data
"""
# convert data to tensor
num_elem = data.shape[0]
data = np.repeat(data, self.num_classes, axis=0)
tf_data = tf.convert_to_tensor(data)
# get latent gaussian features from the encoder
latent = self.network.encoder(tf_data, self.num_classes)
gaussian = latent['gaussian']
# set one-hot values for each category
indices = np.tile(range(self.num_classes), num_elem)
categorical = tf.one_hot(indices, self.num_classes)
# use the gaussian features and categories to generate data from the generator
out = self.network.decoder(gaussian, categorical, self.output_size)
return self.sess.run(out,
feed_dict={
self.network.temperature: self.temperature,
self.w_metric: self._w_metric,
self.learning_rate: self.lr
})
class VAE(object):
def __init__(self, resource):
""" data and external toolkits """
self.d = resource.dh # dataset manager
self.ls = Layers()
self.lf = LossFunctions(self.ls, self.d, self.encoder)
""" placeholders defined outside"""
if c.DO_TRAIN:
self.lr = resource.ph['lr']
def encoder(self, h, is_train, y=None):
if is_train:
_d = self.d
#_ = tf.summary.image('image', tf.reshape(h, [-1, _d.h, _d.w, _d.c]), 10)
scope = 'e_1'
h = self.ls.conv2d(scope + '_1',
h,
128,
filter_size=(2, 2),
strides=(1, 2, 2, 1),
padding="VALID")
h = tf.layers.batch_normalization(h, training=is_train, name=scope)
h = tf.nn.relu(h)
scope = 'e_2'
h = self.ls.conv2d(scope + '_1',
h,
256,
filter_size=(2, 2),
strides=(1, 2, 2, 1),
padding="VALID")
h = tf.layers.batch_normalization(h, training=is_train, name=scope)
h = tf.nn.relu(h)
scope = 'e_3'
h = self.ls.conv2d(scope + '_1',
h,
512,
filter_size=(2, 2),
strides=(1, 2, 2, 1),
padding="VALID")
h = tf.layers.batch_normalization(h, training=is_train, name=scope)
#h = tf.nn.relu(h)
h = tf.nn.tanh(h)
# -> (b, 4, 4, 512)
print('h:', h)
#h = tf.reshape(h, (c.BATCH_SIZE, -1))
h = tf.reshape(h, (-1, 4 * 4 * 512))
print('h:', h)
#sys.exit('aa')
h = self.ls.denseV2('top_of_encoder', h, c.Z_SIZE * 2, activation=None)
print('h:', h)
return self.ls.vae_sampler_w_feature_slice(h, c.Z_SIZE)
def decoder(self, h, is_train):
scope = 'top_of_decoder'
#h = self.ls.denseV2(scope, h, 128, activation=self.ls.lrelu)
h = self.ls.denseV2(scope, h, 512, activation=self.ls.lrelu)
print('h:', scope, h)
h = tf.reshape(h, (-1, 4, 4, 32))
print('h:', scope, h)
scope = 'd_1'
h = self.ls.deconv2d(scope + '_1', h, 512, filter_size=(2, 2))
h = tf.layers.batch_normalization(h, training=is_train, name=scope)
h = tf.nn.relu(h)
print('h:', scope, h)
scope = 'd_2'
h = self.ls.deconv2d(scope + '_2', h, 256, filter_size=(2, 2))
h = tf.layers.batch_normalization(h, training=is_train, name=scope)
h = tf.nn.relu(h)
print('h:', scope, h)
scope = 'd_3'
h = self.ls.deconv2d(scope + '_3', h, 128, filter_size=(2, 2))
h = tf.layers.batch_normalization(h, training=is_train, name=scope)
h = tf.nn.relu(h)
print('h:', scope, h)
scope = 'd_4'
h = self.ls.conv2d(scope + '_4',
h,
3,
filter_size=(1, 1),
strides=(1, 1, 1, 1),
padding="VALID",
activation=tf.nn.sigmoid)
print('h:', scope, h)
return h
def build_graph_train(self, x_l, y_l):
o = dict() # output
loss = 0
if c.IS_AUGMENTATION_ENABLED:
x_l = distorted = self.distort(x_l)
if c.IS_AUG_NOISE_TRUE:
x_l = self.ls.get_corrupted(x_l, 0.15)
z, mu, logsigma = self.encoder(x_l, is_train=True, y=y_l)
x_reconst = self.decoder(z, is_train=True)
""" p(x|z) Reconstruction Loss """
o['Lr'] = self.lf.get_loss_pxz(x_reconst, x_l, 'Bernoulli')
o['x_reconst'] = x_reconst
o['x'] = x_l
loss += o['Lr']
""" VAE KL-Divergence Loss """
LAMBDA_VAE = 0.1
o['mu'], o['logsigma'] = mu, logsigma
# work around. [ToDo] make sure the root cause that makes kl loss inf
#logsigma = tf.clip_by_norm( logsigma, 10)
o['Lz'] = self.lf.get_loss_vae(c.Z_SIZE, mu, logsigma, _lambda=0.0)
loss += LAMBDA_VAE * o['Lz']
""" set losses """
o['loss'] = loss
self.o_train = o
""" set optimizer """
optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.5)
grads = optimizer.compute_gradients(loss)
for i, (g, v) in enumerate(grads):
if g is not None:
#g = tf.Print(g, [g], "g %s = "%(v))
grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients
else:
print('g is None:', v)
v = tf.Print(v, [v], "v = ", summarize=10000)
# update ema in batch_normalization
with tf.control_dependencies(tf.get_collection(
tf.GraphKeys.UPDATE_OPS)):
self.op = optimizer.apply_gradients(grads) # return train_op
def build_graph_test(self, x_l, y_l):
o = dict() # output
loss = 0
z, mu, logsigma = self.encoder(x_l, is_train=False, y=y_l)
x_reconst = self.decoder(mu, is_train=False)
o['x_reconst'] = x_reconst
o['x'] = x_l
#o['Lr'] = self.lf.get_loss_pxz(x_reconst, x_l, 'LeastSquare')
o['Lr'] = self.lf.get_loss_pxz(x_reconst, x_l, 'Bernoulli')
#o['Lr'] = self.lf.get_loss_pxz(x_reconst, x_l, 'DiscretizedLogistic')
#o['Lr'] = tf.reduce_mean(tf.keras.losses.binary_crossentropy(x_l, x_reconst))
loss += o['Lr']
""" set losses """
o['loss'] = loss
self.o_test = o
def distort(self, x):
"""
maybe helpful http://www.redhub.io/Tensorflow/tensorflow-models/src/master/inception/inception/image_processing.py
"""
_d = self.d
def _distort(a_image):
"""
bounding_boxes: A Tensor of type float32.
3-D with shape [batch, N, 4] describing the N bounding boxes associated with the image.
Bounding boxes are supplied and returned as [y_min, x_min, y_max, x_max]
"""
if c.IS_AUG_TRANS_TRUE:
a_image = tf.pad(a_image, [[2, 2], [2, 2], [0, 0]])
a_image = tf.random_crop(a_image, [_d.h, _d.w, _d.c])
if c.IS_AUG_FLIP_TRUE:
a_image = tf.image.random_flip_left_right(a_image)
if c.IS_AUG_ROTATE_TRUE:
from math import pi
radian = tf.random_uniform(shape=(), minval=0,
maxval=360) * pi / 180
a_image = tf.contrib.image.rotate(a_image,
radian,
interpolation='BILINEAR')
if c.IS_AUG_COLOR_TRUE:
a_image = tf.image.random_brightness(a_image, max_delta=0.2)
a_image = tf.image.random_contrast(a_image,
lower=0.2,
upper=1.8)
a_image = tf.image.random_hue(a_image, max_delta=0.2)
if c.IS_AUG_CROP_TRUE:
# shape: [1, 1, 4]
bounding_boxes = tf.constant(
[[[1 / 10, 1 / 10, 9 / 10, 9 / 10]]], dtype=tf.float32)
begin, size, _ = tf.image.sample_distorted_bounding_box(
(_d.h, _d.w, _d.c),
bounding_boxes,
min_object_covered=(9.8 / 10.0),
aspect_ratio_range=[9.5 / 10.0, 10.0 / 9.5])
a_image = tf.slice(a_image, begin, size)
""" for the purpose of distorting not use tf.image.resize_image_with_crop_or_pad under """
a_image = tf.image.resize_images(a_image, [_d.h, _d.w])
""" due to the size of channel returned from tf.image.resize_images is not being given,
specify it manually. """
a_image = tf.reshape(a_image, [_d.h, _d.w, _d.c])
return a_image
""" process batch times in parallel """
return tf.map_fn(_distort, x)
