def __init__(self, args): self.num_epochs = args.epochs self.cuda = args.cuda self.verbose = args.verbose self.batch_size = args.batch_size self.batch_size_val = args.batch_size_val self.learning_rate = args.learning_rate self.decay_epoch = args.decay_epoch self.lr_decay = args.lr_decay self.w_cat = args.w_categ self.w_gauss = args.w_gauss self.w_rec = args.w_rec self.rec_type = args.rec_type self.gaussian_size = args.gaussian_size self.input_size = args.input_size self.output_size = args.output_size self.network = VAENet(self.input_size, self.gaussian_size, self.output_size) self.losses = LossFunctions() self.metrics = Metrics() if self.cuda: self.network = self.network.cuda()
def __init__(self, args): self.num_epochs = args.epochs self.cuda = args.cuda self.verbose = args.verbose self.batch_size = args.batch_size self.batch_size_val = args.batch_size_val self.learning_rate = args.learning_rate self.decay_epoch = args.decay_epoch self.lr_decay = args.lr_decay self.w_cat = args.w_categ self.w_gauss = args.w_gauss self.w_rec = args.w_rec self.rec_type = args.rec_type self.num_classes = args.num_classes self.gaussian_size = args.gaussian_size self.input_size = args.input_size # gumbel self.init_temp = args.init_temp self.decay_temp = args.decay_temp self.hard_gumbel = args.hard_gumbel self.min_temp = args.min_temp self.decay_temp_rate = args.decay_temp_rate self.gumbel_temp = self.init_temp self.network = GMVAENet(self.input_size, self.gaussian_size, self.num_classes) self.losses = LossFunctions() self.metrics = Metrics() if self.cuda: self.network = self.network.cuda()
def __init__(self, d, lr, lambda_z_wu, do_classify, use_kl=True): """ model architecture """ self.MLP_SIZES = [512, 256, 256, 128, 128] self.Z_SIZES = [64, 32, 32, 32, 32] self.L = L = len(self.MLP_SIZES) self.do_classify = do_classify """ flags for regularizers """ self.use_kl = use_kl """ data and external toolkits """ self.d = d # dataset manager self.ls = Layers() self.lf = LossFunctions(self.ls, d, self.encoder) """ placeholders defined outside""" self.lr = lr self.lambda_z_wu = lambda_z_wu """ cache for mu and sigma """ self.e_mus, self.e_logsigmas = [0] * L, [ 0 ] * L # q(z_i+1 | z_i), bottom-up inference as Eq.7-9 self.p_mus, self.p_logsigmas = [0] * L, [ 0 ] * L # p(z_i | z_i+1), top-down prior as Eq.1-3 self.d_mus, self.d_logsigmas = [0] * L, [ 0 ] * L # q(z_i | .), bidirectional inference as Eq.17-19
def __init__(self, params): self.batch_size = params.batch_size self.batch_size_val = params.batch_size_val self.initial_temperature = params.temperature self.decay_temperature = params.decay_temperature self.num_epochs = params.num_epochs self.loss_type = params.loss_type self.num_classes = params.num_classes self.w_gauss = params.w_gaussian self.w_categ = params.w_categorical self.w_recon = params.w_reconstruction self.decay_temp_rate = params.decay_temp_rate self.gaussian_size = params.gaussian_size self.min_temperature = params.min_temperature self.temperature = params.temperature # current temperature self.verbose = params.verbose self.sess = tf.Session() self.network = Networks(params) self.losses = LossFunctions() self.learning_rate = tf.placeholder(tf.float32, []) self.lr = params.learning_rate self.decay_epoch = params.decay_epoch self.lr_decay = params.lr_decay self.dataset = params.dataset self.metrics = Metrics()
def __init__(self, d, lr, lambda_z_wu, read_attn, write_attn, do_classify, do_reconst): self.do_classify = do_classify """ flags for each regularizor """ self.do_reconst = do_reconst self.read_attn = read_attn self.write_attn = write_attn """ dataset information """ self.set_datainfo(d) """ external toolkits """ self.ls = Layers() self.lf = LossFunctions(self.ls, self.d, self.encoder) self.ii = ImageInterface(_is_3d, self.read_attn, self.write_attn, GLIMPSE_SIZE_READ, GLIMPSE_SIZE_WRITE, _h, _w, _c) # for refference from get_loss_kl_draw() self.T = T self.L = L self.Z_SIZES = Z_SIZES """ placeholders defined outside""" self.lr = lr self.lambda_z_wu = lambda_z_wu """sequence of canvases """ self.cs = [0] * T """ initialization """ self.init_lstms() self.init_time_zero() """ workaround for variable_scope(reuse=True) """ self.DO_SHARE = None
def __init__(self, resource): """ data and external toolkits """ self.d = resource.dh # dataset manager self.ls = Layers() self.lf = LossFunctions(self.ls, self.d, self.encoder) """ placeholders defined outside""" if c.DO_TRAIN: self.lr = resource.ph['lr']
def __init__(self, d, lr, lambda_pi_usl, use_pi): """ flags for each regularizor """ self.use_pi = use_pi """ data and external toolkits """ self.d = d # dataset manager self.ls = Layers() self.lf = LossFunctions(self.ls, d, self.encoder) """ placeholders defined outside""" self.lr = lr self.lambda_pi_usl = lambda_pi_usl
def __init__(self, params): self.batch_size = params.batch_size self.batch_size_val = params.batch_size_val self.initial_temperature = params.temperature self.decay_temperature = params.decay_temperature self.num_epochs = params.num_epochs self.loss_type = params.loss_type self.num_classes = params.num_classes self.w_gauss = params.w_gaussian self.w_categ = params.w_categorical self.w_recon = params.w_reconstruction self.decay_temp_rate = params.decay_temp_rate self.gaussian_size = params.gaussian_size self.feature_size = params.feature_size self.min_temperature = params.min_temperature self.temperature = params.temperature # current temperature self.verbose = params.verbose self.sess = tf.Session() self.network = CatVAENetwork(params) self.losses = LossFunctions() self.w_assign = params.w_assign self.num_labeled = params.num_labeled self.knn = params.knn self.metric_loss = params.metric_loss self.w_metric = tf.placeholder(tf.float32, []) self.initial_w_metric = params.w_metric self._w_metric = params.w_metric self.anneal_metric_loss = params.anneal_w_metric self.learning_rate = tf.placeholder(tf.float32, []) self.lr = params.learning_rate self.decay_epoch = params.decay_epoch self.lr_decay = params.lr_decay self.pretrain = params.pretrain self.num_labeled_batch = params.num_labeled_batch self.dataset = params.dataset self.metric_margin = params.metric_margin
class PI(object): def __init__(self, d, lr, lambda_pi_usl, use_pi): """ flags for each regularizor """ self.use_pi = use_pi """ data and external toolkits """ self.d = d # dataset manager self.ls = Layers() self.lf = LossFunctions(self.ls, d, self.encoder) """ placeholders defined outside""" self.lr = lr self.lambda_pi_usl = lambda_pi_usl def encoder(self, x, is_train=True, do_update_bn=True): """ https://arxiv.org/pdf/1610.02242.pdf """ if is_train: h = self.distort(x) h = self.ls.get_corrupted(x, 0.15) else: h = x scope = '1' h = self.ls.conv2d(scope + '_1', h, 128, activation=self.ls.lrelu) h = self.ls.conv2d(scope + '_2', h, 128, activation=self.ls.lrelu) h = self.ls.conv2d(scope + '_3', h, 128, activation=self.ls.lrelu) h = self.ls.max_pool(h) if is_train: h = tf.nn.dropout(h, 0.5) scope = '2' h = self.ls.conv2d(scope + '_1', h, 256, activation=self.ls.lrelu) h = self.ls.conv2d(scope + '_2', h, 256, activation=self.ls.lrelu) h = self.ls.conv2d(scope + '_3', h, 256, activation=self.ls.lrelu) h = self.ls.max_pool(h) if is_train: h = tf.nn.dropout(h, 0.5) scope = '3' h = self.ls.conv2d(scope + '_1', h, 512, activation=self.ls.lrelu) h = self.ls.conv2d(scope + '_2', h, 256, activation=self.ls.lrelu, filter_size=(1, 1)) h = self.ls.conv2d(scope + '_3', h, 128, activation=self.ls.lrelu, filter_size=(1, 1)) h = tf.reduce_mean(h, reduction_indices=[1, 2]) # Global average pooling h = self.ls.dense(scope, h, self.d.l) return h def build_graph_train(self, x_l, y_l, x, is_supervised=True): o = dict() # output loss = 0 logit = self.encoder(x) with tf.variable_scope(tf.get_variable_scope(), reuse=True): logit_l = self.encoder( x_l, is_train=True, do_update_bn=False) # for pyx and vat loss computation """ Classification Loss """ o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_l, y_l) loss += o['Ly'] """ PI Model Loss """ if self.use_pi: with tf.variable_scope(tf.get_variable_scope(), reuse=True): _, _, o['Lp'] = self.lf.get_loss_pi(x, logit, is_train=True) loss += self.lambda_pi_usl * o['Lp'] else: o['Lp'] = tf.constant(0) """ set losses """ o['loss'] = loss self.o_train = o """ set optimizer """ optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.5) #self.op = optimizer.minimize(loss) grads = optimizer.compute_gradients(loss) for i, (g, v) in enumerate(grads): if g is not None: #g = tf.Print(g, [g], "g %s = "%(v)) grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients else: print('g is None:', v) v = tf.Print(v, [v], "v = ", summarize=10000) self.op = optimizer.apply_gradients(grads) # return train_op def build_graph_test(self, x_l, y_l): o = dict() # output loss = 0 logit_l = self.encoder( x_l, is_train=False, do_update_bn=False) # for pyx and vat loss computation """ classification loss """ o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_l, y_l) loss += o['Ly'] """ set losses """ o['loss'] = loss self.o_test = o def distort(self, x): _d = self.d def _distort(a_image): """ bounding_boxes: A Tensor of type float32. 3-D with shape [batch, N, 4] describing the N bounding boxes associated with the image. Bounding boxes are supplied and returned as [y_min, x_min, y_max, x_max] """ # shape: [1, 1, 4] bounding_boxes = tf.constant([[[1 / 10, 1 / 10, 9 / 10, 9 / 10]]], dtype=tf.float32) begin, size, _ = tf.image.sample_distorted_bounding_box( (_d.h, _d.w, _d.c), bounding_boxes, min_object_covered=(8.5 / 10.0), aspect_ratio_range=[7.0 / 10.0, 10.0 / 7.0]) a_image = tf.slice(a_image, begin, size) """ for the purpose of distorting not use tf.image.resize_image_with_crop_or_pad under """ a_image = tf.image.resize_images(a_image, [_d.h, _d.w]) """ due to the size of channel returned from tf.image.resize_images is not being given, specify it manually. """ a_image = tf.reshape(a_image, [_d.h, _d.w, _d.c]) return a_image """ process batch times in parallel """ return tf.map_fn(_distort, x)
class VAE: def __init__(self, args): self.num_epochs = args.epochs self.cuda = args.cuda self.verbose = args.verbose self.batch_size = args.batch_size self.batch_size_val = args.batch_size_val self.learning_rate = args.learning_rate self.decay_epoch = args.decay_epoch self.lr_decay = args.lr_decay self.w_cat = args.w_categ self.w_gauss = args.w_gauss self.w_rec = args.w_rec self.rec_type = args.rec_type self.gaussian_size = args.gaussian_size self.input_size = args.input_size self.output_size = args.output_size self.network = VAENet(self.input_size, self.gaussian_size, self.output_size) self.losses = LossFunctions() self.metrics = Metrics() if self.cuda: self.network = self.network.cuda() # =============================================================================== # This is where all the losses are computed; helper method for train_epoch. def unlabeled_loss(self, chunk, out_net): # obtain network variables z, data_recon = out_net['gaussian'], out_net['x_rec'] mu, var = out_net['mean'], out_net['var'] # reconstruction loss loss_rec = self.losses.reconstruction_loss(chunk, data_recon, self.rec_type) # gaussian loss loss_gauss = self.losses.kl_divergence_v2(mu, var) # total loss loss_total = self.w_rec * loss_rec + self.w_gauss * loss_gauss loss_dic = { 'total': loss_total, 'reconstruction': loss_rec, 'gaussian': loss_gauss, } return loss_dic # =============================================================================== # =============================================================================== # Train the model for one epoch; helper method for train. def train_epoch(self, optimizer, data_loader): self.network.train() total_loss = 0. recon_loss = 0. gauss_loss = 0. accuracy = 0. num_batches = 0. for (path, chunk) in data_loader: if self.cuda == 1: path = path.cuda() chunk = chunk.cuda() optimizer.zero_grad() # flatten data path = path.view(path.size(0), -1) chunk = chunk.view(chunk.size(0), -1) # forward call out_net = self.network(path) unlab_loss_dic = self.unlabeled_loss(chunk, out_net) total = unlab_loss_dic['total'] # accumulate values total_loss += total.item() recon_loss += unlab_loss_dic['reconstruction'].item() gauss_loss += unlab_loss_dic['gaussian'].item() # perform backpropagation total.backward() optimizer.step() num_batches += 1. # average per batch total_loss /= num_batches recon_loss /= num_batches gauss_loss /= num_batches return total_loss, recon_loss, gauss_loss # ============================================================================== # ============================================================================== # Test the model. This method is essentially the same as train_epoch except that # self.network.eval is called at the beginning of this method. def test(self, data_loader): self.network.eval() total_loss = 0. recon_loss = 0. gauss_loss = 0. accuracy = 0. num_batches = 0. with torch.no_grad(): for path, chunk in data_loader: if self.cuda == 1: path = path.cuda() chunk = chunk.cuda() # flatten data path = path.view(path.size(0), -1) chunk = chunk.view(chunk.size(0), -1) # forward call out_net = self.network(path) unlab_loss_dic = self.unlabeled_loss(chunk, out_net) # accumulate values total_loss += unlab_loss_dic['total'].item() recon_loss += unlab_loss_dic['reconstruction'].item() gauss_loss += unlab_loss_dic['gaussian'].item() num_batches += 1. # average per batch total_loss /= num_batches recon_loss /= num_batches gauss_loss /= num_batches return total_loss, recon_loss, gauss_loss # ============================================================================== def train(self, train_loader, val_loader): """Train the model Args: train_loader: (DataLoader) corresponding loader containing the training data val_loader: (DataLoader) corresponding loader containing the validation data Returns: output: (dict) contains the history of train/val loss """ optimizer = optim.Adam(self.network.parameters(), lr=self.learning_rate) train_total_losses = [] train_recon_losses = [] train_gauss_losses = [] valid_total_losses = [] valid_recon_losses = [] valid_gauss_losses = [] for epoch in range(1, self.num_epochs + 1): train_total_loss, train_recon_loss, train_gauss_loss = self.train_epoch( optimizer, train_loader) valid_total_loss, valid_recon_loss, valid_gauss_loss = self.test( val_loader) print( f"Epoch {epoch:5} / {self.num_epochs} | Total loss: {round(valid_total_loss, 2):7} | Recon loss: {round(valid_recon_loss, 2):7} | Gauss loss: {round(valid_gauss_loss, 2):7}" ) train_total_losses.append(train_total_loss) train_recon_losses.append(train_recon_loss) train_gauss_losses.append(train_gauss_loss) valid_total_losses.append(valid_total_loss) valid_recon_losses.append(valid_recon_loss) valid_gauss_losses.append(valid_gauss_loss) return { 'train_total_losses': train_total_losses, 'train_recon_losses': train_recon_losses, 'train_gauss_losses': train_gauss_losses, 'valid_total_losses': valid_total_losses, 'valid_recon_losses': valid_recon_losses, 'valid_gauss_losses': valid_gauss_losses, } def latent_features(self, data_loader): """Obtain latent features learnt by the model Args: data_loader: (DataLoader) loader containing the data return_labels: (boolean) whether to return true labels or not Returns: features: (array) array containing the features from the data """ self.network.eval() N = len(data_loader.dataset) features = np.zeros((N, self.gaussian_size)) start_ind = 0 with torch.no_grad(): for (data, ) in data_loader: if self.cuda == 1: data = data.cuda() data = data.view(data.size(0), -1) out = self.network(data) latent_feat = out['mean'] end_ind = min(start_ind + data.size(0), N + 1) features[start_ind:end_ind] = latent_feat.cpu().numpy() start_ind += data.size(0) return features def reconstruct_data(self, data_loader, sample_size=-1): self.network.eval() # sample random data from loader indices = np.random.randint(0, len(data_loader.dataset), size=sample_size) test_random_loader = torch.utils.data.DataLoader( data_loader.dataset, batch_size=sample_size, sampler=SubsetRandomSampler(indices)) # obtain values it = iter(test_random_loader) test_batch_data, _ = it.next() original = test_batch_data.data.numpy() if self.cuda: test_batch_data = test_batch_data.cuda() # obtain reconstructed data out = self.network(test_batch_data) reconstructed = out['x_rec'] return original, reconstructed.data.cpu().numpy() def plot_latent_space(self, data_loader, save=False): """Plot the latent space learnt by the model Args: data: (array) corresponding array containing the data labels: (array) corresponding array containing the labels save: (bool) whether to save the latent space plot Returns: fig: (figure) plot of the latent space """ # obtain the latent features features = self.latent_features(data_loader) # plot only the first 2 dimensions fig = plt.figure(figsize=(8, 6)) plt.scatter(features[:, 0], features[:, 1], c=labels, marker='o', edgecolor='none', cmap=plt.cm.get_cmap('jet', 10), s=10) plt.colorbar() if (save): fig.savefig('latent_space.png') return fig def random_generation(self, num_chunks): latent_vecs = torch.randn(num_chunks, 64).cuda() chunks = self.network.generative.pxz(latent_vecs) return chunks.cpu().detach().numpy()
class GMVAE: def __init__(self, args): self.num_epochs = args.epochs self.cuda = args.cuda self.verbose = args.verbose self.batch_size = args.batch_size self.batch_size_val = args.batch_size_val self.learning_rate = args.learning_rate self.decay_epoch = args.decay_epoch self.lr_decay = args.lr_decay self.w_cat = args.w_categ self.w_gauss = args.w_gauss self.w_rec = args.w_rec self.rec_type = args.rec_type self.num_classes = args.num_classes self.gaussian_size = args.gaussian_size self.input_size = args.input_size # gumbel self.init_temp = args.init_temp self.decay_temp = args.decay_temp self.hard_gumbel = args.hard_gumbel self.min_temp = args.min_temp self.decay_temp_rate = args.decay_temp_rate self.gumbel_temp = self.init_temp self.network = GMVAENet(self.input_size, self.gaussian_size, self.num_classes) self.losses = LossFunctions() self.metrics = Metrics() if self.cuda: self.network = self.network.cuda() def unlabeled_loss(self, data, out_net): """Method defining the loss functions derived from the variational lower bound Args: data: (array) corresponding array containing the input data out_net: (dict) contains the graph operations or nodes of the network output Returns: loss_dic: (dict) contains the values of each loss function and predictions """ # obtain network variables z, data_recon = out_net['gaussian'], out_net['x_rec'] logits, prob_cat = out_net['logits'], out_net['prob_cat'] y_mu, y_var = out_net['y_mean'], out_net['y_var'] mu, var = out_net['mean'], out_net['var'] # reconstruction loss loss_rec = self.losses.reconstruction_loss(data, data_recon, self.rec_type) # gaussian loss loss_gauss = self.losses.gaussian_loss(z, mu, var, y_mu, y_var) # categorical loss loss_cat = -self.losses.entropy(logits, prob_cat) - np.log(0.1) # total loss loss_total = self.w_rec * loss_rec + self.w_gauss * loss_gauss + self.w_cat * loss_cat # obtain predictions _, predicted_labels = torch.max(logits, dim=1) loss_dic = { 'total': loss_total, 'predicted_labels': predicted_labels, 'reconstruction': loss_rec, 'gaussian': loss_gauss, 'categorical': loss_cat } return loss_dic def train_epoch(self, optimizer, data_loader): """Train the model for one epoch Args: optimizer: (Optim) optimizer to use in backpropagation data_loader: (DataLoader) corresponding loader containing the training data Returns: average of all loss values, accuracy, nmi """ self.network.train() total_loss = 0. recon_loss = 0. cat_loss = 0. gauss_loss = 0. accuracy = 0. nmi = 0. num_batches = 0. true_labels_list = [] predicted_labels_list = [] # iterate over the dataset for (data, labels) in data_loader: if self.cuda == 1: data = data.cuda() optimizer.zero_grad() # flatten data data = data.view(data.size(0), -1) # forward call out_net = self.network(data, self.gumbel_temp, self.hard_gumbel) unlab_loss_dic = self.unlabeled_loss(data, out_net) total = unlab_loss_dic['total'] # accumulate values total_loss += total.item() recon_loss += unlab_loss_dic['reconstruction'].item() gauss_loss += unlab_loss_dic['gaussian'].item() cat_loss += unlab_loss_dic['categorical'].item() # perform backpropagation total.backward() optimizer.step() # save predicted and true labels predicted = unlab_loss_dic['predicted_labels'] true_labels_list.append(labels) predicted_labels_list.append(predicted) num_batches += 1. # average per batch total_loss /= num_batches recon_loss /= num_batches gauss_loss /= num_batches cat_loss /= num_batches # concat all true and predicted labels true_labels = torch.cat(true_labels_list, dim=0).cpu().numpy() predicted_labels = torch.cat(predicted_labels_list, dim=0).cpu().numpy() # compute metrics accuracy = 100.0 * self.metrics.cluster_acc(predicted_labels, true_labels) nmi = 100.0 * self.metrics.nmi(predicted_labels, true_labels) return total_loss, recon_loss, gauss_loss, cat_loss, accuracy, nmi def test(self, data_loader, return_loss=False): """Test the model with new data Args: data_loader: (DataLoader) corresponding loader containing the test/validation data return_loss: (boolean) whether to return the average loss values Return: accuracy and nmi for the given test data """ self.network.eval() total_loss = 0. recon_loss = 0. cat_loss = 0. gauss_loss = 0. accuracy = 0. nmi = 0. num_batches = 0. true_labels_list = [] predicted_labels_list = [] with torch.no_grad(): for data, labels in data_loader: if self.cuda == 1: data = data.cuda() # flatten data data = data.view(data.size(0), -1) # forward call out_net = self.network(data, self.gumbel_temp, self.hard_gumbel) unlab_loss_dic = self.unlabeled_loss(data, out_net) # accumulate values total_loss += unlab_loss_dic['total'].item() recon_loss += unlab_loss_dic['reconstruction'].item() gauss_loss += unlab_loss_dic['gaussian'].item() cat_loss += unlab_loss_dic['categorical'].item() # save predicted and true labels predicted = unlab_loss_dic['predicted_labels'] true_labels_list.append(labels) predicted_labels_list.append(predicted) num_batches += 1. # average per batch if return_loss: total_loss /= num_batches recon_loss /= num_batches gauss_loss /= num_batches cat_loss /= num_batches # concat all true and predicted labels true_labels = torch.cat(true_labels_list, dim=0).cpu().numpy() predicted_labels = torch.cat(predicted_labels_list, dim=0).cpu().numpy() # compute metrics accuracy = 100.0 * self.metrics.cluster_acc(predicted_labels, true_labels) nmi = 100.0 * self.metrics.nmi(predicted_labels, true_labels) if return_loss: return total_loss, recon_loss, gauss_loss, cat_loss, accuracy, nmi else: return accuracy, nmi def train(self, train_loader, val_loader): """Train the model Args: train_loader: (DataLoader) corresponding loader containing the training data val_loader: (DataLoader) corresponding loader containing the validation data Returns: output: (dict) contains the history of train/val loss """ optimizer = optim.Adam(self.network.parameters(), lr=self.learning_rate) train_history_acc, val_history_acc = [], [] train_history_nmi, val_history_nmi = [], [] for epoch in range(1, self.num_epochs + 1): train_loss, train_rec, train_gauss, train_cat, train_acc, train_nmi = self.train_epoch( optimizer, train_loader) val_loss, val_rec, val_gauss, val_cat, val_acc, val_nmi = self.test( val_loader, True) # if verbose then print specific information about training if self.verbose == 1: print("(Epoch %d / %d)" % (epoch, self.num_epochs)) print("Train - REC: %.5lf; Gauss: %.5lf; Cat: %.5lf;" % \ (train_rec, train_gauss, train_cat)) print("Valid - REC: %.5lf; Gauss: %.5lf; Cat: %.5lf;" % \ (val_rec, val_gauss, val_cat)) print("Accuracy=Train: %.5lf; Val: %.5lf NMI=Train: %.5lf; Val: %.5lf Total Loss=Train: %.5lf; Val: %.5lf" % \ (train_acc, val_acc, train_nmi, val_nmi, train_loss, val_loss)) else: print('(Epoch %d / %d) Train_Loss: %.3lf; Val_Loss: %.3lf Train_ACC: %.3lf; Val_ACC: %.3lf Train_NMI: %.3lf; Val_NMI: %.3lf' % \ (epoch, self.num_epochs, train_loss, val_loss, train_acc, val_acc, train_nmi, val_nmi)) # decay gumbel temperature if self.decay_temp == 1: self.gumbel_temp = np.maximum( self.init_temp * np.exp(-self.decay_temp_rate * epoch), self.min_temp) if self.verbose == 1: print("Gumbel Temperature: %.3lf" % self.gumbel_temp) train_history_acc.append(train_acc) val_history_acc.append(val_acc) train_history_nmi.append(train_nmi) val_history_nmi.append(val_nmi) return { 'train_history_nmi': train_history_nmi, 'val_history_nmi': val_history_nmi, 'train_history_acc': train_history_acc, 'val_history_acc': val_history_acc } def latent_features(self, data_loader, return_labels=False): """Obtain latent features learnt by the model Args: data_loader: (DataLoader) loader containing the data return_labels: (boolean) whether to return true labels or not Returns: features: (array) array containing the features from the data """ self.network.eval() N = len(data_loader.dataset) features = np.zeros((N, self.gaussian_size)) if return_labels: true_labels = np.zeros(N, dtype=np.int64) start_ind = 0 with torch.no_grad(): for (data, labels) in data_loader: if self.cuda == 1: data = data.cuda() # flatten data data = data.view(data.size(0), -1) out = self.network.inference(data, self.gumbel_temp, self.hard_gumbel) latent_feat = out['mean'] end_ind = min(start_ind + data.size(0), N + 1) # return true labels if return_labels: true_labels[start_ind:end_ind] = labels.cpu().numpy() features[start_ind:end_ind] = latent_feat.cpu().detach().numpy( ) start_ind += data.size(0) if return_labels: return features, true_labels return features def reconstruct_data(self, data_loader, sample_size=-1): """Reconstruct Data Args: data_loader: (DataLoader) loader containing the data sample_size: (int) size of random data to consider from data_loader Returns: reconstructed: (array) array containing the reconstructed data """ self.network.eval() # sample random data from loader indices = np.random.randint(0, len(data_loader.dataset), size=sample_size) test_random_loader = torch.utils.data.DataLoader( data_loader.dataset, batch_size=sample_size, sampler=SubsetRandomSampler(indices)) # obtain values it = iter(test_random_loader) test_batch_data, _ = it.next() original = test_batch_data.data.numpy() if self.cuda: test_batch_data = test_batch_data.cuda() # obtain reconstructed data out = self.network(test_batch_data, self.gumbel_temp, self.hard_gumbel) reconstructed = out['x_rec'] return original, reconstructed.data.cpu().numpy() def plot_latent_space(self, data_loader, save=False): """Plot the latent space learnt by the model Args: data: (array) corresponding array containing the data labels: (array) corresponding array containing the labels save: (bool) whether to save the latent space plot Returns: fig: (figure) plot of the latent space """ # obtain the latent features features = self.latent_features(data_loader) # plot only the first 2 dimensions fig = plt.figure(figsize=(8, 6)) plt.scatter(features[:, 0], features[:, 1], c=labels, marker='o', edgecolor='none', cmap=plt.cm.get_cmap('jet', 10), s=10) plt.colorbar() if (save): fig.savefig('latent_space.png') return fig def random_generation(self, num_elements=1): """Random generation for each category Args: num_elements: (int) number of elements to generate Returns: generated data according to num_elements """ # categories for each element arr = np.array([]) for i in range(self.num_classes): arr = np.hstack([arr, np.ones(num_elements) * i]) indices = arr.astype(int).tolist() categorical = F.one_hot(torch.tensor(indices), self.num_classes).float() if self.cuda: categorical = categorical.cuda() # infer the gaussian distribution according to the category mean, var = self.network.generative.pzy(categorical) # gaussian random sample by using the mean and variance noise = torch.randn_like(var) std = torch.sqrt(var) gaussian = mean + noise * std # generate new samples with the given gaussian generated = self.network.generative.pxz(gaussian) return generated.cpu().detach().numpy()
class LVAE(object): def __init__(self, d, lr, lambda_z_wu, do_classify, use_kl=True): """ model architecture """ self.MLP_SIZES = [512, 256, 256, 128, 128] self.Z_SIZES = [64, 32, 32, 32, 32] self.L = L = len(self.MLP_SIZES) self.do_classify = do_classify """ flags for regularizers """ self.use_kl = use_kl """ data and external toolkits """ self.d = d # dataset manager self.ls = Layers() self.lf = LossFunctions(self.ls, d, self.encoder) """ placeholders defined outside""" self.lr = lr self.lambda_z_wu = lambda_z_wu """ cache for mu and sigma """ self.e_mus, self.e_logsigmas = [0] * L, [ 0 ] * L # q(z_i+1 | z_i), bottom-up inference as Eq.7-9 self.p_mus, self.p_logsigmas = [0] * L, [ 0 ] * L # p(z_i | z_i+1), top-down prior as Eq.1-3 self.d_mus, self.d_logsigmas = [0] * L, [ 0 ] * L # q(z_i | .), bidirectional inference as Eq.17-19 def encoder(self, x, is_train=True, do_update_bn=True): h = x for l in range(self.L): scope = 'Encode_L' + str(l) h = self.ls.dense(scope, h, self.MLP_SIZES[l]) h = self.ls.bn(scope, h, is_train, do_update_bn, name=scope) h = tf.nn.elu(h) """ prepare for bidirectional inference """ _, self.e_mus[l], self.e_logsigmas[l] = self.ls.vae_sampler( scope, h, self.Z_SIZES[l], tf.nn.softplus) # Eq.13-15 #return h return self.e_mus[-1] def decoder(self, is_train=True, do_update_bn=True): for l in range(self.L - 1, -1, -1): scope = 'Decoder_L' + str(l) if l == self.L - 1: """ At the highest latent layer, mu & sigma are identical to those outputed from encoer. And making actual z is not necessary for the highest layer.""" mu, logsigma = self.e_mus[l], self.e_logsigmas[l] self.d_mus[l], self.d_logsigmas[l] = mu, logsigma z = self.ls.sampler(self.d_mus[l], tf.exp(self.d_logsigmas[l])) """ prior of z_L is set as standard Gaussian, N(0,I). """ self.p_mus[l], self.p_logsigmas[l] = tf.zeros( (mu.get_shape())), tf.zeros((logsigma.get_shape())) else: """ prior is developed from z of the above layer """ _, self.p_mus[l], self.p_logsigmas[l] = self.ls.vae_sampler( scope, z, self.Z_SIZES[l], tf.nn.softplus) # Eq.13-15 z, self.d_mus[l], self.d_logsigmas[ l] = self.ls.precision_weighted_sampler( scope, (self.e_mus[l], tf.exp(self.e_logsigmas[l])), (self.p_mus[l], tf.exp( self.p_logsigmas[l]))) # Eq.17-19 """ go out to the input space """ _d = self.d x = self.ls.dense('bottom', z, _d.img_size, tf.nn.elu) # reconstructed input if _d.is_3d: x = tf.reshape(x, (-1, _d.h, _d.w, _d.c)) return x def build_graph_train(self, x_l, y_l, x): o = dict() # output loss = 0 logit = self.encoder(x) x_reconst = self.decoder() with tf.variable_scope(tf.get_variable_scope(), reuse=True): logit_l = self.encoder( x_l, is_train=True, do_update_bn=False) # for pyx and vat loss computation """ Classification Loss """ if self.do_classify: o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_l, y_l) loss += o['Ly'] """ for visualizationc """ o['z'], o['y'] = logit, y_l """ p(x|z) Reconstruction Loss """ o['Lr'] = self.lf.get_loss_pxz(x_reconst, x, 'DiscretizedLogistic') loss += o['Lr'] o['x'] = x o['cs'] = x_reconst """ VAE KL-Divergence Loss """ if self.use_kl: o['KL1'], o['KL2'], o['Lz'] = self.lf.get_loss_kl(self, _lambda=10.0) loss += self.lambda_z_wu * o['Lz'] else: o['KL1'], o['KL2'], o['Lz'] = tf.constant(0), tf.constant( 0), tf.constant(0) """ set losses """ o['loss'] = loss self.o_train = o """ set optimizer """ optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.5) #self.op = optimizer.minimize(loss) grads = optimizer.compute_gradients(loss) for i, (g, v) in enumerate(grads): if g is not None: #g = tf.Print(g, [g], "g %s = "%(v)) grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients else: print('g is None:', v) v = tf.Print(v, [v], "v = ", summarize=10000) #for v in tf.all_variables(): print("%s : %s" % (v.name,v.get_shape())) self.op = optimizer.apply_gradients(grads) # return train_op def build_graph_test(self, x_l, y_l): o = dict() # output loss = 0 logit_l = self.encoder( x_l, is_train=False, do_update_bn=False) # for pyx and vat loss computation """ classification loss """ if self.do_classify: o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_l, y_l) loss += o['Ly'] """ for visualizationc """ o['z'], o['y'] = logit_l, y_l """ set losses """ o['loss'] = loss self.o_test = o
class GMVAE: def __init__(self, params): self.batch_size = params.batch_size self.batch_size_val = params.batch_size_val self.initial_temperature = params.temperature self.decay_temperature = params.decay_temperature self.num_epochs = params.num_epochs self.loss_type = params.loss_type self.num_classes = params.num_classes self.w_gauss = params.w_gaussian self.w_categ = params.w_categorical self.w_recon = params.w_reconstruction self.decay_temp_rate = params.decay_temp_rate self.gaussian_size = params.gaussian_size self.min_temperature = params.min_temperature self.temperature = params.temperature # current temperature self.verbose = params.verbose self.sess = tf.Session() self.network = Networks(params) self.losses = LossFunctions() self.learning_rate = tf.placeholder(tf.float32, []) self.lr = params.learning_rate self.decay_epoch = params.decay_epoch self.lr_decay = params.lr_decay self.dataset = params.dataset self.metrics = Metrics() def create_dataset(self, is_training, data, labels, batch_size): """Create dataset given input data Args: is_training: (bool) whether to use the train or test pipeline. At training, we shuffle the data and have multiple epochs data: (array) corresponding array containing the input data labels: (array) corresponding array containing the labels of the input data batch_size: (int) size of each batch to consider from the data Returns: output: (dict) contains what will be the input of the tensorflow graph """ num_samples = data.shape[0] # create dataset object if labels is None: dataset = tf.data.Dataset.from_tensor_slices(data) else: dataset = tf.data.Dataset.from_tensor_slices((data, labels)) # shuffle data in training phase if is_training: dataset = dataset.shuffle(num_samples).repeat() dataset = dataset.batch(batch_size) dataset = dataset.prefetch(1) # create reinitializable iterator from dataset iterator = dataset.make_initializable_iterator() if labels is None: data = iterator.get_next() else: data, labels = iterator.get_next() iterator_init = iterator.initializer output = { 'data': data, 'labels': labels, 'iterator_init': iterator_init } return output def unlabeled_loss(self, data, latent_spec, output_size, is_training=True): """Model function defining the loss functions derived from the variational lower bound Args: data: (array) corresponding array containing the input data latent_spec: (dict) contains the graph operations or nodes of the latent variables output_size: (int) size of the output layer is_training: (bool) whether we are in training phase or not Returns: loss_dic: (dict) contains the values of each loss function and predictions """ gaussian, mean, var = latent_spec['gaussian'], latent_spec[ 'mean'], latent_spec['var'] categorical, prob, log_prob = latent_spec['categorical'], latent_spec[ 'prob_cat'], latent_spec['log_prob_cat'] _logits, features = latent_spec['logits'], latent_spec['features'] output, y_mean, y_var = latent_spec['output'], latent_spec[ 'y_mean'], latent_spec['y_var'] # reconstruction loss if self.loss_type == 'bce': loss_rec = self.w_recon * self.losses.binary_cross_entropy( data, output) elif self.loss_type == 'mse': loss_rec = self.w_recon * tf.losses.mean_squared_error( data, output) else: raise "invalid loss function... try bce or mse..." # gaussian loss loss_gaussian = self.w_gauss * self.losses.labeled_loss( gaussian, mean, var, y_mean, y_var) # categorical loss loss_categorical = self.w_categ * -self.losses.entropy(_logits, prob) # obtain predictions predicted_labels = tf.argmax(_logits, axis=1) # total_loss loss_total = loss_rec + loss_gaussian + loss_categorical loss_dic = { 'total': loss_total, 'predicted_labels': predicted_labels, 'reconstruction': loss_rec, 'gaussian': loss_gaussian, 'categorical': loss_categorical } return loss_dic def create_model(self, is_training, inputs, output_size): """Model function defining the graph operations. Args: is_training: (bool) whether we are in training phase or not inputs: (dict) contains the inputs of the graph (features, labels...) this can be `tf.placeholder` or outputs of `tf.data` output_size: (int) size of the output layer Returns: model_spec: (dict) contains the graph operations or nodes needed for training / evaluation """ data, _labels = inputs['data'], inputs['labels'] latent_spec = self.network.encoder(data, self.num_classes, is_training) out_logits, y_mean, y_var, output = self.network.decoder( latent_spec['gaussian'], latent_spec['categorical'], output_size, is_training) latent_spec['output'] = out_logits latent_spec['y_mean'] = y_mean latent_spec['y_var'] = y_var # unlabeled losses unlabeled_loss_dic = self.unlabeled_loss(data, latent_spec, output_size, is_training) loss_total = unlabeled_loss_dic['total'] if is_training: # use adam for optimization optimizer = tf.train.AdamOptimizer(self.learning_rate) # needed for batch normalization layer update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op = optimizer.minimize(loss_total) # create model specification model_spec = inputs model_spec['variable_init_op'] = tf.global_variables_initializer() # optimizers are only available in training phase if is_training: model_spec['train_op'] = train_op else: model_spec['output'] = output model_spec['loss_total'] = loss_total model_spec['loss_rec_ul'] = unlabeled_loss_dic['reconstruction'] model_spec['loss_gauss_ul'] = unlabeled_loss_dic['gaussian'] model_spec['loss_categ_ul'] = unlabeled_loss_dic['categorical'] model_spec['true_labels'] = _labels model_spec['predicted'] = unlabeled_loss_dic['predicted_labels'] return model_spec def evaluate_dataset(self, is_training, num_batches, model_spec): """Evaluate the model Args: is_training: (bool) whether we are training or not num_batches: (int) number of batches to train/test model_spec: (dict) contains the graph operations or nodes needed for evaluation Returns: (dic) average of loss functions and metrics for the given number of batches """ avg_accuracy = 0.0 avg_nmi = 0.0 avg_loss_cat = 0.0 avg_loss_total = 0.0 avg_loss_rec = 0.0 avg_loss_gauss = 0.0 list_predicted_labels = [] list_true_labels = [] # initialize dataset iteratior self.sess.run(model_spec['iterator_init']) if is_training: train_optimizer = model_spec['train_op'] # training phase for j in range(num_batches): _, loss_total, loss_cat_ul, loss_rec_ul, loss_gauss_ul, true_labels, predicted_labels = self.sess.run( [ train_optimizer, model_spec['loss_total'], model_spec['loss_categ_ul'], model_spec['loss_rec_ul'], model_spec['loss_gauss_ul'], model_spec['true_labels'], model_spec['predicted'] ], feed_dict={ self.network.temperature: self.temperature, self.learning_rate: self.lr }) # save values list_predicted_labels.append(predicted_labels) list_true_labels.append(true_labels) avg_loss_rec += loss_rec_ul avg_loss_gauss += loss_gauss_ul avg_loss_cat += loss_cat_ul avg_loss_total += loss_total else: # validation phase for j in range(num_batches): # run the tensorflow flow graph loss_rec_ul, loss_gauss_ul, loss_cat_ul, loss_total, true_labels, predicted_labels = self.sess.run( [ model_spec['loss_rec_ul'], model_spec['loss_gauss_ul'], model_spec['loss_categ_ul'], model_spec['loss_total'], model_spec['true_labels'], model_spec['predicted'] ], feed_dict={ self.network.temperature: self.temperature, self.learning_rate: self.lr }) # save values list_predicted_labels.append(predicted_labels) list_true_labels.append(true_labels) avg_loss_rec += loss_rec_ul avg_loss_gauss += loss_gauss_ul avg_loss_cat += loss_cat_ul avg_loss_total += loss_total # average values by the given number of batches avg_loss_rec /= num_batches avg_loss_gauss /= num_batches avg_loss_cat /= num_batches avg_loss_total /= num_batches # average accuracy and nmi of all the data predicted_labels = np.hstack(list_predicted_labels) true_labels = np.hstack(list_true_labels) avg_nmi = self.metrics.nmi(predicted_labels, true_labels) avg_accuracy = self.metrics.cluster_acc(predicted_labels, true_labels) return { 'loss_rec': avg_loss_rec, 'loss_gauss': avg_loss_gauss, 'loss_cat': avg_loss_cat, 'loss_total': avg_loss_total, 'accuracy': avg_accuracy, 'nmi': avg_nmi } def train(self, train_data, train_labels, val_data, val_labels): """Train the model Args: train_data: (array) corresponding array containing the training data train_labels: (array) corresponding array containing the labels of the training data val_data: (array) corresponding array containing the validation data val_labels: (array) corresponding array containing the labels of the validation data Returns: output: (dict) contains the history of train/val loss """ train_history_loss, val_history_loss = [], [] train_history_acc, val_history_acc = [], [] train_history_nmi, val_history_nmi = [], [] # create training and validation dataset train_dataset = self.create_dataset(True, train_data, train_labels, self.batch_size) val_dataset = self.create_dataset(False, val_data, val_labels, self.batch_size_val) self.output_size = train_data.shape[1] # create train and validation models train_model = self.create_model(True, train_dataset, self.output_size) val_model = self.create_model(False, val_dataset, self.output_size) # set number of batches num_train_batches = int( np.ceil(train_data.shape[0] / (1.0 * self.batch_size))) num_val_batches = int( np.ceil(val_data.shape[0] / (1.0 * self.batch_size_val))) # initialize global variables self.sess.run(train_model['variable_init_op']) # training and validation phases print('Training phase...') for i in range(self.num_epochs): # decay learning rate according to decay_epoch parameter if self.decay_epoch > 0 and (i + 1) % self.decay_epoch == 0: self.lr = self.lr * self.lr_decay print('Decaying learning rate: %lf' % self.lr) # evaluate train and validation datasets train_loss = self.evaluate_dataset(True, num_train_batches, train_model) val_loss = self.evaluate_dataset(False, num_val_batches, val_model) # get training results for printing train_loss_rec = train_loss['loss_rec'] train_loss_gauss = train_loss['loss_gauss'] train_loss_cat = train_loss['loss_cat'] train_accuracy = train_loss['accuracy'] train_nmi = train_loss['nmi'] train_total_loss = train_loss['loss_total'] # get validation results for printing val_loss_rec = val_loss['loss_rec'] val_loss_gauss = val_loss['loss_gauss'] val_loss_cat = val_loss['loss_cat'] val_accuracy = val_loss['accuracy'] val_nmi = val_loss['nmi'] val_total_loss = val_loss['loss_total'] # if verbose then print specific information about training if self.verbose == 1: print("(Epoch %d / %d)" % (i + 1, self.num_epochs)) print("Train - REC: %.5lf; Gauss: %.5lf; Cat: %.5lf;" % \ (train_loss_rec, train_loss_gauss, train_loss_cat)) print("Valid - REC: %.5lf; Gauss: %.5lf; Cat: %.5lf;" % \ (val_loss_rec, val_loss_gauss, val_loss_cat)) print("Accuracy=Train: %.5lf; Val: %.5lf NMI=Train: %.5lf; Val: %.5lf Total Loss=Train: %.5lf; Val: %.5lf" % \ (train_accuracy, val_accuracy, train_nmi, val_nmi, train_total_loss, val_total_loss)) else: print("(Epoch %d / %d) Train Loss: %.5lf; Val Loss: %.5lf Train ACC: %.5lf; Val ACC: %.5lf Train NMI: %.5lf; Val NMI: %.5lf" % \ (i + 1, self.num_epochs, train_total_loss, val_total_loss, train_accuracy, val_accuracy, train_nmi, val_nmi)) # save loss and accuracy of each epoch train_history_loss.append(train_total_loss) val_history_loss.append(val_total_loss) train_history_acc.append(train_accuracy) val_history_acc.append(val_accuracy) if self.decay_temperature == 1: # decay temperature of gumbel-softmax self.temperature = np.maximum( self.initial_temperature * np.exp(-self.decay_temp_rate * (i + 1)), self.min_temperature) if self.verbose == 1: print("Gumbel Temperature: %.5lf" % self.temperature) return { 'train_history_loss': train_history_loss, 'val_history_loss': val_history_loss, 'train_history_acc': train_history_acc, 'val_history_acc': val_history_acc } def test(self, test_data, test_labels, batch_size=-1): """Test the model with new data Args: test_data: (array) corresponding array containing the testing data test_labels: (array) corresponding array containing the labels of the testing data batch_size: (int) batch size used to run the model Return: accuracy for the given test data """ # if batch_size is not specified then use all data if batch_size == -1: batch_size = test_data.shape[0] # create dataset test_dataset = self.create_dataset(False, test_data, test_labels, batch_size) true_labels = test_dataset['labels'] # perform a forward call on the encoder to obtain predicted labels latent = self.network.encoder(test_dataset['data'], self.num_classes) logits = latent['logits'] predicted_labels = tf.argmax(logits, axis=1) # initialize dataset iterator self.sess.run(test_dataset['iterator_init']) # calculate number of batches given batch size num_batches = int(np.ceil(test_data.shape[0] / (1.0 * batch_size))) # evaluate the model list_predicted_labels = [] list_true_labels = [] for j in range(num_batches): _predicted_labels, _true_labels = self.sess.run( [predicted_labels, true_labels], feed_dict={ self.network.temperature: self.temperature, self.learning_rate: self.lr }) # save values list_predicted_labels.append(_predicted_labels) list_true_labels.append(_true_labels) # average accuracy and nmi of all the data predicted_labels = np.hstack(list_predicted_labels) true_labels = np.hstack(list_true_labels) avg_nmi = self.metrics.nmi(predicted_labels, true_labels) avg_accuracy = self.metrics.cluster_acc(predicted_labels, true_labels) return avg_accuracy, avg_nmi def latent_features(self, data, batch_size=-1): """Obtain latent features learnt by the model Args: data: (array) corresponding array containing the data batch_size: (int) size of each batch to consider from the data Returns: features: (array) array containing the features from the data """ # if batch_size is not specified then use all data if batch_size == -1: batch_size = data.shape[0] # create dataset dataset = self.create_dataset(False, data, None, batch_size) # we will use only the encoder network latent = self.network.encoder(dataset['data'], self.num_classes) encoder = latent['features'] # obtain the features from the input data self.sess.run(dataset['iterator_init']) num_batches = data.shape[0] // batch_size features = np.zeros((data.shape[0], self.gaussian_size)) for j in range(num_batches): features[j * batch_size:j * batch_size + batch_size] = self.sess.run(encoder, feed_dict={ self.network.temperature: self.temperature, self.learning_rate: self.lr }) return features def reconstruct_data(self, data, batch_size=-1): """Reconstruct Data Args: data: (array) corresponding array containing the data batch_size: (int) size of each batch to consider from the data Returns: reconstructed: (array) array containing the reconstructed data """ # if batch_size is not specified then use all data if batch_size == -1: batch_size = data.shape[0] # create dataset dataset = self.create_dataset(False, data, None, batch_size) # reuse model used in training model_spec = self.create_model(False, dataset, data.shape[1]) # obtain the reconstructed data self.sess.run(model_spec['iterator_init']) num_batches = data.shape[0] // batch_size reconstructed = np.zeros(data.shape) pos = 0 for j in range(num_batches): reconstructed[pos:pos + batch_size] = self.sess.run( model_spec['output'], feed_dict={ self.network.temperature: self.temperature, self.learning_rate: self.lr }) pos += batch_size return reconstructed def plot_latent_space(self, data, labels, save=False): """Plot the latent space learnt by the model Args: data: (array) corresponding array containing the data labels: (array) corresponding array containing the labels save: (bool) whether to save the latent space plot Returns: fig: (figure) plot of the latent space """ # obtain the latent features features = self.latent_features(data) # plot only the first 2 dimensions fig = plt.figure(figsize=(8, 6)) plt.scatter(features[:, 0], features[:, 1], c=labels, marker='o', edgecolor='none', cmap=plt.cm.get_cmap('jet', 10), s=10) plt.colorbar() if (save): fig.savefig('latent_space.png') return fig def generate_data(self, num_elements=1, category=0): """Generate data for a specified category Args: num_elements: (int) number of elements to generate category: (int) category from which we will generate data Returns: generated data according to num_elements """ indices = (np.ones(num_elements) * category).astype(int).tolist() # category is specified with a one-hot array categorical = tf.one_hot(indices, self.num_classes) # infer the gaussian distribution according to the category mean, var = self.network.gaussian_from_categorical(categorical) # gaussian random sample by using the mean and variance gaussian = tf.random_normal(tf.shape(mean), mean, tf.sqrt(var)) # generate new samples with the given gaussian _, out = self.network.output_from_gaussian(gaussian, self.output_size) return self.sess.run(out, feed_dict={ self.network.temperature: self.temperature, self.learning_rate: self.lr }) def random_generation(self, num_elements=1): """Random generation for each category Args: num_elements: (int) number of elements to generate Returns: generated data according to num_elements """ # categories for each element arr = np.array([]) for i in range(self.num_classes): arr = np.hstack([arr, np.ones(num_elements) * i]) indices = arr.astype(int).tolist() categorical = tf.one_hot(indices, self.num_classes) # infer the gaussian distribution according to the category mean, var = self.network.gaussian_from_categorical(categorical) # gaussian random sample by using the mean and variance gaussian = tf.random_normal(tf.shape(mean), mean, tf.sqrt(var)) # generate new samples with the given gaussian _, out = self.network.output_from_gaussian(gaussian, self.output_size) return self.sess.run(out, feed_dict={ self.network.temperature: self.temperature, self.learning_rate: self.lr })
class ConvDRAW(object): def __init__(self, d, lr, lambda_z_wu, read_attn, write_attn, do_classify, do_reconst): self.do_classify = do_classify """ flags for each regularizor """ self.do_reconst = do_reconst self.read_attn = read_attn self.write_attn = write_attn """ dataset information """ self.set_datainfo(d) """ external toolkits """ self.ls = Layers() self.lf = LossFunctions(self.ls, self.d, self.encoder) self.ii = ImageInterface(_is_3d, self.read_attn, self.write_attn, GLIMPSE_SIZE_READ, GLIMPSE_SIZE_WRITE, _h, _w, _c) # for refference from get_loss_kl_draw() self.T = T self.L = L self.Z_SIZES = Z_SIZES """ placeholders defined outside""" self.lr = lr self.lambda_z_wu = lambda_z_wu """sequence of canvases """ self.cs = [0] * T """ initialization """ self.init_lstms() self.init_time_zero() """ workaround for variable_scope(reuse=True) """ self.DO_SHARE = None def set_datainfo(self, d): self.d = d # dataset manager global _b, _h, _w, _c, _img_size, _is_3d _b = d.batch_size _h = d.h _w = d.w _c = d.c _img_size = d.img_size _is_3d = d.is_3d def init_time_zero(self): self.cs[0] = tf.zeros((_b, _h, _w, _c)) if _is_3d else tf.zeros( (_b, _img_size)) self.h_dec[0][0] = tf.zeros((_b, RNN_SIZES[0])) def init_lstms(self): h_enc, e_mus, e_logsigmas = [[0] * L] * (T + 1), [[0] * L] * ( T + 1), [[0] * L] * (T + 1) # q(z_i+1 | z_i), bottom-up inference h_dec, d_mus, d_logsigmas = [[0] * L] * (T + 1), [[0] * L] * ( T + 1), [[0] * L] * (T + 1) # q(z_i | .), bidirectional inference p_mus, p_logsigmas = [[0] * L] * (T + 1), [[0] * L] * ( T + 1) # p(z_i | z_i+1), top-down prior """ set-up LSTM cells """ e_cells, e_states = [None] * L, [None] * L d_cells, d_states = [None] * L, [None] * L for l in range(L): e_cells[l] = tf.contrib.rnn.core_rnn_cell.LSTMCell(RNN_SIZES[l]) d_cells[l] = tf.contrib.rnn.core_rnn_cell.LSTMCell(RNN_SIZES[l]) e_states[l] = e_cells[l].zero_state(_b, tf.float32) d_states[l] = d_cells[l].zero_state(_b, tf.float32) """ set as standard Gaussian, N(0,I). """ d_mus[0][l], d_logsigmas[0][l] = tf.zeros( (_b, Z_SIZES[l])), tf.zeros((_b, Z_SIZES[l])) p_mus[0][l], p_logsigmas[0][l] = tf.zeros( (_b, Z_SIZES[l])), tf.zeros((_b, Z_SIZES[l])) self.h_enc, self.e_mus, self.e_logsigmas = h_enc, e_mus, e_logsigmas self.h_dec, self.d_mus, self.d_logsigmas = h_dec, d_mus, d_logsigmas self.p_mus, self.p_logsigmas = p_mus, p_logsigmas self.e_cells, self.e_states = e_cells, e_states self.d_cells, self.d_states = d_cells, d_states self.z = [[0] * L] * (T + 1) ########################################### """ LSTM cells """ ########################################### def lstm_encode(self, state, x, l, is_train): scope = 'lstm_encode_' + str(l) x = tf.reshape(x, (_b, -1)) if x.get_shape()[1] != RNN_SIZES[l]: print(scope, ':', x.get_shape()[1:], '=>', RNN_SIZES[l]) x = self.ls.dense(scope, x, RNN_SIZES[l]) return self.e_cells[l](x, state) def lstm_decode(self, state, x, l, is_train): scope = 'lstm_decode_' + str(l) x = tf.reshape(x, (_b, -1)) if x.get_shape()[1] != RNN_SIZES[l]: print(scope, ':', x.get_shape()[1:], '=>', RNN_SIZES[l]) x = self.ls.dense(scope, x, RNN_SIZES[l]) return self.d_cells[l](x, state) ########################################### """ Encoder """ ########################################### def encoder(self, x, t, is_train=True, do_update_bn=True): for l in range(L): scope = 'Encode_L' + str(l) with tf.variable_scope(scope, reuse=self.DO_SHARE): if l == 0: x_hat = x - self.canvase_previous(t) h_dec_lowest_prev = self.h_dec[ t - 1][0] if t == 0 else tf.zeros((_b, RNN_SIZES[0])) input = self.ii.read(x, x_hat, h_dec_lowest_prev) else: input = self.h_enc[t][l - 1] self.h_enc[t][l], self.e_states[l] = self.lstm_encode( self.e_states[l], input, l, is_train) input = self.ls.dense(scope, self.h_enc[t][l], Z_SIZES[l] * 2) self.z[t][l], self.e_mus[t][l], self.e_logsigmas[t][ l] = self.ls.vae_sampler_w_feature_slice( input, Z_SIZES[l]) """ classifier """ logit = self.ls.dense('top', self.h_enc[t][-1], self.d.l, activation=tf.nn.elu) return logit ########################################### """ Decoder """ ########################################### def decoder(self, t, is_train=True, do_update_bn=True): for l in range(L - 1, -1, -1): scope = 'Decoder_L' + str(l) with tf.variable_scope(scope, reuse=self.DO_SHARE): if l == L - 1: input = self.z[t][l] else: input = self.concat(self.z[t][l], self.h_dec[t][l + 1], l) self.h_dec[t][l], self.d_states[l] = self.lstm_decode( self.d_states[l], input, l, is_train) """ go out to the input space """ if l == 0: # [ToDo] replace bellow reconstructor with conv-lstm if _is_3d: o = self.canvase_previous(t) + self.ii.write( self.h_dec[t][l]) #if t == T-1: # for MNIST o = tf.nn.sigmoid(o) self.cs[t] = o else: self.cs[t] = tf.nn.sigmoid( self.canvase_previous(t) + self.ii.write(self.h_dec[t][l])) return self.cs[t] """ set prior after building the decoder """ def prior(self, t): for l in range(L - 1, -1, -1): scope = 'Piror_L' + str(l) """ preparation for p_* for t+1 and d_* for t with the output from lstm-decoder""" if l != 0: input = self.ls.dense(scope, self.h_dec[t][l], Z_SIZES[l] * 2 + Z_SIZES[l - 1] * 2) self.p_mus[t + 1][l], self.p_logsigmas[t + 1][l], self.d_mus[ t][l], self.d_logsigmas[t][l] = self.ls.split( input, 1, [Z_SIZES[l]] * 2 + [Z_SIZES[l - 1]] * 2) else: """ no one uses d_* """ input = self.ls.dense(scope, self.h_dec[t][l], Z_SIZES[l] * 2) self.p_mus[t + 1][l], self.p_logsigmas[t + 1][l] = self.ls.split( input, 1, [Z_SIZES[l]] * 2) """ setting p_mus[0][l] and p_logsigmas[0][l] """ if t == 0: if l == L - 1: """ has already been performed at init() """ pass else: """ by using only decoder's top-down path as prior since p(z) of t-1 does not exist """ self.p_mus[t][l], self.p_logsigmas[t][l] = self.d_mus[t][ l + 1], tf.exp(self.d_logsigmas[t][l + 1]) # Eq.19 at t=0 else: if l == L - 1: """ has already been performed at t-1 """ pass else: """ update p(z) of current t """ _, self.p_mus[t][l], self.p_logsigmas[t][ l] = self.ls.precision_weighted_sampler( scope, (self.p_mus[t][l], tf.exp(self.p_logsigmas[t][l])), (self.d_mus[t][l + 1], tf.exp(self.d_logsigmas[t][l + 1]))) # Eq.19 ########################################### """ Build Graph """ ########################################### def build_graph_train(self, x_l, y_l, x, is_supervised=True): o = dict() # output loss = 0 logit_ls = [] """ Build DRAW """ for t in range(T): logit_ls.append(self.encoder(x, t)) x_reconst = self.decoder(t) self.prior(t) if t == 0: self.DO_SHARE = DO_SHARE = True self.ii.set_do_share(DO_SHARE) self.ls.set_do_share(DO_SHARE) """ p(x|z) Reconstruction Loss """ o['x'] = x o['cs'] = self.cs o['Lr'] = self.lf.get_loss_pxz(x_reconst, x, 'DiscretizedLogistic') loss += o['Lr'] """ VAE KL-Divergence Loss """ o['KL1'], o['KL2'], o['Lz'] = self.lf.get_loss_kl_draw(self) loss += self.lambda_z_wu * o['Lz'] """ set losses """ o['loss'] = loss self.o_train = o """ set optimizer """ optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.5) grads = optimizer.compute_gradients(loss) for i, (g, v) in enumerate(grads): if g is not None: #g = tf.Print(g, [g], "g %s = "%(v)) grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients else: print('g is None:', v) v = tf.Print(v, [v], "v = ", summarize=10000) self.op = optimizer.apply_gradients(grads) # return train_op def build_graph_test(self, x_l, y_l, is_supervised=False): o = dict() # output loss = 0 logit_ls = [] """ Build DRAW """ for t in range(T): logit_ls.append( self.encoder(x_l, t, is_train=False, do_update_bn=False)) x_reconst = self.decoder(t) self.prior(t) if t == 0: self.DO_SHARE = DO_SHARE = True self.ii.set_do_share(DO_SHARE) self.ls.set_do_share(DO_SHARE) """ classification loss """ if is_supervised: o['Ly'], o['accur'] = self.lf.get_loss_pyx(logit_ls[-1], y_l) loss += o['Ly'] """ for visualizationc """ o['z'], o['y'] = logit_ls[-1], y_l """ set losses """ o['loss'] = loss self.o_test = o ########################################### """ Utilities """ ########################################### def canvase_previous(self, t): if _is_3d: c_prev = tf.zeros((_b, _h, _w, _c)) if t == 0 else self.cs[t - 1] else: c_prev = tf.zeros((_b, _img_size)) if t == 0 else self.cs[t - 1] return c_prev def concat(self, x1, x2, l): if False: # [ToDo] x1 = tf.reshape(x1, (_b, IMAGE_SIZES[l][0], IMAGE_SIZES[l][1], -1)) x2 = tf.reshape(x2, (_b, IMAGE_SIZES[l][0], IMAGE_SIZES[l][1], -1)) return tf.concat([x1, x2], 3) else: x1 = tf.reshape(x1, (_b, -1)) x2 = tf.reshape(x2, (_b, -1)) return tf.concat([x1, x2], 1)
class SSVAE: def __init__(self, params): self.batch_size = params.batch_size self.batch_size_val = params.batch_size_val self.initial_temperature = params.temperature self.decay_temperature = params.decay_temperature self.num_epochs = params.num_epochs self.loss_type = params.loss_type self.num_classes = params.num_classes self.w_gauss = params.w_gaussian self.w_categ = params.w_categorical self.w_recon = params.w_reconstruction self.decay_temp_rate = params.decay_temp_rate self.gaussian_size = params.gaussian_size self.feature_size = params.feature_size self.min_temperature = params.min_temperature self.temperature = params.temperature # current temperature self.verbose = params.verbose self.sess = tf.Session() self.network = CatVAENetwork(params) self.losses = LossFunctions() self.w_assign = params.w_assign self.num_labeled = params.num_labeled self.knn = params.knn self.metric_loss = params.metric_loss self.w_metric = tf.placeholder(tf.float32, []) self.initial_w_metric = params.w_metric self._w_metric = params.w_metric self.anneal_metric_loss = params.anneal_w_metric self.learning_rate = tf.placeholder(tf.float32, []) self.lr = params.learning_rate self.decay_epoch = params.decay_epoch self.lr_decay = params.lr_decay self.pretrain = params.pretrain self.num_labeled_batch = params.num_labeled_batch self.dataset = params.dataset self.metric_margin = params.metric_margin def create_dataset(self, is_training, data, labels, batch_size, x_labeled=None, y_labeled=None): """Create dataset given input data Args: is_training: (bool) whether to use the train or test pipeline. At training, we shuffle the data and have multiple epochs data: (array) corresponding array containing the input data labels: (array) corresponding array containing the labels of the input data batch_size: (int) size of each batch to consider from the data x_labeled: (array) corresponding array containing the labeled input data y_labeled: (array) corresponding array containing the labeles of the labeled input data Returns: output: (dict) contains what will be the input of the tensorflow graph """ num_samples = data.shape[0] # create dataset object if labels is None: dataset = tf.data.Dataset.from_tensor_slices(data) else: dataset = tf.data.Dataset.from_tensor_slices((data, labels)) # shuffle data in training phase if is_training: dataset = dataset.shuffle(num_samples).repeat() dataset = dataset.batch(batch_size) dataset = dataset.prefetch(1) # create reinitializable iterator from dataset iterator = dataset.make_initializable_iterator() labeled_data = False if labels is None: data = iterator.get_next() else: data, labels = iterator.get_next() # append labeled data to each batch if x_labeled is not None: labeled_data = True _data = tf.concat([data, x_labeled], 0) _labels = tf.concat([labels, y_labeled], 0) iterator_init = iterator.initializer if labeled_data: output = { 'data': _data, 'labels': _labels, 'iterator_init': iterator_init } output['labels_semisupervised'] = y_labeled else: output = { 'data': data, 'labels': labels, 'iterator_init': iterator_init } output['labels_semisupervised'] = None return output def create_model(self, is_training, inputs, output_size): """Model function defining the graph operations. Args: is_training: (bool) whether we are in training phase or not inputs: (dict) contains the inputs of the graph (features, labels...) this can be `tf.placeholder` or outputs of `tf.data` output_size: (int) size of the output layer Returns: model_spec: (dict) contains the graph operations or nodes needed for training / evaluation """ data, _labels = inputs['data'], inputs['labels'] # create network and obtain latent vectors that will be used in loss functions latent_spec = self.network.encoder(data, self.num_classes, is_training) gaussian, mean, logVar = latent_spec['gaussian'], latent_spec[ 'mean'], latent_spec['logVar'] categorical, prob, log_prob = latent_spec['categorical'], latent_spec[ 'prob_cat'], latent_spec['log_prob_cat'] _logits, features = latent_spec['logits'], latent_spec['features'] output = self.network.decoder(gaussian, categorical, output_size, is_training) # reconstruction loss if self.loss_type == 'bce': loss_rec = self.losses.binary_cross_entropy(data, output) elif self.loss_type == 'mse': loss_rec = tf.losses.mean_squared_error(data, output) else: raise "invalid loss function... try bce or mse..." # kl-divergence loss loss_kl = self.losses.kl_gaussian(mean, logVar) loss_kl_cat = self.losses.kl_categorical(prob, log_prob, self.num_classes) # auxiliary task to assign labels and regularize the feature space if _labels is not None: labeled_ss = inputs['labels_semisupervised'] if labeled_ss is not None: # assignment loss only if labeled data is available (training phase) predicted_labels = assign_labels_semisupervised( features, labeled_ss, self.num_labeled_batch, self.batch_size, self.num_classes, self.knn) # use assigned labels and logits to calculate cross entropy loss loss_assign = tf.losses.sparse_softmax_cross_entropy( labels=predicted_labels, logits=_logits) else: # predict labels from logits or softmax(logits) (validation/testing phase) loss_assign = tf.constant(0.) predicted_labels = tf.argmax(prob, axis=1) # calculate accuracy using the predicted and true labels accuracy = tf.reduce_mean( tf.cast(tf.equal(_labels, predicted_labels), tf.float32)) # metric embedding loss if self.metric_loss == 'triplet': loss_metric = tf.contrib.losses.metric_learning.triplet_semihard_loss( predicted_labels, features, margin=self.metric_margin) elif self.metric_loss == 'lifted': loss_metric = tf.contrib.losses.metric_learning.lifted_struct_loss( predicted_labels, features, margin=self.metric_margin) else: raise "invalid metric loss... currently we support triplet and lifted loss" else: accuracy = tf.constant(0.) loss_assign = tf.constant(0.) loss_metric = tf.constant(0.) predicted_labels = tf.constant(0.) # variational autoencoder loss loss_vae = self.w_recon * loss_rec loss_vae += self.w_gauss * loss_kl loss_vae += self.w_categ * loss_kl_cat # total loss loss_total = loss_vae + self.w_assign * loss_assign + self.w_metric * loss_metric if is_training: # use adam for optimization optimizer = tf.train.AdamOptimizer(self.learning_rate) # needed for batch normalization layer update_ops = tf.get_collection(tf.GraphKeys.UPDATE_OPS) with tf.control_dependencies(update_ops): train_op_vae = optimizer.minimize(loss_vae) train_op_tot = optimizer.minimize(loss_total) # create model specification model_spec = inputs model_spec['variable_init_op'] = tf.global_variables_initializer() model_spec['output'] = output model_spec['features'] = features model_spec['predicted_labels'] = predicted_labels model_spec['true_labels'] = _labels model_spec['loss_rec'] = loss_rec model_spec['loss_kl'] = loss_kl model_spec['loss_kl_cat'] = loss_kl_cat model_spec['loss_total'] = loss_total model_spec['loss_metric'] = loss_metric model_spec['loss_assign'] = loss_assign model_spec['accuracy'] = accuracy # optimizers are only available in training phase if is_training: model_spec['train_op'] = train_op_tot model_spec['train_vae'] = train_op_vae return model_spec def evaluate_dataset(self, is_training, num_batches, model_spec, labeled_data=None, labeled_labels=None): """Evaluate the model Args: is_training: (bool) whether we are training or not num_batches: (int) number of batches to train/test model_spec: (dict) contains the graph operations or nodes needed for evaluation labeled_data: (array) corresponding array containing the labeled input data labeled_labels: (array) corresponding array containing the labeles of the labeled input data Returns: (dic) average of loss functions and metrics for the given number of batches """ avg_accuracy = 0.0 avg_nmi = 0.0 avg_loss_rec = 0.0 avg_loss_kl = 0.0 avg_loss_cat = 0.0 avg_loss_total = 0.0 avg_loss_metric = 0.0 avg_loss_assign = 0.0 # initialize dataset iteratior self.sess.run(model_spec['iterator_init']) if is_training: # pretraining will train only the variational autoencoder losses if self.pretrain < 1: train_optimizer = model_spec['train_op'] else: train_optimizer = model_spec['train_vae'] # training phase for j in range(num_batches): # select randomly subsets of labeled data according to the batch size _x_labeled, _y_labeled, _, _ = create_semisupervised_dataset( labeled_data, labeled_labels, self.num_classes, self.num_labeled_batch) # run the tensorflow flow graph _, loss_rec, loss_kl, loss_metric, loss_assign, loss_cat, loss_total, accuracy = self.sess.run( [ train_optimizer, model_spec['loss_rec'], model_spec['loss_kl'], model_spec['loss_metric'], model_spec['loss_assign'], model_spec['loss_kl_cat'], model_spec['loss_total'], model_spec['accuracy'] ], feed_dict={ self.network.temperature: self.temperature, self.w_metric: self._w_metric, self.learning_rate: self.lr, self.x_labeled: _x_labeled, self.y_labeled: _y_labeled }) # accumulate values avg_accuracy += accuracy avg_loss_rec += loss_rec avg_loss_kl += loss_kl avg_loss_cat += loss_cat avg_loss_total += loss_total avg_loss_metric += loss_metric avg_loss_assign += loss_assign else: # validation phase for j in range(num_batches): # run the tensorflow flow graph loss_rec, loss_kl, loss_metric, loss_assign, loss_cat, loss_total, accuracy = self.sess.run( [ model_spec['loss_rec'], model_spec['loss_kl'], model_spec['loss_metric'], model_spec['loss_assign'], model_spec['loss_kl_cat'], model_spec['loss_total'], model_spec['accuracy'] ], feed_dict={ self.network.temperature: self.temperature, self.w_metric: self._w_metric, self.learning_rate: self.lr }) # accumulate values avg_accuracy += accuracy avg_loss_rec += loss_rec avg_loss_kl += loss_kl avg_loss_cat += loss_cat avg_loss_total += loss_total avg_loss_metric += loss_metric avg_loss_assign += loss_assign # average values by the given number of batches avg_loss_rec /= num_batches avg_loss_kl /= num_batches avg_accuracy /= num_batches avg_loss_cat /= num_batches avg_loss_total /= num_batches avg_loss_metric /= num_batches avg_loss_assign /= num_batches return { 'avg_loss_rec': avg_loss_rec, 'avg_loss_kl': avg_loss_kl, 'avg_loss_cat': avg_loss_cat, 'total_loss': avg_loss_total, 'avg_accuracy': avg_accuracy, 'avg_loss_metric': avg_loss_metric, 'avg_loss_assign': avg_loss_assign } def train(self, train_data, train_labels, val_data, val_labels, labeled_data, labeled_labels): """Train the model Args: train_data: (array) corresponding array containing the training data train_labels: (array) corresponding array containing the labels of the training data val_data: (array) corresponding array containing the validation data val_labels: (array) corresponding array containing the labels of the validation data labeled_data: (array) corresponding array containing the labeled input data labeled_labels: (array) corresponding array containing the labeles of the labeled input data Returns: output: (dict) contains the history of train/val loss """ train_history_loss, val_history_loss = [], [] train_history_acc, val_history_acc = [], [] train_history_nmi, val_history_nmi = [], [] # placeholders for the labeled data self.x_labeled = tf.placeholder( tf.float32, shape=[self.num_labeled_batch, labeled_data.shape[1]]) self.y_labeled = tf.placeholder(tf.int64, shape=[self.num_labeled_batch]) # create training and validation dataset train_dataset = self.create_dataset( True, train_data, train_labels, self.batch_size - self.num_labeled_batch, self.x_labeled, self.y_labeled) val_dataset = self.create_dataset(True, val_data, val_labels, self.batch_size_val) self.output_size = train_data.shape[1] # create train and validation models train_model = self.create_model(True, train_dataset, self.output_size) val_model = self.create_model(False, val_dataset, self.output_size) # set number of batches num_train_batches = int( np.ceil(train_data.shape[0] / (1.0 * (self.batch_size - self.num_labeled_batch)))) num_val_batches = int( np.ceil(val_data.shape[0] / (1.0 * self.batch_size_val))) # initialize global variables self.sess.run(train_model['variable_init_op']) # training and validation phases print('Training phase...') for i in range(self.num_epochs): # pretraining at each epoch if self.pretrain > 0: self.pretrain = self.pretrain - 1 # decay learning rate according to decay_epoch parameter if self.decay_epoch > 0 and (i + 1) % self.decay_epoch == 0: self.lr = self.lr * self.lr_decay print('Decaying learning rate: %lf' % self.lr) # evaluate train and validation datasets train_loss = self.evaluate_dataset(True, num_train_batches, train_model, labeled_data, labeled_labels) val_loss = self.evaluate_dataset(False, num_val_batches, val_model) # get training results for printing train_loss_rec = train_loss['avg_loss_rec'] train_loss_kl = train_loss['avg_loss_kl'] train_loss_cat = train_loss['avg_loss_cat'] train_loss_ass = train_loss['avg_loss_assign'] train_loss_met = train_loss['avg_loss_metric'] train_accuracy = train_loss['avg_accuracy'] train_total_loss = train_loss['total_loss'] # get validation results for printing val_loss_rec = val_loss['avg_loss_rec'] val_loss_kl = val_loss['avg_loss_kl'] val_loss_cat = val_loss['avg_loss_cat'] val_loss_ass = val_loss['avg_loss_assign'] val_loss_met = val_loss['avg_loss_metric'] val_accuracy = val_loss['avg_accuracy'] val_total_loss = val_loss['total_loss'] # if verbose then print specific information about training if self.verbose == 1: print("(Epoch %d / %d) REC=Train: %.5lf; Val: %.5lf KL=Train: %.5lf; Val: %.5lf KL-Cat=Train: %.5lf; Val: %.5lf MET=Train: %.5lf; Val: %.5lf ASS=Train: %.5lf; Val: %.5lf ACC=Train %.5lf; Val %.5lf" % \ (i + 1, self.num_epochs, train_loss_rec, val_loss_rec, train_loss_kl, val_loss_kl, train_loss_cat, val_loss_cat, train_loss_met, val_loss_met, train_loss_ass, val_loss_ass, train_accuracy, val_accuracy)) else: print("(Epoch %d / %d) Train Loss: %.5lf; Val Loss: %.5lf Train Accuracy: %.5lf; Val Accuracy: %.5lf" % \ (i + 1, self.num_epochs, train_total_loss, val_total_loss, train_accuracy, val_accuracy)) # save loss and accuracy of each epoch train_history_loss.append(train_total_loss) val_history_loss.append(val_total_loss) train_history_acc.append(train_accuracy) val_history_acc.append(val_accuracy) if self.anneal_metric_loss == 1: #anneal loss from initial_w_metric to 1 in the first 100 epochs self._w_metric = np.minimum( self.initial_w_metric * np.exp(0.06908 * (i + 1)), 1) if self.verbose == 1: print('Metric Weight: %.5lf' % self._w_metric) if self.decay_temperature == 1: # decay temperature of gumbel-softmax self.temperature = np.maximum( self.initial_temperature * np.exp(-self.decay_temp_rate * (i + 1)), self.min_temperature) if self.verbose == 1: print("Gumbel Temperature: %.5lf" % self.temperature) return { 'train_history_loss': train_history_loss, 'val_history_loss': val_history_loss, 'train_history_acc': train_history_acc, 'val_history_acc': val_history_acc } def test(self, test_data, test_labels, batch_size=-1): """Test the model with new data Args: test_data: (array) corresponding array containing the testing data test_labels: (array) corresponding array containing the labels of the testing data batch_size: (int) batch size used to run the model Return: accuracy for the given test data """ # if batch_size is not specified then use all data if batch_size == -1: batch_size = test_data.shape[0] # create dataset test_dataset = self.create_dataset(False, test_data, test_labels, batch_size) true_labels = test_dataset['labels'] # perform a forward call on the encoder to obtain predicted labels latent = self.network.encoder(test_dataset['data'], self.num_classes) logits = latent['prob_cat'] predicted_labels = tf.argmax(logits, axis=1) # calculate accuracy given the predicted and true labels accuracy = tf.reduce_mean( tf.cast(tf.equal(true_labels, predicted_labels), tf.float32)) # initialize dataset iterator self.sess.run(test_dataset['iterator_init']) # calculate number of batches given batch size num_batches = int(np.ceil(test_data.shape[0] / (1.0 * batch_size))) # evaluate the model avg_accuracy = 0.0 for j in range(num_batches): _accuracy = self.sess.run(accuracy, feed_dict={ self.network.temperature: self.temperature, self.w_metric: self._w_metric, self.learning_rate: self.lr }) avg_accuracy += _accuracy # average the accuracy avg_accuracy /= num_batches return avg_accuracy def latent_features(self, data, batch_size=-1): """Obtain latent features learnt by the model Args: data: (array) corresponding array containing the data batch_size: (int) size of each batch to consider from the data Returns: features: (array) array containing the features from the data """ # if batch_size is not specified then use all data if batch_size == -1: batch_size = data.shape[0] # create dataset dataset = self.create_dataset(False, data, None, batch_size) # we will use only the encoder network latent = self.network.encoder(dataset['data'], self.num_classes) encoder = latent['features'] # obtain the features from the input data self.sess.run(dataset['iterator_init']) num_batches = data.shape[0] // batch_size features = np.zeros((data.shape[0], self.feature_size)) for j in range(num_batches): features[j * batch_size:j * batch_size + batch_size] = self.sess.run(encoder, feed_dict={ self.network.temperature: self.temperature, self.w_metric: self._w_metric, self.learning_rate: self.lr }) return features def reconstruct_data(self, data, batch_size=-1): """Reconstruct Data Args: data: (array) corresponding array containing the data batch_size: (int) size of each batch to consider from the data Returns: reconstructed: (array) array containing the reconstructed data """ # if batch_size is not specified then use all data if batch_size == -1: batch_size = data.shape[0] # create dataset dataset = self.create_dataset(False, data, None, batch_size) # reuse model used in training model_spec = self.create_model(False, dataset, data.shape[1]) # obtain the reconstructed data self.sess.run(model_spec['iterator_init']) num_batches = data.shape[0] // batch_size reconstructed = np.zeros(data.shape) pos = 0 for j in range(num_batches): reconstructed[pos:pos + batch_size] = self.sess.run( model_spec['output'], feed_dict={ self.network.temperature: self.temperature, self.w_metric: self._w_metric, self.learning_rate: self.lr }) pos += batch_size return reconstructed def plot_latent_space(self, data, labels, save=False): """Plot the latent space learnt by the model Args: data: (array) corresponding array containing the data labels: (array) corresponding array containing the labels save: (bool) whether to save the latent space plot Returns: fig: (figure) plot of the latent space """ # obtain the latent features features = self.latent_features(data) # plot only the first 2 dimensions fig = plt.figure(figsize=(8, 6)) plt.scatter(features[:, 0], features[:, 1], c=labels, marker='o', edgecolor='none', cmap=plt.cm.get_cmap('jet', 10), s=10) plt.colorbar() if (save): fig.savefig('latent_space.png') return fig def generate_data(self, num_elements=1, category=0): """Generate data for a specified category Args: num_elements: (int) number of elements to generate category: (int) category from which we will generate data Returns: generated data according to num_elements """ # gaussian noise for each element noise = tf.random_normal([num_elements, self.gaussian_size], mean=0, stddev=1, dtype=tf.float32) indices = (np.ones(num_elements) * category).astype(int).tolist() # category is specified with a one-hot array categorical = tf.one_hot(indices, self.num_classes) # use the gaussian noise and category to generate data from the generator out = self.network.decoder(noise, categorical, self.output_size) return self.sess.run(out, feed_dict={ self.network.temperature: self.temperature, self.w_metric: self._w_metric, self.learning_rate: self.lr }) def random_generation(self, num_elements=1): """Random generation for each category Args: num_elements: (int) number of elements to generate Returns: generated data according to num_elements """ # gaussian noise for each element noise = tf.random_normal( [num_elements * self.num_classes, self.gaussian_size], mean=0, stddev=1, dtype=tf.float32) # categories for each element arr = np.array([]) for i in range(self.num_classes): arr = np.hstack([arr, np.ones(num_elements) * i]) indices = arr.astype(int).tolist() categorical = tf.one_hot(indices, self.num_classes) # use the gaussian noise and categories to generate data from the generator out = self.network.decoder(noise, categorical, self.output_size) return self.sess.run(out, feed_dict={ self.network.temperature: self.temperature, self.w_metric: self._w_metric, self.learning_rate: self.lr }) def style_generation(self, data): """Style transfer generation for each category given a predefined style Args: data: (array) corresponding array containing the input style Returns: generated data according to the style of data """ # convert data to tensor num_elem = data.shape[0] data = np.repeat(data, self.num_classes, axis=0) tf_data = tf.convert_to_tensor(data) # get latent gaussian features from the encoder latent = self.network.encoder(tf_data, self.num_classes) gaussian = latent['gaussian'] # set one-hot values for each category indices = np.tile(range(self.num_classes), num_elem) categorical = tf.one_hot(indices, self.num_classes) # use the gaussian features and categories to generate data from the generator out = self.network.decoder(gaussian, categorical, self.output_size) return self.sess.run(out, feed_dict={ self.network.temperature: self.temperature, self.w_metric: self._w_metric, self.learning_rate: self.lr })
class VAE(object): def __init__(self, resource): """ data and external toolkits """ self.d = resource.dh # dataset manager self.ls = Layers() self.lf = LossFunctions(self.ls, self.d, self.encoder) """ placeholders defined outside""" if c.DO_TRAIN: self.lr = resource.ph['lr'] def encoder(self, h, is_train, y=None): if is_train: _d = self.d #_ = tf.summary.image('image', tf.reshape(h, [-1, _d.h, _d.w, _d.c]), 10) scope = 'e_1' h = self.ls.conv2d(scope + '_1', h, 128, filter_size=(2, 2), strides=(1, 2, 2, 1), padding="VALID") h = tf.layers.batch_normalization(h, training=is_train, name=scope) h = tf.nn.relu(h) scope = 'e_2' h = self.ls.conv2d(scope + '_1', h, 256, filter_size=(2, 2), strides=(1, 2, 2, 1), padding="VALID") h = tf.layers.batch_normalization(h, training=is_train, name=scope) h = tf.nn.relu(h) scope = 'e_3' h = self.ls.conv2d(scope + '_1', h, 512, filter_size=(2, 2), strides=(1, 2, 2, 1), padding="VALID") h = tf.layers.batch_normalization(h, training=is_train, name=scope) #h = tf.nn.relu(h) h = tf.nn.tanh(h) # -> (b, 4, 4, 512) print('h:', h) #h = tf.reshape(h, (c.BATCH_SIZE, -1)) h = tf.reshape(h, (-1, 4 * 4 * 512)) print('h:', h) #sys.exit('aa') h = self.ls.denseV2('top_of_encoder', h, c.Z_SIZE * 2, activation=None) print('h:', h) return self.ls.vae_sampler_w_feature_slice(h, c.Z_SIZE) def decoder(self, h, is_train): scope = 'top_of_decoder' #h = self.ls.denseV2(scope, h, 128, activation=self.ls.lrelu) h = self.ls.denseV2(scope, h, 512, activation=self.ls.lrelu) print('h:', scope, h) h = tf.reshape(h, (-1, 4, 4, 32)) print('h:', scope, h) scope = 'd_1' h = self.ls.deconv2d(scope + '_1', h, 512, filter_size=(2, 2)) h = tf.layers.batch_normalization(h, training=is_train, name=scope) h = tf.nn.relu(h) print('h:', scope, h) scope = 'd_2' h = self.ls.deconv2d(scope + '_2', h, 256, filter_size=(2, 2)) h = tf.layers.batch_normalization(h, training=is_train, name=scope) h = tf.nn.relu(h) print('h:', scope, h) scope = 'd_3' h = self.ls.deconv2d(scope + '_3', h, 128, filter_size=(2, 2)) h = tf.layers.batch_normalization(h, training=is_train, name=scope) h = tf.nn.relu(h) print('h:', scope, h) scope = 'd_4' h = self.ls.conv2d(scope + '_4', h, 3, filter_size=(1, 1), strides=(1, 1, 1, 1), padding="VALID", activation=tf.nn.sigmoid) print('h:', scope, h) return h def build_graph_train(self, x_l, y_l): o = dict() # output loss = 0 if c.IS_AUGMENTATION_ENABLED: x_l = distorted = self.distort(x_l) if c.IS_AUG_NOISE_TRUE: x_l = self.ls.get_corrupted(x_l, 0.15) z, mu, logsigma = self.encoder(x_l, is_train=True, y=y_l) x_reconst = self.decoder(z, is_train=True) """ p(x|z) Reconstruction Loss """ o['Lr'] = self.lf.get_loss_pxz(x_reconst, x_l, 'Bernoulli') o['x_reconst'] = x_reconst o['x'] = x_l loss += o['Lr'] """ VAE KL-Divergence Loss """ LAMBDA_VAE = 0.1 o['mu'], o['logsigma'] = mu, logsigma # work around. [ToDo] make sure the root cause that makes kl loss inf #logsigma = tf.clip_by_norm( logsigma, 10) o['Lz'] = self.lf.get_loss_vae(c.Z_SIZE, mu, logsigma, _lambda=0.0) loss += LAMBDA_VAE * o['Lz'] """ set losses """ o['loss'] = loss self.o_train = o """ set optimizer """ optimizer = tf.train.AdamOptimizer(learning_rate=self.lr, beta1=0.5) grads = optimizer.compute_gradients(loss) for i, (g, v) in enumerate(grads): if g is not None: #g = tf.Print(g, [g], "g %s = "%(v)) grads[i] = (tf.clip_by_norm(g, 5), v) # clip gradients else: print('g is None:', v) v = tf.Print(v, [v], "v = ", summarize=10000) # update ema in batch_normalization with tf.control_dependencies(tf.get_collection( tf.GraphKeys.UPDATE_OPS)): self.op = optimizer.apply_gradients(grads) # return train_op def build_graph_test(self, x_l, y_l): o = dict() # output loss = 0 z, mu, logsigma = self.encoder(x_l, is_train=False, y=y_l) x_reconst = self.decoder(mu, is_train=False) o['x_reconst'] = x_reconst o['x'] = x_l #o['Lr'] = self.lf.get_loss_pxz(x_reconst, x_l, 'LeastSquare') o['Lr'] = self.lf.get_loss_pxz(x_reconst, x_l, 'Bernoulli') #o['Lr'] = self.lf.get_loss_pxz(x_reconst, x_l, 'DiscretizedLogistic') #o['Lr'] = tf.reduce_mean(tf.keras.losses.binary_crossentropy(x_l, x_reconst)) loss += o['Lr'] """ set losses """ o['loss'] = loss self.o_test = o def distort(self, x): """ maybe helpful http://www.redhub.io/Tensorflow/tensorflow-models/src/master/inception/inception/image_processing.py """ _d = self.d def _distort(a_image): """ bounding_boxes: A Tensor of type float32. 3-D with shape [batch, N, 4] describing the N bounding boxes associated with the image. Bounding boxes are supplied and returned as [y_min, x_min, y_max, x_max] """ if c.IS_AUG_TRANS_TRUE: a_image = tf.pad(a_image, [[2, 2], [2, 2], [0, 0]]) a_image = tf.random_crop(a_image, [_d.h, _d.w, _d.c]) if c.IS_AUG_FLIP_TRUE: a_image = tf.image.random_flip_left_right(a_image) if c.IS_AUG_ROTATE_TRUE: from math import pi radian = tf.random_uniform(shape=(), minval=0, maxval=360) * pi / 180 a_image = tf.contrib.image.rotate(a_image, radian, interpolation='BILINEAR') if c.IS_AUG_COLOR_TRUE: a_image = tf.image.random_brightness(a_image, max_delta=0.2) a_image = tf.image.random_contrast(a_image, lower=0.2, upper=1.8) a_image = tf.image.random_hue(a_image, max_delta=0.2) if c.IS_AUG_CROP_TRUE: # shape: [1, 1, 4] bounding_boxes = tf.constant( [[[1 / 10, 1 / 10, 9 / 10, 9 / 10]]], dtype=tf.float32) begin, size, _ = tf.image.sample_distorted_bounding_box( (_d.h, _d.w, _d.c), bounding_boxes, min_object_covered=(9.8 / 10.0), aspect_ratio_range=[9.5 / 10.0, 10.0 / 9.5]) a_image = tf.slice(a_image, begin, size) """ for the purpose of distorting not use tf.image.resize_image_with_crop_or_pad under """ a_image = tf.image.resize_images(a_image, [_d.h, _d.w]) """ due to the size of channel returned from tf.image.resize_images is not being given, specify it manually. """ a_image = tf.reshape(a_image, [_d.h, _d.w, _d.c]) return a_image """ process batch times in parallel """ return tf.map_fn(_distort, x)