def var_dropout(observed, x, n, net_size, n_particles, is_training): with zs.BayesianNet(observed=observed) as model: h = x normalizer_params = { 'is_training': is_training, 'updates_collections': None } for i, [n_in, n_out] in enumerate(zip(net_size[:-1], net_size[1:])): eps_mean = tf.zeros([n, n_in]) # Adding noise to Weights eps = zs.Normal('layer' + str(i) + '/eps', eps_mean, std=1., n_samples=n_particles, group_ndims=1) h = layers.fully_connected(h * eps, n_out, normalizer_fn=layers.batch_norm, normalizer_params=normalizer_params) if i < len(net_size) - 2: h = tf.nn.relu(h) y = zs.Categorical('y', h, group_ndims=0) return model, h
def p_net(observed, seq_len): ''' Decoder: p(x|z) = p(y_v|w) ''' with zs.BayesianNet(observed=observed) as model: cell = BayesianLSTMCell(128, forget_bias=0.) # shape was [max_seq_len, nb_batches, nb_classes] h_list = bayesian_rnn(cell, x, y_i) item_features = tf.get_variable("item_features", shape=[nb_items, embedding_size, nb_classes], initializer=tf.truncated_normal_initializer(stddev=0.02)) relevant_items = tf.nn.embedding_lookup(item_features, y_i, name="feat_items") logits = tf.tensordot(h_list, relevant_items, axes=[[2], [2]]) # That's not even the good shape but anyway _ = zs.Categorical('y_v', logits) # shape of its local_log_prob = [max_seq_len, nb_batches] # because we already observe the true variable (y_v is in observed) return model
def var_dropout(observed, x, n, net_size, n_particles, is_training): with zs.BayesianNet(observed=observed) as model: h = x normalizer_params = { 'is_training': is_training, 'updates_collections': None } for i, [n_in, n_out] in enumerate(zip(net_size[:-1], net_size[1:])): eps_mean = tf.zeros([n, n_in]) # Adding noise to Weights eps = zs.Normal('layer' + str(i) + '/eps', eps_mean, std=1., n_samples=n_particles, group_ndims=1) h = layers.fully_connected(h * eps, n_out, normalizer_fn=layers.batch_norm, normalizer_params=normalizer_params) if i < len(net_size) - 2: h = tf.nn.relu(h) print(np.shape(h)) y_logstd = tf.get_variable('y_logstd', shape=[], initializer=tf.constant_initializer(0.)) noise = tf.random_normal(shape=tf.shape(h), mean=0.0, stddev=0.1, dtype=tf.float32) y = zs.Categorical('y', h + noise) # print(i) print(np.shape(y)) return model, h
def px_z_y(self, observed, captions=None, lengths=None, gen_mode=False, n_x=None): """ Args: observed: for q, parametrized by encoder, used during training Returns: model: zhusuan model object, can be used for getting probabilities """ if captions is not None and lengths is not None: self.captions = captions self.lengths = lengths if n_x is None: n_x = tf.shape(self.images_fv)[0] with zs.BayesianNet(observed) as model: z_mean = tf.zeros([n_x, self.params.latent_size]) z = zs.Normal('z', mean=z_mean, std=self.params.std, group_ndims=1, n_samples=self.params.gen_z_samples) tf.summary.histogram("distributions/z", z) y_logits = tf.zeros([n_x, self.n_classes]) y = zs.OnehotCategorical('y', y_logits, n_samples=self.params.gen_z_samples) with tf.variable_scope("net"): embedding = tf.get_variable( "dec_embeddings", [self.data_dict.vocab_size, self.params.embed_size], dtype=tf.float32) # word dropout before = tf.reshape(self.captions, [-1]) word_drop_keep = self.params.word_dropout_keep if gen_mode: word_drop_keep = 1.0 captions = tf.nn.dropout(tf.to_float(self.captions), word_drop_keep) after = tf.reshape(tf.to_int32(captions), [-1]) mask_after = tf.to_int32(tf.not_equal(before, after)) to_unk = mask_after * self.data_dict.word2idx['<UNK>'] captions = tf.reshape(tf.add(after, to_unk), [tf.shape(self.images_fv)[0], -1]) vect_inputs = tf.nn.embedding_lookup(embedding, captions) dec_lstm_drop = self.params.dec_lstm_drop if gen_mode: dec_lstm_drop = 1.0 cell_0 = make_rnn_cell([self.params.decoder_hidden], base_cell=tf.contrib.rnn.LSTMCell, dropout_keep_prob=dec_lstm_drop) # zero_state0 = cell_0.zero_state( # batch_size=tf.shape(self.images_fv)[0], # dtype=tf.float32) # run this cell to get initial state added_shape = self.params.gen_z_samples * self.params.n_classes +\ self.params.embed_size # added_shape = self.params.embed_size # f_mapping = tf.layers.dense(self.images_fv, added_shape, # name='f_emb2') c = h = tf.layers.dense(self.images_fv, self.params.decoder_hidden, name='dec_init_map') initial_state0 = (tf.nn.rnn_cell.LSTMStateTuple(c, h), ) # vector z, mapped into embed_dim z = tf.concat([z, tf.to_float(y)], 2) z = tf.reshape(z, [n_x, (self.params.latent_size + self.n_classes)\ * self.params.gen_z_samples]) z_dec = layers.dense(z, added_shape, name='z_rnn') _, z_state = cell_0(z_dec, initial_state0) initial_state = rnn_placeholders(z_state) # concat with inputs y_re = tf.to_float( tf.reshape(y, [ tf.shape(self.images_fv)[0], self.params.gen_z_samples * self.params.n_classes ])) y = tf.tile(tf.expand_dims(y_re, 1), [1, tf.shape(vect_inputs)[1], 1]) vect_inputs = tf.concat([vect_inputs, y], 2) # vect_inputs = tf.Print(vect_inputs, [tf.shape(vect_inputs)], # first_n=1) # captions LSTM outputs, final_state = tf.nn.dynamic_rnn( cell_0, inputs=vect_inputs, sequence_length=self.lengths, initial_state=initial_state, swap_memory=True, dtype=tf.float32) # output shape [batch_size, seq_length, self.params.decoder_hidden] if gen_mode: # only interested in the last output outputs = outputs[:, -1, :] outputs_r = tf.reshape(outputs, [-1, cell_0.output_size]) x_logits = tf.layers.dense(outputs_r, units=self.data_dict.vocab_size, name='rnn_logits') x_logits_r = tf.reshape( x_logits, [tf.shape(outputs)[0], tf.shape(outputs)[1], -1]) x = zs.Categorical('x', x_logits_r, group_ndims=1) # for generating sample = None if gen_mode: if self.params.sample_gen == 'sample': sample = tf.multinomial(x_logits / self.params.temperature, 1)[0][0] elif self.params.sample_gen == 'beam_search': sample = tf.nn.softmax(x_logits) else: sample = tf.nn.softmax(x_logits) return model, x_logits, (initial_state, final_state, sample)