def var_dropout(observed, x, n, net_size, n_particles, is_training):
with zs.BayesianNet(observed=observed) as model:
h = x
normalizer_params = {
'is_training': is_training,
'updates_collections': None
}
for i, [n_in, n_out] in enumerate(zip(net_size[:-1], net_size[1:])):
eps_mean = tf.zeros([n, n_in])
# Adding noise to Weights
eps = zs.Normal('layer' + str(i) + '/eps',
eps_mean,
std=1.,
n_samples=n_particles,
group_ndims=1)
h = layers.fully_connected(h * eps,
n_out,
normalizer_fn=layers.batch_norm,
normalizer_params=normalizer_params)
if i < len(net_size) - 2:
h = tf.nn.relu(h)
y = zs.Categorical('y', h, group_ndims=0)
return model, h
def p_net(observed, seq_len):
'''
Decoder: p(x|z) = p(y_v|w)
'''
with zs.BayesianNet(observed=observed) as model:
cell = BayesianLSTMCell(128, forget_bias=0.)
# shape was [max_seq_len, nb_batches, nb_classes]
h_list = bayesian_rnn(cell, x, y_i)
item_features = tf.get_variable("item_features", shape=[nb_items, embedding_size, nb_classes],
initializer=tf.truncated_normal_initializer(stddev=0.02))
relevant_items = tf.nn.embedding_lookup(item_features, y_i, name="feat_items")
logits = tf.tensordot(h_list, relevant_items, axes=[[2], [2]]) # That's not even the good shape but anyway
_ = zs.Categorical('y_v', logits) # shape of its local_log_prob = [max_seq_len, nb_batches]
# because we already observe the true variable (y_v is in observed)
return model
def var_dropout(observed, x, n, net_size, n_particles, is_training):
with zs.BayesianNet(observed=observed) as model:
h = x
normalizer_params = {
'is_training': is_training,
'updates_collections': None
}
for i, [n_in, n_out] in enumerate(zip(net_size[:-1], net_size[1:])):
eps_mean = tf.zeros([n, n_in])
# Adding noise to Weights
eps = zs.Normal('layer' + str(i) + '/eps',
eps_mean,
std=1.,
n_samples=n_particles,
group_ndims=1)
h = layers.fully_connected(h * eps,
n_out,
normalizer_fn=layers.batch_norm,
normalizer_params=normalizer_params)
if i < len(net_size) - 2:
h = tf.nn.relu(h)
print(np.shape(h))
y_logstd = tf.get_variable('y_logstd',
shape=[],
initializer=tf.constant_initializer(0.))
noise = tf.random_normal(shape=tf.shape(h),
mean=0.0,
stddev=0.1,
dtype=tf.float32)
y = zs.Categorical('y', h + noise)
# print(i)
print(np.shape(y))
return model, h
def px_z_y(self,
observed,
captions=None,
lengths=None,
gen_mode=False,
n_x=None):
"""
Args:
observed: for q, parametrized by encoder, used during training
Returns:
model: zhusuan model object, can be used for getting probabilities
"""
if captions is not None and lengths is not None:
self.captions = captions
self.lengths = lengths
if n_x is None:
n_x = tf.shape(self.images_fv)[0]
with zs.BayesianNet(observed) as model:
z_mean = tf.zeros([n_x, self.params.latent_size])
z = zs.Normal('z',
mean=z_mean,
std=self.params.std,
group_ndims=1,
n_samples=self.params.gen_z_samples)
tf.summary.histogram("distributions/z", z)
y_logits = tf.zeros([n_x, self.n_classes])
y = zs.OnehotCategorical('y',
y_logits,
n_samples=self.params.gen_z_samples)
with tf.variable_scope("net"):
embedding = tf.get_variable(
"dec_embeddings",
[self.data_dict.vocab_size, self.params.embed_size],
dtype=tf.float32)
# word dropout
before = tf.reshape(self.captions, [-1])
word_drop_keep = self.params.word_dropout_keep
if gen_mode:
word_drop_keep = 1.0
captions = tf.nn.dropout(tf.to_float(self.captions),
word_drop_keep)
after = tf.reshape(tf.to_int32(captions), [-1])
mask_after = tf.to_int32(tf.not_equal(before, after))
to_unk = mask_after * self.data_dict.word2idx['<UNK>']
captions = tf.reshape(tf.add(after, to_unk),
[tf.shape(self.images_fv)[0], -1])
vect_inputs = tf.nn.embedding_lookup(embedding, captions)
dec_lstm_drop = self.params.dec_lstm_drop
if gen_mode:
dec_lstm_drop = 1.0
cell_0 = make_rnn_cell([self.params.decoder_hidden],
base_cell=tf.contrib.rnn.LSTMCell,
dropout_keep_prob=dec_lstm_drop)
# zero_state0 = cell_0.zero_state(
# batch_size=tf.shape(self.images_fv)[0],
# dtype=tf.float32)
# run this cell to get initial state
added_shape = self.params.gen_z_samples * self.params.n_classes +\
self.params.embed_size
# added_shape = self.params.embed_size
# f_mapping = tf.layers.dense(self.images_fv, added_shape,
# name='f_emb2')
c = h = tf.layers.dense(self.images_fv,
self.params.decoder_hidden,
name='dec_init_map')
initial_state0 = (tf.nn.rnn_cell.LSTMStateTuple(c, h), )
# vector z, mapped into embed_dim
z = tf.concat([z, tf.to_float(y)], 2)
z = tf.reshape(z, [n_x, (self.params.latent_size
+ self.n_classes)\
* self.params.gen_z_samples])
z_dec = layers.dense(z, added_shape, name='z_rnn')
_, z_state = cell_0(z_dec, initial_state0)
initial_state = rnn_placeholders(z_state)
# concat with inputs
y_re = tf.to_float(
tf.reshape(y, [
tf.shape(self.images_fv)[0],
self.params.gen_z_samples * self.params.n_classes
]))
y = tf.tile(tf.expand_dims(y_re, 1),
[1, tf.shape(vect_inputs)[1], 1])
vect_inputs = tf.concat([vect_inputs, y], 2)
# vect_inputs = tf.Print(vect_inputs, [tf.shape(vect_inputs)],
# first_n=1)
# captions LSTM
outputs, final_state = tf.nn.dynamic_rnn(
cell_0,
inputs=vect_inputs,
sequence_length=self.lengths,
initial_state=initial_state,
swap_memory=True,
dtype=tf.float32)
# output shape [batch_size, seq_length, self.params.decoder_hidden]
if gen_mode:
# only interested in the last output
outputs = outputs[:, -1, :]
outputs_r = tf.reshape(outputs, [-1, cell_0.output_size])
x_logits = tf.layers.dense(outputs_r,
units=self.data_dict.vocab_size,
name='rnn_logits')
x_logits_r = tf.reshape(
x_logits, [tf.shape(outputs)[0],
tf.shape(outputs)[1], -1])
x = zs.Categorical('x', x_logits_r, group_ndims=1)
# for generating
sample = None
if gen_mode:
if self.params.sample_gen == 'sample':
sample = tf.multinomial(x_logits / self.params.temperature,
1)[0][0]
elif self.params.sample_gen == 'beam_search':
sample = tf.nn.softmax(x_logits)
else:
sample = tf.nn.softmax(x_logits)
return model, x_logits, (initial_state, final_state, sample)