def __init__(self, sess, state_size, action_size, hidden_dim):
self.sess = sess
with tf.variable_scope('policy_net'):
self.states = tf.placeholder(dtype=tf.float32,
shape=[None, state_size],
name='policy_states')
self.targets = tf.placeholder(dtype=tf.float32,
shape=[None, 1],
name='policy_targets')
self.actions = tf.placeholder(dtype=tf.int32,
shape=[None, 1],
name='policy_actions')
x = Dense(units=hidden_dim, activation='relu')(self.states)
x = Dense(units=hidden_dim, activation='relu')(x)
self.logits = Dense(units=action_size, activation=None)(x)
dist = Categorical(logits=self.logits)
optimizer = tf.train.AdamOptimizer(0.001)
# Get action
self.new_action = dist.sample()
self.new_log_prob = dist.log_prob(self.new_action)
# Calc loss
log_prob = dist.log_prob(tf.squeeze(self.actions))
loss = tf.reduce_mean(tf.squeeze(self.targets) - 0.2 * log_prob)
self.train_op = optimizer.minimize(loss,
var_list=_params('policy_net'))
def generate_samples(self, sess, args, chars, vocab, seq_length = 200,
initial = ' ',
datafile = 'data/gan/fake_reviews.txt'):
''' Generate a batch of reviews'''
state = self.cell_gen.zero_state(args.batch_size, tf.float32).eval()
sequence_matrix = []
for i in xrange(args.batch_size):
sequence_matrix.append([])
char_arr = args.batch_size * [initial]
for n in xrange(seq_length):
x = np.zeros((args.batch_size, 1))
for i, char in enumerate(char_arr):
x[i,0] = vocab[char]
feed = {self.input_data: x, self.initial_state_gen: state}
sample_op = Categorical(self.logits_sequence[0])
[sample_indexes, state] = sess.run(
[sample_op.sample(n = 1), self.final_state_gen], feed)
char_arr = [chars[i] for i in tf.squeeze(sample_indexes)]
for i, char in enumerate(char_arr):
sequence_matrix[i].append(char)
with open(datafile, 'a+') as f:
for line in sequence_matrix:
print ''.join(line)
print>>f, ''.join(line)
return sequence_matrix
def sample(self, time, outputs, state, name=None):
with ops.name_scope(name, "ScheduledEmbeddingTrainingHelperSample",
[time, outputs, state]):
# Return -1s where we did not sample, and sample_ids elsewhere
select_sample_noise = random_ops.random_uniform(
[self.batch_size], seed=self._scheduling_seed)
select_sample = (self._sampling_probability > select_sample_noise)
sample_id_sampler = Categorical(logits=outputs)
return array_ops.where(select_sample,
sample_id_sampler.sample(seed=self._seed),
array_ops.tile([-1], [self.batch_size]))
def get_gaussian_mixture_log_prob(cat_probs, gauss_mu, gauss_sigma):
"""Get the logrithmic p.d.f. of a Gaussian mixture model.
Args:
cat_probs:
`1-D` tensor with unit (reduce) sum, as the categorical probabilities.
gauss_mu:
List of tensors, with the length the shape of `cat_probs`, as the `mu`
values of the Gaussian components. All these tensors shall share the
same shape (as, e.g., `gauss_mu[0]`)
gauss_sigma:
List of tensors, with the length the shape of `cat_probs`, as the `sigma`
values of the Gaussian components. Thus shall be all positive, and shall
be all the same shape as `gauss_mu[0]`.
Returns:
Callable, mapping from tensor of the shape of `gauss_mu[0]` to scalar, as
the p.d.f..
"""
n_cats = cat_probs.shape[0]
cat = Categorical(probs=cat_probs)
components = [
Independent( Normal(gauss_mu[i], gauss_sigma[i]) )
for i in range(n_cats)
]
distribution = Mixture(cat=cat, components=components)
return distribution.log_prob
def mixture(locs, scales, pi, K):
cat = Categorical(probs=pi)
components = [
MultivariateNormalDiag(loc=locs[:, i, :], scale_diag=scales[:, i, :])
for i in range(K)]
# get the mixture distribution
mix = Mixture(cat=cat, components=components)
return mix
def mog_from_out_params(mog_params, use_log_scales):
logit_probs, means, std_params = tf.split(mog_params, num_or_size_splits=3, axis=2)
cat = Categorical(logits=logit_probs)
nr_mix = mog_params.get_shape().as_list()[2] // 3
components = []
for i in range(nr_mix):
gauss_params = tf.stack([means[:, :, i], std_params[:, :, i]], axis=2)
mean, std = mean_std_from_out_params(gauss_params, use_log_scales)
components.append(Normal(loc=mean, scale=std))
distribution = Mixture(cat=cat, components=components)
return distribution
def sampling_func(y_pred):
out_mu, out_sigma, out_pi = tf.split(y_pred, num_or_size_splits=[num_mixes * output_dim,
num_mixes * output_dim,
num_mixes],
axis=1, name='mdn_coef_split')
cat = Categorical(logits=out_pi)
component_splits = [output_dim] * num_mixes
mus = tf.split(out_mu, num_or_size_splits=component_splits, axis=1)
sigs = tf.split(out_sigma, num_or_size_splits=component_splits, axis=1)
coll = [MultivariateNormalDiag(loc=loc, scale_diag=scale) for loc, scale
in zip(mus, sigs)]
mixture = Mixture(cat=cat, components=coll)
samp = mixture.sample()
# Todo: temperature adjustment for sampling function.
return samp
def loss_func(y_true, y_pred):
out_mu, out_sigma, out_pi = tf.split(y_pred, num_or_size_splits=[num_mixes * output_dim,
num_mixes * output_dim,
num_mixes],
axis=1, name='mdn_coef_split')
cat = Categorical(logits=out_pi)
component_splits = [output_dim] * num_mixes
mus = tf.split(out_mu, num_or_size_splits=component_splits, axis=1)
sigs = tf.split(out_sigma, num_or_size_splits=component_splits, axis=1)
coll = [MultivariateNormalDiag(loc=loc, scale_diag=scale) for loc, scale
in zip(mus, sigs)]
mixture = Mixture(cat=cat, components=coll)
loss = mixture.log_prob(y_true)
loss = tf.negative(loss)
loss = tf.reduce_mean(loss)
return loss
def get_trained_q(trained_var):
"""Get the trained inference distribution :math:`q` (c.f. section "Notation"
in the documentation).
Args:
trained_var:
`dict` object with keys contains "a", "mu", and "zeta", and values being
either numpy arraies or TensorFlow tensors (`tf.constant`), as the value
of the trained value of variables in "nn4post".
Returns:
An instance of `Mixture`.
"""
var_names = ['a', 'mu', 'zeta']
for name in var_names:
if name not in trained_var.keys():
e = (
'{0} is not in the keys of {1}.'
).format(name, trained_var)
raise Exception(e)
_trained_var = {
name:
val if isinstance(val, tf.Tensor) \
else tf.constant(val)
for name, val in trained_var.items()
}
cat = Categorical(tf.nn.softmax(_trained_var['a']))
mu_zetas = list(zip(
tf.unstack(_trained_var['mu'], axis=0),
tf.unstack(_trained_var['zeta'], axis=0),
))
components = [
Independent(
NormalWithSoftplusScale(mu, zeta)
) for mu, zeta in mu_zetas
]
mixture = Mixture(cat, components)
return mixture
def model(input_, param):
""" Shall be implemented by TensorFlow. This is an example, as a shallow
neural network.
Args:
input_:
`dict`, like `{'x_1': x_1, 'x_2': x_2}, with values Tensors.
param:
`dict`, like `{'w': w, 'b': b}, with values Tensors.
Returns:
`dict`, like `{'y': Y}`, where `Y` is an instance of
`tf.distributions.Distribution`.
"""
# shape: `[None, n_hiddens]`
hidden = tf.sigmoid(tf.matmul(input_['x'], param['w_h']) + param['b_h'])
# shape: `[None, n_outputs]`
logits = tf.matmul(hidden, param['w_a']) + param['b_a']
Y = Categorical(logits=logits)
return {'y': Y}
def get_trained_posterior(trained_var, param_shape):
"""
Args:
trained_var:
`dict` object with keys contains "a", "mu", and "zeta", and values being
either numpy arraies or TensorFlow tensors (`tf.constant`), as the value
of the trained value of variables in "nn4post".
param_shape:
`dict` with keys the parameter-names and values the assocated shapes (as
lists).
Returns:
Dictionary with keys the parameter-names and values instances of `Mixture`
as the distributions that fit the associated posteriors.
"""
n_c = trained_var['a'].shape[0]
cat = Categorical(logits=trained_var['a'])
parse_param = get_parse_param(param_shape)
mu_list = [parse_param(trained_var['mu'][i]) for i in range(n_c)]
zeta_list = [parse_param(trained_var['zeta'][i]) for i in range(n_c)]
trained_posterior = {}
for param_name in trained_var.keys():
components = [
Independent(NormalWithSoftplusScale(
mu_list[i][param_name], zeta_list[i][param_name]))
for i in range(n_c)
]
mixture = Mixture(cat, components)
trained_posterior[param_name] = mixture
return trained_posterior
#OPTIMIZER = tf.train.AdamOptimizer(LR)
OPTIMIZER = tf.train.RMSPropOptimizer(LR)
DTYPE = 'float32'
SKIP_STEP = 50
# -- Gaussian Mixture Distribution
with tf.name_scope('posterior'):
target_c = tf.constant([0.05, 0.25, 0.70])
target_mu = tf.stack(
[tf.ones([N_D]) * (i - 1) * 3 for i in range(TARGET_N_C)], axis=0)
target_zeta_val = np.zeros([TARGET_N_C, N_D])
#target_zeta_val[1] = np.ones([N_D]) * 5.0
target_zeta = tf.constant(target_zeta_val, dtype='float32')
cat = Categorical(probs=target_c)
components = [
Independent(NormalWithSoftplusScale(target_mu[i], target_zeta[i]))
for i in range(TARGET_N_C)
]
p = Mixture(cat, components)
def log_posterior(theta):
return p.log_prob(theta)
# test!
# test 1
init_var = {
'a':
np.zeros([N_C], dtype=DTYPE),
def sample(probs, name):
dist = Categorical(probs=probs, allow_nan_stats=False,
name=name) # XXX Categorical/logits/Log:0: NaN
return dist.sample()
def categorical_log_prob(p, x):
return Categorical(p).log_prob(x)
def categorical_sample(p):
return Categorical(p).sample()
def inference(self):
encoder = self.x
with tf.variable_scope("ENCODER"):
for i, hidden_size in enumerate(self.hidden_sizes):
encoder = dense_layer(inputs=encoder,
num_outputs=hidden_size,
activation_fn=relu,
use_batch_norm=self.use_batch_norm,
is_training=self.is_training,
scope='{:d}'.format(i + 1))
with tf.variable_scope("Z"):
z_mu = dense_layer(inputs=encoder,
num_outputs=self.latent_size,
activation_fn=None,
use_batch_norm=False,
is_training=self.is_training,
scope='MU')
z_sigma = dense_layer(
inputs=encoder,
num_outputs=self.latent_size,
activation_fn=lambda x: tf.exp(tf.clip_by_value(x, -3, 3)),
use_batch_norm=False,
is_training=self.is_training,
scope='SIGMA')
self.q_z_given_x = Normal(mu=z_mu, sigma=z_sigma)
# Mean of z
self.z_mean = self.q_z_given_x.mean()
# Stochastic layer
self.z = self.q_z_given_x.sample()
# Decoder - Generative model, p(x|z)
if self.use_count_sum:
decoder = tf.concat([self.z, self.n], axis=1, name='Z_N')
else:
decoder = self.z
with tf.variable_scope("DECODER"):
for i, hidden_size in enumerate(reversed(self.hidden_sizes)):
decoder = dense_layer(
inputs=decoder,
num_outputs=hidden_size,
activation_fn=relu,
use_batch_norm=self.use_batch_norm,
is_training=self.is_training,
scope='{:d}'.format(len(self.hidden_sizes) - i))
# Reconstruction distribution parameterisation
with tf.variable_scope("X_TILDE"):
x_theta = {}
for parameter in self.reconstruction_distribution["parameters"]:
parameter_activation_function = \
self.reconstruction_distribution["parameters"]\
[parameter]["activation function"]
p_min, p_max = \
self.reconstruction_distribution["parameters"]\
[parameter]["support"]
x_theta[parameter] = dense_layer(
inputs=decoder,
num_outputs=self.feature_size,
activation_fn=lambda x: tf.clip_by_value(
parameter_activation_function(x), p_min + self.epsilon,
p_max - self.epsilon),
is_training=self.is_training,
scope=parameter.upper())
self.p_x_given_z = self.reconstruction_distribution["class"](
x_theta)
if self.k_max:
x_logits = dense_layer(inputs=decoder,
num_outputs=self.feature_size *
self.k_max,
activation_fn=None,
is_training=self.is_training,
scope="P_K")
x_logits = tf.reshape(x_logits,
[-1, self.feature_size, self.k_max])
self.p_x_given_z = Categorized(
dist=self.p_x_given_z, cat=Categorical(logits=x_logits))
self.x_tilde_mean = self.p_x_given_z.mean()
# Add histogram summaries for the trainable parameters
for parameter in tf.trainable_variables():
tf.summary.histogram(parameter.name, parameter)
def build_model(self):
net = tl.layers.InputLayer(self.input, name='input_layer')
with tf.variable_scope('fc1'):
net = tl.layers.TimeDistributedLayer(
net,
layer_class=tl.layers.DenseLayer,
args={
'n_units': 32,
'act': tf.nn.elu,
'W_init': tf.contrib.layers.variance_scaling_initializer(),
'W_init_args': {
'regularizer':
tf.contrib.layers.l2_regularizer(
self.args.weight_decay)
},
'name': 'fc1_'
},
name='time_dense_fc1')
# net = tl.layers.DropoutLayer(net, keep=self.args.keep_prob, name='fc1_drop')
with tf.variable_scope('highway'):
num_highway = 3
for idx in xrange(num_highway):
highway_args = {
'n_units': 32,
'act': tf.nn.elu,
'W_init': tf.contrib.layers.variance_scaling_initializer(),
'b_init': tf.constant_initializer(value=0.0),
'W_init_args': {
'regularizer':
tf.contrib.layers.l2_regularizer(
self.args.weight_decay)
},
'name': 'highway_%03d_' % idx
}
net = tl.layers.TimeDistributedLayer(
net,
layer_class=utility.Highway,
args=highway_args,
name='time_dense_highway_%d' % idx)
with tf.variable_scope('fc2'):
net = tl.layers.TimeDistributedLayer(
net,
layer_class=tl.layers.DenseLayer,
args={
'n_units': 64,
'act': tf.nn.elu,
'W_init': tf.contrib.layers.variance_scaling_initializer(),
'W_init_args': {
'regularizer':
tf.contrib.layers.l2_regularizer(
self.args.weight_decay)
},
'name': 'highway_to_fc_'
},
name='time_dense_highway_to_fc')
net = tl.layers.DropoutLayer(net,
keep=self.args.keep_prob,
name='hw_to_fc_drop')
with tf.variable_scope('RNN'):
if self.args.rnn_cell == 'lstm':
rnn_cell_fn = tf.contrib.rnn.BasicLSTMCell
elif self.args.rnn_cell == 'gru':
rnn_cell_fn = tf.contrib.rnn.GRUCell
else:
raise ValueError(
'Unimplemented RNN Cell, should be \'lstm\' or \'gru\'')
self.rnn_keep_prob = tf.placeholder(tf.float32)
rnn_layer_name = 'DRNN_layer'
net = tl.layers.DynamicRNNLayer(layer=net,
cell_fn=rnn_cell_fn,
n_hidden=128,
dropout=(1.0, self.rnn_keep_prob),
n_layer=self.args.num_cells,
return_last=True,
name=rnn_layer_name)
rnn_weights_params = [
var for var in net.all_params
if rnn_layer_name in var.name and 'weights' in var.name
]
self.add_regularization_loss(rnn_weights_params)
net = tl.layers.DenseLayer(
net,
n_units=256,
act=tf.nn.elu,
W_init=tf.contrib.layers.variance_scaling_initializer(),
W_init_args={
'regularizer':
tf.contrib.layers.l2_regularizer(self.args.weight_decay)
},
name='fc_3')
net = tl.layers.DenseLayer(
net,
n_units=128,
act=tf.nn.elu,
W_init=tf.contrib.layers.variance_scaling_initializer(),
W_init_args={
'regularizer':
tf.contrib.layers.l2_regularizer(self.args.weight_decay)
},
name='fc_4')
mus_num = self.args.num_mixtures * self.args.gaussian_dim
sigmas_num = self.args.num_mixtures * self.args.gaussian_dim
weights_num = self.args.num_mixtures
num_output = mus_num + sigmas_num + weights_num
net = tl.layers.DenseLayer(
net,
n_units=num_output * self.args.pred_frames_num,
act=tf.identity,
W_init=tf.contrib.layers.variance_scaling_initializer(),
W_init_args={
'regularizer':
tf.contrib.layers.l2_regularizer(self.args.weight_decay)
},
name='nn_output')
self.net = tl.layers.ReshapeLayer(
net,
shape=[-1, self.args.pred_frames_num, num_output],
name='reshape')
output = self.net.outputs
with tf.variable_scope('MDN'):
mus = output[:, :, :mus_num]
sigmas = tf.exp(output[:, :, mus_num:mus_num + sigmas_num])
self.weight_logits = output[:, :, mus_num + sigmas_num:]
self.mus = tf.reshape(
mus, (-1, self.args.pred_frames_num, self.args.num_mixtures,
self.args.gaussian_dim))
self.sigmas = tf.reshape(
sigmas, (-1, self.args.pred_frames_num, self.args.num_mixtures,
self.args.gaussian_dim))
self.weights = tf.nn.softmax(self.weight_logits)
self.y_mix = []
for time_step in xrange(self.args.pred_frames_num):
cat = Categorical(logits=self.weight_logits[:, time_step, :])
components = [
MultivariateNormalDiag(mu=mu, diag_stdev=sigma)
for mu, sigma in zip(
tf.unstack(
tf.transpose(self.mus[:, time_step, :, :], (1, 0,
2))),
tf.unstack(
tf.transpose(self.sigmas[:,
time_step, :, :], (1, 0,
2))))
]
self.y_mix.append(Mixture(cat=cat, components=components))
self.loss = self.get_loss()
"initial value": tf.ones
},
"mus": {
"support": [-inf, inf],
"activation function": identity,
"initial value": lambda x: tf.random_normal(x, stddev=1)
},
"log_sigmas": {
"support": [-3, 3],
"activation function": identity,
"initial value": tf.zeros
}
},
"class":
lambda theta: Mixture(
cat=Categorical(logits=theta["logits"]),
components=[
MultivariateNormalDiag(loc=m, scale_diag=tf.exp(s))
for m, s in zip(theta["mus"], theta["log_sigmas"])
])
},
"categorical": {
"parameters": {
"logits": {
"support": [-inf, inf],
"activation function": identity
}
},
"class": lambda theta: Categorical(logits=theta["logits"]),
},
"bernoulli": {
def __init__(self, lr, s_len, a_len):
self.s = tf.placeholder(dtype=tf.float32,
shape=(None, s_len),
name='s')
t1 = tf.layers.dense(
inputs=self.s,
units=16,
activation=tf.nn.relu,
use_bias=True,
kernel_initializer=tf.keras.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer(),
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
trainable=True,
name='t1')
t2 = tf.layers.dense(
inputs=t1,
units=16,
activation=tf.nn.relu,
use_bias=True,
kernel_initializer=tf.keras.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer(),
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
trainable=True,
name='t2')
t3 = tf.layers.dense(
inputs=t2,
units=16,
activation=tf.nn.sigmoid,
use_bias=True,
kernel_initializer=tf.keras.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer(),
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
trainable=True,
name='t3')
self.p = tf.layers.dense(
inputs=t3,
units=a_len,
activation=tf.nn.softmax,
use_bias=True,
kernel_initializer=tf.keras.initializers.glorot_uniform(),
bias_initializer=tf.zeros_initializer(),
kernel_regularizer=None,
bias_regularizer=None,
activity_regularizer=None,
trainable=True,
name='p')
self.ava = tf.placeholder(dtype=bool,
shape=[None, 13 * 8 * 13],
name='ava')
self.p_unnorm = self.p * tf.cast(self.ava, tf.float32)
self.p_norm = self.p_unnorm / (tf.reduce_sum(self.p_unnorm, axis=1))
self.dist = Categorical(probs=self.p_norm)
self.a = self.dist.sample()
self.G = tf.placeholder(dtype=tf.float32, shape=[None], name='G')
self.a_acted = tf.placeholder(dtype=tf.int32,
shape=[None],
name='a_acted')
a_indices = tf.stack([
tf.range(tf.shape(self.a_acted)[0], dtype=tf.int32), self.a_acted
],
axis=1)
self.gathered_p = tf.gather_nd(params=self.p, indices=a_indices)
self.logpi = tf.log(self.gathered_p)
self.loss = -tf.reduce_mean(self.G * self.logpi)
self.train_op = tf.train.AdamOptimizer(lr).minimize(self.loss)
self.sess = tf.Session()
init = tf.global_variables_initializer()
self.sess.run(init)
self.saver = tf.train.Saver()
return
def sample(probs): dist = Categorical(probs=probs) return dist.sample()
def sample(probs):
dist = Categorical(probs=probs, allow_nan_stats=False)
return dist.sample()
