def z1_pre_encoder(x, z2, rhus=[256, 256]):
"""
Pre-stochastic layer encoder for z1 (latent segment variable)
Args:
x(tf.Tensor): tensor of shape (bs, T, F)
z2(tf.Tensor): tensor of shape (bs, D1)
rhus(list): list of numbers of LSTM layer hidden units
Return:
out(tf.Tensor): concatenation of hidden states of all LSTM layers
"""
bs, T = tf.shape(x)[0], tf.shape(x)[1]
z2 = tf.tile(tf.expand_dims(z2, 1), (1, T, 1))
x_z2 = tf.concat([x, z2], axis=-1)
cell = MultiRNNCell([BasicLSTMCell(rhu) for rhu in rhus])
init_state = cell.zero_state(bs, x.dtype)
name = "z1_enc_lstm_%s" % ("_".join(map(str, rhus)), )
_, final_state = dynamic_rnn(cell,
x_z2,
dtype=x.dtype,
initial_state=init_state,
time_major=False,
scope=name)
out = [l_final_state.h for l_final_state in final_state]
out = tf.concat(out, axis=-1)
return out
class RecurrentController(BaseController):
def network_vars(self):
self.lstm_cell = BasicLSTMCell(256)
self.state = self.lstm_cell.zero_state(self.batch_size, tf.float32)
def network_op(self, X, state):
X = tf.convert_to_tensor(X)
return self.lstm_cell(X, state)
def get_state(self):
return self.state
def update_state(self, new_state):
return tf.no_op()
def decoder(z1, z2, x, rhus=[256, 256], x_mu_nl=None, x_logvar_nl=None):
"""
decoder
Args:
z1(tf.Tensor)
z2(tf.Tensor)
x(tf.Tensor): tensor of shape (bs, T, F). only shape is used
rhus(list)
"""
bs = tf.shape(x)[0]
z1_z2 = tf.concat([z1, z2], axis=-1)
cell = MultiRNNCell([BasicLSTMCell(rhu) for rhu in rhus])
state_t = cell.zero_state(bs, x.dtype)
name = "dec_lstm_%s_step" % ("_".join(map(str, rhus)), )
def cell_step(inp, prev_state):
return cell(inp, prev_state, scope=name)
gdim = x.get_shape().as_list()[2]
gname = "dec_gauss_step"
def glayer_step(inp):
return gauss_layer(inp, gdim, x_mu_nl, x_logvar_nl, gname)
out, x_mu, x_logvar, x_sample = [], [], [], []
for t in xrange(x.get_shape().as_list()[1]):
if t > 0:
tf.get_variable_scope().reuse_variables()
out_t, state_t, x_mu_t, x_logvar_t, x_sample_t = decoder_step(
z1_z2, state_t, cell_step, glayer_step)
out.append(out_t)
x_mu.append(x_mu_t)
x_logvar.append(x_logvar_t)
x_sample.append(x_sample_t)
out = tf.stack(out, axis=1, name="dec_pre_out")
x_mu = tf.stack(x_mu, axis=1, name="dec_x_mu")
x_logvar = tf.stack(x_logvar, axis=1, name="dec_x_logvar")
x_sample = tf.stack(x_sample, axis=1, name="dec_x_sample")
px_z = [x_mu, x_logvar]
return out, px_z, x_sample
def z2_pre_encoder(x, rhus=[256, 256]):
"""
Pre-stochastic layer encoder for z2 (latent sequence variable)
Args:
x(tf.Tensor): tensor of shape (bs, T, F)
rhus(list): list of numbers of LSTM layer hidden units
Return:
out(tf.Tensor): concatenation of hidden states of all LSTM layers
"""
bs = tf.shape(x)[0]
cell = MultiRNNCell([BasicLSTMCell(rhu) for rhu in rhus])
init_state = cell.zero_state(bs, x.dtype)
name = "z2_enc_lstm_%s" % ("_".join(map(str, rhus)),)
_, final_state = dynamic_rnn(cell, x, dtype=x.dtype,
initial_state=init_state, time_major=False, scope=name)
out = [l_final_state.h for l_final_state in final_state]
out = tf.concat(out, axis=-1)
return out
def __init__(self, model_parameters, training_parameters, directories,
**kwargs):
""" Initialization of the RNN Model as TensorFlow computational graph
"""
self.model_parameters = model_parameters
self.training_parameters = training_parameters
self.directories = directories
# Define model hyperparameters Tensors
with tf.name_scope("Parameters"):
self.learning_rate = tf.placeholder(tf.float32,
name="learning_rate")
self.momentum = tf.placeholder(tf.float32, name="momentum")
self.input_keep_probability = tf.placeholder(
tf.float32, name="input_keep_probability")
self.output_keep_probability = tf.placeholder(
tf.float32, name="output_keep_probability")
# Define input, output and initialization Tensors
with tf.name_scope("Input"):
self.inputs = tf.placeholder("float", [
None, self.model_parameters.sequence_length,
self.model_parameters.input_dimension
],
name='input_placeholder')
self.targets = tf.placeholder("float", [
None, self.model_parameters.sequence_length,
self.model_parameters.n_classes
],
name='labels_placeholder')
self.init = tf.placeholder(
tf.float32,
shape=[None, self.model_parameters.state_size],
name="init")
# Define the TensorFlow RNN computational graph
with tf.name_scope("RNN"):
cells = []
# Define the layers
for _ in range(self.model_parameters.n_layers):
if self.model_parameters.model == 'rnn':
cell = BasicRNNCell(self.model_parameters.state_size)
elif self.model_parameters.model == 'gru':
cell = GRUCell(self.model_parameters.state_size)
elif self.model_parameters.model == 'lstm':
cell = BasicLSTMCell(self.model_parameters.state_size,
state_is_tuple=True)
elif self.model_parameters.model == 'nas':
cell = NASCell(self.model_parameters.state_size)
else:
raise Exception("model type not supported: {}".format(
self.model_parameters.model))
if (self.model_parameters.output_keep_probability < 1.0
or self.model_parameters.input_keep_probability < 1.0):
if self.model_parameters.output_keep_probability < 1.0:
cell = DropoutWrapper(
cell,
output_keep_prob=self.output_keep_probability)
if self.model_parameters.input_keep_probability < 1.0:
cell = DropoutWrapper(
cell, input_keep_prob=self.input_keep_probability)
cells.append(cell)
cell = MultiRNNCell(cells)
# Simulate time steps and get RNN cell output
self.outputs, self.next_state = tf.nn.dynamic_rnn(cell,
self.inputs,
dtype=tf.float32)
# Define cost Tensors
with tf.name_scope("Cost"):
# Flatten to apply same weights to all time steps
self.flattened_outputs = tf.reshape(
self.outputs, [-1, self.model_parameters.state_size],
name="flattened_outputs")
self.softmax_w = tf.Variable(tf.truncated_normal([
self.model_parameters.state_size,
self.model_parameters.n_classes
],
stddev=0.01),
name="softmax_weights")
self.softmax_b = tf.Variable(tf.constant(
0.1, shape=[self.model_parameters.n_classes]),
name="softmax_biases")
# Softmax activation layer, using RNN inner loop last output
# logits and labels must have the same shape [batch_size, num_classes]
self.logits = tf.matmul(self.flattened_outputs,
self.softmax_w) + self.softmax_b
self.unshaped_predictions = tf.nn.softmax(
self.logits, name="unshaped_predictions")
tf.summary.histogram('logits', self.logits)
# Return to the initial predictions shape
self.predictions = tf.reshape(self.unshaped_predictions, [
-1, self.model_parameters.sequence_length,
self.model_parameters.n_classes
],
name="predictions")
self.cross_entropy = tf.reduce_mean(-tf.reduce_sum(
self.targets *
tf.log(self.predictions), reduction_indices=[2]))
# Get the most likely label for each input
self.label_prediction = tf.argmax(self.predictions,
2,
name="label_predictions")
# Compare predictions to labels
self.correct_prediction = tf.equal(tf.argmax(self.predictions, 2),
tf.argmax(self.targets, 2),
name="correct_predictions")
self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction,
tf.float32),
name="accuracy")
# Define Training Tensors
with tf.name_scope("Train"):
#self.validation_perplexity = tf.Variable(dtype=tf.float32, initial_value=float("inf"),
#trainable=False,
#name="validation_perplexity")
#self.validation_accuracy = tf.Variable(dtype=tf.float32, initial_value=float("inf"),
#trainable=False,
#name="validation_accuracy")
#tf.scalar_summary(self.validation_perplexity.op.name, self.validation_perplexity)
#tf.scalar_summary(self.validation_accuracy.op.name, self.validation_accuracy)
#self.training_epoch_perplexity = tf.Variable(dtype=tf.float32, initial_value=float("inf"),
#trainable=False,
#name="training_epoch_perplexity")
#self.training_epoch_accuracy = tf.Variable(dtype=tf.float32, initial_value=float("inf"),
#trainable=False,
#name="training_epoch_accuracy")
#tf.scalar_summary(self.training_epoch_perplexity.op.name, self.training_epoch_perplexity)
#tf.scalar_summary(self.training_epoch_accuracy.op.name, self.training_epoch_accuracy)
#self.iteration = tf.Variable(0, dtype=tf.int64, name="iteration", trainable=False)
# Momentum optimisation
self.optimizer = tf.train.MomentumOptimizer(
learning_rate=self.learning_rate,
momentum=self.momentum,
name="optimizer")
self.train_step = self.optimizer.minimize(self.cross_entropy,
name="train_step")
# Initializing the variables
self.initializer = tf.global_variables_initializer()
def network_vars(self):
self.lstm_cell = BasicLSTMCell(64)
self.state = tf.Variable(tf.zeros([self.batch_size, 64]),
trainable=False)
self.output = tf.Variable(tf.zeros([self.batch_size, 64]),
trainable=False)
targets = [decoder_inputs[i+1] for i in range(output_seq_len-1)]
# add one more target
targets.append(tf.placeholder(dtype = tf.int32, shape = [None], name = 'last_target'))
target_weights = [tf.placeholder(dtype = tf.float32, shape = [None], name = 'target_w{}'.format(i)) for i in range(output_seq_len)]
# output projection
size = 512
w_t = tf.get_variable('proj_w', [en_vocab_size, size], tf.float32)
b = tf.get_variable('proj_b', [en_vocab_size], tf.float32)
w = tf.transpose(w_t)
output_projection = (w, b)
outputs, states = seq2seq_lib.embedding_attention_seq2seq(
encoder_inputs,
decoder_inputs,
BasicLSTMCell(size),
num_encoder_symbols = zh_vocab_size,
num_decoder_symbols = en_vocab_size,
embedding_size = 100,
feed_previous = False,
output_projection = output_projection,
dtype = tf.float32)
# define our loss function
def sampled_loss(labels, logits):
return tf.nn.sampled_softmax_loss(
weights = w_t,
biases = b,
labels = tf.reshape(labels, [-1, 1]),
inputs = logits,
def __init__(self, kwd_voc_size, *args, **kwargs):
BasicLSTMCell.__init__(self, *args, **kwargs)
self.key_words_voc_size = kwd_voc_size
def network_vars(self):
self.lstm_cell = BasicLSTMCell(256)
self.state = self.lstm_cell.zero_state(self.batch_size, tf.float32)
def __init__(self,
model_parameters,
training_parameters,
directories,
**kwargs):
""" Initialization of the RNN Model as TensorFlow computational graph
"""
self.model_parameters = model_parameters
self.training_parameters = training_parameters
self.directories = directories
# Define model hyperparameters Tensors
with tf.name_scope("Parameters"):
self.learning_rate = tf.placeholder(tf.float32,
name="learning_rate")
self.momentum = tf.placeholder(tf.float32,
name="momentum")
self.input_keep_probability = tf.placeholder(tf.float32,
name="input_keep_probability")
self.output_keep_probability = tf.placeholder(tf.float32,
name="output_keep_probability")
self.is_training = tf.placeholder(tf.bool)
# Define input, output and initialization Tensors
with tf.name_scope("Input"):
self.inputs = tf.placeholder("float", [None,
self.model_parameters.sequence_length,
self.model_parameters.input_dimension],
name='input_placeholder')
self.targets = tf.placeholder("float", [None,
self.model_parameters.sequence_length,
1],
name='labels_placeholder')
self.init = tf.placeholder(tf.float32, shape=[None,
self.model_parameters.state_size],
name="init")
# Define the TensorFlow RNN computational graph
with tf.name_scope("LSTMRNN_RNN"):
cells = []
# Define the layers
for _ in range(self.model_parameters.n_layers):
if self.model_parameters.model == 'rnn':
cell = BasicRNNCell(self.model_parameters.state_size)
elif self.model_parameters.model == 'gru':
cell = GRUCell(self.model_parameters.state_size)
elif self.model_parameters.model == 'lstm':
cell = BasicLSTMCell(self.model_parameters.state_size, state_is_tuple=True)
elif self.model_parameters.model == 'nas':
cell = NASCell(self.model_parameters.state_size)
else:
raise Exception("model type not supported: {}".format(self.model_parameters.model))
if (self.model_parameters.output_keep_probability < 1.0
or self.model_parameters.input_keep_probability < 1.0):
if self.model_parameters.output_keep_probability < 1.0 :
cell = DropoutWrapper(cell,
output_keep_prob=self.output_keep_probability)
if self.model_parameters.input_keep_probability < 1.0 :
cell = DropoutWrapper(cell,
input_keep_prob=self.input_keep_probability)
cells.append(cell)
cell = MultiRNNCell(cells)
# Simulate time steps and get RNN cell output
self.outputs, self.next_state = tf.nn.dynamic_rnn(cell, self.inputs, dtype = tf.float32)
# Define cost Tensors
with tf.name_scope("LSTMRNN_Cost"):
# Flatten to apply same weights to all time steps
self.flattened_outputs = tf.reshape(self.outputs, [-1,
self.model_parameters.state_size],
name="flattened_outputs")
self.output_w = tf.Variable(tf.truncated_normal([
self.model_parameters.state_size,
1], stddev=0.01),
name="output_weights")
self.variable_summaries(self.output_w, 'output_weights')
self.output_b = tf.Variable(tf.constant(0.1),
name="output_biases")
self.variable_summaries(self.output_w, 'output_biases')
# Define decision threshold Tensor
self.decision_threshold = tf.Variable(self.model_parameters.threshold,
name="decision_threshold")
# Define moving average step Tensor
self.ma_step = tf.Variable(self.model_parameters.ma_step,
name="ma_step")
# Softmax activation layer, using RNN inner loop last output
# logits and labels must have the same shape [batch_size, num_classes]
self.logits = tf.add(tf.matmul(self.flattened_outputs, self.output_w),
self.output_b,
name="logits")
self.logits_bn = self.batch_norm_wrapper(inputs=self.logits,
is_training=self.is_training)
tf.summary.histogram('logits', self.logits)
tf.summary.histogram('logits_bn', self.logits_bn)
self.predictions = tf.reshape(self.logits,
[-1, self.model_parameters.sequence_length, 1],
name="predictions")
self.shaped_predictions = tf.reshape(self.predictions,
[-1],
name="shaped_predictions")
self.tmp_smoothed_predictions = tf.concat([self.shaped_predictions,
tf.fill(tf.expand_dims(self.ma_step-1, 0), self.shaped_predictions[tf.shape(self.shaped_predictions)[0]-1])],
axis=0,
name="tmp_smoothed_predictions")
self.ma_loop_idx = tf.constant(0, dtype='int32')
self.shaped_smoothed_predictions = tf.zeros([0], dtype='float32')
_, self.shaped_smoothed_predictions = tf.while_loop(lambda i, _: i < tf.shape(self.shaped_predictions)[0],
self.ma_while_body,
[self.ma_loop_idx, self.shaped_smoothed_predictions],
shape_invariants=[tf.TensorShape([]),
tf.TensorShape([None])])
self.smoothed_predictions = tf.reshape(self.shaped_smoothed_predictions,
[-1, self.model_parameters.sequence_length, 1],
name="smoothed_predictions")
self.soft_predictions_summary = tf.summary.tensor_summary("soft_predictions", self.smoothed_predictions)
# self.soft_predictions_summary = tf.summary.tensor_summary("soft_predictions", self.predictions)
# self.shaped_logits = tf.reshape(self.logits,
# [-1, self.model_parameters.sequence_length, 1],
# name="shaped_logits")
# Cross-Entropy
# self.cost = tf.reduce_mean(-tf.reduce_sum(
# self.targets * tf.log(self.predictions),
# reduction_indices=[2]), name="cross_entropy")
# self.cross_entropy = tf.reduce_mean(
# tf.nn.sigmoid_cross_entropy_with_logits(_sentinel=None,
# labels=self.targets,
# logits=self.predictions),
# name="cross_entropy")
# self.cross_entropy = tf.nn.sigmoid_cross_entropy_with_logits(
# _sentinel=None,
# labels=self.targets,
# logits=self.shaped_logits,
# name="cross_entropy")
# Root Mean Squared Error
# self.mean_squared_error = tf.losses.mean_squared_error(
# labels=self.targets,
# predictions=self.predictions)
self.cost = tf.sqrt(tf.reduce_mean(
tf.squared_difference(
self.smoothed_predictions, self.targets)))
# self.cost = tf.sqrt(tf.reduce_mean(
# tf.squared_difference(
# self.predictions, self.targets)))
tf.summary.scalar('training_cost', self.cost)
# self.cost = tf.reduce_mean(
# self.cross_entropy,
# name="cost")
voicing_condition = tf.greater(self.smoothed_predictions,
tf.fill(tf.shape(self.smoothed_predictions), self.decision_threshold),
name="thresholding")
# voicing_condition = tf.greater(self.predictions,
# tf.fill(tf.shape(self.predictions), self.decision_threshold),
# name="thresholding")
self.label_predictions = tf.where(voicing_condition,
tf.ones_like(self.smoothed_predictions) ,
tf.zeros_like(self.smoothed_predictions),
name="label_predictions")
# self.label_predictions = tf.where(voicing_condition,
# tf.ones_like(self.predictions) ,
# tf.zeros_like(self.predictions),
# name="label_predictions")
self.hard_predictions_summary = tf.summary.tensor_summary("hard_predictions", self.label_predictions)
self.correct_prediction = tf.equal(self.label_predictions,
self.targets,
name="correct_predictions")
self.r = tf.reshape(self.targets, [-1])
self.h = tf.reshape(self.label_predictions, [-1])
# Defined outside the while loop to avoid problems
self.dump_one = tf.constant(1, dtype=tf.int32, shape=[])
self.temp_pk_miss = tf.Variable([0], tf.int32, name='temp_pk_miss')
self.temp_pk_falsealarm = tf.Variable([0], tf.int32, name='temp_pk_falsealarm')
self.loop_idx = tf.constant(0, dtype=tf.int32, name='loop_idx')
self.loop_vars = self.loop_idx, self.temp_pk_miss, self.temp_pk_falsealarm
_, self.all_temp_pk_miss, self.all_temp_pk_falsealarm = tf.while_loop(
self.while_condition,
self.while_body,
self.loop_vars,
shape_invariants=(self.loop_idx.get_shape(), tf.TensorShape([None]), tf.TensorShape([None])))
self.pk_miss = tf.reduce_mean(
tf.cast(self.all_temp_pk_miss, tf.float32))
tf.summary.scalar('p_miss', self.pk_miss)
self.pk_falsealarm = tf.reduce_mean(
tf.cast(self.all_temp_pk_falsealarm, tf.float32))
tf.summary.scalar('p_falsealarm', self.pk_falsealarm)
self.pk = tf.reduce_mean(
tf.cast(
tf.add(self.all_temp_pk_miss, self.all_temp_pk_falsealarm),
tf.float32),
name='pk')
tf.summary.scalar('pk', self.pk)
self.accuracy = tf.reduce_mean(tf.cast(self.correct_prediction, tf.float32),
name="accuracy")
tf.summary.scalar('accuracy', self.accuracy)
self.recall, self.update_op_recall = tf.metrics.recall(
labels=self.targets,
predictions=self.label_predictions,
name="recall")
tf.summary.scalar('recall', self.recall)
self.precision, self.update_op_precision = tf.metrics.precision(
labels=self.targets,
predictions=self.label_predictions,
name="precision")
tf.summary.scalar('precision', self.precision)
# Define Training Tensors
with tf.name_scope("LSTMRNN_Train"):
# Momentum optimisation
self.optimizer = tf.train.MomentumOptimizer(learning_rate=self.learning_rate,
momentum=self.momentum,
name="optimizer")
self.train_step = self.optimizer.minimize(self.cost,
name="train_step")
# Initializing the variables
self.initializer = tf.group(tf.global_variables_initializer(),
tf.local_variables_initializer())
