def encode(self, inputs, input_seq_length, is_training):
'''
Create the variables and do the forward computation
Args:
inputs: the inputs to the neural network, this is a dictionary of
[batch_size x time x ...] tensors
input_seq_length: The sequence lengths of the input utterances, this
is a dictionary of [batch_size] vectors
is_training: whether or not the network is in training mode
Returns:
- the outputs of the encoder as a dictionary of
[bath_size x time x ...] tensors
- the sequence lengths of the outputs as a dictionary of
[batch_size] tensors
'''
#splice the features
spliced = {}
for inp in inputs:
times = [inputs[inp]]
for i in range(1, int(self.conf['context'])):
times.append(
tf.pad(tensor=inputs[inp][:, i:, :],
paddings=[[0, 0], [0, i], [0, 0]]))
times.append(
tf.pad(tensor=inputs[inp][:, :-i, :],
paddings=[[0, 0], [i, 0], [0, 0]]))
spliced[inp] = tf.concat(times, 2)
#do the forward computation
logits = {}
for inp in spliced:
with tf.variable_scope(inp):
logits[inp] = spliced[inp]
#stack the sequences for effecicience reasons
logits[inp] = ops.stack_seq(logits[inp], input_seq_length[inp])
for i in range(int(self.conf['num_layers'])):
logits[inp] = tf.contrib.layers.fully_connected(
inputs=logits[inp],
num_outputs=int(self.conf['num_units']),
scope='layer%d' % i)
if self.conf['layer_norm'] == 'True':
logits[inp] = tf.contrib.layers.layer_norm(logits[inp])
if float(self.conf['dropout']) < 1 and is_training:
logits[inp] = tf.nn.dropout(
logits[inp], float(self.conf['dropout']))
#unstack the sequences again
logits[inp] = ops.unstack_seq(logits[inp],
input_seq_length[inp])
return logits, input_seq_length
def sigmoid_cross_entropy(targets, logits, logit_seq_length, target_seq_length):
'''
Sigmoid cross entropy
Args:
targets: a dictionary of [batch_size x time x ...] tensor containing
the targets
logits: a dictionary of [batch_size x time x ...] tensor containing
the logits
logit_seq_length: a dictionary of [batch_size] vectors containing
the logit sequence lengths
target_seq_length: a dictionary of [batch_size] vectors containing
the target sequence lengths
Returns:
a scalar value containing the loss
'''
with tf.name_scope('sigmoid_cross_entropy_loss'):
losses = []
for t in targets:
#stack all the logits except the final logits
stacked_logits = ops.stack_seq(logits[t], logit_seq_length[t])
#create the stacked targets
stacked_targets = ops.stack_seq(
tf.to_float(targets[t]),
target_seq_length[t])
stacked_loss = tf.nn.sigmoid_cross_entropy_with_logits(
logits=stacked_logits,
labels=stacked_targets)
unstacked_loss = ops.unstack_seq(stacked_loss, logit_seq_length[t])
losses.append(tf.reduce_mean(tf.reduce_sum(unstacked_loss, 1)))
loss = tf.reduce_sum(losses)
return loss
def cross_entropy(targets, logits, logit_seq_length, target_seq_length):
'''
cross enthropy loss
Args:
targets: a dictionary of [batch_size x time x ...] tensor containing
the targets
logits: a dictionary of [batch_size x time x ...] tensor containing
the logits
logit_seq_length: a dictionary of [batch_size] vectors containing
the logit sequence lengths
target_seq_length: a dictionary of [batch_size] vectors containing
the target sequence lengths
Returns:
a scalar value containing the loss
'''
with tf.name_scope('cross_entropy_loss'):
losses = []
for t in targets:
#stack the logits
stacked_logits = ops.stack_seq(logits[t], logit_seq_length[t])
#create the stacked targets
stacked_targets = ops.stack_seq(targets[t], target_seq_length[t])
stacked_targets = tf.cast(stacked_targets, tf.int32)
stacked_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=stacked_logits,
labels=stacked_targets)
unstacked_loss = ops.unstack_seq(stacked_loss, logit_seq_length[t])
losses.append(tf.reduce_mean(tf.reduce_sum(unstacked_loss, 1)))
loss = tf.reduce_sum(losses)
return loss
def cross_entropy_eos(targets, logits, logit_seq_length, target_seq_length):
'''
cross enthropy loss with an end of sequence label added
Args:
targets: a dictionary of [batch_size x time x ...] tensor containing
the targets
logits: a dictionary of [batch_size x time x ...] tensor containing
the logits
logit_seq_length: a dictionary of [batch_size] vectors containing
the logit sequence lengths
target_seq_length: a dictionary of [batch_size] vectors containing
the target sequence lengths
Returns:
a scalar value containing the loss
'''
with tf.name_scope('cross_entropy_loss'):
losses = []
batch_size = tf.shape(targets.values()[0])[0]
for t in targets:
with tf.name_scope('cross_entropy_loss'):
output_dim = tf.shape(logits[t])[-1]
#get the logits for the final timestep
indices = tf.stack(
[tf.range(batch_size), logit_seq_length[t] - 1], axis=1)
final_logits = tf.gather_nd(logits[t], indices)
#stack all the logits except the final logits
stacked_logits = ops.stack_seq(logits[t],
logit_seq_length[t] - 1)
#create the stacked targets
stacked_targets = ops.stack_seq(targets[t],
target_seq_length[t])
#create the targets for the end of sequence labels
final_targets = tf.tile([output_dim - 1], [batch_size])
#compute the loss for the loss for the sequences
stacked_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=stacked_logits, labels=stacked_targets)
#compute the loss for the eos label
eos_loss = tf.nn.sparse_softmax_cross_entropy_with_logits(
logits=final_logits, labels=final_targets)
unstacked_loss = ops.unstack_seq(stacked_loss,
target_seq_length[t])
losses.append(
tf.reduce_mean(
tf.reduce_sum(unstacked_loss, 1) + eos_loss))
loss = tf.reduce_sum(losses)
return loss