def testCustomPlotFunc(self):
batch_size = 3
data = tf.ones((batch_size, 3, 5))
trim = tf.constant([[3, 2], [2, 5], [3, 3]])
titles = tf.constant(['batch1', 'batch2', 'batch3'])
def TrimAndAddImage(fig, axes, data, trim, title, **kwargs):
plot.AddImage(fig,
axes,
data[:trim[0], :trim[1]],
title=title,
**kwargs)
with self.session() as s:
fig = plot.MatplotlibFigureSummary('fig_custom_plotfunc',
max_outputs=batch_size,
plot_func=TrimAndAddImage)
fig.AddSubplot([data, trim, titles])
im = fig.Finalize()
summary_str = s.run(im)
summary = tf.summary.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), batch_size)
for n, value in enumerate(summary.value):
self.assertEqual(value.tag, 'fig_custom_plotfunc/image/%d' % n)
self.assertGreater(value.image.width, 1)
self.assertGreater(value.image.height, 1)
self.assertEqual(value.image.colorspace, 3)
self.assertNotEqual(value.image.encoded_image_string,
self.default_encoded_image)
def _AddSummary(self, batch, prediction):
"""Adds image summaries for the batch."""
p = self.params
if not self.do_eval or not p.add_image_summary:
# Image summaries only works in evaler/decoder.
return
def Draw(fig, axes, img, label, pred):
plot.AddImage(
fig=fig,
axes=axes,
data=img[:, :, 0] / 256.,
show_colorbar=False,
suppress_xticks=True,
suppress_yticks=True)
axes.text(
x=0.5,
y=0,
s=u'%d vs. %d' % (label, pred),
transform=axes.transAxes,
horizontalalignment='center')
with plot.MatplotlibFigureSummary(
'examples', figsize=(1, 1), max_outputs=10) as fig:
fig.AddSubplot([batch.raw, batch.label, prediction], Draw)
def setUp(self):
with self.session() as s:
fig = plot.MatplotlibFigureSummary('DEFAULT', self.FIGSIZE, max_outputs=1)
batched_data = tf.expand_dims(self.DEFAULT_DATA, 0) # Batch size 1.
fig.AddSubplot([batched_data])
im = fig.Finalize()
summary_str = s.run(im)
summary = tf.summary.Summary.FromString(summary_str)
self.default_encoded_image = summary.value[0].image.encoded_image_string
def testEnforcesConsistentBatchSize(self):
batch_size = 4
tensors = [tf.ones((batch_size, 3, 5)), tf.ones((batch_size - 2, 2, 2))]
with self.session() as s:
fig = plot.MatplotlibFigureSummary('summary', self.FIGSIZE, max_outputs=1)
for t in tensors:
fig.AddSubplot([t])
im = fig.Finalize()
with self.assertRaises(tf.errors.InvalidArgumentError):
s.run(im)
def testLimitsOutputImagesIfBatchIsSmall(self):
batch_size = 1
tensors = [tf.zeros((batch_size, 3, 5)), tf.ones((batch_size, 2, 2))]
with self.session() as s:
fig = plot.MatplotlibFigureSummary('summary', self.FIGSIZE, max_outputs=3)
for t in tensors:
fig.AddSubplot([t])
im = fig.Finalize()
summary_str = s.run(im)
summary = tf.summary.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), 1)
def testOnlyPlotsFirstMaxOutputImages(self):
batch_size = 4
tensors = [tf.ones((batch_size, 3, 5)), tf.ones((batch_size, 2, 2))]
with self.session() as s:
fig = plot.MatplotlibFigureSummary('summary', self.FIGSIZE, max_outputs=2)
for t in tensors:
fig.AddSubplot([t])
im = fig.Finalize()
summary_str = s.run(im)
summary = tf.summary.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), 2)
def testDoesNotDieOnMatplotlibError(self):
invalid_dim_data = tf.ones((5,))
with self.session() as s:
fig = plot.MatplotlibFigureSummary('summary', self.FIGSIZE, max_outputs=1)
fig.AddSubplot([invalid_dim_data])
im = fig.Finalize()
summary_str = s.run(im)
summary = tf.summary.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), 1)
value = summary.value[0]
# Generates dummy 1-pixel image.
self.assertEqual(value.image.width, 1)
self.assertEqual(value.image.height, 1)
def testCanUseAsContextManager(self):
with self.session() as s:
with plot.MatplotlibFigureSummary(
'context_manager_figure', self.FIGSIZE, max_outputs=1) as fig:
batched_data = tf.expand_dims(self.DEFAULT_DATA, 0) # Batch size 1.
fig.AddSubplot([batched_data])
summary_str = s.run(tf.summary.merge_all(scope='context_manager_figure'))
summary = tf.summary.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), 1)
value = summary.value[0]
self.assertEqual(value.image.width, self.EXPECTED_DPI * self.FIGSIZE[0])
self.assertEqual(value.image.height, self.EXPECTED_DPI * self.FIGSIZE[1])
self.assertEqual(value.image.colorspace, 3)
self.assertEqual(value.image.encoded_image_string,
self.default_encoded_image)
def testUnicodeText(self):
with self.session() as s:
fig = plot.MatplotlibFigureSummary(
'matplotlib_uni', self.FIGSIZE, max_outputs=1)
batched_data = tf.expand_dims(self.DEFAULT_DATA, 0) # Batch size 1.
fig.AddSubplot([batched_data], xlabel=u'bździągwa', ylabel='żółć')
im = fig.Finalize()
summary_str = s.run(im)
summary = tf.summary.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), 1)
value = summary.value[0]
self.assertEqual(value.tag, 'matplotlib_uni/image')
self.assertEqual(value.image.width, self.EXPECTED_DPI * self.FIGSIZE[0])
self.assertEqual(value.image.height, self.EXPECTED_DPI * self.FIGSIZE[1])
self.assertEqual(value.image.colorspace, 3)
def _AddAttenProbsImageSummary(self, name, atten_probs):
"""Add image summary of input attention probabilities."""
def PlotAttention(fig, axes, cur_atten_probs, title):
plot.AddImage(fig, axes, cur_atten_probs, title=title)
axes.set_ylabel(plot.ToUnicode('Output sequence index'), wrap=True)
axes.set_xlabel(plot.ToUnicode('Input sequence index'), wrap=True)
with plot.MatplotlibFigureSummary(
name + '/atten_example',
figsize=(10, 10),
max_outputs=1,
subplot_grid_shape=(1, 1)) as fig:
# Extract first entry in batch of attention prob matrices
# [tgt_len, src_len]
fig.AddSubplot([atten_probs], PlotAttention, title='atten_probs')
def AddAttentionSummary(params,
attention_tensors,
src_paddings,
tgt_paddings,
transcripts=None,
max_outputs=3):
"""Adds an image summary showing the attention probability matrix and state.
Args:
params: A param object.
attention_tensors: A list of 3D tensors shaped [target_len, batch_size,
source_len] where attention[i, j, k] is the probability for the i-th
output attending to the k-th input for element j in the batch.
src_paddings: A tensor of binary paddings shaped [source_len, batch] for the
source sequence.
tgt_paddings: A tensor of binary paddings shaped [target_len, batch] for the
target sequence.
transcripts: Optional, transcripts shaped [batch, target_len] for the source
sequence.
max_outputs: Integer maximum number of elements of the batch to plot.
Returns:
The added image summary.
"""
name = attention_tensors[0].name + '/Attention'
if not params.add_summary:
return tf.summary.scalar('disabled_%s' % name, 0)
fig = plot.MatplotlibFigureSummary(name, max_outputs=max_outputs)
src_lens = SequenceLength(tf.transpose(src_paddings))
tgt_lens = SequenceLength(tf.transpose(tgt_paddings))
for n, atten in enumerate(attention_tensors):
# Diagnostic metric that decreases as attention picks up.
max_entropy = tf.log(tf.cast(src_lens, tf.float32))
max_entropy = tf.expand_dims(tf.expand_dims(max_entropy, 0), -1)
atten_normalized_entropy = -atten * tf.log(atten + 1e-10) / max_entropy
scalar(params, 'Attention/average_normalized_entropy/%d' % n,
tf.reduce_mean(atten_normalized_entropy))
args = [tf.transpose(atten, [1, 0, 2]), src_lens, tgt_lens]
if transcripts is not None and n == 0:
args.append(transcripts)
fig.AddSubplot(
args,
TrimPaddingAndPlotAttention,
title=atten.name,
xlabel='Input',
ylabel='Output')
return fig.Finalize()
def testBasic(self):
with self.session() as s:
fig = plot.MatplotlibFigureSummary(
'matplotlib_figure', self.FIGSIZE, max_outputs=1)
batched_data = tf.expand_dims(self.DEFAULT_DATA, 0) # Batch size 1.
fig.AddSubplot([batched_data])
im = fig.Finalize()
summary_str = s.run(im)
summary = tf.summary.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), 1)
value = summary.value[0]
self.assertEqual(value.tag, 'matplotlib_figure/image')
self.assertEqual(value.image.width, self.EXPECTED_DPI * self.FIGSIZE[0])
self.assertEqual(value.image.height, self.EXPECTED_DPI * self.FIGSIZE[1])
self.assertEqual(value.image.colorspace, 3)
self.assertEqual(value.image.encoded_image_string,
self.default_encoded_image)
def testCanChangeFigsize(self):
figsize = (self.FIGSIZE[0], 2 * self.FIGSIZE[1])
with self.session() as s:
fig = plot.MatplotlibFigureSummary('summary', figsize, max_outputs=1)
batched_data = tf.expand_dims(self.DEFAULT_DATA, 0) # Batch size 1.
fig.AddSubplot([batched_data])
im = fig.Finalize()
summary_str = s.run(im)
summary_str = s.run(im)
summary = tf.summary.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), 1)
value = summary.value[0]
self.assertEqual(value.image.width, self.EXPECTED_DPI * figsize[0])
self.assertEqual(value.image.height, self.EXPECTED_DPI * figsize[1])
self.assertNotEqual(value.image.encoded_image_string,
self.default_encoded_image)
def PlotSequenceFeatures(plots, name, **kwargs):
"""Plots a stack of sequence features.
Args:
plots: A list of tuple (tensor, seq_len), as returned by
PrepareSequenceForPlot().
name: A string for the caption of the plot.
**kwargs: Keyword arguments passed to AddSubplot().
"""
if not _ShouldAddSummary():
return
with plot.MatplotlibFigureSummary(name, figsize=(8, len(plots) * 3.5)) as fig:
for i, (tensor, seq_len) in enumerate(plots):
fig.AddSubplot([tensor, seq_len],
TrimPaddingAndPlotSequence,
title=GetTensorName(tensor, name, i),
**kwargs)
def AddAttentionSummaryBatchMajor(attention_tensors,
src_paddings,
tgt_paddings,
transcripts=None,
max_outputs=3):
"""Adds an image summary showing the attention probability matrix and state.
As opposed to AddAttentionSummary() takes all tensors with batch dimension in
axis 0.
Args:
attention_tensors: A list of 3D tensors shaped [batch_size, target_len,
source_len] where attention[b, i, j] is the probability for the i-th
output attending to the j-th input for element b in the batch.
src_paddings: A tensor of binary paddings shaped [batch, source_len] for the
source sequence.
tgt_paddings: A tensor of binary paddings shaped [batch, target_len] for the
target sequence.
transcripts: Optional, transcripts shaped [batch, source_len] for the source
sequence.
max_outputs: Integer maximum number of elements of the batch to plot.
"""
name = attention_tensors[0].name + '/Attention'
if not _ShouldAddSummary():
return
with plot.MatplotlibFigureSummary(name, max_outputs=max_outputs) as fig:
src_lens = SequenceLength(src_paddings)
tgt_lens = SequenceLength(tgt_paddings)
for n, atten in enumerate(attention_tensors):
# Diagnostic metric that decreases as attention picks up.
max_entropy = tf.log(tf.cast(src_lens, tf.float32))
max_entropy = tf.expand_dims(tf.expand_dims(max_entropy, -1), -1)
atten_normalized_entropy = -atten * tf.log(atten +
1e-10) / max_entropy
scalar('Attention/average_normalized_entropy/%d' % n,
tf.reduce_mean(atten_normalized_entropy))
args = [atten, src_lens, tgt_lens]
if transcripts is not None and n == 0:
args.append(transcripts)
fig.AddSubplot(args,
TrimPaddingAndPlotAttention,
title=atten.name,
xlabel='Input',
ylabel='Output')
def testLargerBatch(self):
batch_size = 4
tensors = [tf.ones((batch_size, 3, 5)), tf.ones((batch_size, 2, 2))]
with self.session() as s:
fig = plot.MatplotlibFigureSummary(
'larger_batch', self.FIGSIZE, max_outputs=batch_size)
for t in tensors:
fig.AddSubplot([t])
im = fig.Finalize()
summary_str = s.run(im)
summary = tf.summary.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), batch_size)
for n, value in enumerate(summary.value):
self.assertEqual(value.tag, u'larger_batch/image/%d' % n)
self.assertEqual(value.image.width, self.EXPECTED_DPI * self.FIGSIZE[0])
self.assertEqual(value.image.height, self.EXPECTED_DPI * self.FIGSIZE[1])
self.assertEqual(value.image.colorspace, 3)
self.assertNotEqual(value.image.encoded_image_string,
self.default_encoded_image)
def _AddAttenProbsSummary(self, source_paddings, targets, atten_probs):
"""Add image summary of attention probs.
Args:
source_paddings: source padding, of shape [src_len, src_batch].
targets: A dict of string to tensors representing the targets one try to
predict. Each tensor in targets is of shape [tgt_batch, tgt_len].
atten_probs: a list of attention probs, each element is of shape
[tgt_len, tgt_batch, src_len].
"""
if not self.cluster.add_summary:
return
num_rows = len(atten_probs)
fig = plot.MatplotlibFigureSummary('decoder_example',
figsize=(6, 3 * num_rows),
max_outputs=1,
subplot_grid_shape=(num_rows, 1))
def PlotAttention(fig, axes, cur_atten_probs, title, set_x_label):
plot.AddImage(fig, axes, cur_atten_probs, title=title)
axes.set_ylabel(plot.ToUnicode('Output sequence index'), wrap=True)
if set_x_label:
axes.set_xlabel(plot.ToUnicode('Input sequence index'),
wrap=True)
index = 0
srclen = tf.cast(tf.reduce_sum(1 - source_paddings[:, index]),
tf.int32)
tgtlen = tf.cast(tf.reduce_sum(1 - targets.paddings[index, :]),
tf.int32)
for i, probs in enumerate(atten_probs):
# Extract first entry in batch of attention prob matrices
# [tgt_len, src_len]
probs = probs[:, index, :]
probs = tf.expand_dims(probs[:tgtlen, :srclen], 0)
fig.AddSubplot([probs],
PlotAttention,
title='atten_probs_%d' % i,
set_x_label=(i == len(atten_probs) - 1))
fig.Finalize()
def testAddMultiCurveSubplot(self):
with self.session(graph=tf.Graph(), use_gpu=False) as sess:
fig = plot.MatplotlibFigureSummary('XXX')
batch_size = 2
tensor = tf.ones([batch_size, 3])
paddings = tf.constant([[0., 0., 0.], [0., 1., 1.]])
plot.AddMultiCurveSubplot(fig, [tensor, tensor],
paddings,
labels=['label1', 'label2'],
xlabels=tf.constant(['a', 'b']),
title='Title',
ylabel='Ylabel')
summary_str = sess.run(fig.Finalize())
summary = tf.Summary.FromString(summary_str)
self.assertEqual(len(summary.value), batch_size)
for n, value in enumerate(summary.value):
self.assertEqual(value.tag, 'XXX/image/%d' % n)
self.assertGreater(value.image.width, 0)
self.assertGreater(value.image.height, 0)
self.assertNotEqual(value.image.encoded_image_string,
self.default_encoded_image)
def CameraImageSummary(frontal_images, run_segment_strings, figsize=(6, 4)):
"""Write frontal_images as tf.Summaries.
Args:
frontal_images: Float tensor of frontal camera images: Shape: [batch,
height, width, depth]. Expected aspect ratio of 3:2 for visualization.
run_segment_strings: Tensor of strings: Shape: [batch, 1]. The associated
RunSegment proto for the batch.
figsize: Tuple indicating size of camera image. Default is (6, 4)
indicating a 3:2 aspect ratio for visualization.
"""
# Parse the run segment strings to extract the run segment info.
run_segment_ids = ExtractRunIds(run_segment_strings)
def DrawCameraImage(fig, axes, frontal_image, run_segment_id):
"""Draw camera image for image summary."""
plot.AddImage(fig=fig,
axes=axes,
data=frontal_image / 256.,
show_colorbar=False,
suppress_xticks=True,
suppress_yticks=True)
txt = axes.text(x=0.5,
y=0.01,
s=run_segment_id,
color='blue',
fontsize=14,
transform=axes.transAxes,
horizontalalignment='center')
txt.set_path_effects([
path_effects.Stroke(linewidth=3, foreground='lightblue'),
path_effects.Normal()
])
with plot.MatplotlibFigureSummary('examples',
figsize=figsize,
max_outputs=10) as fig:
# Plot raw frontal image samples for each example.
fig.AddSubplot([frontal_images, run_segment_ids], DrawCameraImage)
def AddAttentionSummaryBatchMajor(attention_tensors,
src_paddings,
tgt_paddings,
transcripts=None,
max_outputs=3):
"""Adds an image summary showing the attention probability matrix and state.
As opposed to AddAttentionSummary() takes all tensors with batch dimension in
axis 0.
Args:
attention_tensors: A list of 3D tensors shaped [batch_size, target_len,
source_len] where attention[b, i, j] is the probability for the i-th
output attending to the j-th input for element b in the batch.
src_paddings: A tensor of binary paddings shaped [batch, source_len] for the
source sequence. Or a list of tensors of the same length as
attention_tensors with a separate paddings for each entry in
attention_tensors.
tgt_paddings: A tensor of binary paddings shaped [batch, target_len] for the
target sequence. Or a list of tensors of the same length as
attention_tensors with a separate paddings for each entry in
attention_tensors.
transcripts: Optional, transcripts shaped [batch, source_len] for the source
sequence.
max_outputs: Integer maximum number of elements of the batch to plot.
"""
def VerifyLen(paddings):
length = len(paddings) if isinstance(paddings, list) else 1
if length != 1 and length != len(attention_tensors):
raise ValueError('Bad length of paddings list {}'.format(length))
VerifyLen(src_paddings)
VerifyLen(tgt_paddings)
name = attention_tensors[0].name + '/Attention'
if not _ShouldAddSummary():
return
def ToLengths(paddings):
paddings = paddings if isinstance(paddings, list) else [paddings]
return [SequenceLength(p) for p in paddings]
def Get(lengths, i):
return lengths[0 if len(lengths) == 1 else i]
src_lens = ToLengths(src_paddings)
tgt_lens = ToLengths(tgt_paddings)
with plot.MatplotlibFigureSummary(name,
max_outputs=max_outputs,
gridspec_kwargs={'hspace': 0.3}) as fig:
for n, atten in enumerate(attention_tensors):
# Diagnostic metric that decreases as attention picks up.
max_entropy = tf.log(tf.cast(Get(src_lens, n), tf.float32))
max_entropy = tf.expand_dims(tf.expand_dims(max_entropy, -1), -1)
atten_normalized_entropy = -atten * tf.log(atten +
1e-10) / max_entropy
scalar('Attention/average_normalized_entropy/%d' % n,
tf.reduce_mean(atten_normalized_entropy))
args = [atten, Get(src_lens, n), Get(tgt_lens, n)]
if transcripts is not None and n == 0:
args.append(transcripts)
fig.AddSubplot(args,
TrimPaddingAndPlotAttention,
title=atten.name,
xlabel='Input',
ylabel='Output')
def FProp(self, theta, batch, state0=None):
"""Encodes source as represented by 'inputs' and 'paddings'.
Args:
theta: A NestedMap object containing weights' values of this
layer and its children layers.
batch: A NestedMap with fields:
- src_inputs - The inputs tensor. It is expected to be of shape [batch,
time, feature_dim, channels].
- paddings - The paddings tensor. It is expected to be of shape [batch,
time].
state0: Recurrent input state. Not supported/ignored by this encoder.
Returns:
A NestedMap containing:
- 'encoded': a feature tensor of shape [time, batch, depth]
- 'padding': a 0/1 tensor of shape [time, batch]
- 'state': the updated recurrent state
- '${layer_type}_${layer_index}': The per-layer encoder output. Each one
is a NestedMap containing 'encoded' and 'padding' similar to regular
final outputs, except that 'encoded' from conv or conv_lstm layers are
of shape [time, batch, depth, channels].
"""
p = self.params
inputs, paddings = batch.src_inputs, batch.paddings
outputs = py_utils.NestedMap()
with tf.name_scope(p.name):
# Adding specAugmentation.
if p.use_specaugment and not p.is_eval:
inputs, paddings = self.specaugment.FProp(
theta.specaugment, inputs, paddings)
# Add a few extra padded timesteps at the end. This is for ensuring the
# correctness of the conv-layers at the edges.
if p.pad_steps > 0:
# inplace_update() is not supported by TPU for now. Since we have done
# padding on the input_generator, we may avoid this additional padding.
assert not py_utils.use_tpu()
inputs_pad = tf.zeros(
inplace_ops.inplace_update(tf.shape(inputs), 1,
p.pad_steps), inputs.dtype)
paddings_pad = tf.ones(
inplace_ops.inplace_update(tf.shape(paddings), 1,
p.pad_steps), paddings.dtype)
inputs = tf.concat([inputs, inputs_pad], 1, name='inputs')
paddings = tf.concat([paddings, paddings_pad], 1)
def ReshapeForPlot(tensor, padding, name):
"""Transposes and flattens channels to [batch, dim, seq_len] shape."""
# Flatten any dimensions beyond the third into the third.
batch_size = tf.shape(tensor)[0]
max_len = tf.shape(tensor)[1]
plot_tensor = tf.reshape(tensor, [batch_size, max_len, -1])
plot_tensor = tf.transpose(plot_tensor, [0, 2, 1], name=name)
return (plot_tensor, summary_utils.SequenceLength(padding))
plots = [
ReshapeForPlot(tf.transpose(inputs, [0, 1, 3, 2]), paddings,
'inputs')
]
conv_out = inputs
out_padding = paddings
for i, conv_layer in enumerate(self.conv):
conv_out, out_padding = conv_layer.FProp(
theta.conv[i], conv_out, out_padding)
if p.extra_per_layer_outputs:
conv_out *= (1.0 -
out_padding[:, :, tf.newaxis, tf.newaxis])
outputs['conv_%d' % i] = py_utils.NestedMap(
encoded=tf.transpose(conv_out,
[1, 0, 2, 3]), # to [t, b, d, c]
padding=tf.transpose(out_padding))
plots.append(
ReshapeForPlot(tf.transpose(conv_out, [0, 1, 3, 2]),
out_padding, 'conv_%d_out' % i))
def TransposeFirstTwoDims(t):
first_dim = tf.shape(t)[0]
second_dim = tf.shape(t)[1]
t_new = tf.transpose(
tf.reshape(t, [first_dim, second_dim, -1]), [1, 0, 2])
t_shape_new = tf.concat([[second_dim], [first_dim],
tf.shape(t)[2:]], 0)
return tf.reshape(t_new, t_shape_new)
# Now the conv-lstm part.
conv_lstm_out = conv_out
conv_lstm_out_padding = out_padding
for i, (rnn, cnn) in enumerate(
zip(self.conv_lstm_rnn, self.conv_lstm_cnn)):
conv_lstm_in = conv_lstm_out
# Move time dimension to be the first.
conv_lstm_in = TransposeFirstTwoDims(conv_lstm_in)
conv_lstm_in = tf.expand_dims(conv_lstm_in, 2)
conv_lstm_in_padding = tf.expand_dims(
tf.transpose(conv_lstm_out_padding), 2)
lstm_out = rnn.FProp(theta.conv_lstm_rnn[i], conv_lstm_in,
conv_lstm_in_padding)
# Move time dimension to be the second.
cnn_in = TransposeFirstTwoDims(lstm_out)
cnn_in = tf.squeeze(cnn_in, 2)
cnn_in_padding = conv_lstm_out_padding
cnn_out, cnn_out_padding = cnn.FProp(theta.conv_lstm_cnn[i],
cnn_in, cnn_in_padding)
conv_lstm_out, conv_lstm_out_padding = cnn_out, cnn_out_padding
if p.extra_per_layer_outputs:
conv_lstm_out *= (
1.0 -
conv_lstm_out_padding[:, :, tf.newaxis, tf.newaxis])
outputs['conv_lstm_%d' % i] = py_utils.NestedMap(
encoded=tf.transpose(conv_lstm_out,
[1, 0, 2, 3]), # to [t, b, d, c]
padding=tf.transpose(conv_lstm_out_padding))
plots.append(
ReshapeForPlot(conv_lstm_out, conv_lstm_out_padding,
'conv_lstm_%d_out' % i))
# Need to do a reshape before starting the rnn layers.
conv_lstm_out = py_utils.HasRank(conv_lstm_out, 4)
conv_lstm_out_shape = tf.shape(conv_lstm_out)
new_shape = tf.concat([conv_lstm_out_shape[:2], [-1]], 0)
conv_lstm_out = tf.reshape(conv_lstm_out, new_shape)
if self._first_lstm_input_dim_pad:
conv_lstm_out = tf.pad(
conv_lstm_out,
[[0, 0], [0, 0], [0, self._first_lstm_input_dim_pad]])
conv_lstm_out = py_utils.HasShape(
conv_lstm_out, [-1, -1, self._first_lstm_input_dim])
# Transpose to move the time dimension to be the first.
rnn_in = tf.transpose(conv_lstm_out, [1, 0, 2])
rnn_padding = tf.expand_dims(tf.transpose(conv_lstm_out_padding),
2)
# rnn_in is of shape [time, batch, depth]
# rnn_padding is of shape [time, batch, 1]
# Now the rnn layers.
num_skips = 0
for i in range(p.num_lstm_layers):
rnn_out = self.rnn[i].FProp(theta.rnn[i], rnn_in, rnn_padding)
residual_index = i - p.residual_start + 1
if p.residual_start > 0 and residual_index >= 0:
if residual_index % p.residual_stride == 0:
residual_in = rnn_in
if residual_index % p.residual_stride == p.residual_stride - 1:
# Highway skip connection.
if p.highway_skip:
rnn_out = self.highway_skip[num_skips].FProp(
theta.highway_skip[num_skips], residual_in,
rnn_out)
num_skips += 1
else:
# Residual skip connection.
rnn_out += py_utils.HasShape(
residual_in, tf.shape(rnn_out))
if p.project_lstm_output and (i < p.num_lstm_layers - 1):
# Projection layers.
rnn_out = self.proj[i].FProp(theta.proj[i], rnn_out,
rnn_padding)
if i == p.num_lstm_layers - 1:
rnn_out *= (1.0 - rnn_padding)
if p.extra_per_layer_outputs:
rnn_out *= (1.0 - rnn_padding)
outputs['rnn_%d' % i] = py_utils.NestedMap(
encoded=rnn_out, padding=tf.squeeze(rnn_padding, [2]))
plots.append(
ReshapeForPlot(tf.transpose(rnn_out, [1, 0, 2]),
tf.transpose(rnn_padding, [1, 0, 2]),
'rnn_%d_out' % i))
rnn_in = rnn_out
final_out = rnn_in
if self.cluster.add_summary:
fig = plot.MatplotlibFigureSummary('encoder_example',
figsize=(8,
len(plots) * 3.5))
# Order layers from bottom to top.
plots.reverse()
for tensor, seq_len in plots:
fig.AddSubplot([tensor, seq_len],
summary_utils.TrimPaddingAndPlotSequence,
title=tensor.name,
xlabel='Time')
fig.Finalize()
outputs['encoded'] = final_out
outputs['padding'] = tf.squeeze(rnn_padding, [2])
outputs['state'] = py_utils.NestedMap()
return outputs
