def test_pack_padded_long_sequence_forward_backward(total_length, padding_value,
batch_first, shapes, seed, ctx, func_name):
if not func_name.endswith("Cuda"):
pytest.skip(
"PackPaddedSequence tests except for Cuda for very long sequence skips.")
from nbla_test_utils import function_tester
rng = np.random.RandomState(seed)
sequences = [rng.randn(*shape).astype(np.float32) for shape in shapes]
padded_sequence = pad_sequence(sequences, batch_first)
lengths = np.array([seq.shape[0] for seq in sequences])
inputs = [padded_sequence, lengths]
func_args0 = [batch_first]
func_args1 = [batch_first, padding_value, total_length]
insert_identity = [True, False]
# Forward
function_tester(rng, F.pack_padded_sequence, ref_pack_padded_sequence, inputs,
ctx=ctx, func_name=func_name, func_args=func_args0,
backward=[False, False],
atol_f=1e-3, atol_b=1e-2, insert_identity=insert_identity)
# Backward
import nnabla as nn
padded_sequence0 = nn.Variable.from_numpy_array(
inputs[0]).apply(need_grad=True)
lengths = nn.Variable.from_numpy_array(inputs[1])
with nn.context_scope(ctx), nn.auto_forward():
# Pack backward
padded_sequence0.g = rng.randn(*padded_sequence0.shape)
packed_sequence0, batch_sizes = F.pack_padded_sequence(
padded_sequence0, lengths, *func_args0)
g = rng.randn(*packed_sequence0.shape)
packed_sequence0.g = g
packed_sequence0.parent.backward([padded_sequence0, lengths], [packed_sequence0, batch_sizes],
[False, False])
# Unpack
packed_sequence1 = nn.Variable.from_numpy_array(g)
padded_sequence1, lengths = F.pad_packed_sequence(
packed_sequence1, batch_sizes, *func_args1)
# Compare w/o accum
np.testing.assert_allclose(padded_sequence0.g.flatten(),
padded_sequence1.d.flatten(
)[:np.prod(padded_sequence0.shape)],
atol=1e-4,
err_msg="{} test (w/o accum) with long sequence failed.".format(func_name))
# Compare w/ accum
packed_sequence0.parent.backward([padded_sequence0, lengths], [packed_sequence0, batch_sizes],
[True, False])
np.testing.assert_allclose(padded_sequence0.g.flatten() / 2,
padded_sequence1.d.flatten(
)[:np.prod(padded_sequence0.shape)],
atol=1e-4,
err_msg="{} test (w/ accum) with long sequence failed.".format(func_name))
def pad_packed_sequence(sequence,
batch_first=False,
padding_value=0.0,
total_length=None):
"""Pad packed sequence.
This method unpacks the packed sequqnce and pad it, the inverse operation of :func:`pack_padded_sequence`.
:math:`T_i` is the length of the :math:`i`-th Variable in the sequences.
:math:`B` is the batch size equal to the length of the sequences.
:math:`T` is the max of :math:`T_i` for all :math:`i`.
:math:`*` is the remaining dimensions including none.
.. note::
This function **must** be used the dynamic computation mode.
Example:
.. code-block:: python
import numpy as np
import nnabla as nn
import nnabla.functions as F
import nnabla.utils.rnn as rnn_utils
nn.set_auto_forward(True)
l2v = lambda ldata: nn.Variable.from_numpy_array(np.asarray(ldata))
a = l2v([3, 3])
b = l2v([2, 2, 2])
c = l2v([2, 2, 2])
d = l2v([1, 1, 1, 1])
e = l2v([3, 3])
sequences = [a, b, c, d, e]
packed_sequence = rnn_utils.pack_sequence(sequences, enforce_sorted=False)
print(packed_sequence.data.d)
print(packed_sequence.batch_sizes.d)
padded_sequence, lengths = rnn_utils.pad_packed_sequence(packed_sequence)
print(padded_sequence.d)
print(lengths.d)
Args:
sequence (:obj:`PackedSequence`): PackedSequence.
batch_first (bool): If False, output is of (:math:`T`, :math:`B`, :math:`*`) shape,
otherwise (:math:`B`, :math:`T`, :math:`*`).
padding_value (float): Padding value.
total_length (int): If not None, the outputs are padded up to the `total_length`.
If the `total_length` is less than the max length in the `sequences`,
the error is thrown.
This is normally used in the distributed training to align with
the longest sequence in a distributed system.
Returns:
:obj:`nnabla.Variable` of (:math:`T`, :math:`B`, :math:`*`) or (:math:`B`, :math:`T`, :math:`*`) shape
"""
packed_sequence = sequence.data
batch_sizes = sequence.batch_sizes
sorted_indices = sequence.sorted_indices
unsorted_indices = sequence.unsorted_indices
T = batch_sizes.shape[0]
if total_length is not None:
if total_length < T:
raise ValueError(
"`total length ({})` must be greater than or equal to the maximum length ({})."
.format(total_length, T))
padded_sequence, lengths = F.pad_packed_sequence(packed_sequence,
batch_sizes, batch_first,
padding_value,
total_length)
if unsorted_indices is not None:
axis = 0 if batch_first else 1
padded_sequence = F.gather(padded_sequence, unsorted_indices, axis)
lengths = lengths[unsorted_indices]
return padded_sequence, lengths