def __init__(self,
num_layers,
num_units,
reduction_factor=2,
cell_class=tf.contrib.rnn.LSTMCell,
dropout=0.3):
"""Initializes the parameters of the encoder.
Args:
num_layers: The number of layers.
num_units: The number of units in each layer.
reduction_factor: The time reduction factor.
cell_class: The inner cell class or a callable taking :obj:`num_units` as
argument and returning a cell.
dropout: The probability to drop units in each layer output.
"""
self.reduction_factor = reduction_factor
self.state_reducer = JoinReducer()
self.layers = []
for _ in range(num_layers):
self.layers.append(
BidirectionalRNNEncoder(1,
num_units,
reducer=ConcatReducer(),
cell_class=cell_class,
dropout=dropout))
def __init__(self,
encoders,
outputs_reducer=ConcatReducer(axis=1),
states_reducer=JoinReducer(),
outputs_layer_fn=None,
combined_output_layer_fn=None,
share_parameters=False):
"""Initializes the parameters of the encoder.
Args:
encoders: A list of :class:`opennmt.encoders.encoder.Encoder` or a single
one, in which case the same encoder is applied to each input.
outputs_reducer: A :class:`opennmt.layers.reducer.Reducer` to merge all
outputs. If ``None``, defaults to
:class:`opennmt.layers.reducer.JoinReducer`.
states_reducer: A :class:`opennmt.layers.reducer.Reducer` to merge all
states. If ``None``, defaults to
:class:`opennmt.layers.reducer.JoinReducer`.
outputs_layer_fn: A callable or list of callables applied to the
encoders outputs If it is a single callable, it is on each encoder
output. Otherwise, the ``i`` th callable is applied on encoder ``i``
output.
combined_output_layer_fn: A callable to apply on the combined output
(i.e. the output of :obj:`outputs_reducer`).
share_parameters: If ``True``, share parameters between the parallel
encoders. For stateful encoders, simply pass a single encoder instance
to :obj:`encoders` for parameter sharing.
Raises:
ValueError: if :obj:`outputs_layer_fn` is a list with a size not equal
to the number of encoders.
"""
if (isinstance(encoders, list) and outputs_layer_fn is not None
and isinstance(outputs_layer_fn, list)
and len(outputs_layer_fn) != len(encoders)):
raise ValueError(
"The number of output layers must match the number of encoders; "
"expected %d layers but got %d." %
(len(encoders), len(outputs_layer_fn)))
super(ParallelEncoder, self).__init__()
self.encoders = encoders
self.outputs_reducer = outputs_reducer if outputs_reducer is not None else JoinReducer(
)
self.states_reducer = states_reducer if states_reducer is not None else JoinReducer(
)
self.outputs_layer_fn = outputs_layer_fn
self.combined_output_layer_fn = combined_output_layer_fn
self.share_parameters = share_parameters
def __init__(self,
encoders,
states_reducer=JoinReducer(),
transition_layer_fn=None):
"""Initializes the parameters of the encoder.
Args:
encoders: A list of :class:`opennmt.encoders.Encoder`.
states_reducer: A :class:`opennmt.layers.Reducer` to merge all
states.
transition_layer_fn: A callable or list of callables applied to the
output of an encoder before passing it as input to the next. If it is a
single callable, it is applied between every encoders. Otherwise, the
``i`` th callable will be applied between encoders ``i`` and ``i + 1``.
Raises:
ValueError: if :obj:`transition_layer_fn` is a list with a size not equal
to the number of encoder transitions ``len(encoders) - 1``.
"""
if (transition_layer_fn is not None
and isinstance(transition_layer_fn, list)
and len(transition_layer_fn) != len(encoders) - 1):
raise ValueError(
"The number of transition layers must match the number of encoder "
"transitions, expected %d layers but got %d." %
(len(encoders) - 1, len(transition_layer_fn)))
super(SequentialEncoder, self).__init__()
self.encoders = encoders
self.states_reducer = states_reducer
self.transition_layer_fn = transition_layer_fn
def __init__(self,
num_layers,
num_units,
reduction_factor=2,
cell_class=tf.contrib.rnn.LSTMCell,
dropout=0.3):
"""Initializes the parameters of the encoder.
Args:
num_layers: The number of layers.
num_units: The number of units in each layer.
reduction_factor: The time reduction factor.
cell_class: The inner cell class or a callable taking :obj:`num_units` as
argument and returning a cell.
dropout: The probability to drop units in each layer output.
"""
self.reduction_factor = reduction_factor
self.state_reducer = JoinReducer()
self.layers = []
for _ in range(num_layers):
self.layers.append(BidirectionalRNNEncoder(
1,
num_units,
reducer=ConcatReducer(),
cell_class=cell_class,
dropout=dropout))
def encode(self,
inputs,
sequence_length=None,
mode=tf.estimator.ModeKeys.TRAIN):
inputs = tf.layers.dropout(
inputs,
rate=self._dropout,
training=mode == tf.estimator.ModeKeys.TRAIN)
states = []
for i, layer in enumerate(self._layers):
with tf.variable_scope("layer_%d" % i):
outputs, state, sequence_length = layer.encode(
inputs, sequence_length=sequence_length, mode=mode)
outputs = tf.layers.dropout(
outputs,
rate=self._dropout,
training=mode == tf.estimator.ModeKeys.TRAIN)
inputs = outputs + inputs if i >= 2 else outputs
states.append(state)
with tf.variable_scope("projection"):
projected = tf.layers.dense(inputs, self._num_units)
state = JoinReducer()(states)
return (projected, state, sequence_length)
def __init__(self,
num_layers,
num_units,
reduction_factor=2,
cell_class=None,
dropout=0.3):
"""Initializes the parameters of the encoder.
Args:
num_layers: The number of layers.
num_units: The number of units in each layer.
reduction_factor: The time reduction factor.
cell_class: The inner cell class or a callable taking :obj:`num_units` as
argument and returning a cell. Defaults to a LSTM cell.
dropout: The probability to drop units in each layer output.
"""
super().__init__()
self.reduction_factor = reduction_factor
self.state_reducer = JoinReducer()
self.layers = [
RNNEncoder(
1,
num_units // 2,
bidirectional=True,
reducer=ConcatReducer(),
cell_class=cell_class,
dropout=dropout,
) for _ in range(num_layers)
]
def __init__(self, encoders, states_reducer=JoinReducer()):
"""Initializes the parameters of the encoder.
Args:
encoders: A list of :class:`opennmt.encoders.encoder.Encoder`.
states_reducer: A :class:`opennmt.layers.reducer.Reducer` to merge all
states.
"""
self.encoders = encoders
self.states_reducer = states_reducer
def encode(self,
inputs,
sequence_length=None,
mode=tf.estimator.ModeKeys.TRAIN):
encoder_outputs, bidirectional_state, sequence_length = self.bidirectional.encode(
inputs, sequence_length=sequence_length, mode=mode)
encoder_outputs, unidirectional_state, sequence_length = self.unidirectional.encode(
encoder_outputs, sequence_length=sequence_length, mode=mode)
encoder_state = JoinReducer()(
[bidirectional_state, unidirectional_state])
return (encoder_outputs, encoder_state, sequence_length)
def __init__(self,
encoders,
outputs_reducer=ConcatReducer(axis=1),
states_reducer=JoinReducer(),
outputs_layer_fn=None,
combined_output_layer_fn=None):
"""Initializes the parameters of the encoder.
Args:
encoders: A list of :class:`opennmt.encoders.encoder.Encoder`.
outputs_reducer: A :class:`opennmt.layers.reducer.Reducer` to merge all
outputs.
states_reducer: A :class:`opennmt.layers.reducer.Reducer` to merge all
states.
outputs_layer_fn: A callable or list of callables applied to the
encoders outputs If it is a single callable, it is on each encoder
output. Otherwise, the ``i`` th callable is applied on encoder ``i``
output.
combined_output_layer_fn: A callable to apply on the combined output
(i.e. the output of :obj:`outputs_reducer`).
Raises:
ValueError: if :obj:`outputs_layer_fn` is a list with a size not equal
to the number of encoders.
"""
if (outputs_layer_fn is not None
and isinstance(outputs_layer_fn, list)
and len(outputs_layer_fn) != len(encoders)):
raise ValueError(
"The number of output layers must match the number of encoders; "
"expected %d layers but got %d." %
(len(encoders), len(outputs_layer_fn)))
self.encoders = encoders
self.outputs_reducer = outputs_reducer
self.states_reducer = states_reducer
self.outputs_layer_fn = outputs_layer_fn
self.combined_output_layer_fn = combined_output_layer_fn
class PyramidalRNNEncoder(Encoder):
"""An encoder that reduces the time dimension after each bidirectional layer."""
def __init__(self,
num_layers,
num_units,
reduction_factor=2,
cell_class=tf.contrib.rnn.LSTMCell,
dropout=0.3):
"""Initializes the parameters of the encoder.
Args:
num_layers: The number of layers.
num_units: The number of units in each layer.
reduction_factor: The time reduction factor.
cell_class: The inner cell class or a callable taking :obj:`num_units` as
argument and returning a cell.
dropout: The probability to drop units in each layer output.
"""
self.reduction_factor = reduction_factor
self.state_reducer = JoinReducer()
self.layers = []
for _ in range(num_layers):
self.layers.append(BidirectionalRNNEncoder(
1,
num_units,
reducer=ConcatReducer(),
cell_class=cell_class,
dropout=dropout))
def encode(self, inputs, sequence_length=None, mode=tf.estimator.ModeKeys.TRAIN):
encoder_state = []
for layer_index, layer in enumerate(self.layers):
input_depth = inputs.get_shape().as_list()[-1]
if layer_index == 0:
# For the first input, make the number of timesteps a multiple of the
# total reduction factor.
total_reduction_factor = pow(self.reduction_factor, len(self.layers) - 1)
current_length = tf.shape(inputs)[1]
factor = tf.divide(tf.cast(current_length, tf.float32), total_reduction_factor)
new_length = tf.cast(tf.ceil(factor), tf.int32) * total_reduction_factor
padding = new_length - current_length
inputs = tf.pad(
inputs,
[[0, 0], [0, padding], [0, 0]])
inputs.set_shape((None, None, input_depth))
else:
# In other cases, reduce the time dimension.
inputs = tf.reshape(
inputs,
[tf.shape(inputs)[0], -1, input_depth * self.reduction_factor])
if sequence_length is not None:
sequence_length = tf.div(sequence_length, self.reduction_factor)
with tf.variable_scope("layer_{}".format(layer_index)):
outputs, state, sequence_length = layer.encode(
inputs,
sequence_length=sequence_length,
mode=mode)
encoder_state.append(state)
inputs = outputs
return (
outputs,
self.state_reducer.reduce(encoder_state),
sequence_length)
class PyramidalRNNEncoder(Encoder):
"""An encoder that reduces the time dimension after each bidirectional layer."""
def __init__(self,
num_layers,
num_units,
reduction_factor=2,
cell_class=tf.contrib.rnn.LSTMCell,
dropout=0.3):
"""Initializes the parameters of the encoder.
Args:
num_layers: The number of layers.
num_units: The number of units in each layer.
reduction_factor: The time reduction factor.
cell_class: The inner cell class or a callable taking :obj:`num_units` as
argument and returning a cell.
dropout: The probability to drop units in each layer output.
"""
self.reduction_factor = reduction_factor
self.state_reducer = JoinReducer()
self.layers = []
for _ in range(num_layers):
self.layers.append(
BidirectionalRNNEncoder(1,
num_units,
reducer=ConcatReducer(),
cell_class=cell_class,
dropout=dropout))
def encode(self,
inputs,
sequence_length=None,
mode=tf.estimator.ModeKeys.TRAIN):
encoder_state = []
for layer_index, layer in enumerate(self.layers):
input_depth = inputs.get_shape().as_list()[-1]
if layer_index == 0:
# For the first input, make the number of timesteps a multiple of the
# total reduction factor.
total_reduction_factor = pow(self.reduction_factor,
len(self.layers) - 1)
current_length = tf.shape(inputs)[1]
factor = tf.divide(tf.cast(current_length, tf.float32),
total_reduction_factor)
new_length = tf.cast(tf.ceil(factor),
tf.int32) * total_reduction_factor
inputs = pad_in_time(inputs, new_length - current_length)
# Lengths should not be smaller than the total reduction factor.
sequence_length = tf.maximum(sequence_length,
total_reduction_factor)
else:
# In other cases, reduce the time dimension.
inputs = tf.reshape(inputs, [
tf.shape(inputs)[0], -1,
input_depth * self.reduction_factor
])
if sequence_length is not None:
sequence_length = tf.div(sequence_length,
self.reduction_factor)
with tf.variable_scope("layer_{}".format(layer_index)):
outputs, state, sequence_length = layer.encode(
inputs, sequence_length=sequence_length, mode=mode)
encoder_state.append(state)
inputs = outputs
return (outputs, self.state_reducer.reduce(encoder_state),
sequence_length)
