def __init__(
self,
context_length: int,
prediction_length: int,
d_input: int,
d_hidden: int,
**kwargs,
) -> None:
super(TemporalFusionEncoder, self).__init__(**kwargs)
self.context_length = context_length
self.prediction_length = prediction_length
with self.name_scope():
self.encoder_lstm = rnn.HybridSequentialRNNCell(prefix="encoder_")
self.encoder_lstm.add(
rnn.LSTMCell(
hidden_size=d_hidden,
input_size=d_input,
))
self.decoder_lstm = rnn.HybridSequentialRNNCell(prefix="decoder_")
self.decoder_lstm.add(
rnn.LSTMCell(
hidden_size=d_hidden,
input_size=d_input,
))
self.gate = nn.HybridSequential()
self.gate.add(nn.Dense(d_hidden * 2, flatten=False))
self.gate.add(GatedLinearUnit(axis=-1, nonlinear=False))
if d_input != d_hidden:
self.skip_proj = nn.Dense(d_hidden, flatten=False)
self.add_skip = True
else:
self.add_skip = False
self.lnorm = nn.LayerNorm(axis=-1)
def _make_layers(self,
mode,
num_layers,
input_size,
hidden_size,
dropout,
var_drop_in,
var_drop_state,
var_drop_out,
skip_connection,
proj_size=None):
layers = rnn.HybridSequentialRNNCell()
for idx in range(num_layers):
if idx == num_layers - 1:
dropout = 0
if idx == 0:
sc = False
else:
sc = skip_connection
layer = _get_rnn_cell(mode, 1, input_size, hidden_size, dropout,
var_drop_in, var_drop_state, var_drop_out,
sc, proj_size)
layers.add(layer)
input_size = hidden_size
return layers
def _get_encoder(self):
block = rnn.HybridSequentialRNNCell()
with block.name_scope():
for _ in range(self._num_layers):
block.add(contrib.rnn.LSTMPCell(self._hidden_size, self._projection_size))
if self._encode_dropout:
block.add(rnn.DropoutCell(self._encode_dropout))
return block
def _get_rnn_cell(mode,
num_layers,
input_size,
hidden_size,
dropout,
weight_dropout,
var_drop_in,
var_drop_state,
var_drop_out,
skip_connection,
proj_size=None,
cell_clip=None,
proj_clip=None):
"""create rnn cell given specs
Parameters
----------
mode : str
The type of RNN cell to use. Options are 'lstmpc', 'rnn_tanh', 'rnn_relu', 'lstm', 'gru'.
num_layers : int
The number of RNN cells in the encoder.
input_size : int
The initial input size of in the RNN cell.
hidden_size : int
The hidden size of the RNN cell.
dropout : float
The dropout rate to use for encoder output.
weight_dropout: float
The dropout rate to the hidden to hidden connections.
var_drop_in: float
The variational dropout rate for inputs. Won’t apply dropout if it equals 0.
var_drop_state: float
The variational dropout rate for state inputs on the first state channel.
Won’t apply dropout if it equals 0.
var_drop_out: float
The variational dropout rate for outputs. Won’t apply dropout if it equals 0.
skip_connection : bool
Whether to add skip connections (add RNN cell input to output)
proj_size : int
The projection size of each LSTMPCellWithClip cell.
Only available when the mode=lstmpc.
cell_clip : float
Clip cell state between [-cellclip, cell_clip] in LSTMPCellWithClip cell.
Only available when the mode=lstmpc.
proj_clip : float
Clip projection between [-projclip, projclip] in LSTMPCellWithClip cell
Only available when the mode=lstmpc.
"""
assert mode == 'lstmpc' or proj_size is None, \
'proj_size takes effect only when mode is lstmpc'
assert mode == 'lstmpc' or cell_clip is None, \
'cell_clip takes effect only when mode is lstmpc'
assert mode == 'lstmpc' or proj_clip is None, \
'proj_clip takes effect only when mode is lstmpc'
rnn_cell = rnn.HybridSequentialRNNCell()
with rnn_cell.name_scope():
for i in range(num_layers):
if mode == 'rnn_relu':
cell = rnn.RNNCell(hidden_size, 'relu', input_size=input_size)
elif mode == 'rnn_tanh':
cell = rnn.RNNCell(hidden_size, 'tanh', input_size=input_size)
elif mode == 'lstm':
cell = rnn.LSTMCell(hidden_size, input_size=input_size)
elif mode == 'gru':
cell = rnn.GRUCell(hidden_size, input_size=input_size)
elif mode == 'lstmpc':
cell = LSTMPCellWithClip(hidden_size,
proj_size,
cell_clip=cell_clip,
projection_clip=proj_clip,
input_size=input_size)
if var_drop_in + var_drop_state + var_drop_out != 0:
cell = contrib.rnn.VariationalDropoutCell(
cell, var_drop_in, var_drop_state, var_drop_out)
if skip_connection:
cell = rnn.ResidualCell(cell)
rnn_cell.add(cell)
if i != num_layers - 1 and dropout != 0:
rnn_cell.add(rnn.DropoutCell(dropout))
if weight_dropout:
apply_weight_drop(rnn_cell, 'h2h_weight', rate=weight_dropout)
return rnn_cell
def _get_rnn_cell(mode,
num_layers,
input_size,
hidden_size,
dropout,
var_drop_in,
var_drop_state,
var_drop_out,
skip_connection,
proj_size=None):
"""create rnn cell given specs
Parameters
----------
mode : str
The type of RNN cell to use. Options are 'rnn_tanh', 'rnn_relu', 'lstm', 'lstmp', 'gru'.
num_layers : int
The number of RNN cells in the encoder.
input_size : int
The initial input size of in the RNN cell.
hidden_size : int
The hidden size of the RNN cell.
dropout : float
The dropout rate to use for encoder output.
var_drop_in: float
The variational dropout rate for inputs. Won’t apply dropout if it equals 0.
var_drop_state: float
The variational dropout rate for state inputs on the first state channel.
Won’t apply dropout if it equals 0.
var_drop_out: float
The variational dropout rate for outputs. Won’t apply dropout if it equals 0.
skip_connection : bool
Whether to add skip connections (add RNN cell input to output)
proj_size : int
The projection size of each LSTMPCell cell.
Only available when the mode=lstmpc.
"""
if mode == 'lstmps':
assert proj_size is not None, \
'proj_size takes effect only when mode is lstmp'
rnn_cell = rnn.HybridSequentialRNNCell()
with rnn_cell.name_scope():
for i in range(num_layers):
if mode == 'rnn_relu':
cell = rnn.RNNCell(hidden_size, 'relu', input_size=input_size)
elif mode == 'rnn_tanh':
cell = rnn.RNNCell(hidden_size, 'tanh', input_size=input_size)
elif mode == 'lstm':
cell = rnn.LSTMCell(hidden_size, input_size=input_size)
elif mode == 'lstmp':
cell = gluon.contrib.rnn.LSTMPCell(hidden_size, input_size,
proj_size)
elif mode == 'gru':
cell = rnn.GRUCell(hidden_size, input_size=input_size)
if var_drop_in + var_drop_state + var_drop_out != 0:
cell = gluon.contrib.rnn.VariationalDropoutCell(
cell, var_drop_in, var_drop_state, var_drop_out)
if skip_connection:
cell = rnn.ResidualCell(cell)
rnn_cell.add(cell)
if i != num_layers - 1 and dropout != 0:
rnn_cell.add(rnn.DropoutCell(dropout))
return rnn_cell
def __init__(self, mode, num_layers, input_size, hidden_size, dropout=0.0,
skip_connection=True, proj_size=None, cell_clip=None, proj_clip=None, **kwargs):
super(BiLMEncoder, self).__init__(**kwargs)
self._mode = mode
self._num_layers = num_layers
self._input_size = input_size
self._hidden_size = hidden_size
self._dropout = dropout
self._skip_connection = skip_connection
self._proj_size = proj_size
self._cell_clip = cell_clip
self._proj_clip = proj_clip
with self.name_scope():
lstm_input_size = self._input_size
self.forward_layers = rnn.HybridSequentialRNNCell()
with self.forward_layers.name_scope():
for layer_index in range(self._num_layers):
forward_layer = _get_rnn_cell(mode=self._mode,
num_layers=1,
input_size=lstm_input_size,
hidden_size=self._hidden_size,
dropout=0
if layer_index == num_layers - 1
else self._dropout,
weight_dropout=0,
var_drop_in=0,
var_drop_state=0,
var_drop_out=0,
skip_connection=False
if layer_index == 0
else self._skip_connection,
proj_size=self._proj_size,
cell_clip=self._cell_clip,
proj_clip=self._proj_clip)
self.forward_layers.add(forward_layer)
lstm_input_size = self._hidden_size \
if self._proj_size is None else self._proj_size
lstm_input_size = self._input_size
self.backward_layers = rnn.HybridSequentialRNNCell()
with self.backward_layers.name_scope():
for layer_index in range(self._num_layers):
backward_layer = _get_rnn_cell(mode=self._mode,
num_layers=1,
input_size=lstm_input_size,
hidden_size=self._hidden_size,
dropout=0
if layer_index == num_layers - 1
else self._dropout,
weight_dropout=0,
var_drop_in=0,
var_drop_state=0,
var_drop_out=0,
skip_connection=False
if layer_index == 0
else self._skip_connection,
proj_size=self._proj_size,
cell_clip=self._cell_clip,
proj_clip=self._proj_clip)
self.backward_layers.add(backward_layer)
lstm_input_size = self._hidden_size \
if self._proj_size is None else self._proj_size
