def __init__( self, context_length: int, prediction_length: int, d_input: int, d_hidden: int, **kwargs, ) -> None: super(TemporalFusionEncoder, self).__init__(**kwargs) self.context_length = context_length self.prediction_length = prediction_length with self.name_scope(): self.encoder_lstm = rnn.HybridSequentialRNNCell(prefix="encoder_") self.encoder_lstm.add( rnn.LSTMCell( hidden_size=d_hidden, input_size=d_input, )) self.decoder_lstm = rnn.HybridSequentialRNNCell(prefix="decoder_") self.decoder_lstm.add( rnn.LSTMCell( hidden_size=d_hidden, input_size=d_input, )) self.gate = nn.HybridSequential() self.gate.add(nn.Dense(d_hidden * 2, flatten=False)) self.gate.add(GatedLinearUnit(axis=-1, nonlinear=False)) if d_input != d_hidden: self.skip_proj = nn.Dense(d_hidden, flatten=False) self.add_skip = True else: self.add_skip = False self.lnorm = nn.LayerNorm(axis=-1)
def _make_layers(self, mode, num_layers, input_size, hidden_size, dropout, var_drop_in, var_drop_state, var_drop_out, skip_connection, proj_size=None): layers = rnn.HybridSequentialRNNCell() for idx in range(num_layers): if idx == num_layers - 1: dropout = 0 if idx == 0: sc = False else: sc = skip_connection layer = _get_rnn_cell(mode, 1, input_size, hidden_size, dropout, var_drop_in, var_drop_state, var_drop_out, sc, proj_size) layers.add(layer) input_size = hidden_size return layers
def _get_encoder(self): block = rnn.HybridSequentialRNNCell() with block.name_scope(): for _ in range(self._num_layers): block.add(contrib.rnn.LSTMPCell(self._hidden_size, self._projection_size)) if self._encode_dropout: block.add(rnn.DropoutCell(self._encode_dropout)) return block
def _get_rnn_cell(mode, num_layers, input_size, hidden_size, dropout, weight_dropout, var_drop_in, var_drop_state, var_drop_out, skip_connection, proj_size=None, cell_clip=None, proj_clip=None): """create rnn cell given specs Parameters ---------- mode : str The type of RNN cell to use. Options are 'lstmpc', 'rnn_tanh', 'rnn_relu', 'lstm', 'gru'. num_layers : int The number of RNN cells in the encoder. input_size : int The initial input size of in the RNN cell. hidden_size : int The hidden size of the RNN cell. dropout : float The dropout rate to use for encoder output. weight_dropout: float The dropout rate to the hidden to hidden connections. var_drop_in: float The variational dropout rate for inputs. Won’t apply dropout if it equals 0. var_drop_state: float The variational dropout rate for state inputs on the first state channel. Won’t apply dropout if it equals 0. var_drop_out: float The variational dropout rate for outputs. Won’t apply dropout if it equals 0. skip_connection : bool Whether to add skip connections (add RNN cell input to output) proj_size : int The projection size of each LSTMPCellWithClip cell. Only available when the mode=lstmpc. cell_clip : float Clip cell state between [-cellclip, cell_clip] in LSTMPCellWithClip cell. Only available when the mode=lstmpc. proj_clip : float Clip projection between [-projclip, projclip] in LSTMPCellWithClip cell Only available when the mode=lstmpc. """ assert mode == 'lstmpc' or proj_size is None, \ 'proj_size takes effect only when mode is lstmpc' assert mode == 'lstmpc' or cell_clip is None, \ 'cell_clip takes effect only when mode is lstmpc' assert mode == 'lstmpc' or proj_clip is None, \ 'proj_clip takes effect only when mode is lstmpc' rnn_cell = rnn.HybridSequentialRNNCell() with rnn_cell.name_scope(): for i in range(num_layers): if mode == 'rnn_relu': cell = rnn.RNNCell(hidden_size, 'relu', input_size=input_size) elif mode == 'rnn_tanh': cell = rnn.RNNCell(hidden_size, 'tanh', input_size=input_size) elif mode == 'lstm': cell = rnn.LSTMCell(hidden_size, input_size=input_size) elif mode == 'gru': cell = rnn.GRUCell(hidden_size, input_size=input_size) elif mode == 'lstmpc': cell = LSTMPCellWithClip(hidden_size, proj_size, cell_clip=cell_clip, projection_clip=proj_clip, input_size=input_size) if var_drop_in + var_drop_state + var_drop_out != 0: cell = contrib.rnn.VariationalDropoutCell( cell, var_drop_in, var_drop_state, var_drop_out) if skip_connection: cell = rnn.ResidualCell(cell) rnn_cell.add(cell) if i != num_layers - 1 and dropout != 0: rnn_cell.add(rnn.DropoutCell(dropout)) if weight_dropout: apply_weight_drop(rnn_cell, 'h2h_weight', rate=weight_dropout) return rnn_cell
def _get_rnn_cell(mode, num_layers, input_size, hidden_size, dropout, var_drop_in, var_drop_state, var_drop_out, skip_connection, proj_size=None): """create rnn cell given specs Parameters ---------- mode : str The type of RNN cell to use. Options are 'rnn_tanh', 'rnn_relu', 'lstm', 'lstmp', 'gru'. num_layers : int The number of RNN cells in the encoder. input_size : int The initial input size of in the RNN cell. hidden_size : int The hidden size of the RNN cell. dropout : float The dropout rate to use for encoder output. var_drop_in: float The variational dropout rate for inputs. Won’t apply dropout if it equals 0. var_drop_state: float The variational dropout rate for state inputs on the first state channel. Won’t apply dropout if it equals 0. var_drop_out: float The variational dropout rate for outputs. Won’t apply dropout if it equals 0. skip_connection : bool Whether to add skip connections (add RNN cell input to output) proj_size : int The projection size of each LSTMPCell cell. Only available when the mode=lstmpc. """ if mode == 'lstmps': assert proj_size is not None, \ 'proj_size takes effect only when mode is lstmp' rnn_cell = rnn.HybridSequentialRNNCell() with rnn_cell.name_scope(): for i in range(num_layers): if mode == 'rnn_relu': cell = rnn.RNNCell(hidden_size, 'relu', input_size=input_size) elif mode == 'rnn_tanh': cell = rnn.RNNCell(hidden_size, 'tanh', input_size=input_size) elif mode == 'lstm': cell = rnn.LSTMCell(hidden_size, input_size=input_size) elif mode == 'lstmp': cell = gluon.contrib.rnn.LSTMPCell(hidden_size, input_size, proj_size) elif mode == 'gru': cell = rnn.GRUCell(hidden_size, input_size=input_size) if var_drop_in + var_drop_state + var_drop_out != 0: cell = gluon.contrib.rnn.VariationalDropoutCell( cell, var_drop_in, var_drop_state, var_drop_out) if skip_connection: cell = rnn.ResidualCell(cell) rnn_cell.add(cell) if i != num_layers - 1 and dropout != 0: rnn_cell.add(rnn.DropoutCell(dropout)) return rnn_cell
def __init__(self, mode, num_layers, input_size, hidden_size, dropout=0.0, skip_connection=True, proj_size=None, cell_clip=None, proj_clip=None, **kwargs): super(BiLMEncoder, self).__init__(**kwargs) self._mode = mode self._num_layers = num_layers self._input_size = input_size self._hidden_size = hidden_size self._dropout = dropout self._skip_connection = skip_connection self._proj_size = proj_size self._cell_clip = cell_clip self._proj_clip = proj_clip with self.name_scope(): lstm_input_size = self._input_size self.forward_layers = rnn.HybridSequentialRNNCell() with self.forward_layers.name_scope(): for layer_index in range(self._num_layers): forward_layer = _get_rnn_cell(mode=self._mode, num_layers=1, input_size=lstm_input_size, hidden_size=self._hidden_size, dropout=0 if layer_index == num_layers - 1 else self._dropout, weight_dropout=0, var_drop_in=0, var_drop_state=0, var_drop_out=0, skip_connection=False if layer_index == 0 else self._skip_connection, proj_size=self._proj_size, cell_clip=self._cell_clip, proj_clip=self._proj_clip) self.forward_layers.add(forward_layer) lstm_input_size = self._hidden_size \ if self._proj_size is None else self._proj_size lstm_input_size = self._input_size self.backward_layers = rnn.HybridSequentialRNNCell() with self.backward_layers.name_scope(): for layer_index in range(self._num_layers): backward_layer = _get_rnn_cell(mode=self._mode, num_layers=1, input_size=lstm_input_size, hidden_size=self._hidden_size, dropout=0 if layer_index == num_layers - 1 else self._dropout, weight_dropout=0, var_drop_in=0, var_drop_state=0, var_drop_out=0, skip_connection=False if layer_index == 0 else self._skip_connection, proj_size=self._proj_size, cell_clip=self._cell_clip, proj_clip=self._proj_clip) self.backward_layers.add(backward_layer) lstm_input_size = self._hidden_size \ if self._proj_size is None else self._proj_size