class LSTMLayer(MakiLayer):
TYPE = 'LSTMLayer'
NUM_CELLS = 'num_cells'
INPUT_DIM = 'input_dim'
SEQ_LENGTH = 'seq_length'
DYNAMIC = 'dynamic'
BIDIRECTIONAL = 'bidirectional'
ACTIVATION = 'activation'
OUTPUT_HIDDEN_STATE = 'HIDDEN_STATE'
OUTPUT_LAST_CANDIDATE = 'LAST_CANDIDATE'
OUTPUT_LAST_HIDDEN_STATE = 'LAST_HIDDEN_STATE'
def __init__(self, in_d, out_d, name, activation=tf.nn.tanh, dynamic=True):
"""
Parameters
----------
in_d : int
Number of neurons in the layer.
out_d : int
Dimensionality of the input vectors, e.t. number of features. Dimensionality:
[batch_size, seq_length, num_features(this is input_dim in this case)].
seq_length : int
Max length of the input sequences.
activation : tensorflow function
Activation function of the layer.
dynamic : boolean
Influences whether the layer will be working as dynamic RNN or static. The difference
between static and dynamic is that in case of static TensorFlow builds static graph and the RNN
will always go through each time step in the sequence. In case of dynamic TensorFlow will be
creating RNN `in a while loop`, that is to say that using dynamic RNN you can pass sequences of
variable length, but you have to provide list of sequences' lengthes. Currently API for using
dynamic RNNs is not provided.
WARNING! THIS PARAMETER DOESN'T PLAY ANY ROLE IF YOU'RE GONNA STACK RNN LAYERS.
"""
self._num_cells = in_d
self._input_dim = in_d
self._f = activation
self._cell = LSTMCell(num_units=out_d, activation=activation, dtype=tf.float32)
self._cell.build(input_shape=[out_d])
self._dynamic = dynamic
params = self._cell.variables
param_common_name = name + f'_{in_d}_{out_d}'
named_params_dict = {(param_common_name + '_' + str(i)): param for i, param in enumerate(params)}
super().__init__(
name=name,
params=params,
regularize_params=params,
named_params_dict=named_params_dict,
outputs_names=[
LSTMLayer.OUTPUT_HIDDEN_STATE,
LSTMLayer.OUTPUT_LAST_CANDIDATE,
LSTMLayer.OUTPUT_LAST_HIDDEN_STATE
]
)
def forward(self, x, computation_mode=MakiRestorable.INFERENCE_MODE):
if self._dynamic:
dynamic_x = dynamic_rnn(self._cell, x, dtype=tf.float32)
# hidden states, (last candidate value, last hidden state)
hs, (c_last, h_last) = dynamic_x
return hs, c_last, h_last
else:
unstack_x = tf.unstack(x, axis=1)
static_x = static_rnn(self._cell, unstack_x, dtype=tf.float32)
hs_list, (c_last, h_last) = static_x
hs = tf.stack(hs_list, axis=1)
return hs, c_last, h_last
def training_forward(self, x):
return self.forward(x)
@staticmethod
def build(params: dict):
num_cells = params[LSTMLayer.NUM_CELLS]
input_dim = params[LSTMLayer.INPUT_DIM]
seq_length = params[LSTMLayer.SEQ_LENGTH]
name = params[MakiRestorable.NAME]
dynamic = params[LSTMLayer.DYNAMIC]
bidirectional = params[LSTMLayer.BIDIRECTIONAL]
activation = ActivationConverter.str_to_activation(params[LSTMLayer.ACTIVATION])
return LSTMLayer(
in_d=num_cells,
out_d=input_dim,
seq_length=seq_length,
name=name,
activation=activation,
dynamic=dynamic,
bidirectional=bidirectional
)
def to_dict(self):
return {
MakiRestorable.FIELD_TYPE: LSTMLayer.TYPE,
MakiRestorable.PARAMS: {
MakiRestorable.NAME: self._name,
LSTMLayer.NUM_CELLS: self._num_cells,
LSTMLayer.INPUT_DIM: self._input_dim,
LSTMLayer.SEQ_LENGTH: self._seq_length,
LSTMLayer.DYNAMIC: self._dynamic,
LSTMLayer.BIDIRECTIONAL: self._bidirectional,
LSTMLayer.ACTIVATION: ActivationConverter.activation_to_str(self._f)
}
}
class LSTMLayer(MakiLayer):
def __init__(self,
num_cells,
input_dim,
seq_length,
name,
activation=tf.nn.tanh,
dynamic=False,
bidirectional=False):
"""
Parameters
----------
num_cells : int
Number of neurons in the layer.
input_dim : int
Dimensionality of the input vectors, e.t. number of features. Dimensionality
example: [batch_size, seq_length, num_features(this is input_dim in this case)].
seq_length : int
Max length of the input sequences.
activation : tensorflow function
Activation function of the layer.
dynamic : boolean
Influences whether the layer will be working as dynamic RNN or static. The difference
between static and dynamic is that in case of static TensorFlow builds static graph and the RNN
will always go through each time step in the sequence. In case of dynamic TensorFlow will be
creating RNN `in a while loop`, that is to say that using dynamic RNN you can pass sequences of
variable length, but you have to provide list of sequences' lengthes. Currently API for using
dynamic RNNs is not provided.
WARNING! THIS PARAMETER DOESN'T PLAY ANY ROLE IF YOU'RE GONNA STACK RNN LAYERS.
bidirectional : boolean
Influences whether the layer will be bidirectional.
WARNING! THIS PARAMETER DOESN'T PLAY ANY ROLE IF YOU'RE GONNA STACK RNN LAYERS.
"""
self.name = str(name)
self.num_cells = num_cells
self.input_dim = input_dim
self.seq_length = seq_length
self.dynamic = dynamic
self.bidirectional = bidirectional
self.f = activation
self.cells = LSTMCell(num_units=num_cells,
activation=activation,
dtype=tf.float32)
# Responsible for being RNN whether bidirectional or vanilla
self.cell_type = CellType.get_cell_type(bidirectional, dynamic)
self.cells.build(inputs_shape=[None, tf.Dimension(self.input_dim)])
self.params = self.cells.variables
self.param_common_name = self.name + f'_{num_cells}_{input_dim}_{seq_length}'
self.named_params_dict = {(self.param_common_name + '_' + str(i)):
param
for i, param in enumerate(self.params)}
def forward(self, X, is_training=False):
if self.cell_type == CellType.Bidir_Dynamic:
return bidirectional_dynamic_rnn(cell_fw=self.cells,
cell_bw=self.cells,
inputs=X,
dtype=tf.float32)
elif self.cell_type == CellType.Bidir_Static:
X = tf.unstack(X, num=self.seq_length, axis=1)
return static_bidirectional_rnn(cell_fw=self.cells,
cell_bw=self.cells,
inputs=X,
dtype=tf.float32)
elif self.cell_type == CellType.Dynamic:
return dynamic_rnn(self.cells, X, dtype=tf.float32)
elif self.cell_type == CellType.Static:
X = tf.unstack(X, num=self.seq_length, axis=1)
return static_rnn(self.cells, X, dtype=tf.float32)
def get_params(self):
return self.params
def get_params_dict(self):
return self.named_params_dict
def to_dict(self):
return {
'type': 'LSTMLayer',
'params': {
'num_cells': self.num_cells,
'input_dim': self.input_dim,
'seq_length': self.seq_length,
'name': self.name,
'dynamic': self.dynamic,
'bidirectional': self.bidirectional,
'activation': ActivationConverter.activation_to_str(self.f)
}
}
