def _additional_test_with_residuals(self, inputs, **kwargs):
"""Runs the cell in a session"""
inputs = tf.convert_to_tensor(inputs)
state = (
tf.constant(np.random.randn(1, 1)),
tf.constant(np.random.randn(1, 1)))
with tf.variable_scope("root", initializer=tf.constant_initializer(0.5)):
test_cell = rnn_cell.ExtendedMultiRNNCell(
[tf.contrib.rnn.GRUCell(1) for _ in range(2)], residual_connections=True, **kwargs)
res_test = test_cell(inputs, state, scope="test")
with self.test_session() as sess:
sess.run([tf.global_variables_initializer()])
return sess.run(res_test)
def get_rnn_cell(cell_class,
cell_params,
num_layers=1,
dropout_input_keep_prob=1.0,
dropout_output_keep_prob=1.0,
residual_connections=False,
residual_combiner="add",
residual_dense=False):
"""Creates a new RNN Cell
Args:
cell_class: Name of the cell class, e.g. "BasicLSTMCell".
cell_params: A dictionary of parameters to pass to the cell constructor.
num_layers: Number of layers. The cell will be wrapped with
`tf.contrib.rnn.MultiRNNCell`
dropout_input_keep_prob: Dropout keep probability applied
to the input of cell *at each layer*
dropout_output_keep_prob: Dropout keep probability applied
to the output of cell *at each layer*
residual_connections: If true, add residual connections
between all cells
Returns:
An instance of `tf.contrib.rnn.RNNCell`.
"""
#pylint: disable=redefined-variable-type
cells = []
for _ in range(num_layers):
cell = cell_from_spec(cell_class, cell_params)
if dropout_input_keep_prob < 1.0 or dropout_output_keep_prob < 1.0:
cell = tf.contrib.rnn.DropoutWrapper(
cell=cell,
input_keep_prob=dropout_input_keep_prob,
output_keep_prob=dropout_output_keep_prob)
cells.append(cell)
if len(cells) > 1:
final_cell = rnn_cell.ExtendedMultiRNNCell(
cells=cells,
residual_connections=residual_connections,
residual_combiner=residual_combiner,
residual_dense=residual_dense)
else:
final_cell = cells[0]
return final_cell
def get_rnn_cell(cell_spec,
num_layers=1,
dropout_input_keep_prob=1.0,
dropout_output_keep_prob=1.0,
residual_connections=False,
residual_combiner="add",
residual_dense=False):
"""Creates a new RNN Cell.
Args:
cell_spec: A JSON string that defines how to create a cell instance.
See `cell_from_spec` for more details.
num_layers: Number of layers. The cell will be wrapped with
`tf.contrib.rnn.MultiRNNCell`
dropout_input_keep_prob: Dropout keep probability applied
to the input of cell *at each layer*
dropout_output_keep_prob: Dropout keep probability applied
to the output of cell *at each layer*
residual_connections: If true, add residual connections
between all cells
Returns:
An instance of `tf.contrib.rnn.RNNCell`.
"""
#pylint: disable=redefined-variable-type
cell = cell_from_spec(cell_spec)
if dropout_input_keep_prob < 1.0 or dropout_output_keep_prob < 1.0:
cell = tf.contrib.rnn.DropoutWrapper(
cell=cell,
input_keep_prob=dropout_input_keep_prob,
output_keep_prob=dropout_output_keep_prob)
if num_layers > 1:
cell = rnn_cell.ExtendedMultiRNNCell(
cells=[cell] * num_layers,
residual_connections=residual_connections,
residual_combiner=residual_combiner,
residual_dense=residual_dense)
return cell
def test_without_residuals(self):
inputs = tf.constant(np.random.randn(1, 2))
state = (
tf.constant(np.random.randn(1, 2)),
tf.constant(np.random.randn(1, 2)))
with tf.variable_scope("root", initializer=tf.constant_initializer(0.5)):
standard_cell = tf.contrib.rnn.MultiRNNCell(
[tf.contrib.rnn.GRUCell(2) for _ in range(2)], state_is_tuple=True)
res_standard = standard_cell(inputs, state, scope="standard")
test_cell = rnn_cell.ExtendedMultiRNNCell(
[tf.contrib.rnn.GRUCell(2) for _ in range(2)])
res_test = test_cell(inputs, state, scope="test")
with self.test_session() as sess:
sess.run([tf.global_variables_initializer()])
res_standard_, res_test_, = sess.run([res_standard, res_test])
# Make sure it produces the same results as the standard cell
self.assertAllClose(res_standard_[0], res_test_[0])
self.assertAllClose(res_standard_[1][0], res_test_[1][0])
self.assertAllClose(res_standard_[1][1], res_test_[1][1])
def get_rnn_cell(
cell_class,
cell_params,
num_layers=1,
dropout_input_keep_prob=1.0,
dropout_output_keep_prob=1.0,
residual_connections=False,
residual_combiner="add",
residual_dense=False,
distributed=True,
device_name=''
): #TODO update it to get gpu number for bidirectional encoder for faster training :LD
"""Creates a new RNN Cell
Args:
cell_class: Name of the cell class, e.g. "BasicLSTMCell".
cell_params: A dictionary of parameters to pass to the cell constructor.
num_layers: Number of layers. The cell will be wrapped with
`tf.contrib.rnn.MultiRNNCell`
dropout_input_keep_prob: Dropout keep probability applied
to the input of cell *at each layer*
dropout_output_keep_prob: Dropout keep probability applied
to the output of cell *at each layer*
residual_connections: If true, add residual connections
between all cells
Returns:
An instance of `tf.contrib.rnn.RNNCell`.
"""
cells = []
for i in range(num_layers):
if "add_residual" in cell_params and cell_params["add_residual"]:
cell_params['residual_connections'] = (
i >=
cell_params['num_layers'] - cell_params['num_residual_layers'])
if distributed:
cell = cell_from_spec(cell_class,
cell_params,
device_name=getDeviceName(i))
else:
cell = cell_from_spec(cell_class,
cell_params,
device_name=device_name)
if dropout_input_keep_prob < 1.0 or dropout_output_keep_prob < 1.0:
cell = tf.contrib.rnn.DropoutWrapper(
cell=cell,
input_keep_prob=dropout_input_keep_prob,
output_keep_prob=dropout_output_keep_prob)
cells.append(cell)
if len(cells) > 1:
final_cell = rnn_cell.ExtendedMultiRNNCell(
cells=cells,
residual_connections=residual_connections,
residual_combiner=residual_combiner,
residual_dense=residual_dense)
else:
final_cell = cells[0]
return final_cell