def _additional_test_with_residuals(self, inputs, **kwargs): """Runs the cell in a session""" inputs = tf.convert_to_tensor(inputs) state = ( tf.constant(np.random.randn(1, 1)), tf.constant(np.random.randn(1, 1))) with tf.variable_scope("root", initializer=tf.constant_initializer(0.5)): test_cell = rnn_cell.ExtendedMultiRNNCell( [tf.contrib.rnn.GRUCell(1) for _ in range(2)], residual_connections=True, **kwargs) res_test = test_cell(inputs, state, scope="test") with self.test_session() as sess: sess.run([tf.global_variables_initializer()]) return sess.run(res_test)
def get_rnn_cell(cell_class, cell_params, num_layers=1, dropout_input_keep_prob=1.0, dropout_output_keep_prob=1.0, residual_connections=False, residual_combiner="add", residual_dense=False): """Creates a new RNN Cell Args: cell_class: Name of the cell class, e.g. "BasicLSTMCell". cell_params: A dictionary of parameters to pass to the cell constructor. num_layers: Number of layers. The cell will be wrapped with `tf.contrib.rnn.MultiRNNCell` dropout_input_keep_prob: Dropout keep probability applied to the input of cell *at each layer* dropout_output_keep_prob: Dropout keep probability applied to the output of cell *at each layer* residual_connections: If true, add residual connections between all cells Returns: An instance of `tf.contrib.rnn.RNNCell`. """ #pylint: disable=redefined-variable-type cells = [] for _ in range(num_layers): cell = cell_from_spec(cell_class, cell_params) if dropout_input_keep_prob < 1.0 or dropout_output_keep_prob < 1.0: cell = tf.contrib.rnn.DropoutWrapper( cell=cell, input_keep_prob=dropout_input_keep_prob, output_keep_prob=dropout_output_keep_prob) cells.append(cell) if len(cells) > 1: final_cell = rnn_cell.ExtendedMultiRNNCell( cells=cells, residual_connections=residual_connections, residual_combiner=residual_combiner, residual_dense=residual_dense) else: final_cell = cells[0] return final_cell
def get_rnn_cell(cell_spec, num_layers=1, dropout_input_keep_prob=1.0, dropout_output_keep_prob=1.0, residual_connections=False, residual_combiner="add", residual_dense=False): """Creates a new RNN Cell. Args: cell_spec: A JSON string that defines how to create a cell instance. See `cell_from_spec` for more details. num_layers: Number of layers. The cell will be wrapped with `tf.contrib.rnn.MultiRNNCell` dropout_input_keep_prob: Dropout keep probability applied to the input of cell *at each layer* dropout_output_keep_prob: Dropout keep probability applied to the output of cell *at each layer* residual_connections: If true, add residual connections between all cells Returns: An instance of `tf.contrib.rnn.RNNCell`. """ #pylint: disable=redefined-variable-type cell = cell_from_spec(cell_spec) if dropout_input_keep_prob < 1.0 or dropout_output_keep_prob < 1.0: cell = tf.contrib.rnn.DropoutWrapper( cell=cell, input_keep_prob=dropout_input_keep_prob, output_keep_prob=dropout_output_keep_prob) if num_layers > 1: cell = rnn_cell.ExtendedMultiRNNCell( cells=[cell] * num_layers, residual_connections=residual_connections, residual_combiner=residual_combiner, residual_dense=residual_dense) return cell
def test_without_residuals(self): inputs = tf.constant(np.random.randn(1, 2)) state = ( tf.constant(np.random.randn(1, 2)), tf.constant(np.random.randn(1, 2))) with tf.variable_scope("root", initializer=tf.constant_initializer(0.5)): standard_cell = tf.contrib.rnn.MultiRNNCell( [tf.contrib.rnn.GRUCell(2) for _ in range(2)], state_is_tuple=True) res_standard = standard_cell(inputs, state, scope="standard") test_cell = rnn_cell.ExtendedMultiRNNCell( [tf.contrib.rnn.GRUCell(2) for _ in range(2)]) res_test = test_cell(inputs, state, scope="test") with self.test_session() as sess: sess.run([tf.global_variables_initializer()]) res_standard_, res_test_, = sess.run([res_standard, res_test]) # Make sure it produces the same results as the standard cell self.assertAllClose(res_standard_[0], res_test_[0]) self.assertAllClose(res_standard_[1][0], res_test_[1][0]) self.assertAllClose(res_standard_[1][1], res_test_[1][1])
def get_rnn_cell( cell_class, cell_params, num_layers=1, dropout_input_keep_prob=1.0, dropout_output_keep_prob=1.0, residual_connections=False, residual_combiner="add", residual_dense=False, distributed=True, device_name='' ): #TODO update it to get gpu number for bidirectional encoder for faster training :LD """Creates a new RNN Cell Args: cell_class: Name of the cell class, e.g. "BasicLSTMCell". cell_params: A dictionary of parameters to pass to the cell constructor. num_layers: Number of layers. The cell will be wrapped with `tf.contrib.rnn.MultiRNNCell` dropout_input_keep_prob: Dropout keep probability applied to the input of cell *at each layer* dropout_output_keep_prob: Dropout keep probability applied to the output of cell *at each layer* residual_connections: If true, add residual connections between all cells Returns: An instance of `tf.contrib.rnn.RNNCell`. """ cells = [] for i in range(num_layers): if "add_residual" in cell_params and cell_params["add_residual"]: cell_params['residual_connections'] = ( i >= cell_params['num_layers'] - cell_params['num_residual_layers']) if distributed: cell = cell_from_spec(cell_class, cell_params, device_name=getDeviceName(i)) else: cell = cell_from_spec(cell_class, cell_params, device_name=device_name) if dropout_input_keep_prob < 1.0 or dropout_output_keep_prob < 1.0: cell = tf.contrib.rnn.DropoutWrapper( cell=cell, input_keep_prob=dropout_input_keep_prob, output_keep_prob=dropout_output_keep_prob) cells.append(cell) if len(cells) > 1: final_cell = rnn_cell.ExtendedMultiRNNCell( cells=cells, residual_connections=residual_connections, residual_combiner=residual_combiner, residual_dense=residual_dense) else: final_cell = cells[0] return final_cell