def _single_cell(unit_type, num_units, forget_bias, dropout, mode, residual_connection=False, device_str=None, residual_fn=None): """ 创建一个RNN单元。 :param unit_type: RNN类型 :param num_units: 隐层神经元个数 :param forget_bias: 遗忘门偏置 :param dropout: dropout比例 :param mode: 训练模式(只有train模式下才设置dropout) :param residual_connection: 是否使用残差连接 :param device_str: 设备 :param residual_fn: 残差方法 :return: """ # dropout (= 1 - keep_prob) is set to 0 during eval and infer dropout = dropout if mode == tf.contrib.learn.ModeKeys.TRAIN else 0.0 # Cell Type if unit_type == "lstm": print(" LSTM, forget_bias=%g" % forget_bias, end='') single_cell = rnn.BasicLSTMCell(num_units, forget_bias=forget_bias) elif unit_type == "gru": print(" GRU", end='') single_cell = rnn.GRUCell(num_units) elif unit_type == "layer_norm_lstm": print(" Layer Normalized LSTM, forget_bias=%g" % forget_bias, end='') single_cell = rnn.LayerNormBasicLSTMCell(num_units, forget_bias=forget_bias, layer_norm=True) elif unit_type == "nas": print(" NASCell", end='') single_cell = rnn.NASCell(num_units) else: raise ValueError("Unknown unit type %s!" % unit_type) # Dropout (= 1 - keep_prob) if dropout > 0.0: single_cell = rnn.DropoutWrapper(cell=single_cell, input_keep_prob=(1.0 - dropout)) print(" %s, dropout=%g " % (type(single_cell).__name__, dropout), end='') # Residual if residual_connection: single_cell = rnn.ResidualWrapper(single_cell, residual_fn=residual_fn) print(" %s" % type(single_cell).__name__, end='') # Device Wrapper if device_str: single_cell = rnn.DeviceWrapper(single_cell, device_str) print(" %s, device=%s" % (type(single_cell).__name__, device_str), end='') return single_cell
def make_rnn_cell(rnn_layer_sizes, dropout_keep_prob=1.0, attn_length=0, base_cell=contrib_rnn.BasicLSTMCell, residual_connections=False): cells = [] for i in range(len(rnn_layer_sizes)): cell = base_cell(rnn_layer_sizes[i]) if attn_length and not cells: # Add attention wrapper to first layer. cell = contrib_rnn.AttentionCellWrapper(cell, attn_length, state_is_tuple=True) if residual_connections: cell = contrib_rnn.ResidualWrapper(cell) if i == 0 or rnn_layer_sizes[i] != rnn_layer_sizes[i - 1]: cell = contrib_rnn.InputProjectionWrapper( cell, rnn_layer_sizes[i]) cell = contrib_rnn.DropoutWrapper(cell, output_keep_prob=dropout_keep_prob) cells.append(cell) cell = contrib_rnn.MultiRNNCell(cells) return cell
def create_rnn_layer(layer_num, dim): if layer_num == 0: return tf_rnn.LSTMCell(dim, name='layer_%s' % layer_num) cell = tf_rnn.LSTMCell(dim, name='layer_%s' % layer_num) cell = tf_rnn.ResidualWrapper(cell) return cell
def inputs_encoder(inputs, embeddings, sequence_length, size=None, activation=tf.nn.tanh, num_residual_layers=0, dropout=0.0): assert (size is not None) and isinstance(size, int) assert isinstance(num_residual_layers, int) and (num_residual_layers >= 0) with tf.variable_scope('inputs_encoder'): encoder_inputs = tf.concat([inputs, embeddings], -1) stacked_cell = rnn.MultiRNNCell([ rnn.DropoutWrapper(rnn.GRUCell(size, activation=activation), output_keep_prob=(1. - dropout)) ] + [ rnn.DropoutWrapper(rnn.ResidualWrapper( rnn.GRUCell(size, activation=activation)), output_keep_prob=(1. - dropout)) for _ in range(num_residual_layers) ]) _outputs, _state = tf.nn.dynamic_rnn(cell=stacked_cell, inputs=encoder_inputs, sequence_length=sequence_length, dtype=tf.float32) return _outputs, tf.concat(_state, -1)
def create_rnn_layer(layer_num, dim): cell = tf_rnn.LSTMCell(dim, name='layer_%s' % layer_num) if use_dropout and dropout_keep < 1.: cell = tf_rnn.DropoutWrapper(cell, output_keep_prob=dropout_keep) if layer_num > 0: cell = tf_rnn.ResidualWrapper(cell) return cell
def build_single_cell(self,layer=1): cell_type = rnn.LSTMCell if (self.cell_type.lower() == 'gru'): cell_type = rnn.GRUCell cell = cell_type(self.hidden_units * layer) if self.use_dropout: cell = rnn.DropoutWrapper(cell,dtype=self.dtype,output_keep_prob=self.keep_prob_placeholder) if self.use_residual: cell = rnn.ResidualWrapper(cell) return cell
def rnn_cell(rnn_cell_size, dropout_keep_prob, residual, is_training=True): """Builds an LSTMBlockCell based on the given parameters.""" dropout_keep_prob = dropout_keep_prob if is_training else 1.0 cells = [] for i in range(len(rnn_cell_size)): cell = rnn.LSTMBlockCell(rnn_cell_size[i]) if residual: cell = rnn.ResidualWrapper(cell) if i == 0 or rnn_cell_size[i] != rnn_cell_size[i - 1]: cell = rnn.InputProjectionWrapper(cell, rnn_cell_size[i]) cell = rnn.DropoutWrapper(cell, input_keep_prob=dropout_keep_prob) cells.append(cell) return rnn.MultiRNNCell(cells)
def _rnn_cell(self, size=None, activation=None, dropout=None, residual=False): cell = rnn.GRUCell((size or self._hidden_size), activation=activation) if residual: cell = rnn.ResidualWrapper(cell) if dropout is not None: cell = rnn.DropoutWrapper(cell, input_keep_prob=(1.0 - dropout)) return cell
def _new_cell_wrapper(residual_connection=False, device_id=None): c = _new_cell() if input_keep_prob < 1.0 or output_keep_prob < 1.0: c = rnn.DropoutWrapper(c, input_keep_prob=input_keep_prob, output_keep_prob=output_keep_prob) if residual_connection: c = rnn.ResidualWrapper(c) if device_id: c = rnn.DeviceWrapper(c, device_id) return c
def _single_cell(unit_type, num_units, forget_bias, dropout, mode, residual_connection=False, residual_fn=None, trainable=True): """Create an instance of a single RNN cell.""" # dropout (= 1 - keep_prob) is set to 0 during eval and infer dropout = dropout if mode == tf.estimator.ModeKeys.TRAIN else 0.0 # Cell Type if unit_type == "lstm": single_cell = contrib_rnn.LSTMCell(num_units, forget_bias=forget_bias, trainable=trainable) elif unit_type == "gru": single_cell = contrib_rnn.GRUCell(num_units, trainable=trainable) elif unit_type == "layer_norm_lstm": single_cell = contrib_rnn.LayerNormBasicLSTMCell( num_units, forget_bias=forget_bias, layer_norm=True, trainable=trainable) elif unit_type == "nas": single_cell = contrib_rnn.NASCell(num_units, trainable=trainable) else: raise ValueError("Unknown unit type %s!" % unit_type) # Dropout (= 1 - keep_prob). if dropout > 0.0: single_cell = contrib_rnn.DropoutWrapper(cell=single_cell, input_keep_prob=(1.0 - dropout)) # Residual. if residual_connection: single_cell = contrib_rnn.ResidualWrapper(single_cell, residual_fn=residual_fn) return single_cell
def make_rnn_cell(rnn_layer_sizes, dropout_keep_prob=1.0, attn_length=0, base_cell=contrib_rnn.BasicLSTMCell, residual_connections=False): """Makes a RNN cell from the given hyperparameters. Args: rnn_layer_sizes: A list of integer sizes (in units) for each layer of the RNN. dropout_keep_prob: The float probability to keep the output of any given sub-cell. attn_length: The size of the attention vector. base_cell: The base tf.contrib.rnn.RNNCell to use for sub-cells. residual_connections: Whether or not to use residual connections (via tf.contrib.rnn.ResidualWrapper). Returns: A tf.contrib.rnn.MultiRNNCell based on the given hyperparameters. """ cells = [] for i in range(len(rnn_layer_sizes)): cell = base_cell(rnn_layer_sizes[i]) if attn_length and not cells: # Add attention wrapper to first layer. cell = contrib_rnn.AttentionCellWrapper(cell, attn_length, state_is_tuple=True) if residual_connections: cell = contrib_rnn.ResidualWrapper(cell) if i == 0 or rnn_layer_sizes[i] != rnn_layer_sizes[i - 1]: cell = contrib_rnn.InputProjectionWrapper( cell, rnn_layer_sizes[i]) cell = contrib_rnn.DropoutWrapper(cell, output_keep_prob=dropout_keep_prob) cells.append(cell) cell = contrib_rnn.MultiRNNCell(cells) return cell
def model(inputs, inputs_length, reuse=None): embedded = tf.layers.dense(inputs=inputs, units=FLAGS.num_hidden, kernel_initializer=xavier_initializer(), name='embedding_layer', reuse=reuse) cell = rnn.GRUCell(FLAGS.num_hidden, reuse=reuse) cell = rnn.DropoutWrapper(cell, output_keep_prob=keep_prob) cell = rnn.MultiRNNCell([rnn.ResidualWrapper(cell) for i in range(FLAGS.num_layer)]) outputs, final_state = tf.nn.dynamic_rnn(cell, embedded, inputs_length, dtype=tf.float32) # cell.zero_state(batch_size, dtype=tf.float32)) results = tf.layers.dense(inputs=final_state[FLAGS.num_layer - 1], units=2, kernel_initializer=xavier_initializer(), name='output_layer', reuse=reuse) return results
def get_rnn_cell(hparams=None, mode=None): """Creates an RNN cell. See :func:`~texar.core.default_rnn_cell_hparams` for all hyperparameters and default values. Args: hparams (dict or HParams, optional): Cell hyperparameters. Missing hyperparameters are set to default values. mode (optional): A Tensor taking value in :tf_main:`tf.estimator.ModeKeys <estimator/ModeKeys>`, including `TRAIN`, `EVAL`, and `PREDICT`. If `None`, dropout will be controlled by :func:`texar.global_mode`. Returns: A cell instance. Raises: ValueError: If hparams["num_layers"]>1 and hparams["type"] is a class instance. ValueError: The cell is not an :tf_main:`RNNCell <contrib/rnn/RNNCell>` instance. """ if hparams is None or isinstance(hparams, dict): hparams = HParams(hparams, default_rnn_cell_hparams()) d_hp = hparams["dropout"] if d_hp["variational_recurrent"] and \ len(d_hp["input_size"]) != hparams["num_layers"]: raise ValueError( "If variational_recurrent=True, input_size must be a list of " "num_layers(%d) integers. Got len(input_size)=%d." % (hparams["num_layers"], len(d_hp["input_size"]))) cells = [] cell_kwargs = hparams["kwargs"].todict() num_layers = hparams["num_layers"] for layer_i in range(num_layers): # Create the basic cell cell_type = hparams["type"] if not is_str(cell_type) and not isinstance(cell_type, type): if num_layers > 1: raise ValueError( "If 'num_layers'>1, then 'type' must be a cell class or " "its name/module path, rather than a cell instance.") cell_modules = ['tensorflow.contrib.rnn', 'texar.custom'] cell = utils.check_or_get_instance(cell_type, cell_kwargs, cell_modules, rnn.RNNCell) # Optionally add dropout if d_hp["input_keep_prob"] < 1.0 or \ d_hp["output_keep_prob"] < 1.0 or \ d_hp["state_keep_prob"] < 1.0: vr_kwargs = {} if d_hp["variational_recurrent"]: vr_kwargs = { "variational_recurrent": True, "input_size": d_hp["input_size"][layer_i], "dtype": tf.float32 } input_keep_prob = switch_dropout(d_hp["input_keep_prob"], mode) output_keep_prob = switch_dropout(d_hp["output_keep_prob"], mode) state_keep_prob = switch_dropout(d_hp["state_keep_prob"], mode) cell = rnn.DropoutWrapper(cell=cell, input_keep_prob=input_keep_prob, output_keep_prob=output_keep_prob, state_keep_prob=state_keep_prob, **vr_kwargs) # Optionally add residual and highway connections if layer_i > 0: if hparams["residual"]: cell = rnn.ResidualWrapper(cell) if hparams["highway"]: cell = rnn.HighwayWrapper(cell) cells.append(cell) if hparams["num_layers"] > 1: cell = rnn.MultiRNNCell(cells) else: cell = cells[0] return cell