def _single_cell(unit_type,
num_units,
forget_bias,
dropout,
mode,
residual_connection=False,
device_str=None,
residual_fn=None):
"""
创建一个RNN单元。
:param unit_type: RNN类型
:param num_units: 隐层神经元个数
:param forget_bias: 遗忘门偏置
:param dropout: dropout比例
:param mode: 训练模式（只有train模式下才设置dropout）
:param residual_connection: 是否使用残差连接
:param device_str: 设备
:param residual_fn: 残差方法
:return:
"""
# dropout (= 1 - keep_prob) is set to 0 during eval and infer
dropout = dropout if mode == tf.contrib.learn.ModeKeys.TRAIN else 0.0
# Cell Type
if unit_type == "lstm":
print(" LSTM, forget_bias=%g" % forget_bias, end='')
single_cell = rnn.BasicLSTMCell(num_units, forget_bias=forget_bias)
elif unit_type == "gru":
print(" GRU", end='')
single_cell = rnn.GRUCell(num_units)
elif unit_type == "layer_norm_lstm":
print(" Layer Normalized LSTM, forget_bias=%g" % forget_bias, end='')
single_cell = rnn.LayerNormBasicLSTMCell(num_units,
forget_bias=forget_bias,
layer_norm=True)
elif unit_type == "nas":
print(" NASCell", end='')
single_cell = rnn.NASCell(num_units)
else:
raise ValueError("Unknown unit type %s!" % unit_type)
# Dropout (= 1 - keep_prob)
if dropout > 0.0:
single_cell = rnn.DropoutWrapper(cell=single_cell,
input_keep_prob=(1.0 - dropout))
print(" %s, dropout=%g " % (type(single_cell).__name__, dropout),
end='')
# Residual
if residual_connection:
single_cell = rnn.ResidualWrapper(single_cell, residual_fn=residual_fn)
print(" %s" % type(single_cell).__name__, end='')
# Device Wrapper
if device_str:
single_cell = rnn.DeviceWrapper(single_cell, device_str)
print(" %s, device=%s" % (type(single_cell).__name__, device_str),
end='')
return single_cell
def make_rnn_cell(rnn_layer_sizes,
dropout_keep_prob=1.0,
attn_length=0,
base_cell=contrib_rnn.BasicLSTMCell,
residual_connections=False):
cells = []
for i in range(len(rnn_layer_sizes)):
cell = base_cell(rnn_layer_sizes[i])
if attn_length and not cells:
# Add attention wrapper to first layer.
cell = contrib_rnn.AttentionCellWrapper(cell,
attn_length,
state_is_tuple=True)
if residual_connections:
cell = contrib_rnn.ResidualWrapper(cell)
if i == 0 or rnn_layer_sizes[i] != rnn_layer_sizes[i - 1]:
cell = contrib_rnn.InputProjectionWrapper(
cell, rnn_layer_sizes[i])
cell = contrib_rnn.DropoutWrapper(cell,
output_keep_prob=dropout_keep_prob)
cells.append(cell)
cell = contrib_rnn.MultiRNNCell(cells)
return cell
def create_rnn_layer(layer_num, dim):
if layer_num == 0:
return tf_rnn.LSTMCell(dim, name='layer_%s' % layer_num)
cell = tf_rnn.LSTMCell(dim, name='layer_%s' % layer_num)
cell = tf_rnn.ResidualWrapper(cell)
return cell
def inputs_encoder(inputs,
embeddings,
sequence_length,
size=None,
activation=tf.nn.tanh,
num_residual_layers=0,
dropout=0.0):
assert (size is not None) and isinstance(size, int)
assert isinstance(num_residual_layers, int) and (num_residual_layers >= 0)
with tf.variable_scope('inputs_encoder'):
encoder_inputs = tf.concat([inputs, embeddings], -1)
stacked_cell = rnn.MultiRNNCell([
rnn.DropoutWrapper(rnn.GRUCell(size, activation=activation),
output_keep_prob=(1. - dropout))
] + [
rnn.DropoutWrapper(rnn.ResidualWrapper(
rnn.GRUCell(size, activation=activation)),
output_keep_prob=(1. - dropout))
for _ in range(num_residual_layers)
])
_outputs, _state = tf.nn.dynamic_rnn(cell=stacked_cell,
inputs=encoder_inputs,
sequence_length=sequence_length,
dtype=tf.float32)
return _outputs, tf.concat(_state, -1)
def create_rnn_layer(layer_num, dim):
cell = tf_rnn.LSTMCell(dim, name='layer_%s' % layer_num)
if use_dropout and dropout_keep < 1.:
cell = tf_rnn.DropoutWrapper(cell, output_keep_prob=dropout_keep)
if layer_num > 0:
cell = tf_rnn.ResidualWrapper(cell)
return cell
def build_single_cell(self,layer=1):
cell_type = rnn.LSTMCell
if (self.cell_type.lower() == 'gru'):
cell_type = rnn.GRUCell
cell = cell_type(self.hidden_units * layer)
if self.use_dropout:
cell = rnn.DropoutWrapper(cell,dtype=self.dtype,output_keep_prob=self.keep_prob_placeholder)
if self.use_residual:
cell = rnn.ResidualWrapper(cell)
return cell
def rnn_cell(rnn_cell_size, dropout_keep_prob, residual, is_training=True):
"""Builds an LSTMBlockCell based on the given parameters."""
dropout_keep_prob = dropout_keep_prob if is_training else 1.0
cells = []
for i in range(len(rnn_cell_size)):
cell = rnn.LSTMBlockCell(rnn_cell_size[i])
if residual:
cell = rnn.ResidualWrapper(cell)
if i == 0 or rnn_cell_size[i] != rnn_cell_size[i - 1]:
cell = rnn.InputProjectionWrapper(cell, rnn_cell_size[i])
cell = rnn.DropoutWrapper(cell, input_keep_prob=dropout_keep_prob)
cells.append(cell)
return rnn.MultiRNNCell(cells)
def _rnn_cell(self,
size=None,
activation=None,
dropout=None,
residual=False):
cell = rnn.GRUCell((size or self._hidden_size), activation=activation)
if residual:
cell = rnn.ResidualWrapper(cell)
if dropout is not None:
cell = rnn.DropoutWrapper(cell, input_keep_prob=(1.0 - dropout))
return cell
def _new_cell_wrapper(residual_connection=False, device_id=None):
c = _new_cell()
if input_keep_prob < 1.0 or output_keep_prob < 1.0:
c = rnn.DropoutWrapper(c,
input_keep_prob=input_keep_prob,
output_keep_prob=output_keep_prob)
if residual_connection:
c = rnn.ResidualWrapper(c)
if device_id:
c = rnn.DeviceWrapper(c, device_id)
return c
def _single_cell(unit_type,
num_units,
forget_bias,
dropout,
mode,
residual_connection=False,
residual_fn=None,
trainable=True):
"""Create an instance of a single RNN cell."""
# dropout (= 1 - keep_prob) is set to 0 during eval and infer
dropout = dropout if mode == tf.estimator.ModeKeys.TRAIN else 0.0
# Cell Type
if unit_type == "lstm":
single_cell = contrib_rnn.LSTMCell(num_units,
forget_bias=forget_bias,
trainable=trainable)
elif unit_type == "gru":
single_cell = contrib_rnn.GRUCell(num_units, trainable=trainable)
elif unit_type == "layer_norm_lstm":
single_cell = contrib_rnn.LayerNormBasicLSTMCell(
num_units,
forget_bias=forget_bias,
layer_norm=True,
trainable=trainable)
elif unit_type == "nas":
single_cell = contrib_rnn.NASCell(num_units, trainable=trainable)
else:
raise ValueError("Unknown unit type %s!" % unit_type)
# Dropout (= 1 - keep_prob).
if dropout > 0.0:
single_cell = contrib_rnn.DropoutWrapper(cell=single_cell,
input_keep_prob=(1.0 -
dropout))
# Residual.
if residual_connection:
single_cell = contrib_rnn.ResidualWrapper(single_cell,
residual_fn=residual_fn)
return single_cell
def make_rnn_cell(rnn_layer_sizes,
dropout_keep_prob=1.0,
attn_length=0,
base_cell=contrib_rnn.BasicLSTMCell,
residual_connections=False):
"""Makes a RNN cell from the given hyperparameters.
Args:
rnn_layer_sizes: A list of integer sizes (in units) for each layer of the
RNN.
dropout_keep_prob: The float probability to keep the output of any given
sub-cell.
attn_length: The size of the attention vector.
base_cell: The base tf.contrib.rnn.RNNCell to use for sub-cells.
residual_connections: Whether or not to use residual connections (via
tf.contrib.rnn.ResidualWrapper).
Returns:
A tf.contrib.rnn.MultiRNNCell based on the given hyperparameters.
"""
cells = []
for i in range(len(rnn_layer_sizes)):
cell = base_cell(rnn_layer_sizes[i])
if attn_length and not cells:
# Add attention wrapper to first layer.
cell = contrib_rnn.AttentionCellWrapper(cell,
attn_length,
state_is_tuple=True)
if residual_connections:
cell = contrib_rnn.ResidualWrapper(cell)
if i == 0 or rnn_layer_sizes[i] != rnn_layer_sizes[i - 1]:
cell = contrib_rnn.InputProjectionWrapper(
cell, rnn_layer_sizes[i])
cell = contrib_rnn.DropoutWrapper(cell,
output_keep_prob=dropout_keep_prob)
cells.append(cell)
cell = contrib_rnn.MultiRNNCell(cells)
return cell
def model(inputs, inputs_length, reuse=None):
embedded = tf.layers.dense(inputs=inputs,
units=FLAGS.num_hidden,
kernel_initializer=xavier_initializer(),
name='embedding_layer',
reuse=reuse)
cell = rnn.GRUCell(FLAGS.num_hidden, reuse=reuse)
cell = rnn.DropoutWrapper(cell, output_keep_prob=keep_prob)
cell = rnn.MultiRNNCell([rnn.ResidualWrapper(cell) for i in range(FLAGS.num_layer)])
outputs, final_state = tf.nn.dynamic_rnn(cell,
embedded,
inputs_length,
dtype=tf.float32)
# cell.zero_state(batch_size, dtype=tf.float32))
results = tf.layers.dense(inputs=final_state[FLAGS.num_layer - 1],
units=2,
kernel_initializer=xavier_initializer(),
name='output_layer',
reuse=reuse)
return results
def get_rnn_cell(hparams=None, mode=None):
"""Creates an RNN cell.
See :func:`~texar.core.default_rnn_cell_hparams` for all
hyperparameters and default values.
Args:
hparams (dict or HParams, optional): Cell hyperparameters. Missing
hyperparameters are set to default values.
mode (optional): A Tensor taking value in
:tf_main:`tf.estimator.ModeKeys <estimator/ModeKeys>`, including
`TRAIN`, `EVAL`, and `PREDICT`. If `None`, dropout will be
controlled by :func:`texar.global_mode`.
Returns:
A cell instance.
Raises:
ValueError: If hparams["num_layers"]>1 and hparams["type"] is a class
instance.
ValueError: The cell is not an
:tf_main:`RNNCell <contrib/rnn/RNNCell>` instance.
"""
if hparams is None or isinstance(hparams, dict):
hparams = HParams(hparams, default_rnn_cell_hparams())
d_hp = hparams["dropout"]
if d_hp["variational_recurrent"] and \
len(d_hp["input_size"]) != hparams["num_layers"]:
raise ValueError(
"If variational_recurrent=True, input_size must be a list of "
"num_layers(%d) integers. Got len(input_size)=%d." %
(hparams["num_layers"], len(d_hp["input_size"])))
cells = []
cell_kwargs = hparams["kwargs"].todict()
num_layers = hparams["num_layers"]
for layer_i in range(num_layers):
# Create the basic cell
cell_type = hparams["type"]
if not is_str(cell_type) and not isinstance(cell_type, type):
if num_layers > 1:
raise ValueError(
"If 'num_layers'>1, then 'type' must be a cell class or "
"its name/module path, rather than a cell instance.")
cell_modules = ['tensorflow.contrib.rnn', 'texar.custom']
cell = utils.check_or_get_instance(cell_type, cell_kwargs,
cell_modules, rnn.RNNCell)
# Optionally add dropout
if d_hp["input_keep_prob"] < 1.0 or \
d_hp["output_keep_prob"] < 1.0 or \
d_hp["state_keep_prob"] < 1.0:
vr_kwargs = {}
if d_hp["variational_recurrent"]:
vr_kwargs = {
"variational_recurrent": True,
"input_size": d_hp["input_size"][layer_i],
"dtype": tf.float32
}
input_keep_prob = switch_dropout(d_hp["input_keep_prob"], mode)
output_keep_prob = switch_dropout(d_hp["output_keep_prob"], mode)
state_keep_prob = switch_dropout(d_hp["state_keep_prob"], mode)
cell = rnn.DropoutWrapper(cell=cell,
input_keep_prob=input_keep_prob,
output_keep_prob=output_keep_prob,
state_keep_prob=state_keep_prob,
**vr_kwargs)
# Optionally add residual and highway connections
if layer_i > 0:
if hparams["residual"]:
cell = rnn.ResidualWrapper(cell)
if hparams["highway"]:
cell = rnn.HighwayWrapper(cell)
cells.append(cell)
if hparams["num_layers"] > 1:
cell = rnn.MultiRNNCell(cells)
else:
cell = cells[0]
return cell
