def __init__(self,
hidden_size,
keep_prob,
cell_type="gru",
scope="encoder"):
"""
Inputs:
hidden_size:
int.
Hidden size of the RNN
keep_prob:
Tensor
containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
if cell_type == "gru":
rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size)
rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size)
elif cell_type == "lstm":
rnn_cell_fw = rnn_cell.LSTMCell(self.hidden_size)
rnn_cell_bw = rnn_cell.LSTMCell(self.hidden_size)
else:
assert (False, "No such cell type for RNN encoder!")
self.rnn_cell_fw = DropoutWrapper(rnn_cell_fw,
input_keep_prob=self.keep_prob)
self.rnn_cell_bw = DropoutWrapper(rnn_cell_bw,
input_keep_prob=self.keep_prob)
self.scope = scope
logger.info("Encoder created: {} | hidden_size = {}".format(
cell_type, hidden_size))
def add_conversational_context(self, sentence_summary_size):
line_vectors_as_timesteps = tf.expand_dims(self.sentence_summaries, 0)
if self.config.bidirectional_conversations:
forwardcell = DropoutWrapper(GRUCell(sentence_summary_size),
self.dropout_placeholder,
self.dropout_placeholder)
backwardcell = DropoutWrapper(GRUCell(sentence_summary_size),
self.dropout_placeholder,
self.dropout_placeholder)
outputs, sf, sb = bidirectional_dynamic_rnn(
forwardcell,
backwardcell,
line_vectors_as_timesteps,
tf.slice(tf.shape(line_vectors_as_timesteps), [1],
[1]), # what the f*****g f**k
dtype=tf.float32,
scope="ChapterRNN")
self.conversation_state = tf.squeeze(outputs)
return 2 * sentence_summary_size
else:
rnncell = DropoutWrapper(GRUCell(sentence_summary_size),
self.dropout_placeholder,
self.dropout_placeholder)
outputs, state = tf.nn.dynamic_rnn(
rnncell,
line_vectors_as_timesteps,
tf.slice(tf.shape(line_vectors_as_timesteps), [1],
[1]), # what the f*****g f**k
dtype=tf.float32,
scope="ChapterRNN")
self.conversation_state = tf.squeeze(outputs)
return sentence_summary_size
def __init__(self, size, num_layers, tied_weights=True):
self.size = size
self.keep_prob = tf.placeholder(tf.float32)
self.tied_weights = tied_weights
# when we use different weights, we force the model to concatenate
if not self.tied_weights:
FLAGS.concat = True
if FLAGS.rnn == "lstm":
cell = rnn_cell.BasicLSTMCell(self.size)
state_is_tuple = True
else:
cell = rnn_cell.GRUCell(self.size)
state_is_tuple = False
cell = DropoutWrapper(cell, input_keep_prob=self.keep_prob, seed=123)
self.encoder_cell = tf.nn.rnn_cell.MultiRNNCell(
[cell] * num_layers, state_is_tuple=state_is_tuple)
if not tied_weights:
if FLAGS.rnn == "lstm":
cell_back = rnn_cell.BasicLSTMCell(self.size)
state_is_tuple = True
else:
cell_back = rnn_cell.GRUCell(self.size)
state_is_tuple = False
cell_back = DropoutWrapper(cell_back,
input_keep_prob=self.keep_prob,
seed=123)
self.encoder_cell_bw = tf.nn.rnn_cell.MultiRNNCell(
[cell_back] * num_layers, state_is_tuple=state_is_tuple)
def __init__(self, hidden_size, keep_prob, num_layers):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.num_layers = num_layers
self.rnn_cell_fw = [
rnn_cell.GRUCell(self.hidden_size) for _ in range(self.num_layers)
]
self.rnn_cell_fw = [
DropoutWrapper(cell, input_keep_prob=self.keep_prob)
for cell in self.rnn_cell_fw
]
self.multi_rnn_cell_fw = rnn_cell.MultiRNNCell(self.rnn_cell_fw,
state_is_tuple=False)
self.rnn_cell_bw = [
rnn_cell.GRUCell(self.hidden_size) for _ in range(self.num_layers)
]
self.rnn_cell_bw = [
DropoutWrapper(cell, input_keep_prob=self.keep_prob)
for cell in self.rnn_cell_bw
]
self.multi_rnn_cell_bw = rnn_cell.MultiRNNCell(self.rnn_cell_bw,
state_is_tuple=False)
def add_sentence_summaries(self, wordvector_embed_size):
if self.config.bidirectional_sentences:
forwardcell = DropoutWrapper(GRUCell(wordvector_embed_size),
self.dropout_placeholder,
self.dropout_placeholder)
backwardcell = DropoutWrapper(GRUCell(wordvector_embed_size),
self.dropout_placeholder,
self.dropout_placeholder)
_, statefw, statebw = bidirectional_dynamic_rnn(
forwardcell,
backwardcell,
self.embedded_lines,
self.line_length_placeholder,
dtype=tf.float32,
scope="LineRNN")
self.sentence_summaries = tf.concat(1, [statefw, statebw])
return 2 * wordvector_embed_size
else:
rnncell = DropoutWrapper(GRUCell(wordvector_embed_size),
self.dropout_placeholder,
self.dropout_placeholder)
_, self.sentence_summaries = tf.nn.dynamic_rnn(
rnncell,
self.embedded_lines,
self.line_length_placeholder,
dtype=tf.float32,
scope="LineRNN")
return wordvector_embed_size
def __init__(self, hidden_size, keep_prob):
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.rnn_cell_fw = tf.contrib.rnn.LSTMCell(self.hidden_size, forget_bias=1.0)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob)
self.rnn_cell_bw = tf.contrib.rnn.LSTMCell(self.hidden_size, forget_bias=1.0)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def __init__(self, hidden_size, keep_prob):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
#self.rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size)
self.rnn_cell_fw_layer1 = rnn_cell.LSTMCell(self.hidden_size)
self.rnn_cell_fw_layer1 = DropoutWrapper(
self.rnn_cell_fw_layer1, input_keep_prob=self.keep_prob)
#self.rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size)
self.rnn_cell_bw_layer1 = rnn_cell.LSTMCell(self.hidden_size)
self.rnn_cell_bw_layer1 = DropoutWrapper(
self.rnn_cell_bw_layer1, input_keep_prob=self.keep_prob)
self.rnn_cell_fw_layer2 = rnn_cell.LSTMCell(self.hidden_size)
self.rnn_cell_fw_layer2 = DropoutWrapper(
self.rnn_cell_fw_layer2, input_keep_prob=self.keep_prob)
self.rnn_cell_bw_layer2 = rnn_cell.LSTMCell(self.hidden_size)
self.rnn_cell_bw_layer2 = DropoutWrapper(
self.rnn_cell_bw_layer2, input_keep_prob=self.keep_prob)
def __init__(self, hidden_size, keep_prob):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
# layer 1
self.rnn_cell_fw_1 = rnn_cell.BasicLSTMCell(self.hidden_size,
forget_bias=1.0)
self.rnn_cell_fw_1 = DropoutWrapper(self.rnn_cell_fw_1,
input_keep_prob=self.keep_prob)
self.rnn_cell_bw_1 = rnn_cell.BasicLSTMCell(self.hidden_size,
forget_bias=1.0)
self.rnn_cell_bw_1 = DropoutWrapper(self.rnn_cell_bw_1,
input_keep_prob=self.keep_prob)
# layer 2
self.rnn_cell_fw_2 = rnn_cell.BasicLSTMCell(self.hidden_size,
forget_bias=1.0)
self.rnn_cell_fw_2 = DropoutWrapper(self.rnn_cell_fw_2,
input_keep_prob=self.keep_prob)
self.rnn_cell_bw_2 = rnn_cell.BasicLSTMCell(self.hidden_size,
forget_bias=1.0)
self.rnn_cell_bw_2 = DropoutWrapper(self.rnn_cell_bw_2,
input_keep_prob=self.keep_prob)
def __init__(self, hidden_size, keep_prob):
self.hidden_size = hidden_size # this should be 2 * self.FLAGS.hidden_size
self.keep_prob = keep_prob
self.fwd = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size / 2),
input_keep_prob=self.keep_prob)
self.back = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size / 2),
input_keep_prob=self.keep_prob)
def s2v(self):
sqrt3 = math.sqrt(3.0)
initializer = tf.random_uniform_initializer(-sqrt3,
sqrt3,
dtype=self.dtype)
# word embedding layer
if self.pre_trained_word_emb is not None:
self.word_embeddings = tf.get_variable(
name='word_embedding',
initializer=self.pre_trained_word_emb,
dtype=self.dtype)
else:
self.word_embeddings = tf.get_variable(
name='word_embedding',
shape=[self.voc_size, self.emb_size],
initializer=initializer,
dtype=self.dtype)
self.embedded_sentence = tf.nn.embedding_lookup(
self.word_embeddings, self.sentence)
self.embedded_sentence = tf.nn.dropout(
self.embedded_sentence, keep_prob=self.keep_word_prob_placeholder)
# create the rnn cell
if self.rnn_cell_type.lower() == 'gru':
rnn_cell = GRUCell
else:
rnn_cell = LSTMCell
rnn_cell = rnn_cell(self.hidden_units)
if self.use_lstm_dropout:
rnn_cell = DropoutWrapper(
rnn_cell,
dtype=tf.float32,
output_keep_prob=self.keep_lstm_prob_placeholder)
if self.rnn_model == 'leap-lstm':
self.sentence_emb, self.skip_dis_output = self.leap_lstm(rnn_cell)
elif self.rnn_model == 'rnn':
if self.rnn_pattern == 1:
self.sentence_emb = self.general_rnn(rnn_cell, out='LAST')
else:
self.sentence_emb = self.general_rnn_for_pattern(
rnn_cell, out='LAST') # for test the training time
elif self.rnn_model == 'brnn':
self.sentence_emb = self.general_brnn()
elif self.rnn_model == 'skip-rnn-2017':
self.sentence_emb, self.budget_loss, self.updated_states, self.rnn_final_states, self.rnn_outputs = self.skip_rnn_2017(
)
elif self.rnn_model == 'skim-rnn':
small_rnn_cell = LSTMCell(5) # small size 5
small_rnn_cell = DropoutWrapper(
small_rnn_cell,
dtype=tf.float32,
output_keep_prob=self.keep_lstm_prob_placeholder)
self.sentence_emb, self.skip_dis_output, self.skim_loss = self.skim_rnn(
rnn_cell, small_rnn_cell) # skim-rnn的设定直接按照github上源码来就可以了
else:
print("bad rnn model!")
exit()
def __init__(self, hidden_size, keep_prob):
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.rnn_cell_fw = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_size)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw,
input_keep_prob=self.keep_prob)
self.rnn_cell_bw = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_size)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw,
input_keep_prob=self.keep_prob)
def getCell(self, is_training, dp, config):
# code for RNN
if is_training == True:
print("==> Construct ", config.cell_type, " graph for training")
else:
print("==> Construct ", config.cell_type, " graph for testing")
if config.cell_type == "LSTM":
if config.num_layer == 1:
basicCell = LSTMCell(config.hidden_size, forget_bias=0.0, state_is_tuple=True)
elif config.num_layer == 2:
basicCell = LSTMCell(config.hidden_size, forget_bias=0.0, state_is_tuple=True)
basicCell_2 = LSTMCell(config.hidden_size_2, forget_bias=0.0, state_is_tuple=True)
else:
raise ValueError("config.num_layer should be 1:2 ")
elif config.cell_type == "RNN":
if config.num_layer == 1:
basicCell = BasicRNNCell(config.hidden_size)
elif config.num_layer == 2:
basicCell = BasicRNNCell(config.hidden_size)
basicCell_2 = BasicRNNCell(config.hidden_size_2)
else:
raise ValueError("config.num_layer should be [1-3] ")
elif config.cell_type == "GRU":
if config.num_layer == 1:
basicCell = GRUCell(config.hidden_size, forget_bias=0.0, state_is_tuple=True)
elif config.num_layer == 2:
basicCell = GRUCell(config.hidden_size, forget_bias=0.0, state_is_tuple=True)
basicCell_2 = GRUCell(config.hidden_size_2, forget_bias=0.0, state_is_tuple=True)
else:
raise ValueError("only support 1-2 layers ")
else:
raise ValueError("cell type should be GRU,LSTM,RNN")
# add dropout layer between hidden layers
if is_training and config.keep_prob < 1:
if config.num_layer == 1:
basicCell = DropoutWrapper(basicCell, input_keep_prob=config.keep_prob,
output_keep_prob=config.keep_prob)
elif config.num_layer == 2:
basicCell = DropoutWrapper(basicCell, input_keep_prob=config.keep_prob,
output_keep_prob=config.keep_prob)
basicCell_2 = DropoutWrapper(basicCell_2, input_keep_prob=config.keep_prob,
output_keep_prob=config.keep_prob)
else:
pass
if config.num_layer == 1:
cell = rnn_cell.MultiRNNCell([basicCell], state_is_tuple=True)
elif config.num_layer == 2:
cell = rnn_cell.MultiRNNCell([basicCell, basicCell_2], state_is_tuple=True)
return cell
def inference_layer(self, inputs):
if self.dblstm:
with tf.name_scope('deep_bidirectional_rnn'):
rnn_outputs, _ = deep_bidirectional_dynamic_rnn(
[self._dblstm_cell() for _ in range(self.num_layers)],
inputs,
sequence_length=self.sequence_lengths)
state_dim = self.state_dim
else:
cell_fw = DropoutWrapper(LSTMCell(num_units=self.state_dim),
variational_recurrent=True,
state_keep_prob=self.dropout_keep_prob,
output_keep_prob=self.dropout_keep_prob,
dtype=tf.float32)
cell_bw = DropoutWrapper(LSTMCell(num_units=self.state_dim),
variational_recurrent=True,
state_keep_prob=self.dropout_keep_prob,
output_keep_prob=self.dropout_keep_prob,
dtype=tf.float32)
with tf.name_scope('bidirectional_rnn'):
rnn_outputs, _ = tf.nn.bidirectional_dynamic_rnn(
cell_fw,
cell_bw,
inputs,
sequence_length=self.sequence_lengths,
dtype=tf.float32)
rnn_outputs = tf.concat(rnn_outputs, 2)
state_dim = self.state_dim * 2
with tf.name_scope('linear_projection'):
softmax_weights = tf.get_variable(
'softmax_W', [state_dim, self.num_classes],
initializer=tf.random_normal_initializer(0, 0.01))
softmax_bias = tf.get_variable('softmax_b', [self.num_classes],
initializer=tf.zeros_initializer)
time_steps = tf.shape(rnn_outputs)[1]
rnn_outputs = tf.reshape(
rnn_outputs, [-1, state_dim],
name="flatten_rnn_outputs_for_linear_projection")
logits = tf.nn.xw_plus_b(x=rnn_outputs,
weights=softmax_weights,
biases=softmax_bias,
name="softmax_projection")
self.scores = tf.reshape(logits,
[-1, time_steps, self.num_classes],
name="unflatten_logits")
if self.crf:
self.transition_params = tf.get_variable(
"transitions", [self.num_classes, self.num_classes])
def __init__(self, hidden_size, keep_prob,model_name="RNNModelEncoder"):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.rnn_cell_fw = rnn_cell.LSTMCell(self.hidden_size)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob)
self.rnn_cell_bw = rnn_cell.LSTMCell(self.hidden_size)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
self.model_name=model_name
def __init__(self, hidden_size, keep_prob):
"""
hidden_size: number of hidden size of RNN
keep_prob: keep prob. for dropout
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.fw_rnn = rnn_cell.GRUCell(self.hidden_size)
self.fw_rnn = DropoutWrapper(self.fw_rnn,
input_keep_prob=self.keep_prob)
self.bw_rnn = rnn_cell.GRUCell(self.hidden_size)
self.bw_rnn = DropoutWrapper(self.bw_rnn,
input_keep_prob=self.keep_prob)
def build_graph(self):
input_lens = tf.reduce_sum(self.X_mask_placeholder, axis=1)
inputs = self.X_placeholder
char_embedding = self.convolve()
inputs = tf.concat([self.X_placeholder, char_embedding], axis=-1)
inputs = tf.concat([inputs, self.features], axis=-1)
for i in range(0, self.depth):
lstm_cell_forward = rnn_cell.LSTMCell(self.hidden_size)
lstm_cell_forward = DropoutWrapper(lstm_cell_forward,
input_keep_prob=self.keep_prob)
lstm_cell_backward = rnn_cell.LSTMCell(self.hidden_size)
lstm_cell_backward = DropoutWrapper(lstm_cell_backward,
input_keep_prob=self.keep_prob)
(fw_out, bw_out), _ = tf.nn.bidirectional_dynamic_rnn(
lstm_cell_forward,
lstm_cell_backward,
inputs,
input_lens,
dtype=tf.float32,
scope='layer' + str(i))
out = tf.concat([fw_out, bw_out], 2)
out = tf.nn.dropout(out, self.keep_prob)
inputs = out
h = tf.contrib.layers.fully_connected(out,
num_outputs=self.hidden_size,
activation_fn=tf.nn.relu)
rows = tf.range(0, tf.shape(input_lens)[-1])
indices = tf.subtract(input_lens, tf.ones_like(input_lens))
indices = tf.nn.relu(indices)
slicer = tf.stack([rows, indices], axis=1)
h = tf.gather_nd(h, slicer)
weights = tf.get_variable(
"W",
shape=[self.hidden_size, self.num_classes],
initializer=tf.contrib.layers.xavier_initializer())
bias = tf.get_variable("b",
shape=[self.num_classes],
initializer=tf.zeros_initializer())
logits = tf.nn.xw_plus_b(h, weights, bias, name="logits")
preds = tf.argmax(logits, 1)
return logits, preds
def __init__(self, hidden_size, keep_prob, attention_mechanism=None, name="RNNEncoder"):
with vs.variable_scope(name):
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size)
if attention_mechanism is not None:
self.rnn_cell_fw = AttentionWrapper(self.rnn_cell_fw, attention_mechanism)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob)
self.rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size)
if attention_mechanism is not None:
self.rnn_cell_bw = AttentionWrapper(self.rnn_cell_bw, attention_mechanism)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
self.name = name
def __init__(self, keep_prob, qn_vec_size, cxt_vec_size):
"""
Inputs:
keep_prob: tensor containing a single scalar that is the keep probability (for dropout)
qn_vec_size: size of the question vectors. int
cxt_vec_size: size of the context vectors. int
"""
self.keep_prob = keep_prob
self.qn_vec_size = qn_vec_size
self.cxt_vec_size = cxt_vec_size
self.rnn_cell_fw = rnn_cell.LSTMCell(cxt_vec_size/2, reuse=tf.AUTO_REUSE)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob)
self.rnn_cell_bw = rnn_cell.LSTMCell(cxt_vec_size/2, reuse=tf.AUTO_REUSE)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def __init__(self, keep_prob, key_vec_size, value_vec_size):
"""
Inputs:
keep_prob: tensor containing a single scalar that is the keep probability (for dropout)
key_vec_size: size of the key vectors. int
value_vec_size: size of the value vectors. int
"""
self.keep_prob = keep_prob
self.key_vec_size = key_vec_size
self.value_vec_size = value_vec_size
self.rnn_cell_fw = rnn_cell.LSTMCell(value_vec_size/2, reuse=tf.AUTO_REUSE)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob)
self.rnn_cell_bw = rnn_cell.LSTMCell(value_vec_size/2, reuse=tf.AUTO_REUSE)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def __init__(self, hidden_size, keep_prob, scope_name):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw,
input_keep_prob=self.keep_prob)
self.rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw,
input_keep_prob=self.keep_prob)
self.scope_name = scope_name
def __init__(self, hidden_size, keep_prob):
#(self.FLAGS.hidden_size, self.keep_prob)
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.rnn_cell_fw = rnn_cell.BasicLSTMCell(self.hidden_size)
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw,
input_keep_prob=self.keep_prob)
self.rnn_cell_bw = rnn_cell.BasicLSTMCell(self.hidden_size)
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw,
input_keep_prob=self.keep_prob)
def biLSTM(inputs, hiddenSize, keepProb, scopeName, mask=None):
with vs.variable_scope(scopeName):
fwCell = rnn_cell.LSTMCell(hiddenSize)
fwCell = DropoutWrapper(fwCell, input_keep_prob=keepProb)
bwCell = rnn_cell.LSTMCell(hiddenSize)
bwCell = DropoutWrapper(bwCell, input_keep_prob=keepProb)
if mask is not None:
mask = tf.reduce_sum(mask, reduction_indices=1)
(fwOut, bwOut), _ = tf.nn.bidirectional_dynamic_rnn(fwCell,
bwCell,
inputs,
mask,
dtype=tf.float32)
out = tf.concat([fwOut, bwOut], 2)
out = tf.nn.dropout(out, keepProb)
return out
def single_cell(cell_class,
cell_params,
dp_input_keep_prob=1.0,
dp_output_keep_prob=1.0,
residual_connections=False):
"""Creates an instance of the rnn cell.
Such cell describes one step one layer and can include residual connection
and/or dropout
Args:
cell_class: Tensorflow RNN cell class
cell_params (dict): cell parameters
dp_input_keep_prob (float): (default: 1.0) input dropout keep probability
dp_output_keep_prob (float): (default: 1.0) output dropout keep probability
residual_connections (bool): whether to add residual connection
Returns:
TF RNN instance
"""
cell = cell_class(**cell_params)
if residual_connections:
cell = ResidualWrapper(cell)
if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0:
cell = DropoutWrapper(cell,
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
return cell
def __init__(self, hidden_size, keep_prob, num_layers=1, cell_type='gru',
scope='RNNEncoder', combiner='concat'):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
def get_cell():
if cell_type == 'gru':
return rnn_cell.GRUCell(self.hidden_size)
elif cell_type == 'lstm':
return rnn_cell.LSTMCell(self.hidden_size)
elif cell_type == 'layer_norm':
return LayerNormBasicLSTMCell(self.hidden_size,
dropout_keep_prob=keep_prob)
else:
raise Exception('Unknown cell type: {}'.format(cell_type))
dropout = lambda: DropoutWrapper(get_cell(), input_keep_prob=self.keep_prob)
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.use_multi_layer_rnn = num_layers > 1
self.rnn_cell_fw = dropout()
if self.use_multi_layer_rnn:
self.rnn_cell_fw = [dropout() for _ in xrange(num_layers)]
self.rnn_cell_bw = dropout()
if self.use_multi_layer_rnn:
self.rnn_cell_bw = [dropout() for _ in xrange(num_layers)]
self.scope = scope
self.combiner = combiner
def __init__(self, hidden_size, keep_prob):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.rnn_cell_fw = tf.nn.rnn_cell.MultiRNNCell(
[rnn_cell.GRUCell(self.hidden_size) for i in range(2)])
self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw,
input_keep_prob=self.keep_prob)
self.rnn_cell_bw = tf.nn.rnn_cell.MultiRNNCell(
[rnn_cell.GRUCell(self.hidden_size) for i in range(2)])
self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw,
input_keep_prob=self.keep_prob)
def single_cell(num_units):
if cell_type == 'rnn':
cell_class = BasicRNNCell
elif cell_type == 'lstm':
cell_class = LSTMCell
elif cell_type == 'gru':
cell_class = GRUCell
else:
raise ValueError('Cell Type Not Supported! ')
if activation is not None:
if activation == 'relu':
activation_f = tf.nn.relu
elif activation == 'sigmoid':
activation_f = tf.sigmoid
elif activation == 'elu':
activation_f = tf.nn.elu
else:
raise ValueError('Activation Function Not Supported! ')
else:
activation_f = None
if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0:
return DropoutWrapper(cell_class(num_units=num_units,
activation=activation_f),
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob)
else:
return cell_class(num_units=num_units)
def __init__(self,
hidden_size,
keep_prob,
num_layers,
use_cudnn_lstm=False,
batch_size=None,
cudnn_dropout=None):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.use_cudnn_lstm = use_cudnn_lstm
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.num_layers = num_layers
self.cudnn_dropout = cudnn_dropout
if self.use_cudnn_lstm:
print('Using cudnn lstm')
self.direction = 'bidirectional'
self.cudnn_cell = cudnn_rnn.CudnnLSTM(self.num_layers,
self.hidden_size,
direction=self.direction,
dropout=cudnn_dropout)
else:
self.rnn_cell_fw = [
tf.contrib.rnn.LSTMCell(self.hidden_size,
name='lstmf' + str(i))
for i in range(num_layers)
]
self.rnn_cell_fw = [
DropoutWrapper(self.rnn_cell_fw[i],
input_keep_prob=self.keep_prob)
for i in range(num_layers)
]
self.rnn_cell_bw = [
tf.contrib.rnn.LSTMCell(self.hidden_size,
name='lstmb' + str(i))
for i in range(num_layers)
]
self.rnn_cell_bw = [
DropoutWrapper(self.rnn_cell_bw[i],
input_keep_prob=self.keep_prob)
for i in range(num_layers)
]
def single_cell(
cell_class,
cell_params,
dp_input_keep_prob=1.0,
dp_output_keep_prob=1.0,
recurrent_keep_prob=1.0,
input_weight_keep_prob=1.0,
recurrent_weight_keep_prob=1.0,
weight_variational=False,
dropout_seed=None,
zoneout_prob=0.0,
training=True,
residual_connections=False,
awd_initializer=False,
variational_recurrent=False, # in case they want to use DropoutWrapper
dtype=None,
):
"""Creates an instance of the rnn cell.
Such cell describes one step one layer and can include residual connection
and/or dropout
Args:
cell_class: Tensorflow RNN cell class
cell_params (dict): cell parameters
dp_input_keep_prob (float): (default: 1.0) input dropout keep
probability.
dp_output_keep_prob (float): (default: 1.0) output dropout keep
probability.
zoneout_prob(float): zoneout probability. Applying both zoneout and
droupout is currently not supported
residual_connections (bool): whether to add residual connection
Returns:
TF RNN instance
"""
if awd_initializer:
val = 1.0 / math.sqrt(cell_params['num_units'])
cell_params['initializer'] = tf.random_uniform_initializer(minval=-val,
maxval=val)
cell = cell_class(**cell_params)
if residual_connections:
cell = ResidualWrapper(cell)
if zoneout_prob > 0.0 and (dp_input_keep_prob < 1.0
or dp_output_keep_prob < 1.0):
raise ValueError(
'Currently applying both dropout and zoneout on the same cell.'
'This is currently not supported.')
if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0 and training:
cell = DropoutWrapper(
cell,
input_keep_prob=dp_input_keep_prob,
output_keep_prob=dp_output_keep_prob,
variational_recurrent=variational_recurrent,
dtype=dtype,
seed=dropout_seed,
)
if zoneout_prob > 0.0:
cell = ZoneoutWrapper(cell, zoneout_prob, is_training=training)
return cell
def _create_single_rnn_cell(self, num_units):
cell = GRUCell(
num_units) if self.cfg["cell_type"] == "gru" else LSTMCell(
num_units)
if self.cfg["use_dropout"]:
cell = DropoutWrapper(cell, output_keep_prob=self.rnn_keep_prob)
if self.cfg["use_residual"]:
cell = ResidualWrapper(cell)
return cell
def __init__(self, hidden_size, keep_prob):
"""
Inputs:
hidden_size: int. Hidden size of the RNN
keep_prob: Tensor containing a single scalar that is the keep probability (for dropout)
"""
self.hidden_size = hidden_size
self.keep_prob = keep_prob
self.g0_fwd = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size),
input_keep_prob=self.keep_prob)
self.g0_back = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size),
input_keep_prob=self.keep_prob)
self.g1_fwd = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size),
input_keep_prob=self.keep_prob)
self.g1_back = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size),
input_keep_prob=self.keep_prob)
self.multi_fwd = rnn_cell.MultiRNNCell([self.g0_fwd, self.g1_fwd])
self.multi_back = rnn_cell.MultiRNNCell([self.g0_back, self.g1_back])
