def __init__(self, hidden_size, keep_prob, cell_type="gru", scope="encoder"): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.hidden_size = hidden_size self.keep_prob = keep_prob if cell_type == "gru": rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size) rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size) elif cell_type == "lstm": rnn_cell_fw = rnn_cell.LSTMCell(self.hidden_size) rnn_cell_bw = rnn_cell.LSTMCell(self.hidden_size) else: assert (False, "No such cell type for RNN encoder!") self.rnn_cell_fw = DropoutWrapper(rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = DropoutWrapper(rnn_cell_bw, input_keep_prob=self.keep_prob) self.scope = scope logger.info("Encoder created: {} | hidden_size = {}".format( cell_type, hidden_size))
def add_conversational_context(self, sentence_summary_size): line_vectors_as_timesteps = tf.expand_dims(self.sentence_summaries, 0) if self.config.bidirectional_conversations: forwardcell = DropoutWrapper(GRUCell(sentence_summary_size), self.dropout_placeholder, self.dropout_placeholder) backwardcell = DropoutWrapper(GRUCell(sentence_summary_size), self.dropout_placeholder, self.dropout_placeholder) outputs, sf, sb = bidirectional_dynamic_rnn( forwardcell, backwardcell, line_vectors_as_timesteps, tf.slice(tf.shape(line_vectors_as_timesteps), [1], [1]), # what the f*****g f**k dtype=tf.float32, scope="ChapterRNN") self.conversation_state = tf.squeeze(outputs) return 2 * sentence_summary_size else: rnncell = DropoutWrapper(GRUCell(sentence_summary_size), self.dropout_placeholder, self.dropout_placeholder) outputs, state = tf.nn.dynamic_rnn( rnncell, line_vectors_as_timesteps, tf.slice(tf.shape(line_vectors_as_timesteps), [1], [1]), # what the f*****g f**k dtype=tf.float32, scope="ChapterRNN") self.conversation_state = tf.squeeze(outputs) return sentence_summary_size
def __init__(self, size, num_layers, tied_weights=True): self.size = size self.keep_prob = tf.placeholder(tf.float32) self.tied_weights = tied_weights # when we use different weights, we force the model to concatenate if not self.tied_weights: FLAGS.concat = True if FLAGS.rnn == "lstm": cell = rnn_cell.BasicLSTMCell(self.size) state_is_tuple = True else: cell = rnn_cell.GRUCell(self.size) state_is_tuple = False cell = DropoutWrapper(cell, input_keep_prob=self.keep_prob, seed=123) self.encoder_cell = tf.nn.rnn_cell.MultiRNNCell( [cell] * num_layers, state_is_tuple=state_is_tuple) if not tied_weights: if FLAGS.rnn == "lstm": cell_back = rnn_cell.BasicLSTMCell(self.size) state_is_tuple = True else: cell_back = rnn_cell.GRUCell(self.size) state_is_tuple = False cell_back = DropoutWrapper(cell_back, input_keep_prob=self.keep_prob, seed=123) self.encoder_cell_bw = tf.nn.rnn_cell.MultiRNNCell( [cell_back] * num_layers, state_is_tuple=state_is_tuple)
def __init__(self, hidden_size, keep_prob, num_layers): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.hidden_size = hidden_size self.keep_prob = keep_prob self.num_layers = num_layers self.rnn_cell_fw = [ rnn_cell.GRUCell(self.hidden_size) for _ in range(self.num_layers) ] self.rnn_cell_fw = [ DropoutWrapper(cell, input_keep_prob=self.keep_prob) for cell in self.rnn_cell_fw ] self.multi_rnn_cell_fw = rnn_cell.MultiRNNCell(self.rnn_cell_fw, state_is_tuple=False) self.rnn_cell_bw = [ rnn_cell.GRUCell(self.hidden_size) for _ in range(self.num_layers) ] self.rnn_cell_bw = [ DropoutWrapper(cell, input_keep_prob=self.keep_prob) for cell in self.rnn_cell_bw ] self.multi_rnn_cell_bw = rnn_cell.MultiRNNCell(self.rnn_cell_bw, state_is_tuple=False)
def add_sentence_summaries(self, wordvector_embed_size): if self.config.bidirectional_sentences: forwardcell = DropoutWrapper(GRUCell(wordvector_embed_size), self.dropout_placeholder, self.dropout_placeholder) backwardcell = DropoutWrapper(GRUCell(wordvector_embed_size), self.dropout_placeholder, self.dropout_placeholder) _, statefw, statebw = bidirectional_dynamic_rnn( forwardcell, backwardcell, self.embedded_lines, self.line_length_placeholder, dtype=tf.float32, scope="LineRNN") self.sentence_summaries = tf.concat(1, [statefw, statebw]) return 2 * wordvector_embed_size else: rnncell = DropoutWrapper(GRUCell(wordvector_embed_size), self.dropout_placeholder, self.dropout_placeholder) _, self.sentence_summaries = tf.nn.dynamic_rnn( rnncell, self.embedded_lines, self.line_length_placeholder, dtype=tf.float32, scope="LineRNN") return wordvector_embed_size
def __init__(self, hidden_size, keep_prob): self.hidden_size = hidden_size self.keep_prob = keep_prob self.rnn_cell_fw = tf.contrib.rnn.LSTMCell(self.hidden_size, forget_bias=1.0) self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = tf.contrib.rnn.LSTMCell(self.hidden_size, forget_bias=1.0) self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def __init__(self, hidden_size, keep_prob): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.hidden_size = hidden_size self.keep_prob = keep_prob #self.rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size) self.rnn_cell_fw_layer1 = rnn_cell.LSTMCell(self.hidden_size) self.rnn_cell_fw_layer1 = DropoutWrapper( self.rnn_cell_fw_layer1, input_keep_prob=self.keep_prob) #self.rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size) self.rnn_cell_bw_layer1 = rnn_cell.LSTMCell(self.hidden_size) self.rnn_cell_bw_layer1 = DropoutWrapper( self.rnn_cell_bw_layer1, input_keep_prob=self.keep_prob) self.rnn_cell_fw_layer2 = rnn_cell.LSTMCell(self.hidden_size) self.rnn_cell_fw_layer2 = DropoutWrapper( self.rnn_cell_fw_layer2, input_keep_prob=self.keep_prob) self.rnn_cell_bw_layer2 = rnn_cell.LSTMCell(self.hidden_size) self.rnn_cell_bw_layer2 = DropoutWrapper( self.rnn_cell_bw_layer2, input_keep_prob=self.keep_prob)
def __init__(self, hidden_size, keep_prob): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.hidden_size = hidden_size self.keep_prob = keep_prob # layer 1 self.rnn_cell_fw_1 = rnn_cell.BasicLSTMCell(self.hidden_size, forget_bias=1.0) self.rnn_cell_fw_1 = DropoutWrapper(self.rnn_cell_fw_1, input_keep_prob=self.keep_prob) self.rnn_cell_bw_1 = rnn_cell.BasicLSTMCell(self.hidden_size, forget_bias=1.0) self.rnn_cell_bw_1 = DropoutWrapper(self.rnn_cell_bw_1, input_keep_prob=self.keep_prob) # layer 2 self.rnn_cell_fw_2 = rnn_cell.BasicLSTMCell(self.hidden_size, forget_bias=1.0) self.rnn_cell_fw_2 = DropoutWrapper(self.rnn_cell_fw_2, input_keep_prob=self.keep_prob) self.rnn_cell_bw_2 = rnn_cell.BasicLSTMCell(self.hidden_size, forget_bias=1.0) self.rnn_cell_bw_2 = DropoutWrapper(self.rnn_cell_bw_2, input_keep_prob=self.keep_prob)
def __init__(self, hidden_size, keep_prob): self.hidden_size = hidden_size # this should be 2 * self.FLAGS.hidden_size self.keep_prob = keep_prob self.fwd = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size / 2), input_keep_prob=self.keep_prob) self.back = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size / 2), input_keep_prob=self.keep_prob)
def s2v(self): sqrt3 = math.sqrt(3.0) initializer = tf.random_uniform_initializer(-sqrt3, sqrt3, dtype=self.dtype) # word embedding layer if self.pre_trained_word_emb is not None: self.word_embeddings = tf.get_variable( name='word_embedding', initializer=self.pre_trained_word_emb, dtype=self.dtype) else: self.word_embeddings = tf.get_variable( name='word_embedding', shape=[self.voc_size, self.emb_size], initializer=initializer, dtype=self.dtype) self.embedded_sentence = tf.nn.embedding_lookup( self.word_embeddings, self.sentence) self.embedded_sentence = tf.nn.dropout( self.embedded_sentence, keep_prob=self.keep_word_prob_placeholder) # create the rnn cell if self.rnn_cell_type.lower() == 'gru': rnn_cell = GRUCell else: rnn_cell = LSTMCell rnn_cell = rnn_cell(self.hidden_units) if self.use_lstm_dropout: rnn_cell = DropoutWrapper( rnn_cell, dtype=tf.float32, output_keep_prob=self.keep_lstm_prob_placeholder) if self.rnn_model == 'leap-lstm': self.sentence_emb, self.skip_dis_output = self.leap_lstm(rnn_cell) elif self.rnn_model == 'rnn': if self.rnn_pattern == 1: self.sentence_emb = self.general_rnn(rnn_cell, out='LAST') else: self.sentence_emb = self.general_rnn_for_pattern( rnn_cell, out='LAST') # for test the training time elif self.rnn_model == 'brnn': self.sentence_emb = self.general_brnn() elif self.rnn_model == 'skip-rnn-2017': self.sentence_emb, self.budget_loss, self.updated_states, self.rnn_final_states, self.rnn_outputs = self.skip_rnn_2017( ) elif self.rnn_model == 'skim-rnn': small_rnn_cell = LSTMCell(5) # small size 5 small_rnn_cell = DropoutWrapper( small_rnn_cell, dtype=tf.float32, output_keep_prob=self.keep_lstm_prob_placeholder) self.sentence_emb, self.skip_dis_output, self.skim_loss = self.skim_rnn( rnn_cell, small_rnn_cell) # skim-rnn的设定直接按照github上源码来就可以了 else: print("bad rnn model!") exit()
def __init__(self, hidden_size, keep_prob): self.hidden_size = hidden_size self.keep_prob = keep_prob self.rnn_cell_fw = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_size) self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = tf.nn.rnn_cell.BasicLSTMCell(self.hidden_size) self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def getCell(self, is_training, dp, config): # code for RNN if is_training == True: print("==> Construct ", config.cell_type, " graph for training") else: print("==> Construct ", config.cell_type, " graph for testing") if config.cell_type == "LSTM": if config.num_layer == 1: basicCell = LSTMCell(config.hidden_size, forget_bias=0.0, state_is_tuple=True) elif config.num_layer == 2: basicCell = LSTMCell(config.hidden_size, forget_bias=0.0, state_is_tuple=True) basicCell_2 = LSTMCell(config.hidden_size_2, forget_bias=0.0, state_is_tuple=True) else: raise ValueError("config.num_layer should be 1:2 ") elif config.cell_type == "RNN": if config.num_layer == 1: basicCell = BasicRNNCell(config.hidden_size) elif config.num_layer == 2: basicCell = BasicRNNCell(config.hidden_size) basicCell_2 = BasicRNNCell(config.hidden_size_2) else: raise ValueError("config.num_layer should be [1-3] ") elif config.cell_type == "GRU": if config.num_layer == 1: basicCell = GRUCell(config.hidden_size, forget_bias=0.0, state_is_tuple=True) elif config.num_layer == 2: basicCell = GRUCell(config.hidden_size, forget_bias=0.0, state_is_tuple=True) basicCell_2 = GRUCell(config.hidden_size_2, forget_bias=0.0, state_is_tuple=True) else: raise ValueError("only support 1-2 layers ") else: raise ValueError("cell type should be GRU,LSTM,RNN") # add dropout layer between hidden layers if is_training and config.keep_prob < 1: if config.num_layer == 1: basicCell = DropoutWrapper(basicCell, input_keep_prob=config.keep_prob, output_keep_prob=config.keep_prob) elif config.num_layer == 2: basicCell = DropoutWrapper(basicCell, input_keep_prob=config.keep_prob, output_keep_prob=config.keep_prob) basicCell_2 = DropoutWrapper(basicCell_2, input_keep_prob=config.keep_prob, output_keep_prob=config.keep_prob) else: pass if config.num_layer == 1: cell = rnn_cell.MultiRNNCell([basicCell], state_is_tuple=True) elif config.num_layer == 2: cell = rnn_cell.MultiRNNCell([basicCell, basicCell_2], state_is_tuple=True) return cell
def inference_layer(self, inputs): if self.dblstm: with tf.name_scope('deep_bidirectional_rnn'): rnn_outputs, _ = deep_bidirectional_dynamic_rnn( [self._dblstm_cell() for _ in range(self.num_layers)], inputs, sequence_length=self.sequence_lengths) state_dim = self.state_dim else: cell_fw = DropoutWrapper(LSTMCell(num_units=self.state_dim), variational_recurrent=True, state_keep_prob=self.dropout_keep_prob, output_keep_prob=self.dropout_keep_prob, dtype=tf.float32) cell_bw = DropoutWrapper(LSTMCell(num_units=self.state_dim), variational_recurrent=True, state_keep_prob=self.dropout_keep_prob, output_keep_prob=self.dropout_keep_prob, dtype=tf.float32) with tf.name_scope('bidirectional_rnn'): rnn_outputs, _ = tf.nn.bidirectional_dynamic_rnn( cell_fw, cell_bw, inputs, sequence_length=self.sequence_lengths, dtype=tf.float32) rnn_outputs = tf.concat(rnn_outputs, 2) state_dim = self.state_dim * 2 with tf.name_scope('linear_projection'): softmax_weights = tf.get_variable( 'softmax_W', [state_dim, self.num_classes], initializer=tf.random_normal_initializer(0, 0.01)) softmax_bias = tf.get_variable('softmax_b', [self.num_classes], initializer=tf.zeros_initializer) time_steps = tf.shape(rnn_outputs)[1] rnn_outputs = tf.reshape( rnn_outputs, [-1, state_dim], name="flatten_rnn_outputs_for_linear_projection") logits = tf.nn.xw_plus_b(x=rnn_outputs, weights=softmax_weights, biases=softmax_bias, name="softmax_projection") self.scores = tf.reshape(logits, [-1, time_steps, self.num_classes], name="unflatten_logits") if self.crf: self.transition_params = tf.get_variable( "transitions", [self.num_classes, self.num_classes])
def __init__(self, hidden_size, keep_prob,model_name="RNNModelEncoder"): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.hidden_size = hidden_size self.keep_prob = keep_prob self.rnn_cell_fw = rnn_cell.LSTMCell(self.hidden_size) self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = rnn_cell.LSTMCell(self.hidden_size) self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob) self.model_name=model_name
def __init__(self, hidden_size, keep_prob): """ hidden_size: number of hidden size of RNN keep_prob: keep prob. for dropout """ self.hidden_size = hidden_size self.keep_prob = keep_prob self.fw_rnn = rnn_cell.GRUCell(self.hidden_size) self.fw_rnn = DropoutWrapper(self.fw_rnn, input_keep_prob=self.keep_prob) self.bw_rnn = rnn_cell.GRUCell(self.hidden_size) self.bw_rnn = DropoutWrapper(self.bw_rnn, input_keep_prob=self.keep_prob)
def build_graph(self): input_lens = tf.reduce_sum(self.X_mask_placeholder, axis=1) inputs = self.X_placeholder char_embedding = self.convolve() inputs = tf.concat([self.X_placeholder, char_embedding], axis=-1) inputs = tf.concat([inputs, self.features], axis=-1) for i in range(0, self.depth): lstm_cell_forward = rnn_cell.LSTMCell(self.hidden_size) lstm_cell_forward = DropoutWrapper(lstm_cell_forward, input_keep_prob=self.keep_prob) lstm_cell_backward = rnn_cell.LSTMCell(self.hidden_size) lstm_cell_backward = DropoutWrapper(lstm_cell_backward, input_keep_prob=self.keep_prob) (fw_out, bw_out), _ = tf.nn.bidirectional_dynamic_rnn( lstm_cell_forward, lstm_cell_backward, inputs, input_lens, dtype=tf.float32, scope='layer' + str(i)) out = tf.concat([fw_out, bw_out], 2) out = tf.nn.dropout(out, self.keep_prob) inputs = out h = tf.contrib.layers.fully_connected(out, num_outputs=self.hidden_size, activation_fn=tf.nn.relu) rows = tf.range(0, tf.shape(input_lens)[-1]) indices = tf.subtract(input_lens, tf.ones_like(input_lens)) indices = tf.nn.relu(indices) slicer = tf.stack([rows, indices], axis=1) h = tf.gather_nd(h, slicer) weights = tf.get_variable( "W", shape=[self.hidden_size, self.num_classes], initializer=tf.contrib.layers.xavier_initializer()) bias = tf.get_variable("b", shape=[self.num_classes], initializer=tf.zeros_initializer()) logits = tf.nn.xw_plus_b(h, weights, bias, name="logits") preds = tf.argmax(logits, 1) return logits, preds
def __init__(self, hidden_size, keep_prob, attention_mechanism=None, name="RNNEncoder"): with vs.variable_scope(name): self.hidden_size = hidden_size self.keep_prob = keep_prob self.rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size) if attention_mechanism is not None: self.rnn_cell_fw = AttentionWrapper(self.rnn_cell_fw, attention_mechanism) self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size) if attention_mechanism is not None: self.rnn_cell_bw = AttentionWrapper(self.rnn_cell_bw, attention_mechanism) self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob) self.name = name
def __init__(self, keep_prob, qn_vec_size, cxt_vec_size): """ Inputs: keep_prob: tensor containing a single scalar that is the keep probability (for dropout) qn_vec_size: size of the question vectors. int cxt_vec_size: size of the context vectors. int """ self.keep_prob = keep_prob self.qn_vec_size = qn_vec_size self.cxt_vec_size = cxt_vec_size self.rnn_cell_fw = rnn_cell.LSTMCell(cxt_vec_size/2, reuse=tf.AUTO_REUSE) self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = rnn_cell.LSTMCell(cxt_vec_size/2, reuse=tf.AUTO_REUSE) self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def __init__(self, keep_prob, key_vec_size, value_vec_size): """ Inputs: keep_prob: tensor containing a single scalar that is the keep probability (for dropout) key_vec_size: size of the key vectors. int value_vec_size: size of the value vectors. int """ self.keep_prob = keep_prob self.key_vec_size = key_vec_size self.value_vec_size = value_vec_size self.rnn_cell_fw = rnn_cell.LSTMCell(value_vec_size/2, reuse=tf.AUTO_REUSE) self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = rnn_cell.LSTMCell(value_vec_size/2, reuse=tf.AUTO_REUSE) self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def __init__(self, hidden_size, keep_prob, scope_name): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.hidden_size = hidden_size self.keep_prob = keep_prob self.rnn_cell_fw = rnn_cell.GRUCell(self.hidden_size) self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = rnn_cell.GRUCell(self.hidden_size) self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob) self.scope_name = scope_name
def __init__(self, hidden_size, keep_prob): #(self.FLAGS.hidden_size, self.keep_prob) """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.hidden_size = hidden_size self.keep_prob = keep_prob self.rnn_cell_fw = rnn_cell.BasicLSTMCell(self.hidden_size) self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = rnn_cell.BasicLSTMCell(self.hidden_size) self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def biLSTM(inputs, hiddenSize, keepProb, scopeName, mask=None): with vs.variable_scope(scopeName): fwCell = rnn_cell.LSTMCell(hiddenSize) fwCell = DropoutWrapper(fwCell, input_keep_prob=keepProb) bwCell = rnn_cell.LSTMCell(hiddenSize) bwCell = DropoutWrapper(bwCell, input_keep_prob=keepProb) if mask is not None: mask = tf.reduce_sum(mask, reduction_indices=1) (fwOut, bwOut), _ = tf.nn.bidirectional_dynamic_rnn(fwCell, bwCell, inputs, mask, dtype=tf.float32) out = tf.concat([fwOut, bwOut], 2) out = tf.nn.dropout(out, keepProb) return out
def single_cell(cell_class, cell_params, dp_input_keep_prob=1.0, dp_output_keep_prob=1.0, residual_connections=False): """Creates an instance of the rnn cell. Such cell describes one step one layer and can include residual connection and/or dropout Args: cell_class: Tensorflow RNN cell class cell_params (dict): cell parameters dp_input_keep_prob (float): (default: 1.0) input dropout keep probability dp_output_keep_prob (float): (default: 1.0) output dropout keep probability residual_connections (bool): whether to add residual connection Returns: TF RNN instance """ cell = cell_class(**cell_params) if residual_connections: cell = ResidualWrapper(cell) if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0: cell = DropoutWrapper(cell, input_keep_prob=dp_input_keep_prob, output_keep_prob=dp_output_keep_prob) return cell
def __init__(self, hidden_size, keep_prob, num_layers=1, cell_type='gru', scope='RNNEncoder', combiner='concat'): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ def get_cell(): if cell_type == 'gru': return rnn_cell.GRUCell(self.hidden_size) elif cell_type == 'lstm': return rnn_cell.LSTMCell(self.hidden_size) elif cell_type == 'layer_norm': return LayerNormBasicLSTMCell(self.hidden_size, dropout_keep_prob=keep_prob) else: raise Exception('Unknown cell type: {}'.format(cell_type)) dropout = lambda: DropoutWrapper(get_cell(), input_keep_prob=self.keep_prob) self.hidden_size = hidden_size self.keep_prob = keep_prob self.use_multi_layer_rnn = num_layers > 1 self.rnn_cell_fw = dropout() if self.use_multi_layer_rnn: self.rnn_cell_fw = [dropout() for _ in xrange(num_layers)] self.rnn_cell_bw = dropout() if self.use_multi_layer_rnn: self.rnn_cell_bw = [dropout() for _ in xrange(num_layers)] self.scope = scope self.combiner = combiner
def __init__(self, hidden_size, keep_prob): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.hidden_size = hidden_size self.keep_prob = keep_prob self.rnn_cell_fw = tf.nn.rnn_cell.MultiRNNCell( [rnn_cell.GRUCell(self.hidden_size) for i in range(2)]) self.rnn_cell_fw = DropoutWrapper(self.rnn_cell_fw, input_keep_prob=self.keep_prob) self.rnn_cell_bw = tf.nn.rnn_cell.MultiRNNCell( [rnn_cell.GRUCell(self.hidden_size) for i in range(2)]) self.rnn_cell_bw = DropoutWrapper(self.rnn_cell_bw, input_keep_prob=self.keep_prob)
def single_cell(num_units): if cell_type == 'rnn': cell_class = BasicRNNCell elif cell_type == 'lstm': cell_class = LSTMCell elif cell_type == 'gru': cell_class = GRUCell else: raise ValueError('Cell Type Not Supported! ') if activation is not None: if activation == 'relu': activation_f = tf.nn.relu elif activation == 'sigmoid': activation_f = tf.sigmoid elif activation == 'elu': activation_f = tf.nn.elu else: raise ValueError('Activation Function Not Supported! ') else: activation_f = None if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0: return DropoutWrapper(cell_class(num_units=num_units, activation=activation_f), input_keep_prob=dp_input_keep_prob, output_keep_prob=dp_output_keep_prob) else: return cell_class(num_units=num_units)
def __init__(self, hidden_size, keep_prob, num_layers, use_cudnn_lstm=False, batch_size=None, cudnn_dropout=None): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.use_cudnn_lstm = use_cudnn_lstm self.hidden_size = hidden_size self.keep_prob = keep_prob self.num_layers = num_layers self.cudnn_dropout = cudnn_dropout if self.use_cudnn_lstm: print('Using cudnn lstm') self.direction = 'bidirectional' self.cudnn_cell = cudnn_rnn.CudnnLSTM(self.num_layers, self.hidden_size, direction=self.direction, dropout=cudnn_dropout) else: self.rnn_cell_fw = [ tf.contrib.rnn.LSTMCell(self.hidden_size, name='lstmf' + str(i)) for i in range(num_layers) ] self.rnn_cell_fw = [ DropoutWrapper(self.rnn_cell_fw[i], input_keep_prob=self.keep_prob) for i in range(num_layers) ] self.rnn_cell_bw = [ tf.contrib.rnn.LSTMCell(self.hidden_size, name='lstmb' + str(i)) for i in range(num_layers) ] self.rnn_cell_bw = [ DropoutWrapper(self.rnn_cell_bw[i], input_keep_prob=self.keep_prob) for i in range(num_layers) ]
def single_cell( cell_class, cell_params, dp_input_keep_prob=1.0, dp_output_keep_prob=1.0, recurrent_keep_prob=1.0, input_weight_keep_prob=1.0, recurrent_weight_keep_prob=1.0, weight_variational=False, dropout_seed=None, zoneout_prob=0.0, training=True, residual_connections=False, awd_initializer=False, variational_recurrent=False, # in case they want to use DropoutWrapper dtype=None, ): """Creates an instance of the rnn cell. Such cell describes one step one layer and can include residual connection and/or dropout Args: cell_class: Tensorflow RNN cell class cell_params (dict): cell parameters dp_input_keep_prob (float): (default: 1.0) input dropout keep probability. dp_output_keep_prob (float): (default: 1.0) output dropout keep probability. zoneout_prob(float): zoneout probability. Applying both zoneout and droupout is currently not supported residual_connections (bool): whether to add residual connection Returns: TF RNN instance """ if awd_initializer: val = 1.0 / math.sqrt(cell_params['num_units']) cell_params['initializer'] = tf.random_uniform_initializer(minval=-val, maxval=val) cell = cell_class(**cell_params) if residual_connections: cell = ResidualWrapper(cell) if zoneout_prob > 0.0 and (dp_input_keep_prob < 1.0 or dp_output_keep_prob < 1.0): raise ValueError( 'Currently applying both dropout and zoneout on the same cell.' 'This is currently not supported.') if dp_input_keep_prob != 1.0 or dp_output_keep_prob != 1.0 and training: cell = DropoutWrapper( cell, input_keep_prob=dp_input_keep_prob, output_keep_prob=dp_output_keep_prob, variational_recurrent=variational_recurrent, dtype=dtype, seed=dropout_seed, ) if zoneout_prob > 0.0: cell = ZoneoutWrapper(cell, zoneout_prob, is_training=training) return cell
def _create_single_rnn_cell(self, num_units): cell = GRUCell( num_units) if self.cfg["cell_type"] == "gru" else LSTMCell( num_units) if self.cfg["use_dropout"]: cell = DropoutWrapper(cell, output_keep_prob=self.rnn_keep_prob) if self.cfg["use_residual"]: cell = ResidualWrapper(cell) return cell
def __init__(self, hidden_size, keep_prob): """ Inputs: hidden_size: int. Hidden size of the RNN keep_prob: Tensor containing a single scalar that is the keep probability (for dropout) """ self.hidden_size = hidden_size self.keep_prob = keep_prob self.g0_fwd = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size), input_keep_prob=self.keep_prob) self.g0_back = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size), input_keep_prob=self.keep_prob) self.g1_fwd = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size), input_keep_prob=self.keep_prob) self.g1_back = DropoutWrapper(rnn_cell.LSTMCell(self.hidden_size), input_keep_prob=self.keep_prob) self.multi_fwd = rnn_cell.MultiRNNCell([self.g0_fwd, self.g1_fwd]) self.multi_back = rnn_cell.MultiRNNCell([self.g0_back, self.g1_back])