def __init__(self, word_emb: np.ndarray, char_emb: np.ndarray, context_limit: int = 450, question_limit: int = 150, char_limit: int = 16, train_char_emb: bool = True, char_hidden_size: int = 100, encoder_hidden_size: int = 75, attention_hidden_size: int = 75, keep_prob: float = 0.7, min_learning_rate: float = 0.001, noans_token: bool = False, **kwargs) -> None: super().__init__(**kwargs) self.init_word_emb = word_emb self.init_char_emb = char_emb self.context_limit = context_limit self.question_limit = question_limit self.char_limit = char_limit self.train_char_emb = train_char_emb self.char_hidden_size = char_hidden_size self.hidden_size = encoder_hidden_size self.attention_hidden_size = attention_hidden_size self.keep_prob = keep_prob self.min_learning_rate = min_learning_rate self.noans_token = noans_token self.word_emb_dim = self.init_word_emb.shape[1] self.char_emb_dim = self.init_char_emb.shape[1] self.last_impatience = 0 self.lr_impatience = 0 if check_gpu_existence(): self.GRU = CudnnGRU else: self.GRU = CudnnCompatibleGRU self.sess_config = tf.ConfigProto(allow_soft_placement=True) self.sess_config.gpu_options.allow_growth = True self.sess = tf.Session(config=self.sess_config) self._init_graph() self._init_optimizer() self.sess.run(tf.global_variables_initializer()) # Try to load the model (if there are some model files the model will be loaded from them) if self.load_path is not None: self.load()
def cudnn_lstm_wrapper(units, n_hidden, n_layers=1, trainable_initial_states=None, seq_lengths=None, initial_h=None, initial_c=None, name='cudnn_lstm', reuse=False): if check_gpu_existence(): return cudnn_lstm(units, n_hidden, n_layers, trainable_initial_states, seq_lengths, initial_h, initial_c, name, reuse) log.info('\nWarning! tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell is used. ' 'It is okay for inference mode, but ' 'if you train your model with this cell it could NOT be used with ' 'tf.contrib.cudnn_rnn.CudnnLSTMCell later. ' ) return cudnn_compatible_lstm(units, n_hidden, n_layers, trainable_initial_states, seq_lengths, initial_h, initial_c, name, reuse)
def cudnn_lstm_wrapper(units, n_hidden, n_layers=1, trainable_initial_states=None, seq_lengths=None, initial_h=None, initial_c=None, name='cudnn_lstm', reuse=False): if check_gpu_existence(): return cudnn_lstm(units, n_hidden, n_layers, trainable_initial_states, seq_lengths, initial_h, initial_c, name, reuse) log.info('\nWarning! tf.contrib.cudnn_rnn.CudnnCompatibleLSTMCell is used. ' 'It is okay for inference mode, but ' 'if you train your model with this cell it could NOT be used with ' 'tf.contrib.cudnn_rnn.CudnnLSTMCell later. ' ) return cudnn_compatible_lstm(units, n_hidden, n_layers, trainable_initial_states, seq_lengths, initial_h, initial_c, name, reuse)
def _build_cudnn_rnn(self, units, n_hidden_list, cell_type, intra_layer_dropout, mask): if not check_gpu_existence(): raise RuntimeError('Usage of cuDNN RNN layers require GPU along with cuDNN library') sequence_lengths = tf.to_int32(tf.reduce_sum(mask, axis=1)) for n, n_hidden in enumerate(n_hidden_list): with tf.variable_scope(cell_type.upper() + '_' + str(n)): if cell_type.lower() == 'lstm': units, _ = cudnn_bi_lstm(units, n_hidden, sequence_lengths) elif cell_type.lower() == 'gru': units, _ = cudnn_bi_gru(units, n_hidden, sequence_lengths) else: raise RuntimeError('Wrong cell type "{}"! Only "gru" and "lstm"!'.format(cell_type)) units = tf.concat(units, -1) if intra_layer_dropout and n != len(n_hidden_list) - 1: units = variational_dropout(units, self._dropout_ph) return units
def __init__(self, word_emb: np.ndarray, char_emb: np.ndarray, context_limit: int = 450, question_limit: int = 150, char_limit: int = 16, train_char_emb: bool = True, char_hidden_size: int = 100, encoder_hidden_size: int = 75, attention_hidden_size: int = 75, keep_prob: float = 0.7, min_learning_rate: float = 0.001, noans_token: bool = False, **kwargs) -> None: super().__init__(**kwargs) self.init_word_emb = word_emb self.init_char_emb = char_emb self.context_limit = context_limit self.question_limit = question_limit self.char_limit = char_limit self.train_char_emb = train_char_emb self.char_hidden_size = char_hidden_size self.hidden_size = encoder_hidden_size self.attention_hidden_size = attention_hidden_size self.keep_prob = keep_prob self.min_learning_rate = min_learning_rate self.noans_token = noans_token self.word_emb_dim = self.init_word_emb.shape[1] self.char_emb_dim = self.init_char_emb.shape[1] self.last_impatience = 0 self.lr_impatience = 0 if check_gpu_existence(): self.GRU = CudnnGRU else: self.GRU = CudnnCompatibleGRU self.sess_config = tf.ConfigProto(allow_soft_placement=True) self.sess_config.gpu_options.allow_growth = True self.sess = tf.Session(config=self.sess_config) self._init_graph() self._init_optimizer() self.sess.run(tf.global_variables_initializer()) # Try to load the model (if there are some model files the model will be loaded from them) if self.load_path is not None: self.load()
def __init__(self, **kwargs): if not check_gpu_existence(): raise RuntimeError('SquadModel requires GPU') self.opt = deepcopy(kwargs) self.init_word_emb = self.opt['word_emb'] self.init_char_emb = self.opt['char_emb'] self.context_limit = self.opt['context_limit'] self.question_limit = self.opt['question_limit'] self.char_limit = self.opt['char_limit'] self.char_hidden_size = self.opt['char_hidden_size'] self.hidden_size = self.opt['encoder_hidden_size'] self.attention_hidden_size = self.opt['attention_hidden_size'] self.keep_prob = self.opt['keep_prob'] self.learning_rate = self.opt['learning_rate'] self.min_learning_rate = self.opt['min_learning_rate'] self.learning_rate_patience = self.opt['learning_rate_patience'] self.grad_clip = self.opt['grad_clip'] self.weight_decay = self.opt['weight_decay'] self.word_emb_dim = self.init_word_emb.shape[1] self.char_emb_dim = self.init_char_emb.shape[1] self.last_impatience = 0 self.lr_impatience = 0 self.sess_config = tf.ConfigProto(allow_soft_placement=True) self.sess_config.gpu_options.allow_growth = True self.sess = tf.Session(config=self.sess_config) self._init_graph() self._init_optimizer() self.sess.run(tf.global_variables_initializer()) super().__init__(**kwargs) # Try to load the model (if there are some model files the model will be loaded from them) if self.load_path is not None: self.load() if self.weight_decay < 1.0: self.sess.run(self.assign_vars)
def __init__(self, n_classes: int = 2, dropout_keep_prob: float = 0.5, return_probas: bool = False, **kwargs): """ Args: n_classes: number of classes for classification dropout_keep_prob: Probability of keeping the hidden state, values from 0 to 1. 0.5 works well in most cases. return_probas: whether to return confidences of the relation to be appropriate or not **kwargs: """ kwargs.setdefault('learning_rate_drop_div', 10.0) kwargs.setdefault('learning_rate_drop_patience', 5.0) kwargs.setdefault('clip_norm', 5.0) super().__init__(**kwargs) self.n_classes = n_classes self.dropout_keep_prob = dropout_keep_prob self.return_probas = return_probas config = tf.ConfigProto() config.gpu_options.allow_growth = True if check_gpu_existence(): self.GRU = CudnnGRU else: self.GRU = CudnnCompatibleGRU self.question_ph = tf.placeholder(tf.float32, [None, None, 300]) self.rel_emb_ph = tf.placeholder(tf.float32, [None, None, 300]) r_mask_2 = tf.cast(self.rel_emb_ph, tf.bool) r_len_2 = tf.reduce_sum(tf.cast(r_mask_2, tf.int32), axis=2) r_mask = tf.cast(r_len_2, tf.bool) r_len = tf.reduce_sum(tf.cast(r_mask, tf.int32), axis=1) rel_emb = tf.math.divide_no_nan(tf.reduce_sum(self.rel_emb_ph, axis=1), tf.cast(tf.expand_dims(r_len, axis=1), tf.float32)) self.y_ph = tf.placeholder(tf.int32, shape=(None,)) self.one_hot_labels = tf.one_hot(self.y_ph, depth=self.n_classes, dtype=tf.float32) self.keep_prob_ph = tf.placeholder_with_default(1.0, shape=[], name='keep_prob_ph') q_mask_2 = tf.cast(self.question_ph, tf.bool) q_len_2 = tf.reduce_sum(tf.cast(q_mask_2, tf.int32), axis=2) q_mask = tf.cast(q_len_2, tf.bool) q_len = tf.reduce_sum(tf.cast(q_mask, tf.int32), axis=1) question_dr = variational_dropout(self.question_ph, keep_prob=self.keep_prob_ph) b_size = tf.shape(self.question_ph)[0] with tf.variable_scope("question_encode"): rnn = self.GRU(num_layers=2, num_units=75, batch_size=b_size, input_size=300, keep_prob=self.keep_prob_ph) q = rnn(question_dr, seq_len=q_len) with tf.variable_scope("attention"): rel_emb_exp = tf.expand_dims(rel_emb, axis=1) dot_products = tf.reduce_sum(tf.multiply(q, rel_emb_exp), axis=2, keep_dims=False) s_mask = softmax_mask(dot_products, q_mask) att_weights = tf.expand_dims(tf.nn.softmax(s_mask), axis=2) self.s_r = tf.reduce_sum(tf.multiply(att_weights, q), axis=1) self.logits = tf.layers.dense(tf.multiply(self.s_r, rel_emb), 2, activation=None, use_bias=False) self.y_pred = tf.argmax(self.logits, axis=-1) loss_tensor = tf.nn.sigmoid_cross_entropy_with_logits(labels=self.one_hot_labels, logits=self.logits) self.loss = tf.reduce_mean(loss_tensor) self.train_op = self.get_train_op(self.loss) self.sess = tf.Session(config=config) self.sess.run(tf.global_variables_initializer()) self.load()
def __init__(self, embedder, tag_vocab, ner_vocab, pos_vocab, sess=None): # check gpu if not check_gpu_existence(): raise RuntimeError('Ontonotes NER model requires GPU with cuDNN!') n_hidden = (256, 256, 256) token_embeddings_dim = 100 n_tags = len(tag_vocab) # Create placeholders x_word = tf.placeholder(dtype=tf.float32, shape=[None, None, token_embeddings_dim], name='x_word') x_char = tf.placeholder(dtype=tf.int32, shape=[None, None, None], name='x_char') # Features x_pos = tf.placeholder(dtype=tf.float32, shape=[None, None, len(pos_vocab)], name='x_pos') # Senna x_ner = tf.placeholder(dtype=tf.float32, shape=[None, None, len(ner_vocab)], name='x_ner') # Senna x_capi = tf.placeholder(dtype=tf.float32, shape=[None, None], name='x_capi') y_true = tf.placeholder(dtype=tf.int32, shape=[None, None], name='y_tag') mask = tf.placeholder(dtype=tf.float32, shape=[None, None], name='mask') sequence_lengths = tf.reduce_sum(mask, axis=1) # Concat features to embeddings emb = tf.concat( [x_word, tf.expand_dims(x_capi, 2), x_pos, x_ner], axis=2) # The network units = emb for n, n_h in enumerate(n_hidden): with tf.variable_scope('RNN_' + str(n)): units, _ = cudnn_bi_lstm(units, n_h, tf.to_int32(sequence_lengths)) # Classifier with tf.variable_scope('Classifier'): units = tf.layers.dense(units, n_hidden[-1], kernel_initializer=xavier_initializer()) logits = tf.layers.dense(units, n_tags, kernel_initializer=xavier_initializer()) # CRF _, trainsition_params = tf.contrib.crf.crf_log_likelihood( logits, y_true, sequence_lengths) # Initialize session if sess is None: sess = tf.Session() self._ner_tagger = SennaNERTagger('download/senna/') self._pos_tagger = SennaChunkTagger('download/senna/') self._x_w = x_word self._x_c = x_char self._x_capi = x_capi self.x_pos = x_pos self.x_ner = x_ner self._y_true = y_true self._mask = mask self._sequence_lengths = sequence_lengths self._token_embeddings_dim = token_embeddings_dim self._pos_dict = pos_vocab self._ner_dict = ner_vocab self._tag_dict = tag_vocab self._logits = logits self._trainsition_params = trainsition_params self._sess = sess sess.run(tf.global_variables_initializer()) self._embedder = embedder