def _build_embeddings_op(self): tf.summary.scalar('dropout_keep_probability', self.keep_prob) tf.summary.scalar("learning_rate", self.lr) with tf.variable_scope('words'): if self.cfg.use_pretrained: _word_embeddings = tf.Variable(self.cfg.glove_embeddings, name='_word_embeddings', dtype=tf.float32, trainable=self.cfg.finetune_emb) else: _word_embeddings = tf.get_variable(name='_word_embeddings', dtype=tf.float32, trainable=True, shape=[self.cfg.word_vocab_size, self.cfg.word_dim]) word_embeddings = tf.nn.embedding_lookup(_word_embeddings, self.word_ids, name="word_embeddings") with tf.variable_scope('char_rep_method'): if self.cfg.use_char_emb: _char_embeddings = tf.get_variable(name='_char_embeddings', dtype=tf.float32, trainable=True, shape=[self.cfg.char_vocab_size, self.cfg.char_dim]) char_embeddings = tf.nn.embedding_lookup(_char_embeddings, self.char_ids, name="char_embeddings") if self.cfg.char_rep_method == 'rnn': char_rnn = BiRNN(self.cfg.num_units_char) char_output = char_rnn(char_embeddings, self.word_lengths, return_last_state=True) else: # cnn model for char representation char_output = multi_conv1d(char_embeddings, self.cfg.filter_sizes, self.cfg.heights, "VALID", self.is_train, self.keep_prob) word_embeddings = tf.concat([word_embeddings, char_output], axis=-1) if self.cfg.use_highway: with tf.variable_scope("highway"): self.word_embeddings = highway_network(word_embeddings, self.cfg.highway_num_layers, bias=True, is_train=self.is_train, keep_prob=self.keep_prob) else: # directly dropout before model_op self.word_embeddings = dropout(word_embeddings, keep_prob=self.keep_prob, is_train=self.is_train) print("embeddings shape: {}".format(self.word_embeddings.get_shape().as_list()))
def _build_embeddings_op(self): tf.summary.scalar('dropout_keep_probability', self.keep_prob) tf.summary.scalar("learning_rate", self.lr) with tf.variable_scope('words'): if self.cfg.use_pretrained: _word_embeddings = tf.Variable(self.cfg.glove_embeddings, name='_word_embeddings', dtype=tf.float32, trainable=self.cfg.finetune_emb) else: _word_embeddings = tf.get_variable( name='_word_embeddings', dtype=tf.float32, trainable=True, shape=[self.cfg.word_vocab_size, self.cfg.word_dim]) word_embeddings = tf.nn.embedding_lookup(_word_embeddings, self.word_ids, name="word_embeddings") with tf.variable_scope('char_represent'): if self.cfg.use_char_emb: _char_embeddings = tf.get_variable( name='_char_embeddings', dtype=tf.float32, trainable=True, shape=[self.cfg.char_vocab_size, self.cfg.char_dim]) char_embeddings = tf.nn.embedding_lookup( _char_embeddings, self.char_ids, name="char_embeddings") s = tf.shape( char_embeddings ) # [batch size, max length of sentence, max length of word, char_dim] output = multi_conv1d(char_embeddings, self.cfg.filter_sizes, self.cfg.heights, "VALID", self.is_train, self.keep_prob, scope="char_cnn") # shape = (batch size, max sentence length, char representation size) self.char_output = tf.reshape( output, [s[0], s[1], self.cfg.char_out_size]) word_embeddings = tf.concat( [word_embeddings, self.char_output], axis=-1) if self.cfg.use_highway: with tf.variable_scope("highway"): self.word_embeddings = highway_network( word_embeddings, self.cfg.highway_num_layers, bias=True, is_train=self.is_train, keep_prob=self.keep_prob) else: # directly dropout before model_op self.word_embeddings = dropout(word_embeddings, keep_prob=self.keep_prob, is_train=self.is_train) print('word embeddings shape: {}'.format( self.word_embeddings.get_shape().as_list()))
def _build_embedding_op(self): with tf.variable_scope("embeddings"): if not self.cfg["use_pretrained"]: self.word_embeddings = tf.get_variable( name="emb", dtype=tf.float32, trainable=True, shape=[self.word_vocab_size, self.cfg["emb_dim"]]) else: self.word_embeddings = tf.Variable( np.load(self.cfg["pretrained_emb"])["embeddings"], name="emb", dtype=tf.float32, trainable=self.cfg["tuning_emb"]) word_emb = tf.nn.embedding_lookup(self.word_embeddings, self.words, name="word_emb") print("word embedding shape: {}".format( word_emb.get_shape().as_list())) if self.cfg["use_chars"]: self.char_embeddings = tf.get_variable( name="c_emb", dtype=tf.float32, trainable=True, shape=[self.char_vocab_size, self.cfg["char_emb_dim"]]) char_emb = tf.nn.embedding_lookup(self.char_embeddings, self.chars, name="chars_emb") char_represent = multi_conv1d(char_emb, self.cfg["filter_sizes"], self.cfg["channel_sizes"], drop_rate=self.drop_rate, is_train=self.is_train) print("chars representation shape: {}".format( char_represent.get_shape().as_list())) word_emb = tf.concat([word_emb, char_represent], axis=-1) if self.cfg["use_highway"]: self.word_emb = highway_network(word_emb, self.cfg["highway_layers"], use_bias=True, bias_init=0.0, keep_prob=self.keep_prob, is_train=self.is_train) else: self.word_emb = tf.layers.dropout(word_emb, rate=self.drop_rate, training=self.is_train) print("word and chars concatenation shape: {}".format( self.word_emb.get_shape().as_list()))