def _build_embeddings_op(self):
tf.summary.scalar('dropout_keep_probability', self.keep_prob)
tf.summary.scalar("learning_rate", self.lr)
with tf.variable_scope('words'):
if self.cfg.use_pretrained:
_word_embeddings = tf.Variable(self.cfg.glove_embeddings, name='_word_embeddings', dtype=tf.float32,
trainable=self.cfg.finetune_emb)
else:
_word_embeddings = tf.get_variable(name='_word_embeddings', dtype=tf.float32, trainable=True,
shape=[self.cfg.word_vocab_size, self.cfg.word_dim])
word_embeddings = tf.nn.embedding_lookup(_word_embeddings, self.word_ids, name="word_embeddings")
with tf.variable_scope('char_rep_method'):
if self.cfg.use_char_emb:
_char_embeddings = tf.get_variable(name='_char_embeddings', dtype=tf.float32, trainable=True,
shape=[self.cfg.char_vocab_size, self.cfg.char_dim])
char_embeddings = tf.nn.embedding_lookup(_char_embeddings, self.char_ids, name="char_embeddings")
if self.cfg.char_rep_method == 'rnn':
char_rnn = BiRNN(self.cfg.num_units_char)
char_output = char_rnn(char_embeddings, self.word_lengths, return_last_state=True)
else: # cnn model for char representation
char_output = multi_conv1d(char_embeddings, self.cfg.filter_sizes, self.cfg.heights, "VALID",
self.is_train, self.keep_prob)
word_embeddings = tf.concat([word_embeddings, char_output], axis=-1)
if self.cfg.use_highway:
with tf.variable_scope("highway"):
self.word_embeddings = highway_network(word_embeddings, self.cfg.highway_num_layers, bias=True,
is_train=self.is_train, keep_prob=self.keep_prob)
else: # directly dropout before model_op
self.word_embeddings = dropout(word_embeddings, keep_prob=self.keep_prob, is_train=self.is_train)
print("embeddings shape: {}".format(self.word_embeddings.get_shape().as_list()))
def _build_embeddings_op(self):
tf.summary.scalar('dropout_keep_probability', self.keep_prob)
tf.summary.scalar("learning_rate", self.lr)
with tf.variable_scope('words'):
if self.cfg.use_pretrained:
_word_embeddings = tf.Variable(self.cfg.glove_embeddings,
name='_word_embeddings',
dtype=tf.float32,
trainable=self.cfg.finetune_emb)
else:
_word_embeddings = tf.get_variable(
name='_word_embeddings',
dtype=tf.float32,
trainable=True,
shape=[self.cfg.word_vocab_size, self.cfg.word_dim])
word_embeddings = tf.nn.embedding_lookup(_word_embeddings,
self.word_ids,
name="word_embeddings")
with tf.variable_scope('char_represent'):
if self.cfg.use_char_emb:
_char_embeddings = tf.get_variable(
name='_char_embeddings',
dtype=tf.float32,
trainable=True,
shape=[self.cfg.char_vocab_size, self.cfg.char_dim])
char_embeddings = tf.nn.embedding_lookup(
_char_embeddings, self.char_ids, name="char_embeddings")
s = tf.shape(
char_embeddings
) # [batch size, max length of sentence, max length of word, char_dim]
output = multi_conv1d(char_embeddings,
self.cfg.filter_sizes,
self.cfg.heights,
"VALID",
self.is_train,
self.keep_prob,
scope="char_cnn")
# shape = (batch size, max sentence length, char representation size)
self.char_output = tf.reshape(
output, [s[0], s[1], self.cfg.char_out_size])
word_embeddings = tf.concat(
[word_embeddings, self.char_output], axis=-1)
if self.cfg.use_highway:
with tf.variable_scope("highway"):
self.word_embeddings = highway_network(
word_embeddings,
self.cfg.highway_num_layers,
bias=True,
is_train=self.is_train,
keep_prob=self.keep_prob)
else: # directly dropout before model_op
self.word_embeddings = dropout(word_embeddings,
keep_prob=self.keep_prob,
is_train=self.is_train)
print('word embeddings shape: {}'.format(
self.word_embeddings.get_shape().as_list()))
def _build_embedding_op(self):
with tf.variable_scope("embeddings"):
if not self.cfg["use_pretrained"]:
self.word_embeddings = tf.get_variable(
name="emb",
dtype=tf.float32,
trainable=True,
shape=[self.word_vocab_size, self.cfg["emb_dim"]])
else:
self.word_embeddings = tf.Variable(
np.load(self.cfg["pretrained_emb"])["embeddings"],
name="emb",
dtype=tf.float32,
trainable=self.cfg["tuning_emb"])
word_emb = tf.nn.embedding_lookup(self.word_embeddings,
self.words,
name="word_emb")
print("word embedding shape: {}".format(
word_emb.get_shape().as_list()))
if self.cfg["use_chars"]:
self.char_embeddings = tf.get_variable(
name="c_emb",
dtype=tf.float32,
trainable=True,
shape=[self.char_vocab_size, self.cfg["char_emb_dim"]])
char_emb = tf.nn.embedding_lookup(self.char_embeddings,
self.chars,
name="chars_emb")
char_represent = multi_conv1d(char_emb,
self.cfg["filter_sizes"],
self.cfg["channel_sizes"],
drop_rate=self.drop_rate,
is_train=self.is_train)
print("chars representation shape: {}".format(
char_represent.get_shape().as_list()))
word_emb = tf.concat([word_emb, char_represent], axis=-1)
if self.cfg["use_highway"]:
self.word_emb = highway_network(word_emb,
self.cfg["highway_layers"],
use_bias=True,
bias_init=0.0,
keep_prob=self.keep_prob,
is_train=self.is_train)
else:
self.word_emb = tf.layers.dropout(word_emb,
rate=self.drop_rate,
training=self.is_train)
print("word and chars concatenation shape: {}".format(
self.word_emb.get_shape().as_list()))
def _build_embedding_op(self):
with tf.variable_scope("embeddings"):
if not self.cfg["use_pretrained"]:
self.word_embeddings = tf.get_variable(name="emb", dtype=tf.float32, trainable=True,
shape=[self.word_vocab_size, self.cfg["emb_dim"]])
else:
word_emb_1 = tf.Variable(np.load(self.cfg["pretrained_emb"])["embeddings"], name="word_emb_1",
dtype=tf.float32, trainable=self.cfg["tuning_emb"])
word_emb_2 = tf.get_variable(name="word_emb_2", shape=[3, self.cfg["emb_dim"]], dtype=tf.float32,
trainable=True) # For UNK, NUM and END
self.word_embeddings = tf.concat([word_emb_1, word_emb_2], axis=0)
word_emb = tf.nn.embedding_lookup(self.word_embeddings, self.words, name="word_emb")
print("word embedding shape: {}".format(word_emb.get_shape().as_list()))
if self.cfg["use_highway"]:
self.word_emb = highway_network(word_emb, self.cfg["highway_layers"], use_bias=True, bias_init=0.0,
keep_prob=self.keep_prob, is_train=self.is_train)
else:
self.word_emb = tf.layers.dropout(word_emb, rate=self.drop_rate, training=self.is_train)