def add_embedding(self):
with tf.variable_scope("feature_lookup"):
self.word_embedding_matrix = random_uniform_initializer(self.word_embeddings.shape, "word_embedding_matrix",
0.01, trainable=True)
self.pos_embedding_matrix = random_uniform_initializer(self.pos_embeddings.shape, "pos_embedding_matrix",
0.01, trainable=True)
self.dep_embedding_matrix = random_uniform_initializer(self.dep_embeddings.shape, "dep_embedding_matrix",
0.01, trainable=True)
word_context_embeddings = tf.nn.embedding_lookup(self.word_embedding_matrix, self.word_input_placeholder)
pos_context_embeddings = tf.nn.embedding_lookup(self.pos_embedding_matrix, self.pos_input_placeholder)
dep_context_embeddings = tf.nn.embedding_lookup(self.dep_embedding_matrix, self.dep_input_placeholder)
word_embeddings = tf.reshape(word_context_embeddings,
[-1, self.config.word_features_types * self.config.embedding_dim],
name="word_context_embeddings")
pos_embeddings = tf.reshape(pos_context_embeddings,
[-1, self.config.pos_features_types * self.config.embedding_dim],
name="pos_context_embeddings")
dep_embeddings = tf.reshape(dep_context_embeddings,
[-1, self.config.dep_features_types * self.config.embedding_dim],
name="dep_context_embeddings")
with tf.variable_scope("batch_inputs"):
embeddings = tf.concat([word_embeddings, pos_embeddings, dep_embeddings], 1, name="batch_feature_matrix")
return embeddings, word_embeddings, pos_embeddings, dep_embeddings
def add_embedding(self):
with tf.variable_scope("feature_lookup"):
self.word_embedding_matrix = random_uniform_initializer(
self.word_embeddings.shape,
"word_embedding_matrix",
0.01,
trainable=True)
self.char_embedding_matrix = random_uniform_initializer(
self.char_embeddings.shape,
"char_embedding_matrix",
0.01,
trainable=True)
word_context_embeddings = tf.nn.embedding_lookup(
self.word_embedding_matrix, self.word_input_placeholder)
char_context_embeddings = tf.nn.embedding_lookup(
self.char_embedding_matrix, self.char_input_placeholder)
# temp = tf.cond(self.is_training, True, False)
if self.config.add_gaussian_noise is True and self.is_training is not None:
word_context_embeddings = tf.cond(
self.is_training, lambda: self.gaussian_noise_layer(
word_context_embeddings, 0.1),
lambda: tf.identity(word_context_embeddings))
char_context_embeddings = tf.cond(
self.is_training, lambda: self.gaussian_noise_layer(
char_context_embeddings, 0.01),
lambda: tf.identity(char_context_embeddings))
# [N, H, W] -> [N, H, W, C]
word_context_embeddings_expanded = tf.expand_dims(
word_context_embeddings, -1)
char_context_embeddings_expanded = tf.expand_dims(
char_context_embeddings, -1)
return word_context_embeddings, word_context_embeddings_expanded, char_context_embeddings, char_context_embeddings_expanded
def add_cube_prediction_op(self):
print "***Building network with CUBE activation***"
_, word_embeddings, pos_embeddings, dep_embeddings = self.add_embedding()
with tf.variable_scope("layer_connections"):
with tf.variable_scope("layer_1"):
w11 = random_uniform_initializer((self.config.word_features_types * self.config.embedding_dim,
self.config.l1_hidden_size), "w11",
0.01, trainable=True)
w12 = random_uniform_initializer((self.config.pos_features_types * self.config.embedding_dim,
self.config.l1_hidden_size), "w12",
0.01, trainable=True)
w13 = random_uniform_initializer((self.config.dep_features_types * self.config.embedding_dim,
self.config.l1_hidden_size), "w13",
0.01, trainable=True)
b1 = random_uniform_initializer((self.config.l1_hidden_size,), "bias1",
0.01, trainable=True)
"""
w11 = xavier_initializer((self.config.word_features_types * self.config.embedding_dim,
self.config.l1_hidden_size), "w11")
w12 = xavier_initializer((self.config.pos_features_types * self.config.embedding_dim,
self.config.l1_hidden_size), "w12")
w13 = xavier_initializer((self.config.dep_features_types * self.config.embedding_dim,
self.config.l1_hidden_size), "w13")
b1 = xavier_initializer((self.config.l1_hidden_size,), "bias1")
"""
# for visualization
preactivations = tf.pow(tf.add_n([tf.matmul(word_embeddings, w11),
tf.matmul(pos_embeddings, w12),
tf.matmul(dep_embeddings, w13)]) + b1, 3, name="preactivations")
tf.summary.histogram("preactivations", preactivations)
# non_positive_activation_fraction = tf.reduce_mean(tf.cast(tf.less_equal(preactivations, 0),
# tf.float32))
# tf.summary.scalar("non_positive_activations_fraction", non_positive_activation_fraction)
h1 = tf.nn.dropout(preactivations,
keep_prob=self.dropout_placeholder,
name="output_activations")
with tf.variable_scope("layer_2"):
"""
w2 = xavier_initializer((self.config.l1_hidden_size, self.config.l2_hidden_size), "w2")
b2 = xavier_initializer((self.config.l2_hidden_size,), "bias2")
"""
w2 = random_uniform_initializer((self.config.l1_hidden_size, self.config.l2_hidden_size), "w2",
0.01, trainable=True)
b2 = random_uniform_initializer((self.config.l2_hidden_size,), "bias2",
0.01, trainable=True)
h2 = tf.nn.relu(tf.add(tf.matmul(h1, w2), b2), name="activations")
with tf.variable_scope("layer_3"):
"""
w3 = xavier_initializer((self.config.l2_hidden_size, self.config.num_classes), "w3")
b3 = xavier_initializer((self.config.num_classes,), "bias3")
"""
w3 = random_uniform_initializer((self.config.l2_hidden_size, self.config.num_classes), "w3",
0.01, trainable=True)
b3 = random_uniform_initializer((self.config.num_classes,), "bias3", 0.01, trainable=True)
with tf.variable_scope("predictions"):
predictions = tf.add(tf.matmul(h2, w3), b3, name="prediction_logits")
return predictions
