def add_embedding(self): with tf.variable_scope("feature_lookup"): self.word_embedding_matrix = random_uniform_initializer(self.word_embeddings.shape, "word_embedding_matrix", 0.01, trainable=True) self.pos_embedding_matrix = random_uniform_initializer(self.pos_embeddings.shape, "pos_embedding_matrix", 0.01, trainable=True) self.dep_embedding_matrix = random_uniform_initializer(self.dep_embeddings.shape, "dep_embedding_matrix", 0.01, trainable=True) word_context_embeddings = tf.nn.embedding_lookup(self.word_embedding_matrix, self.word_input_placeholder) pos_context_embeddings = tf.nn.embedding_lookup(self.pos_embedding_matrix, self.pos_input_placeholder) dep_context_embeddings = tf.nn.embedding_lookup(self.dep_embedding_matrix, self.dep_input_placeholder) word_embeddings = tf.reshape(word_context_embeddings, [-1, self.config.word_features_types * self.config.embedding_dim], name="word_context_embeddings") pos_embeddings = tf.reshape(pos_context_embeddings, [-1, self.config.pos_features_types * self.config.embedding_dim], name="pos_context_embeddings") dep_embeddings = tf.reshape(dep_context_embeddings, [-1, self.config.dep_features_types * self.config.embedding_dim], name="dep_context_embeddings") with tf.variable_scope("batch_inputs"): embeddings = tf.concat([word_embeddings, pos_embeddings, dep_embeddings], 1, name="batch_feature_matrix") return embeddings, word_embeddings, pos_embeddings, dep_embeddings
def add_embedding(self): with tf.variable_scope("feature_lookup"): self.word_embedding_matrix = random_uniform_initializer( self.word_embeddings.shape, "word_embedding_matrix", 0.01, trainable=True) self.char_embedding_matrix = random_uniform_initializer( self.char_embeddings.shape, "char_embedding_matrix", 0.01, trainable=True) word_context_embeddings = tf.nn.embedding_lookup( self.word_embedding_matrix, self.word_input_placeholder) char_context_embeddings = tf.nn.embedding_lookup( self.char_embedding_matrix, self.char_input_placeholder) # temp = tf.cond(self.is_training, True, False) if self.config.add_gaussian_noise is True and self.is_training is not None: word_context_embeddings = tf.cond( self.is_training, lambda: self.gaussian_noise_layer( word_context_embeddings, 0.1), lambda: tf.identity(word_context_embeddings)) char_context_embeddings = tf.cond( self.is_training, lambda: self.gaussian_noise_layer( char_context_embeddings, 0.01), lambda: tf.identity(char_context_embeddings)) # [N, H, W] -> [N, H, W, C] word_context_embeddings_expanded = tf.expand_dims( word_context_embeddings, -1) char_context_embeddings_expanded = tf.expand_dims( char_context_embeddings, -1) return word_context_embeddings, word_context_embeddings_expanded, char_context_embeddings, char_context_embeddings_expanded
def add_cube_prediction_op(self): print "***Building network with CUBE activation***" _, word_embeddings, pos_embeddings, dep_embeddings = self.add_embedding() with tf.variable_scope("layer_connections"): with tf.variable_scope("layer_1"): w11 = random_uniform_initializer((self.config.word_features_types * self.config.embedding_dim, self.config.l1_hidden_size), "w11", 0.01, trainable=True) w12 = random_uniform_initializer((self.config.pos_features_types * self.config.embedding_dim, self.config.l1_hidden_size), "w12", 0.01, trainable=True) w13 = random_uniform_initializer((self.config.dep_features_types * self.config.embedding_dim, self.config.l1_hidden_size), "w13", 0.01, trainable=True) b1 = random_uniform_initializer((self.config.l1_hidden_size,), "bias1", 0.01, trainable=True) """ w11 = xavier_initializer((self.config.word_features_types * self.config.embedding_dim, self.config.l1_hidden_size), "w11") w12 = xavier_initializer((self.config.pos_features_types * self.config.embedding_dim, self.config.l1_hidden_size), "w12") w13 = xavier_initializer((self.config.dep_features_types * self.config.embedding_dim, self.config.l1_hidden_size), "w13") b1 = xavier_initializer((self.config.l1_hidden_size,), "bias1") """ # for visualization preactivations = tf.pow(tf.add_n([tf.matmul(word_embeddings, w11), tf.matmul(pos_embeddings, w12), tf.matmul(dep_embeddings, w13)]) + b1, 3, name="preactivations") tf.summary.histogram("preactivations", preactivations) # non_positive_activation_fraction = tf.reduce_mean(tf.cast(tf.less_equal(preactivations, 0), # tf.float32)) # tf.summary.scalar("non_positive_activations_fraction", non_positive_activation_fraction) h1 = tf.nn.dropout(preactivations, keep_prob=self.dropout_placeholder, name="output_activations") with tf.variable_scope("layer_2"): """ w2 = xavier_initializer((self.config.l1_hidden_size, self.config.l2_hidden_size), "w2") b2 = xavier_initializer((self.config.l2_hidden_size,), "bias2") """ w2 = random_uniform_initializer((self.config.l1_hidden_size, self.config.l2_hidden_size), "w2", 0.01, trainable=True) b2 = random_uniform_initializer((self.config.l2_hidden_size,), "bias2", 0.01, trainable=True) h2 = tf.nn.relu(tf.add(tf.matmul(h1, w2), b2), name="activations") with tf.variable_scope("layer_3"): """ w3 = xavier_initializer((self.config.l2_hidden_size, self.config.num_classes), "w3") b3 = xavier_initializer((self.config.num_classes,), "bias3") """ w3 = random_uniform_initializer((self.config.l2_hidden_size, self.config.num_classes), "w3", 0.01, trainable=True) b3 = random_uniform_initializer((self.config.num_classes,), "bias3", 0.01, trainable=True) with tf.variable_scope("predictions"): predictions = tf.add(tf.matmul(h2, w3), b3, name="prediction_logits") return predictions