def call(self, input_vectors, use_context): # input_vectors : [num_window, hidden_size] batch_size, seq_length, hidden_dim = bc.get_shape_list2(input_vectors) # Add position embedding input_vectors = bc.embedding_postprocessor2( input_tensor=input_vectors, token_type_table=self.token_type_table, full_position_embeddings=self.full_position_embeddings, use_token_type=False, token_type_ids=None, token_type_vocab_size=1, use_position_embeddings=True, max_position_embeddings=self.config.max_num_window, dropout_prob=self.config.hidden_dropout_prob) input_shape = [batch_size, seq_length] attention_mask = tf.ones([batch_size, seq_length, seq_length], tf.int32) * tf.expand_dims(use_context, 2) with tf.compat.v1.variable_scope("mid"): prev_output = bc.reshape_to_matrix(input_vectors) for layer_idx in range(self.n_layers): with tf.compat.v1.variable_scope("layer_%d" % layer_idx): intermediate_output, prev_output = self.layer_list[ layer_idx].apply(prev_output, batch_size, seq_length, attention_mask) final_output = bc.reshape_from_matrix2( prev_output, input_shape) self.all_layer_outputs.append(final_output) return prev_output
def apply(self, input_ids, segment_ids, initializer_range, vocab_size, hidden_size, type_vocab_size, max_position_embeddings, hidden_dropout_prob, use_one_hot_embeddings): initializer = bc.create_initializer(initializer_range) self.embedding_table = tf.compat.v1.get_variable( name="word_embeddings", shape=[vocab_size, hidden_size], initializer=initializer) self.token_type_table = tf.compat.v1.get_variable( name="token_type_embeddings", shape=[type_vocab_size, hidden_size], initializer=initializer) self.full_position_embeddings = tf.compat.v1.get_variable( name="position_embeddings", shape=[max_position_embeddings, hidden_size], initializer=initializer) # Perform embedding lookup on the word ids. (self.embedding_output, self.embedding_table) = bc.embedding_lookup2( input_ids=input_ids, embedding_table=self.embedding_table, vocab_size=vocab_size, embedding_size=hidden_size, use_one_hot_embeddings=use_one_hot_embeddings) # Add positional embeddings and token type embeddings, then layer # normalize and perform dropout. self.embedding_output = bc.embedding_postprocessor2( input_tensor=self.embedding_output, token_type_table=self.token_type_table, full_position_embeddings=self.full_position_embeddings, use_token_type=True, token_type_ids=segment_ids, token_type_vocab_size=type_vocab_size, use_position_embeddings=True, max_position_embeddings=max_position_embeddings, dropout_prob=hidden_dropout_prob) return self.embedding_output