def add_global_voting_op(self): with tf.variable_scope("global_voting"): self.final_scores_before_global = - (1 - self.loss_mask) * 50 + self.final_scores gmask = tf.to_float(((self.final_scores_before_global - self.args.global_thr) >= 0)) # [b,s,30] masked_entity_emb = self.pure_entity_embeddings * tf.expand_dims(gmask, axis=3) # [b,s,30,300] * [b,s,30,1] batch_size = tf.shape(masked_entity_emb)[0] all_voters_emb = tf.reduce_sum(tf.reshape(masked_entity_emb, [batch_size, -1, 300]), axis=1, keep_dims=True) # [b, 1, 300] span_voters_emb = tf.reduce_sum(masked_entity_emb, axis=2) # [batch, num_of_spans, 300] valid_voters_emb = all_voters_emb - span_voters_emb # [b, 1, 300] - [batch, spans, 300] = [batch, spans, 300] (broadcasting) # [300] - [batch, spans, 300] = [batch, spans, 300] (broadcasting) valid_voters_emb = tf.nn.l2_normalize(valid_voters_emb, dim=2) self.global_voting_scores = tf.squeeze(tf.matmul(self.pure_entity_embeddings, tf.expand_dims(valid_voters_emb, axis=3)), axis=3) # [b,s,30,300] matmul [b,s,300,1] --> [b,s,30,1]-->[b,s,30] scalar_predictors = tf.stack([self.final_scores_before_global, self.global_voting_scores], 3) #print("scalar_predictors = ", scalar_predictors) #[b, s, 30, 2] with tf.variable_scope("psi_and_global_ffnn"): if self.args.global_score_ffnn[0] == 0: self.final_scores = util.projection(scalar_predictors, 1) else: hidden_layers, hidden_size = self.args.global_score_ffnn[0], self.args.global_score_ffnn[1] self.final_scores = util.ffnn(scalar_predictors, hidden_layers, hidden_size, 1, self.dropout if self.args.ffnn_dropout else None) # [batch, num_mentions, 30, 1] squeeze to [batch, num_mentions, 30] self.final_scores = tf.squeeze(self.final_scores, axis=3)
def add_cand_ent_scores_op(self): self.log_cand_entities_scores = tf.log( tf.minimum(1.0, tf.maximum(self.args.zero, self.cand_entities_scores))) stack_values = [] if self.args.nn_components.find("lstm") != -1: stack_values.append(self.similarity_scores) if self.args.nn_components.find("pem") != -1: stack_values.append(self.log_cand_entities_scores) if self.args.nn_components.find("attention") != -1: stack_values.append(self.attention_scores) scalar_predictors = tf.stack(stack_values, 3) #print("scalar_predictors = ", scalar_predictors) # [batch, num_mentions, 30, 3] with tf.variable_scope("similarity_and_prior_ffnn"): if self.args.final_score_ffnn[0] == 0: self.final_scores = util.projection( scalar_predictors, 1) # [batch, num_mentions, 30, 1] else: hidden_layers, hidden_size = self.args.final_score_ffnn[ 0], self.args.final_score_ffnn[1] self.final_scores = util.ffnn( scalar_predictors, hidden_layers, hidden_size, 1, self.dropout if self.args.ffnn_dropout else None) self.final_scores = tf.squeeze( self.final_scores, axis=3) # squeeze to [batch, num_mentions, 30]
def add_cand_ent_scores_op(self): # now add the cand_entity_scores maybe also some extra features and through a simple ffnn stack_values = [] if self.args.nn_components.find("lstm") != -1: stack_values.append(self.similarity_scores) if self.args.nn_components.find("pem") != -1: # TODO rename to pem_scores self.log_cand_entities_scores = self.custom_pem( self.args.pem_without_log, self.args.pem_buckets_boundaries) stack_values.append(self.log_cand_entities_scores) if self.args.nn_components.find("attention") != -1: stack_values.append(self.attention_scores) if len(stack_values) == 1: # since only one scalar omit the final ffnn self.final_scores = stack_values[0] return scalar_predictors = tf.stack(stack_values, 3) #print("scalar_predictors = ", scalar_predictors) #[batch, num_mentions, 30, 2] with tf.variable_scope("similarity_and_prior_ffnn"): if self.args.final_score_ffnn[0] == 0: self.final_scores = util.projection(scalar_predictors, 1, model=self) else: hidden_layers, hidden_size = self.args.final_score_ffnn[ 0], self.args.final_score_ffnn[1] self.final_scores = util.ffnn( scalar_predictors, hidden_layers, hidden_size, 1, self.dropout if self.args.ffnn_dropout else None, model=self) self.final_scores = tf.squeeze(self.final_scores, axis=3)
def add_lstm_score_op(self): #print("cand_entities = ", self.cand_entities) with tf.variable_scope("span_emb_ffnn"): # [batch, num_mentions, 300] # the span embedding can have different size depending on the chosen hyperparameters. We project it to 300 # dims to match the entity embeddings (formula 4) if self.args.span_emb_ffnn[0] == 0: span_emb_projected = util.projection(self.span_emb, 300, model=self) else: hidden_layers, hidden_size = self.args.span_emb_ffnn[ 0], self.args.span_emb_ffnn[1] span_emb_projected = util.ffnn( self.span_emb, hidden_layers, hidden_size, 300, self.dropout if self.args.ffnn_dropout else None, model=self) #print("span_emb_projected = ", span_emb_projected) # formula (6) <x^m, y_j> computation. this is the lstm score scores = tf.matmul(tf.expand_dims(span_emb_projected, 2), self.entity_embeddings, transpose_b=True) #print("scores = ", scores) self.similarity_scores = tf.squeeze( scores, axis=2) # [batch, num_mentions, 1, 30]
def add_lstm_score_op(self): #print("cand_entities = ", self.cand_entities) with tf.variable_scope("span_emb_ffnn"): # [batch, num_mentions, 300] if self.args.span_emb_ffnn[0] == 0: span_emb_projected = util.projection(self.span_emb, 300, model=self) else: hidden_layers, hidden_size = self.args.span_emb_ffnn[ 0], self.args.span_emb_ffnn[1] span_emb_projected = util.ffnn( self.span_emb, hidden_layers, hidden_size, 300, self.dropout if self.args.ffnn_dropout else None, model=self) #print("span_emb_projected = ", span_emb_projected) scores = tf.matmul(tf.expand_dims(span_emb_projected, 2), self.entity_embeddings, transpose_b=True) #print("scores = ", scores) self.similarity_scores = tf.squeeze( scores, axis=2) # [batch, num_mentions, 1, 30]
def add_lstm_score_op(self): with tf.variable_scope("span_emb_ffnn"): # [batch, num_mentions, 300] # the span embedding can have different size depending on the chosen hyperparameters. We project it to 300 # dims to match the entity embeddings (formula 4) if self.args.span_emb_ffnn[0] == 0: span_emb_projected = util.projection(self.span_emb, 256) else: hidden_layers, hidden_size = self.args.span_emb_ffnn[0], self.args.span_emb_ffnn[1] span_emb_projected = util.ffnn(self.span_emb, hidden_layers, hidden_size, 300, self.dropout if self.args.ffnn_dropout else None) #print("span_emb_projected = ", span_emb_projected) # formula (6) <x^m, y_j> computation. this is the lstm score coeffs = tf.nn.softmax(tf.matmul(span_emb_projected[:, :, None, None, :], self.entity_embeddings, transpose_b=True)) coeffs = tf.transpose(coeffs, [0, 1, 2, 4, 3]) ent_emb = tf.reduce_sum(coeffs * self.entity_embeddings, -2) scores = tf.matmul(tf.expand_dims(span_emb_projected, 2), ent_emb, transpose_b=True) #print("scores = ", scores) self.similarity_scores = tf.squeeze(scores, axis=2) # [batch, num_mentions, 1, 30]
def add_global_voting_op(self): with tf.variable_scope("global_voting"): self.final_scores_before_global = -( 1 - self.loss_mask) * 50 + self.final_scores if self.args.global_topkfromallspans: batch_num = tf.shape(self.final_scores)[0] spans_num = tf.shape(self.final_scores)[1] # num of spans cand_ent_num = tf.shape(self.final_scores)[2] # 30 new_size = spans_num * cand_ent_num temp = tf.diag(tf.ones([spans_num])) temp = tf.tile(tf.expand_dims(temp, axis=2), [1, 1, cand_ent_num]) temp = tf.reshape(temp, [spans_num, new_size]) mask = tf.reshape( tf.tile(tf.expand_dims(temp, axis=1), [1, cand_ent_num, 1]), [new_size, new_size]) mask = 1 - mask all_entities = tf.reshape(self.pure_entity_embeddings, [batch_num, new_size, 300]) all_scores = tf.matmul( all_entities, all_entities, transpose_b=True) # [batch, new_size, new_size] filtered_scores = all_scores * mask top_values, _ = tf.nn.top_k(filtered_scores, self.args.global_topkfromallspans) # [batch, new_size, K] if self.args.global_topkfromallspans_onlypositive: top_values = tf.maximum(top_values, self.args.zero) # so to avoid keeping cand ent that have score < of this value even if they are the self.global_voting_scores = tf.reduce_mean( top_values, axis=2) # [batch, new_size] self.global_voting_scores = tf.reshape( self.global_voting_scores, [batch_num, spans_num, cand_ent_num]) else: if self.args.global_gmask_unambigious: gmask = self._sequence_mask_v13( tf.equal(self.cand_entities_len, 1), tf.shape(self.final_scores)[2]) elif not self.args.global_topk: gmask = tf.to_float( ((self.final_scores_before_global - self.args.global_thr) >= 0)) # [b,s,30] else: top_values, _ = tf.nn.top_k( self.final_scores_before_global, self.args.global_topk) # [batch, num_of_spans, K] K_value = top_values[:, :, -1] # [batch, num_of_spans] #if hasattr(self.args, 'global_topkthr'): if self.args.global_topkthr: K_value = tf.maximum(self.args.global_topkthr, K_value) # so to avoid keeping cand ent that have score < of this value even if they are the # top for this span. 30 threshold = tf.tile( tf.expand_dims(K_value, 2), [1, 1, tf.shape(self.final_scores)[-1]]) # [batch, num_of_spans, 30] gmask = tf.to_float( ((self.final_scores_before_global - threshold) >= 0)) gmask = gmask * self.loss_mask if self.args.global_mask_scale_each_mention_voters_to_one: temp = tf.reduce_sum( gmask, axis=2, keep_dims=True) # [batch, num_of_spans, 1] temp = tf.where(tf.less(temp, 1e-4), temp, 1. / (temp + 1e-4)) gmask = gmask * temp elif self.args.global_gmask_based_on_localscore: gmask = gmask * tf.nn.softmax( self.final_scores_before_global) self.gmask = gmask masked_entity_emb = self.pure_entity_embeddings * tf.expand_dims( gmask, axis=3) # [b,s,30,300] * [b,s,30,1] batch_size = tf.shape(masked_entity_emb)[0] all_voters_emb = tf.reduce_sum(tf.reshape( masked_entity_emb, [batch_size, -1, 300]), axis=1, keep_dims=True) # [b, 1, 300] span_voters_emb = tf.reduce_sum( masked_entity_emb, axis=2) # [batch, num_of_spans, 300] valid_voters_emb = all_voters_emb - span_voters_emb # [b, 1, 300] - [batch, spans, 300] = [batch, spans, 300] (broadcasting) # [300] - [batch, spans, 300] = [batch, spans, 300] (broadcasting) if self.args.global_norm_or_mean == "norm": valid_voters_emb = tf.nn.l2_normalize(valid_voters_emb, dim=2) else: all_voters_num = tf.reduce_sum(gmask) # scalar span_voters_num = tf.reduce_sum(gmask, axis=2) # [batch, spans] valid_voters_emb = valid_voters_emb / tf.expand_dims( all_voters_num - span_voters_num, axis=2) self.global_voting_scores = tf.squeeze(tf.matmul( self.pure_entity_embeddings, tf.expand_dims(valid_voters_emb, axis=3)), axis=3) # [b,s,30,300] matmul [b,s,300,1] --> [b,s,30,1]-->[b,s,30] stack_values = [] if self.args.stage2_nn_components.find("pem") != -1: # TODO rename to pem_scores self.gpem_scores = self.custom_pem( self.args.gpem_without_log, self.args.gpem_buckets_boundaries) stack_values.append(self.gpem_scores) if self.args.stage2_nn_components.find("local") != -1: stack_values.append(self.final_scores_before_global) stack_values.append(self.global_voting_scores) scalar_predictors = tf.stack(stack_values, 3) #print("scalar_predictors = ", scalar_predictors) #[b, s, 30, 2] with tf.variable_scope("psi_and_global_ffnn"): if self.args.global_score_ffnn[0] == 0: self.final_scores = util.projection(scalar_predictors, 1, model=self) else: hidden_layers, hidden_size = self.args.global_score_ffnn[ 0], self.args.global_score_ffnn[1] self.final_scores = util.ffnn( scalar_predictors, hidden_layers, hidden_size, 1, self.dropout if self.args.ffnn_dropout else None, model=self) # [batch, num_mentions, 30, 1] squeeze to [batch, num_mentions, 30] self.final_scores = tf.squeeze(self.final_scores, axis=3)