def add_global_voting_op(self):
with tf.variable_scope("global_voting"):
self.final_scores_before_global = - (1 - self.loss_mask) * 50 + self.final_scores
gmask = tf.to_float(((self.final_scores_before_global - self.args.global_thr) >= 0)) # [b,s,30]
masked_entity_emb = self.pure_entity_embeddings * tf.expand_dims(gmask, axis=3) # [b,s,30,300] * [b,s,30,1]
batch_size = tf.shape(masked_entity_emb)[0]
all_voters_emb = tf.reduce_sum(tf.reshape(masked_entity_emb, [batch_size, -1, 300]), axis=1,
keep_dims=True) # [b, 1, 300]
span_voters_emb = tf.reduce_sum(masked_entity_emb, axis=2) # [batch, num_of_spans, 300]
valid_voters_emb = all_voters_emb - span_voters_emb
# [b, 1, 300] - [batch, spans, 300] = [batch, spans, 300] (broadcasting)
# [300] - [batch, spans, 300] = [batch, spans, 300] (broadcasting)
valid_voters_emb = tf.nn.l2_normalize(valid_voters_emb, dim=2)
self.global_voting_scores = tf.squeeze(tf.matmul(self.pure_entity_embeddings, tf.expand_dims(valid_voters_emb, axis=3)), axis=3)
# [b,s,30,300] matmul [b,s,300,1] --> [b,s,30,1]-->[b,s,30]
scalar_predictors = tf.stack([self.final_scores_before_global, self.global_voting_scores], 3)
#print("scalar_predictors = ", scalar_predictors) #[b, s, 30, 2]
with tf.variable_scope("psi_and_global_ffnn"):
if self.args.global_score_ffnn[0] == 0:
self.final_scores = util.projection(scalar_predictors, 1)
else:
hidden_layers, hidden_size = self.args.global_score_ffnn[0], self.args.global_score_ffnn[1]
self.final_scores = util.ffnn(scalar_predictors, hidden_layers, hidden_size, 1,
self.dropout if self.args.ffnn_dropout else None)
# [batch, num_mentions, 30, 1] squeeze to [batch, num_mentions, 30]
self.final_scores = tf.squeeze(self.final_scores, axis=3)
def add_cand_ent_scores_op(self):
self.log_cand_entities_scores = tf.log(
tf.minimum(1.0,
tf.maximum(self.args.zero, self.cand_entities_scores)))
stack_values = []
if self.args.nn_components.find("lstm") != -1:
stack_values.append(self.similarity_scores)
if self.args.nn_components.find("pem") != -1:
stack_values.append(self.log_cand_entities_scores)
if self.args.nn_components.find("attention") != -1:
stack_values.append(self.attention_scores)
scalar_predictors = tf.stack(stack_values, 3)
#print("scalar_predictors = ", scalar_predictors) # [batch, num_mentions, 30, 3]
with tf.variable_scope("similarity_and_prior_ffnn"):
if self.args.final_score_ffnn[0] == 0:
self.final_scores = util.projection(
scalar_predictors, 1) # [batch, num_mentions, 30, 1]
else:
hidden_layers, hidden_size = self.args.final_score_ffnn[
0], self.args.final_score_ffnn[1]
self.final_scores = util.ffnn(
scalar_predictors, hidden_layers, hidden_size, 1,
self.dropout if self.args.ffnn_dropout else None)
self.final_scores = tf.squeeze(
self.final_scores,
axis=3) # squeeze to [batch, num_mentions, 30]
def add_cand_ent_scores_op(self):
# now add the cand_entity_scores maybe also some extra features and through a simple ffnn
stack_values = []
if self.args.nn_components.find("lstm") != -1:
stack_values.append(self.similarity_scores)
if self.args.nn_components.find("pem") != -1:
# TODO rename to pem_scores
self.log_cand_entities_scores = self.custom_pem(
self.args.pem_without_log, self.args.pem_buckets_boundaries)
stack_values.append(self.log_cand_entities_scores)
if self.args.nn_components.find("attention") != -1:
stack_values.append(self.attention_scores)
if len(stack_values) == 1:
# since only one scalar omit the final ffnn
self.final_scores = stack_values[0]
return
scalar_predictors = tf.stack(stack_values, 3)
#print("scalar_predictors = ", scalar_predictors) #[batch, num_mentions, 30, 2]
with tf.variable_scope("similarity_and_prior_ffnn"):
if self.args.final_score_ffnn[0] == 0:
self.final_scores = util.projection(scalar_predictors,
1,
model=self)
else:
hidden_layers, hidden_size = self.args.final_score_ffnn[
0], self.args.final_score_ffnn[1]
self.final_scores = util.ffnn(
scalar_predictors,
hidden_layers,
hidden_size,
1,
self.dropout if self.args.ffnn_dropout else None,
model=self)
self.final_scores = tf.squeeze(self.final_scores, axis=3)
def add_lstm_score_op(self):
#print("cand_entities = ", self.cand_entities)
with tf.variable_scope("span_emb_ffnn"):
# [batch, num_mentions, 300]
# the span embedding can have different size depending on the chosen hyperparameters. We project it to 300
# dims to match the entity embeddings (formula 4)
if self.args.span_emb_ffnn[0] == 0:
span_emb_projected = util.projection(self.span_emb,
300,
model=self)
else:
hidden_layers, hidden_size = self.args.span_emb_ffnn[
0], self.args.span_emb_ffnn[1]
span_emb_projected = util.ffnn(
self.span_emb,
hidden_layers,
hidden_size,
300,
self.dropout if self.args.ffnn_dropout else None,
model=self)
#print("span_emb_projected = ", span_emb_projected)
# formula (6) <x^m, y_j> computation. this is the lstm score
scores = tf.matmul(tf.expand_dims(span_emb_projected, 2),
self.entity_embeddings,
transpose_b=True)
#print("scores = ", scores)
self.similarity_scores = tf.squeeze(
scores, axis=2) # [batch, num_mentions, 1, 30]
def add_lstm_score_op(self):
#print("cand_entities = ", self.cand_entities)
with tf.variable_scope("span_emb_ffnn"):
# [batch, num_mentions, 300]
if self.args.span_emb_ffnn[0] == 0:
span_emb_projected = util.projection(self.span_emb,
300,
model=self)
else:
hidden_layers, hidden_size = self.args.span_emb_ffnn[
0], self.args.span_emb_ffnn[1]
span_emb_projected = util.ffnn(
self.span_emb,
hidden_layers,
hidden_size,
300,
self.dropout if self.args.ffnn_dropout else None,
model=self)
#print("span_emb_projected = ", span_emb_projected)
scores = tf.matmul(tf.expand_dims(span_emb_projected, 2),
self.entity_embeddings,
transpose_b=True)
#print("scores = ", scores)
self.similarity_scores = tf.squeeze(
scores, axis=2) # [batch, num_mentions, 1, 30]
def add_lstm_score_op(self):
with tf.variable_scope("span_emb_ffnn"):
# [batch, num_mentions, 300]
# the span embedding can have different size depending on the chosen hyperparameters. We project it to 300
# dims to match the entity embeddings (formula 4)
if self.args.span_emb_ffnn[0] == 0:
span_emb_projected = util.projection(self.span_emb, 256)
else:
hidden_layers, hidden_size = self.args.span_emb_ffnn[0], self.args.span_emb_ffnn[1]
span_emb_projected = util.ffnn(self.span_emb, hidden_layers, hidden_size, 300,
self.dropout if self.args.ffnn_dropout else None)
#print("span_emb_projected = ", span_emb_projected)
# formula (6) <x^m, y_j> computation. this is the lstm score
coeffs = tf.nn.softmax(tf.matmul(span_emb_projected[:, :, None, None, :], self.entity_embeddings, transpose_b=True))
coeffs = tf.transpose(coeffs, [0, 1, 2, 4, 3])
ent_emb = tf.reduce_sum(coeffs * self.entity_embeddings, -2)
scores = tf.matmul(tf.expand_dims(span_emb_projected, 2), ent_emb, transpose_b=True)
#print("scores = ", scores)
self.similarity_scores = tf.squeeze(scores, axis=2) # [batch, num_mentions, 1, 30]
def add_global_voting_op(self):
with tf.variable_scope("global_voting"):
self.final_scores_before_global = -(
1 - self.loss_mask) * 50 + self.final_scores
if self.args.global_topkfromallspans:
batch_num = tf.shape(self.final_scores)[0]
spans_num = tf.shape(self.final_scores)[1] # num of spans
cand_ent_num = tf.shape(self.final_scores)[2] # 30
new_size = spans_num * cand_ent_num
temp = tf.diag(tf.ones([spans_num]))
temp = tf.tile(tf.expand_dims(temp, axis=2),
[1, 1, cand_ent_num])
temp = tf.reshape(temp, [spans_num, new_size])
mask = tf.reshape(
tf.tile(tf.expand_dims(temp, axis=1),
[1, cand_ent_num, 1]), [new_size, new_size])
mask = 1 - mask
all_entities = tf.reshape(self.pure_entity_embeddings,
[batch_num, new_size, 300])
all_scores = tf.matmul(
all_entities, all_entities,
transpose_b=True) # [batch, new_size, new_size]
filtered_scores = all_scores * mask
top_values, _ = tf.nn.top_k(filtered_scores,
self.args.global_topkfromallspans)
# [batch, new_size, K]
if self.args.global_topkfromallspans_onlypositive:
top_values = tf.maximum(top_values, self.args.zero)
# so to avoid keeping cand ent that have score < of this value even if they are the
self.global_voting_scores = tf.reduce_mean(
top_values, axis=2) # [batch, new_size]
self.global_voting_scores = tf.reshape(
self.global_voting_scores,
[batch_num, spans_num, cand_ent_num])
else:
if self.args.global_gmask_unambigious:
gmask = self._sequence_mask_v13(
tf.equal(self.cand_entities_len, 1),
tf.shape(self.final_scores)[2])
elif not self.args.global_topk:
gmask = tf.to_float(
((self.final_scores_before_global -
self.args.global_thr) >= 0)) # [b,s,30]
else:
top_values, _ = tf.nn.top_k(
self.final_scores_before_global, self.args.global_topk)
# [batch, num_of_spans, K]
K_value = top_values[:, :, -1] # [batch, num_of_spans]
#if hasattr(self.args, 'global_topkthr'):
if self.args.global_topkthr:
K_value = tf.maximum(self.args.global_topkthr, K_value)
# so to avoid keeping cand ent that have score < of this value even if they are the
# top for this span. 30
threshold = tf.tile(
tf.expand_dims(K_value, 2),
[1, 1, tf.shape(self.final_scores)[-1]])
# [batch, num_of_spans, 30]
gmask = tf.to_float(
((self.final_scores_before_global - threshold) >= 0))
gmask = gmask * self.loss_mask
if self.args.global_mask_scale_each_mention_voters_to_one:
temp = tf.reduce_sum(
gmask, axis=2,
keep_dims=True) # [batch, num_of_spans, 1]
temp = tf.where(tf.less(temp, 1e-4), temp,
1. / (temp + 1e-4))
gmask = gmask * temp
elif self.args.global_gmask_based_on_localscore:
gmask = gmask * tf.nn.softmax(
self.final_scores_before_global)
self.gmask = gmask
masked_entity_emb = self.pure_entity_embeddings * tf.expand_dims(
gmask, axis=3) # [b,s,30,300] * [b,s,30,1]
batch_size = tf.shape(masked_entity_emb)[0]
all_voters_emb = tf.reduce_sum(tf.reshape(
masked_entity_emb, [batch_size, -1, 300]),
axis=1,
keep_dims=True) # [b, 1, 300]
span_voters_emb = tf.reduce_sum(
masked_entity_emb, axis=2) # [batch, num_of_spans, 300]
valid_voters_emb = all_voters_emb - span_voters_emb
# [b, 1, 300] - [batch, spans, 300] = [batch, spans, 300] (broadcasting)
# [300] - [batch, spans, 300] = [batch, spans, 300] (broadcasting)
if self.args.global_norm_or_mean == "norm":
valid_voters_emb = tf.nn.l2_normalize(valid_voters_emb,
dim=2)
else:
all_voters_num = tf.reduce_sum(gmask) # scalar
span_voters_num = tf.reduce_sum(gmask,
axis=2) # [batch, spans]
valid_voters_emb = valid_voters_emb / tf.expand_dims(
all_voters_num - span_voters_num, axis=2)
self.global_voting_scores = tf.squeeze(tf.matmul(
self.pure_entity_embeddings,
tf.expand_dims(valid_voters_emb, axis=3)),
axis=3)
# [b,s,30,300] matmul [b,s,300,1] --> [b,s,30,1]-->[b,s,30]
stack_values = []
if self.args.stage2_nn_components.find("pem") != -1:
# TODO rename to pem_scores
self.gpem_scores = self.custom_pem(
self.args.gpem_without_log,
self.args.gpem_buckets_boundaries)
stack_values.append(self.gpem_scores)
if self.args.stage2_nn_components.find("local") != -1:
stack_values.append(self.final_scores_before_global)
stack_values.append(self.global_voting_scores)
scalar_predictors = tf.stack(stack_values, 3)
#print("scalar_predictors = ", scalar_predictors) #[b, s, 30, 2]
with tf.variable_scope("psi_and_global_ffnn"):
if self.args.global_score_ffnn[0] == 0:
self.final_scores = util.projection(scalar_predictors,
1,
model=self)
else:
hidden_layers, hidden_size = self.args.global_score_ffnn[
0], self.args.global_score_ffnn[1]
self.final_scores = util.ffnn(
scalar_predictors,
hidden_layers,
hidden_size,
1,
self.dropout if self.args.ffnn_dropout else None,
model=self)
# [batch, num_mentions, 30, 1] squeeze to [batch, num_mentions, 30]
self.final_scores = tf.squeeze(self.final_scores, axis=3)
