def getLabelsEMA(self, article_index, query_index,
supporting_article_index, entities, identifier):
#Extract out label for this article (ie. is label correct)
labels = []
gold_entities = identifier[article_index]
new_entities = entities[article_index][query_index][
supporting_article_index]
orig_entities = entities[article_index][query_index][0]
for ind in range(len(gold_entities)):
ent = new_entities[ind].lower().strip()
orig_ent = orig_entities[ind].lower().strip()
gold = gold_entities[ind].lower().strip()
match = evaluatePrediction(ent, gold)
orig_match = evaluatePrediction(orig_ent, gold)
if match == 1:
labels.append(ind)
# if not orig_match == 1:
# labels.append(ind)
if set(labels) == set([0, 1, 2]):
labels = [3]
elif labels == []:
labels = [4]
assert (len(labels) > 0)
return labels
def getLabelsEMA(self, article_index, query_index, supporting_article_index, entities, identifier):
#Extract out label for this article (ie. is label correct)
labels = []
gold_entities = identifier[article_index]
new_entities = entities[article_index][query_index][supporting_article_index]
orig_entities = entities[article_index][query_index][0]
for ind in range(len(gold_entities)):
ent = new_entities[ind].lower().strip()
orig_ent = orig_entities[ind].lower().strip()
gold = gold_entities[ind].lower().strip()
match = evaluatePrediction(ent, gold)
orig_match = evaluatePrediction(orig_ent, gold)
if match == 1:
labels.append(ind)
# if not orig_match == 1:
# labels.append(ind)
if set(labels) == set([0, 1, 2]):
labels = [3]
elif labels == []:
labels = [4]
assert (len(labels) > 0)
return labels
def evaluateSansBaseline(self, predicted_identifiers, test_identifiers,
num_entities, COUNT_ZERO):
predicted_correct = [0.] * num_entities
total_predicted = [0.] * num_entities
total_gold = [0.] * num_entities
print 'len(test_identifiers)', len(test_identifiers)
print 'num_entities', num_entities
for entity_index in range(num_entities):
for article_index in range(len(predicted_identifiers)):
predicted = self.TEST_ENTITIES[article_index][0][0][
entity_index].strip().lower()
gold = test_identifiers[article_index][entity_index].strip(
).lower()
match = evaluatePrediction(predicted, gold)
if match == 'skip':
continue
else:
total_gold[entity_index] += 1
if match == "no_predict":
continue
if match == 1:
predicted_correct[entity_index] += 1
total_predicted[entity_index] += 1
helper.printScores(predicted_correct, total_predicted, total_gold)
def evaluateSansBaseline(self, predicted_identifiers, test_identifiers, num_entities, COUNT_ZERO):
predicted_correct = [0.] * num_entities
total_predicted = [0.] * num_entities
total_gold = [0.] * num_entities
print 'len(test_identifiers)', len(test_identifiers)
print 'num_entities', num_entities
for entity_index in range(num_entities):
for article_index in range(len(predicted_identifiers)):
predicted = self.TEST_ENTITIES[article_index][0][0][entity_index].strip().lower()
gold = test_identifiers[article_index][entity_index].strip().lower()
match = evaluatePrediction(predicted, gold)
if match == 'skip':
continue
else:
total_gold[entity_index] += 1
if match == "no_predict":
continue
if match == 1:
predicted_correct[entity_index] += 1
total_predicted[entity_index] += 1
helper.printScores(predicted_correct, total_predicted,total_gold)
