def getLabelsEMA(self, article_index, query_index, supporting_article_index, entities, identifier): #Extract out label for this article (ie. is label correct) labels = [] gold_entities = identifier[article_index] new_entities = entities[article_index][query_index][ supporting_article_index] orig_entities = entities[article_index][query_index][0] for ind in range(len(gold_entities)): ent = new_entities[ind].lower().strip() orig_ent = orig_entities[ind].lower().strip() gold = gold_entities[ind].lower().strip() match = evaluatePrediction(ent, gold) orig_match = evaluatePrediction(orig_ent, gold) if match == 1: labels.append(ind) # if not orig_match == 1: # labels.append(ind) if set(labels) == set([0, 1, 2]): labels = [3] elif labels == []: labels = [4] assert (len(labels) > 0) return labels
def getLabelsEMA(self, article_index, query_index, supporting_article_index, entities, identifier): #Extract out label for this article (ie. is label correct) labels = [] gold_entities = identifier[article_index] new_entities = entities[article_index][query_index][supporting_article_index] orig_entities = entities[article_index][query_index][0] for ind in range(len(gold_entities)): ent = new_entities[ind].lower().strip() orig_ent = orig_entities[ind].lower().strip() gold = gold_entities[ind].lower().strip() match = evaluatePrediction(ent, gold) orig_match = evaluatePrediction(orig_ent, gold) if match == 1: labels.append(ind) # if not orig_match == 1: # labels.append(ind) if set(labels) == set([0, 1, 2]): labels = [3] elif labels == []: labels = [4] assert (len(labels) > 0) return labels
def evaluateSansBaseline(self, predicted_identifiers, test_identifiers, num_entities, COUNT_ZERO): predicted_correct = [0.] * num_entities total_predicted = [0.] * num_entities total_gold = [0.] * num_entities print 'len(test_identifiers)', len(test_identifiers) print 'num_entities', num_entities for entity_index in range(num_entities): for article_index in range(len(predicted_identifiers)): predicted = self.TEST_ENTITIES[article_index][0][0][ entity_index].strip().lower() gold = test_identifiers[article_index][entity_index].strip( ).lower() match = evaluatePrediction(predicted, gold) if match == 'skip': continue else: total_gold[entity_index] += 1 if match == "no_predict": continue if match == 1: predicted_correct[entity_index] += 1 total_predicted[entity_index] += 1 helper.printScores(predicted_correct, total_predicted, total_gold)
def evaluateSansBaseline(self, predicted_identifiers, test_identifiers, num_entities, COUNT_ZERO): predicted_correct = [0.] * num_entities total_predicted = [0.] * num_entities total_gold = [0.] * num_entities print 'len(test_identifiers)', len(test_identifiers) print 'num_entities', num_entities for entity_index in range(num_entities): for article_index in range(len(predicted_identifiers)): predicted = self.TEST_ENTITIES[article_index][0][0][entity_index].strip().lower() gold = test_identifiers[article_index][entity_index].strip().lower() match = evaluatePrediction(predicted, gold) if match == 'skip': continue else: total_gold[entity_index] += 1 if match == "no_predict": continue if match == 1: predicted_correct[entity_index] += 1 total_predicted[entity_index] += 1 helper.printScores(predicted_correct, total_predicted,total_gold)