def evaluateSansBaseline(self, predicted_identifiers, test_identifiers, num_entities, COUNT_ZERO): predicted_correct = [0.] * num_entities total_predicted = [0.] * num_entities total_gold = [0.] * num_entities print 'len(test_identifiers)', len(test_identifiers) print 'num_entities', num_entities for entity_index in range(num_entities): for article_index in range(len(predicted_identifiers)): predicted = self.TEST_ENTITIES[article_index][0][0][ entity_index].strip().lower() gold = test_identifiers[article_index][entity_index].strip( ).lower() match = evaluatePrediction(predicted, gold) if match == 'skip': continue else: total_gold[entity_index] += 1 if match == "no_predict": continue if match == 1: predicted_correct[entity_index] += 1 total_predicted[entity_index] += 1 helper.printScores(predicted_correct, total_predicted, total_gold)
def evaluateSansBaseline(self, predicted_identifiers, test_identifiers, num_entities, COUNT_ZERO): predicted_correct = [0.] * num_entities total_predicted = [0.] * num_entities total_gold = [0.] * num_entities print 'len(test_identifiers)', len(test_identifiers) print 'num_entities', num_entities for entity_index in range(num_entities): for article_index in range(len(predicted_identifiers)): predicted = self.TEST_ENTITIES[article_index][0][0][entity_index].strip().lower() gold = test_identifiers[article_index][entity_index].strip().lower() match = evaluatePrediction(predicted, gold) if match == 'skip': continue else: total_gold[entity_index] += 1 if match == "no_predict": continue if match == 1: predicted_correct[entity_index] += 1 total_predicted[entity_index] += 1 helper.printScores(predicted_correct, total_predicted,total_gold)
def evaluatePredictions(predicted, gold): range_toks = range(len(predicted)) num_tags = len(int2tagsTag) correct = [0] * num_tags guessed = [0] * num_tags gold_c = [0] * num_tags for i in range(len(int2tagsTag)): correct[i] = sum([predicted[j] == gold[j] and gold[j] == i for j in range_toks]) guessed[i] = sum([predicted[j] == i for j in range_toks]) gold_c[i] = sum([gold[j] == i for j in range_toks]) helper.printScores(correct, guessed, gold_c)
def eval_mode_batch(output_tags, confidences, cities): tagged_data, identifier = load_data(output_tags) num_tags = len(int2tags) - 1 correct = [0]* num_tags guessed = [0] * num_tags gold_correct = [0] * num_tags assert len(tagged_data) == len(confidences) for i in range(len(tagged_data)): sentence = tagged_data[i][0] tags = tagged_data[i][1] tag_confs = confidences[i] ident = identifier[i] gold_ents = ident.split(',')[:num_tags] #Throw away title output_pred_line, entity_confidences, entity_cnts = predict_mode(sentence, tags, tag_confs, cities) predictions = output_pred_line.split(" ### ") if not len(gold_ents) == len(predictions): print 'ident', ident print 'gold_ents', gold_ents raw_input() continue for index in range(len(gold_ents)): match = evaluatePrediction(predictions[index], gold_ents[index]) debugCity = False if index == 3 and debugCity: print 'predictions[index]', predictions[index] print 'gold_ents[index]', gold_ents[index] print 'match', match raw_input() if match == 'skip': continue else: gold_correct[index] += 1 if match == "no_predict": continue if match == 1: correct[index] += 1 guessed[index] += 1 helper.printScores(correct, guessed, gold_correct, False)
def eval_mode_batch(output_tags, confidences, cities): tagged_data, identifier = load_data(output_tags) num_tags = len(int2tags) - 1 correct = [0] * num_tags guessed = [0] * num_tags gold_correct = [0] * num_tags assert len(tagged_data) == len(confidences) for i in range(len(tagged_data)): sentence = tagged_data[i][0] tags = tagged_data[i][1] tag_confs = confidences[i] ident = identifier[i] gold_ents = ident.split(',')[:num_tags] #Throw away title output_pred_line, entity_confidences, entity_cnts = predict_mode( sentence, tags, tag_confs, cities) predictions = output_pred_line.split(" ### ") if not len(gold_ents) == len(predictions): print 'ident', ident print 'gold_ents', gold_ents raw_input() continue for index in range(len(gold_ents)): match = evaluatePrediction(predictions[index], gold_ents[index]) debugCity = False if index == 3 and debugCity: print 'predictions[index]', predictions[index] print 'gold_ents[index]', gold_ents[index] print 'match', match raw_input() if match == 'skip': continue else: gold_correct[index] += 1 if match == "no_predict": continue if match == 1: correct[index] += 1 guessed[index] += 1 helper.printScores(correct, guessed, gold_correct, False)