def evaluateSansBaseline(self, predicted_identifiers, test_identifiers,
num_entities, COUNT_ZERO):
predicted_correct = [0.] * num_entities
total_predicted = [0.] * num_entities
total_gold = [0.] * num_entities
print 'len(test_identifiers)', len(test_identifiers)
print 'num_entities', num_entities
for entity_index in range(num_entities):
for article_index in range(len(predicted_identifiers)):
predicted = self.TEST_ENTITIES[article_index][0][0][
entity_index].strip().lower()
gold = test_identifiers[article_index][entity_index].strip(
).lower()
match = evaluatePrediction(predicted, gold)
if match == 'skip':
continue
else:
total_gold[entity_index] += 1
if match == "no_predict":
continue
if match == 1:
predicted_correct[entity_index] += 1
total_predicted[entity_index] += 1
helper.printScores(predicted_correct, total_predicted, total_gold)
def evaluateSansBaseline(self, predicted_identifiers, test_identifiers, num_entities, COUNT_ZERO):
predicted_correct = [0.] * num_entities
total_predicted = [0.] * num_entities
total_gold = [0.] * num_entities
print 'len(test_identifiers)', len(test_identifiers)
print 'num_entities', num_entities
for entity_index in range(num_entities):
for article_index in range(len(predicted_identifiers)):
predicted = self.TEST_ENTITIES[article_index][0][0][entity_index].strip().lower()
gold = test_identifiers[article_index][entity_index].strip().lower()
match = evaluatePrediction(predicted, gold)
if match == 'skip':
continue
else:
total_gold[entity_index] += 1
if match == "no_predict":
continue
if match == 1:
predicted_correct[entity_index] += 1
total_predicted[entity_index] += 1
helper.printScores(predicted_correct, total_predicted,total_gold)
def evaluatePredictions(predicted, gold):
range_toks = range(len(predicted))
num_tags = len(int2tagsTag)
correct = [0] * num_tags
guessed = [0] * num_tags
gold_c = [0] * num_tags
for i in range(len(int2tagsTag)):
correct[i] = sum([predicted[j] == gold[j] and gold[j] == i for j in range_toks])
guessed[i] = sum([predicted[j] == i for j in range_toks])
gold_c[i] = sum([gold[j] == i for j in range_toks])
helper.printScores(correct, guessed, gold_c)
def evaluatePredictions(predicted, gold):
range_toks = range(len(predicted))
num_tags = len(int2tagsTag)
correct = [0] * num_tags
guessed = [0] * num_tags
gold_c = [0] * num_tags
for i in range(len(int2tagsTag)):
correct[i] = sum([predicted[j] == gold[j] and gold[j] == i for j in range_toks])
guessed[i] = sum([predicted[j] == i for j in range_toks])
gold_c[i] = sum([gold[j] == i for j in range_toks])
helper.printScores(correct, guessed, gold_c)
def eval_mode_batch(output_tags, confidences, cities):
tagged_data, identifier = load_data(output_tags)
num_tags = len(int2tags) - 1
correct = [0]* num_tags
guessed = [0] * num_tags
gold_correct = [0] * num_tags
assert len(tagged_data) == len(confidences)
for i in range(len(tagged_data)):
sentence = tagged_data[i][0]
tags = tagged_data[i][1]
tag_confs = confidences[i]
ident = identifier[i]
gold_ents = ident.split(',')[:num_tags] #Throw away title
output_pred_line, entity_confidences, entity_cnts = predict_mode(sentence, tags, tag_confs, cities)
predictions = output_pred_line.split(" ### ")
if not len(gold_ents) == len(predictions):
print 'ident', ident
print 'gold_ents', gold_ents
raw_input()
continue
for index in range(len(gold_ents)):
match = evaluatePrediction(predictions[index], gold_ents[index])
debugCity = False
if index == 3 and debugCity:
print 'predictions[index]', predictions[index]
print 'gold_ents[index]', gold_ents[index]
print 'match', match
raw_input()
if match == 'skip':
continue
else:
gold_correct[index] += 1
if match == "no_predict":
continue
if match == 1:
correct[index] += 1
guessed[index] += 1
helper.printScores(correct, guessed, gold_correct, False)
def eval_mode_batch(output_tags, confidences, cities):
tagged_data, identifier = load_data(output_tags)
num_tags = len(int2tags) - 1
correct = [0] * num_tags
guessed = [0] * num_tags
gold_correct = [0] * num_tags
assert len(tagged_data) == len(confidences)
for i in range(len(tagged_data)):
sentence = tagged_data[i][0]
tags = tagged_data[i][1]
tag_confs = confidences[i]
ident = identifier[i]
gold_ents = ident.split(',')[:num_tags] #Throw away title
output_pred_line, entity_confidences, entity_cnts = predict_mode(
sentence, tags, tag_confs, cities)
predictions = output_pred_line.split(" ### ")
if not len(gold_ents) == len(predictions):
print 'ident', ident
print 'gold_ents', gold_ents
raw_input()
continue
for index in range(len(gold_ents)):
match = evaluatePrediction(predictions[index], gold_ents[index])
debugCity = False
if index == 3 and debugCity:
print 'predictions[index]', predictions[index]
print 'gold_ents[index]', gold_ents[index]
print 'match', match
raw_input()
if match == 'skip':
continue
else:
gold_correct[index] += 1
if match == "no_predict":
continue
if match == 1:
correct[index] += 1
guessed[index] += 1
helper.printScores(correct, guessed, gold_correct, False)
