def iter_results(lbs_or_file, test_file, schema):
predictions = []
if isinstance(lbs_or_file, str):
with open_file(lbs_or_file) as predict:
for line in predict:
label = get_label(int(line.split(';')[0]), schema)
predictions.append(label)
else:
for lb in lbs_or_file:
label = get_label(lb, schema)
predictions.append(label)
idx = 0
with open_file(test_file) as test_file:
line_num = 0
evi_num = 0
for line in test_file:
line_num += 1
q_tokens, evi_tokens_list, golden_answers_list = parse_line(line)
for e_tokens, golden_answers in \
zip(evi_tokens_list, golden_answers_list):
tags = predictions[idx:idx + len(e_tokens)]
evi_num += 1
yield q_tokens, e_tokens, tags, golden_answers
idx += len(e_tokens)
# one question has been parsed, needed by voters
yield None, None, None, None
# print(idx, len(predictions))
assert idx == len(predictions)
def main():
parser = argparse.ArgumentParser()
parser.add_argument('raw_prediction')
parser.add_argument('test_file')
parser.add_argument('-s',
'--schema',
default='BIO2',
choices=['BO', 'BO2', 'BIO', 'BIO2', 'BIO3'])
parser.add_argument('-o', '--output', default='-')
parser.add_argument('-f',
'--fuzzy',
action='store_true',
help='fuzzy evaluation')
options = parser.parse_args()
stats = F1Stats(options.fuzzy)
for q_tokens, e_tokens, tags, golden_answers in \
iter_results(options.raw_prediction, options.test_file,
options.schema):
if q_tokens is None: continue # one question has been processed
pred_answers = get_tagging_results(e_tokens, tags)
stats.update(golden_answers, pred_answers)
output = sys.stdout if options.output == '-' else open_file(
options.output, 'w')
print >> output, stats.get_metrics_str()
output.close()
def iter_results(raw_prediction_file, test_file, schema):
predictions = []
with open_file(raw_prediction_file) as predict:
for line in predict:
label = get_label(int(line.split(';')[0]), schema)
predictions.append(label)
idx = 0
with open_file(test_file) as test_file:
for line in test_file:
q_tokens, evi_tokens_list, golden_answers_list = parse_line(line)
for e_tokens, golden_answers in \
zip(evi_tokens_list, golden_answers_list):
tags = predictions[idx:idx + len(e_tokens)]
yield q_tokens, e_tokens, tags, golden_answers
idx += len(e_tokens)
# one question has been parsed, needed by voters
yield None, None, None, None
assert idx == len(predictions)