def process_document(doc_name, part_name, gold_doc, auto_doc, out, remove_singletons=True):
for ofile in [out['out'], out['short out']]:
print >> ofile
print >> ofile, '-' * 79
print >> ofile, doc_name, part_name
print >> ofile, '-' * 79
print >> ofile
text = gold_doc['text']
gold_parses = gold_doc['parses']
gold_heads = gold_doc['heads']
gold_mentions = gold_doc['mentions']
gold_clusters = gold_doc['clusters']
auto_mentions = auto_doc['mentions'].copy()
auto_clusters = auto_doc['clusters'].copy()
if remove_singletons:
to_remove = set()
for cluster in auto_clusters:
if len(auto_clusters[cluster]) == 1:
to_remove.add(cluster)
for mention in auto_clusters[cluster]:
auto_mentions.pop(mention)
for cluster in to_remove:
auto_clusters.pop(cluster)
gold_cluster_set = coreference.set_of_clusters(gold_clusters)
auto_cluster_set = coreference.set_of_clusters(auto_clusters)
gold_mention_set = coreference.set_of_mentions(gold_clusters)
auto_mention_set = coreference.set_of_mentions(auto_clusters)
coreference_rendering.print_conll_style_part(out['system output'], text, auto_mentions, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['gold'], text, gold_mentions, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: original'], text, auto_mentions, doc_name, part_name)
# Fix boundary match errors
errors = []
span_errors = match_boundaries(gold_mention_set, auto_mention_set, auto_mentions, auto_clusters, text, gold_parses, gold_heads)
if len(span_errors) == 0:
print >> out['out'], "No",
print >> out['short out'], "No",
print >> out['out'], "Span Errors: (system, gold)"
print >> out['short out'], "Span Errors: (system, gold)"
for error in span_errors:
errors.append(('span mismatch', error))
before = coreference_rendering.print_mention(None, False, gold_parses, gold_heads, text, error[0], return_str=True)
after = coreference_rendering.print_mention(None, False, gold_parses, gold_heads, text, error[1], return_str=True)
print >> out['out'], '{:<50} {:<50}'.format(before, after)
print >> out['short out'], '{:<50} {:<50}'.format(before, after)
print >> out['out']
print >> out['short out']
for error in errors:
print >> out['out'], 'span mismatch', error
print >> out['properties'], ['span error'] + list(error[1])
print >> out['out']
print >> out['out'], '-' * 79
print >> out['short out']
print >> out['short out'], '-' * 79
coreference_rendering.print_conll_style_part(out['error: span mismatch'], text, auto_mentions, doc_name, part_name)
auto_mentions_split = auto_mentions.copy()
auto_mentions_extra_mention = auto_mentions.copy()
auto_mentions_extra_entity = auto_mentions.copy()
auto_mentions_merge = auto_mentions.copy()
auto_mentions_missing_mention = auto_mentions.copy()
auto_mentions_missing_entity = auto_mentions.copy()
auto_mentions_extra_mention_prog = auto_mentions.copy()
auto_mentions_extra_entity_prog = auto_mentions.copy()
auto_mentions_merge_prog = auto_mentions.copy()
auto_mentions_missing_mention_prog = auto_mentions.copy()
auto_mentions_missing_entity_prog = auto_mentions.copy()
max_cluster = 0
if len(auto_mentions) > 0:
max_cluster = auto_mentions[max(auto_mentions, key=lambda mention: auto_mentions[mention])]
groups = coreference.confusion_groups(gold_mentions, auto_mentions, gold_clusters, auto_clusters)
for auto, gold in groups:
### print_pre_change_info(out, auto, gold, auto_mentions, gold_mention_set, text, gold_parses, gold_heads, gold_clusters, gold_mentions, gold_doc, auto_clusters)
if nlp_eval.coreference_cluster_match(gold, auto):
continue
# Print clusters with errors shown
print >> out['out']
print >> out['short out']
colours = coreference_rendering.print_cluster_error_group([auto, gold], out['out'], text, gold_parses, gold_heads, gold_mentions)
colours2 = coreference_rendering.print_cluster_error_group([auto, gold], out['short out'], text, gold_parses, gold_heads, gold_mentions)
# Work out the errors
changes = repair(auto, gold, auto_mentions, gold_mention_set, text, gold_parses, gold_heads, gold_clusters, gold_mentions, gold_doc)
print >> out['out'], "\nRaw changes:"
for name in changes:
print >> out['out'], name, len(changes[name])
for change in changes[name]:
errors.append(('raw ' + name, change))
# Categorise
changes = categorise(auto, gold, changes, text, gold_parses, gold_heads, gold_mention_set, auto_mentions, gold_doc)
# Apply updates to corrected sets
if 'split' in changes:
for change in changes['split']:
max_cluster += 1
for mention in change[0]:
auto_mentions_split[mention] = max_cluster
auto_mentions_extra_mention_prog[mention] = max_cluster
auto_mentions_extra_entity_prog[mention] = max_cluster
auto_mentions_merge_prog[mention] = max_cluster
auto_mentions_missing_mention_prog[mention] = max_cluster
auto_mentions_missing_entity_prog[mention] = max_cluster
rest = change[1].difference(change[0])
if len(rest) == 1:
rest = iter(rest).next()
if rest not in gold_mentions:
auto_mentions_split.pop(rest)
auto_mentions_extra_mention_prog.pop(rest)
auto_mentions_extra_entity_prog.pop(rest)
auto_mentions_merge_prog.pop(rest)
auto_mentions_missing_mention_prog.pop(rest)
auto_mentions_missing_entity_prog.pop(rest)
if 'extra mention' in changes:
for change in changes['extra mention']:
for mention in change[0]:
auto_mentions_extra_mention.pop(mention)
auto_mentions_extra_mention_prog.pop(mention)
auto_mentions_extra_entity_prog.pop(mention)
auto_mentions_merge_prog.pop(mention)
auto_mentions_missing_mention_prog.pop(mention)
auto_mentions_missing_entity_prog.pop(mention)
if 'extra entity' in changes:
for change in changes['extra entity']:
for mention in change[0]:
auto_mentions_extra_entity.pop(mention)
auto_mentions_extra_entity_prog.pop(mention)
auto_mentions_merge_prog.pop(mention)
auto_mentions_missing_mention_prog.pop(mention)
auto_mentions_missing_entity_prog.pop(mention)
if 'merge' in changes:
for change in changes['merge']:
for cauto_mentions in [auto_mentions_merge, auto_mentions_merge_prog, auto_mentions_missing_mention_prog, auto_mentions_missing_entity_prog]:
non_pronoun = min_non_pronoun(change[1], text, gold_parses, gold_heads)
if non_pronoun is None:
non_pronoun = min(change[1])
if non_pronoun not in cauto_mentions:
max_cluster += 1
cauto_mentions[non_pronoun] = max_cluster
ncluster_id = cauto_mentions[non_pronoun]
done = set()
for mention in change[0]:
if mention not in cauto_mentions:
cauto_mentions[mention] = ncluster_id
elif cauto_mentions[mention] not in done:
pcluster_id = cauto_mentions[mention]
done.add(pcluster_id)
for smention in cauto_mentions:
if cauto_mentions[smention] == pcluster_id:
cauto_mentions[smention] = ncluster_id
if 'missing mention' in changes:
for change in changes['missing mention']:
for cauto_mentions in [auto_mentions_missing_mention, auto_mentions_missing_mention_prog, auto_mentions_missing_entity_prog]:
min_in_goal = None
for mention in change[1]:
if mention in cauto_mentions:
if min_in_goal is None or min_in_goal > mention:
min_in_goal = mention
mention = iter(change[0]).next()
if min_in_goal is not None:
cauto_mentions[mention] = cauto_mentions[min_in_goal]
else:
min_mention = min(change[1])
max_cluster += 1
cauto_mentions[min_mention] = max_cluster
cauto_mentions[mention] = max_cluster
if 'missing entity' in changes:
for change in changes['missing entity']:
max_cluster += 1
for mention in change[0]:
auto_mentions_missing_entity[mention] = max_cluster
auto_mentions_missing_entity_prog[mention] = max_cluster
# Aggregate and count errors
print >> out['out'], "\nCategorised:"
print >> out['short out'], "\nErrors:"
rename = {
'span mismatch': "Span Error",
'split': 'Conflated Entities',
'extra mention': 'Extra Mention',
'extra entity': 'Extra Entity',
'merge': 'Divided Entity',
'missing mention': 'Missing Mention',
'missing entity': 'Missing Entity'
}
for name in changes:
if len(changes[name]) > 0:
print >> out['out'], len(changes[name]), rename[name]
print >> out['short out'], len(changes[name]), rename[name]
print >> out['out'], '\nDetailed error listing:'
for name in changes:
for change in changes[name]:
mention = None
if len(change[0]) == 1:
mention = change[0].copy().pop()
if mention is not None:
print >> out['out'], name,
if mention in gold_mentions:
colour = 15
if gold_mentions[mention] in colours:
colour = colours[gold_mentions[mention]]
coreference_rendering.print_mention(out['out'], False, gold_parses, gold_heads, text, mention, colour)
else:
coreference_rendering.print_mention(out['out'], False, gold_parses, gold_heads, text, mention, extra=True)
print >> out['out'], name, change
print >> out['out'], "Properties included:", name, change[-1]
print >> out['properties'], [name] + change[-1]
errors.append((name, change))
print >> out['out']
print >> out['out'], '-' * 79
print >> out['short out']
print >> out['short out'], '-' * 79
# Print corrected output
coreference_rendering.print_conll_style_part(out['error: split'], text, auto_mentions_split, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: extra mention'], text, auto_mentions_extra_mention, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: extra entity'], text, auto_mentions_extra_entity, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: merge'], text, auto_mentions_merge, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: missing mention'], text, auto_mentions_missing_mention, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: missing entity'], text, auto_mentions_missing_entity, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: extra mention prog'], text, auto_mentions_extra_mention_prog, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: extra entity prog'], text, auto_mentions_extra_entity_prog, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: merge prog'], text, auto_mentions_merge_prog, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: missing mention prog'], text, auto_mentions_missing_mention_prog, doc_name, part_name)
coreference_rendering.print_conll_style_part(out['error: missing entity prog'], text, auto_mentions_missing_entity_prog, doc_name, part_name)
return errors
for out in out_files: print >> out, "\n# %s %s\n" % (doc, part) text = gold[doc][part]['text'] gold_parses = gold[doc][part]['parses'] gold_heads = gold[doc][part]['heads'] gold_mentions = gold[doc][part]['mentions'] gold_clusters = gold[doc][part]['clusters'] auto_mentions = auto[doc][part]['mentions'] auto_clusters = auto[doc][part]['clusters'] gold_cluster_set = coreference.set_of_clusters(gold_clusters) auto_cluster_set = coreference.set_of_clusters(auto_clusters) gold_mention_set = coreference.set_of_mentions(gold_clusters) auto_mention_set = coreference.set_of_mentions(auto_clusters) if len(sys.argv) > 4 and sys.argv[4] == 'T': coreference_rendering.match_boundaries(gold_mention_set, auto_mention_set, auto_mentions, auto_clusters, auto_cluster_set, text, gold_parses, gold_heads) # Coloured mention output coreference_rendering.print_mention_list(out_mention_list, gold_mentions, auto_mention_set, gold_parses, gold_heads, text) coreference_rendering.print_mention_text(out_mention_text, gold_mentions, auto_mention_set, gold_parses, gold_heads, text) # Coloured cluster output, grouped groups = coreference.confusion_groups(gold_mentions, auto_mentions, gold_clusters, auto_clusters) covered = coreference_rendering.print_cluster_errors(groups, out_cluster_errors, out_cluster_context, text, gold_parses, gold_heads, auto_clusters, gold_clusters, gold_mentions) print >> out_cluster_errors, "Entirely missing or extra\n" print >> out_cluster_context, "Entirely missing or extra\n"
def main():
try:
opts, args = getopt.gnu_getopt(sys.argv[1:], '',
['resolvespanerrors', 'lang='])
output_prefix, gold_dir, test_file = args
except (getopt.GetoptError, ValueError):
print('Print coreference resolution errors')
print(('./%s <prefix> <gold_dir> <test_file> '
'[--resolvespanerrors] [--lang=<en|nl>]' % sys.argv[0]))
return
opts = dict(opts)
lang = opts.get('--lang', 'en')
auto = coreference_reading.read_conll_coref_system_output(test_file)
gold = coreference_reading.read_conll_matching_files(auto, gold_dir, lang)
out_cluster_errors = open(output_prefix + '.cluster_errors', 'w')
out_cluster_context = open(output_prefix + '.cluster_context', 'w')
out_cluster_missing = open(output_prefix + '.cluster_missing', 'w')
out_cluster_extra = open(output_prefix + '.cluster_extra', 'w')
out_mention_list = open(output_prefix + '.mention_list', 'w')
out_mention_text = open(output_prefix + '.mention_text', 'w')
out_files = [
out_cluster_errors, out_cluster_context, out_cluster_missing,
out_cluster_extra, out_mention_list, out_mention_text
]
for out in out_files:
init.header(sys.argv, out)
for function, outfile in [
(coreference_rendering.print_mention_text, out_mention_text),
(coreference_rendering.print_mention_list, out_mention_list),
(coreference_rendering.print_cluster_errors, out_cluster_errors),
(coreference_rendering.print_cluster_errors, out_cluster_context),
(coreference_rendering.print_cluster_extra, out_cluster_extra),
(coreference_rendering.print_cluster_missing, out_cluster_missing)
]:
instructions = function.__doc__.split('\n')
instructions = ['# ' + inst for inst in instructions]
print('\n'.join(instructions), file=outfile)
# Define an order
order = []
for doc in auto:
for part in auto[doc]:
order.append((doc, part))
order.sort()
for doc, part in order:
# Setup
for out in out_files:
print("\n# %s %s\n" % (doc, part), file=out)
text = gold[doc][part]['text']
gold_parses = gold[doc][part]['parses']
gold_heads = gold[doc][part]['heads']
gold_mentions = gold[doc][part]['mentions']
gold_clusters = gold[doc][part]['clusters']
auto_mentions = auto[doc][part]['mentions']
auto_clusters = auto[doc][part]['clusters']
gold_cluster_set = coreference.set_of_clusters(gold_clusters)
auto_cluster_set = coreference.set_of_clusters(auto_clusters)
gold_mention_set = coreference.set_of_mentions(gold_clusters)
auto_mention_set = coreference.set_of_mentions(auto_clusters)
if '--resolvespanerrors' in opts:
coreference_rendering.match_boundaries(
gold_mention_set, auto_mention_set, auto_mentions,
auto_clusters, auto_cluster_set, text, gold_parses, gold_heads)
# Coloured mention output
coreference_rendering.print_mention_list(out_mention_list,
gold_mentions,
auto_mention_set, gold_parses,
gold_heads, text)
coreference_rendering.print_mention_text(out_mention_text,
gold_mentions,
auto_mention_set, gold_parses,
gold_heads, text)
# Coloured cluster output, grouped
groups = coreference.confusion_groups(gold_mentions, auto_mentions,
gold_clusters, auto_clusters)
covered = coreference_rendering.print_cluster_errors(
groups, out_cluster_errors, out_cluster_context, text, gold_parses,
gold_heads, auto_clusters, gold_clusters, gold_mentions)
print("Entirely missing or extra\n", file=out_cluster_errors)
print("Entirely missing or extra\n", file=out_cluster_context)
coreference_rendering.print_cluster_missing(out_cluster_errors,
out_cluster_context,
out_cluster_missing, text,
gold_cluster_set, covered,
gold_parses, gold_heads)
coreference_rendering.print_cluster_extra(out_cluster_errors,
out_cluster_context,
out_cluster_extra, text,
auto_cluster_set, covered,
gold_parses, gold_heads)
