def process_document(doc_name, part_name, gold_doc, auto_doc, out, remove_singletons=True):
	for ofile in [out['out'], out['short out']]:
		print >> ofile
		print >> ofile, '-' * 79
		print >> ofile, doc_name, part_name
		print >> ofile, '-' * 79
		print >> ofile
	text = gold_doc['text']

	gold_parses = gold_doc['parses']
	gold_heads = gold_doc['heads']
	gold_mentions = gold_doc['mentions']
	gold_clusters = gold_doc['clusters']

	auto_mentions = auto_doc['mentions'].copy()
	auto_clusters = auto_doc['clusters'].copy()

	if remove_singletons:
		to_remove = set()
		for cluster in auto_clusters:
			if len(auto_clusters[cluster]) == 1:
				to_remove.add(cluster)
				for mention in auto_clusters[cluster]:
					auto_mentions.pop(mention)
		for cluster in to_remove:
			auto_clusters.pop(cluster)

	gold_cluster_set = coreference.set_of_clusters(gold_clusters)
	auto_cluster_set = coreference.set_of_clusters(auto_clusters)
	gold_mention_set = coreference.set_of_mentions(gold_clusters)
	auto_mention_set = coreference.set_of_mentions(auto_clusters)

	coreference_rendering.print_conll_style_part(out['system output'], text, auto_mentions, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['gold'], text, gold_mentions, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: original'], text, auto_mentions, doc_name, part_name)

	# Fix boundary match errors
	errors = []
	span_errors = match_boundaries(gold_mention_set, auto_mention_set, auto_mentions, auto_clusters, text, gold_parses, gold_heads)
	if len(span_errors) == 0:
		print >> out['out'], "No",
		print >> out['short out'], "No",
	print >> out['out'], "Span Errors: (system, gold)"
	print >> out['short out'], "Span Errors: (system, gold)"
	for error in span_errors:
		errors.append(('span mismatch', error))
		before = coreference_rendering.print_mention(None, False, gold_parses, gold_heads, text, error[0], return_str=True)
		after = coreference_rendering.print_mention(None, False, gold_parses, gold_heads, text, error[1], return_str=True)
		print >> out['out'], '{:<50}    {:<50}'.format(before, after)
		print >> out['short out'], '{:<50}    {:<50}'.format(before, after)
	print >> out['out']
	print >> out['short out']
	for error in errors:
		print >> out['out'], 'span mismatch', error
		print >> out['properties'], ['span error'] + list(error[1])
	print >> out['out']
	print >> out['out'], '-' * 79
	print >> out['short out']
	print >> out['short out'], '-' * 79

	coreference_rendering.print_conll_style_part(out['error: span mismatch'], text, auto_mentions, doc_name, part_name)

	auto_mentions_split = auto_mentions.copy()
	auto_mentions_extra_mention = auto_mentions.copy()
	auto_mentions_extra_entity = auto_mentions.copy()
	auto_mentions_merge = auto_mentions.copy()
	auto_mentions_missing_mention = auto_mentions.copy()
	auto_mentions_missing_entity = auto_mentions.copy()
	auto_mentions_extra_mention_prog = auto_mentions.copy()
	auto_mentions_extra_entity_prog = auto_mentions.copy()
	auto_mentions_merge_prog = auto_mentions.copy()
	auto_mentions_missing_mention_prog = auto_mentions.copy()
	auto_mentions_missing_entity_prog = auto_mentions.copy()
	max_cluster = 0
	if len(auto_mentions) > 0:
		max_cluster = auto_mentions[max(auto_mentions, key=lambda mention: auto_mentions[mention])]

	groups = coreference.confusion_groups(gold_mentions, auto_mentions, gold_clusters, auto_clusters)
	for auto, gold in groups:
###		print_pre_change_info(out, auto, gold, auto_mentions, gold_mention_set, text, gold_parses, gold_heads, gold_clusters, gold_mentions, gold_doc, auto_clusters)

		if nlp_eval.coreference_cluster_match(gold, auto):
			continue

		# Print clusters with errors shown
		print >> out['out']
		print >> out['short out']
		colours = coreference_rendering.print_cluster_error_group([auto, gold], out['out'], text, gold_parses, gold_heads, gold_mentions)
		colours2 = coreference_rendering.print_cluster_error_group([auto, gold], out['short out'], text, gold_parses, gold_heads, gold_mentions)

		# Work out the errors
		changes = repair(auto, gold, auto_mentions, gold_mention_set, text, gold_parses, gold_heads, gold_clusters, gold_mentions, gold_doc)
		print >> out['out'], "\nRaw changes:"
		for name in changes:
			print >> out['out'], name, len(changes[name])
			for change in changes[name]:
				errors.append(('raw ' + name, change))

		# Categorise
		changes = categorise(auto, gold, changes, text, gold_parses, gold_heads, gold_mention_set, auto_mentions, gold_doc)

		# Apply updates to corrected sets
		if 'split' in changes:
			for change in changes['split']:
				max_cluster += 1
				for mention in change[0]:
					auto_mentions_split[mention] = max_cluster
					auto_mentions_extra_mention_prog[mention] = max_cluster
					auto_mentions_extra_entity_prog[mention] = max_cluster
					auto_mentions_merge_prog[mention] = max_cluster
					auto_mentions_missing_mention_prog[mention] = max_cluster
					auto_mentions_missing_entity_prog[mention] = max_cluster
				rest = change[1].difference(change[0])
				if len(rest) == 1:
					rest = iter(rest).next()
					if rest not in gold_mentions:
						auto_mentions_split.pop(rest)
						auto_mentions_extra_mention_prog.pop(rest)
						auto_mentions_extra_entity_prog.pop(rest)
						auto_mentions_merge_prog.pop(rest)
						auto_mentions_missing_mention_prog.pop(rest)
						auto_mentions_missing_entity_prog.pop(rest)

		if 'extra mention' in changes:
			for change in changes['extra mention']:
				for mention in change[0]:
					auto_mentions_extra_mention.pop(mention)
					auto_mentions_extra_mention_prog.pop(mention)
					auto_mentions_extra_entity_prog.pop(mention)
					auto_mentions_merge_prog.pop(mention)
					auto_mentions_missing_mention_prog.pop(mention)
					auto_mentions_missing_entity_prog.pop(mention)

		if 'extra entity' in changes:
			for change in changes['extra entity']:
				for mention in change[0]:
					auto_mentions_extra_entity.pop(mention)
					auto_mentions_extra_entity_prog.pop(mention)
					auto_mentions_merge_prog.pop(mention)
					auto_mentions_missing_mention_prog.pop(mention)
					auto_mentions_missing_entity_prog.pop(mention)

		if 'merge' in changes:
			for change in changes['merge']:
				for cauto_mentions in [auto_mentions_merge, auto_mentions_merge_prog, auto_mentions_missing_mention_prog, auto_mentions_missing_entity_prog]:
					non_pronoun = min_non_pronoun(change[1], text, gold_parses, gold_heads)
					if non_pronoun is None:
						non_pronoun = min(change[1])
					if non_pronoun not in cauto_mentions:
						max_cluster += 1
						cauto_mentions[non_pronoun] = max_cluster
					ncluster_id = cauto_mentions[non_pronoun]
					done = set()
					for mention in change[0]:
						if mention not in cauto_mentions:
							cauto_mentions[mention] = ncluster_id
						elif cauto_mentions[mention] not in done:
							pcluster_id = cauto_mentions[mention]
							done.add(pcluster_id)
							for smention in cauto_mentions:
								if cauto_mentions[smention] == pcluster_id:
									cauto_mentions[smention] = ncluster_id

		if 'missing mention' in changes:
			for change in changes['missing mention']:
				for cauto_mentions in [auto_mentions_missing_mention, auto_mentions_missing_mention_prog, auto_mentions_missing_entity_prog]:
					min_in_goal = None
					for mention in change[1]:
						if mention in cauto_mentions:
							if min_in_goal is None or min_in_goal > mention:
								min_in_goal = mention
					mention = iter(change[0]).next()
					if min_in_goal is not None:
						cauto_mentions[mention] = cauto_mentions[min_in_goal]
					else:
						min_mention = min(change[1])
						max_cluster += 1
						cauto_mentions[min_mention] = max_cluster
						cauto_mentions[mention] = max_cluster

		if 'missing entity' in changes:
			for change in changes['missing entity']:
				max_cluster += 1
				for mention in change[0]:
					auto_mentions_missing_entity[mention] = max_cluster
					auto_mentions_missing_entity_prog[mention] = max_cluster

		# Aggregate and count errors
		print >> out['out'], "\nCategorised:"
		print >> out['short out'], "\nErrors:"
		rename = {
			'span mismatch': "Span Error",
			'split': 'Conflated Entities',
			'extra mention': 'Extra Mention',
			'extra entity': 'Extra Entity',
			'merge': 'Divided Entity',
			'missing mention': 'Missing Mention',
			'missing entity': 'Missing Entity'
		}
		for name in changes:
			if len(changes[name]) > 0:
				print >> out['out'], len(changes[name]), rename[name]
				print >> out['short out'], len(changes[name]), rename[name]
		print >> out['out'], '\nDetailed error listing:'
		for name in changes:
			for change in changes[name]:
				mention = None
				if len(change[0]) == 1:
					mention = change[0].copy().pop()
				if mention is not None:
					print >> out['out'], name,
					if mention in gold_mentions:
						colour = 15
						if gold_mentions[mention] in colours:
							colour = colours[gold_mentions[mention]]
						coreference_rendering.print_mention(out['out'], False, gold_parses, gold_heads, text, mention, colour)
					else:
						coreference_rendering.print_mention(out['out'], False, gold_parses, gold_heads, text, mention, extra=True)
				print >> out['out'], name, change
				print >> out['out'], "Properties included:", name, change[-1]
				print >> out['properties'], [name] + change[-1]
				errors.append((name, change))
		print >> out['out']
		print >> out['out'], '-' * 79
		print >> out['short out']
		print >> out['short out'], '-' * 79

	# Print corrected output
	coreference_rendering.print_conll_style_part(out['error: split'], text, auto_mentions_split, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: extra mention'], text, auto_mentions_extra_mention, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: extra entity'], text, auto_mentions_extra_entity, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: merge'], text, auto_mentions_merge, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: missing mention'], text, auto_mentions_missing_mention, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: missing entity'], text, auto_mentions_missing_entity, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: extra mention prog'], text, auto_mentions_extra_mention_prog, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: extra entity prog'], text, auto_mentions_extra_entity_prog, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: merge prog'], text, auto_mentions_merge_prog, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: missing mention prog'], text, auto_mentions_missing_mention_prog, doc_name, part_name)
	coreference_rendering.print_conll_style_part(out['error: missing entity prog'], text, auto_mentions_missing_entity_prog, doc_name, part_name)

	return errors
	for doc, part in order:
		# Setup
		for out in out_files:
			print >> out, "\n# %s %s\n" % (doc, part)

		text = gold[doc][part]['text']

		gold_parses = gold[doc][part]['parses']
		gold_heads = gold[doc][part]['heads']
		gold_mentions = gold[doc][part]['mentions']
		gold_clusters = gold[doc][part]['clusters']

		auto_mentions = auto[doc][part]['mentions']
		auto_clusters = auto[doc][part]['clusters']

		gold_cluster_set = coreference.set_of_clusters(gold_clusters)
		auto_cluster_set = coreference.set_of_clusters(auto_clusters)
		gold_mention_set = coreference.set_of_mentions(gold_clusters)
		auto_mention_set = coreference.set_of_mentions(auto_clusters)

		if len(sys.argv) > 4 and sys.argv[4] == 'T':
			coreference_rendering.match_boundaries(gold_mention_set, auto_mention_set, auto_mentions, auto_clusters, auto_cluster_set, text, gold_parses, gold_heads)

		# Coloured mention output
		coreference_rendering.print_mention_list(out_mention_list, gold_mentions, auto_mention_set, gold_parses, gold_heads, text)
		coreference_rendering.print_mention_text(out_mention_text, gold_mentions, auto_mention_set, gold_parses, gold_heads, text)

		# Coloured cluster output, grouped
		groups = coreference.confusion_groups(gold_mentions, auto_mentions, gold_clusters, auto_clusters)

		covered = coreference_rendering.print_cluster_errors(groups, out_cluster_errors, out_cluster_context, text, gold_parses, gold_heads, auto_clusters, gold_clusters, gold_mentions)
def main():
    try:
        opts, args = getopt.gnu_getopt(sys.argv[1:], '',
                                       ['resolvespanerrors', 'lang='])
        output_prefix, gold_dir, test_file = args
    except (getopt.GetoptError, ValueError):
        print('Print coreference resolution errors')
        print(('./%s <prefix> <gold_dir> <test_file> '
               '[--resolvespanerrors] [--lang=<en|nl>]' % sys.argv[0]))
        return
    opts = dict(opts)
    lang = opts.get('--lang', 'en')
    auto = coreference_reading.read_conll_coref_system_output(test_file)
    gold = coreference_reading.read_conll_matching_files(auto, gold_dir, lang)

    out_cluster_errors = open(output_prefix + '.cluster_errors', 'w')
    out_cluster_context = open(output_prefix + '.cluster_context', 'w')
    out_cluster_missing = open(output_prefix + '.cluster_missing', 'w')
    out_cluster_extra = open(output_prefix + '.cluster_extra', 'w')
    out_mention_list = open(output_prefix + '.mention_list', 'w')
    out_mention_text = open(output_prefix + '.mention_text', 'w')
    out_files = [
        out_cluster_errors, out_cluster_context, out_cluster_missing,
        out_cluster_extra, out_mention_list, out_mention_text
    ]
    for out in out_files:
        init.header(sys.argv, out)

    for function, outfile in [
        (coreference_rendering.print_mention_text, out_mention_text),
        (coreference_rendering.print_mention_list, out_mention_list),
        (coreference_rendering.print_cluster_errors, out_cluster_errors),
        (coreference_rendering.print_cluster_errors, out_cluster_context),
        (coreference_rendering.print_cluster_extra, out_cluster_extra),
        (coreference_rendering.print_cluster_missing, out_cluster_missing)
    ]:
        instructions = function.__doc__.split('\n')
        instructions = ['# ' + inst for inst in instructions]
        print('\n'.join(instructions), file=outfile)

    # Define an order
    order = []
    for doc in auto:
        for part in auto[doc]:
            order.append((doc, part))
    order.sort()

    for doc, part in order:
        # Setup
        for out in out_files:
            print("\n# %s %s\n" % (doc, part), file=out)

        text = gold[doc][part]['text']

        gold_parses = gold[doc][part]['parses']
        gold_heads = gold[doc][part]['heads']
        gold_mentions = gold[doc][part]['mentions']
        gold_clusters = gold[doc][part]['clusters']

        auto_mentions = auto[doc][part]['mentions']
        auto_clusters = auto[doc][part]['clusters']

        gold_cluster_set = coreference.set_of_clusters(gold_clusters)
        auto_cluster_set = coreference.set_of_clusters(auto_clusters)
        gold_mention_set = coreference.set_of_mentions(gold_clusters)
        auto_mention_set = coreference.set_of_mentions(auto_clusters)

        if '--resolvespanerrors' in opts:
            coreference_rendering.match_boundaries(
                gold_mention_set, auto_mention_set, auto_mentions,
                auto_clusters, auto_cluster_set, text, gold_parses, gold_heads)

        # Coloured mention output
        coreference_rendering.print_mention_list(out_mention_list,
                                                 gold_mentions,
                                                 auto_mention_set, gold_parses,
                                                 gold_heads, text)
        coreference_rendering.print_mention_text(out_mention_text,
                                                 gold_mentions,
                                                 auto_mention_set, gold_parses,
                                                 gold_heads, text)

        # Coloured cluster output, grouped
        groups = coreference.confusion_groups(gold_mentions, auto_mentions,
                                              gold_clusters, auto_clusters)

        covered = coreference_rendering.print_cluster_errors(
            groups, out_cluster_errors, out_cluster_context, text, gold_parses,
            gold_heads, auto_clusters, gold_clusters, gold_mentions)
        print("Entirely missing or extra\n", file=out_cluster_errors)
        print("Entirely missing or extra\n", file=out_cluster_context)
        coreference_rendering.print_cluster_missing(out_cluster_errors,
                                                    out_cluster_context,
                                                    out_cluster_missing, text,
                                                    gold_cluster_set, covered,
                                                    gold_parses, gold_heads)
        coreference_rendering.print_cluster_extra(out_cluster_errors,
                                                  out_cluster_context,
                                                  out_cluster_extra, text,
                                                  auto_cluster_set, covered,
                                                  gold_parses, gold_heads)