def generate_sentence_examples(ref,
outs,
src=None,
score_type='sentbleu',
report_length=10,
compare_directions='0-1',
title=None,
case_insensitive=False):
"""
Generate examples of sentences that satisfy some criterion, usually score of one system better
Args:
ref: Tokens from the reference
outs: Tokens from the output file(s)
src: Tokens from the source (optional)
score_type: The type of scorer to use
report_length: Number of sentences to print for each system being better or worse
compare_directions: A string specifying which systems to compare
title: A string specifying the caption of the printed table
case_insensitive: A boolean specifying whether to turn on the case insensitive option
"""
report_length = int(report_length)
case_insensitive = True if case_insensitive == 'True' else False
scorer = scorers.create_scorer_from_profile(
score_type, case_insensitive=case_insensitive)
direcs = arg_utils.parse_compare_directions(compare_directions)
scorediff_lists = []
for (left, right) in direcs:
scorediff_list = []
deduplicate_set = set()
for i, (o1, o2, r) in enumerate(zip(outs[left], outs[right], ref)):
if (tuple(o1), tuple(o2), tuple(r)) in deduplicate_set:
continue
deduplicate_set.add((tuple(o1), tuple(o2), tuple(r)))
s1, str1 = scorer.score_sentence(r, o1)
s2, str2 = scorer.score_sentence(r, o2)
scorediff_list.append((s2 - s1, s1, s2, str1, str2, i))
scorediff_list.sort()
scorediff_lists.append(scorediff_list)
reporter = reporters.SentenceExampleReport(report_length=report_length,
scorediff_lists=scorediff_lists,
scorer=scorer,
ref=ref,
outs=outs,
src=src,
compare_directions=direcs,
title=title)
reporter.generate_report()
return reporter
def generate_sentence_examples(ref,
outs,
src=None,
score_type='sentbleu',
report_length=10,
compare_directions='0-1',
title=None,
case_insensitive=False,
to_cache=False,
cache_dicts=None):
"""
Generate examples of sentences that satisfy some criterion, usually score of one system better
Args:
ref: Tokens from the reference
outs: Tokens from the output file(s)
src: Tokens from the source (optional)
score_type: The type of scorer to use
report_length: Number of sentences to print for each system being better or worse
compare_directions: A string specifying which systems to compare
title: A string specifying the caption of the printed table
case_insensitive: A boolean specifying whether to turn on the case insensitive option
to_cache: Return a list of computed statistics if True
cache_dicts: A list of dictionaries that store cached statistics for each output
"""
# check and set parameters
report_length = int(report_length)
if type(case_insensitive) == str:
case_insensitive = True if case_insensitive == 'True' else False
# compute statistics
scorer = scorers.create_scorer_from_profile(
score_type, case_insensitive=case_insensitive)
cache_key_list = ['scores', 'strs']
scores, strs = cache_utils.extract_cache_dicts(cache_dicts, cache_key_list,
len(outs))
src = [None for _ in ref] if src is None else src
if cache_dicts is None:
scores, strs = [], []
for out in outs:
scores_i, strs_i = [], []
for (r, o, s) in zip(ref, out, src):
score, string = scorer.score_sentence(r, o, s)
scores_i.append(score)
strs_i.append(string)
scores.append(scores_i)
strs.append(strs_i)
if to_cache:
cache_dict = cache_utils.return_cache_dict(cache_key_list,
[scores, strs])
return cache_dict
direcs = arg_utils.parse_compare_directions(compare_directions)
scorediff_lists = []
for (left, right) in direcs:
scorediff_list = []
deduplicate_set = set()
for i, (o1, o2, r) in enumerate(zip(outs[left], outs[right], ref)):
if (tuple(o1), tuple(o2), tuple(r)) in deduplicate_set:
continue
deduplicate_set.add((tuple(o1), tuple(o2), tuple(r)))
s1, str1 = scores[left][i], strs[left][i]
s2, str2 = scores[right][i], strs[right][i]
scorediff_list.append((s2 - s1, s1, s2, str1, str2, i))
scorediff_list.sort()
scorediff_lists.append(scorediff_list)
# generate reports
reporter = reporters.SentenceExampleReport(report_length=report_length,
scorediff_lists=scorediff_lists,
scorer=scorer,
ref=ref,
outs=outs,
src=src,
compare_directions=direcs,
title=title)
reporter.generate_report()
return reporter