def test_power_misaligned_asked(self):
ref = """They said Yes We asked them how happy they were and then we gave them an envelope"""
hyp = """they said yes we gave we ask them how happy they were and then we gave them on low"""
aligner = PowerAligner(ref, hyp, lowercase=True, lexicon=self.lex)
aligner.align()
# print "WER"
# print aligner.wer_alignment
# print "POWER"
# print aligner.power_alignment
expected_ref = [
"They", "said", "Yes", "We", "", "", "asked", "them", "how",
"happy", "they", "were", "and", "then", "we", "gave", "them", "an",
"envelope"
]
expected_hyp = [
"they", "said", "yes", "we", "gave", "we", "ask", "them", "how",
"happy", "they", "were", "and", "then", "we", "gave", "them", "on",
"low"
]
expected_align = [
"C", "C", "C", "C", "I", "I", "S", "C", "C", "C", "C", "C", "C",
"C", "C", "C", "C", "S", "S"
]
self.maxDiff = None
self.assertEqual(expected_align, aligner.power_alignment.align)
self.assertEqual(expected_ref, aligner.power_alignment.s1)
self.assertEqual(expected_hyp, aligner.power_alignment.s2)
expected = hyp.split()
actual = aligner.wer_alignment.s2_string().split()
self.assertEqual(actual, expected)
actual = aligner.power_alignment.s2_string().split()
self.assertEqual(actual, expected)
def test_charToWordAlign_extra_hyp_syl_no_overlap(self):
ref = """_ _ asked""".split()
hyp = """gave we ask""".split()
align = """I I S""".split()
refwords = ' '.join([r for r in ref if r != '_'])
hypwords = ' '.join([h for h in hyp if h != "_"])
ref_phones = """| # _ _ _ _ _ _ _ _ _ ae s k t |""".split(
)
hyp_phones = """| # g ey v | # w iy | # ae s k _ |""".split(
)
align_phones = """C C I I I I I I I I I C C C D C""".split(
)
ref_phones = [r.replace('_', '') for r in ref_phones]
hyp_phones = [r.replace('_', '') for r in hyp_phones]
lev = Levenshtein.align(ref_phones, hyp_phones,
PowerAligner.reserve_list,
PowerAligner.exclusive_sets)
lev.editops()
phone_align = lev.expandAlign()
# self.assertEqual(phone_align.align, align_phones)
word_align, phone_align = PowerAligner.phoneAlignToWordAlign(
refwords.split(), hypwords.split(), ref_phones, hyp_phones)
self.assertEqual(word_align.align, align)
self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref])
self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_charToWordAlign_extra_hyp_syl_overlap(self):
ref = """_ butchering""".split()
hyp = """the maturing""".split()
align = """I S""".split()
refwords = ' '.join([r for r in ref if r != '_'])
hypwords = ' '.join([h for h in hyp if h != "_"])
ref_phones = """| # b uh ch # er # ih ng |""".split(
)
hyp_phones = """| # dh ax | # m ax ch # uh r # ih ng |""".split(
)
align_phones = """C C S S I I I I C C S I C C C C""".split(
)
lev = Levenshtein.align(ref_phones, hyp_phones,
PowerAligner.reserve_list,
PowerAligner.exclusive_sets)
lev.editops()
phone_align = lev.expandAlign()
word_align, phone_align = PowerAligner.phoneAlignToWordAlign(
refwords.split(), hypwords.split(), ref_phones, hyp_phones)
self.assertEqual(word_align.align, align)
self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref])
self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_power_punct_preservation(self):
hyp = "so to address this we developed with the doctor brahmin stanford virtual dissection table"
ref = "So to address this we developed with a Dr. Brown in Stanford virtual dissection table"
aligner = PowerAligner(ref, hyp, lowercase=True, lexicon=self.lex)
aligner.align()
# print "WER"
# print aligner.wer_alignment
# print "POWER"
# print aligner.power_alignment
expected = ref.split()
actual = aligner.wer_alignment.s1_string().split()
self.assertEqual(actual, expected)
actual = aligner.power_alignment.s1_string().split()
self.assertEqual(actual, expected)
expected = hyp.split()
actual = aligner.wer_alignment.s2_string().split()
self.assertEqual(actual, expected)
actual = aligner.power_alignment.s2_string().split()
self.assertEqual(actual, expected)
def test_power_word_fix(self):
#hyp = """you might have a low bar pneumonia for example and they could give you"""
hyp = """an 'anti serum and injection of rabbit anti bodies""" # to the bacterium streptococcus if the interned sub typed it correctly"""
#ref = """You might have a lobar pneumonia, for example, and they could give you"""
ref = """an antiserum, an injection of rabid antibodies""" # to the bacterium streptococcus, if the intern sub-typed it correctly."""
aligner = PowerAligner(ref, hyp, lowercase=True, lexicon=self.lex)
aligner.align()
# print "WER"
# print aligner.wer_alignment
# print "POWER"
# print aligner.power_alignment
expected = ref.split()
actual = aligner.wer_alignment.s1_string().split()
self.assertEqual(actual, expected)
actual = aligner.power_alignment.s1_string().split()
self.assertEqual(actual, expected)
expected = hyp.split()
actual = aligner.wer_alignment.s2_string().split()
self.assertEqual(actual, expected)
actual = aligner.power_alignment.s2_string().split()
self.assertEqual(actual, expected)
def test_fbk_u268(self):
ref = """all at""".split()
hyp = """or _""".split()
align = """S D""".split()
refwords = ' '.join([r for r in ref if r != '_'])
hypwords = ' '.join([h for h in hyp if h != "_"])
# What the wrong alignment used to be.
ref_phones = """| # ao l | # ae t |""".split()
hyp_phones = """| # ao r |""".split()
align_phones = """C C C D D D D S C""".split()
correct_phones = """C C C S C D D D D""".split()
word_align, phone_align = PowerAligner.phoneAlignToWordAlign(
refwords.split(), hypwords.split(), ref_phones, hyp_phones)
print(word_align)
print(phone_align)
self.assertEqual(word_align.align, align)
self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref])
self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_fbk_u198_b(self):
ref = """and is going to pull it in""".split()
hyp = """an _ _ _ _ _ end""".split()
align = """S D D D D D S""".split()
ref_phones = """| # ae n d | # ih z | # g ow # ih ng | # t ax | # p uh l | # ih t | # ih n |""".split(
)
hyp_phones = """ | # ae n | # eh n d |""".split(
)
align_phones = """D D D D D D D D D D D D D D D D D D D D D D D D D C C S S C C S C I C""".split(
)
correct_phones = """C C C C D D D D D D D D D D D D D D D D D D D D D D D D D C C S C I C""".split(
)
refwords = ' '.join([r for r in ref if r != '_'])
hypwords = ' '.join([h for h in hyp if h != "_"])
word_align, phone_align = PowerAligner.phoneAlignToWordAlign(
refwords.split(), hypwords.split(), ref_phones, hyp_phones)
print(word_align)
print(phone_align)
self.assertEqual(word_align.align, align)
self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref])
self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_fbk_u264_a(self):
ref = ["So the", "other", "elephant"]
hyp = """set in an""".split()
align = """S S S""".split()
ref_phones = """| # s ow | # dh ax | # ah dh # er | # eh l # ax f # ax n t |""".split(
)
hyp_phones = """| # s eh t | # ih n | # ae n d |""".split(
)
align_phones = """C C C S D D S D C C S S D D C D D D D D D C S C S C""".split(
)
correct_phones = """C C C S D D S D C C S S D D C C D D D D D D S C S C""".split(
)
refwords = ' '.join([r for r in ref if r != '_'])
hypwords = ' '.join([h for h in hyp if h != "_"])
word_align, phone_align = PowerAligner.phoneAlignToWordAlign(
refwords.split(), hypwords.split(), ref_phones, hyp_phones)
print(word_align)
print(phone_align)
self.assertEqual(word_align.align, align)
self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref])
self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_fbk_u264_b(self):
ref = ["clear to", "all", "of", "you"]
hyp = """fit for _ the""".split()
align = """S S D S """.split()
ref_phones = """| # k l ih r | # t ax | # ao l | # ah v | # y uw |""".split(
)
hyp_phones = """| # f ih t | # f ao r | # dh ax |""".split(
)
align_phones = """C C D S C D D D C D C C I C D D D D S C C S S C""".split(
)
correct_phones = """C C D S C D D D C D C C I C S D D D D C C S S C""".split(
)
refwords = ' '.join([r for r in ref if r != '_'])
hypwords = ' '.join([h for h in hyp if h != "_"])
word_align, phone_align = PowerAligner.phoneAlignToWordAlign(
refwords.split(), hypwords.split(), ref_phones, hyp_phones)
print(word_align)
print(phone_align)
self.assertEqual(word_align.align, align)
self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref])
self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_kit_u264_a(self):
ref = """aortic _ root graft""".split()
hyp = """able to group graph""".split()
align = """S I S S""".split()
ref_phones = """| # ao r t # ih k | # r uw t | # g r ae f t |""".split(
)
hyp_phones = """| # ey b # ax l | # t ax | # g r uw p | # g r ae f |""".split(
)
align_phones = """C C S S D C S S C C I I I I I C C S C C C C C C D C""".split(
)
correct_phones = """C C S S D C S S C C I I I I I C C S C C C C C C D C""".split(
)
refwords = ' '.join([r for r in ref if r != '_'])
hypwords = ' '.join([h for h in hyp if h != "_"])
word_align, phone_align = PowerAligner.phoneAlignToWordAlign(
refwords.split(), hypwords.split(), ref_phones, hyp_phones)
print(word_align)
print(phone_align)
self.assertEqual(word_align.align, align)
self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref])
self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def main(argv):
parser = argparse.ArgumentParser("power.py")
parser.add_argument('--ref',
dest='reffile',
required=True,
help="Define the reference file")
parser.add_argument('--hyp',
dest='hypfile',
required=True,
help="Define the hypothesis file")
parser.add_argument('-v',
'--verbose',
dest='verbose',
action='store_true',
help="Verbose output",
default=False)
parser.add_argument('-o',
'--output',
dest='output',
help="Output file prefix",
required=True)
parser.add_argument('-f',
'--format',
dest='format',
nargs='+',
help="Output formats",
choices=['sgml', 'snt', 'json', 'align'],
default=['snt'])
parser.add_argument(
'--print-wer',
dest='print_wer',
action='store_true',
help=
"Whether to print the original WER info in all of the output formats",
default=False)
parser.add_argument('--compare',
dest='compare',
action='store_true',
help="Print comparison results between POWER and WER",
default=False)
parser.add_argument('--show-phonemes',
dest="show_phonemes",
action='store_true',
help="Show phonetic alignments on error regions",
default=False)
parser.add_argument('--show-confusions',
dest="show_confusions",
choices=['txt', 'json'],
help="Output the phonetic confusions",
nargs='+')
parser.add_argument('--case-sensitive',
dest="lowercase",
action='store_false',
help="Perform case-sensitive alignment",
default=True)
parser.add_argument(
'--word-align-weights',
dest="word_align_weights",
required=False,
nargs=4,
type=int,
help='Weights for the Levenshtein word aligner (C S D I)')
parser.add_argument(
'--lexicon',
dest="lexicon",
default=None,
required=True,
help="Path to pronunciation lexicon (json key/value dict)")
#parser.set_defaults(verbose=False, format=['sgml'], print_wer=False, compare_wer=False, show_phonemes=False)
args = parser.parse_args(argv)
wer_score_components = Counter()
power_score_components = Counter()
wer_confusions = defaultdict(Counter)
power_confusions = defaultdict(Counter)
if args.verbose:
print(args)
# Open the two files for reading
with open(args.reffile, 'r') as f_ref, open(args.hypfile, 'r') as f_hyp:
linecount = 0
wer_writers = []
power_writers = []
for i in range(len(args.format)):
# Open files for writing POWER
filepath = "%s.power.%s" % (args.output, args.format[i])
power_writers.append(
writers.CreateWriter(args.format[i], filepath, args.hypfile,
args.reffile))
if args.print_wer:
# Open files for writing WER
filepath = "%s.wer.%s" % (args.output, args.format[i])
wer_writers.append(
writers.CreateWriter(args.format[i], filepath,
args.hypfile, args.reffile))
# Assume that line counts match
for refline in f_ref:
linecount += 1
if args.verbose:
print('===========')
print('Segment %d:' % linecount)
print('===========')
refline = refline.strip()
hypline = f_hyp.readline().strip()
blank_lines = False
if not refline and not hypline:
# Nothing to compare
blank_lines = True
else:
if args.verbose:
print 'REF: "%s"' % refline
print 'HYP: "%s"' % hypline
aligner = None
if args.word_align_weights:
keys = ['C', 'S', 'D', 'I']
word_align_weights = dict(
zip(keys, args.word_align_weights))
aligner = PowerAligner(
refline,
hypline,
lowercase=args.lowercase,
verbose=args.verbose,
lexicon=args.lexicon,
word_align_weights=word_align_weights)
else:
aligner = PowerAligner(refline,
hypline,
lowercase=args.lowercase,
verbose=args.verbose,
lexicon=args.lexicon)
wer_score_components += Counter(aligner.wer_components)
if args.print_wer:
for writer in wer_writers:
if blank_lines:
writer.write_blank()
else:
writer.write(linecount, aligner.wer_components,
aligner.wer_alignment)
if args.verbose:
print 'WER alignment:'
print aligner.wer_alignment
print 'WER: ', aligner.wer
print 'Errors:', aligner.wer_components
print '==============='
aligner.align()
power_score_components += Counter(aligner.power_components)
if args.show_confusions:
if args.print_wer:
# TODO: Refactor
cp = aligner.wer_alignment.confusion_pairs()
for key in cp.iterkeys():
wer_confusions[key] += cp[key]
cp = aligner.power_alignment.confusion_pairs()
for key in cp.iterkeys():
power_confusions[key] += cp[key]
if args.verbose:
print 'Error Regions:'
for i in aligner.error_indexes:
print aligner.split_regions[i]
print aligner.phonetic_alignments[i]
print '-----'
print '==============='
print 'POWER alignment:'
print aligner.power_alignment
print 'POWER: ', aligner.power
print 'Errors:', aligner.power_components
print '==============='
print ""
# Write POWER info
for writer in power_writers:
if blank_lines:
writer.write_blank()
elif args.show_phonemes:
writer.write(linecount, aligner.power_components,
aligner.power_alignment,
aligner.phonetic_alignments)
else:
writer.write(linecount, aligner.power_components,
aligner.power_alignment)
# Close all output files
for writer in wer_writers:
writer.finalize()
for writer in power_writers:
writer.finalize()
# Compare final WER to POWER
print "============="
print "Final scores:"
final_wer = (wer_score_components['S'] + wer_score_components['D'] +
wer_score_components['I']) / wer_score_components['L']
final_power = (power_score_components['S'] + power_score_components['D'] +
power_score_components['I']) / power_score_components['L']
print "WER: ", final_wer
print wer_score_components
print "POWER:", final_power
print power_score_components
diff_score = final_power - final_wer
diff_components = copy.deepcopy(power_score_components)
diff_components.subtract(wer_score_components)
print ""
print "Score component difference (POWER vs WER):"
print "Diff: ", diff_score
print diff_components
print "============="
if args.compare:
writers.CompareWriter.write_comparison(
"%s.rsum" % args.output, args.hypfile, args.reffile, linecount,
final_power, final_wer, power_score_components,
wer_score_components, diff_score, diff_components)
if args.show_confusions:
if 'txt' in args.show_confusions:
writers.ConfusionPairWriter.write("%s.power.conf" % args.output,
args.hypfile, args.reffile,
power_confusions)
if args.print_wer:
writers.ConfusionPairWriter.write("%s.wer.conf" % args.output,
args.hypfile, args.reffile,
wer_confusions)
if 'json' in args.show_confusions:
writers.ConfusionPairWriter.write_json(
"%s.power.conf.json" % args.output, args.hypfile, args.reffile,
power_confusions)
if args.print_wer:
writers.ConfusionPairWriter.write_json(
"%s.wer.conf.json" % args.output, args.hypfile,
args.reffile, wer_confusions)