def test_power_misaligned_asked(self): ref = """They said Yes We asked them how happy they were and then we gave them an envelope""" hyp = """they said yes we gave we ask them how happy they were and then we gave them on low""" aligner = PowerAligner(ref, hyp, lowercase=True, lexicon=self.lex) aligner.align() # print "WER" # print aligner.wer_alignment # print "POWER" # print aligner.power_alignment expected_ref = [ "They", "said", "Yes", "We", "", "", "asked", "them", "how", "happy", "they", "were", "and", "then", "we", "gave", "them", "an", "envelope" ] expected_hyp = [ "they", "said", "yes", "we", "gave", "we", "ask", "them", "how", "happy", "they", "were", "and", "then", "we", "gave", "them", "on", "low" ] expected_align = [ "C", "C", "C", "C", "I", "I", "S", "C", "C", "C", "C", "C", "C", "C", "C", "C", "C", "S", "S" ] self.maxDiff = None self.assertEqual(expected_align, aligner.power_alignment.align) self.assertEqual(expected_ref, aligner.power_alignment.s1) self.assertEqual(expected_hyp, aligner.power_alignment.s2) expected = hyp.split() actual = aligner.wer_alignment.s2_string().split() self.assertEqual(actual, expected) actual = aligner.power_alignment.s2_string().split() self.assertEqual(actual, expected)
def test_charToWordAlign_extra_hyp_syl_no_overlap(self): ref = """_ _ asked""".split() hyp = """gave we ask""".split() align = """I I S""".split() refwords = ' '.join([r for r in ref if r != '_']) hypwords = ' '.join([h for h in hyp if h != "_"]) ref_phones = """| # _ _ _ _ _ _ _ _ _ ae s k t |""".split( ) hyp_phones = """| # g ey v | # w iy | # ae s k _ |""".split( ) align_phones = """C C I I I I I I I I I C C C D C""".split( ) ref_phones = [r.replace('_', '') for r in ref_phones] hyp_phones = [r.replace('_', '') for r in hyp_phones] lev = Levenshtein.align(ref_phones, hyp_phones, PowerAligner.reserve_list, PowerAligner.exclusive_sets) lev.editops() phone_align = lev.expandAlign() # self.assertEqual(phone_align.align, align_phones) word_align, phone_align = PowerAligner.phoneAlignToWordAlign( refwords.split(), hypwords.split(), ref_phones, hyp_phones) self.assertEqual(word_align.align, align) self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref]) self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_charToWordAlign_extra_hyp_syl_overlap(self): ref = """_ butchering""".split() hyp = """the maturing""".split() align = """I S""".split() refwords = ' '.join([r for r in ref if r != '_']) hypwords = ' '.join([h for h in hyp if h != "_"]) ref_phones = """| # b uh ch # er # ih ng |""".split( ) hyp_phones = """| # dh ax | # m ax ch # uh r # ih ng |""".split( ) align_phones = """C C S S I I I I C C S I C C C C""".split( ) lev = Levenshtein.align(ref_phones, hyp_phones, PowerAligner.reserve_list, PowerAligner.exclusive_sets) lev.editops() phone_align = lev.expandAlign() word_align, phone_align = PowerAligner.phoneAlignToWordAlign( refwords.split(), hypwords.split(), ref_phones, hyp_phones) self.assertEqual(word_align.align, align) self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref]) self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_power_punct_preservation(self): hyp = "so to address this we developed with the doctor brahmin stanford virtual dissection table" ref = "So to address this we developed with a Dr. Brown in Stanford virtual dissection table" aligner = PowerAligner(ref, hyp, lowercase=True, lexicon=self.lex) aligner.align() # print "WER" # print aligner.wer_alignment # print "POWER" # print aligner.power_alignment expected = ref.split() actual = aligner.wer_alignment.s1_string().split() self.assertEqual(actual, expected) actual = aligner.power_alignment.s1_string().split() self.assertEqual(actual, expected) expected = hyp.split() actual = aligner.wer_alignment.s2_string().split() self.assertEqual(actual, expected) actual = aligner.power_alignment.s2_string().split() self.assertEqual(actual, expected)
def test_power_word_fix(self): #hyp = """you might have a low bar pneumonia for example and they could give you""" hyp = """an 'anti serum and injection of rabbit anti bodies""" # to the bacterium streptococcus if the interned sub typed it correctly""" #ref = """You might have a lobar pneumonia, for example, and they could give you""" ref = """an antiserum, an injection of rabid antibodies""" # to the bacterium streptococcus, if the intern sub-typed it correctly.""" aligner = PowerAligner(ref, hyp, lowercase=True, lexicon=self.lex) aligner.align() # print "WER" # print aligner.wer_alignment # print "POWER" # print aligner.power_alignment expected = ref.split() actual = aligner.wer_alignment.s1_string().split() self.assertEqual(actual, expected) actual = aligner.power_alignment.s1_string().split() self.assertEqual(actual, expected) expected = hyp.split() actual = aligner.wer_alignment.s2_string().split() self.assertEqual(actual, expected) actual = aligner.power_alignment.s2_string().split() self.assertEqual(actual, expected)
def test_fbk_u268(self): ref = """all at""".split() hyp = """or _""".split() align = """S D""".split() refwords = ' '.join([r for r in ref if r != '_']) hypwords = ' '.join([h for h in hyp if h != "_"]) # What the wrong alignment used to be. ref_phones = """| # ao l | # ae t |""".split() hyp_phones = """| # ao r |""".split() align_phones = """C C C D D D D S C""".split() correct_phones = """C C C S C D D D D""".split() word_align, phone_align = PowerAligner.phoneAlignToWordAlign( refwords.split(), hypwords.split(), ref_phones, hyp_phones) print(word_align) print(phone_align) self.assertEqual(word_align.align, align) self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref]) self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_fbk_u198_b(self): ref = """and is going to pull it in""".split() hyp = """an _ _ _ _ _ end""".split() align = """S D D D D D S""".split() ref_phones = """| # ae n d | # ih z | # g ow # ih ng | # t ax | # p uh l | # ih t | # ih n |""".split( ) hyp_phones = """ | # ae n | # eh n d |""".split( ) align_phones = """D D D D D D D D D D D D D D D D D D D D D D D D D C C S S C C S C I C""".split( ) correct_phones = """C C C C D D D D D D D D D D D D D D D D D D D D D D D D D C C S C I C""".split( ) refwords = ' '.join([r for r in ref if r != '_']) hypwords = ' '.join([h for h in hyp if h != "_"]) word_align, phone_align = PowerAligner.phoneAlignToWordAlign( refwords.split(), hypwords.split(), ref_phones, hyp_phones) print(word_align) print(phone_align) self.assertEqual(word_align.align, align) self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref]) self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_fbk_u264_a(self): ref = ["So the", "other", "elephant"] hyp = """set in an""".split() align = """S S S""".split() ref_phones = """| # s ow | # dh ax | # ah dh # er | # eh l # ax f # ax n t |""".split( ) hyp_phones = """| # s eh t | # ih n | # ae n d |""".split( ) align_phones = """C C C S D D S D C C S S D D C D D D D D D C S C S C""".split( ) correct_phones = """C C C S D D S D C C S S D D C C D D D D D D S C S C""".split( ) refwords = ' '.join([r for r in ref if r != '_']) hypwords = ' '.join([h for h in hyp if h != "_"]) word_align, phone_align = PowerAligner.phoneAlignToWordAlign( refwords.split(), hypwords.split(), ref_phones, hyp_phones) print(word_align) print(phone_align) self.assertEqual(word_align.align, align) self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref]) self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_fbk_u264_b(self): ref = ["clear to", "all", "of", "you"] hyp = """fit for _ the""".split() align = """S S D S """.split() ref_phones = """| # k l ih r | # t ax | # ao l | # ah v | # y uw |""".split( ) hyp_phones = """| # f ih t | # f ao r | # dh ax |""".split( ) align_phones = """C C D S C D D D C D C C I C D D D D S C C S S C""".split( ) correct_phones = """C C D S C D D D C D C C I C S D D D D C C S S C""".split( ) refwords = ' '.join([r for r in ref if r != '_']) hypwords = ' '.join([h for h in hyp if h != "_"]) word_align, phone_align = PowerAligner.phoneAlignToWordAlign( refwords.split(), hypwords.split(), ref_phones, hyp_phones) print(word_align) print(phone_align) self.assertEqual(word_align.align, align) self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref]) self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def test_kit_u264_a(self): ref = """aortic _ root graft""".split() hyp = """able to group graph""".split() align = """S I S S""".split() ref_phones = """| # ao r t # ih k | # r uw t | # g r ae f t |""".split( ) hyp_phones = """| # ey b # ax l | # t ax | # g r uw p | # g r ae f |""".split( ) align_phones = """C C S S D C S S C C I I I I I C C S C C C C C C D C""".split( ) correct_phones = """C C S S D C S S C C I I I I I C C S C C C C C C D C""".split( ) refwords = ' '.join([r for r in ref if r != '_']) hypwords = ' '.join([h for h in hyp if h != "_"]) word_align, phone_align = PowerAligner.phoneAlignToWordAlign( refwords.split(), hypwords.split(), ref_phones, hyp_phones) print(word_align) print(phone_align) self.assertEqual(word_align.align, align) self.assertEqual(word_align.s1, [x if x != "_" else "" for x in ref]) self.assertEqual(word_align.s2, [x if x != "_" else "" for x in hyp])
def main(argv): parser = argparse.ArgumentParser("power.py") parser.add_argument('--ref', dest='reffile', required=True, help="Define the reference file") parser.add_argument('--hyp', dest='hypfile', required=True, help="Define the hypothesis file") parser.add_argument('-v', '--verbose', dest='verbose', action='store_true', help="Verbose output", default=False) parser.add_argument('-o', '--output', dest='output', help="Output file prefix", required=True) parser.add_argument('-f', '--format', dest='format', nargs='+', help="Output formats", choices=['sgml', 'snt', 'json', 'align'], default=['snt']) parser.add_argument( '--print-wer', dest='print_wer', action='store_true', help= "Whether to print the original WER info in all of the output formats", default=False) parser.add_argument('--compare', dest='compare', action='store_true', help="Print comparison results between POWER and WER", default=False) parser.add_argument('--show-phonemes', dest="show_phonemes", action='store_true', help="Show phonetic alignments on error regions", default=False) parser.add_argument('--show-confusions', dest="show_confusions", choices=['txt', 'json'], help="Output the phonetic confusions", nargs='+') parser.add_argument('--case-sensitive', dest="lowercase", action='store_false', help="Perform case-sensitive alignment", default=True) parser.add_argument( '--word-align-weights', dest="word_align_weights", required=False, nargs=4, type=int, help='Weights for the Levenshtein word aligner (C S D I)') parser.add_argument( '--lexicon', dest="lexicon", default=None, required=True, help="Path to pronunciation lexicon (json key/value dict)") #parser.set_defaults(verbose=False, format=['sgml'], print_wer=False, compare_wer=False, show_phonemes=False) args = parser.parse_args(argv) wer_score_components = Counter() power_score_components = Counter() wer_confusions = defaultdict(Counter) power_confusions = defaultdict(Counter) if args.verbose: print(args) # Open the two files for reading with open(args.reffile, 'r') as f_ref, open(args.hypfile, 'r') as f_hyp: linecount = 0 wer_writers = [] power_writers = [] for i in range(len(args.format)): # Open files for writing POWER filepath = "%s.power.%s" % (args.output, args.format[i]) power_writers.append( writers.CreateWriter(args.format[i], filepath, args.hypfile, args.reffile)) if args.print_wer: # Open files for writing WER filepath = "%s.wer.%s" % (args.output, args.format[i]) wer_writers.append( writers.CreateWriter(args.format[i], filepath, args.hypfile, args.reffile)) # Assume that line counts match for refline in f_ref: linecount += 1 if args.verbose: print('===========') print('Segment %d:' % linecount) print('===========') refline = refline.strip() hypline = f_hyp.readline().strip() blank_lines = False if not refline and not hypline: # Nothing to compare blank_lines = True else: if args.verbose: print 'REF: "%s"' % refline print 'HYP: "%s"' % hypline aligner = None if args.word_align_weights: keys = ['C', 'S', 'D', 'I'] word_align_weights = dict( zip(keys, args.word_align_weights)) aligner = PowerAligner( refline, hypline, lowercase=args.lowercase, verbose=args.verbose, lexicon=args.lexicon, word_align_weights=word_align_weights) else: aligner = PowerAligner(refline, hypline, lowercase=args.lowercase, verbose=args.verbose, lexicon=args.lexicon) wer_score_components += Counter(aligner.wer_components) if args.print_wer: for writer in wer_writers: if blank_lines: writer.write_blank() else: writer.write(linecount, aligner.wer_components, aligner.wer_alignment) if args.verbose: print 'WER alignment:' print aligner.wer_alignment print 'WER: ', aligner.wer print 'Errors:', aligner.wer_components print '===============' aligner.align() power_score_components += Counter(aligner.power_components) if args.show_confusions: if args.print_wer: # TODO: Refactor cp = aligner.wer_alignment.confusion_pairs() for key in cp.iterkeys(): wer_confusions[key] += cp[key] cp = aligner.power_alignment.confusion_pairs() for key in cp.iterkeys(): power_confusions[key] += cp[key] if args.verbose: print 'Error Regions:' for i in aligner.error_indexes: print aligner.split_regions[i] print aligner.phonetic_alignments[i] print '-----' print '===============' print 'POWER alignment:' print aligner.power_alignment print 'POWER: ', aligner.power print 'Errors:', aligner.power_components print '===============' print "" # Write POWER info for writer in power_writers: if blank_lines: writer.write_blank() elif args.show_phonemes: writer.write(linecount, aligner.power_components, aligner.power_alignment, aligner.phonetic_alignments) else: writer.write(linecount, aligner.power_components, aligner.power_alignment) # Close all output files for writer in wer_writers: writer.finalize() for writer in power_writers: writer.finalize() # Compare final WER to POWER print "=============" print "Final scores:" final_wer = (wer_score_components['S'] + wer_score_components['D'] + wer_score_components['I']) / wer_score_components['L'] final_power = (power_score_components['S'] + power_score_components['D'] + power_score_components['I']) / power_score_components['L'] print "WER: ", final_wer print wer_score_components print "POWER:", final_power print power_score_components diff_score = final_power - final_wer diff_components = copy.deepcopy(power_score_components) diff_components.subtract(wer_score_components) print "" print "Score component difference (POWER vs WER):" print "Diff: ", diff_score print diff_components print "=============" if args.compare: writers.CompareWriter.write_comparison( "%s.rsum" % args.output, args.hypfile, args.reffile, linecount, final_power, final_wer, power_score_components, wer_score_components, diff_score, diff_components) if args.show_confusions: if 'txt' in args.show_confusions: writers.ConfusionPairWriter.write("%s.power.conf" % args.output, args.hypfile, args.reffile, power_confusions) if args.print_wer: writers.ConfusionPairWriter.write("%s.wer.conf" % args.output, args.hypfile, args.reffile, wer_confusions) if 'json' in args.show_confusions: writers.ConfusionPairWriter.write_json( "%s.power.conf.json" % args.output, args.hypfile, args.reffile, power_confusions) if args.print_wer: writers.ConfusionPairWriter.write_json( "%s.wer.conf.json" % args.output, args.hypfile, args.reffile, wer_confusions)