methods.append("swbc-gold+fp-et+res") methods.append("swbc-gold+fp-et-res") methods.append("swbc-gold-fp+et+res") methods.append("swbc-gold-fp+et-res") methods.append("swbc-gold-fp-et+res") methods.append("swbc-gold-fp-et-res") for method in methods: # For each method, compare system outputs to gold transcripts cleaned with given method. c_gold_A = "./results/cleaned-golds/" + method + "/sw" + code + "-A.out" c_gold_B = "./results/cleaned-golds/" + method + "/sw" + code + "-B.out" trans_a = "./results/" + sub_dir + "split-system-trans-text/left-sw0" + code + "-trans.txt" trans_b = "./results/" + sub_dir + "split-system-trans-text/right-sw0" + code + "-trans.txt" # Find WER assuming left channel is speaker A. left_a = full_wer(c_gold_A, trans_a) right_b = full_wer(c_gold_B, trans_b) mean1 = (left_a + right_b)/2 # Find WER assuming left channel is speaker B. left_b = full_wer(c_gold_B, trans_a) right_a = full_wer(c_gold_A, trans_b) mean2 = (left_b + right_a)/2 # Store minimum as that is WER when channel and speaker are correctly matched. correct_mean = min(mean1, mean2) outfile = "./results/" + sub_dir + "split-wer/" + method + "/wer-" + code + "-mean.txt" with open(outfile, 'w') as f: f.write(str(correct_mean))
split_path = "./results/" + system + "/split-wer/original/" max_diff = 0 for wer_file in os.listdir(split_path): # For each split WER file (output from split format scripts), gather all relevant files. code = re.search(r"[0-9]{4}", wer_file).group(0) trans_full = "./results/" + system + "/system-trans-text/" + system + "-sw0" + code + "-mono-trans.txt" gold_full = "./results/gold-trans-text/" + code + "-full-joined-transcript.txt" trans_left = "./results/" + system + "/split-system-trans-text/left-sw0" + code + "-trans.txt" trans_right = "./results/" + system + "/split-system-trans-text/right-sw0" + code + "-trans.txt" gold_a = "./results/gold-split-text/a/" + code + "-A-joined-transcript.txt" gold_b = "./results/gold-split-text/b/" + code + "-B-joined-transcript.txt" try: # Calc WER on transcript with both speakers. joined_wer = full_wer(gold_full, trans_full) # Find WERs assuming left channel = speaker A left_a = full_wer(gold_a, trans_left) right_b = full_wer(gold_b, trans_right) # Find WERs assuming left channel = speaker B left_b = full_wer(gold_b, trans_left) right_a = full_wer(gold_a, trans_right) except FileNotFoundError: # Note any instances where a file is missing. print(code) pass if joined_wer and left_a and left_b and left_b and right_b: # Calculate overall split WERs by finding the mean of Speakers A & B.
prep = "sw0" + code + "-mono" incsv = "./results/msoft/universal/msoft-" + prep + ".csv" intext = "./results/msoft/system-trans-text/msoft-" + prep + "-trans.txt" gold_text = "./results/gold-trans-text/" + code + "-full-joined-transcript.txt" gold_original = "./results/gold-timings/" + code + "-full-transcript.json" # Now we have all the files, the final metrics are calculated. # Stability stab_file = "./results/msoft/stability/" + prep + ".json" stability = calc_stability(incsv) with open(stab_file, 'w') as outfile: json.dump(stability, outfile) # WER full_wer_file = "./results/msoft/full-wer/" + prep + ".txt" word_error_rate = full_wer(gold_text, intext) with open(full_wer_file, 'w') as wer_out: wer_out.write(str(word_error_rate)) # Latencies fo_latency_file = "./results/msoft/latency/fo/" + prep + ".json" fd_latency_file = "./results/msoft/latency/fd/" + prep + ".json" fo_latencies, fd_latencies = latency(intext, incsv, gold_original, 'microsoft') with open(fo_latency_file, 'w') as fo_out: json.dump(fo_latencies, fo_out) with open(fd_latency_file, 'w') as fd_out: json.dump(fd_latencies, fd_out)
else: print("System not implemented" ) ## TODO Add elif statements as new systems are added. # These are the files needed to calculate the mean WER over the split channels (per swb code) trans_left = "./results/" + sub_dir + "split-system-trans-text/left-sw0" + code + "-trans.txt" trans_right = "./results/" + sub_dir + "split-system-trans-text/right-sw0" + code + "-trans.txt" gold_a = "./results/gold-split-text/a/" + code + "-A-joined-transcript.txt" gold_b = "./results/gold-split-text/b/" + code + "-B-joined-transcript.txt" # The left audio channel may not be speaker A (and vice versa). # To find this out, we find the mean WER for (left=A, right=B) & (left=B, right=A) # Then we output the minimum as that is clearly when the audio channel transcript (from the system) matches the speaker transcript. # This calculation finds the mean WER assuming the left channel is speaker A. left_a = full_wer(gold_a, trans_left) right_b = full_wer(gold_b, trans_right) mean1 = (left_a + right_b) / 2 # This calculation finds the mean WER assuming the left channel is speaker B. left_b = full_wer(gold_b, trans_left) right_a = full_wer(gold_a, trans_right) mean2 = (left_b + right_a) / 2 # Again, the mimimum of these means is when the channels and speakers are corectly matched. # Obviously the other mean is larger because the transcript is matched with the wrong speaker. correct_mean = min(mean1, mean2) # Store. outfile = "./results/" + sub_dir + "split-wer/original/wer-" + code + "-mean.txt" with open(outfile, 'w') as f: