methods.append("swbc-gold+fp-et+res")
methods.append("swbc-gold+fp-et-res")
methods.append("swbc-gold-fp+et+res")
methods.append("swbc-gold-fp+et-res")
methods.append("swbc-gold-fp-et+res")
methods.append("swbc-gold-fp-et-res")
for method in methods:
# For each method, compare system outputs to gold transcripts cleaned with given method.
c_gold_A = "./results/cleaned-golds/" + method + "/sw" + code + "-A.out"
c_gold_B = "./results/cleaned-golds/" + method + "/sw" + code + "-B.out"
trans_a = "./results/" + sub_dir + "split-system-trans-text/left-sw0" + code + "-trans.txt"
trans_b = "./results/" + sub_dir + "split-system-trans-text/right-sw0" + code + "-trans.txt"
# Find WER assuming left channel is speaker A.
left_a = full_wer(c_gold_A, trans_a)
right_b = full_wer(c_gold_B, trans_b)
mean1 = (left_a + right_b)/2
# Find WER assuming left channel is speaker B.
left_b = full_wer(c_gold_B, trans_a)
right_a = full_wer(c_gold_A, trans_b)
mean2 = (left_b + right_a)/2
# Store minimum as that is WER when channel and speaker are correctly matched.
correct_mean = min(mean1, mean2)
outfile = "./results/" + sub_dir + "split-wer/" + method + "/wer-" + code + "-mean.txt"
with open(outfile, 'w') as f:
f.write(str(correct_mean))
split_path = "./results/" + system + "/split-wer/original/"
max_diff = 0
for wer_file in os.listdir(split_path):
# For each split WER file (output from split format scripts), gather all relevant files.
code = re.search(r"[0-9]{4}", wer_file).group(0)
trans_full = "./results/" + system + "/system-trans-text/" + system + "-sw0" + code + "-mono-trans.txt"
gold_full = "./results/gold-trans-text/" + code + "-full-joined-transcript.txt"
trans_left = "./results/" + system + "/split-system-trans-text/left-sw0" + code + "-trans.txt"
trans_right = "./results/" + system + "/split-system-trans-text/right-sw0" + code + "-trans.txt"
gold_a = "./results/gold-split-text/a/" + code + "-A-joined-transcript.txt"
gold_b = "./results/gold-split-text/b/" + code + "-B-joined-transcript.txt"
try:
# Calc WER on transcript with both speakers.
joined_wer = full_wer(gold_full, trans_full)
# Find WERs assuming left channel = speaker A
left_a = full_wer(gold_a, trans_left)
right_b = full_wer(gold_b, trans_right)
# Find WERs assuming left channel = speaker B
left_b = full_wer(gold_b, trans_left)
right_a = full_wer(gold_a, trans_right)
except FileNotFoundError:
# Note any instances where a file is missing.
print(code)
pass
if joined_wer and left_a and left_b and left_b and right_b:
# Calculate overall split WERs by finding the mean of Speakers A & B.
prep = "sw0" + code + "-mono"
incsv = "./results/msoft/universal/msoft-" + prep + ".csv"
intext = "./results/msoft/system-trans-text/msoft-" + prep + "-trans.txt"
gold_text = "./results/gold-trans-text/" + code + "-full-joined-transcript.txt"
gold_original = "./results/gold-timings/" + code + "-full-transcript.json"
# Now we have all the files, the final metrics are calculated.
# Stability
stab_file = "./results/msoft/stability/" + prep + ".json"
stability = calc_stability(incsv)
with open(stab_file, 'w') as outfile:
json.dump(stability, outfile)
# WER
full_wer_file = "./results/msoft/full-wer/" + prep + ".txt"
word_error_rate = full_wer(gold_text, intext)
with open(full_wer_file, 'w') as wer_out:
wer_out.write(str(word_error_rate))
# Latencies
fo_latency_file = "./results/msoft/latency/fo/" + prep + ".json"
fd_latency_file = "./results/msoft/latency/fd/" + prep + ".json"
fo_latencies, fd_latencies = latency(intext, incsv, gold_original, 'microsoft')
with open(fo_latency_file, 'w') as fo_out:
json.dump(fo_latencies, fo_out)
with open(fd_latency_file, 'w') as fd_out:
json.dump(fd_latencies, fd_out)
else:
print("System not implemented"
) ## TODO Add elif statements as new systems are added.
# These are the files needed to calculate the mean WER over the split channels (per swb code)
trans_left = "./results/" + sub_dir + "split-system-trans-text/left-sw0" + code + "-trans.txt"
trans_right = "./results/" + sub_dir + "split-system-trans-text/right-sw0" + code + "-trans.txt"
gold_a = "./results/gold-split-text/a/" + code + "-A-joined-transcript.txt"
gold_b = "./results/gold-split-text/b/" + code + "-B-joined-transcript.txt"
# The left audio channel may not be speaker A (and vice versa).
# To find this out, we find the mean WER for (left=A, right=B) & (left=B, right=A)
# Then we output the minimum as that is clearly when the audio channel transcript (from the system) matches the speaker transcript.
# This calculation finds the mean WER assuming the left channel is speaker A.
left_a = full_wer(gold_a, trans_left)
right_b = full_wer(gold_b, trans_right)
mean1 = (left_a + right_b) / 2
# This calculation finds the mean WER assuming the left channel is speaker B.
left_b = full_wer(gold_b, trans_left)
right_a = full_wer(gold_a, trans_right)
mean2 = (left_b + right_a) / 2
# Again, the mimimum of these means is when the channels and speakers are corectly matched.
# Obviously the other mean is larger because the transcript is matched with the wrong speaker.
correct_mean = min(mean1, mean2)
# Store.
outfile = "./results/" + sub_dir + "split-wer/original/wer-" + code + "-mean.txt"
with open(outfile, 'w') as f: