def score(ref_trn=None, hyp_trn=None):
# ref_trn="hello world"
# hyp_trn="hello duck"
ref_trn = "/Users/user/github/spoken_langauge_understanding/ms_speech_recognition_system/M1_Introduction/misc/ref.trn"
hyp_trn = "/Users/user/github/spoken_langauge_understanding/ms_speech_recognition_system/M1_Introduction/misc/hyp.trn"
ref_trn_list = normalize_sent(ref_trn)
hyp_trn_list = normalize_sent(hyp_trn)
for ref, hyp in zip(ref_trn_list, hyp_trn_list):
num_tokens, num_errors, num_deletions, num_insertions, num_substitutions = wer.string_edit_distance(ref=ref, hyp=hyp)
wer_result = (num_substitutions + num_deletions + num_insertions) / num_tokens
print(wer_result)
print("-----------------------------------")
print("Sentence Error Rate")
print("-----------------------------------")
print("-----------------------------------")
print("Word Error Rate")
print("Sum: N={}, Err={}, Sub={}, Del={}, Ins={}".format(num_tokens, num_errors, \
num_deletions, num_insertions, num_substitutions))
print("Avg: N={}, Err={}%, Sub={}%, Del={}%, Ins={}%".format(
num_tokens, percent_round_off(num_errors, num_tokens), \
percent_round_off(num_deletions, num_tokens), percent_round_off(num_insertions, num_tokens), \
percent_round_off(num_substitutions, num_tokens)))
print("-----------------------------------")
return
def score(ref_trn=None, hyp_trn=None):
sentence_error = 0
num_words = 0
word_error = 0
sub_error = 0
del_error = 0
ins_error = 0
ref_trn_tokens = {}
hyp_trn_tokens = {}
fh = open(ref_trn)
for line in fh:
ref_trn_tokens[line.split()[-1]] = line.split()[:-1]
fh.close()
fh = open(hyp_trn)
for line in fh:
hyp_trn_tokens[line.split()[-1]] = line.split()[:-1]
fh.close()
for key, value in ref_trn_tokens.items():
num_tokens, num_errors, num_deletions, num_insertions, num_substitutions = wer.string_edit_distance(
ref=value, hyp=hyp_trn_tokens[key])
num_words += num_tokens
if (num_errors != 0):
sentence_error += 1
word_error += num_errors
sub_error += num_substitutions
del_error += num_deletions
ins_error += num_insertions
print("id: {0:s}".format(key))
print("Scores: N={0:d}, S={4:d}, D={2:d}, I={3:d}".format(
num_tokens, num_errors, num_deletions, num_insertions,
num_substitutions))
print()
print('---------------------------------')
print('Sentence Error Rate:')
print('Sum: N={0:d}, Err={1:d}'.format(len(ref_trn_tokens),
sentence_error))
print('Avg: N={0:d}, Err={1:0.2f}%'.format(
len(ref_trn_tokens), 100 * sentence_error / len(ref_trn_tokens)))
print('---------------------------------')
print('Word Error Rate:')
print('Sum: N={0:d}, Err={1:d}, Sub={2:d}, Del={3:d}, Ins={4:d}'.format(
num_words, word_error, sub_error, del_error, ins_error))
print(
'Avg: N={0:d}, Err={1:0.2f}%, Sub={2:0.2f}%, Del={3:0.2f}%, Ins={4:0.2f}%'
.format(num_words, 100 * word_error / num_words,
100 * sub_error / num_words, 100 * del_error / num_words,
100 * ins_error / num_words))
return
def score(ref_trn=None, hyp_trn=None):
reference_string = ""
hypothesis_string = ""
num_sentences = 0
num_words = 0
num_sentences_error = 0
num_deletions = 0
num_insertions = 0
num_substitutions = 0
num_errors = 0
with open(ref_trn) as reference, open(hyp_trn) as hypothesis:
for lineR, lineH in zip(reference, hypothesis):
num_sentences += 1
id_pattern = re.compile(r'\(.+\)')
id_ref_match = id_pattern.search(lineR)
reference_string = lineR[:id_ref_match.start()]
id_hyp_match = id_pattern.search(lineH)
hypothesis_string = lineH[:id_hyp_match.start()]
ref_word_list = reference_string.split()
hyp_word_list = hypothesis_string.split()
num_words += len(reference_string.split())
#print(reference_string + id_ref_match[0])
#print(hypothesis_string + id_hyp_match[0])
tokens, edits, deletions, insertions, substitutions = wer.string_edit_distance(
ref=ref_word_list, hyp=hyp_word_list)
if deletions != 0 or insertions != 0 or substitutions != 0:
num_sentences_error += 1
num_errors += edits
num_deletions += deletions
num_insertions += insertions
num_substitutions += substitutions
print(f'''id: {id_ref_match[0]}
Scores: N={tokens}, E={edits}, D={deletions}, I={insertions}, S={substitutions}
''')
print(f'''-----------------------------------
Sentence Error Rate:
Sum: N={num_sentences}, Err={num_sentences_error}
Avg: N={num_sentences}, Err={num_sentences_error/num_sentences*100:.2f}%
-----------------------------------
Word Error Rate:
Sum: N={num_words}, Err={num_errors}, Sub={num_substitutions}, Del={num_deletions}, Ins={num_insertions}
Avg: N={num_words}, Err={num_errors/num_words*100:.2f}%, Sub={num_substitutions/num_words*100:.2f}%, Del={num_deletions/num_words*100:.2f}%, Ins={num_insertions/num_words*100:.2f}%
-----------------------------------''')
return
def score(ref_trn=None, hyp_trn=None):
ref_dict = read_trn(ref_trn)
hyp_dict = read_trn(hyp_trn)
assert len(ref_dict) == len(hyp_dict)
word2idx = {}
word_to_idx(ref_dict, word2idx)
word_to_idx(hyp_dict, word2idx)
Err = 0
Sub = 0
Del = 0
Ins = 0
Tks = 0
ErrSen = 0
for hyp_k, hyp_v in hyp_dict.items():
if hyp_k not in ref_dict:
raise RuntimeError("key {} not found in ref".format(hyp_k))
if hyp_k == '3170-137482-0006':
print("hi")
ref_v = ref_dict[hyp_k]
ref_s = sen_to_idxs(ref_v, word2idx)
hyp_s = sen_to_idxs(hyp_v, word2idx)
(N, E, D, I, S) = wer.string_edit_distance(ref_s, hyp_s)
if E > 0:
ErrSen = ErrSen + 1
Tks = Tks + N
Err = Err + E
Sub = Sub + S
Del = Del + D
Ins = Ins + I
print("id: ({})".format(hyp_k))
print("Scores: N={}, S={}, D={}, I={}\n".format(N, S, D, I))
N = len(ref_dict)
print("-----------------------------------")
print("Sentence Error Rate:")
print("Sum: N={}, Err={}".format(N, ErrSen))
print("Avg: N={}, Err={:0.2f}%".format(N, 100. * ErrSen / N))
print("-----------------------------------")
print("Word Error Rate:")
print("Sum: N={}, Err={}, Sub={}, Del={}, Ins={}".format(
Tks, Err, Sub, Del, Ins))
print("Avg: N={}, Err={:0.2f}%, Sub={:0.2f}%, Del={:0.2f}%, Ins={:0.2f}%".
format(Tks, 100. * Err / Tks, 100. * Sub / Tks, 100. * Del / Tks,
100. * Ins / Tks))
print("-----------------------------------")
def score(ref_trn=None, hyp_trn=None):
wer_sum = wer_err = wer_del = wer_ins = wer_sub = 0
ser_sum = ser_err = 0
for ref_line, hyp_line in zip(open(ref_trn), open(hyp_trn)):
ref_m = re.search('(.*) \((.*)\)', ref_line)
hyp_m = re.search('(.*) \((.*)\)', hyp_line)
track_id = ref_m.group(2)
ref_string = ref_m.group(1).split()
hyp_string = hyp_m.group(1).split()
print("id: ({})".format(track_id))
print("ref: {}".format(ref_string))
print("hyp: {}".format(hyp_string))
num_tokens, num_errors, num_deletions, num_insertions, num_substitutions = wer.string_edit_distance(
ref=ref_string, hyp=hyp_string)
print("Score N={} S={} D={} I={}".format(num_tokens, num_substitutions,
num_deletions,
num_insertions))
wer_sum += num_tokens
wer_err += num_errors
wer_del += num_deletions
wer_sub += num_substitutions
wer_ins += num_insertions
ser_sum += 1
ser_err += 0 if num_errors == 0 else 1
print("")
print("-----------------")
print("Sentence Error Rate:")
print("Sum: N={} Err={}".format(ser_sum, ser_err))
print("Avg: N={} Err={:.0%}".format(ser_sum, ser_err / ser_sum))
print("-----------------")
print("Sentence Error Rate:")
print("Sum: N={} Err={} Sub={} Del={} Ins={}".format(
wer_sum, wer_err, wer_sub, wer_del, wer_ins))
print("Avg: N={} Err={:.0%} Sub={:.0%} Del={:.0%} Ins={:.0%}".format(
wer_sum, wer_err / wer_sum, wer_sub / wer_sum, wer_del / wer_sum,
wer_ins / wer_sum))
print("-----------------")
return
def score(ref_trn=None, hyp_trn=None):
ref = trn_to_dict('misc/ref.trn')
hyp = trn_to_dict('misc/hyp.trn')
n_error_sentences = 0
n_sentences = 0
n_word = 0
n_error_word = 0
total_del = 0
total_insert = 0
total_sub = 0
for key in ref:
hyp_trans = hyp.get(key)
ref_trans = ref[key]
num_tokens, num_errors, num_deletions, num_insertions, num_substitutions = wer.string_edit_distance(ref_trans, hyp_trans)
n_word += num_tokens
n_error_word += num_errors
total_del += num_deletions
total_insert += num_insertions
total_sub += num_substitutions
n_sentences += 1
n_error_sentences += (1 if num_errors > 0 else 0)
arg = {
'nsentence': n_sentences,
'n_error_sentence': n_error_sentences,
'ser': n_error_sentences//n_sentences,
'nwords': n_word,
'nsub': num_substitutions,
'ninsert': num_insertions,
'ndel': num_deletions,
'psub': num_substitutions/n_word * 100,
'pins': num_insertions/n_word * 100,
'pdel': num_deletions/n_word * 100,
'wer': n_error_word/n_word * 100,
}
report_str = ('Total sendtences: {nsentence},\n'
'Num sentences with an error: {n_error_sentence},\n'
'SER: {ser},\n'
'Total words: {nwords},\n'
'nsub, ninsert, ndel: {nsub}, {ninsert}, {ndel},\n'
'psub, pinsert, pdel: {psub}%, {pins}%, {pdel}%,\n'
'WER: {wer}').format(**arg)
print(report_str)
return
def score(ref_trn=None, hyp_trn=None):
# Lectura de archivos .trf
filehand_ref = open(ref_trn, 'r')
filehand_hyp = open(hyp_trn, 'r')
# Conteo de la cantidad de frases
number_sentences = 0
# Variable para contar las frases con errores
sentences_error = 0
# Palabras dentro del archivo REF
reference_words = 0
# Errores globales
errors = 0
sust = 0
inser = 0
delet = 0
print('\nResultados:\n')
# Loop dentro del archio REF
while True:
line_r = filehand_ref.readline()
if not line_r:
break
number_sentences += 1
line_list_r = line_r.split()
# Separo una línea en PRHASE e ID
phrase_ID = line_list_r.pop(-1)
phrase_words_r = line_list_r
reference_words = reference_words + len(phrase_words_r)
# LOOP dentro de archivo HYP para cada linea del Archivo REF, usando el ID como tag
while True:
line_h = filehand_hyp.readline()
line_list_h = line_h.split()
if line_list_h[-1] == phrase_ID:
line_list_h.pop(-1)
phrase_words_h = line_list_h
num_tokens, num_errors, num_deletions, num_insertions, num_substitutions = wer.string_edit_distance(
ref=phrase_words_r, hyp=phrase_words_h)
errors = errors + num_errors
sust = sust + num_substitutions
inser = inser + num_insertions
delet = delet + num_deletions
WER = (num_substitutions + num_insertions +
num_deletions) / len(phrase_words_r)
if not num_errors == 0:
sentences_error += 1
print('ID:', phrase_ID, '\n'
'Palabras de referencia:', phrase_words_r, '\n'
'Palabras Adivinadas:', phrase_words_h, '\n'
'E:', num_errors, 'D:', num_deletions, 'S:',
num_substitutions, 'I:', num_insertions)
break
sentences_error_perc = round(sentences_error / number_sentences * 100)
WER_global = round(errors / reference_words * 100)
Del_global = round(delet / reference_words * 100)
Sust_global = round(sust / reference_words * 100)
Ins_global = round(inser / reference_words * 100)
# Salida en formato text
print('El numero de sentencias es:', number_sentences,
'\nEl numero de Sentencias con errores es:', sentences_error, '\n'
'El porcentaje de sentencias con errores es', sentences_error_perc,
'% \n'
'El numero total de palabras en el archivo REF es', reference_words,
'\n'
'Errores totales:', errors, 'WER:', Sust_global, '% \n'
'Sustituciones totales:', sust, Sust_global, ' % \n'
'Inserciones totales:', inser, Ins_global, ' % \n'
'Supreciones totales:', delet, Del_global, ' % \n')
return
def score(ref_trn=None, hyp_trn=None):
t_sen = 0
t_esen = 0
t_w = 0
t_i = 0
t_d = 0
t_s = 0
t_e = 0
with open(ref_trn) as f1, open(hyp_trn) as f2:
for x, y in zip(f1, f2):
x = x.split()
y = y.split()
#try:
# x.remove('')
# y.remove('')
#except:
# pass
num_tokens, num_errors, num_deletions, num_insertions, num_substitutions = wer.string_edit_distance(
ref=x, hyp=y)
print("Id:" + str(x[len(x) - 1]) +
"Scores: N=%i ,S=%i ,D=%i ,I=%i" %
(num_tokens - 1, num_substitutions, num_deletions,
num_insertions))
print(" ")
t_sen = t_sen + 1
if num_errors > 0:
t_esen = t_esen + 1
t_w = t_w + num_tokens - 1
t_d = t_d + num_deletions
t_s = t_s + num_substitutions
t_i = t_i + num_insertions
t_e = t_e + num_errors
print("Sentence Error Rate:")
print("Sum: N=%i ,Error Sentences=%i" % (t_sen, t_esen))
print("Average: N=%i ,Error precentage=%f" % (t_sen,
(t_esen / t_sen) * 100))
print("Word Error Rate:")
print(
"Sum: N=%i ,Total Error=%i, Substitution=%i, Deletion=%i, Insertion=%i"
% (t_w, t_e, t_s, t_d, t_i))
print(
"Average: N=%i ,Error precentage=%f,Subs. Error precentage=%f,Del. Error precentage=%f,Inser. Error precentage=%f"
% (t_w, (t_e / t_w) * 100, (t_s / t_w) * 100, (t_d / t_w) * 100,
(t_i / t_w) * 100))
return
def score(ref_trn=None, hyp_trn=None):
ref_trn_dic, hyp_trn_dic = build_transcriptions_dics(ref_trn, hyp_trn)
total_num_of_words = 0
total_num_of_ref_sent = len(ref_trn_dic)
sentences_with_errors = 0
total_errors = 0
total_substitutions, total_insertions, total_deletions = 0, 0, 0
for (key, value) in hyp_trn_dic.items():
reference_string = ref_trn_dic[key]
hypothesis_string = value
for ref_word, hyp_word in zip(reference_string.split(),
hypothesis_string.split()):
num_tokens, num_errors, num_deletions, num_insertions, num_substitutions = wer.string_edit_distance(
ref=ref_word, hyp=hyp_word)
total_num_of_words += num_tokens
total_errors += num_errors
total_deletions += num_deletions
total_insertions += num_insertions
total_substitutions += num_substitutions
if num_errors > 0:
sentences_with_errors += 1
print()
print(
"====================================================================")
print("Total number of reference sentences: " + str(total_num_of_ref_sent))
print("Num of sentences with error: " + str(sentences_with_errors))
print("SER: " + __format_str(total_num_of_ref_sent, sentences_with_errors))
print()
print("Total number of:")
print("\tWords: " + str(total_num_of_words))
print("\tErrors: " + str(total_errors))
print("\tDeletions: " + str(total_deletions))
print("\tInsertions: " + str(total_insertions))
print("\tSubstitutions: " + str(total_substitutions))
print(
"WER: " +
__format_str(total_substitutions + total_insertions +
total_deletions, total_num_of_words))
print(
"====================================================================")
return
def score(ref_trn=None, hyp_trn=None):
num_ref_sent, ref_dict = read_file(ref_trn) #refernce transcription
_, hyp_dict = read_file(hyp_trn)
num_err_sents = 0 #number of sentences with errors
total_ref_w = 0 #total number of reference words
w_errors = 0 #total number of word errors
total_deletions = 0
total_insertions = 0
total_substitutions = 0
for k in ref_dict:
if k in hyp_dict:
#print(k)
num_tokens, num_errors, num_deletions, num_insertions, num_substitutions = wer.string_edit_distance(
ref_dict[k], hyp_dict[k])
# print(num_tokens, num_errors, num_deletions, num_insertions, num_substitutions)
total_ref_w += num_tokens #increas number of reference words
if num_errors != 0:
num_err_sents += 1 #increase number of error sentences
w_errors += num_errors #increase number of error words
total_deletions += num_deletions
total_insertions += num_insertions
total_substitutions += num_substitutions
###reports##
WER = w_errors / total_ref_w #word error rate
SER = num_err_sents / num_ref_sent #sentence error rate
deletions_per = total_deletions / total_ref_w #he percentage of deletions
insertions_per = total_insertions / total_ref_w #The percentage of insertions
substitutions_per = total_substitutions / total_ref_w #The percentage of substitutions
output = """Total number of reference sentences in the test set: {} sentences \n
Number of sentences with an error :{} \n
SER :{:2f} % \n
Total number of reference words : {} words \n
Total number of word errors : {} \n
Total number of word substitutions, insertions, and deletions : {} , {} , {} \n
WER :{:2f} % \n "
percentage of substitutions, insertions, and deletions : {:2f} % ,{:2f} % , {:2f} % """.format(
num_ref_sent, num_err_sents, 100 * SER, total_ref_w, w_errors,
total_substitutions, total_insertions, total_deletions, 100 * WER,
100 * substitutions_per, 100 * insertions_per, 100 * deletions_per)
print(output)
return WER, SER
