def evaluate(self, test_lines):
"""Calculate and print out word error rate (WER) and Accuracy
on test sample.
Args:
test_lines: List of test dictionary. Each element of list must be String
containing word and its pronounciation (e.g., "word W ER D");
"""
test_dic = data_utils.collect_pronunciations(test_lines)
if len(test_dic) < 1:
print("Test dictionary is empty")
return
if self.mode == 'g2p':
print('Beginning calculation word error rate (WER) on test sample.')
errors = self.calc_error(test_dic)
print("Words: %d" % len(test_dic))
print("Errors: %d" % errors)
print("WER: %.3f" % (float(errors)/len(test_dic)))
print("Accuracy: %.3f" % float(1-(errors/len(test_dic))))
else:
print('Beginning calculation pronunciation error rate (PER) on test sample.')
errors, total, total_pronunciations = self.calc_error(test_dic)
print("Pronunciations: %d" % total_pronunciations)
print("Errors: %d" % errors)
print("PER: ",float(errors)/float(total))
print("Accuracy: %.3f" % float(1-(float(errors)/total)))
def test_evaluate(self):
model_dir = "tests/models/decode"
with g2p.tf.Graph().as_default():
g2p_model = g2p.G2PModel(model_dir)
test_lines = g2p.codecs.open(
"tests/data/toydict.test", "r", "utf-8").readlines()
g2p_model.evaluate(test_lines)
test_dic = data_utils.collect_pronunciations(test_lines)
errors = g2p_model.calc_error(test_dic)
self.assertAlmostEqual(float(errors)/len(test_dic), 0.667, places=3)
def test_evaluate(self):
model_dir = "models/decode"
with g2p.tf.Graph().as_default():
g2p_model = g2p.G2PModel(model_dir)
g2p_model.load_decode_model()
test_lines = open("data/toydict.test").readlines()
g2p_model.evaluate(test_lines)
test_dic = data_utils.collect_pronunciations(test_lines)
errors = g2p_model.calc_error(test_dic)
self.assertAlmostEqual(float(errors)/len(test_dic), 0.667, places=3)
def test_evaluate(self):
model_dir = "tests/models/decode"
with g2p.tf.Graph().as_default():
g2p_model = g2p.G2PModel(model_dir, 'p2g')
g2p_model.load_decode_model()
with open("tests/data/toydict.test") as f:
test_lines = f.readlines()
g2p_model.evaluate(test_lines)
test_dic = data_utils.collect_pronunciations(test_lines)
errors, total, total_pronunciations = g2p_model.calc_error(
test_dic)
self.assertAlmostEqual(float(errors) / float(total),
0.630,
places=3)
def evaluate(self, test_lines):
"""Calculate and print out word error rate (WER) and Accuracy
on test sample.
Args:
test_lines: List of test dictionary. Each element of list must be String
containing word and its pronounciation (e.g., "word W ER D");
"""
if not hasattr(self, "model"):
raise RuntimeError("Model not found in %s" % self.model_dir)
test_dic = data_utils.collect_pronunciations(test_lines)
print('Beginning calculation word error rate (WER) on test sample.')
errors = self.calc_error(test_dic)
print("Words : %d" % len(test_dic))
print("Errors: %d" % errors)
print("WER : %.3f" % (float(errors)/len(test_dic)))
print("Accuracy : %.3f" % float(1-(errors/len(test_dic))))
def evaluate(self, test_lines):
"""Calculate and print out word error rate (WER) and Accuracy
on test sample.
Args:
test_lines: List of test dictionary. Each element of list must be String
containing word and its pronounciation (e.g., "word W ER D");
"""
if not hasattr(self, "model"):
raise RuntimeError("Model not found in %s" % self.model_dir)
test_dic = data_utils.collect_pronunciations(test_lines)
print('Beginning calculation word error rate (WER) on test sample.')
errors = self.calc_error(test_dic)
print("Words : %d" % len(test_dic))
print("Errors: %d" % errors)
print("WER : %.3f" % (float(errors) / len(test_dic)))
print("Accuracy : %.3f" % float(1 - (errors / len(test_dic))))
def evaluate(self, test_lines):
"""Calculate and print out word error rate (WER) and Accuracy
on test sample.
Args:
test_lines: List of test dictionary. Each element of list must be String
containing word and its pronounciation (e.g., "word W ER D");
"""
test_dic = data_utils.collect_pronunciations(test_lines)
if len(test_dic) < 1:
print("Test dictionary is empty")
return
print('Beginning calculation word error rate (WER) on test sample.')
errors = self.calc_error(test_dic)
print("Words: %d" % len(test_dic))
print("Errors: %d" % errors)
print("WER: %.3f" % (float(errors)/len(test_dic)))
print("Accuracy: %.3f" % float(1-(errors/len(test_dic))))
