def evaluate(self, test_lines): """Calculate and print out word error rate (WER) and Accuracy on test sample. Args: test_lines: List of test dictionary. Each element of list must be String containing word and its pronounciation (e.g., "word W ER D"); """ test_dic = data_utils.collect_pronunciations(test_lines) if len(test_dic) < 1: print("Test dictionary is empty") return if self.mode == 'g2p': print('Beginning calculation word error rate (WER) on test sample.') errors = self.calc_error(test_dic) print("Words: %d" % len(test_dic)) print("Errors: %d" % errors) print("WER: %.3f" % (float(errors)/len(test_dic))) print("Accuracy: %.3f" % float(1-(errors/len(test_dic)))) else: print('Beginning calculation pronunciation error rate (PER) on test sample.') errors, total, total_pronunciations = self.calc_error(test_dic) print("Pronunciations: %d" % total_pronunciations) print("Errors: %d" % errors) print("PER: ",float(errors)/float(total)) print("Accuracy: %.3f" % float(1-(float(errors)/total)))
def test_evaluate(self): model_dir = "tests/models/decode" with g2p.tf.Graph().as_default(): g2p_model = g2p.G2PModel(model_dir) test_lines = g2p.codecs.open( "tests/data/toydict.test", "r", "utf-8").readlines() g2p_model.evaluate(test_lines) test_dic = data_utils.collect_pronunciations(test_lines) errors = g2p_model.calc_error(test_dic) self.assertAlmostEqual(float(errors)/len(test_dic), 0.667, places=3)
def test_evaluate(self): model_dir = "models/decode" with g2p.tf.Graph().as_default(): g2p_model = g2p.G2PModel(model_dir) g2p_model.load_decode_model() test_lines = open("data/toydict.test").readlines() g2p_model.evaluate(test_lines) test_dic = data_utils.collect_pronunciations(test_lines) errors = g2p_model.calc_error(test_dic) self.assertAlmostEqual(float(errors)/len(test_dic), 0.667, places=3)
def test_evaluate(self): model_dir = "tests/models/decode" with g2p.tf.Graph().as_default(): g2p_model = g2p.G2PModel(model_dir, 'p2g') g2p_model.load_decode_model() with open("tests/data/toydict.test") as f: test_lines = f.readlines() g2p_model.evaluate(test_lines) test_dic = data_utils.collect_pronunciations(test_lines) errors, total, total_pronunciations = g2p_model.calc_error( test_dic) self.assertAlmostEqual(float(errors) / float(total), 0.630, places=3)
def evaluate(self, test_lines): """Calculate and print out word error rate (WER) and Accuracy on test sample. Args: test_lines: List of test dictionary. Each element of list must be String containing word and its pronounciation (e.g., "word W ER D"); """ if not hasattr(self, "model"): raise RuntimeError("Model not found in %s" % self.model_dir) test_dic = data_utils.collect_pronunciations(test_lines) print('Beginning calculation word error rate (WER) on test sample.') errors = self.calc_error(test_dic) print("Words : %d" % len(test_dic)) print("Errors: %d" % errors) print("WER : %.3f" % (float(errors)/len(test_dic))) print("Accuracy : %.3f" % float(1-(errors/len(test_dic))))
def evaluate(self, test_lines): """Calculate and print out word error rate (WER) and Accuracy on test sample. Args: test_lines: List of test dictionary. Each element of list must be String containing word and its pronounciation (e.g., "word W ER D"); """ if not hasattr(self, "model"): raise RuntimeError("Model not found in %s" % self.model_dir) test_dic = data_utils.collect_pronunciations(test_lines) print('Beginning calculation word error rate (WER) on test sample.') errors = self.calc_error(test_dic) print("Words : %d" % len(test_dic)) print("Errors: %d" % errors) print("WER : %.3f" % (float(errors) / len(test_dic))) print("Accuracy : %.3f" % float(1 - (errors / len(test_dic))))
def evaluate(self, test_lines): """Calculate and print out word error rate (WER) and Accuracy on test sample. Args: test_lines: List of test dictionary. Each element of list must be String containing word and its pronounciation (e.g., "word W ER D"); """ test_dic = data_utils.collect_pronunciations(test_lines) if len(test_dic) < 1: print("Test dictionary is empty") return print('Beginning calculation word error rate (WER) on test sample.') errors = self.calc_error(test_dic) print("Words: %d" % len(test_dic)) print("Errors: %d" % errors) print("WER: %.3f" % (float(errors)/len(test_dic))) print("Accuracy: %.3f" % float(1-(errors/len(test_dic))))