def test_benchmark_not_none(self):
self.assertIsNotNone(
word_tokenization.benchmark(
["วัน", "จัน", "ทร์", "สี", "เหลือง"],
["วัน", "จันทร์", "สี", "เหลือง"],
)
)
def test_benchmark(self):
expected = []
actual = []
for pair in TEST_DATA["sentences"]:
expected.append(pair["expected"])
actual.append(pair["actual"])
df = word_tokenization.benchmark(expected, actual)
self.assertIsNotNone(df)
def __init__(self, name, argv):
parser = argparse.ArgumentParser(**cli.make_usage("benchmark " + name))
parser.add_argument(
"--input-file",
action="store",
help="Path to input file to compare against the test file",
)
parser.add_argument(
"--test-file",
action="store",
help="Path to test file i.e. ground truth",
)
parser.add_argument(
"--save-details",
default=False,
action="store_true",
help=("Save comparison details to files (eval-XXX.json"
" and eval-details-XXX.json)"),
)
args = parser.parse_args(argv)
actual = _read_file(args.input_file)
expected = _read_file(args.test_file)
assert len(actual) == len(
expected
), "Input and test files do not have the same number of samples"
print("Benchmarking %s against %s with %d samples in total" %
(args.input_file, args.test_file, len(actual)))
df_raw = word_tokenization.benchmark(expected, actual)
columns = [
"char_level:tp",
"char_level:fp",
"char_level:tn",
"char_level:fn",
"word_level:correctly_tokenised_words",
"word_level:total_words_in_sample",
"word_level:total_words_in_ref_sample",
]
statistics = dict()
for c in columns:
statistics[c] = float(df_raw[c].sum())
statistics["char_level:precision"] = statistics["char_level:tp"] / (
statistics["char_level:tp"] + statistics["char_level:fp"])
statistics["char_level:recall"] = statistics["char_level:tp"] / (
statistics["char_level:tp"] + statistics["char_level:fn"])
statistics["word_level:precision"] = (
statistics["word_level:correctly_tokenised_words"] /
statistics["word_level:total_words_in_sample"])
statistics["word_level:recall"] = (
statistics["word_level:correctly_tokenised_words"] /
statistics["word_level:total_words_in_ref_sample"])
print("============== Benchmark Result ==============")
for c in ["tp", "fn", "tn", "fp", "precision", "recall"]:
c = f"char_level:{c}"
v = statistics[c]
print(f"{c:>40s} {v:.4f}")
for c in [
"total_words_in_sample",
"total_words_in_ref_sample",
"correctly_tokenised_words",
"precision",
"recall",
]:
c = f"word_level:{c}"
v = statistics[c]
print(f"{c:>40s} {v:.4f}")
if args.save_details:
dir_name = os.path.dirname(args.input_file)
file_name = args.input_file.split("/")[-1].split(".")[0]
res_path = "%s/eval-%s.yml" % (dir_name, file_name)
print("Evaluation result is saved to %s" % res_path)
with open(res_path, "w", encoding="utf-8") as outfile:
yaml.dump(statistics, outfile, default_flow_style=False)
res_path = "%s/eval-details-%s.json" % (dir_name, file_name)
print("Details of comparisons is saved to %s" % res_path)
with open(res_path, "w", encoding="utf-8") as f:
samples = []
for i, r in enumerate(df_raw.to_dict("records")):
expected, actual = r["expected"], r["actual"]
del r["expected"]
del r["actual"]
samples.append(
dict(metrics=r, expected=expected, actual=actual,
id=i))
details = dict(metrics=statistics, samples=samples)
json.dump(details, f, ensure_ascii=False)