def test_renaming(): generate_artifacts() cmd_args = [ "yttm", "encode", f"--model={RENAME_ID_MODEL_FILE}", "--output_type=id", "--bos", "--n_threads=1", ] run(cmd_args, stdin=open(TEST_FILE, "r"), stdout=open("log.txt", "w"), check=True) file_starts_with("log.txt", "29") cmd_args = [ "yttm", "encode", f"--model={RENAME_ID_MODEL_FILE}", "--output_type=id", "--eos", "--reverse", "--n_threads=1", ] run(cmd_args, stdin=open(TEST_FILE, "r"), stdout=open("log.txt", "w"), check=True) file_starts_with("log.txt", "1148") os.remove("log.txt")
def test_bos_eos_reverse(): generate_artifacts() cmd_args = [ "yttm", "encode", f"--model={BASE_MODEL_FILE}", "--output_type=subword", "--n_threads=1", "--bos", ] run(cmd_args, stdin=open(TEST_FILE, "r"), stdout=open("log.txt", "w"), check=True) file_starts_with("log.txt", "<BOS>") cmd_args = [ "yttm", "encode", f"--model={BASE_MODEL_FILE}", "--output_type=subword", "--n_threads=1", "--reverse", "--eos", ] run(cmd_args, stdin=open(TEST_FILE, "r"), stdout=open("log.txt", "w"), check=True) file_starts_with("log.txt", "<EOS>") cmd_args = [ "yttm", "encode", f"--model={BASE_MODEL_FILE}", "--output_type=id", "--n_threads=1", "--bos", ] run(cmd_args, stdin=open(TEST_FILE, "r"), stdout=open("log.txt", "w"), check=True) file_starts_with("log.txt", "2") cmd_args = [ "yttm", "encode", f"--model={BASE_MODEL_FILE}", "--output_type=id", "--n_threads=1", "--reverse", "--eos", ] run(cmd_args, stdin=open(TEST_FILE, "r"), stdout=open("log.txt", "w"), check=True) file_starts_with("log.txt", "3") os.remove("log.txt")
def test_renaming_unknown(): generate_artifacts() with open("local_test.txt", "w") as fout: fout.write("z") cmd_args = [ "yttm", "encode", f"--model={RENAME_ID_MODEL_FILE}", "--output_type=id", "--reverse", "--n_threads=1", ] run( cmd_args, stdin=open("local_test.txt", "r"), stdout=open("log.txt", "w"), check=True, ) file_starts_with("log.txt", "2922") os.remove("local_test.txt") os.remove("log.txt") return