def _convert_to_bchar(in_path_prefix: str, src: str, tgt: str, out_path: str):
print(f"convert to bchar {in_path_prefix}")
with open(out_path, "w") as f_o:
for lang in [src, tgt]:
with open(f"{in_path_prefix}.{lang}") as f:
for s in f:
f_o.write(byte_encode(s.strip()) + "\n")
def _convert_to_bchar(in_path_prefix: str, src: str, tgt: str, out_path: str):
with open(out_path, 'w') as f_o:
for lang in [src, tgt]:
with open(f'{in_path_prefix}.{lang}') as f:
for s in f:
f_o.write(byte_encode(s.strip()) + '\n')
def encode(x: str) -> str:
encoded = byte_encode(x)
escaped = encoded.replace(SPACE, SPACE_ESCAPE)
return SPACE.join(list(escaped))
def encode(self, x: str) -> str:
byte_encoded = byte_encode(x)
return SPACE.join(self.sp.EncodeAsPieces(byte_encoded))
