def test_smart_open_without_suffix(): with TemporaryDirectory() as temp: fname = os.path.join(temp, 'test') _touch_file(fname, compressed=True, empty=False) with utils.smart_open(fname) as fin: assert len(fin.readlines()) == 10 _touch_file(fname, compressed=False, empty=False) with utils.smart_open(fname) as fin: assert len(fin.readlines()) == 10
def get_output_handler(output_type: str, output_fname: Optional[str] = None) -> 'OutputHandler': """ :param output_type: Type of output handler. :param output_fname: Output filename. If none sys.stdout is used. :raises: ValueError for unknown output_type. :return: Output handler. """ output_stream = sys.stdout if output_fname is None else smart_open( output_fname, mode='w') if output_type == C.OUTPUT_HANDLER_TRANSLATION: return StringOutputHandler(output_stream) elif output_type == C.OUTPUT_HANDLER_SCORE: return ScoreOutputHandler(output_stream) elif output_type == C.OUTPUT_HANDLER_PAIR_WITH_SCORE: return PairWithScoreOutputHandler(output_stream) elif output_type == C.OUTPUT_HANDLER_TRANSLATION_WITH_SCORE: return StringWithScoreOutputHandler(output_stream) elif output_type == C.OUTPUT_HANDLER_BENCHMARK: return BenchmarkOutputHandler(output_stream) elif output_type == C.OUTPUT_HANDLER_JSON: return JSONOutputHandler(output_stream) elif output_type == C.OUTPUT_HANDLER_TRANSLATION_WITH_FACTORS: return FactoredStringOutputHandler(output_stream) else: raise ValueError("unknown output type")
def merge_spm(input_fname: str, output_fname: str): with utils.smart_open(input_fname, "r") as inp, open(output_fname, "w", encoding="utf-8") as out: for line in inp: sentence = line.replace(' ', '').replace('\u2581', ' ').lstrip() out.write(sentence)
def make_inputs( input_file: Optional[str], translator: inference.Translator, input_is_json: bool, input_factors: Optional[List[str]] = None ) -> Generator[inference.TranslatorInput, None, None]: """ Generates TranslatorInput instances from input. If input is None, reads from stdin. If num_input_factors > 1, the function will look for factors attached to each token, separated by '|'. If source is not None, reads from the source file. If num_source_factors > 1, num_source_factors source factor filenames are required. :param input_file: The source file (possibly None). :param translator: Translator that will translate each line of input. :param input_is_json: Whether the input is in json format. :param input_factors: Source factor files. :return: TranslatorInput objects. """ if input_file is None: check_condition( input_factors is None, "Translating from STDIN, not expecting any factor files.") for sentence_id, line in enumerate(sys.stdin, 1): if input_is_json: yield inference.make_input_from_json_string( sentence_id=sentence_id, json_string=line, translator=translator) else: yield inference.make_input_from_factored_string( sentence_id=sentence_id, factored_string=line, translator=translator) else: input_factors = [] if input_factors is None else input_factors inputs = [input_file] + input_factors if not input_is_json: check_condition( translator.num_source_factors == len(inputs), "Model(s) require %d factors, but %d given (through --input and --input-factors)." % (translator.num_source_factors, len(inputs))) with ExitStack() as exit_stack: streams = [exit_stack.enter_context(smart_open(i)) for i in inputs] for sentence_id, inputs in enumerate(zip(*streams), 1): if input_is_json: yield inference.make_input_from_json_string( sentence_id=sentence_id, json_string=inputs[0], translator=translator) else: yield inference.make_input_from_multiple_strings( sentence_id=sentence_id, strings=list(inputs))
def merge_bpe(input_fname: str, output_fname: str): """ Merge byte-pair encoded sub-words. :param input_fname: Path of byte-pair encoded input file, plain text or gzipped. :param output_fname: Path of tokenized output file, plain text. """ with utils.smart_open(input_fname, "r") as inp, open(output_fname, "w", encoding="utf-8") as out: for line in inp: # Merge on special markers and strip stray markers (end of line) merged = line.replace(SUBWORD_SPECIAL + " ", "").replace(SUBWORD_SPECIAL, "") out.write(merged)
def merge_bpe(input_fname: str, output_fname: str): """ Merge byte-pair encoded sub-words. :param input_fname: Path of byte-pair encoded input file, plain text or gzipped. :param output_fname: Path of tokenized output file, plain text. """ with utils.smart_open(input_fname, "r") as inp, open(output_fname, "w", encoding="utf-8") as out: for line in inp: # Merge on special markers and strip stray markers (end of line) merged = line.replace(SUBWORD_SPECIAL + " ", "").replace(SUBWORD_SPECIAL, "") out.write(merged)