decoder_tokenizer_legacy=args.decoder_tokenizer_legacy,
)
_, _ = MTDataPreproc.preprocess_parallel_dataset(
clean=args.clean,
src_fname=args.src_fname,
tgt_fname=args.tgt_fname,
out_dir=args.out_dir,
encoder_tokenizer_name=args.encoder_tokenizer_name,
encoder_model_name=args.encoder_model_name,
encoder_tokenizer_model=encoder_tokenizer_model,
encoder_bpe_dropout=args.encoder_tokenizer_bpe_dropout,
encoder_tokenizer_r2l=args.encoder_tokenizer_r2l,
decoder_tokenizer_name=args.decoder_tokenizer_name,
decoder_model_name=args.decoder_model_name,
decoder_tokenizer_model=decoder_tokenizer_model,
decoder_tokenizer_r2l=args.decoder_tokenizer_r2l,
decoder_bpe_dropout=args.decoder_tokenizer_bpe_dropout,
max_seq_length=args.max_seq_length,
min_seq_length=args.min_seq_length,
tokens_in_batch=args.tokens_in_batch,
lines_per_dataset_fragment=args.lines_per_dataset_fragment,
num_batches_per_tarfile=args.num_batches_per_tarfile,
tar_file_prefix=args.tar_file_prefix,
global_rank=0,
world_size=1,
n_jobs=args.n_preproc_jobs,
encoder_tokenizer_legacy=args.encoder_tokenizer_legacy,
decoder_tokenizer_legacy=args.decoder_tokenizer_legacy,
)
global_rank=0,
)
else:
encoder_tokenizer_model, decoder_tokenizer_model = args.encoder_tokenizer_model, args.decoder_tokenizer_model
encoder_tokenizer, decoder_tokenizer = MTDataPreproc.get_enc_dec_tokenizers(
encoder_tokenizer_name=args.encoder_tokenizer_name,
encoder_tokenizer_model=encoder_tokenizer_model,
encoder_bpe_dropout=args.encoder_tokenizer_bpe_dropout,
decoder_tokenizer_name=args.decoder_tokenizer_name,
decoder_tokenizer_model=decoder_tokenizer_model,
decoder_bpe_dropout=args.decoder_tokenizer_bpe_dropout,
)
_, _ = MTDataPreproc.preprocess_parallel_dataset(
clean=args.clean,
src_fname=args.src_fname,
tgt_fname=args.tgt_fname,
out_dir=args.out_dir,
encoder_tokenizer=encoder_tokenizer,
decoder_tokenizer=decoder_tokenizer,
max_seq_length=args.max_seq_length,
min_seq_length=args.min_seq_length,
tokens_in_batch=args.tokens_in_batch,
lines_per_dataset_fragment=args.lines_per_dataset_fragment,
num_batches_per_tarfile=args.num_batches_per_tarfile,
pkl_file_prefix=args.pkl_file_prefix,
global_rank=0,
world_size=1,
)