def test01_preprocess_snips(self):
shutil.rmtree(DATA_BIN_SNIPS, ignore_errors=True)
args = [
"--data",
DATA_SNIPS,
"--text-tokenizer",
"bert-base-cased",
"--output-dir",
DATA_BIN_SNIPS,
]
args = preprocess.parse_args(args)
preprocess.main(args)
def test01_preprocess_split(self):
shutil.rmtree(DATA_BIN, ignore_errors=True)
args = [
"--data",
DATA,
"--text-tokenizer",
"bert-base-cased",
"--output-dir",
DATA_BIN,
"--split-amount",
"0.1",
]
args = preprocess.parse_args(args)
preprocess.main(args)