def test_preprocess1(self):
preprocessor2.preprocess(
self.corpus_as_list,
self.filelist,
modelist=["Incorrect_RV", "Gold"],
destlist=[self.output_dir + "/gold.txt", self.output_dir + "/test.txt"],
)
raise Exception
def main(corpus_dir="", output_dir="", working_dir="", preprocess=False):
import preprocessor2
corpus, filelist = read(corpus_dir=corpus_dir, output_dir=output_dir, working_dir=working_dir)
if preprocess:
corpus_as_dict = preprocessor2.preprocess(corpus, filelist, modelist=["Incorrect_RV", "Gold"])
store_to_pickle(output_dir+"/corpusdict_", corpus_as_dict)