예제 #1
0
 def test_preprocess1(self):
     preprocessor2.preprocess(
         self.corpus_as_list,
         self.filelist,
         modelist=["Incorrect_RV", "Gold"],
         destlist=[self.output_dir + "/gold.txt", self.output_dir + "/test.txt"],
     )
     raise Exception
예제 #2
0
def main(corpus_dir="", output_dir="", working_dir="", preprocess=False):
    import preprocessor2
    corpus, filelist = read(corpus_dir=corpus_dir, output_dir=output_dir, working_dir=working_dir)
    if preprocess:
        corpus_as_dict = preprocessor2.preprocess(corpus, filelist, modelist=["Incorrect_RV", "Gold"])
        store_to_pickle(output_dir+"/corpusdict_", corpus_as_dict)