def raw_to_corpus(): for f in ["train.txt", "dev.txt", "test.txt"]: tagged_corpus = TaggedCorpus() input = join(dirname(dirname(__file__)), "raw", "vlsp2016", f) tagged_corpus.load(input) tagged_corpus.sentences = preprocess(tagged_corpus.sentences) output = join(dirname(dirname(__file__)), "corpus", "vlsp2016", f) tagged_corpus.save(output)
def raw_to_sample_corpus(n): for f in ["train.txt", "dev.txt", "test.txt"]: tagged_corpus = TaggedCorpus() input = join(dirname(dirname(__file__)), "raw", "treebank", f) tagged_corpus.load(input) tagged_corpus.sentences = preprocess(tagged_corpus.sentences)[:n] output = join(dirname(dirname(__file__)), "corpus", "sample_treebank", f) tagged_corpus.save(output)
def raw_to_corpus(): for f1, f2 in [("chunk_train.txt", "train.txt"), ("chunk_dev.txt", "dev.txt"), ("chunk_test.txt", "test.txt")]: tagged_corpus = TaggedCorpus() input = join(dirname(dirname(__file__)), "raw", "treebank", f1) tagged_corpus.load(input) tagged_corpus.sentences = preprocess(tagged_corpus.sentences) output = join(dirname(dirname(__file__)), "corpus", "treebank", f2) tagged_corpus.save(output)