def treebank_eda(): total_corpus = TaggedCorpus() for file in ["train", "test", "dev"]: input_file = join("corpus", "treebank", file + ".txt") output_folder = join("eda", "treebank", file) corpus = TaggedCorpus() corpus.load(input_file) total_corpus.sentences += corpus.sentences corpus.analyze(output_folder=output_folder, auto_remove=True) total_corpus.analyze(output_folder=join("eda", "treebank", "total"))
def sample_data(n=200): tagged_corpus = TaggedCorpus() file = join(dirname(__file__), "corpus", "vlsp_chunk", "train.txt") tagged_corpus.load(file) sentences = tagged_corpus.sentences[:n] sample_corpus = TaggedCorpus(sentences) file = join(dirname(__file__), "corpus", "vlsp_chunk_sample", "train.txt") sample_corpus.save(file)
def raw_to_corpus(): for f in ["train.txt", "dev.txt", "test.txt"]: tagged_corpus = TaggedCorpus() input = join(dirname(dirname(__file__)), "raw", "vlsp2016", f) tagged_corpus.load(input) tagged_corpus.sentences = preprocess(tagged_corpus.sentences) output = join(dirname(dirname(__file__)), "corpus", "vlsp2016", f) tagged_corpus.save(output)
def raw_to_sample_corpus(n): for f in ["train.txt", "dev.txt", "test.txt"]: tagged_corpus = TaggedCorpus() input = join(dirname(dirname(__file__)), "raw", "treebank", f) tagged_corpus.load(input) tagged_corpus.sentences = preprocess(tagged_corpus.sentences)[:n] output = join(dirname(dirname(__file__)), "corpus", "sample_treebank", f) tagged_corpus.save(output)
def raw_to_corpus(): for f1, f2 in [("chunk_train.txt", "train.txt"), ("chunk_dev.txt", "dev.txt"), ("chunk_test.txt", "test.txt")]: tagged_corpus = TaggedCorpus() input = join(dirname(dirname(__file__)), "raw", "treebank", f1) tagged_corpus.load(input) tagged_corpus.sentences = preprocess(tagged_corpus.sentences) output = join(dirname(dirname(__file__)), "corpus", "treebank", f2) tagged_corpus.save(output)
def load_data(file): tagged_corpus = TaggedCorpus() tagged_corpus.load(file) sentences = tagged_corpus.sentences return sentences
from os.path import join from underthesea_flow.reader.tagged_corpus import TaggedCorpus if __name__ == '__main__': total_corpus = TaggedCorpus() for file in ["train", "test", "dev"]: input_file = join("corpus", "vlsp2016", file + ".txt") output_folder = join("eda", "vlsp2016", file) corpus = TaggedCorpus() corpus.load(input_file) total_corpus.sentences += corpus.sentences corpus.analyze(output_folder=output_folder, auto_remove=True) total_corpus.analyze(output_folder=join("eda", "vlsp2016", "total"))