def sample_data(n=200):
tagged_corpus = TaggedCorpus()
file = join(dirname(__file__), "corpus", "vlsp_chunk", "train.txt")
tagged_corpus.load(file)
sentences = tagged_corpus.sentences[:n]
sample_corpus = TaggedCorpus(sentences)
file = join(dirname(__file__), "corpus", "vlsp_chunk_sample", "train.txt")
sample_corpus.save(file)
def raw_to_corpus():
for f in ["train.txt", "dev.txt", "test.txt"]:
tagged_corpus = TaggedCorpus()
input = join(dirname(dirname(__file__)), "raw", "vlsp2016", f)
tagged_corpus.load(input)
tagged_corpus.sentences = preprocess(tagged_corpus.sentences)
output = join(dirname(dirname(__file__)), "corpus", "vlsp2016", f)
tagged_corpus.save(output)
def raw_to_sample_corpus(n):
for f in ["train.txt", "dev.txt", "test.txt"]:
tagged_corpus = TaggedCorpus()
input = join(dirname(dirname(__file__)), "raw", "treebank", f)
tagged_corpus.load(input)
tagged_corpus.sentences = preprocess(tagged_corpus.sentences)[:n]
output = join(dirname(dirname(__file__)), "corpus", "sample_treebank",
f)
tagged_corpus.save(output)
def raw_to_corpus():
for f1, f2 in [("chunk_train.txt", "train.txt"),
("chunk_dev.txt", "dev.txt"),
("chunk_test.txt", "test.txt")]:
tagged_corpus = TaggedCorpus()
input = join(dirname(dirname(__file__)), "raw", "treebank", f1)
tagged_corpus.load(input)
tagged_corpus.sentences = preprocess(tagged_corpus.sentences)
output = join(dirname(dirname(__file__)), "corpus", "treebank", f2)
tagged_corpus.save(output)
def treebank_eda():
total_corpus = TaggedCorpus()
for file in ["train", "test", "dev"]:
input_file = join("corpus", "treebank", file + ".txt")
output_folder = join("eda", "treebank", file)
corpus = TaggedCorpus()
corpus.load(input_file)
total_corpus.sentences += corpus.sentences
corpus.analyze(output_folder=output_folder, auto_remove=True)
total_corpus.analyze(output_folder=join("eda", "treebank", "total"))
def load_data(file):
tagged_corpus = TaggedCorpus()
tagged_corpus.load(file)
sentences = tagged_corpus.sentences
return sentences
from os.path import join
from underthesea_flow.reader.tagged_corpus import TaggedCorpus
if __name__ == '__main__':
total_corpus = TaggedCorpus()
for file in ["train", "test", "dev"]:
input_file = join("corpus", "vlsp2016", file + ".txt")
output_folder = join("eda", "vlsp2016", file)
corpus = TaggedCorpus()
corpus.load(input_file)
total_corpus.sentences += corpus.sentences
corpus.analyze(output_folder=output_folder, auto_remove=True)
total_corpus.analyze(output_folder=join("eda", "vlsp2016", "total"))