def extract_important_tokens(corpus_file, min_count=1): corpus = SimpleCorpus(corpus_file) word_counts = defaultdict(int) for context in corpus.get_texts(): for word in context: word_counts[word] += 1 return set([k for k, v in word_counts.items() if v >= min_count])
def get_corpus_file(corpus_file, label): corpus = SimpleCorpus(corpus_file) return (label, corpus.get_texts())
def parse_corpus_contexts(corpus_file, interesting_tokens=None, tag=1): corpus = SimpleCorpus(corpus_file) return list_of_target_contexts(corpus.get_texts(), interesting_tokens, tag=tag)
def setUp(self): self.interesting_tokens = set(['the','it']) module_path = os.path.dirname(__file__) self.corpus_path = os.path.join(module_path, 'test_data/corpus.en.1000') self.corpus = SimpleCorpus(self.corpus_path)