def extract_important_tokens(corpus_file, min_count=1):
corpus = SimpleCorpus(corpus_file)
word_counts = defaultdict(int)
for context in corpus.get_texts():
for word in context:
word_counts[word] += 1
return set([k for k, v in word_counts.items() if v >= min_count])
def get_corpus_file(corpus_file, label):
corpus = SimpleCorpus(corpus_file)
return (label, corpus.get_texts())
def parse_corpus_contexts(corpus_file, interesting_tokens=None, tag=1):
corpus = SimpleCorpus(corpus_file)
return list_of_target_contexts(corpus.get_texts(),
interesting_tokens,
tag=tag)
def setUp(self):
self.interesting_tokens = set(['the','it'])
module_path = os.path.dirname(__file__)
self.corpus_path = os.path.join(module_path, 'test_data/corpus.en.1000')
self.corpus = SimpleCorpus(self.corpus_path)
