def test_token_index_simple(self):
tk_str = 'adams wat badcat xcat $9000 xcat'
ti = token_index(tk_str)
print(ti)
self.assertIn(1, ti['wat'])
self.assertIn(3, ti['xcat'])
self.assertIn(5, ti['xcat'])
def test_freq_dist_dict_full(self):
with open('{}{}'.format(base_resources, '2011-1-19raw.txt'), 'r')\
as f:
text = f.read().decode('utf-8')
text = remove_punctuation(text)
stopped = stop_words(text)
ti = token_index(stopped)
#print(pformat(ti), file=stderr)
with open('{}{}'.format(target_out, '2011-1-19token_index'),\
'w') as out_file:
out_file.write(pformat(ti))