def _load_gensim_dictionary(self):
setattr(self, '_index2id', {})
gensim_dict = GensimDictionary()
gensim_dict.num_docs = self.num_docs
gensim_dict.num_pos = self.num_pos
gensim_dict.num_nnz = self.num_nnz
for dic_token in self.dic_tokens.all():
self._index2id[dic_token.index] = dic_token.id
gensim_dict.token2id[dic_token.text] = dic_token.index
gensim_dict.dfs[dic_token.index] = dic_token.document_frequency
logger.info("Dictionary contains %d tokens" % len(gensim_dict.token2id))
return gensim_dict
