def _load_gensim_dictionary(self): setattr(self, '_index2id', {}) gensim_dict = GensimDictionary() gensim_dict.num_docs = self.num_docs gensim_dict.num_pos = self.num_pos gensim_dict.num_nnz = self.num_nnz for dic_token in self.dic_tokens.all(): self._index2id[dic_token.index] = dic_token.id gensim_dict.token2id[dic_token.text] = dic_token.index gensim_dict.dfs[dic_token.index] = dic_token.document_frequency logger.info("Dictionary contains %d tokens" % len(gensim_dict.token2id)) return gensim_dict