def pregenerate(self):
# overriding as we have to calculate stuff for all users
print "Generating global TFIDF data"
pbar = ProgressBar()
messages = self.threads.messages_by_from_user()
users = messages.keys()
self.counters = []
for user in pbar(users):
corpus = Corpus.from_messages(messages[user], self.cache)
corpus.process()
self.counters.append(corpus.counter())
def pregenerate(self):
# overriding as we have to calculate stuff for all users
print "Generating global TFIDF data"
pbar = ProgressBar()
messages = self.threads.messages_by_from_user()
users = messages.keys()
self.counters = []
for user in pbar(users):
corpus = Corpus.from_messages(messages[user], self.cache)
corpus.process()
self.counters.append(corpus.counter())
def generate_for_user(self, user):
corpus = Corpus.from_messages(self.threads.messages_including_from_user(user), self.cache)
corpus.process()
frequencies = corpus.frequencies(limit = self.WORD_LIMIT)
# this step is slow, so don't redo it if the file already exists and the hash of frequencies
# hasn't changed
filename = "%s/wordcloud_%s.png" % (self.PLOTS_DIR, self.slug(user))
freqs_hash = hashlib.md5(pickle.dumps(frequencies)).hexdigest()
cache_key = "wordcloudgenerator_%s" % filename
if not (os.path.exists(filename) and self.cache.get(cache_key) == freqs_hash):
self.cache.set(cache_key, freqs_hash)
WordCloudWrapper.save(frequencies, filename)
def generate_for_user(self, user):
corpus = Corpus.from_messages(
self.threads.messages_including_from_user(user), self.cache)
corpus.process()
frequencies = corpus.frequencies(limit=self.WORD_LIMIT)
# this step is slow, so don't redo it if the file already exists and the hash of frequencies
# hasn't changed
filename = "%s/wordcloud_%s.png" % (self.PLOTS_DIR, self.slug(user))
freqs_hash = hashlib.md5(pickle.dumps(frequencies)).hexdigest()
cache_key = "wordcloudgenerator_%s" % filename
if not (os.path.exists(filename)
and self.cache.get(cache_key) == freqs_hash):
self.cache.set(cache_key, freqs_hash)
WordCloudWrapper.save(frequencies, filename)