def __load_data(self):
if os.path.isfile(self.fname):
return utils.unpickled_content(self.fname)
else:
data = {"subreddit": self.subred, "nstories": 0, "urls": []}
self.__save_data(data)
return data
def fetch_ngram(self, url, ng=1):
fname = "%s/%s.words_%d_gram" % (self.storage_dir, hashlib.sha1(url).hexdigest(), ng)
print fname
if not os.path.isfile(fname):
if ng > 1:
words = self.fetch_ngram(url, ng=1)
ngrams = self.make_n_gram(words, ng)
utils.save_pickled(fname, ngrams)
else:
words = self.word
s_preprocessor.process(CachedContentFetcher.fetch(self, url))
utils.save_pickled(fname, words)
return utils.unpickled_content(fname)
