def __load_data(self): if os.path.isfile(self.fname): return utils.unpickled_content(self.fname) else: data = {"subreddit": self.subred, "nstories": 0, "urls": []} self.__save_data(data) return data
def fetch_ngram(self, url, ng=1): fname = "%s/%s.words_%d_gram" % (self.storage_dir, hashlib.sha1(url).hexdigest(), ng) print fname if not os.path.isfile(fname): if ng > 1: words = self.fetch_ngram(url, ng=1) ngrams = self.make_n_gram(words, ng) utils.save_pickled(fname, ngrams) else: words = self.word s_preprocessor.process(CachedContentFetcher.fetch(self, url)) utils.save_pickled(fname, words) return utils.unpickled_content(fname)