Ejemplo n.º 1
0
 def fetch_ngram(self, url, ng=1):
     fname = "%s/%s.words_%d_gram" % (self.storage_dir, hashlib.sha1(url).hexdigest(), ng)
     print fname
     if not os.path.isfile(fname):
         if ng > 1:
             words = self.fetch_ngram(url, ng=1)
             ngrams = self.make_n_gram(words, ng)
             utils.save_pickled(fname, ngrams)
         else:
             words = self.word
             s_preprocessor.process(CachedContentFetcher.fetch(self, url))
             utils.save_pickled(fname, words)
     return utils.unpickled_content(fname)
Ejemplo n.º 2
0
 def __save_data(self, data):
     utils.save_pickled(self.fname, data)
 def save_metadata(self):
     utils.save_pickled(self.labels_metadata_fname, self.labels_metadata)