def savegetvar(self, var): h = DumpTruck(dbname = '/tmp/test.db') h.save_var(u'weird', var) h.close() h = DumpTruck(dbname = '/tmp/test.db') t=os.stat('/tmp/test.db').st_mtime self.assertEqual(h.get_var(u'weird'), var) h.close() assert os.stat('/tmp/test.db').st_mtime==t
def savegetvar(self, var): h = DumpTruck(dbname="/tmp/test.db") h.save_var(u"weird", var) h.close() h = DumpTruck(dbname="/tmp/test.db") t = os.stat("/tmp/test.db").st_mtime self.assertEqual(h.get_var(u"weird"), var) h.close() assert os.stat("/tmp/test.db").st_mtime == t
def get(self, key, value): h = DumpTruck(dbname = u'/tmp/test.db') self.assertEqual(h.get_var(key), value) h.close()
def savegetvar(self, var): h = DumpTruck(dbname = '/tmp/test.db') h.save_var(u'weird', var) self.assertEqual(h.get_var(u'weird'), var) h.close()
kin = {"kwargs": cb.kwargs} kin.update(reference) dt.insert(kin, cb.bucket) # The first entry has no ancestors, so it has to make its own entry. if self.motherbucket == None: dt.insert({"scraper_run": scraper_run, "kwargs": self.kwargs}, self.bucket) return childbuckets def reference(self): # For linking scraped data to this row return {"kwargs": self.kwargs, "motherkwargs": self.motherkwargs, "scraper_run": scraper_run} try: scraper_run = dt.get_var("scraper_run") except: scraper_run = datetime.date.today().isoformat() dt.save_var("scraper_run", scraper_run) def excavate(bucketclasses=[], startingbuckets=[]): "Start everything." # Bucket classes (page types) if bucketclasses == []: for g in globals().values(): if isinstance(g, BucketMold) and g != BucketMold: bucketclasses.append(g) bag = Bag(buckets=bucketclasses)