def setUp(self):
self.cleanUp()
h = DumpTruck(dbname=u"/tmp/test.db")
h.save_var(u"birthday", u"November 30, 1888")
h.close()
connection = sqlite3.connect(u"/tmp/test.db")
self.cursor = connection.cursor()
def setUp(self): self.cleanUp() h = DumpTruck(dbname = u'/tmp/test.db') h.save_var(u'birthday', u'November 30, 1888') h.close() connection=sqlite3.connect(u'/tmp/test.db') self.cursor=connection.cursor()
def savegetvar(self, var):
h = DumpTruck(dbname="/tmp/test.db")
h.save_var(u"weird", var)
h.close()
h = DumpTruck(dbname="/tmp/test.db")
t = os.stat("/tmp/test.db").st_mtime
self.assertEqual(h.get_var(u"weird"), var)
h.close()
assert os.stat("/tmp/test.db").st_mtime == t
def savegetvar(self, var):
h = DumpTruck(dbname = '/tmp/test.db')
h.save_var(u'weird', var)
h.close()
h = DumpTruck(dbname = '/tmp/test.db')
t=os.stat('/tmp/test.db').st_mtime
self.assertEqual(h.get_var(u'weird'), var)
h.close()
assert os.stat('/tmp/test.db').st_mtime==t
def save(self, key, value): h = DumpTruck(dbname = u'/tmp/test.db') h.save_var(key, value) h.close()
def savegetvar(self, var): h = DumpTruck(dbname = '/tmp/test.db') h.save_var(u'weird', var) self.assertEqual(h.get_var(u'weird'), var) h.close()
# We initialize DumpTruck, with dummy data that won't be inserted.
dump_truck.create_table(
{
"stored_filename": "filename.20130227205616.tgz",
"size": 1,
"metadata": {"is_enc": False},
"backup_date": 1361994976,
"filename": "filename",
"backend": "s3",
"is_deleted": False,
"last_updated": 1361994976,
"tags": [],
"backend_hash": "backendhash",
},
"backups",
)
dump_truck.create_index(["stored_filename"], "backups", unique=True)
if not "inventory" in dump_truck.tables():
dump_truck.create_table({"filename": "filename", "archive_id": "glacier-archive-id"}, "inventory")
dump_truck.create_index(["filename"], "inventory", unique=True)
if not "jobs" in dump_truck.tables():
dump_truck.create_table({"filename": "filename", "job_id": "job_id"}, "jobs")
dump_truck.create_index(["filename"], "jobs", unique=True)
if not "config" in dump_truck.tables():
dump_truck.save_var("client_id", "")
dump_truck.save_var("sync_ts", 0)
dump_truck.save_var("tags", set())
# The first entry has no ancestors, so it has to make its own entry.
if self.motherbucket == None:
dt.insert({"scraper_run": scraper_run, "kwargs": self.kwargs}, self.bucket)
return childbuckets
def reference(self):
# For linking scraped data to this row
return {"kwargs": self.kwargs, "motherkwargs": self.motherkwargs, "scraper_run": scraper_run}
try:
scraper_run = dt.get_var("scraper_run")
except:
scraper_run = datetime.date.today().isoformat()
dt.save_var("scraper_run", scraper_run)
def excavate(bucketclasses=[], startingbuckets=[]):
"Start everything."
# Bucket classes (page types)
if bucketclasses == []:
for g in globals().values():
if isinstance(g, BucketMold) and g != BucketMold:
bucketclasses.append(g)
bag = Bag(buckets=bucketclasses)
# The seed buckets
if dt.execute('select count(*) as "c" from `%s`' % bag._table_name)[0]["c"] == 0:
for b in startingbuckets: