Ejemplo n.º 1
0
 def setUp(self):
     self.cleanUp()
     h = DumpTruck(dbname=u"/tmp/test.db")
     h.save_var(u"birthday", u"November 30, 1888")
     h.close()
     connection = sqlite3.connect(u"/tmp/test.db")
     self.cursor = connection.cursor()
Ejemplo n.º 2
0
 def setUp(self):
   self.cleanUp()
   h = DumpTruck(dbname = u'/tmp/test.db')
   h.save_var(u'birthday', u'November 30, 1888')
   h.close()
   connection=sqlite3.connect(u'/tmp/test.db')
   self.cursor=connection.cursor()
Ejemplo n.º 3
0
 def savegetvar(self, var):
     h = DumpTruck(dbname="/tmp/test.db")
     h.save_var(u"weird", var)
     h.close()
     h = DumpTruck(dbname="/tmp/test.db")
     t = os.stat("/tmp/test.db").st_mtime
     self.assertEqual(h.get_var(u"weird"), var)
     h.close()
     assert os.stat("/tmp/test.db").st_mtime == t
Ejemplo n.º 4
0
 def savegetvar(self, var):
   h = DumpTruck(dbname = '/tmp/test.db')
   h.save_var(u'weird', var)
   h.close()
   h = DumpTruck(dbname = '/tmp/test.db')
   t=os.stat('/tmp/test.db').st_mtime
   self.assertEqual(h.get_var(u'weird'), var)
   h.close()
   assert os.stat('/tmp/test.db').st_mtime==t
Ejemplo n.º 5
0
 def save(self, key, value):
   h = DumpTruck(dbname = u'/tmp/test.db')
   h.save_var(key, value)
   h.close()
Ejemplo n.º 6
0
 def savegetvar(self, var):
   h = DumpTruck(dbname = '/tmp/test.db')
   h.save_var(u'weird', var)
   self.assertEqual(h.get_var(u'weird'), var)
   h.close()
Ejemplo n.º 7
0
    # We initialize DumpTruck, with dummy data that won't be inserted.
    dump_truck.create_table(
        {
            "stored_filename": "filename.20130227205616.tgz",
            "size": 1,
            "metadata": {"is_enc": False},
            "backup_date": 1361994976,
            "filename": "filename",
            "backend": "s3",
            "is_deleted": False,
            "last_updated": 1361994976,
            "tags": [],
            "backend_hash": "backendhash",
        },
        "backups",
    )
    dump_truck.create_index(["stored_filename"], "backups", unique=True)

if not "inventory" in dump_truck.tables():
    dump_truck.create_table({"filename": "filename", "archive_id": "glacier-archive-id"}, "inventory")
    dump_truck.create_index(["filename"], "inventory", unique=True)

if not "jobs" in dump_truck.tables():
    dump_truck.create_table({"filename": "filename", "job_id": "job_id"}, "jobs")
    dump_truck.create_index(["filename"], "jobs", unique=True)

if not "config" in dump_truck.tables():
    dump_truck.save_var("client_id", "")
    dump_truck.save_var("sync_ts", 0)
    dump_truck.save_var("tags", set())
Ejemplo n.º 8
0
        # The first entry has no ancestors, so it has to make its own entry.
        if self.motherbucket == None:
            dt.insert({"scraper_run": scraper_run, "kwargs": self.kwargs}, self.bucket)
        return childbuckets

    def reference(self):
        # For linking scraped data to this row
        return {"kwargs": self.kwargs, "motherkwargs": self.motherkwargs, "scraper_run": scraper_run}


try:
    scraper_run = dt.get_var("scraper_run")
except:
    scraper_run = datetime.date.today().isoformat()
    dt.save_var("scraper_run", scraper_run)


def excavate(bucketclasses=[], startingbuckets=[]):
    "Start everything."

    # Bucket classes (page types)
    if bucketclasses == []:
        for g in globals().values():
            if isinstance(g, BucketMold) and g != BucketMold:
                bucketclasses.append(g)
    bag = Bag(buckets=bucketclasses)

    # The seed buckets
    if dt.execute('select count(*) as "c" from `%s`' % bag._table_name)[0]["c"] == 0:
        for b in startingbuckets: