def store(self, obj):
if self.on_store:
self.on_store(obj)
bits, hash = self.lookup(obj)
filename = utils.substitute(self.config["src"], dict(bits=bits, hash=hash))
exists = os.path.exists(filename)
# Make directories.
try:
os.makedirs(os.path.dirname(filename))
except os.error:
pass
# Connect.
try:
conn = sqlite3.connect(filename)
conn.row_factory = sqlite3.Row
c = conn.cursor()
except sqlite3.Error as e:
raise RuntimeError("%s: %s" % (filename, e))
# Initialize database.
if not exists:
for query in self.config["init"]:
q, params = utils.substitute_sql(query, obj)
try:
c.execute(q, params)
except sqlite3.Error as e:
raise RuntimeError("%s: %s: %s" % (filename, q, e))
# Insert query.
hash1 = self.hashof(obj)
q, params = utils.substitute_sql(
self.config["insert"],
dict(obj, _id=int(hash1, 16), _hash=hash1, sha1=lambda arg: hashlib.sha1(arg).hexdigest()),
)
try:
c.execute(q, params)
except sqlite3.Error as e:
raise RuntimeError("%s: %s: %s" % (filename, q, e))
# Commit & close.
try:
conn.commit()
conn.close()
except sqlite3.Error as e:
raise RuntimeError("%s: %s" % (filename, e))
try:
stat = os.stat(filename)
size = stat.st_size
except OSError as e:
raise RuntimeError("%s: %s" % (filename, e))
self.index_update_size(bits, hash, size)
def store(self, obj):
if self.on_store:
self.on_store(obj)
filename = utils.substitute(self.config["src"], obj)
try:
os.makedirs(os.path.dirname(filename))
except os.error:
pass
with open(filename, "w") as f:
f.write(obj["raw_data"])
def retrieve(self, obj, exclude):
filename = utils.substitute(self.config['src'], obj)
if not os.path.exists(filename): return None
conn = sqlite3.connect(filename)
conn.row_factory = sqlite3.Row
c = conn.cursor()
q, params = utils.substitute_sql(self.config['select'], obj)
try: rows = c.execute(q, params)
except sqlite3.Error as e: raise RuntimeError("%s: %s" % (q, e))
row = rows.fetchone()
if row is None: return None
o = obj.copy()
o.update(dict(zip(row.keys(), row)))
conn.close()
return Driver.retrieve(self, o, exclude)
def retrieve(self, obj, exclude=[]):
filename = utils.substitute(self.config["src"], obj)
o = obj.copy()
if not "modified" in exclude:
try:
stat = os.stat(filename)
o["modified"] = int(stat.st_mtime)
except OSError:
return None
if "raw_data" in exclude:
return Driver.retrieve(self, o, exclude)
try:
with open(filename) as f:
o["raw_data"] = f.read()
except IOError:
return None
return Driver.retrieve(self, o, exclude)
def store(self, obj):
if self.on_store: self.on_store(obj)
filename = utils.substitute(self.config['src'], obj)
exists = os.path.exists(filename)
# Make directories.
try: os.makedirs(os.path.dirname(filename))
except os.error: pass
# Connect.
try:
conn = sqlite3.connect(filename)
c = conn.cursor()
except sqlite3.Error as e:
raise RuntimeError('%s: %s' % (filename, e))
q = 'PRAGMA synchronous = OFF'
try: c.execute(q)
except sqlite3.Error as e:
raise RuntimeError('%s: %s: %s' % (filename, q, e))
# Initialize database.
if not exists:
for query in self.config['init']:
q, params = utils.substitute_sql(query, obj)
c.execute(q, params)
# Insert query.
q, params = utils.substitute_sql(self.config['insert'], obj)
try: c.execute(q, params)
except sqlite3.Error as e:
raise RuntimeError('%s: %s: %s' % (filename, q, e))
# Commit & close.
try:
conn.commit()
conn.close()
except sqlite3.Error as e:
raise RuntimeError('%s: %s' % (filename, e))
def retrieve(self, obj, exclude=[]):
bits, hash = self.lookup(obj)
filename = utils.substitute(self.config["src"], dict(bits=bits, hash=hash))
if not os.path.exists(filename):
return None
# Connect.
try:
conn = sqlite3.connect(filename)
conn.row_factory = sqlite3.Row
c = conn.cursor()
except sqlite3.Error as e:
raise RuntimeError("%s: %s" % (filename, e))
# Select query.
q, params = utils.substitute_sql(
self.config["select"], dict(obj, sha1=lambda arg: hashlib.sha1(arg).hexdigest())
)
try:
rows = c.execute(q, params)
except sqlite3.Error as e:
raise RuntimeError("%s: %s: %s" % (filename, q, e))
row = rows.fetchone()
if row is None:
return None
o = obj.copy()
o.update(dict(zip(row.keys(), row)))
# Close.
try:
conn.close()
except sqlite3.Error as e:
raise RuntimeError("%s: %s" % (filename, e))
return Driver.retrieve(self, o, exclude)
def keyof(self, obj):
sha1 = lambda arg: hashlib.sha1(arg).hexdigest()
return utils.substitute(self.config["key"], dict(obj, sha1=sha1))