def _build_index(self):
self.fp.seek(0)
if self.fp.read(len(self.MAGIC)) != self.MAGIC:
raise IOError, "invalid storage (missing magic in %r)" % self.fp
index_offset = u64(self.fp.read(8))
assert index_offset > 0
self.fp.seek(index_offset)
index_size = u64(self.fp.read(8))
self.index = loads(decompress(self.fp.read(index_size)))
while 1:
# Read one transaction each time here.
oids = {}
transaction_offset = self.fp.tell()
try:
while 1:
object_record_offset = self.fp.tell()
record = self._read_block()
if len(record) == 0:
break # normal termination
if len(record) < 12:
raise ValueError("Bad record size")
oid = record[0:8]
oids[oid] = object_record_offset
# We've reached the normal end of a transaction.
self.index.update(oids)
oids.clear()
except (ValueError, IOError), exc:
if self.fp.tell() > transaction_offset:
if not self.repair:
raise
# The transaction was malformed. Attempt repair.
if self.fp.mode == 'r':
raise RuntimeError(
"Can't repair readonly file.\n%s" % exc)
self.fp.seek(transaction_offset)
self.fp.truncate()
break
def __init__(self, filename=None, readonly=False, repair=False, fp=None):
"""(filename:str=None, readonly:bool=False, repair:bool=False)
If filename is empty (or None), a temporary file will be used.
"""
self.oid = 0
self.filename = filename
if fp is not None:
self.fp = fp
elif readonly:
if not filename:
raise ValueError(
"A filename is required for a readonly storage.")
if repair:
raise ValueError("A readonly storage can't be repaired.")
self.fp = open(self.filename, 'rb')
else:
if not filename:
self.fp = NamedTemporaryFile(suffix=".durus", mode="w+b")
elif (os.path.exists(self.filename) and
os.stat(self.filename).st_size > 0):
self.fp = open(self.filename, 'a+b')
else:
self.fp = open(self.filename, 'w+b')
try:
lock_file(self.fp)
except IOError:
self.fp.close()
raise RuntimeError(
"\n %s is locked."
"\n There is probably a Durus storage server (or a client)"
"\n using it.\n" % self.get_filename())
self.pending_records = {}
self.pack_extra = None
self.repair = repair
self._set_concrete_class_for_magic()
self.index = {}
self._build_index()
max_oid = 0
for oid in self.index:
max_oid = max(max_oid, u64(oid))
self.oid = max_oid
def _build_index(self):
self.index = {}
self.fp.seek(0)
if self.fp.read(len(self.MAGIC)) != self.MAGIC:
raise IOError, "invalid storage (missing magic in %r)" % self.fp
max_tid = 0
while 1:
# Read one transaction each time here.
transaction_offset = self.fp.tell()
oids = {}
try:
while 1:
object_record_offset = self.fp.tell()
trecord = self._read_block()
if len(trecord) == 0:
break # normal termination
if len(trecord) < 16:
raise ValueError("Bad record size")
tid = trecord[0:8]
oid = trecord[8:16]
max_tid = max(max_tid, u64(tid))
oids[oid] = object_record_offset
# We've reached the normal end of a transaction.
self.index.update(oids)
oids.clear()
except (ValueError, IOError), exc:
if self.fp.tell() > transaction_offset:
if not self.repair:
raise
# The transaction was malformed. Attempt repair.
if self.fp.mode == 'r':
raise RuntimeError(
"Can't repair readonly file.\n%s" % exc)
self.fp.seek(transaction_offset)
self.fp.truncate()
break
def test_check_p64_u64(self):
for x in range(3):
assert len(p64(x)) == 8
assert u64(p64(x)) == x
