def handle_L(self, s):
# load
oid = recv(s, 8)
if oid in self._find_client(s).invalid:
s.sendall(STATUS_INVALID)
else:
try:
record = self.storage.load(oid)
except KeyError:
log(10, 'KeyError %s', u64(oid))
s.sendall(STATUS_KEYERROR)
else:
if is_logging(5):
log(5, 'Load %-7s %s', u64(oid), extract_class_name(record))
s.sendall(STATUS_OKAY + p32(len(record)) + record)
def handle_C(self, s):
# commit
client = self._find_client(s)
s.sendall(p32(len(client.invalid)) + ''.join(client.invalid))
client.invalid.clear()
tlen = u32(recv(s, 4))
if tlen == 0:
return # client decided not to commit (e.g. conflict)
tdata = recv(s, tlen)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', tlen)
self.storage.begin()
i = 0
oids = []
while i < len(tdata):
rlen = u32(tdata[i:i+4])
i += 4
oid = tdata[i:i+8]
record = tdata[i+8:i+rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s', u64(oid), rlen, class_name)
self.storage.store(oid, record)
oids.append(oid)
assert i == len(tdata)
self.storage.end()
log(20, 'Committed %3s objects %s bytes at %s',
len(oids), tlen, datetime.now())
s.sendall(STATUS_OKAY)
for c in self.clients:
if c is not client:
c.invalid.update(oids)
def gen_oid_record(self):
"""() -> sequence([oid:str, record:str])
A FileStorage will do a better job of this.
"""
for oid_num in xrange(u64(self.new_oid())):
try:
oid = p64(oid_num)
record = self.load(oid)
yield oid, record
except DurusKeyError:
pass
def _build_index(self):
self.fp.seek(0)
if self.fp.read(len(self.MAGIC)) != self.MAGIC:
raise IOError, "invalid storage (missing magic in %r)" % self.fp
index_offset = u64(self.fp.read(8))
assert index_offset > 0
self.fp.seek(index_offset)
index_size = u64(self.fp.read(8))
self.index = loads(decompress(self.fp.read(index_size)))
while 1:
# Read one transaction each time here.
oids = {}
transaction_offset = self.fp.tell()
try:
while 1:
object_record_offset = self.fp.tell()
record = self._read_block()
if len(record) == 0:
break # normal termination
if len(record) < 12:
raise ValueError("Bad record size")
oid = record[0:8]
oids[oid] = object_record_offset
# We've reached the normal end of a transaction.
self.index.update(oids)
oids.clear()
except (ValueError, IOError), exc:
if self.fp.tell() > transaction_offset:
if not self.repair:
raise
# The transaction was malformed. Attempt repair.
if self.fp.mode == 'r':
raise RuntimeError(
"Can't repair readonly file.\n%s" % exc)
self.fp.seek(transaction_offset)
self.fp.truncate()
break
def __init__(self, filename=None, readonly=False, repair=False):
"""(filename:str=None, readonly:bool=False, repair:bool=False)
If filename is empty (or None), a temporary file will be used.
"""
self.oid = 0
self.filename = filename
if readonly:
if not filename:
raise ValueError(
"A filename is required for a readonly storage.")
if repair:
raise ValueError("A readonly storage can't be repaired.")
self.fp = open(self.filename, 'rb')
else:
if not filename:
self.fp = NamedTemporaryFile(suffix=".durus", mode="w+b")
elif os.path.exists(self.filename):
self.fp = open(self.filename, 'a+b')
else:
self.fp = open(self.filename, 'w+b')
try:
lock_file(self.fp)
except IOError:
self.fp.close()
raise RuntimeError(
"\n %s is locked."
"\n There is probably a Durus storage server (or a client)"
"\n using it.\n" % self.get_filename())
self.pending_records = {}
self.pack_extra = None
self.repair = repair
self._set_concrete_class_for_magic()
self.index = {}
self._build_index()
max_oid = 0
for oid in self.index:
max_oid = max(max_oid, u64(oid))
self.oid = max_oid
def _build_index(self):
self.index = {}
self.fp.seek(0)
if self.fp.read(len(self.MAGIC)) != self.MAGIC:
raise IOError, "invalid storage (missing magic in %r)" % self.fp
max_tid = 0
while 1:
# Read one transaction each time here.
transaction_offset = self.fp.tell()
oids = {}
try:
while 1:
object_record_offset = self.fp.tell()
trecord = self._read_block()
if len(trecord) == 0:
break # normal termination
if len(trecord) < 16:
raise ValueError("Bad record size")
tid = trecord[0:8]
oid = trecord[8:16]
max_tid = max(max_tid, u64(tid))
oids[oid] = object_record_offset
# We've reached the normal end of a transaction.
self.index.update(oids)
oids.clear()
except (ValueError, IOError), exc:
if self.fp.tell() > transaction_offset:
if not self.repair:
raise
# The transaction was malformed. Attempt repair.
if self.fp.mode == 'r':
raise RuntimeError(
"Can't repair readonly file.\n%s" % exc)
self.fp.seek(transaction_offset)
self.fp.truncate()
break