def _report_load_record(self):
if self.load_record and is_logging(5):
log(
5, '[%s]\n' % getpid() +
'\n'.join("%8s: %s" % (item[1], item[0])
for item in sorted(self.load_record.items())))
self.load_record.clear()
def handle_C(self, s):
# commit
client = self._find_client(s)
s.sendall(p32(len(client.invalid)) + ''.join(client.invalid))
client.invalid.clear()
tlen = u32(recv(s, 4))
if tlen == 0:
return # client decided not to commit (e.g. conflict)
tdata = recv(s, tlen)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', tlen)
self.storage.begin()
i = 0
oids = []
while i < len(tdata):
rlen = u32(tdata[i:i+4])
i += 4
oid = tdata[i:i+8]
record = tdata[i+8:i+rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s', u64(oid), rlen, class_name)
self.storage.store(oid, record)
oids.append(oid)
assert i == len(tdata)
self.storage.end()
log(20, 'Committed %3s objects %s bytes at %s',
len(oids), tlen, datetime.now())
s.sendall(STATUS_OKAY)
for c in self.clients:
if c is not client:
c.invalid.update(oids)
def handle_commit(self, client, db_name):
# C
log(20, 'Commit %s' % db_name)
storage = self.storages[db_name]
self._sync_storage(db_name, storage)
invalid = client.invalid[db_name]
yield client.write(int4_to_str(len(invalid)))
yield client.write(join_bytes(invalid))
yield client.flush()
invalid.clear()
tdata_len = str_to_int4((yield client.read(4)))
if tdata_len == 0:
# Client decided not to commit (e.g. conflict)
return
tdata = yield client.read(tdata_len)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', tdata_len)
storage.begin()
i = 0
oids = []
while i < tdata_len:
rlen = str_to_int4(tdata[i:i+4])
i += 4
oid = tdata[i:i+8]
record = tdata[i+8:i+rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s',
str_to_int8(oid), rlen, class_name)
storage.store(oid, record)
oids.append(oid)
assert i == tdata_len
oid_set = set(oids)
for c in self.clients:
if c is not client:
if oid_set.intersection(c.unused_oids[db_name]):
raise ClientError('invalid oid: %r' % oid)
try:
handle_invalidations = (
lambda oids: self._handle_invalidations(db_name, oids))
storage.end(handle_invalidations=handle_invalidations)
except ConflictError:
log(20, 'Conflict during commit')
yield client.write(STATUS_INVALID)
else:
self._report_load_record(storage)
log(20, 'Committed %3s objects %s bytes at %s',
len(oids), tdata_len, datetime.now())
yield client.write(STATUS_OKAY)
client.unused_oids[db_name] -= oid_set
for c in self.clients:
if c is not client:
c.invalid[db_name].update(oids)
storage.d_bytes_since_pack += tdata_len + 8
def end(self, handle_invalidations=None):
self.shelf.store(iteritems(self.pending_records))
if is_logging(20):
shelf_file = self.shelf.get_file()
shelf_file.seek_end()
pos = shelf_file.tell()
log(20, "Transaction at [%s] end=%s" % (datetime.now(), pos))
if self.pack_extra is not None:
self.pack_extra.update(self.pending_records)
self.allocated_unused_oids -= set(self.pending_records)
self.begin()
def end(self, handle_invalidations=None):
self.shelf.store(iteritems(self.pending_records))
if is_logging(10):
shelf_file = self.shelf.get_file()
shelf_file.seek_end()
pos = shelf_file.tell()
log(10, "Transaction at [%s] end=%s" % (datetime.now(), pos))
if self.pack_extra is not None:
self.pack_extra.update(self.pending_records)
self.allocated_unused_oids -= set(self.pending_records)
self.begin()
def handle_L(self, s):
# load
oid = recv(s, 8)
if oid in self._find_client(s).invalid:
s.sendall(STATUS_INVALID)
else:
try:
record = self.storage.load(oid)
except KeyError:
log(10, 'KeyError %s', u64(oid))
s.sendall(STATUS_KEYERROR)
else:
if is_logging(5):
log(5, 'Load %-7s %s', u64(oid), extract_class_name(record))
s.sendall(STATUS_OKAY + p32(len(record)) + record)
def handle_C(self, s):
# commit
self._sync_storage()
client = self._find_client(s)
write_all(s,
int4_to_str(len(client.invalid)), join_bytes(client.invalid))
client.invalid.clear()
tdata = read_int4_str(s)
if len(tdata) == 0:
return # client decided not to commit (e.g. conflict)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', len(tdata))
self.storage.begin()
i = 0
oids = []
while i < len(tdata):
rlen = str_to_int4(tdata[i:i+4])
i += 4
oid = tdata[i:i+8]
record = tdata[i+8:i+rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s',
str_to_int8(oid), rlen, class_name)
self.storage.store(oid, record)
oids.append(oid)
assert i == len(tdata)
oid_set = set(oids)
for other_client in self.clients:
if other_client is not client:
if oid_set.intersection(other_client.unused_oids):
raise ClientError("invalid oid: %r" % oid)
try:
self.storage.end(handle_invalidations=self._handle_invalidations)
except ConflictError:
log(20, 'Conflict during commit')
write(s, STATUS_INVALID)
else:
self._report_load_record()
log(20, 'Committed %3s objects %s bytes at %s',
len(oids), len(tdata), datetime.now())
write(s, STATUS_OKAY)
client.unused_oids -= oid_set
for c in self.clients:
if c is not client:
c.invalid.update(oids)
self.bytes_since_pack += len(tdata) + 8
def handle_C(self, s):
# commit
self._sync_storage()
client = self._find_client(s)
write_all(s, int4_to_str(len(client.invalid)),
join_bytes(client.invalid))
client.invalid.clear()
tdata = read_int4_str(s)
if len(tdata) == 0:
return # client decided not to commit (e.g. conflict)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', len(tdata))
self.storage.begin()
i = 0
oids = []
while i < len(tdata):
rlen = str_to_int4(tdata[i:i + 4])
i += 4
oid = tdata[i:i + 8]
record = tdata[i + 8:i + rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen,
class_name)
self.storage.store(oid, record)
oids.append(oid)
assert i == len(tdata)
oid_set = set(oids)
for other_client in self.clients:
if other_client is not client:
if oid_set.intersection(other_client.unused_oids):
raise ClientError("invalid oid: %r" % oid)
try:
self.storage.end(handle_invalidations=self._handle_invalidations)
except ConflictError:
log(20, 'Conflict during commit')
write(s, STATUS_INVALID)
else:
self._report_load_record()
log(20, 'Committed %3s objects %s bytes at %s', len(oids),
len(tdata), datetime.now())
write(s, STATUS_OKAY)
client.unused_oids -= oid_set
for c in self.clients:
if c is not client:
c.invalid.update(oids)
self.bytes_since_pack += len(tdata) + 8
def _send_load_response(self, s, oid):
if oid in self._find_client(s).invalid:
write(s, STATUS_INVALID)
else:
try:
record = self.storage.load(oid)
except KeyError:
log(10, 'KeyError %s', str_to_int8(oid))
write(s, STATUS_KEYERROR)
except ReadConflictError:
log(10, 'ReadConflictError %s', str_to_int8(oid))
write(s, STATUS_INVALID)
else:
if is_logging(5):
class_name = extract_class_name(record)
if class_name in self.load_record:
self.load_record[class_name] += 1
else:
self.load_record[class_name] = 1
log(4, 'Load %-7s %s', str_to_int8(oid), class_name)
write(s, STATUS_OKAY)
write_int4_str(s, record)
def _send_load_response(self, s, oid):
if oid in self._find_client(s).invalid:
write(s, STATUS_INVALID)
else:
try:
record = self.storage.load(oid)
except KeyError:
log(10, 'KeyError %s', str_to_int8(oid))
write(s, STATUS_KEYERROR)
except ReadConflictError:
log(10, 'ReadConflictError %s', str_to_int8(oid))
write(s, STATUS_INVALID)
else:
if is_logging(5):
class_name = extract_class_name(record)
if class_name in self.load_record:
self.load_record[class_name] += 1
else:
self.load_record[class_name] = 1
log(4, 'Load %-7s %s', str_to_int8(oid), class_name)
write(s, STATUS_OKAY)
write_int4_str(s, record)
def _send_load_response(self, client, db_name, storage, oid):
if oid in client.invalid[db_name]:
yield client.write(STATUS_INVALID)
else:
try:
record = storage.load(oid)
except KeyError:
log(10, 'KeyError %s', str_to_int8(oid))
yield client.write(STATUS_KEYERROR)
except ReadConflictError:
log(10, 'ReadConflictError %s', str_to_int8(oid))
yield client.write(STATUS_INVALID)
else:
if is_logging(5):
class_name = extract_class_name(record)
if class_name in storage.d_load_record:
storage.d_load_record[class_name] += 1
else:
storage.d_load_record[class_name] = 1
log(4, 'Load %-7s %s', str_to_int8(oid), class_name)
yield client.write(STATUS_OKAY)
yield client.write(int4_to_str(len(record)))
yield client.write(record)
def end(self, handle_invalidations=None):
self._store_records(self.pending_records)
if is_logging(20):
log(20, "Transaction at [%s]" % datetime.now())
self.begin()
def _report_load_record(self):
if self.load_record and is_logging(5):
log(5, '[%s]\n' % getpid() + '\n'.join(
"%8s: %s" % (item[1], item[0])
for item in sorted(self.load_record.items())))
self.load_record.clear()
def _report_load_record(self, storage):
if storage.d_load_record and is_logging(5):
log(5, '\n'.join('%8s: %s' % (value, key)
for key, value
in sorted(storage.d_load_record.items())))
storage.d_load_record.clear()