def _report_load_record(self): if self.load_record and is_logging(5): log( 5, '[%s]\n' % getpid() + '\n'.join("%8s: %s" % (item[1], item[0]) for item in sorted(self.load_record.items()))) self.load_record.clear()
def handle_C(self, s): # commit client = self._find_client(s) s.sendall(p32(len(client.invalid)) + ''.join(client.invalid)) client.invalid.clear() tlen = u32(recv(s, 4)) if tlen == 0: return # client decided not to commit (e.g. conflict) tdata = recv(s, tlen) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', tlen) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = u32(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', u64(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) self.storage.end() log(20, 'Committed %3s objects %s bytes at %s', len(oids), tlen, datetime.now()) s.sendall(STATUS_OKAY) for c in self.clients: if c is not client: c.invalid.update(oids)
def handle_commit(self, client, db_name): # C log(20, 'Commit %s' % db_name) storage = self.storages[db_name] self._sync_storage(db_name, storage) invalid = client.invalid[db_name] yield client.write(int4_to_str(len(invalid))) yield client.write(join_bytes(invalid)) yield client.flush() invalid.clear() tdata_len = str_to_int4((yield client.read(4))) if tdata_len == 0: # Client decided not to commit (e.g. conflict) return tdata = yield client.read(tdata_len) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', tdata_len) storage.begin() i = 0 oids = [] while i < tdata_len: rlen = str_to_int4(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) storage.store(oid, record) oids.append(oid) assert i == tdata_len oid_set = set(oids) for c in self.clients: if c is not client: if oid_set.intersection(c.unused_oids[db_name]): raise ClientError('invalid oid: %r' % oid) try: handle_invalidations = ( lambda oids: self._handle_invalidations(db_name, oids)) storage.end(handle_invalidations=handle_invalidations) except ConflictError: log(20, 'Conflict during commit') yield client.write(STATUS_INVALID) else: self._report_load_record(storage) log(20, 'Committed %3s objects %s bytes at %s', len(oids), tdata_len, datetime.now()) yield client.write(STATUS_OKAY) client.unused_oids[db_name] -= oid_set for c in self.clients: if c is not client: c.invalid[db_name].update(oids) storage.d_bytes_since_pack += tdata_len + 8
def end(self, handle_invalidations=None): self.shelf.store(iteritems(self.pending_records)) if is_logging(20): shelf_file = self.shelf.get_file() shelf_file.seek_end() pos = shelf_file.tell() log(20, "Transaction at [%s] end=%s" % (datetime.now(), pos)) if self.pack_extra is not None: self.pack_extra.update(self.pending_records) self.allocated_unused_oids -= set(self.pending_records) self.begin()
def end(self, handle_invalidations=None): self.shelf.store(iteritems(self.pending_records)) if is_logging(10): shelf_file = self.shelf.get_file() shelf_file.seek_end() pos = shelf_file.tell() log(10, "Transaction at [%s] end=%s" % (datetime.now(), pos)) if self.pack_extra is not None: self.pack_extra.update(self.pending_records) self.allocated_unused_oids -= set(self.pending_records) self.begin()
def handle_L(self, s): # load oid = recv(s, 8) if oid in self._find_client(s).invalid: s.sendall(STATUS_INVALID) else: try: record = self.storage.load(oid) except KeyError: log(10, 'KeyError %s', u64(oid)) s.sendall(STATUS_KEYERROR) else: if is_logging(5): log(5, 'Load %-7s %s', u64(oid), extract_class_name(record)) s.sendall(STATUS_OKAY + p32(len(record)) + record)
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i + 4]) i += 4 oid = tdata[i:i + 8] record = tdata[i + 8:i + rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def _send_load_response(self, s, oid): if oid in self._find_client(s).invalid: write(s, STATUS_INVALID) else: try: record = self.storage.load(oid) except KeyError: log(10, 'KeyError %s', str_to_int8(oid)) write(s, STATUS_KEYERROR) except ReadConflictError: log(10, 'ReadConflictError %s', str_to_int8(oid)) write(s, STATUS_INVALID) else: if is_logging(5): class_name = extract_class_name(record) if class_name in self.load_record: self.load_record[class_name] += 1 else: self.load_record[class_name] = 1 log(4, 'Load %-7s %s', str_to_int8(oid), class_name) write(s, STATUS_OKAY) write_int4_str(s, record)
def _send_load_response(self, client, db_name, storage, oid): if oid in client.invalid[db_name]: yield client.write(STATUS_INVALID) else: try: record = storage.load(oid) except KeyError: log(10, 'KeyError %s', str_to_int8(oid)) yield client.write(STATUS_KEYERROR) except ReadConflictError: log(10, 'ReadConflictError %s', str_to_int8(oid)) yield client.write(STATUS_INVALID) else: if is_logging(5): class_name = extract_class_name(record) if class_name in storage.d_load_record: storage.d_load_record[class_name] += 1 else: storage.d_load_record[class_name] = 1 log(4, 'Load %-7s %s', str_to_int8(oid), class_name) yield client.write(STATUS_OKAY) yield client.write(int4_to_str(len(record))) yield client.write(record)
def end(self, handle_invalidations=None): self._store_records(self.pending_records) if is_logging(20): log(20, "Transaction at [%s]" % datetime.now()) self.begin()
def _report_load_record(self): if self.load_record and is_logging(5): log(5, '[%s]\n' % getpid() + '\n'.join( "%8s: %s" % (item[1], item[0]) for item in sorted(self.load_record.items()))) self.load_record.clear()
def _report_load_record(self, storage): if storage.d_load_record and is_logging(5): log(5, '\n'.join('%8s: %s' % (value, key) for key, value in sorted(storage.d_load_record.items()))) storage.d_load_record.clear()