def end(self, handle_invalidations=None): write(self.s, 'C') n = read_int4(self.s) oid_list = [] if n != 0: packed_oids = read(self.s, n * 8) oid_list = split_oids(packed_oids) try: handle_invalidations(oid_list) except ConflictError: self.transaction_new_oids.reverse() self.oid_pool.extend(self.transaction_new_oids) assert len(self.oid_pool) == len(set(self.oid_pool)) self.begin() # clear out records and transaction_new_oids. write_int4(self.s, 0) # Tell server we are done. raise tdata = [] for oid, record in iteritems(self.records): tdata.append(int4_to_str(8 + len(record))) tdata.append(as_bytes(oid)) tdata.append(record) tdata = join_bytes(tdata) write_int4_str(self.s, tdata) self.records.clear() if len(tdata) > 0: status = read(self.s, 1) if status == STATUS_OKAY: pass elif status == STATUS_INVALID: raise WriteConflictError() else: raise ProtocolError('server returned invalid status %r' % status)
def end(self, handle_invalidations=None): write(self.s, 'C') n = read_int4(self.s) oid_list = [] if n != 0: packed_oids = read(self.s, n*8) oid_list = split_oids(packed_oids) try: handle_invalidations(oid_list) except ConflictError: self.transaction_new_oids.reverse() self.oid_pool.extend(self.transaction_new_oids) assert len(self.oid_pool) == len(set(self.oid_pool)) self.begin() # clear out records and transaction_new_oids. write_int4(self.s, 0) # Tell server we are done. raise tdata = [] for oid, record in iteritems(self.records): tdata.append(int4_to_str(8 + len(record))) tdata.append(as_bytes(oid)) tdata.append(record) tdata = join_bytes(tdata) write_int4_str(self.s, tdata) self.records.clear() if len(tdata) > 0: status = read(self.s, 1) if status == STATUS_OKAY: pass elif status == STATUS_INVALID: raise WriteConflictError() else: raise ProtocolError('server returned invalid status %r' % status)
def handle_new_oids(self, client, db_name): # M log(20, 'New OIDs %s' % db_name) storage = self.storages[db_name] count = ord((yield client.read(1))) log(10, 'oids: %s', count) yield client.write( join_bytes(self._new_oids(client, db_name, storage, count)))
def bulk_load(self, oids): oid_str = join_bytes(oids) num_oids, remainder = divmod(len(oid_str), 8) assert remainder == 0, remainder write_all(self.s, 'B', int4_to_str(num_oids), oid_str) records = [self._get_load_response(oid) for oid in oids] for record in records: yield record
def handle_S(self, s): # sync client = self._find_client(s) self._report_load_record() self._sync_storage() log(8, 'Sync %s', len(client.invalid)) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear()
def handle_commit(self, client, db_name): # C log(20, 'Commit %s' % db_name) storage = self.storages[db_name] self._sync_storage(db_name, storage) invalid = client.invalid[db_name] yield client.write(int4_to_str(len(invalid))) yield client.write(join_bytes(invalid)) yield client.flush() invalid.clear() tdata_len = str_to_int4((yield client.read(4))) if tdata_len == 0: # Client decided not to commit (e.g. conflict) return tdata = yield client.read(tdata_len) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', tdata_len) storage.begin() i = 0 oids = [] while i < tdata_len: rlen = str_to_int4(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) storage.store(oid, record) oids.append(oid) assert i == tdata_len oid_set = set(oids) for c in self.clients: if c is not client: if oid_set.intersection(c.unused_oids[db_name]): raise ClientError('invalid oid: %r' % oid) try: handle_invalidations = ( lambda oids: self._handle_invalidations(db_name, oids)) storage.end(handle_invalidations=handle_invalidations) except ConflictError: log(20, 'Conflict during commit') yield client.write(STATUS_INVALID) else: self._report_load_record(storage) log(20, 'Committed %3s objects %s bytes at %s', len(oids), tdata_len, datetime.now()) yield client.write(STATUS_OKAY) client.unused_oids[db_name] -= oid_set for c in self.clients: if c is not client: c.invalid[db_name].update(oids) storage.d_bytes_since_pack += tdata_len + 8
def handle_sync(self, client, db_name): # S log(20, 'Sync %s' % db_name) storage = self.storages[db_name] self._report_load_record(storage) self._sync_storage(db_name, storage) invalid = client.invalid[db_name] log(8, 'Sync %s', len(invalid)) yield client.write(int4_to_str(len(invalid))) yield client.write(join_bytes(invalid)) invalid.clear()
def check_record_pack_unpack(self): oid = as_bytes('0'*8) data = as_bytes('sample') reflist = ['1'*8, '2'*8] reflist = list(map(as_bytes, reflist)) refs = join_bytes(reflist) result=unpack_record(pack_record(oid, data, refs)) assert result[0] == oid assert result[1] == data assert split_oids(result[2]) == reflist assert split_oids('') == []
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i + 4]) i += 4 oid = tdata[i:i + 8] record = tdata[i + 8:i + rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def get_state(self, obj): self.sio.seek(0) # recycle BytesIO instance self.sio.truncate() self.pickler.clear_memo() self.pickler.dump(type(obj)) self.refs.clear() position = self.sio.tell() self.pickler.dump(obj.__getstate__()) uncompressed = self.sio.getvalue() pickled_type = uncompressed[:position] pickled_state = uncompressed[position:] if WRITE_COMPRESSED_STATE_PICKLES: state = compress(pickled_state) else: state = pickled_state data = pickled_type + state self.refs.discard(obj._p_oid) return data, join_bytes(self.refs)
def pack_record(oid, data, refs): """(oid:str, data:str, refs:str) -> record:str """ return join_bytes([oid, int4_to_str(len(data)), data, refs])
def handle_M(self, s): # new OIDs count = ord(read(s, 1)) log(10, "oids: %s", count) write(s, join_bytes(self._new_oids(s, count)))
def __init__(self, *args): self.io = BytesIO(join_bytes(as_bytes(a) for a in args))