def handle_commit(self, client, db_name): # C log(20, 'Commit %s' % db_name) storage = self.storages[db_name] self._sync_storage(db_name, storage) invalid = client.invalid[db_name] yield client.write(int4_to_str(len(invalid))) yield client.write(join_bytes(invalid)) yield client.flush() invalid.clear() tdata_len = str_to_int4((yield client.read(4))) if tdata_len == 0: # Client decided not to commit (e.g. conflict) return tdata = yield client.read(tdata_len) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', tdata_len) storage.begin() i = 0 oids = [] while i < tdata_len: rlen = str_to_int4(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) storage.store(oid, record) oids.append(oid) assert i == tdata_len oid_set = set(oids) for c in self.clients: if c is not client: if oid_set.intersection(c.unused_oids[db_name]): raise ClientError('invalid oid: %r' % oid) try: handle_invalidations = ( lambda oids: self._handle_invalidations(db_name, oids)) storage.end(handle_invalidations=handle_invalidations) except ConflictError: log(20, 'Conflict during commit') yield client.write(STATUS_INVALID) else: self._report_load_record(storage) log(20, 'Committed %3s objects %s bytes at %s', len(oids), tdata_len, datetime.now()) yield client.write(STATUS_OKAY) client.unused_oids[db_name] -= oid_set for c in self.clients: if c is not client: c.invalid[db_name].update(oids) storage.d_bytes_since_pack += tdata_len + 8
def serve_to_client(self, client_socket): client = ConnectedClient(client_socket) # Initialize per-storage state for the new client. client.invalid = dict( (db_name, set()) for db_name in self.storages) client.unused_oids = dict( (db_name, set()) for db_name in self.storages) self.clients.add(client) while not client.closed: try: command = yield client.read(1) except (ConnectionClosed, SocketError): break else: if command in self.handlers: handler_name = self.handlers[command] handler = getattr(self, handler_name) yield handler(client) elif command in self.db_handlers: handler_name = self.db_handlers[command] handler = getattr(self, handler_name) # Get database name. name_length = str_to_int4((yield client.read(4))) db_name = yield client.read(name_length) yield handler(client, db_name) yield client.flush() log(20, 'Connection closed.') self.clients.remove(client)
def handle_V(self, s): # Verify protocol version match. client_protocol = read(s, 4) log(10, 'Client Protocol: %s', str_to_int4(client_protocol)) assert len(self.protocol) == 4 write(s, self.protocol) if client_protocol != self.protocol: raise ClientError("Protocol not supported.")
def unpack_record(record): """(record:str) -> oid:str, data:str, refs:str The inverse of pack_record(). """ oid = record[:8] data_length = str_to_int4(record[8:12]) data_end = 12 + data_length data = record[12:data_end] refs = record[data_end:] return oid, data, refs
def handle_bulk_read(self, client, db_name): # B log(20, 'Bulk read %s' % db_name) storage = self.storages[db_name] number_of_oids = str_to_int4((yield client.read(4))) oid_str_len = 8 * number_of_oids oid_str = yield client.read(oid_str_len) oids = split_oids(oid_str) for oid in oids: yield self._send_load_response(client, db_name, storage, oid)
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i + 4]) i += 4 oid = tdata[i:i + 8] record = tdata[i + 8:i + rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def check_int4_to_str_str_to_int4(self): for x in range(3): assert len(int4_to_str(x)) == 4 assert x == str_to_int4(int4_to_str(x))