def __init__(self, client, db_name): self.client = client self.db_name = int4_to_str(len(db_name)) + db_name self.socket = client.socket self.oid_pool = [] self.oid_pool_size = 32 self.begin()
def end(self, handle_invalidations=None): write(self.s, 'C') n = read_int4(self.s) oid_list = [] if n != 0: packed_oids = read(self.s, n * 8) oid_list = split_oids(packed_oids) try: handle_invalidations(oid_list) except ConflictError: self.transaction_new_oids.reverse() self.oid_pool.extend(self.transaction_new_oids) assert len(self.oid_pool) == len(set(self.oid_pool)) self.begin() # clear out records and transaction_new_oids. write_int4(self.s, 0) # Tell server we are done. raise tdata = [] for oid, record in iteritems(self.records): tdata.append(int4_to_str(8 + len(record))) tdata.append(as_bytes(oid)) tdata.append(record) tdata = join_bytes(tdata) write_int4_str(self.s, tdata) self.records.clear() if len(tdata) > 0: status = read(self.s, 1) if status == STATUS_OKAY: pass elif status == STATUS_INVALID: raise WriteConflictError() else: raise ProtocolError('server returned invalid status %r' % status)
def end(self, handle_invalidations=None): write(self.s, 'C') n = read_int4(self.s) oid_list = [] if n != 0: packed_oids = read(self.s, n*8) oid_list = split_oids(packed_oids) try: handle_invalidations(oid_list) except ConflictError: self.transaction_new_oids.reverse() self.oid_pool.extend(self.transaction_new_oids) assert len(self.oid_pool) == len(set(self.oid_pool)) self.begin() # clear out records and transaction_new_oids. write_int4(self.s, 0) # Tell server we are done. raise tdata = [] for oid, record in iteritems(self.records): tdata.append(int4_to_str(8 + len(record))) tdata.append(as_bytes(oid)) tdata.append(record) tdata = join_bytes(tdata) write_int4_str(self.s, tdata) self.records.clear() if len(tdata) > 0: status = read(self.s, 1) if status == STATUS_OKAY: pass elif status == STATUS_INVALID: raise WriteConflictError() else: raise ProtocolError('server returned invalid status %r' % status)
def bulk_load(self, oids): oid_str = join_bytes(oids) num_oids, remainder = divmod(len(oid_str), 8) assert remainder == 0, remainder write_all(self.s, 'B', int4_to_str(num_oids), oid_str) records = [self._get_load_response(oid) for oid in oids] for record in records: yield record
def handle_S(self, s): # sync client = self._find_client(s) self._report_load_record() self._sync_storage() log(8, 'Sync %s', len(client.invalid)) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear()
def handle_commit(self, client, db_name): # C log(20, 'Commit %s' % db_name) storage = self.storages[db_name] self._sync_storage(db_name, storage) invalid = client.invalid[db_name] yield client.write(int4_to_str(len(invalid))) yield client.write(join_bytes(invalid)) yield client.flush() invalid.clear() tdata_len = str_to_int4((yield client.read(4))) if tdata_len == 0: # Client decided not to commit (e.g. conflict) return tdata = yield client.read(tdata_len) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', tdata_len) storage.begin() i = 0 oids = [] while i < tdata_len: rlen = str_to_int4(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) storage.store(oid, record) oids.append(oid) assert i == tdata_len oid_set = set(oids) for c in self.clients: if c is not client: if oid_set.intersection(c.unused_oids[db_name]): raise ClientError('invalid oid: %r' % oid) try: handle_invalidations = ( lambda oids: self._handle_invalidations(db_name, oids)) storage.end(handle_invalidations=handle_invalidations) except ConflictError: log(20, 'Conflict during commit') yield client.write(STATUS_INVALID) else: self._report_load_record(storage) log(20, 'Committed %3s objects %s bytes at %s', len(oids), tdata_len, datetime.now()) yield client.write(STATUS_OKAY) client.unused_oids[db_name] -= oid_set for c in self.clients: if c is not client: c.invalid[db_name].update(oids) storage.d_bytes_since_pack += tdata_len + 8
def handle_sync(self, client, db_name): # S log(20, 'Sync %s' % db_name) storage = self.storages[db_name] self._report_load_record(storage) self._sync_storage(db_name, storage) invalid = client.invalid[db_name] log(8, 'Sync %s', len(invalid)) yield client.write(int4_to_str(len(invalid))) yield client.write(join_bytes(invalid)) invalid.clear()
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i+4]) i += 4 oid = tdata[i:i+8] record = tdata[i+8:i+rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i + 4]) i += 4 oid = tdata[i:i + 8] record = tdata[i + 8:i + rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8
def _send_load_response(self, client, db_name, storage, oid): if oid in client.invalid[db_name]: yield client.write(STATUS_INVALID) else: try: record = storage.load(oid) except KeyError: log(10, 'KeyError %s', str_to_int8(oid)) yield client.write(STATUS_KEYERROR) except ReadConflictError: log(10, 'ReadConflictError %s', str_to_int8(oid)) yield client.write(STATUS_INVALID) else: if is_logging(5): class_name = extract_class_name(record) if class_name in storage.d_load_record: storage.d_load_record[class_name] += 1 else: storage.d_load_record[class_name] = 1 log(4, 'Load %-7s %s', str_to_int8(oid), class_name) yield client.write(STATUS_OKAY) yield client.write(int4_to_str(len(record))) yield client.write(record)
def _handle_enumerate_database_names(self, client, names): yield client.write(int4_to_str(len(names))) for name in names: yield client.write(int4_to_str(len(name))) yield client.write(name)
class StorageServer(object): protocol = int4_to_str(1) def __init__(self, storage, host=DEFAULT_HOST, port=DEFAULT_PORT, address=None, gcbytes=DEFAULT_GCBYTES): self.storage = storage self.clients = [] self.sockets = [] self.packer = None self.address = SocketAddress.new(address or (host, port)) self.load_record = {} self.bytes_since_pack = 0 self.gcbytes = gcbytes # Trigger a pack after this many bytes. assert isinstance(gcbytes, (int, float)) def serve(self): sock = self.address.get_listening_socket() log(20, 'Ready on %s', self.address) self.sockets.append(sock) try: while 1: if self.packer is not None: timeout = 0.0 else: timeout = None r, w, e = select.select(self.sockets, [], [], timeout) for s in r: if s is sock: # new connection conn, addr = s.accept() self.address.set_connection_options(conn) self.clients.append(_Client(conn, addr)) self.sockets.append(conn) else: # command from client try: self.handle(s) except (ClientError, socket.error, socket.timeout, IOError): exc = sys.exc_info()[1] log(10, '%s', ''.join(map(str, exc.args))) self.sockets.remove(s) self.clients.remove(self._find_client(s)) s.close() if (self.packer is None and 0 < self.gcbytes <= self.bytes_since_pack): self.packer = self.storage.get_packer() if self.packer is not None: log(20, 'gc started at %s' % datetime.now()) if not r and self.packer is not None: try: pack_step = next(self.packer) if isinstance(pack_step, str): log(15, 'gc ' + pack_step) except StopIteration: log(20, 'gc at %s' % datetime.now()) self.packer = None # done packing self.bytes_since_pack = 0 # reset finally: self.address.close(sock) def handle(self, s): command_byte = read(s, 1)[0] if type(command_byte) is int: command_code = chr(command_byte) else: command_code = command_byte handler = getattr(self, 'handle_%s' % command_code, None) if handler is None: raise ClientError('No such command code: %r' % command_code) handler(s) def _find_client(self, s): for client in self.clients: if client.s is s: return client assert 0 def _new_oids(self, s, count): oids = [] while len(oids) < count: oid = self.storage.new_oid() for client in self.clients: if oid in client.invalid: oid = None break if oid is not None: oids.append(oid) self._find_client(s).unused_oids.update(oids) return oids def handle_N(self, s): # new OID write(s, self._new_oids(s, 1)[0]) def handle_M(self, s): # new OIDs count = ord(read(s, 1)) log(10, "oids: %s", count) write(s, join_bytes(self._new_oids(s, count))) def handle_L(self, s): # load oid = read(s, 8) self._send_load_response(s, oid) def _send_load_response(self, s, oid): if oid in self._find_client(s).invalid: write(s, STATUS_INVALID) else: try: record = self.storage.load(oid) except KeyError: log(10, 'KeyError %s', str_to_int8(oid)) write(s, STATUS_KEYERROR) except ReadConflictError: log(10, 'ReadConflictError %s', str_to_int8(oid)) write(s, STATUS_INVALID) else: if is_logging(5): class_name = extract_class_name(record) if class_name in self.load_record: self.load_record[class_name] += 1 else: self.load_record[class_name] = 1 log(4, 'Load %-7s %s', str_to_int8(oid), class_name) write(s, STATUS_OKAY) write_int4_str(s, record) def handle_C(self, s): # commit self._sync_storage() client = self._find_client(s) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() tdata = read_int4_str(s) if len(tdata) == 0: return # client decided not to commit (e.g. conflict) logging_debug = is_logging(10) logging_debug and log(10, 'Committing %s bytes', len(tdata)) self.storage.begin() i = 0 oids = [] while i < len(tdata): rlen = str_to_int4(tdata[i:i + 4]) i += 4 oid = tdata[i:i + 8] record = tdata[i + 8:i + rlen] i += rlen if logging_debug: class_name = extract_class_name(record) log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen, class_name) self.storage.store(oid, record) oids.append(oid) assert i == len(tdata) oid_set = set(oids) for other_client in self.clients: if other_client is not client: if oid_set.intersection(other_client.unused_oids): raise ClientError("invalid oid: %r" % oid) try: self.storage.end(handle_invalidations=self._handle_invalidations) except ConflictError: log(20, 'Conflict during commit') write(s, STATUS_INVALID) else: self._report_load_record() log(20, 'Committed %3s objects %s bytes at %s', len(oids), len(tdata), datetime.now()) write(s, STATUS_OKAY) client.unused_oids -= oid_set for c in self.clients: if c is not client: c.invalid.update(oids) self.bytes_since_pack += len(tdata) + 8 def _report_load_record(self): if self.load_record and is_logging(5): log( 5, '[%s]\n' % getpid() + '\n'.join("%8s: %s" % (item[1], item[0]) for item in sorted(self.load_record.items()))) self.load_record.clear() def _handle_invalidations(self, oids): for c in self.clients: c.invalid.update(oids) def _sync_storage(self): self._handle_invalidations(self.storage.sync()) def handle_S(self, s): # sync client = self._find_client(s) self._report_load_record() self._sync_storage() log(8, 'Sync %s', len(client.invalid)) write_all(s, int4_to_str(len(client.invalid)), join_bytes(client.invalid)) client.invalid.clear() def handle_P(self, s): # pack if self.packer is None: log(20, 'Pack started at %s' % datetime.now()) self.packer = self.storage.get_packer() if self.packer is None: self.storage.pack() log(20, 'Pack completed at %s' % datetime.now()) else: log(20, 'Pack already in progress at %s' % datetime.now()) write(s, STATUS_OKAY) def handle_B(self, s): # bulk read of objects number_of_oids = read_int4(s) oid_str = read(s, 8 * number_of_oids) oids = split_oids(oid_str) for oid in oids: self._send_load_response(s, oid) def handle_Q(self, s): # graceful quit log(20, 'Quit') self.storage.close() raise SystemExit def handle_V(self, s): # Verify protocol version match. client_protocol = read(s, 4) log(10, 'Client Protocol: %s', str_to_int4(client_protocol)) assert len(self.protocol) == 4 write(s, self.protocol) if client_protocol != self.protocol: raise ClientError("Protocol not supported.")
def pack_record(oid, data, refs): """(oid:str, data:str, refs:str) -> record:str """ return join_bytes([oid, int4_to_str(len(data)), data, refs])
from durus.serialize import extract_class_name, split_oids from durus.storage_server import ( DEFAULT_GCBYTES, STATUS_OKAY, STATUS_KEYERROR, STATUS_INVALID, ClientError, ) from durus.utils import ( int4_to_str, int8_to_str, str_to_int4, str_to_int8, join_bytes, ) DEFAULT_HOST = '127.0.0.1' DEFAULT_PORT = 22972 PROTOCOL = int4_to_str(20001) EXTENSION = '.durus' def database_names(path): """Return a list of all Durus database names in a given path.""" for filename in os.listdir(path): name, ext = os.path.splitext(filename) if ext == EXTENSION: yield name class ConnectedClient(object): def __init__(self, client_socket):
def check_int4_to_str_str_to_int4(self): for x in range(3): assert len(int4_to_str(x)) == 4 assert x == str_to_int4(int4_to_str(x))