def __init__(self, client, db_name):
self.client = client
self.db_name = int4_to_str(len(db_name)) + db_name
self.socket = client.socket
self.oid_pool = []
self.oid_pool_size = 32
self.begin()
def end(self, handle_invalidations=None):
write(self.s, 'C')
n = read_int4(self.s)
oid_list = []
if n != 0:
packed_oids = read(self.s, n * 8)
oid_list = split_oids(packed_oids)
try:
handle_invalidations(oid_list)
except ConflictError:
self.transaction_new_oids.reverse()
self.oid_pool.extend(self.transaction_new_oids)
assert len(self.oid_pool) == len(set(self.oid_pool))
self.begin() # clear out records and transaction_new_oids.
write_int4(self.s, 0) # Tell server we are done.
raise
tdata = []
for oid, record in iteritems(self.records):
tdata.append(int4_to_str(8 + len(record)))
tdata.append(as_bytes(oid))
tdata.append(record)
tdata = join_bytes(tdata)
write_int4_str(self.s, tdata)
self.records.clear()
if len(tdata) > 0:
status = read(self.s, 1)
if status == STATUS_OKAY:
pass
elif status == STATUS_INVALID:
raise WriteConflictError()
else:
raise ProtocolError('server returned invalid status %r' %
status)
def end(self, handle_invalidations=None):
write(self.s, 'C')
n = read_int4(self.s)
oid_list = []
if n != 0:
packed_oids = read(self.s, n*8)
oid_list = split_oids(packed_oids)
try:
handle_invalidations(oid_list)
except ConflictError:
self.transaction_new_oids.reverse()
self.oid_pool.extend(self.transaction_new_oids)
assert len(self.oid_pool) == len(set(self.oid_pool))
self.begin() # clear out records and transaction_new_oids.
write_int4(self.s, 0) # Tell server we are done.
raise
tdata = []
for oid, record in iteritems(self.records):
tdata.append(int4_to_str(8 + len(record)))
tdata.append(as_bytes(oid))
tdata.append(record)
tdata = join_bytes(tdata)
write_int4_str(self.s, tdata)
self.records.clear()
if len(tdata) > 0:
status = read(self.s, 1)
if status == STATUS_OKAY:
pass
elif status == STATUS_INVALID:
raise WriteConflictError()
else:
raise ProtocolError('server returned invalid status %r' % status)
def bulk_load(self, oids):
oid_str = join_bytes(oids)
num_oids, remainder = divmod(len(oid_str), 8)
assert remainder == 0, remainder
write_all(self.s, 'B', int4_to_str(num_oids), oid_str)
records = [self._get_load_response(oid) for oid in oids]
for record in records:
yield record
def bulk_load(self, oids):
oid_str = join_bytes(oids)
num_oids, remainder = divmod(len(oid_str), 8)
assert remainder == 0, remainder
write_all(self.s, 'B', int4_to_str(num_oids), oid_str)
records = [self._get_load_response(oid) for oid in oids]
for record in records:
yield record
def handle_S(self, s):
# sync
client = self._find_client(s)
self._report_load_record()
self._sync_storage()
log(8, 'Sync %s', len(client.invalid))
write_all(s,
int4_to_str(len(client.invalid)), join_bytes(client.invalid))
client.invalid.clear()
def handle_S(self, s):
# sync
client = self._find_client(s)
self._report_load_record()
self._sync_storage()
log(8, 'Sync %s', len(client.invalid))
write_all(s, int4_to_str(len(client.invalid)),
join_bytes(client.invalid))
client.invalid.clear()
def handle_commit(self, client, db_name):
# C
log(20, 'Commit %s' % db_name)
storage = self.storages[db_name]
self._sync_storage(db_name, storage)
invalid = client.invalid[db_name]
yield client.write(int4_to_str(len(invalid)))
yield client.write(join_bytes(invalid))
yield client.flush()
invalid.clear()
tdata_len = str_to_int4((yield client.read(4)))
if tdata_len == 0:
# Client decided not to commit (e.g. conflict)
return
tdata = yield client.read(tdata_len)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', tdata_len)
storage.begin()
i = 0
oids = []
while i < tdata_len:
rlen = str_to_int4(tdata[i:i+4])
i += 4
oid = tdata[i:i+8]
record = tdata[i+8:i+rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s',
str_to_int8(oid), rlen, class_name)
storage.store(oid, record)
oids.append(oid)
assert i == tdata_len
oid_set = set(oids)
for c in self.clients:
if c is not client:
if oid_set.intersection(c.unused_oids[db_name]):
raise ClientError('invalid oid: %r' % oid)
try:
handle_invalidations = (
lambda oids: self._handle_invalidations(db_name, oids))
storage.end(handle_invalidations=handle_invalidations)
except ConflictError:
log(20, 'Conflict during commit')
yield client.write(STATUS_INVALID)
else:
self._report_load_record(storage)
log(20, 'Committed %3s objects %s bytes at %s',
len(oids), tdata_len, datetime.now())
yield client.write(STATUS_OKAY)
client.unused_oids[db_name] -= oid_set
for c in self.clients:
if c is not client:
c.invalid[db_name].update(oids)
storage.d_bytes_since_pack += tdata_len + 8
def handle_sync(self, client, db_name):
# S
log(20, 'Sync %s' % db_name)
storage = self.storages[db_name]
self._report_load_record(storage)
self._sync_storage(db_name, storage)
invalid = client.invalid[db_name]
log(8, 'Sync %s', len(invalid))
yield client.write(int4_to_str(len(invalid)))
yield client.write(join_bytes(invalid))
invalid.clear()
def handle_C(self, s):
# commit
self._sync_storage()
client = self._find_client(s)
write_all(s,
int4_to_str(len(client.invalid)), join_bytes(client.invalid))
client.invalid.clear()
tdata = read_int4_str(s)
if len(tdata) == 0:
return # client decided not to commit (e.g. conflict)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', len(tdata))
self.storage.begin()
i = 0
oids = []
while i < len(tdata):
rlen = str_to_int4(tdata[i:i+4])
i += 4
oid = tdata[i:i+8]
record = tdata[i+8:i+rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s',
str_to_int8(oid), rlen, class_name)
self.storage.store(oid, record)
oids.append(oid)
assert i == len(tdata)
oid_set = set(oids)
for other_client in self.clients:
if other_client is not client:
if oid_set.intersection(other_client.unused_oids):
raise ClientError("invalid oid: %r" % oid)
try:
self.storage.end(handle_invalidations=self._handle_invalidations)
except ConflictError:
log(20, 'Conflict during commit')
write(s, STATUS_INVALID)
else:
self._report_load_record()
log(20, 'Committed %3s objects %s bytes at %s',
len(oids), len(tdata), datetime.now())
write(s, STATUS_OKAY)
client.unused_oids -= oid_set
for c in self.clients:
if c is not client:
c.invalid.update(oids)
self.bytes_since_pack += len(tdata) + 8
def handle_C(self, s):
# commit
self._sync_storage()
client = self._find_client(s)
write_all(s, int4_to_str(len(client.invalid)),
join_bytes(client.invalid))
client.invalid.clear()
tdata = read_int4_str(s)
if len(tdata) == 0:
return # client decided not to commit (e.g. conflict)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', len(tdata))
self.storage.begin()
i = 0
oids = []
while i < len(tdata):
rlen = str_to_int4(tdata[i:i + 4])
i += 4
oid = tdata[i:i + 8]
record = tdata[i + 8:i + rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen,
class_name)
self.storage.store(oid, record)
oids.append(oid)
assert i == len(tdata)
oid_set = set(oids)
for other_client in self.clients:
if other_client is not client:
if oid_set.intersection(other_client.unused_oids):
raise ClientError("invalid oid: %r" % oid)
try:
self.storage.end(handle_invalidations=self._handle_invalidations)
except ConflictError:
log(20, 'Conflict during commit')
write(s, STATUS_INVALID)
else:
self._report_load_record()
log(20, 'Committed %3s objects %s bytes at %s', len(oids),
len(tdata), datetime.now())
write(s, STATUS_OKAY)
client.unused_oids -= oid_set
for c in self.clients:
if c is not client:
c.invalid.update(oids)
self.bytes_since_pack += len(tdata) + 8
def _send_load_response(self, client, db_name, storage, oid):
if oid in client.invalid[db_name]:
yield client.write(STATUS_INVALID)
else:
try:
record = storage.load(oid)
except KeyError:
log(10, 'KeyError %s', str_to_int8(oid))
yield client.write(STATUS_KEYERROR)
except ReadConflictError:
log(10, 'ReadConflictError %s', str_to_int8(oid))
yield client.write(STATUS_INVALID)
else:
if is_logging(5):
class_name = extract_class_name(record)
if class_name in storage.d_load_record:
storage.d_load_record[class_name] += 1
else:
storage.d_load_record[class_name] = 1
log(4, 'Load %-7s %s', str_to_int8(oid), class_name)
yield client.write(STATUS_OKAY)
yield client.write(int4_to_str(len(record)))
yield client.write(record)
def _handle_enumerate_database_names(self, client, names):
yield client.write(int4_to_str(len(names)))
for name in names:
yield client.write(int4_to_str(len(name)))
yield client.write(name)
class StorageServer(object):
protocol = int4_to_str(1)
def __init__(self,
storage,
host=DEFAULT_HOST,
port=DEFAULT_PORT,
address=None,
gcbytes=DEFAULT_GCBYTES):
self.storage = storage
self.clients = []
self.sockets = []
self.packer = None
self.address = SocketAddress.new(address or (host, port))
self.load_record = {}
self.bytes_since_pack = 0
self.gcbytes = gcbytes # Trigger a pack after this many bytes.
assert isinstance(gcbytes, (int, float))
def serve(self):
sock = self.address.get_listening_socket()
log(20, 'Ready on %s', self.address)
self.sockets.append(sock)
try:
while 1:
if self.packer is not None:
timeout = 0.0
else:
timeout = None
r, w, e = select.select(self.sockets, [], [], timeout)
for s in r:
if s is sock:
# new connection
conn, addr = s.accept()
self.address.set_connection_options(conn)
self.clients.append(_Client(conn, addr))
self.sockets.append(conn)
else:
# command from client
try:
self.handle(s)
except (ClientError, socket.error, socket.timeout,
IOError):
exc = sys.exc_info()[1]
log(10, '%s', ''.join(map(str, exc.args)))
self.sockets.remove(s)
self.clients.remove(self._find_client(s))
s.close()
if (self.packer is None
and 0 < self.gcbytes <= self.bytes_since_pack):
self.packer = self.storage.get_packer()
if self.packer is not None:
log(20, 'gc started at %s' % datetime.now())
if not r and self.packer is not None:
try:
pack_step = next(self.packer)
if isinstance(pack_step, str):
log(15, 'gc ' + pack_step)
except StopIteration:
log(20, 'gc at %s' % datetime.now())
self.packer = None # done packing
self.bytes_since_pack = 0 # reset
finally:
self.address.close(sock)
def handle(self, s):
command_byte = read(s, 1)[0]
if type(command_byte) is int:
command_code = chr(command_byte)
else:
command_code = command_byte
handler = getattr(self, 'handle_%s' % command_code, None)
if handler is None:
raise ClientError('No such command code: %r' % command_code)
handler(s)
def _find_client(self, s):
for client in self.clients:
if client.s is s:
return client
assert 0
def _new_oids(self, s, count):
oids = []
while len(oids) < count:
oid = self.storage.new_oid()
for client in self.clients:
if oid in client.invalid:
oid = None
break
if oid is not None:
oids.append(oid)
self._find_client(s).unused_oids.update(oids)
return oids
def handle_N(self, s):
# new OID
write(s, self._new_oids(s, 1)[0])
def handle_M(self, s):
# new OIDs
count = ord(read(s, 1))
log(10, "oids: %s", count)
write(s, join_bytes(self._new_oids(s, count)))
def handle_L(self, s):
# load
oid = read(s, 8)
self._send_load_response(s, oid)
def _send_load_response(self, s, oid):
if oid in self._find_client(s).invalid:
write(s, STATUS_INVALID)
else:
try:
record = self.storage.load(oid)
except KeyError:
log(10, 'KeyError %s', str_to_int8(oid))
write(s, STATUS_KEYERROR)
except ReadConflictError:
log(10, 'ReadConflictError %s', str_to_int8(oid))
write(s, STATUS_INVALID)
else:
if is_logging(5):
class_name = extract_class_name(record)
if class_name in self.load_record:
self.load_record[class_name] += 1
else:
self.load_record[class_name] = 1
log(4, 'Load %-7s %s', str_to_int8(oid), class_name)
write(s, STATUS_OKAY)
write_int4_str(s, record)
def handle_C(self, s):
# commit
self._sync_storage()
client = self._find_client(s)
write_all(s, int4_to_str(len(client.invalid)),
join_bytes(client.invalid))
client.invalid.clear()
tdata = read_int4_str(s)
if len(tdata) == 0:
return # client decided not to commit (e.g. conflict)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', len(tdata))
self.storage.begin()
i = 0
oids = []
while i < len(tdata):
rlen = str_to_int4(tdata[i:i + 4])
i += 4
oid = tdata[i:i + 8]
record = tdata[i + 8:i + rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen,
class_name)
self.storage.store(oid, record)
oids.append(oid)
assert i == len(tdata)
oid_set = set(oids)
for other_client in self.clients:
if other_client is not client:
if oid_set.intersection(other_client.unused_oids):
raise ClientError("invalid oid: %r" % oid)
try:
self.storage.end(handle_invalidations=self._handle_invalidations)
except ConflictError:
log(20, 'Conflict during commit')
write(s, STATUS_INVALID)
else:
self._report_load_record()
log(20, 'Committed %3s objects %s bytes at %s', len(oids),
len(tdata), datetime.now())
write(s, STATUS_OKAY)
client.unused_oids -= oid_set
for c in self.clients:
if c is not client:
c.invalid.update(oids)
self.bytes_since_pack += len(tdata) + 8
def _report_load_record(self):
if self.load_record and is_logging(5):
log(
5, '[%s]\n' % getpid() +
'\n'.join("%8s: %s" % (item[1], item[0])
for item in sorted(self.load_record.items())))
self.load_record.clear()
def _handle_invalidations(self, oids):
for c in self.clients:
c.invalid.update(oids)
def _sync_storage(self):
self._handle_invalidations(self.storage.sync())
def handle_S(self, s):
# sync
client = self._find_client(s)
self._report_load_record()
self._sync_storage()
log(8, 'Sync %s', len(client.invalid))
write_all(s, int4_to_str(len(client.invalid)),
join_bytes(client.invalid))
client.invalid.clear()
def handle_P(self, s):
# pack
if self.packer is None:
log(20, 'Pack started at %s' % datetime.now())
self.packer = self.storage.get_packer()
if self.packer is None:
self.storage.pack()
log(20, 'Pack completed at %s' % datetime.now())
else:
log(20, 'Pack already in progress at %s' % datetime.now())
write(s, STATUS_OKAY)
def handle_B(self, s):
# bulk read of objects
number_of_oids = read_int4(s)
oid_str = read(s, 8 * number_of_oids)
oids = split_oids(oid_str)
for oid in oids:
self._send_load_response(s, oid)
def handle_Q(self, s):
# graceful quit
log(20, 'Quit')
self.storage.close()
raise SystemExit
def handle_V(self, s):
# Verify protocol version match.
client_protocol = read(s, 4)
log(10, 'Client Protocol: %s', str_to_int4(client_protocol))
assert len(self.protocol) == 4
write(s, self.protocol)
if client_protocol != self.protocol:
raise ClientError("Protocol not supported.")
def pack_record(oid, data, refs):
"""(oid:str, data:str, refs:str) -> record:str
"""
return join_bytes([oid, int4_to_str(len(data)), data, refs])
from durus.serialize import extract_class_name, split_oids
from durus.storage_server import (
DEFAULT_GCBYTES,
STATUS_OKAY, STATUS_KEYERROR, STATUS_INVALID,
ClientError,
)
from durus.utils import (
int4_to_str, int8_to_str, str_to_int4, str_to_int8,
join_bytes,
)
DEFAULT_HOST = '127.0.0.1'
DEFAULT_PORT = 22972
PROTOCOL = int4_to_str(20001)
EXTENSION = '.durus'
def database_names(path):
"""Return a list of all Durus database names in a given path."""
for filename in os.listdir(path):
name, ext = os.path.splitext(filename)
if ext == EXTENSION:
yield name
class ConnectedClient(object):
def __init__(self, client_socket):
def check_int4_to_str_str_to_int4(self):
for x in range(3):
assert len(int4_to_str(x)) == 4
assert x == str_to_int4(int4_to_str(x))
def pack_record(oid, data, refs):
"""(oid:str, data:str, refs:str) -> record:str
"""
return join_bytes([oid, int4_to_str(len(data)), data, refs])