def list_open(self):
"""List open databases on server.
:rtype: List of strings
"""
write(self.socket, 'E')
return list(self._enumerate_database_names())
def end(self, handle_invalidations=None):
write(self.s, 'C')
n = read_int4(self.s)
oid_list = []
if n != 0:
packed_oids = read(self.s, n * 8)
oid_list = split_oids(packed_oids)
try:
handle_invalidations(oid_list)
except ConflictError:
self.transaction_new_oids.reverse()
self.oid_pool.extend(self.transaction_new_oids)
assert len(self.oid_pool) == len(set(self.oid_pool))
self.begin() # clear out records and transaction_new_oids.
write_int4(self.s, 0) # Tell server we are done.
raise
tdata = []
for oid, record in iteritems(self.records):
tdata.append(int4_to_str(8 + len(record)))
tdata.append(as_bytes(oid))
tdata.append(record)
tdata = join_bytes(tdata)
write_int4_str(self.s, tdata)
self.records.clear()
if len(tdata) > 0:
status = read(self.s, 1)
if status == STATUS_OKAY:
pass
elif status == STATUS_INVALID:
raise WriteConflictError()
else:
raise ProtocolError('server returned invalid status %r' %
status)
def end(self, handle_invalidations=None):
write(self.s, 'C')
n = read_int4(self.s)
oid_list = []
if n != 0:
packed_oids = read(self.s, n*8)
oid_list = split_oids(packed_oids)
try:
handle_invalidations(oid_list)
except ConflictError:
self.transaction_new_oids.reverse()
self.oid_pool.extend(self.transaction_new_oids)
assert len(self.oid_pool) == len(set(self.oid_pool))
self.begin() # clear out records and transaction_new_oids.
write_int4(self.s, 0) # Tell server we are done.
raise
tdata = []
for oid, record in iteritems(self.records):
tdata.append(int4_to_str(8 + len(record)))
tdata.append(as_bytes(oid))
tdata.append(record)
tdata = join_bytes(tdata)
write_int4_str(self.s, tdata)
self.records.clear()
if len(tdata) > 0:
status = read(self.s, 1)
if status == STATUS_OKAY:
pass
elif status == STATUS_INVALID:
raise WriteConflictError()
else:
raise ProtocolError('server returned invalid status %r' % status)
def list_all(self):
"""List all databases available on the server.
:rtype: list of strings
"""
write(self.socket, 'A')
return list(self._enumerate_database_names())
def store(self, name_value_sequence):
"""([(str, str)]) -> [(str, int|None, int)]
Record all of the items in the sequence.
Return a list of triples, each giving a name, an old position (or None
if this is a new name), and a new position.
"""
self.file.seek_end()
start = self.file.tell()
write_int8(self.file, 0)
result = []
index = {}
try:
for name, value in name_value_sequence:
new_position = self.file.tell()
old_position = self.get_position(name)
index[name] = new_position
result.append((name, old_position, new_position))
write_int8(self.file, len(name) + len(value))
write(self.file, name)
write(self.file, value)
except:
# Revert before raising.
self.file.seek(start)
self.file.truncate()
raise
end = self.file.tell()
self.file.seek(start)
write_int8(self.file, end - start - 8)
self.file.seek(end)
self.memory_index.update(index)
return result
def store(self, name_value_sequence):
"""([(str, str)]) -> [(str, int|None, int)]
Record all of the items in the sequence.
Return a list of triples, each giving a name, an old position (or None
if this is a new name), and a new position.
"""
self.file.seek_end()
start = self.file.tell()
write_int8(self.file, 0)
result = []
index = {}
try:
for name, value in name_value_sequence:
new_position = self.file.tell()
old_position = self.get_position(name)
index[name] = new_position
result.append((name, old_position, new_position))
write_int8(self.file, len(name) + len(value))
write(self.file, name)
write(self.file, value)
except:
# Revert before raising.
self.file.seek(start)
self.file.truncate()
raise
end = self.file.tell()
self.file.seek(start)
write_int8(self.file, end - start - 8)
self.file.seek(end)
self.memory_index.update(index)
return result
def server_protocol(self):
"""Get the protocol used by the server.
:rtype: 4-byte string
"""
write(self.socket, 'V')
return read(self.socket, 4)
def open(self, db_name):
"""Open the named database.
:param db_name: Name of database to open.
:ptype db_name: string
"""
write(self.socket, 'O')
self._write_database_name(db_name)
def handle_V(self, s):
# Verify protocol version match.
client_protocol = read(s, 4)
log(10, 'Client Protocol: %s', str_to_int4(client_protocol))
assert len(self.protocol) == 4
write(s, self.protocol)
if client_protocol != self.protocol:
raise ClientError("Protocol not supported.")
def close(self, db_name):
"""Close the named database.
:param db_name: Name of database to close.
:ptype db_name: string
"""
write(self.socket, 'X')
self._write_database_name(db_name)
def sync(self):
write(self.s, 'S')
n = read_int4(self.s)
if n == 0:
packed_oids = ''
else:
packed_oids = read(self.s, n*8)
return split_oids(packed_oids)
def sync(self):
write(self.s, 'S')
n = read_int4(self.s)
if n == 0:
packed_oids = ''
else:
packed_oids = read(self.s, n * 8)
return split_oids(packed_oids)
def b(self):
s = BytesIO()
for x in ('', 'a', 'ab', 'a' * 1000):
x = as_bytes(x)
s.seek(0)
write(s, x)
s.seek(0)
assert x == read(s, len(x))
def handle_V(self, s):
# Verify protocol version match.
client_protocol = read(s, 4)
log(10, 'Client Protocol: %s', str_to_int4(client_protocol))
assert len(self.protocol) == 4
write(s, self.protocol)
if client_protocol != self.protocol:
raise ClientError("Protocol not supported.")
def b(self):
s = BytesIO()
for x in ("", "a", "ab", "a" * 1000):
x = as_bytes(x)
s.seek(0)
write(s, x)
s.seek(0)
assert x == read(s, len(x))
def handle_P(self, s):
# pack
if self.packer is None:
log(20, 'Pack started at %s' % datetime.now())
self.packer = self.storage.get_packer()
if self.packer is None:
self.storage.pack()
log(20, 'Pack completed at %s' % datetime.now())
else:
log(20, 'Pack already in progress at %s' % datetime.now())
write(s, STATUS_OKAY)
def quit(self):
"""Shut down the server process and disconnect.
.. todo::
Determine if it is necessary to keep "quit" in the protocol
or if a server should only be shut down by process control
on the system that the server is running on.
"""
write(self.socket, 'Q')
self.disconnect()
def new_oid(self):
if not self.oid_pool:
batch = self.oid_pool_size
write(self.s, 'M%s' % chr(batch))
self.oid_pool = split_oids(read(self.s, 8 * batch))
self.oid_pool.reverse()
assert len(self.oid_pool) == len(set(self.oid_pool))
oid = self.oid_pool.pop()
assert oid not in self.oid_pool
self.transaction_new_oids.append(oid)
return oid
def handle_P(self, s):
# pack
if self.packer is None:
log(20, 'Pack started at %s' % datetime.now())
self.packer = self.storage.get_packer()
if self.packer is None:
self.storage.pack()
log(20, 'Pack completed at %s' % datetime.now())
else:
log(20, 'Pack already in progress at %s' % datetime.now())
write(s, STATUS_OKAY)
def new_oid(self):
if not self.oid_pool:
batch = self.oid_pool_size
write(self.s, 'M%s' % chr(batch))
self.oid_pool = split_oids(read(self.s, 8 * batch))
self.oid_pool.reverse()
assert len(self.oid_pool) == len(set(self.oid_pool))
oid = self.oid_pool.pop()
assert oid not in self.oid_pool
self.transaction_new_oids.append(oid)
return oid
def f(self):
class FakeSocket(object):
def recv(self, n):
if n > 10:
return as_bytes('')
return as_bytes('x')
def send(self, s):
return len(s)
s = FakeSocket()
write(s, 'x' * 2000000)
read(s, 8)
raises(ShortRead, read, s, 11)
def new_oid(self):
if not self.oid_pool:
batch = self.oid_pool_size
self._send_command('M')
write(self.socket, chr(batch))
self.oid_pool = split_oids(read(self.socket, 8 * batch))
self.oid_pool.reverse()
assert len(self.oid_pool) == len(set(self.oid_pool))
oid = self.oid_pool.pop()
assert oid not in self.oid_pool
self.transaction_new_oids.append(oid)
return oid
def handle_C(self, s):
# commit
self._sync_storage()
client = self._find_client(s)
write_all(s, int4_to_str(len(client.invalid)),
join_bytes(client.invalid))
client.invalid.clear()
tdata = read_int4_str(s)
if len(tdata) == 0:
return # client decided not to commit (e.g. conflict)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', len(tdata))
self.storage.begin()
i = 0
oids = []
while i < len(tdata):
rlen = str_to_int4(tdata[i:i + 4])
i += 4
oid = tdata[i:i + 8]
record = tdata[i + 8:i + rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s', str_to_int8(oid), rlen,
class_name)
self.storage.store(oid, record)
oids.append(oid)
assert i == len(tdata)
oid_set = set(oids)
for other_client in self.clients:
if other_client is not client:
if oid_set.intersection(other_client.unused_oids):
raise ClientError("invalid oid: %r" % oid)
try:
self.storage.end(handle_invalidations=self._handle_invalidations)
except ConflictError:
log(20, 'Conflict during commit')
write(s, STATUS_INVALID)
else:
self._report_load_record()
log(20, 'Committed %3s objects %s bytes at %s', len(oids),
len(tdata), datetime.now())
write(s, STATUS_OKAY)
client.unused_oids -= oid_set
for c in self.clients:
if c is not client:
c.invalid.update(oids)
self.bytes_since_pack += len(tdata) + 8
def handle_C(self, s):
# commit
self._sync_storage()
client = self._find_client(s)
write_all(s,
int4_to_str(len(client.invalid)), join_bytes(client.invalid))
client.invalid.clear()
tdata = read_int4_str(s)
if len(tdata) == 0:
return # client decided not to commit (e.g. conflict)
logging_debug = is_logging(10)
logging_debug and log(10, 'Committing %s bytes', len(tdata))
self.storage.begin()
i = 0
oids = []
while i < len(tdata):
rlen = str_to_int4(tdata[i:i+4])
i += 4
oid = tdata[i:i+8]
record = tdata[i+8:i+rlen]
i += rlen
if logging_debug:
class_name = extract_class_name(record)
log(10, ' oid=%-6s rlen=%-6s %s',
str_to_int8(oid), rlen, class_name)
self.storage.store(oid, record)
oids.append(oid)
assert i == len(tdata)
oid_set = set(oids)
for other_client in self.clients:
if other_client is not client:
if oid_set.intersection(other_client.unused_oids):
raise ClientError("invalid oid: %r" % oid)
try:
self.storage.end(handle_invalidations=self._handle_invalidations)
except ConflictError:
log(20, 'Conflict during commit')
write(s, STATUS_INVALID)
else:
self._report_load_record()
log(20, 'Committed %3s objects %s bytes at %s',
len(oids), len(tdata), datetime.now())
write(s, STATUS_OKAY)
client.unused_oids -= oid_set
for c in self.clients:
if c is not client:
c.invalid.update(oids)
self.bytes_since_pack += len(tdata) + 8
def stop_durus(address):
socket_address = SocketAddress.new(address)
sock = socket_address.get_connected_socket()
if sock is None:
log(20, "Durus server %s doesn't seem to be running." % str(address))
return False
write(sock, 'Q') # graceful exit message
sock.close()
# Try to wait until the address is free.
for attempt in range(20):
sleep(0.5)
sock = socket_address.get_connected_socket()
if sock is None:
break
sock.close()
return True
def destroy(self, db_name):
"""Destroy the named database.
:param db_name: Name of database to destroy.
:ptype db_name: string
.. note::
The database will not be destroyed if it is currently open.
.. warning::
This will permanently delete the file containing the
database on the server.
"""
write(self.socket, 'D')
self._write_database_name(db_name)
def stop_durus(address):
socket_address = SocketAddress.new(address)
sock = socket_address.get_connected_socket()
if sock is None:
log(20, "Durus server %s doesn't seem to be running." %
str(address))
return False
write(sock, 'Q') # graceful exit message
sock.close()
# Try to wait until the address is free.
for attempt in range(20):
sleep(0.5)
sock = socket_address.get_connected_socket()
if sock is None:
break
sock.close()
return True
def generate_shelf(klass, file, items):
"""(File, [(str, str)])
This returns a generator that writes a new Shelf into file,
iterating once through the given items.
The use of an iterator makes it possible to build a new Shelf
incrementally.
"""
file.seek_end()
if not file.tell() == 0:
raise ValueError("Expected %s to be empty." % file)
write(file, klass.prefix)
if not items:
# Just write an empty transaction.
write_int8(file, 0)
# Write an empty index array.
offset_map = OffsetMap(file)
else:
# Write a transaction here with the given items.
transaction_start = file.tell()
# Write a placeholder for the length.
write_int8(file, 0)
# Loop over the items, writing their records.
# Keep track of max_key and max_offset.
max_key = 0
max_offset = 0
n = 0
for name, value in items:
max_key = max(max_key, str_to_int8(name))
max_offset = max(max_offset, file.tell())
write_int8(file, len(name) + len(value))
write(file, name)
write(file, value)
n += 1
yield n
transaction_end = file.tell()
# Write the correct transaction length.
file.seek(transaction_start)
write_int8(file, transaction_end - transaction_start - 8)
# Write the empty array with the calculated dimensions.
file.seek(transaction_end)
for step in OffsetMap.generate(file, max_key, max_offset):
yield step
offset_map = OffsetMap(file)
# Now read through the records and record the offsets in the array.
file.seek(transaction_start + 8)
while file.tell() < transaction_end:
position = file.tell()
record_length = read_int8(file)
name = read(file, 8)
k = str_to_int8(name)
offset_map[k] = position
file.seek(position + 8 + record_length)
n -= 1
yield n
for index in offset_map.gen_stitch():
yield index
def generate_shelf(klass, file, items):
"""(File, [(str, str)])
This returns a generator that writes a new Shelf into file,
iterating once through the given items.
The use of an iterator makes it possible to build a new Shelf
incrementally.
"""
file.seek_end()
if not file.tell() == 0:
raise ValueError("Expected %s to be empty." % file)
write(file, klass.prefix)
if not items:
# Just write an empty transaction.
write_int8(file, 0)
# Write an empty index array.
offset_map = OffsetMap(file)
else:
# Write a transaction here with the given items.
transaction_start = file.tell()
# Write a placeholder for the length.
write_int8(file, 0)
# Loop over the items, writing their records.
# Keep track of max_key and max_offset.
max_key = 0
max_offset = 0
n = 0
for name, value in items:
max_key = max(max_key, str_to_int8(name))
max_offset = max(max_offset, file.tell())
write_int8(file, len(name) + len(value))
write(file, name)
write(file, value)
n += 1
yield n
transaction_end = file.tell()
# Write the correct transaction length.
file.seek(transaction_start)
write_int8(file, transaction_end - transaction_start - 8)
# Write the empty array with the calculated dimensions.
file.seek(transaction_end)
for step in OffsetMap.generate(file, max_key, max_offset):
yield step
offset_map = OffsetMap(file)
# Now read through the records and record the offsets in the array.
file.seek(transaction_start + 8)
while file.tell() < transaction_end:
position = file.tell()
record_length = read_int8(file)
name = read(file, 8)
k = str_to_int8(name)
offset_map[k] = position
file.seek(position + 8 + record_length)
n -= 1
yield n
for index in offset_map.gen_stitch():
yield index
def _send_load_response(self, s, oid):
if oid in self._find_client(s).invalid:
write(s, STATUS_INVALID)
else:
try:
record = self.storage.load(oid)
except KeyError:
log(10, 'KeyError %s', str_to_int8(oid))
write(s, STATUS_KEYERROR)
except ReadConflictError:
log(10, 'ReadConflictError %s', str_to_int8(oid))
write(s, STATUS_INVALID)
else:
if is_logging(5):
class_name = extract_class_name(record)
if class_name in self.load_record:
self.load_record[class_name] += 1
else:
self.load_record[class_name] = 1
log(4, 'Load %-7s %s', str_to_int8(oid), class_name)
write(s, STATUS_OKAY)
write_int4_str(s, record)
def _send_load_response(self, s, oid):
if oid in self._find_client(s).invalid:
write(s, STATUS_INVALID)
else:
try:
record = self.storage.load(oid)
except KeyError:
log(10, 'KeyError %s', str_to_int8(oid))
write(s, STATUS_KEYERROR)
except ReadConflictError:
log(10, 'ReadConflictError %s', str_to_int8(oid))
write(s, STATUS_INVALID)
else:
if is_logging(5):
class_name = extract_class_name(record)
if class_name in self.load_record:
self.load_record[class_name] += 1
else:
self.load_record[class_name] = 1
log(4, 'Load %-7s %s', str_to_int8(oid), class_name)
write(s, STATUS_OKAY)
write_int4_str(s, record)
def load(self, oid):
self._send_command('L')
write(self.socket, oid)
return self._get_load_response(oid)
def pack(self):
write(self.s, 'P')
status = read(self.s, 1)
if status != STATUS_OKAY:
raise ProtocolError('server returned invalid status %r' % status)
def pack(self):
write(self.s, 'P')
status = read(self.s, 1)
if status != STATUS_OKAY:
raise ProtocolError('server returned invalid status %r' % status)
def close(self):
write(self.s, '.') # Closes the server side.
self.s.close()
def close(self):
write(self.s, '.') # Closes the server side.
self.s.close()
def handle_N(self, s):
# new OID
write(s, self._new_oids(s, 1)[0])
def handle_M(self, s):
# new OIDs
count = ord(read(s, 1))
log(10, "oids: %s", count)
write(s, join_bytes(self._new_oids(s, count)))
def handle_N(self, s):
# new OID
write(s, self._new_oids(s, 1)[0])
def _write_header(self, fp):
fp.seek(0, 2)
assert fp.tell() == 0
write(fp, self.MAGIC)
write_int8(fp, 0) # index offset
def handle_M(self, s):
# new OIDs
count = ord(read(s, 1))
log(10, "oids: %s", count)
write(s, join_bytes(self._new_oids(s, count)))
def disconnect(self):
"""Disconnect from the server."""
if self.socket is not None:
write(self.socket, '.')
self.socket.close()
self.socket = None
def _write_database_name(self, db_name):
write_int4(self.socket, len(db_name))
write(self.socket, db_name)
