def test_staff_deadlock_two_cycles(self):
lock_table = {}
store = InMemoryKVStore()
t2 = TransactionHandler(lock_table, 2, store)
t4 = TransactionHandler(lock_table, 4, store)
t6 = TransactionHandler(lock_table, 6, store)
t8 = TransactionHandler(lock_table, 8, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
# t2 has S lock on a; t4 wants X lock
self.assertEqual(t2.perform_get('a'), 'No such key')
self.assertEqual(t4.perform_put('a', 'a1'), None)
# t4 has S lock on b; t2 wants X lock
self.assertEqual(t4.perform_get('b'), 'No such key')
self.assertEqual(t2.perform_put('b', 'b1'), None)
# t6 has S lock on a; t8 wants X lock
self.assertEqual(t6.perform_get('c'), 'No such key')
self.assertEqual(t8.perform_put('c', 'c1'), None)
# t8 has S lock on b; t6 wants X lock
self.assertEqual(t8.perform_get('d'), 'No such key')
self.assertEqual(t6.perform_put('d', 'd1'), None)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 2 or abort_id == 4 or abort_id == 6 or abort_id == 8)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 2 or abort_id == 4 or abort_id == 6 or abort_id == 8)
def test_deadlock_rw_rw(self):
# Should pass after 2
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
coordinator = TransactionCoordinator(lock_table)
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_get('a'), 'No such key')
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_get('b'), 'No such key')
self.assertEqual(t0.perform_put('a', 'a0'), 'Success')
self.assertEqual(t0.perform_put('b', 'b0'), 'Success')
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_get('a'), 'a0') # T1 R(a)
self.assertEqual(t2.perform_get('b'), 'b0') # T2 R(b)
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_put('b', 'b1'), None) # T1 W(b)
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t2.perform_put('a', 'a1'), None) # T2 W(a)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 1 or abort_id == 2)
def __init__(self,
kvstore_class=KVSTORE_CLASS,
log_level=logging.WARNING,
max_handlers=None):
"""
Initializes the server. Does not start the polling loop. After the
constructor returns, there can be no other servers.
"""
self._logger = logging.getLogger('<%s>' % (self.__class__.__name__))
self._logger.setLevel(log_level)
self._remaining_handlers = max_handlers
self._stats = [0, 0]
self._lock_table = {}
self._next_xid = 0
self._store = kvstore_class()
self._log_level = log_level
self._txn_map = {}
self._coordinator = TransactionCoordinator(self._lock_table)
# Raise an exception if we can connect to an existing server. If a
# context switch occurs in the middle of this code segment, or before
# the socket is bound to the socket file, then this will fail. We
# assume that the probability that the user constructs two servers in
# two different processes at the same time is low, and we ignore this
# edge case. It should not be a problem in any of the test files - most
# of these only create a single instance of the server class.
if os.path.exists(SOCKET_FILE):
test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
test_sock.connect(SOCKET_FILE)
raise KVStoreError('Server seems to be running')
except socket.error as e:
pass
finally:
test_sock.close()
os.unlink(SOCKET_FILE)
try:
asyncore.dispatcher.__init__(self)
# Create socket, but do not connect to anything yet
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind(SOCKET_FILE)
self.listen(5)
except Exception as e:
exc = traceback.format_exc()
self._logger.error(
'Uncaught exception in __init__, closing server\n%s', exc[:-1])
self.close()
raise e
self._logger.debug('Constructed server')
def test_deadlock_identical(self):
# Should pass after 2
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t2 = TransactionHandler(lock_table, 2, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_put('a', 'a0'), 'Success')
self.assertEqual(t0.perform_put('b', 'b0'), 'Success')
self.assertEqual(t2.perform_get('a'), None)
self.assertEqual(t2.perform_get('b'), None)
def test_unlock_rrr(self):
# Should pass after 1.3
lock_table = {}
store = InMemoryKVStore()
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
self.assertEqual(t0.perform_get('a'), 'No such key') # T0 R(a)
self.assertEqual(t1.perform_get('a'), 'No such key') # T1 R(a)
self.assertEqual(t2.perform_get('a'), 'No such key')
self.assertEqual(t3.perform_put('a', '0'), None) # T0 W(a)
self.assertEqual(coordinator.detect_deadlocks(), None)
def test_staff_deadlock_upgrade(self):
lock_table = {}
store = InMemoryKVStore()
t2 = TransactionHandler(lock_table, 2, store)
t4 = TransactionHandler(lock_table, 4, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
# t2, t4 has S lock on a; t2 wants X lock
self.assertEqual(t2.perform_get('a'), 'No such key')
self.assertEqual(t4.perform_get('a'), 'No such key')
self.assertEqual(t2.perform_put('a', 'a1'), None)
# t4 now wants X lock
self.assertEqual(t4.perform_put('a', 'a2'), None)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 2 or abort_id == 4)
def test_staff_deadlock_ww_rw(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
# t0 has X lock on a; t1 wants X lock
self.assertEqual(t0.perform_put('a', 'a0'), 'Success')
self.assertEqual(t1.perform_put('a', 'a1'), None)
# t1 has S lock on b; t0 wants X lock
self.assertEqual(t1.perform_get('b'), 'No such key')
self.assertEqual(t0.perform_put('b', 'b0'), None)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 0 or abort_id == 1)
def test_deadlock_gap_queue(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t2.perform_put('a', 'a0'), 'Success')
self.assertEqual(t0.perform_get('a'), None)
self.assertEqual(t3.perform_put('b', 'b0'), 'Success')
self.assertEqual(t0.perform_get('b'), None)
self.assertEqual(t1.perform_put('b', 'b1'), None)
self.assertEqual(t2.perform_get('b'), None)
abort_id = coordinator.detect_deadlocks()
self.assertEqual(abort_id, None)
def test_multi_deadlock(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
t4 = TransactionHandler(lock_table, 4, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_put('a', 'apple'), 'Success')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.perform_get('a'), 'apple')
self.assertEqual(t2.perform_get('a'), 'apple')
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_put('a', 'banana'), None)
self.assertEqual(t2.perform_put('a', 'pear'), None)
self.assertEqual(t3.perform_put('a', 'cherry'), None)
self.assertEqual(t4.perform_put('a', 'orange'), None)
aid = coordinator.detect_deadlocks()
self.assertTrue(aid == 2 or aid == 1)
self.assertEqual(t2.check_lock(), None)
self.assertEqual(t1.abort(USER), 'User Abort')
self.assertEqual(t2.check_lock(), 'Success')
self.assertEqual(t2.perform_get('a'), 'pear')
self.assertEqual(t3.check_lock(), None)
self.assertEqual(t4.check_lock(), None)
aa = coordinator.detect_deadlocks()
self.assertEqual(aa, None)
def __init__(self, kvstore_class=KVSTORE_CLASS, log_level=logging.WARNING, max_handlers=None):
"""
Initializes the server. Does not start the polling loop. After the
constructor returns, there can be no other servers.
"""
self._logger = logging.getLogger('<%s>' % (self.__class__.__name__))
self._logger.setLevel(log_level)
self._remaining_handlers = max_handlers
self._stats = [0, 0]
self._lock_table = {}
self._next_xid = 0
self._store = kvstore_class()
self._log_level = log_level
self._txn_map = {}
self._coordinator = TransactionCoordinator(self._lock_table)
# Raise an exception if we can connect to an existing server. If a
# context switch occurs in the middle of this code segment, or before
# the socket is bound to the socket file, then this will fail. We
# assume that the probability that the user constructs two servers in
# two different processes at the same time is low, and we ignore this
# edge case. It should not be a problem in any of the test files - most
# of these only create a single instance of the server class.
if os.path.exists(SOCKET_FILE):
test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
test_sock.connect(SOCKET_FILE)
raise KVStoreError('Server seems to be running')
except socket.error as e:
pass
finally:
test_sock.close()
os.unlink(SOCKET_FILE)
try:
asyncore.dispatcher.__init__(self)
# Create socket, but do not connect to anything yet
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind(SOCKET_FILE)
self.listen(5)
except Exception as e:
exc = traceback.format_exc()
self._logger.error('Uncaught exception in __init__, closing server\n%s', exc[:-1])
self.close()
raise e
self._logger.debug('Constructed server')
def test_deadlock_rw_rw(self):
# Should pass after 2
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_get('b'), 'No such key')
self.assertEqual(t0.perform_put('a', 'a0'), 'Success')
self.assertEqual(t0.perform_put('b', 'b0'), 'Success')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_get('a'), 'a0') # T1 R(a)
self.assertEqual(t2.perform_get('b'), 'b0') # T2 R(b)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_put('b', 'b1'), None) # T1 W(b)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t2.perform_put('a', 'a1'), None) # T2 W(a)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 1 or abort_id == 2)
class KVStoreServer(asyncore.dispatcher):
"""
The server listens for incoming connections from clients and spawns a
handler to communicate with each client. The run() method of the server
starts the polling loop.
"""
@classmethod
def get_poll_timeout(cls, poll_timeout, ttl, elapsed_time):
if ttl is None:
return poll_timeout
else:
return min(poll_timeout, ttl - elapsed_time)
def __init__(self, kvstore_class=KVSTORE_CLASS, log_level=logging.WARNING, max_handlers=None):
"""
Initializes the server. Does not start the polling loop. After the
constructor returns, there can be no other servers.
"""
self._logger = logging.getLogger('<%s>' % (self.__class__.__name__))
self._logger.setLevel(log_level)
self._remaining_handlers = max_handlers
self._stats = [0, 0]
self._lock_table = {}
self._next_xid = 0
self._store = kvstore_class()
self._log_level = log_level
self._txn_map = {}
self._coordinator = TransactionCoordinator(self._lock_table)
# Raise an exception if we can connect to an existing server. If a
# context switch occurs in the middle of this code segment, or before
# the socket is bound to the socket file, then this will fail. We
# assume that the probability that the user constructs two servers in
# two different processes at the same time is low, and we ignore this
# edge case. It should not be a problem in any of the test files - most
# of these only create a single instance of the server class.
if os.path.exists(SOCKET_FILE):
test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
test_sock.connect(SOCKET_FILE)
raise KVStoreError('Server seems to be running')
except socket.error as e:
pass
finally:
test_sock.close()
os.unlink(SOCKET_FILE)
try:
asyncore.dispatcher.__init__(self)
# Create socket, but do not connect to anything yet
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind(SOCKET_FILE)
self.listen(5)
except Exception as e:
exc = traceback.format_exc()
self._logger.error('Uncaught exception in __init__, closing server\n%s', exc[:-1])
self.close()
raise e
self._logger.debug('Constructed server')
def remove_transaction(self, xid):
"""
Server handler calls this when it is closed. No method in the server
should have to call this directly.
"""
self._txn_map.pop(xid, None)
def run(self, check_deadlock_fn=None, poll_timeout=1.0, ttl=None):
"""
Runs the polling loop. This does not create any separate processes or
threads, and it only returns after all handlers and the server itself
have been closed. For debugging purposes, we can also specify a time-
to-live (ttl), which specifies the number of seconds that the server
and server handlers are allowed to run before they are forcibly closed.
This is a monkeypatched version of asyncore.loop(). This was the
cleanest way to add the deadlock detector and ttl.
"""
if check_deadlock_fn is None:
check_deadlock_fn = lambda get_count, put_count: True
start_time = time.time()
timed_out = False
while len(asyncore.socket_map) > 0:
elapsed_time = time.time() - start_time
if ttl is not None and elapsed_time > ttl:
timed_out = True
break
new_poll_timeout = self.get_poll_timeout(poll_timeout, ttl, elapsed_time)
# Run select syscall and readable() and writable() on all handlers,
# then run handle_read() and handle_write() on appropriate handlers
asyncore.poll(new_poll_timeout, asyncore.socket_map)
if check_deadlock_fn(self._stats[0], self._stats[1]):
abort_id = self._coordinator.detect_deadlocks()
if abort_id is not None:
self._txn_map[abort_id].deadlock_abort()
for fd, obj in asyncore.socket_map.items():
if obj != self:
obj.close()
self.close()
self._logger.debug('No more open connections')
if timed_out:
raise KVStoreError('Server timed out')
def readable(self):
"""
handle_accept() is called if readable() is True and the select syscall
says the socket is ready to read.
We always want to call handle_accept() if there is data to be read.
"""
return True
def writable(self):
"""
handle_write() is called if writable() is True and the select syscall
says the socket is ready to write.
We never want to call handle_write().
"""
return False
def handle_accept(self):
"""
Accepts a connection from a client, and spawns a new handler to perform
all future communication with that client. The handler is closed
immediately if an exception occurs in the constructor, otherwise it
is added to the polling loop.
"""
pair = self.accept()
if pair is not None:
sock, _ = pair
self._logger.debug('Accepted connection')
xid = self._next_xid
self._next_xid += 1
# After it is initialized, the handler only interacts with the
# server through the variables passed (by reference) into the
# constructor
server_handler = KVStoreServerHandler(sock, self, self._store, self._stats, self._lock_table, xid, self._log_level)
if server_handler.is_open():
# Constructor did not raise an exception
self._txn_map[xid] = server_handler
if self._remaining_handlers is not None:
self._remaining_handlers -= 1
if self._remaining_handlers == 0:
self.handle_close()
def close(self):
asyncore.dispatcher.close(self)
self._logger.debug('Closed server')
def handle_close(self):
"""
Closes the server and removes it from the polling loop. If handlers
have not been closed yet, then run() will not return.
Called when the server has created the maximum number of handlers.
"""
self._logger.debug('Server is not accepting more connections')
self.close()
def handle_error(self):
"""
Closes the server and removes it from the polling loop. If handlers
have not been closed yet, then run() will not return.
Called when one of the handle_*() methods of the class raises an
exception. Prints the stack trace and closes the server.
"""
exc = traceback.format_exc()
self._logger.error('Uncaught exception, closing server\n%s', exc[:-1])
self.close()
class KVStoreServer(asyncore.dispatcher):
"""
The server listens for incoming connections from clients and spawns a
handler to communicate with each client. The run() method of the server
starts the polling loop.
"""
@classmethod
def get_poll_timeout(cls, poll_timeout, ttl, elapsed_time):
if ttl is None:
return poll_timeout
else:
return min(poll_timeout, ttl - elapsed_time)
def __init__(self,
kvstore_class=KVSTORE_CLASS,
log_level=logging.WARNING,
max_handlers=None):
"""
Initializes the server. Does not start the polling loop. After the
constructor returns, there can be no other servers.
"""
self._logger = logging.getLogger('<%s>' % (self.__class__.__name__))
self._logger.setLevel(log_level)
self._remaining_handlers = max_handlers
self._stats = [0, 0]
self._lock_table = {}
self._next_xid = 0
self._store = kvstore_class()
self._log_level = log_level
self._txn_map = {}
self._coordinator = TransactionCoordinator(self._lock_table)
# Raise an exception if we can connect to an existing server. If a
# context switch occurs in the middle of this code segment, or before
# the socket is bound to the socket file, then this will fail. We
# assume that the probability that the user constructs two servers in
# two different processes at the same time is low, and we ignore this
# edge case. It should not be a problem in any of the test files - most
# of these only create a single instance of the server class.
if os.path.exists(SOCKET_FILE):
test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
try:
test_sock.connect(SOCKET_FILE)
raise KVStoreError('Server seems to be running')
except socket.error as e:
pass
finally:
test_sock.close()
os.unlink(SOCKET_FILE)
try:
asyncore.dispatcher.__init__(self)
# Create socket, but do not connect to anything yet
self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM)
self.set_reuse_addr()
self.bind(SOCKET_FILE)
self.listen(5)
except Exception as e:
exc = traceback.format_exc()
self._logger.error(
'Uncaught exception in __init__, closing server\n%s', exc[:-1])
self.close()
raise e
self._logger.debug('Constructed server')
def remove_transaction(self, xid):
"""
Server handler calls this when it is closed. No method in the server
should have to call this directly.
"""
self._txn_map.pop(xid, None)
def run(self, check_deadlock_fn=None, poll_timeout=1.0, ttl=None):
"""
Runs the polling loop. This does not create any separate processes or
threads, and it only returns after all handlers and the server itself
have been closed. For debugging purposes, we can also specify a time-
to-live (ttl), which specifies the number of seconds that the server
and server handlers are allowed to run before they are forcibly closed.
This is a monkeypatched version of asyncore.loop(). This was the
cleanest way to add the deadlock detector and ttl.
"""
if check_deadlock_fn is None:
check_deadlock_fn = lambda get_count, put_count: True
start_time = time.time()
timed_out = False
while len(asyncore.socket_map) > 0:
elapsed_time = time.time() - start_time
if ttl is not None and elapsed_time > ttl:
timed_out = True
break
new_poll_timeout = self.get_poll_timeout(poll_timeout, ttl,
elapsed_time)
# Run select syscall and readable() and writable() on all handlers,
# then run handle_read() and handle_write() on appropriate handlers
asyncore.poll(new_poll_timeout, asyncore.socket_map)
if check_deadlock_fn(self._stats[0], self._stats[1]):
abort_id = self._coordinator.detect_deadlocks()
if abort_id is not None:
self._txn_map[abort_id].deadlock_abort()
for fd, obj in asyncore.socket_map.items():
if obj != self:
obj.close()
self.close()
self._logger.debug('No more open connections')
if timed_out:
raise KVStoreError('Server timed out')
def readable(self):
"""
handle_accept() is called if readable() is True and the select syscall
says the socket is ready to read.
We always want to call handle_accept() if there is data to be read.
"""
return True
def writable(self):
"""
handle_write() is called if writable() is True and the select syscall
says the socket is ready to write.
We never want to call handle_write().
"""
return False
def handle_accept(self):
"""
Accepts a connection from a client, and spawns a new handler to perform
all future communication with that client. The handler is closed
immediately if an exception occurs in the constructor, otherwise it
is added to the polling loop.
"""
pair = self.accept()
if pair is not None:
sock, _ = pair
self._logger.debug('Accepted connection')
xid = self._next_xid
self._next_xid += 1
# After it is initialized, the handler only interacts with the
# server through the variables passed (by reference) into the
# constructor
server_handler = KVStoreServerHandler(sock, self, self._store,
self._stats,
self._lock_table, xid,
self._log_level)
if server_handler.is_open():
# Constructor did not raise an exception
self._txn_map[xid] = server_handler
if self._remaining_handlers is not None:
self._remaining_handlers -= 1
if self._remaining_handlers == 0:
self.handle_close()
def close(self):
asyncore.dispatcher.close(self)
self._logger.debug('Closed server')
def handle_close(self):
"""
Closes the server and removes it from the polling loop. If handlers
have not been closed yet, then run() will not return.
Called when the server has created the maximum number of handlers.
"""
self._logger.debug('Server is not accepting more connections')
self.close()
def handle_error(self):
"""
Closes the server and removes it from the polling loop. If handlers
have not been closed yet, then run() will not return.
Called when one of the handle_*() methods of the class raises an
exception. Prints the stack trace and closes the server.
"""
exc = traceback.format_exc()
self._logger.error('Uncaught exception, closing server\n%s', exc[:-1])
self.close()
