def test_staff_deadlock_two_cycles(self): lock_table = {} store = InMemoryKVStore() t2 = TransactionHandler(lock_table, 2, store) t4 = TransactionHandler(lock_table, 4, store) t6 = TransactionHandler(lock_table, 6, store) t8 = TransactionHandler(lock_table, 8, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) # t2 has S lock on a; t4 wants X lock self.assertEqual(t2.perform_get('a'), 'No such key') self.assertEqual(t4.perform_put('a', 'a1'), None) # t4 has S lock on b; t2 wants X lock self.assertEqual(t4.perform_get('b'), 'No such key') self.assertEqual(t2.perform_put('b', 'b1'), None) # t6 has S lock on a; t8 wants X lock self.assertEqual(t6.perform_get('c'), 'No such key') self.assertEqual(t8.perform_put('c', 'c1'), None) # t8 has S lock on b; t6 wants X lock self.assertEqual(t8.perform_get('d'), 'No such key') self.assertEqual(t6.perform_put('d', 'd1'), None) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 2 or abort_id == 4 or abort_id == 6 or abort_id == 8) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 2 or abort_id == 4 or abort_id == 6 or abort_id == 8)
def test_deadlock_rw_rw(self): # Should pass after 2 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) coordinator = TransactionCoordinator(lock_table) # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_get('a'), 'No such key') # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_get('b'), 'No such key') self.assertEqual(t0.perform_put('a', 'a0'), 'Success') self.assertEqual(t0.perform_put('b', 'b0'), 'Success') # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.commit(), 'Transaction Completed') # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_get('a'), 'a0') # T1 R(a) self.assertEqual(t2.perform_get('b'), 'b0') # T2 R(b) # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_put('b', 'b1'), None) # T1 W(b) # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.check_lock(), None) self.assertEqual(t2.perform_put('a', 'a1'), None) # T2 W(a) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 1 or abort_id == 2)
def __init__(self, kvstore_class=KVSTORE_CLASS, log_level=logging.WARNING, max_handlers=None): """ Initializes the server. Does not start the polling loop. After the constructor returns, there can be no other servers. """ self._logger = logging.getLogger('<%s>' % (self.__class__.__name__)) self._logger.setLevel(log_level) self._remaining_handlers = max_handlers self._stats = [0, 0] self._lock_table = {} self._next_xid = 0 self._store = kvstore_class() self._log_level = log_level self._txn_map = {} self._coordinator = TransactionCoordinator(self._lock_table) # Raise an exception if we can connect to an existing server. If a # context switch occurs in the middle of this code segment, or before # the socket is bound to the socket file, then this will fail. We # assume that the probability that the user constructs two servers in # two different processes at the same time is low, and we ignore this # edge case. It should not be a problem in any of the test files - most # of these only create a single instance of the server class. if os.path.exists(SOCKET_FILE): test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: test_sock.connect(SOCKET_FILE) raise KVStoreError('Server seems to be running') except socket.error as e: pass finally: test_sock.close() os.unlink(SOCKET_FILE) try: asyncore.dispatcher.__init__(self) # Create socket, but do not connect to anything yet self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM) self.set_reuse_addr() self.bind(SOCKET_FILE) self.listen(5) except Exception as e: exc = traceback.format_exc() self._logger.error( 'Uncaught exception in __init__, closing server\n%s', exc[:-1]) self.close() raise e self._logger.debug('Constructed server')
def test_deadlock_identical(self): # Should pass after 2 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t2 = TransactionHandler(lock_table, 2, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_put('a', 'a0'), 'Success') self.assertEqual(t0.perform_put('b', 'b0'), 'Success') self.assertEqual(t2.perform_get('a'), None) self.assertEqual(t2.perform_get('b'), None)
def test_unlock_rrr(self): # Should pass after 1.3 lock_table = {} store = InMemoryKVStore() coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) self.assertEqual(t0.perform_get('a'), 'No such key') # T0 R(a) self.assertEqual(t1.perform_get('a'), 'No such key') # T1 R(a) self.assertEqual(t2.perform_get('a'), 'No such key') self.assertEqual(t3.perform_put('a', '0'), None) # T0 W(a) self.assertEqual(coordinator.detect_deadlocks(), None)
def test_staff_deadlock_upgrade(self): lock_table = {} store = InMemoryKVStore() t2 = TransactionHandler(lock_table, 2, store) t4 = TransactionHandler(lock_table, 4, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) # t2, t4 has S lock on a; t2 wants X lock self.assertEqual(t2.perform_get('a'), 'No such key') self.assertEqual(t4.perform_get('a'), 'No such key') self.assertEqual(t2.perform_put('a', 'a1'), None) # t4 now wants X lock self.assertEqual(t4.perform_put('a', 'a2'), None) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 2 or abort_id == 4)
def test_staff_deadlock_ww_rw(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) # t0 has X lock on a; t1 wants X lock self.assertEqual(t0.perform_put('a', 'a0'), 'Success') self.assertEqual(t1.perform_put('a', 'a1'), None) # t1 has S lock on b; t0 wants X lock self.assertEqual(t1.perform_get('b'), 'No such key') self.assertEqual(t0.perform_put('b', 'b0'), None) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 0 or abort_id == 1)
def test_deadlock_gap_queue(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t2.perform_put('a', 'a0'), 'Success') self.assertEqual(t0.perform_get('a'), None) self.assertEqual(t3.perform_put('b', 'b0'), 'Success') self.assertEqual(t0.perform_get('b'), None) self.assertEqual(t1.perform_put('b', 'b1'), None) self.assertEqual(t2.perform_get('b'), None) abort_id = coordinator.detect_deadlocks() self.assertEqual(abort_id, None)
def test_multi_deadlock(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) t4 = TransactionHandler(lock_table, 4, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_put('a', 'apple'), 'Success') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.perform_get('a'), 'apple') self.assertEqual(t2.perform_get('a'), 'apple') self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_put('a', 'banana'), None) self.assertEqual(t2.perform_put('a', 'pear'), None) self.assertEqual(t3.perform_put('a', 'cherry'), None) self.assertEqual(t4.perform_put('a', 'orange'), None) aid = coordinator.detect_deadlocks() self.assertTrue(aid == 2 or aid == 1) self.assertEqual(t2.check_lock(), None) self.assertEqual(t1.abort(USER), 'User Abort') self.assertEqual(t2.check_lock(), 'Success') self.assertEqual(t2.perform_get('a'), 'pear') self.assertEqual(t3.check_lock(), None) self.assertEqual(t4.check_lock(), None) aa = coordinator.detect_deadlocks() self.assertEqual(aa, None)
def __init__(self, kvstore_class=KVSTORE_CLASS, log_level=logging.WARNING, max_handlers=None): """ Initializes the server. Does not start the polling loop. After the constructor returns, there can be no other servers. """ self._logger = logging.getLogger('<%s>' % (self.__class__.__name__)) self._logger.setLevel(log_level) self._remaining_handlers = max_handlers self._stats = [0, 0] self._lock_table = {} self._next_xid = 0 self._store = kvstore_class() self._log_level = log_level self._txn_map = {} self._coordinator = TransactionCoordinator(self._lock_table) # Raise an exception if we can connect to an existing server. If a # context switch occurs in the middle of this code segment, or before # the socket is bound to the socket file, then this will fail. We # assume that the probability that the user constructs two servers in # two different processes at the same time is low, and we ignore this # edge case. It should not be a problem in any of the test files - most # of these only create a single instance of the server class. if os.path.exists(SOCKET_FILE): test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: test_sock.connect(SOCKET_FILE) raise KVStoreError('Server seems to be running') except socket.error as e: pass finally: test_sock.close() os.unlink(SOCKET_FILE) try: asyncore.dispatcher.__init__(self) # Create socket, but do not connect to anything yet self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM) self.set_reuse_addr() self.bind(SOCKET_FILE) self.listen(5) except Exception as e: exc = traceback.format_exc() self._logger.error('Uncaught exception in __init__, closing server\n%s', exc[:-1]) self.close() raise e self._logger.debug('Constructed server')
def test_deadlock_rw_rw(self): # Should pass after 2 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_get('b'), 'No such key') self.assertEqual(t0.perform_put('a', 'a0'), 'Success') self.assertEqual(t0.perform_put('b', 'b0'), 'Success') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_get('a'), 'a0') # T1 R(a) self.assertEqual(t2.perform_get('b'), 'b0') # T2 R(b) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_put('b', 'b1'), None) # T1 W(b) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.check_lock(), None) self.assertEqual(t2.perform_put('a', 'a1'), None) # T2 W(a) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 1 or abort_id == 2)
class KVStoreServer(asyncore.dispatcher): """ The server listens for incoming connections from clients and spawns a handler to communicate with each client. The run() method of the server starts the polling loop. """ @classmethod def get_poll_timeout(cls, poll_timeout, ttl, elapsed_time): if ttl is None: return poll_timeout else: return min(poll_timeout, ttl - elapsed_time) def __init__(self, kvstore_class=KVSTORE_CLASS, log_level=logging.WARNING, max_handlers=None): """ Initializes the server. Does not start the polling loop. After the constructor returns, there can be no other servers. """ self._logger = logging.getLogger('<%s>' % (self.__class__.__name__)) self._logger.setLevel(log_level) self._remaining_handlers = max_handlers self._stats = [0, 0] self._lock_table = {} self._next_xid = 0 self._store = kvstore_class() self._log_level = log_level self._txn_map = {} self._coordinator = TransactionCoordinator(self._lock_table) # Raise an exception if we can connect to an existing server. If a # context switch occurs in the middle of this code segment, or before # the socket is bound to the socket file, then this will fail. We # assume that the probability that the user constructs two servers in # two different processes at the same time is low, and we ignore this # edge case. It should not be a problem in any of the test files - most # of these only create a single instance of the server class. if os.path.exists(SOCKET_FILE): test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: test_sock.connect(SOCKET_FILE) raise KVStoreError('Server seems to be running') except socket.error as e: pass finally: test_sock.close() os.unlink(SOCKET_FILE) try: asyncore.dispatcher.__init__(self) # Create socket, but do not connect to anything yet self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM) self.set_reuse_addr() self.bind(SOCKET_FILE) self.listen(5) except Exception as e: exc = traceback.format_exc() self._logger.error('Uncaught exception in __init__, closing server\n%s', exc[:-1]) self.close() raise e self._logger.debug('Constructed server') def remove_transaction(self, xid): """ Server handler calls this when it is closed. No method in the server should have to call this directly. """ self._txn_map.pop(xid, None) def run(self, check_deadlock_fn=None, poll_timeout=1.0, ttl=None): """ Runs the polling loop. This does not create any separate processes or threads, and it only returns after all handlers and the server itself have been closed. For debugging purposes, we can also specify a time- to-live (ttl), which specifies the number of seconds that the server and server handlers are allowed to run before they are forcibly closed. This is a monkeypatched version of asyncore.loop(). This was the cleanest way to add the deadlock detector and ttl. """ if check_deadlock_fn is None: check_deadlock_fn = lambda get_count, put_count: True start_time = time.time() timed_out = False while len(asyncore.socket_map) > 0: elapsed_time = time.time() - start_time if ttl is not None and elapsed_time > ttl: timed_out = True break new_poll_timeout = self.get_poll_timeout(poll_timeout, ttl, elapsed_time) # Run select syscall and readable() and writable() on all handlers, # then run handle_read() and handle_write() on appropriate handlers asyncore.poll(new_poll_timeout, asyncore.socket_map) if check_deadlock_fn(self._stats[0], self._stats[1]): abort_id = self._coordinator.detect_deadlocks() if abort_id is not None: self._txn_map[abort_id].deadlock_abort() for fd, obj in asyncore.socket_map.items(): if obj != self: obj.close() self.close() self._logger.debug('No more open connections') if timed_out: raise KVStoreError('Server timed out') def readable(self): """ handle_accept() is called if readable() is True and the select syscall says the socket is ready to read. We always want to call handle_accept() if there is data to be read. """ return True def writable(self): """ handle_write() is called if writable() is True and the select syscall says the socket is ready to write. We never want to call handle_write(). """ return False def handle_accept(self): """ Accepts a connection from a client, and spawns a new handler to perform all future communication with that client. The handler is closed immediately if an exception occurs in the constructor, otherwise it is added to the polling loop. """ pair = self.accept() if pair is not None: sock, _ = pair self._logger.debug('Accepted connection') xid = self._next_xid self._next_xid += 1 # After it is initialized, the handler only interacts with the # server through the variables passed (by reference) into the # constructor server_handler = KVStoreServerHandler(sock, self, self._store, self._stats, self._lock_table, xid, self._log_level) if server_handler.is_open(): # Constructor did not raise an exception self._txn_map[xid] = server_handler if self._remaining_handlers is not None: self._remaining_handlers -= 1 if self._remaining_handlers == 0: self.handle_close() def close(self): asyncore.dispatcher.close(self) self._logger.debug('Closed server') def handle_close(self): """ Closes the server and removes it from the polling loop. If handlers have not been closed yet, then run() will not return. Called when the server has created the maximum number of handlers. """ self._logger.debug('Server is not accepting more connections') self.close() def handle_error(self): """ Closes the server and removes it from the polling loop. If handlers have not been closed yet, then run() will not return. Called when one of the handle_*() methods of the class raises an exception. Prints the stack trace and closes the server. """ exc = traceback.format_exc() self._logger.error('Uncaught exception, closing server\n%s', exc[:-1]) self.close()
class KVStoreServer(asyncore.dispatcher): """ The server listens for incoming connections from clients and spawns a handler to communicate with each client. The run() method of the server starts the polling loop. """ @classmethod def get_poll_timeout(cls, poll_timeout, ttl, elapsed_time): if ttl is None: return poll_timeout else: return min(poll_timeout, ttl - elapsed_time) def __init__(self, kvstore_class=KVSTORE_CLASS, log_level=logging.WARNING, max_handlers=None): """ Initializes the server. Does not start the polling loop. After the constructor returns, there can be no other servers. """ self._logger = logging.getLogger('<%s>' % (self.__class__.__name__)) self._logger.setLevel(log_level) self._remaining_handlers = max_handlers self._stats = [0, 0] self._lock_table = {} self._next_xid = 0 self._store = kvstore_class() self._log_level = log_level self._txn_map = {} self._coordinator = TransactionCoordinator(self._lock_table) # Raise an exception if we can connect to an existing server. If a # context switch occurs in the middle of this code segment, or before # the socket is bound to the socket file, then this will fail. We # assume that the probability that the user constructs two servers in # two different processes at the same time is low, and we ignore this # edge case. It should not be a problem in any of the test files - most # of these only create a single instance of the server class. if os.path.exists(SOCKET_FILE): test_sock = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) try: test_sock.connect(SOCKET_FILE) raise KVStoreError('Server seems to be running') except socket.error as e: pass finally: test_sock.close() os.unlink(SOCKET_FILE) try: asyncore.dispatcher.__init__(self) # Create socket, but do not connect to anything yet self.create_socket(socket.AF_UNIX, socket.SOCK_STREAM) self.set_reuse_addr() self.bind(SOCKET_FILE) self.listen(5) except Exception as e: exc = traceback.format_exc() self._logger.error( 'Uncaught exception in __init__, closing server\n%s', exc[:-1]) self.close() raise e self._logger.debug('Constructed server') def remove_transaction(self, xid): """ Server handler calls this when it is closed. No method in the server should have to call this directly. """ self._txn_map.pop(xid, None) def run(self, check_deadlock_fn=None, poll_timeout=1.0, ttl=None): """ Runs the polling loop. This does not create any separate processes or threads, and it only returns after all handlers and the server itself have been closed. For debugging purposes, we can also specify a time- to-live (ttl), which specifies the number of seconds that the server and server handlers are allowed to run before they are forcibly closed. This is a monkeypatched version of asyncore.loop(). This was the cleanest way to add the deadlock detector and ttl. """ if check_deadlock_fn is None: check_deadlock_fn = lambda get_count, put_count: True start_time = time.time() timed_out = False while len(asyncore.socket_map) > 0: elapsed_time = time.time() - start_time if ttl is not None and elapsed_time > ttl: timed_out = True break new_poll_timeout = self.get_poll_timeout(poll_timeout, ttl, elapsed_time) # Run select syscall and readable() and writable() on all handlers, # then run handle_read() and handle_write() on appropriate handlers asyncore.poll(new_poll_timeout, asyncore.socket_map) if check_deadlock_fn(self._stats[0], self._stats[1]): abort_id = self._coordinator.detect_deadlocks() if abort_id is not None: self._txn_map[abort_id].deadlock_abort() for fd, obj in asyncore.socket_map.items(): if obj != self: obj.close() self.close() self._logger.debug('No more open connections') if timed_out: raise KVStoreError('Server timed out') def readable(self): """ handle_accept() is called if readable() is True and the select syscall says the socket is ready to read. We always want to call handle_accept() if there is data to be read. """ return True def writable(self): """ handle_write() is called if writable() is True and the select syscall says the socket is ready to write. We never want to call handle_write(). """ return False def handle_accept(self): """ Accepts a connection from a client, and spawns a new handler to perform all future communication with that client. The handler is closed immediately if an exception occurs in the constructor, otherwise it is added to the polling loop. """ pair = self.accept() if pair is not None: sock, _ = pair self._logger.debug('Accepted connection') xid = self._next_xid self._next_xid += 1 # After it is initialized, the handler only interacts with the # server through the variables passed (by reference) into the # constructor server_handler = KVStoreServerHandler(sock, self, self._store, self._stats, self._lock_table, xid, self._log_level) if server_handler.is_open(): # Constructor did not raise an exception self._txn_map[xid] = server_handler if self._remaining_handlers is not None: self._remaining_handlers -= 1 if self._remaining_handlers == 0: self.handle_close() def close(self): asyncore.dispatcher.close(self) self._logger.debug('Closed server') def handle_close(self): """ Closes the server and removes it from the polling loop. If handlers have not been closed yet, then run() will not return. Called when the server has created the maximum number of handlers. """ self._logger.debug('Server is not accepting more connections') self.close() def handle_error(self): """ Closes the server and removes it from the polling loop. If handlers have not been closed yet, then run() will not return. Called when one of the handle_*() methods of the class raises an exception. Prints the stack trace and closes the server. """ exc = traceback.format_exc() self._logger.error('Uncaught exception, closing server\n%s', exc[:-1]) self.close()