def test_staff_read_x_lock(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_put('a', '1'), 'Success') self.assertEqual(t0.perform_get('a'), '1') self.assertEqual(t0.perform_put('a', '3'), 'Success')
def test_staff_multiple_write_abort(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_put('a', '1'), 'Success') self.assertEqual(t0.perform_put('a', '2'), 'Success') self.assertEqual(t0.perform_put('a', '3'), 'Success') self.assertEqual(t0.abort(USER), 'User Abort') self.assertEqual(store.get('a'), None)
def test_staff_multiple_write_commit(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_put('a', '1'), 'Success') self.assertEqual(t0.perform_put('a', '2'), 'Success') self.assertEqual(t0.perform_put('a', '3'), 'Success') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(store.get('a'), '3')
def test_abort(self): # Sanity check lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t0.abort(USER), 'User Abort') self.assertEqual(store.get('a'), None)
def test_commit(self): # Sanity check lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(store.get('a'), '0')
def test_rwr(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t1.perform_put('a', '1'), None) self.assertEqual(t2.perform_get('a'), None)
def test_abort_queue(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t1.perform_get('a'), None) self.assertEqual(t1.perform_put('b', '1'), 'Success') self.assertEqual(t1.abort(USER), 'User Abort') self.assertEqual(store.get('a'), '0') self.assertEqual(store.get('b'), None)
def test_upgrade_in_put(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) self.assertEqual(t0.perform_get('apple'), 'No such key') self.assertEqual(t1.perform_put('apple', '333'), None) self.assertEqual(t0.perform_put('apple', 'aaa'), 'Success') self.assertEqual(t0.perform_get('apple'), 'aaa')
def test_abort_commit(self): # Should pass after 1.2 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t0.abort(USER), 'User Abort') self.assertEqual(t1.perform_get('a'), 'No such key') self.assertEqual(t1.perform_put('a', '1'), 'Success') self.assertEqual(t1.perform_get('a'), '1') self.assertEqual(t1.commit(), 'Transaction Completed') self.assertEqual(store.get('a'), '1')
def test_wr(self): # Should pass after 1.1 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t1.perform_get('a'), None) self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(store.get('a'), '0')
def __init__(self, sock, server, store, stats, lock_table, xid, log_level): """ Initializes the handler and adds it to the polling loop. If an exception is raised during the constructor, then the handler is closed and removed from the polling loop. """ self._logger = logging.getLogger('<%s %s>' % (self.__class__.__name__, xid)) self._logger.setLevel(log_level) self._server = server self._store = store self._stats = stats # Global lock table. Each key maps to a list of four lists. # The first list contains txns holding the key's s lock. The second list contains txns holding the key's x lock (list length should be either 0 or 1) # The third list contains txns waiting to acquire the s lock. The fourth list contains txns waiting to acquire the x lock self._lock_table = lock_table self._xid = xid # A string storing data passed from handle_read() to handle_write(). self._data = str(self._xid) # One of the state constants listed above self._state = RESPONDING # self.connected is inherited self._txn_handler = TransactionHandler(self._lock_table, self._xid, self._store) try: asyncore.dispatcher.__init__(self, sock) except: # Exceptions from constructor are not caught by handle_error(), so # we catch them in the constructor itself exc = traceback.format_exc() self._logger.error('Uncaught exception in __init__, cannot construct server handler\n%s', exc[:-1]) self.close() return self._logger.debug('Constructed server handler')
def test_deadlock_naive(self): # Should pass after 2 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t2 = TransactionHandler(lock_table, 2, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_put('a', 'a0'), 'Success') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t0.perform_get('a'), 'a0') self.assertEqual(t2.perform_get('a'), 'a0') self.assertEqual(t0.perform_put('a', 'ab'), None) self.assertEqual(coordinator.detect_deadlocks(), None)
def test_multiple_read(self): # Sanity check lock_table = {} store = InMemoryKVStore() store.put('a', '0') t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t1.perform_get('a'), '0') self.assertEqual(t2.perform_get('a'), '0') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.abort(USER), 'User Abort') self.assertEqual(t2.perform_get('a'), '0')
def test_unlock_ww(self): # Should pass after 1.3 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') # T0 W(a) self.assertEqual(t1.perform_put('a', '1'), None) # T1 W(a) self.assertEqual(t1.check_lock(), None) self.assertEqual(t1.check_lock(), None) self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.check_lock(), 'Success') self.assertEqual(t1.perform_get('a'), '1')
def __init__(self, sock, server, store, stats, lock_table, xid, log_level): """ Initializes the handler and adds it to the polling loop. If an exception is raised during the constructor, then the handler is closed and removed from the polling loop. """ self._logger = logging.getLogger('<%s %s>' % (self.__class__.__name__, xid)) self._logger.setLevel(log_level) self._server = server self._store = store self._stats = stats # Global lock table. Each key maps to a list of four lists. # The first list contains txns holding the key's s lock. The second list contains txns holding the key's x lock (list length should be either 0 or 1) # The third list contains txns waiting to acquire the s lock. The fourth list contains txns waiting to acquire the x lock self._lock_table = lock_table self._xid = xid # A string storing data passed from handle_read() to handle_write(). self._data = str(self._xid) # One of the state constants listed above self._state = RESPONDING # self.connected is inherited self._txn_handler = TransactionHandler(self._lock_table, self._xid, self._store) try: asyncore.dispatcher.__init__(self, sock) except: # Exceptions from constructor are not caught by handle_error(), so # we catch them in the constructor itself exc = traceback.format_exc() self._logger.error( 'Uncaught exception in __init__, cannot construct server handler\n%s', exc[:-1]) self.close() return self._logger.debug('Constructed server handler')
def test_staff_deadlock_upgrade(self): lock_table = {} store = InMemoryKVStore() t2 = TransactionHandler(lock_table, 2, store) t4 = TransactionHandler(lock_table, 4, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) # t2, t4 has S lock on a; t2 wants X lock self.assertEqual(t2.perform_get('a'), 'No such key') self.assertEqual(t4.perform_get('a'), 'No such key') self.assertEqual(t2.perform_put('a', 'a1'), None) # t4 now wants X lock self.assertEqual(t4.perform_put('a', 'a2'), None) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 2 or abort_id == 4)
def test_staff_deadlock_ww_rw(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) # t0 has X lock on a; t1 wants X lock self.assertEqual(t0.perform_put('a', 'a0'), 'Success') self.assertEqual(t1.perform_put('a', 'a1'), None) # t1 has S lock on b; t0 wants X lock self.assertEqual(t1.perform_get('b'), 'No such key') self.assertEqual(t0.perform_put('b', 'b0'), None) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 0 or abort_id == 1)
def test_commit(self): # Sanity check lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(store.get('a'), '0')
def test_abort(self): # Sanity check lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t0.abort(USER), 'User Abort') self.assertEqual(store.get('a'), None)
def test_rw(self): # Should pass after 1.1 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t1.perform_get('a'), 'No such key') self.assertEqual(t1.perform_put('a', '0'), None) self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(store.get('a'), None)
def test_single_abort(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) self.assertEqual(t0.perform_get('apple'), 'No such key') self.assertEqual(t1.perform_get('apple'), 'No such key') self.assertEqual(t2.perform_get('apple'), 'No such key') self.assertEqual(t0.perform_put('apple', '11'), None) self.assertEqual(t0.check_lock(), None) self.assertEqual(t1.commit(), 'Transaction Completed') self.assertEqual(t0.check_lock(), None) self.assertEqual(t2.commit(), 'Transaction Completed') self.assertEqual(t0.check_lock(), "Success") self.assertEqual(t0.perform_get('apple'), '11') self.assertEqual(t0.perform_get('banana'), 'No such key') self.assertEqual(t0.perform_put("banana", '22'), "Success") self.assertEqual(t1.perform_get('apple'), None) self.assertEqual(t0.abort(USER), 'User Abort') self.assertEqual(t1.check_lock(), 'No such key')
def test_multiple_read(self): # Sanity check lock_table = {} store = InMemoryKVStore() store.put('a', '0') t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t1.perform_get('a'), '0') self.assertEqual(t2.perform_get('a'), '0') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.abort(USER), 'User Abort') self.assertEqual(t2.perform_get('a'), '0')
def test_upgrade_s_x_multi(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) self.assertEqual(t0.perform_put('noob', '555'), 'Success') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t0.perform_get('noob'), '555') self.assertEqual(t1.perform_get('noob'), '555') self.assertEqual(t2.perform_get('noob'), '555') self.assertEqual(t1.perform_put('noob', '111'), None) self.assertEqual(t1.check_lock(), None) self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.check_lock(), None) self.assertEqual(t2.commit(), 'Transaction Completed') self.assertEqual(t1.check_lock(), 'Success') self.assertEqual(t1.perform_get('noob'), '111')
def test_commit_abort_commit(self): # Should pass after 1.2 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.perform_get('a'), '0') self.assertEqual(t1.perform_put('a', '1'), 'Success') self.assertEqual(t1.perform_get('a'), '1') self.assertEqual(t1.abort(USER), 'User Abort') self.assertEqual(t2.perform_get('a'), '0') self.assertEqual(t2.perform_put('a', '2'), 'Success') self.assertEqual(t2.perform_get('a'), '2') self.assertEqual(t2.commit(), 'Transaction Completed') self.assertEqual(store.get('a'), '2')
def test_unlock_rrr(self): # Should pass after 1.3 lock_table = {} store = InMemoryKVStore() coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) self.assertEqual(t0.perform_get('a'), 'No such key') # T0 R(a) self.assertEqual(t1.perform_get('a'), 'No such key') # T1 R(a) self.assertEqual(t2.perform_get('a'), 'No such key') self.assertEqual(t3.perform_put('a', '0'), None) # T0 W(a) self.assertEqual(coordinator.detect_deadlocks(), None)
def test_staff_lock_upgrade(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t1.perform_get('a'), 'No such key') self.assertEqual(t2.perform_put('a', '1'), None) self.assertEqual(t0.perform_put('a', '1'), None) self.assertEqual(t1.commit(), 'Transaction Completed') self.assertEqual(t0.check_lock(), 'Success') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t2.check_lock(), 'Success')
def test_staff_fifo_queue(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) t4 = TransactionHandler(lock_table, 4, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') # T0 W(a) self.assertEqual(t1.perform_get('a'), None) # T1 R(a) self.assertEqual(t2.perform_put('a', '2'), None) self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.check_lock(), '0') self.assertEqual(t1.perform_get('a'), '0') self.assertEqual(t3.perform_get('a'), None) self.assertEqual(t4.perform_get('a'), None) self.assertEqual(t1.commit(), 'Transaction Completed') self.assertEqual(t2.check_lock(), 'Success') self.assertEqual(t2.commit(), 'Transaction Completed') self.assertEqual(t3.check_lock(), '2') self.assertEqual(t4.check_lock(), '2')
def test_staff_deadlock_two_cycles(self): lock_table = {} store = InMemoryKVStore() t2 = TransactionHandler(lock_table, 2, store) t4 = TransactionHandler(lock_table, 4, store) t6 = TransactionHandler(lock_table, 6, store) t8 = TransactionHandler(lock_table, 8, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) # t2 has S lock on a; t4 wants X lock self.assertEqual(t2.perform_get('a'), 'No such key') self.assertEqual(t4.perform_put('a', 'a1'), None) # t4 has S lock on b; t2 wants X lock self.assertEqual(t4.perform_get('b'), 'No such key') self.assertEqual(t2.perform_put('b', 'b1'), None) # t6 has S lock on a; t8 wants X lock self.assertEqual(t6.perform_get('c'), 'No such key') self.assertEqual(t8.perform_put('c', 'c1'), None) # t8 has S lock on b; t6 wants X lock self.assertEqual(t8.perform_get('d'), 'No such key') self.assertEqual(t6.perform_put('d', 'd1'), None) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 2 or abort_id == 4 or abort_id == 6 or abort_id == 8) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 2 or abort_id == 4 or abort_id == 6 or abort_id == 8)
class KVStoreServerHandler(asyncore.dispatcher): """ Each handler communicates with a single client. If the handler raises an exception, during the constructor or from a handle_*() method, then it is automatically closed. The handle_*() methods of all handlers (that have not yet been closed) are called by the polling loop whenever appropriate. """ def __init__(self, sock, server, store, stats, lock_table, xid, log_level): """ Initializes the handler and adds it to the polling loop. If an exception is raised during the constructor, then the handler is closed and removed from the polling loop. """ self._logger = logging.getLogger('<%s %s>' % (self.__class__.__name__, xid)) self._logger.setLevel(log_level) self._server = server self._store = store self._stats = stats # Global lock table. Each key maps to a list of four lists. # The first list contains txns holding the key's s lock. The second list contains txns holding the key's x lock (list length should be either 0 or 1) # The third list contains txns waiting to acquire the s lock. The fourth list contains txns waiting to acquire the x lock self._lock_table = lock_table self._xid = xid # A string storing data passed from handle_read() to handle_write(). self._data = str(self._xid) # One of the state constants listed above self._state = RESPONDING # self.connected is inherited self._txn_handler = TransactionHandler(self._lock_table, self._xid, self._store) try: asyncore.dispatcher.__init__(self, sock) except: # Exceptions from constructor are not caught by handle_error(), so # we catch them in the constructor itself exc = traceback.format_exc() self._logger.error( 'Uncaught exception in __init__, cannot construct server handler\n%s', exc[:-1]) self.close() return self._logger.debug('Constructed server handler') def reliable_send(self, msg): """ asyncore.dispatcher does not have sendall, so we implement it here. We stop trying to send if the handler is ever closed. """ if not self.connected: return sent = self.send(msg) while sent < len(msg) and self.connected: sent += self.send(msg[sent:]) def end_transaction(self): """ Marks the end of a transaction, after a COMMIT or ABORT. Notifies the user that the COMMIT or ABORT completed. """ self.close() def readable(self): """ handle_read() is called if readable() is True and the select syscall says the socket is ready to read. We always want to call handle_read() if there is data to be read. """ return True def writable(self): """ handle_write() is called if writable() is True and the select syscall says the socket is ready to write. We avoid calling handle_write() when there is no data to be written. """ return self._state != WAITING def handle_read(self): """ Called by the polling loop if readable() is True and the socket is ready to read. We read the data by calling recv(). recv() returns '' if the connection was closed, so we check for this explicitly. If the connection is still open, handle_read() must set self._data to a non-None value. """ # TODO: implement reliable recv. Create input buffer, in case the input takes too long to arrive or arrives in multiple chunks, and delimiter to mark end of stream. data = self.recv(CHUNK_SIZE) if data == '': # The connection was closed (and handle_close() was already called) return if data[0] == ' ': self._state, self._data = RESPONDING, 'Command begins with whitespace' return if re.search(r'[^A-Za-z0-9_ ]', data): self._state, self._data = RESPONDING, 'Special characters in message' return tokens = data.split(' ', 3) if tokens[0] == 'GET': if len(tokens) == 2: key = tokens[1] if key == '': self._state, self._data = RESPONDING, 'Bad format for GET' else: result = self._txn_handler.perform_get(key) self._stats[0] += 1 if result is None: self._state, self._data = LOCKING, None elif isinstance(result, str): self._state, self._data = RESPONDING, result else: raise KVStoreError( 'T%s.perform_get() returned %r, which is not a string or None' % (self._xid, result)) else: self._state, self._data = RESPONDING, 'Bad format for GET' elif tokens[0] == 'PUT': if len(tokens) == 3: key, value = tokens[1], tokens[2] if key == '' or value == '': self._state, self._data = RESPONDING, 'Bad format for PUT' else: result = self._txn_handler.perform_put(key, value) self._stats[1] += 1 if result is None: self._state, self._data = LOCKING, None elif isinstance(result, str): self._state, self._data = RESPONDING, result else: raise KVStoreError( 'T%s.perform_put() returned %r, which is not a string or None' % (self._xid, result)) else: self._state, self._data = RESPONDING, 'Bad format for PUT' elif tokens[0] == 'COMMIT': if len(tokens) == 1: result = self._txn_handler.commit() if not isinstance(result, str): raise KVStoreError( 'T%s.commit() returned %r, which is not a string' % (self._xid, result)) self._state, self._data = COMMITTING, result else: self._state, self._data = RESPONDING, 'Bad format for COMMIT' elif tokens[0] == 'ABORT': if len(tokens) == 1: result = self._txn_handler.abort(USER) if not isinstance(result, str): raise KVStoreError( 'T%s.abort() returned %r, which is not a string' % (self._xid, result)) self._state, self._data = ABORTING, result else: self._state, self._data = RESPONDING, 'Bad format for ABORT' else: self._state, self._data = RESPONDING, 'Unrecognized command' self._logger.debug('Lock table is %r', self._lock_table) def handle_write(self): """ Called by the polling loop if writable() is True and the socket is ready to write. We write the data by calling send(). """ if self._state != LOCKING: self.reliable_send(self._data) if self._state == ABORTING or self._state == COMMITTING: self.end_transaction() self._data = None self._state = WAITING else: # Try to acquire the necessary locks result = self._txn_handler.check_lock() if result is not None: if not isinstance(result, str): raise KVStoreError( 'T%s.check_lock() returned %r, which is not a string or None' % (self._xid, result)) self._state, self._data = RESPONDING, result def deadlock_abort(self): result = self._txn_handler.abort(DEADLOCK) if not isinstance(result, str): raise KVStoreError( 'T%s.abort() returned %r, which is not a string' % (self._xid, result)) self._state, self._data = ABORTING, result def is_open(self): return self.connected def close(self): asyncore.dispatcher.close(self) self._server.remove_transaction(self._xid) self._logger.debug('Closed server handler') def handle_close(self): """ Closes the handler and removes it from the polling loop. Called when the client has closed the connection. Because the polling loop uses the select syscall, handle_close() is called immediately after the connection is closed. """ self._logger.debug('Client disconnected, closing server handler') self.close() def handle_error(self): """ Closes the handler and removes it from the polling loop. Called when one of the handle_*() methods of the class raises an exception. Prints the stack trace and closes the handler. """ exc = traceback.format_exc() self._logger.error('Uncaught exception, closing server handler\n%s', exc[:-1]) self.close()
def test_staff_block_read_write_read(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) self.assertEqual(t0.perform_put('a', '0'), 'Success') self.assertEqual(t1.perform_get('a'), None) self.assertEqual(t2.perform_put('a', '2'), None) self.assertEqual(t3.perform_get('a'), None) self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.check_lock(), '0') self.assertEqual(t2.check_lock(), None) self.assertEqual(t3.check_lock(), None) self.assertEqual(t1.commit(), 'Transaction Completed') self.assertEqual(t2.check_lock(), 'Success') self.assertEqual(t3.check_lock(), None) self.assertEqual(t2.commit(), 'Transaction Completed') self.assertEqual(t3.check_lock(), '2')
class KVStoreServerHandler(asyncore.dispatcher): """ Each handler communicates with a single client. If the handler raises an exception, during the constructor or from a handle_*() method, then it is automatically closed. The handle_*() methods of all handlers (that have not yet been closed) are called by the polling loop whenever appropriate. """ def __init__(self, sock, server, store, stats, lock_table, xid, log_level): """ Initializes the handler and adds it to the polling loop. If an exception is raised during the constructor, then the handler is closed and removed from the polling loop. """ self._logger = logging.getLogger('<%s %s>' % (self.__class__.__name__, xid)) self._logger.setLevel(log_level) self._server = server self._store = store self._stats = stats # Global lock table. Each key maps to a list of four lists. # The first list contains txns holding the key's s lock. The second list contains txns holding the key's x lock (list length should be either 0 or 1) # The third list contains txns waiting to acquire the s lock. The fourth list contains txns waiting to acquire the x lock self._lock_table = lock_table self._xid = xid # A string storing data passed from handle_read() to handle_write(). self._data = str(self._xid) # One of the state constants listed above self._state = RESPONDING # self.connected is inherited self._txn_handler = TransactionHandler(self._lock_table, self._xid, self._store) try: asyncore.dispatcher.__init__(self, sock) except: # Exceptions from constructor are not caught by handle_error(), so # we catch them in the constructor itself exc = traceback.format_exc() self._logger.error('Uncaught exception in __init__, cannot construct server handler\n%s', exc[:-1]) self.close() return self._logger.debug('Constructed server handler') def reliable_send(self, msg): """ asyncore.dispatcher does not have sendall, so we implement it here. We stop trying to send if the handler is ever closed. """ if not self.connected: return sent = self.send(msg) while sent < len(msg) and self.connected: sent += self.send(msg[sent:]) def end_transaction(self): """ Marks the end of a transaction, after a COMMIT or ABORT. Notifies the user that the COMMIT or ABORT completed. """ self.close() def readable(self): """ handle_read() is called if readable() is True and the select syscall says the socket is ready to read. We always want to call handle_read() if there is data to be read. """ return True def writable(self): """ handle_write() is called if writable() is True and the select syscall says the socket is ready to write. We avoid calling handle_write() when there is no data to be written. """ return self._state != WAITING def handle_read(self): """ Called by the polling loop if readable() is True and the socket is ready to read. We read the data by calling recv(). recv() returns '' if the connection was closed, so we check for this explicitly. If the connection is still open, handle_read() must set self._data to a non-None value. """ # TODO: implement reliable recv. Create input buffer, in case the input takes too long to arrive or arrives in multiple chunks, and delimiter to mark end of stream. data = self.recv(CHUNK_SIZE) if data == '': # The connection was closed (and handle_close() was already called) return if data[0] == ' ': self._state, self._data = RESPONDING, 'Command begins with whitespace' return if re.search(r'[^A-Za-z0-9_ ]', data): self._state, self._data = RESPONDING, 'Special characters in message' return tokens = data.split(' ', 3) if tokens[0] == 'GET': if len(tokens) == 2: key = tokens[1] if key == '': self._state, self._data = RESPONDING, 'Bad format for GET' else: result = self._txn_handler.perform_get(key) self._stats[0] += 1 if result is None: self._state, self._data = LOCKING, None elif isinstance(result, str): self._state, self._data = RESPONDING, result else: raise KVStoreError('T%s.perform_get() returned %r, which is not a string or None' % (self._xid, result)) else: self._state, self._data = RESPONDING, 'Bad format for GET' elif tokens[0] == 'PUT': if len(tokens) == 3: key, value = tokens[1], tokens[2] if key == '' or value == '': self._state, self._data = RESPONDING, 'Bad format for PUT' else: result = self._txn_handler.perform_put(key, value) self._stats[1] += 1 if result is None: self._state, self._data = LOCKING, None elif isinstance(result, str): self._state, self._data = RESPONDING, result else: raise KVStoreError('T%s.perform_put() returned %r, which is not a string or None' % (self._xid, result)) else: self._state, self._data = RESPONDING, 'Bad format for PUT' elif tokens[0] == 'COMMIT': if len(tokens) == 1: result = self._txn_handler.commit() if not isinstance(result, str): raise KVStoreError('T%s.commit() returned %r, which is not a string' % (self._xid, result)) self._state, self._data = COMMITTING, result else: self._state, self._data = RESPONDING, 'Bad format for COMMIT' elif tokens[0] == 'ABORT': if len(tokens) == 1: result = self._txn_handler.abort(USER) if not isinstance(result, str): raise KVStoreError('T%s.abort() returned %r, which is not a string' % (self._xid, result)) self._state, self._data = ABORTING, result else: self._state, self._data = RESPONDING, 'Bad format for ABORT' else: self._state, self._data = RESPONDING, 'Unrecognized command' self._logger.debug('Lock table is %r', self._lock_table) def handle_write(self): """ Called by the polling loop if writable() is True and the socket is ready to write. We write the data by calling send(). """ if self._state != LOCKING: self.reliable_send(self._data) if self._state == ABORTING or self._state == COMMITTING: self.end_transaction() self._data = None self._state = WAITING else: # Try to acquire the necessary locks result = self._txn_handler.check_lock() if result is not None: if not isinstance(result, str): raise KVStoreError('T%s.check_lock() returned %r, which is not a string or None' % (self._xid, result)) self._state, self._data = RESPONDING, result def deadlock_abort(self): result = self._txn_handler.abort(DEADLOCK) if not isinstance(result, str): raise KVStoreError('T%s.abort() returned %r, which is not a string' % (self._xid, result)) self._state, self._data = ABORTING, result def is_open(self): return self.connected def close(self): asyncore.dispatcher.close(self) self._server.remove_transaction(self._xid) self._logger.debug('Closed server handler') def handle_close(self): """ Closes the handler and removes it from the polling loop. Called when the client has closed the connection. Because the polling loop uses the select syscall, handle_close() is called immediately after the connection is closed. """ self._logger.debug('Client disconnected, closing server handler') self.close() def handle_error(self): """ Closes the handler and removes it from the polling loop. Called when one of the handle_*() methods of the class raises an exception. Prints the stack trace and closes the handler. """ exc = traceback.format_exc() self._logger.error('Uncaught exception, closing server handler\n%s', exc[:-1]) self.close()
def test_1_thread_on_multiple_keys(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) self.assertEqual(t0.perform_get('apple'), 'No such key') self.assertEqual(t1.perform_get('apple'), 'No such key') self.assertEqual(t2.perform_get('apple'), 'No such key') self.assertEqual(t0.perform_put('apple', '11'), None) self.assertEqual(t0.check_lock(), None) self.assertEqual(t1.commit(), 'Transaction Completed') self.assertEqual(t0.check_lock(), None) self.assertEqual(t2.commit(), 'Transaction Completed') self.assertEqual(t0.check_lock(), "Success") self.assertEqual(t0.perform_get('apple'), '11') self.assertEqual(t0.perform_put('apple', "10000"), 'Success') self.assertEqual(t0.perform_get('apple'), '10000') self.assertEqual(t0.perform_put("banana", '22'), "Success") self.assertEqual(t0.perform_get('banana'), '22') self.assertEqual(t1.perform_get('apple'), None) self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.check_lock(), '10000') self.assertEqual(t1.perform_get("apple"), "10000") self.assertEqual(t2.perform_get("apple"), "10000") self.assertEqual(t3.perform_get("apple"), "10000")
def test_deadlock_rw_rw(self): # Should pass after 2 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) coordinator = TransactionCoordinator(lock_table) # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_get('a'), 'No such key') # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_get('b'), 'No such key') self.assertEqual(t0.perform_put('a', 'a0'), 'Success') self.assertEqual(t0.perform_put('b', 'b0'), 'Success') # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.commit(), 'Transaction Completed') # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_get('a'), 'a0') # T1 R(a) self.assertEqual(t2.perform_get('b'), 'b0') # T2 R(b) # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_put('b', 'b1'), None) # T1 W(b) # self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.check_lock(), None) self.assertEqual(t2.perform_put('a', 'a1'), None) # T2 W(a) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 1 or abort_id == 2)
def test_multi_deadlock(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) t4 = TransactionHandler(lock_table, 4, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_put('a', 'apple'), 'Success') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.perform_get('a'), 'apple') self.assertEqual(t2.perform_get('a'), 'apple') self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_put('a', 'banana'), None) self.assertEqual(t2.perform_put('a', 'pear'), None) self.assertEqual(t3.perform_put('a', 'cherry'), None) self.assertEqual(t4.perform_put('a', 'orange'), None) aid = coordinator.detect_deadlocks() self.assertTrue(aid == 2 or aid == 1) self.assertEqual(t2.check_lock(), None) self.assertEqual(t1.abort(USER), 'User Abort') self.assertEqual(t2.check_lock(), 'Success') self.assertEqual(t2.perform_get('a'), 'pear') self.assertEqual(t3.check_lock(), None) self.assertEqual(t4.check_lock(), None) aa = coordinator.detect_deadlocks() self.assertEqual(aa, None)
def test_deadlock_gap_queue(self): lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) t3 = TransactionHandler(lock_table, 3, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t2.perform_put('a', 'a0'), 'Success') self.assertEqual(t0.perform_get('a'), None) self.assertEqual(t3.perform_put('b', 'b0'), 'Success') self.assertEqual(t0.perform_get('b'), None) self.assertEqual(t1.perform_put('b', 'b1'), None) self.assertEqual(t2.perform_get('b'), None) abort_id = coordinator.detect_deadlocks() self.assertEqual(abort_id, None)
def test_deadlock_rw_rw(self): # Should pass after 2 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) t2 = TransactionHandler(lock_table, 2, store) coordinator = TransactionCoordinator(lock_table) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_get('b'), 'No such key') self.assertEqual(t0.perform_put('a', 'a0'), 'Success') self.assertEqual(t0.perform_put('b', 'b0'), 'Success') self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_get('a'), 'a0') # T1 R(a) self.assertEqual(t2.perform_get('b'), 'b0') # T2 R(b) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.perform_put('b', 'b1'), None) # T1 W(b) self.assertEqual(coordinator.detect_deadlocks(), None) self.assertEqual(t1.check_lock(), None) self.assertEqual(t2.perform_put('a', 'a1'), None) # T2 W(a) abort_id = coordinator.detect_deadlocks() self.assertTrue(abort_id == 1 or abort_id == 2)
def test_unlock_ww(self): # Should pass after 1.3 lock_table = {} store = InMemoryKVStore() t0 = TransactionHandler(lock_table, 0, store) t1 = TransactionHandler(lock_table, 1, store) self.assertEqual(t0.perform_get('a'), 'No such key') self.assertEqual(t0.perform_put('a', '0'), 'Success') # T0 W(a) self.assertEqual(t1.perform_put('a', '1'), None) # T1 W(a) self.assertEqual(t0.perform_get('a'), '0') self.assertEqual(t1.check_lock(), None) self.assertEqual(t1.check_lock(), None) self.assertEqual(t0.commit(), 'Transaction Completed') self.assertEqual(t1.check_lock(), 'Success') # AssertionError: None != 'Success' self.assertEqual(t1.perform_get('a'), '1')