def test_staff_read_x_lock(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_put('a', '1'), 'Success')
self.assertEqual(t0.perform_get('a'), '1')
self.assertEqual(t0.perform_put('a', '3'), 'Success')
def test_staff_multiple_write_abort(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_put('a', '1'), 'Success')
self.assertEqual(t0.perform_put('a', '2'), 'Success')
self.assertEqual(t0.perform_put('a', '3'), 'Success')
self.assertEqual(t0.abort(USER), 'User Abort')
self.assertEqual(store.get('a'), None)
def test_staff_multiple_write_commit(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_put('a', '1'), 'Success')
self.assertEqual(t0.perform_put('a', '2'), 'Success')
self.assertEqual(t0.perform_put('a', '3'), 'Success')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(store.get('a'), '3')
def test_abort(self):
# Sanity check
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t0.abort(USER), 'User Abort')
self.assertEqual(store.get('a'), None)
def test_commit(self):
# Sanity check
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(store.get('a'), '0')
def test_rwr(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t1.perform_put('a', '1'), None)
self.assertEqual(t2.perform_get('a'), None)
def test_abort_queue(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t1.perform_get('a'), None)
self.assertEqual(t1.perform_put('b', '1'), 'Success')
self.assertEqual(t1.abort(USER), 'User Abort')
self.assertEqual(store.get('a'), '0')
self.assertEqual(store.get('b'), None)
def test_upgrade_in_put(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
self.assertEqual(t0.perform_get('apple'), 'No such key')
self.assertEqual(t1.perform_put('apple', '333'), None)
self.assertEqual(t0.perform_put('apple', 'aaa'), 'Success')
self.assertEqual(t0.perform_get('apple'), 'aaa')
def test_abort_commit(self):
# Should pass after 1.2
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t0.abort(USER), 'User Abort')
self.assertEqual(t1.perform_get('a'), 'No such key')
self.assertEqual(t1.perform_put('a', '1'), 'Success')
self.assertEqual(t1.perform_get('a'), '1')
self.assertEqual(t1.commit(), 'Transaction Completed')
self.assertEqual(store.get('a'), '1')
def test_wr(self):
# Should pass after 1.1
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t1.perform_get('a'), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(store.get('a'), '0')
def __init__(self, sock, server, store, stats, lock_table, xid, log_level):
"""
Initializes the handler and adds it to the polling loop. If an
exception is raised during the constructor, then the handler is closed
and removed from the polling loop.
"""
self._logger = logging.getLogger('<%s %s>' % (self.__class__.__name__, xid))
self._logger.setLevel(log_level)
self._server = server
self._store = store
self._stats = stats
# Global lock table. Each key maps to a list of four lists.
# The first list contains txns holding the key's s lock. The second list contains txns holding the key's x lock (list length should be either 0 or 1)
# The third list contains txns waiting to acquire the s lock. The fourth list contains txns waiting to acquire the x lock
self._lock_table = lock_table
self._xid = xid
# A string storing data passed from handle_read() to handle_write().
self._data = str(self._xid)
# One of the state constants listed above
self._state = RESPONDING
# self.connected is inherited
self._txn_handler = TransactionHandler(self._lock_table, self._xid, self._store)
try:
asyncore.dispatcher.__init__(self, sock)
except:
# Exceptions from constructor are not caught by handle_error(), so
# we catch them in the constructor itself
exc = traceback.format_exc()
self._logger.error('Uncaught exception in __init__, cannot construct server handler\n%s', exc[:-1])
self.close()
return
self._logger.debug('Constructed server handler')
def test_deadlock_naive(self):
# Should pass after 2
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t2 = TransactionHandler(lock_table, 2, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_put('a', 'a0'), 'Success')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t0.perform_get('a'), 'a0')
self.assertEqual(t2.perform_get('a'), 'a0')
self.assertEqual(t0.perform_put('a', 'ab'), None)
self.assertEqual(coordinator.detect_deadlocks(), None)
def test_multiple_read(self):
# Sanity check
lock_table = {}
store = InMemoryKVStore()
store.put('a', '0')
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t1.perform_get('a'), '0')
self.assertEqual(t2.perform_get('a'), '0')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.abort(USER), 'User Abort')
self.assertEqual(t2.perform_get('a'), '0')
def test_unlock_ww(self):
# Should pass after 1.3
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success') # T0 W(a)
self.assertEqual(t1.perform_put('a', '1'), None) # T1 W(a)
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.check_lock(), 'Success')
self.assertEqual(t1.perform_get('a'), '1')
def __init__(self, sock, server, store, stats, lock_table, xid, log_level):
"""
Initializes the handler and adds it to the polling loop. If an
exception is raised during the constructor, then the handler is closed
and removed from the polling loop.
"""
self._logger = logging.getLogger('<%s %s>' %
(self.__class__.__name__, xid))
self._logger.setLevel(log_level)
self._server = server
self._store = store
self._stats = stats
# Global lock table. Each key maps to a list of four lists.
# The first list contains txns holding the key's s lock. The second list contains txns holding the key's x lock (list length should be either 0 or 1)
# The third list contains txns waiting to acquire the s lock. The fourth list contains txns waiting to acquire the x lock
self._lock_table = lock_table
self._xid = xid
# A string storing data passed from handle_read() to handle_write().
self._data = str(self._xid)
# One of the state constants listed above
self._state = RESPONDING
# self.connected is inherited
self._txn_handler = TransactionHandler(self._lock_table, self._xid,
self._store)
try:
asyncore.dispatcher.__init__(self, sock)
except:
# Exceptions from constructor are not caught by handle_error(), so
# we catch them in the constructor itself
exc = traceback.format_exc()
self._logger.error(
'Uncaught exception in __init__, cannot construct server handler\n%s',
exc[:-1])
self.close()
return
self._logger.debug('Constructed server handler')
def test_staff_deadlock_upgrade(self):
lock_table = {}
store = InMemoryKVStore()
t2 = TransactionHandler(lock_table, 2, store)
t4 = TransactionHandler(lock_table, 4, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
# t2, t4 has S lock on a; t2 wants X lock
self.assertEqual(t2.perform_get('a'), 'No such key')
self.assertEqual(t4.perform_get('a'), 'No such key')
self.assertEqual(t2.perform_put('a', 'a1'), None)
# t4 now wants X lock
self.assertEqual(t4.perform_put('a', 'a2'), None)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 2 or abort_id == 4)
def test_staff_deadlock_ww_rw(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
# t0 has X lock on a; t1 wants X lock
self.assertEqual(t0.perform_put('a', 'a0'), 'Success')
self.assertEqual(t1.perform_put('a', 'a1'), None)
# t1 has S lock on b; t0 wants X lock
self.assertEqual(t1.perform_get('b'), 'No such key')
self.assertEqual(t0.perform_put('b', 'b0'), None)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 0 or abort_id == 1)
def test_commit(self):
# Sanity check
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(store.get('a'), '0')
def test_abort(self):
# Sanity check
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t0.abort(USER), 'User Abort')
self.assertEqual(store.get('a'), None)
def test_rw(self):
# Should pass after 1.1
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t1.perform_get('a'), 'No such key')
self.assertEqual(t1.perform_put('a', '0'), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(store.get('a'), None)
def test_single_abort(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
self.assertEqual(t0.perform_get('apple'), 'No such key')
self.assertEqual(t1.perform_get('apple'), 'No such key')
self.assertEqual(t2.perform_get('apple'), 'No such key')
self.assertEqual(t0.perform_put('apple', '11'), None)
self.assertEqual(t0.check_lock(), None)
self.assertEqual(t1.commit(), 'Transaction Completed')
self.assertEqual(t0.check_lock(), None)
self.assertEqual(t2.commit(), 'Transaction Completed')
self.assertEqual(t0.check_lock(), "Success")
self.assertEqual(t0.perform_get('apple'), '11')
self.assertEqual(t0.perform_get('banana'), 'No such key')
self.assertEqual(t0.perform_put("banana", '22'), "Success")
self.assertEqual(t1.perform_get('apple'), None)
self.assertEqual(t0.abort(USER), 'User Abort')
self.assertEqual(t1.check_lock(), 'No such key')
def test_multiple_read(self):
# Sanity check
lock_table = {}
store = InMemoryKVStore()
store.put('a', '0')
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t1.perform_get('a'), '0')
self.assertEqual(t2.perform_get('a'), '0')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.abort(USER), 'User Abort')
self.assertEqual(t2.perform_get('a'), '0')
def test_upgrade_s_x_multi(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
self.assertEqual(t0.perform_put('noob', '555'), 'Success')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t0.perform_get('noob'), '555')
self.assertEqual(t1.perform_get('noob'), '555')
self.assertEqual(t2.perform_get('noob'), '555')
self.assertEqual(t1.perform_put('noob', '111'), None)
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t2.commit(), 'Transaction Completed')
self.assertEqual(t1.check_lock(), 'Success')
self.assertEqual(t1.perform_get('noob'), '111')
def test_commit_abort_commit(self):
# Should pass after 1.2
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.perform_get('a'), '0')
self.assertEqual(t1.perform_put('a', '1'), 'Success')
self.assertEqual(t1.perform_get('a'), '1')
self.assertEqual(t1.abort(USER), 'User Abort')
self.assertEqual(t2.perform_get('a'), '0')
self.assertEqual(t2.perform_put('a', '2'), 'Success')
self.assertEqual(t2.perform_get('a'), '2')
self.assertEqual(t2.commit(), 'Transaction Completed')
self.assertEqual(store.get('a'), '2')
def test_unlock_rrr(self):
# Should pass after 1.3
lock_table = {}
store = InMemoryKVStore()
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
self.assertEqual(t0.perform_get('a'), 'No such key') # T0 R(a)
self.assertEqual(t1.perform_get('a'), 'No such key') # T1 R(a)
self.assertEqual(t2.perform_get('a'), 'No such key')
self.assertEqual(t3.perform_put('a', '0'), None) # T0 W(a)
self.assertEqual(coordinator.detect_deadlocks(), None)
def test_staff_lock_upgrade(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t1.perform_get('a'), 'No such key')
self.assertEqual(t2.perform_put('a', '1'), None)
self.assertEqual(t0.perform_put('a', '1'), None)
self.assertEqual(t1.commit(), 'Transaction Completed')
self.assertEqual(t0.check_lock(), 'Success')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t2.check_lock(), 'Success')
def test_staff_fifo_queue(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
t4 = TransactionHandler(lock_table, 4, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success') # T0 W(a)
self.assertEqual(t1.perform_get('a'), None) # T1 R(a)
self.assertEqual(t2.perform_put('a', '2'), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.check_lock(), '0')
self.assertEqual(t1.perform_get('a'), '0')
self.assertEqual(t3.perform_get('a'), None)
self.assertEqual(t4.perform_get('a'), None)
self.assertEqual(t1.commit(), 'Transaction Completed')
self.assertEqual(t2.check_lock(), 'Success')
self.assertEqual(t2.commit(), 'Transaction Completed')
self.assertEqual(t3.check_lock(), '2')
self.assertEqual(t4.check_lock(), '2')
def test_staff_deadlock_two_cycles(self):
lock_table = {}
store = InMemoryKVStore()
t2 = TransactionHandler(lock_table, 2, store)
t4 = TransactionHandler(lock_table, 4, store)
t6 = TransactionHandler(lock_table, 6, store)
t8 = TransactionHandler(lock_table, 8, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
# t2 has S lock on a; t4 wants X lock
self.assertEqual(t2.perform_get('a'), 'No such key')
self.assertEqual(t4.perform_put('a', 'a1'), None)
# t4 has S lock on b; t2 wants X lock
self.assertEqual(t4.perform_get('b'), 'No such key')
self.assertEqual(t2.perform_put('b', 'b1'), None)
# t6 has S lock on a; t8 wants X lock
self.assertEqual(t6.perform_get('c'), 'No such key')
self.assertEqual(t8.perform_put('c', 'c1'), None)
# t8 has S lock on b; t6 wants X lock
self.assertEqual(t8.perform_get('d'), 'No such key')
self.assertEqual(t6.perform_put('d', 'd1'), None)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 2 or abort_id == 4 or abort_id == 6 or abort_id == 8)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 2 or abort_id == 4 or abort_id == 6 or abort_id == 8)
class KVStoreServerHandler(asyncore.dispatcher):
"""
Each handler communicates with a single client. If the handler raises an
exception, during the constructor or from a handle_*() method, then it is
automatically closed. The handle_*() methods of all handlers (that have not
yet been closed) are called by the polling loop whenever appropriate.
"""
def __init__(self, sock, server, store, stats, lock_table, xid, log_level):
"""
Initializes the handler and adds it to the polling loop. If an
exception is raised during the constructor, then the handler is closed
and removed from the polling loop.
"""
self._logger = logging.getLogger('<%s %s>' %
(self.__class__.__name__, xid))
self._logger.setLevel(log_level)
self._server = server
self._store = store
self._stats = stats
# Global lock table. Each key maps to a list of four lists.
# The first list contains txns holding the key's s lock. The second list contains txns holding the key's x lock (list length should be either 0 or 1)
# The third list contains txns waiting to acquire the s lock. The fourth list contains txns waiting to acquire the x lock
self._lock_table = lock_table
self._xid = xid
# A string storing data passed from handle_read() to handle_write().
self._data = str(self._xid)
# One of the state constants listed above
self._state = RESPONDING
# self.connected is inherited
self._txn_handler = TransactionHandler(self._lock_table, self._xid,
self._store)
try:
asyncore.dispatcher.__init__(self, sock)
except:
# Exceptions from constructor are not caught by handle_error(), so
# we catch them in the constructor itself
exc = traceback.format_exc()
self._logger.error(
'Uncaught exception in __init__, cannot construct server handler\n%s',
exc[:-1])
self.close()
return
self._logger.debug('Constructed server handler')
def reliable_send(self, msg):
"""
asyncore.dispatcher does not have sendall, so we implement it here.
We stop trying to send if the handler is ever closed.
"""
if not self.connected:
return
sent = self.send(msg)
while sent < len(msg) and self.connected:
sent += self.send(msg[sent:])
def end_transaction(self):
"""
Marks the end of a transaction, after a COMMIT or ABORT. Notifies the
user that the COMMIT or ABORT completed.
"""
self.close()
def readable(self):
"""
handle_read() is called if readable() is True and the select syscall
says the socket is ready to read.
We always want to call handle_read() if there is data to be read.
"""
return True
def writable(self):
"""
handle_write() is called if writable() is True and the select syscall
says the socket is ready to write.
We avoid calling handle_write() when there is no data to be written.
"""
return self._state != WAITING
def handle_read(self):
"""
Called by the polling loop if readable() is True and the socket is
ready to read. We read the data by calling recv(). recv() returns '' if
the connection was closed, so we check for this explicitly. If the
connection is still open, handle_read() must set self._data to a
non-None value.
"""
# TODO: implement reliable recv. Create input buffer, in case the input takes too long to arrive or arrives in multiple chunks, and delimiter to mark end of stream.
data = self.recv(CHUNK_SIZE)
if data == '':
# The connection was closed (and handle_close() was already called)
return
if data[0] == ' ':
self._state, self._data = RESPONDING, 'Command begins with whitespace'
return
if re.search(r'[^A-Za-z0-9_ ]', data):
self._state, self._data = RESPONDING, 'Special characters in message'
return
tokens = data.split(' ', 3)
if tokens[0] == 'GET':
if len(tokens) == 2:
key = tokens[1]
if key == '':
self._state, self._data = RESPONDING, 'Bad format for GET'
else:
result = self._txn_handler.perform_get(key)
self._stats[0] += 1
if result is None:
self._state, self._data = LOCKING, None
elif isinstance(result, str):
self._state, self._data = RESPONDING, result
else:
raise KVStoreError(
'T%s.perform_get() returned %r, which is not a string or None'
% (self._xid, result))
else:
self._state, self._data = RESPONDING, 'Bad format for GET'
elif tokens[0] == 'PUT':
if len(tokens) == 3:
key, value = tokens[1], tokens[2]
if key == '' or value == '':
self._state, self._data = RESPONDING, 'Bad format for PUT'
else:
result = self._txn_handler.perform_put(key, value)
self._stats[1] += 1
if result is None:
self._state, self._data = LOCKING, None
elif isinstance(result, str):
self._state, self._data = RESPONDING, result
else:
raise KVStoreError(
'T%s.perform_put() returned %r, which is not a string or None'
% (self._xid, result))
else:
self._state, self._data = RESPONDING, 'Bad format for PUT'
elif tokens[0] == 'COMMIT':
if len(tokens) == 1:
result = self._txn_handler.commit()
if not isinstance(result, str):
raise KVStoreError(
'T%s.commit() returned %r, which is not a string' %
(self._xid, result))
self._state, self._data = COMMITTING, result
else:
self._state, self._data = RESPONDING, 'Bad format for COMMIT'
elif tokens[0] == 'ABORT':
if len(tokens) == 1:
result = self._txn_handler.abort(USER)
if not isinstance(result, str):
raise KVStoreError(
'T%s.abort() returned %r, which is not a string' %
(self._xid, result))
self._state, self._data = ABORTING, result
else:
self._state, self._data = RESPONDING, 'Bad format for ABORT'
else:
self._state, self._data = RESPONDING, 'Unrecognized command'
self._logger.debug('Lock table is %r', self._lock_table)
def handle_write(self):
"""
Called by the polling loop if writable() is True and the socket is
ready to write. We write the data by calling send().
"""
if self._state != LOCKING:
self.reliable_send(self._data)
if self._state == ABORTING or self._state == COMMITTING:
self.end_transaction()
self._data = None
self._state = WAITING
else:
# Try to acquire the necessary locks
result = self._txn_handler.check_lock()
if result is not None:
if not isinstance(result, str):
raise KVStoreError(
'T%s.check_lock() returned %r, which is not a string or None'
% (self._xid, result))
self._state, self._data = RESPONDING, result
def deadlock_abort(self):
result = self._txn_handler.abort(DEADLOCK)
if not isinstance(result, str):
raise KVStoreError(
'T%s.abort() returned %r, which is not a string' %
(self._xid, result))
self._state, self._data = ABORTING, result
def is_open(self):
return self.connected
def close(self):
asyncore.dispatcher.close(self)
self._server.remove_transaction(self._xid)
self._logger.debug('Closed server handler')
def handle_close(self):
"""
Closes the handler and removes it from the polling loop.
Called when the client has closed the connection. Because the polling
loop uses the select syscall, handle_close() is called immediately
after the connection is closed.
"""
self._logger.debug('Client disconnected, closing server handler')
self.close()
def handle_error(self):
"""
Closes the handler and removes it from the polling loop.
Called when one of the handle_*() methods of the class raises an
exception. Prints the stack trace and closes the handler.
"""
exc = traceback.format_exc()
self._logger.error('Uncaught exception, closing server handler\n%s',
exc[:-1])
self.close()
def test_staff_block_read_write_read(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
self.assertEqual(t0.perform_put('a', '0'), 'Success')
self.assertEqual(t1.perform_get('a'), None)
self.assertEqual(t2.perform_put('a', '2'), None)
self.assertEqual(t3.perform_get('a'), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.check_lock(), '0')
self.assertEqual(t2.check_lock(), None)
self.assertEqual(t3.check_lock(), None)
self.assertEqual(t1.commit(), 'Transaction Completed')
self.assertEqual(t2.check_lock(), 'Success')
self.assertEqual(t3.check_lock(), None)
self.assertEqual(t2.commit(), 'Transaction Completed')
self.assertEqual(t3.check_lock(), '2')
class KVStoreServerHandler(asyncore.dispatcher):
"""
Each handler communicates with a single client. If the handler raises an
exception, during the constructor or from a handle_*() method, then it is
automatically closed. The handle_*() methods of all handlers (that have not
yet been closed) are called by the polling loop whenever appropriate.
"""
def __init__(self, sock, server, store, stats, lock_table, xid, log_level):
"""
Initializes the handler and adds it to the polling loop. If an
exception is raised during the constructor, then the handler is closed
and removed from the polling loop.
"""
self._logger = logging.getLogger('<%s %s>' % (self.__class__.__name__, xid))
self._logger.setLevel(log_level)
self._server = server
self._store = store
self._stats = stats
# Global lock table. Each key maps to a list of four lists.
# The first list contains txns holding the key's s lock. The second list contains txns holding the key's x lock (list length should be either 0 or 1)
# The third list contains txns waiting to acquire the s lock. The fourth list contains txns waiting to acquire the x lock
self._lock_table = lock_table
self._xid = xid
# A string storing data passed from handle_read() to handle_write().
self._data = str(self._xid)
# One of the state constants listed above
self._state = RESPONDING
# self.connected is inherited
self._txn_handler = TransactionHandler(self._lock_table, self._xid, self._store)
try:
asyncore.dispatcher.__init__(self, sock)
except:
# Exceptions from constructor are not caught by handle_error(), so
# we catch them in the constructor itself
exc = traceback.format_exc()
self._logger.error('Uncaught exception in __init__, cannot construct server handler\n%s', exc[:-1])
self.close()
return
self._logger.debug('Constructed server handler')
def reliable_send(self, msg):
"""
asyncore.dispatcher does not have sendall, so we implement it here.
We stop trying to send if the handler is ever closed.
"""
if not self.connected:
return
sent = self.send(msg)
while sent < len(msg) and self.connected:
sent += self.send(msg[sent:])
def end_transaction(self):
"""
Marks the end of a transaction, after a COMMIT or ABORT. Notifies the
user that the COMMIT or ABORT completed.
"""
self.close()
def readable(self):
"""
handle_read() is called if readable() is True and the select syscall
says the socket is ready to read.
We always want to call handle_read() if there is data to be read.
"""
return True
def writable(self):
"""
handle_write() is called if writable() is True and the select syscall
says the socket is ready to write.
We avoid calling handle_write() when there is no data to be written.
"""
return self._state != WAITING
def handle_read(self):
"""
Called by the polling loop if readable() is True and the socket is
ready to read. We read the data by calling recv(). recv() returns '' if
the connection was closed, so we check for this explicitly. If the
connection is still open, handle_read() must set self._data to a
non-None value.
"""
# TODO: implement reliable recv. Create input buffer, in case the input takes too long to arrive or arrives in multiple chunks, and delimiter to mark end of stream.
data = self.recv(CHUNK_SIZE)
if data == '':
# The connection was closed (and handle_close() was already called)
return
if data[0] == ' ':
self._state, self._data = RESPONDING, 'Command begins with whitespace'
return
if re.search(r'[^A-Za-z0-9_ ]', data):
self._state, self._data = RESPONDING, 'Special characters in message'
return
tokens = data.split(' ', 3)
if tokens[0] == 'GET':
if len(tokens) == 2:
key = tokens[1]
if key == '':
self._state, self._data = RESPONDING, 'Bad format for GET'
else:
result = self._txn_handler.perform_get(key)
self._stats[0] += 1
if result is None:
self._state, self._data = LOCKING, None
elif isinstance(result, str):
self._state, self._data = RESPONDING, result
else:
raise KVStoreError('T%s.perform_get() returned %r, which is not a string or None' % (self._xid, result))
else:
self._state, self._data = RESPONDING, 'Bad format for GET'
elif tokens[0] == 'PUT':
if len(tokens) == 3:
key, value = tokens[1], tokens[2]
if key == '' or value == '':
self._state, self._data = RESPONDING, 'Bad format for PUT'
else:
result = self._txn_handler.perform_put(key, value)
self._stats[1] += 1
if result is None:
self._state, self._data = LOCKING, None
elif isinstance(result, str):
self._state, self._data = RESPONDING, result
else:
raise KVStoreError('T%s.perform_put() returned %r, which is not a string or None' % (self._xid, result))
else:
self._state, self._data = RESPONDING, 'Bad format for PUT'
elif tokens[0] == 'COMMIT':
if len(tokens) == 1:
result = self._txn_handler.commit()
if not isinstance(result, str):
raise KVStoreError('T%s.commit() returned %r, which is not a string' % (self._xid, result))
self._state, self._data = COMMITTING, result
else:
self._state, self._data = RESPONDING, 'Bad format for COMMIT'
elif tokens[0] == 'ABORT':
if len(tokens) == 1:
result = self._txn_handler.abort(USER)
if not isinstance(result, str):
raise KVStoreError('T%s.abort() returned %r, which is not a string' % (self._xid, result))
self._state, self._data = ABORTING, result
else:
self._state, self._data = RESPONDING, 'Bad format for ABORT'
else:
self._state, self._data = RESPONDING, 'Unrecognized command'
self._logger.debug('Lock table is %r', self._lock_table)
def handle_write(self):
"""
Called by the polling loop if writable() is True and the socket is
ready to write. We write the data by calling send().
"""
if self._state != LOCKING:
self.reliable_send(self._data)
if self._state == ABORTING or self._state == COMMITTING:
self.end_transaction()
self._data = None
self._state = WAITING
else:
# Try to acquire the necessary locks
result = self._txn_handler.check_lock()
if result is not None:
if not isinstance(result, str):
raise KVStoreError('T%s.check_lock() returned %r, which is not a string or None' % (self._xid, result))
self._state, self._data = RESPONDING, result
def deadlock_abort(self):
result = self._txn_handler.abort(DEADLOCK)
if not isinstance(result, str):
raise KVStoreError('T%s.abort() returned %r, which is not a string' % (self._xid, result))
self._state, self._data = ABORTING, result
def is_open(self):
return self.connected
def close(self):
asyncore.dispatcher.close(self)
self._server.remove_transaction(self._xid)
self._logger.debug('Closed server handler')
def handle_close(self):
"""
Closes the handler and removes it from the polling loop.
Called when the client has closed the connection. Because the polling
loop uses the select syscall, handle_close() is called immediately
after the connection is closed.
"""
self._logger.debug('Client disconnected, closing server handler')
self.close()
def handle_error(self):
"""
Closes the handler and removes it from the polling loop.
Called when one of the handle_*() methods of the class raises an
exception. Prints the stack trace and closes the handler.
"""
exc = traceback.format_exc()
self._logger.error('Uncaught exception, closing server handler\n%s', exc[:-1])
self.close()
def test_1_thread_on_multiple_keys(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
self.assertEqual(t0.perform_get('apple'), 'No such key')
self.assertEqual(t1.perform_get('apple'), 'No such key')
self.assertEqual(t2.perform_get('apple'), 'No such key')
self.assertEqual(t0.perform_put('apple', '11'), None)
self.assertEqual(t0.check_lock(), None)
self.assertEqual(t1.commit(), 'Transaction Completed')
self.assertEqual(t0.check_lock(), None)
self.assertEqual(t2.commit(), 'Transaction Completed')
self.assertEqual(t0.check_lock(), "Success")
self.assertEqual(t0.perform_get('apple'), '11')
self.assertEqual(t0.perform_put('apple', "10000"), 'Success')
self.assertEqual(t0.perform_get('apple'), '10000')
self.assertEqual(t0.perform_put("banana", '22'), "Success")
self.assertEqual(t0.perform_get('banana'), '22')
self.assertEqual(t1.perform_get('apple'), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.check_lock(), '10000')
self.assertEqual(t1.perform_get("apple"), "10000")
self.assertEqual(t2.perform_get("apple"), "10000")
self.assertEqual(t3.perform_get("apple"), "10000")
def test_deadlock_rw_rw(self):
# Should pass after 2
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
coordinator = TransactionCoordinator(lock_table)
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_get('a'), 'No such key')
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_get('b'), 'No such key')
self.assertEqual(t0.perform_put('a', 'a0'), 'Success')
self.assertEqual(t0.perform_put('b', 'b0'), 'Success')
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_get('a'), 'a0') # T1 R(a)
self.assertEqual(t2.perform_get('b'), 'b0') # T2 R(b)
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_put('b', 'b1'), None) # T1 W(b)
# self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t2.perform_put('a', 'a1'), None) # T2 W(a)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 1 or abort_id == 2)
def test_multi_deadlock(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
t4 = TransactionHandler(lock_table, 4, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_put('a', 'apple'), 'Success')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.perform_get('a'), 'apple')
self.assertEqual(t2.perform_get('a'), 'apple')
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_put('a', 'banana'), None)
self.assertEqual(t2.perform_put('a', 'pear'), None)
self.assertEqual(t3.perform_put('a', 'cherry'), None)
self.assertEqual(t4.perform_put('a', 'orange'), None)
aid = coordinator.detect_deadlocks()
self.assertTrue(aid == 2 or aid == 1)
self.assertEqual(t2.check_lock(), None)
self.assertEqual(t1.abort(USER), 'User Abort')
self.assertEqual(t2.check_lock(), 'Success')
self.assertEqual(t2.perform_get('a'), 'pear')
self.assertEqual(t3.check_lock(), None)
self.assertEqual(t4.check_lock(), None)
aa = coordinator.detect_deadlocks()
self.assertEqual(aa, None)
def test_deadlock_gap_queue(self):
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
t3 = TransactionHandler(lock_table, 3, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t2.perform_put('a', 'a0'), 'Success')
self.assertEqual(t0.perform_get('a'), None)
self.assertEqual(t3.perform_put('b', 'b0'), 'Success')
self.assertEqual(t0.perform_get('b'), None)
self.assertEqual(t1.perform_put('b', 'b1'), None)
self.assertEqual(t2.perform_get('b'), None)
abort_id = coordinator.detect_deadlocks()
self.assertEqual(abort_id, None)
def test_deadlock_rw_rw(self):
# Should pass after 2
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
t2 = TransactionHandler(lock_table, 2, store)
coordinator = TransactionCoordinator(lock_table)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_get('b'), 'No such key')
self.assertEqual(t0.perform_put('a', 'a0'), 'Success')
self.assertEqual(t0.perform_put('b', 'b0'), 'Success')
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_get('a'), 'a0') # T1 R(a)
self.assertEqual(t2.perform_get('b'), 'b0') # T2 R(b)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.perform_put('b', 'b1'), None) # T1 W(b)
self.assertEqual(coordinator.detect_deadlocks(), None)
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t2.perform_put('a', 'a1'), None) # T2 W(a)
abort_id = coordinator.detect_deadlocks()
self.assertTrue(abort_id == 1 or abort_id == 2)
def test_unlock_ww(self):
# Should pass after 1.3
lock_table = {}
store = InMemoryKVStore()
t0 = TransactionHandler(lock_table, 0, store)
t1 = TransactionHandler(lock_table, 1, store)
self.assertEqual(t0.perform_get('a'), 'No such key')
self.assertEqual(t0.perform_put('a', '0'), 'Success') # T0 W(a)
self.assertEqual(t1.perform_put('a', '1'), None) # T1 W(a)
self.assertEqual(t0.perform_get('a'), '0')
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t1.check_lock(), None)
self.assertEqual(t0.commit(), 'Transaction Completed')
self.assertEqual(t1.check_lock(),
'Success') # AssertionError: None != 'Success'
self.assertEqual(t1.perform_get('a'), '1')
