def test_read_is_stale_with_concurrent_writes(self): """ Read a value that doesn't exist when concurrent with 2 reads. This should raise an exception. """ self.test_sucessful_write() writeset_1 = {'a': 'b'} writeset_2 = {'a': 'c'} read_block_id = 1 client0 = 0 client1 = 1 client2 = 2 # Another successful write commits. # All subsequent ops should see at least state {'a':'b'} self.tracker.send_write(client0, 1, set(), writeset_1, read_block_id) self.tracker.handle_write_reply(client0, 1, skvbc.WriteReply(True, 2)) # A concurrent write and a concurrent read self.tracker.send_write(client1, 1, set(), writeset_2, read_block_id) self.tracker.send_read(client2, 1, ["a"]) # block3/ writeset_2 self.tracker.handle_write_reply(client1, 1, skvbc.WriteReply(True, 3)) # Read a stale value. Key "a" was updated to "b" before the read was # sent. "a" can only be "b" or "c". self.tracker.handle_read_reply(client2, 1, {"a": "a"}) with self.assertRaises(InvalidReadError) as err: self.tracker.verify() self.assertEqual(1, len(err.exception.concurrent_requests))
def test_read_does_not_linearize_two_concurrent_writes(self): """ Read a value that doesn't exist when concurrent with 2 reads. This should raise an exception. """ self.test_sucessful_write() writeset_1 = {'a': 'b'} writeset_2 = {'a': 'c'} read_block_id = 1 client0 = 0 client1 = 1 client2 = 2 # 2 concurrent writes and a concurrent read self.tracker.send_write(client0, 1, set(), writeset_1, read_block_id) self.tracker.send_write(client1, 1, set(), writeset_2, read_block_id) self.tracker.send_read(client2, 1, ["a"]) # block2/ writeset_2 self.tracker.handle_write_reply(client1, 1, skvbc.WriteReply(True, 2)) # block3/ writeset_1 self.tracker.handle_write_reply(client0, 1, skvbc.WriteReply(True, 3)) # Read a value that was never written self.tracker.handle_read_reply(client2, 1, {"a": "d"}) with self.assertRaises(InvalidReadError) as err: self.tracker.verify() self.assertEqual(2, len(err.exception.concurrent_requests))
def test_read_between_2_successful_writes(self): """ Send 2 concurrent writes that don't conflict, but overwrite the same value, along with a concurrent read. The read sees the first overwrite, and should linearize after the first overwritten value correctly. """ # A non-concurrent write self.test_sucessful_write() writeset_1 = {'a': 'b'} writeset_2 = {'a': 'c'} read_block_id = 1 client0 = 0 client1 = 1 client2 = 2 # 2 concurrent writes and a concurrent read self.tracker.send_write(client0, 1, set(), writeset_1, read_block_id) self.tracker.send_write(client1, 1, set(), writeset_2, read_block_id) self.tracker.send_read(client2, 1, ["a"]) # block2/ writeset_2 self.tracker.handle_write_reply(client1, 1, skvbc.WriteReply(True, 2)) # block3/ writeset_1 self.tracker.handle_write_reply(client0, 1, skvbc.WriteReply(True, 3)) self.tracker.handle_read_reply(client2, 1, {"a": "b"}) self.tracker.verify()
def test_contentious_writes_one_success_one_fail(self): """ Two writes contend for the same key. Only one should succeed. The checker should not see an error. """ # First create an initial block so we can contend with it self.test_sucessful_write() # Send 2 concurrent writes with the same readset and writeset readset = set("a") writeset = {'a': 'a'} self.tracker.send_write(0, 1, readset, writeset, 1) self.tracker.send_write(1, 1, readset, writeset, 1) self.tracker.handle_write_reply(0, 1, skvbc.WriteReply(False, 0)) self.tracker.handle_write_reply(1, 1, skvbc.WriteReply(True, 2)) self.tracker.verify()
def test_contentious_writes_both_succeed_same_block(self): """ Two writes contend for the same key. Both succeed, apparently creating the same block. The checker should raise a ConflictingBlockWriteError, since two requests cannot create the same block. """ # First create an initial block so we can contend with it self.test_sucessful_write() # Send 2 concurrent writes with the same readset and writeset readset = set("a") writeset = {'a': 'a'} self.tracker.send_write(0, 1, readset, writeset, 1) self.tracker.send_write(1, 1, readset, writeset, 1) self.tracker.handle_write_reply(1, 1, skvbc.WriteReply(True, 2)) with self.assertRaises(ConflictingBlockWriteError) as err: self.tracker.handle_write_reply(0, 1, skvbc.WriteReply(True, 2)) # The conflicting block id was block 2 self.assertEqual(err.exception.block_id, 2)
def test_contentious_writes_both_succeed(self): """ Two writes contend for the same key. Both succeed. The checker should raise a StaleReadError, since only one should have succeeded. """ # First create an initial block so we can contend with it self.test_sucessful_write() # Send 2 concurrent writes with the same readset and writeset readset = set("a") writeset = {'a': 'a'} self.tracker.send_write(0, 1, readset, writeset, 1) self.tracker.send_write(1, 1, readset, writeset, 1) self.tracker.handle_write_reply(1, 1, skvbc.WriteReply(True, 2)) self.tracker.handle_write_reply(0, 1, skvbc.WriteReply(True, 3)) with self.assertRaises(StaleReadError) as err: self.tracker.verify() # Check the exception detected the error correctly self.assertEqual(err.exception.readset_block_id, 1) self.assertEqual(err.exception.block_with_conflicting_writeset, 2) self.assertEqual(err.exception.block_being_checked, 3)
def test_2_concurrent_writes_one_missing_success_two_concurrent_reads( self): """ Two concurrent writes are sent with two concurrent reads. One write does not respond. One read should linearize correctly based on the write that returned, and the other based on the one that didn't return. """ # A non-concurrent write self.test_sucessful_write() writeset_1 = {'a': 'b'} writeset_2 = {'a': 'c'} read_block_id = 1 client0 = 0 client1 = 1 client2 = 2 client3 = 3 # 2 concurrent unconditional writes and a concurrent read self.tracker.send_write(client0, 1, set(), writeset_1, read_block_id) self.tracker.send_write(client1, 1, set(), writeset_2, read_block_id) self.tracker.send_read(client2, 1, ["a"]) self.tracker.send_read(client3, 1, ["a"]) # writeset_2 written at block 3 self.tracker.handle_write_reply(client1, 1, skvbc.WriteReply(True, 3)) self.tracker.handle_read_reply(client2, 1, writeset_1) self.tracker.handle_read_reply(client3, 1, writeset_2) self.assertEqual(3, self.tracker.last_known_block) # The test should call these methods when it stops sending # operations. # The test must ask the tracker what blocks are missing missing_block_id = 2 last_block = 3 missing_blocks = self.tracker.get_missing_blocks(last_block) self.assertSetEqual(set([missing_block_id]), missing_blocks) self.assertEqual(3, self.tracker.last_known_block) # The test must retrieve the missing blocks from skvbc TesterReplicas # using the clients and them inform the tracker. This is the call that # informs the tracker. self.tracker.fill_missing_blocks({missing_block_id: writeset_1}) # The tracker now has all the information it needs. Tell it to verify # that it's read responses linearize. self.tracker.verify()
def test_long_failed_history(self): """ Launch a batch of unconditional writes along with reads concurrently a few times. Insert a single incorrect read reply. """ self.test_sucessful_write() writeset_1 = {'a': 'b'} writeset_2 = {'a': 'c'} read_block_id = 1 stale_read = {'a': 'a'} written_block_id = 1 for seq_num in range(0, 5): # track requests for client_id in range(0, 20): if client_id % 3 == 0: self.tracker.send_read(client_id, seq_num, ["a"]) else: w = writeset_1 if client_id % 2 == 0: w = writeset_2 self.tracker.send_write(client_id, seq_num, set(), w, read_block_id) # track replies for client_id in range(0, 20): if client_id % 3 == 0: # expected doesn't really matter for correctness # it just has to be one of the two written values because # all writes and reads in a batch are concurrent. expected = writeset_2 if client_id % 2 == 0: expected = writeset_1 if client_id == 6 and seq_num == 3: self.tracker.handle_read_reply(client_id, seq_num, stale_read) else: self.tracker.handle_read_reply(client_id, seq_num, expected) else: written_block_id += 1 reply = skvbc.WriteReply(True, written_block_id) self.tracker.handle_write_reply(client_id, seq_num, reply) with self.assertRaises(InvalidReadError) as err: self.tracker.verify() self.assertEqual(19, len(err.exception.concurrent_requests)) self.assertEqual(stale_read, err.exception.read.kvpairs)
def test_sucessful_write(self): """ A single request results in a single successful reply. The checker finds no errors. """ client_id = 0 seq_num = 0 readset = set() read_block_id = 0 writeset = {'a': 'a'} self.tracker.send_write(client_id, seq_num, readset, writeset, read_block_id) reply = skvbc.WriteReply(success=True, last_block_id=1) self.tracker.handle_write_reply(client_id, seq_num, reply) self.tracker.verify()
def test_2_concurrent_writes_one_missing_failure_two_concurrent_reads( self): """ Two concurrent writes are sent with two concurrent reads. One write does not respond. One read should linearize correctly based on the write that returned, and the other based on the one that didn't return. """ # A non-concurrent write self.test_sucessful_write() writeset_1 = {'a': 'b'} writeset_2 = {'a': 'c'} read_block_id = 1 client0 = 0 client1 = 1 client2 = 2 client3 = 3 # 2 concurrent unconditional writes and a concurrent read self.tracker.send_write(client0, 1, set("a"), writeset_1, read_block_id) self.tracker.send_write(client1, 1, set("a"), writeset_2, read_block_id) self.tracker.send_read(client2, 1, ["a"]) self.tracker.send_read(client3, 1, ["a"]) # writeset_2 written at block 3 self.tracker.handle_write_reply(client1, 1, skvbc.WriteReply(True, 2)) self.tracker.handle_read_reply(client2, 1, writeset_1) self.tracker.handle_read_reply(client3, 1, writeset_2) self.assertEqual(2, self.tracker.last_known_block) # the last block is 2 because one of the writes conflicted last_block = 2 missing_blocks = self.tracker.get_missing_blocks(last_block) self.assertSetEqual(set(), missing_blocks) with self.assertRaises(InvalidReadError) as err: self.tracker.verify() self.assertEqual(3, len(err.exception.concurrent_requests)) # The read of writeset_1 fails, since that was the failed request that # never returned. self.assertEqual(writeset_1, err.exception.read.kvpairs)
def test_failed_write(self): """ A single request results in a single failed reply. The checker finds a NoConflictError, because there were no concurrent requests that would have caused the request to fail explicitly. """ client_id = 0 seq_num = 0 readset = set() read_block_id = 0 writeset = {'a': 'a'} self.tracker.send_write(client_id, seq_num, readset, writeset, read_block_id) reply = skvbc.WriteReply(success=False, last_block_id=1) self.tracker.handle_write_reply(client_id, seq_num, reply) with self.assertRaises(NoConflictError) as err: self.tracker.verify() self.assertEqual(0, err.exception.causal_state.req_index)
def test_long_correct_history(self): """ Launch a batch of unconditional writes along with reads concurrently a few times. The responses should always be correct. """ self.test_sucessful_write() writeset_1 = {'a': 'b'} writeset_2 = {'a': 'c'} read_block_id = 1 written_block_id = 1 for seq_num in range(0, 5): # track requests for client_id in range(0, 20): if client_id % 3 == 0: self.tracker.send_read(client_id, seq_num, ["a"]) else: w = writeset_1 if client_id % 2 == 0: w = writeset_2 self.tracker.send_write(client_id, seq_num, set(), w, read_block_id) # track replies for client_id in range(0, 20): if client_id % 3 == 0: # expected doesn't really matter for correctness # it just has to be one of the two written values because # all writes and reads in a batch are concurrent. expected = writeset_2 if client_id % 2 == 0: expected = writeset_1 self.tracker.handle_read_reply(client_id, seq_num, expected) else: written_block_id += 1 reply = skvbc.WriteReply(True, written_block_id) self.tracker.handle_write_reply(client_id, seq_num, reply) self.tracker.verify()
def test_phantom_block_fill_extra_block(self): """ Create a phantom block that fills in a block at the end of the block chain with an already used request, and ensure a PhantomBlockError gets raised. All write_requests matched in this case, but we ran out of them, as there was 1 more block than write request. """ self.test_sucessful_write() writeset_1 = {'a': 'b'} writeset_2 = {'a': 'c'} read_block_id = 1 client0 = 0 client1 = 1 self.tracker.send_write(client0, 1, set("a"), writeset_1, read_block_id) self.tracker.send_write(client1, 1, set("a"), writeset_2, read_block_id) self.tracker.handle_write_reply(client1, 1, skvbc.WriteReply(True, 3)) # We are missing blocks 2 and 4 last_block = 4 missing_block_ids = [2, 4] missing_blocks = self.tracker.get_missing_blocks(last_block) self.assertSetEqual(set(missing_block_ids), missing_blocks) # Fill in a block correctly matching writeset1, and then an extra block # with no matching writeset, since we already matched the last request # that used writeset_1. blocks = {2: writeset_1, 4: writeset_1} self.tracker.fill_missing_blocks(blocks) with self.assertRaises(PhantomBlockError) as err: self.tracker.verify() self.assertEqual(4, err.exception.block_id) self.assertEqual(writeset_1, err.exception.kvpairs) self.assertEqual(1, len(err.exception.matched_blocks)) self.assertEqual(0, len(err.exception.unmatched_requests))
def test_phantom_block_fill_hole(self): """ Create a phantom block that fills in a missing block in the middle of the block chain, and ensure a PhantomBlockError gets raised. """ self.test_sucessful_write() writeset_1 = {'a': 'b'} writeset_2 = {'a': 'c'} phantom_writeset = {'a': 'd'} read_block_id = 1 client0 = 0 client1 = 1 self.tracker.send_write(client0, 1, set("a"), writeset_1, read_block_id) self.tracker.send_write(client1, 1, set("a"), writeset_2, read_block_id) self.tracker.handle_write_reply(client1, 1, skvbc.WriteReply(True, 3)) # We are missing block 2 last_block = 3 missing_block_id = 2 missing_blocks = self.tracker.get_missing_blocks(last_block) self.assertSetEqual(set([missing_block_id]), missing_blocks) # Fill in block 2 with a block that could never have been generated by a # write request self.tracker.fill_missing_blocks({missing_block_id: phantom_writeset}) with self.assertRaises(PhantomBlockError) as err: self.tracker.verify() self.assertEqual(missing_block_id, err.exception.block_id) self.assertEqual(phantom_writeset, err.exception.kvpairs) self.assertEqual(0, len(err.exception.matched_blocks)) self.assertEqual(1, len(err.exception.unmatched_requests))