def test_state_store_get_and_set(self): """Tests that we correctly get and set state changes to a StateDeltaStore. This tests sets a list of state change values and then gets them back, ensuring that the data is the same. """ database = DictDatabase() delta_store = StateDeltaStore(database) changes = [StateChange(address='a100000' + str(i), value=str(i).encode(), type=StateChange.SET) for i in range(0, 10)] delta_store.save_state_deltas('my_state_root_hash', changes) stored_changes = delta_store.get_state_deltas('my_state_root_hash') # This is a list-like repeated field, but to make it comparable we'll # have to generate a new list stored_changes = [c for c in stored_changes] self.assertEqual(changes, stored_changes)
def test_raise_key_error_on_missing_root_hash(self): """Tests that we correctly raise key error on a missing hash """ database = DictDatabase() delta_store = StateDeltaStore(database) with self.assertRaises(KeyError): delta_store.get_state_deltas('unknown_state_root_hash')
def setUp(self): self.database_of_record = dict_database.DictDatabase() self.state_delta_store = StateDeltaStore(dict_database.DictDatabase()) self.context_manager = context_manager.ContextManager( self.database_of_record, self.state_delta_store) self.first_state_hash = self.context_manager.get_first_root() # used for replicating state hash through direct merkle tree updates self.database_results = dict_database.DictDatabase()
def test_add_subscriber(self): """Test adding a subscriber, who has no known blocks. This scenerio is valid for subscribers who have never connected and would need to receive all deltas since the genesis block. On registration, the subscriber should receive one event, comprised of the state changes for the genesis block. """ mock_service = Mock() block_tree_manager = BlockTreeManager() delta_store = StateDeltaStore(DictDatabase()) delta_processor = StateDeltaProcessor( service=mock_service, state_delta_store=delta_store, block_store=block_tree_manager.block_store) delta_store.save_state_deltas( block_tree_manager.chain_head.state_root_hash, [StateChange(address='deadbeef0000000', value='my_genesis_value'.encode(), type=StateChange.SET), StateChange(address='a14ea01', value='some other state value'.encode(), type=StateChange.SET)]) delta_processor.add_subscriber( 'test_conn_id', [], ['deadbeef']) self.assertEqual(['test_conn_id'], delta_processor.subscriber_ids) # test that it catches up, and receives the events from the chain head. # In this case, it should just be once, as the chain head is the # genesis block chain_head = block_tree_manager.chain_head mock_service.send.assert_called_with( validator_pb2.Message.STATE_DELTA_EVENT, StateDeltaEvent( block_id=chain_head.identifier, block_num=chain_head.block_num, state_root_hash=chain_head.state_root_hash, previous_block_id=chain_head.previous_block_id, state_changes=[StateChange(address='deadbeef0000000', value='my_genesis_value'.encode(), type=StateChange.SET)] ).SerializeToString(), connection_id='test_conn_id')
def test_publish_deltas_no_state_changes(self): """Given a block transition, where no state changes happened (e.g. it only had transactions which did not change state), the StateDeltaProcessor should still publish an event with the block change information. """ mock_service = Mock() block_tree_manager = BlockTreeManager() database = DictDatabase() delta_store = StateDeltaStore(database) delta_processor = StateDeltaProcessor( service=mock_service, state_delta_store=delta_store, block_store=block_tree_manager.block_store) delta_processor.add_subscriber( 'subscriber_conn_id', [block_tree_manager.chain_head.identifier], ['000000']) next_block = block_tree_manager.generate_block() delta_processor.publish_deltas(next_block) mock_service.send.assert_called_with( validator_pb2.Message.STATE_DELTA_EVENT, StateDeltaEvent( block_id=next_block.identifier, block_num=next_block.header.block_num, state_root_hash=next_block.header.state_root_hash, previous_block_id=next_block.header.previous_block_id, state_changes=[]).SerializeToString(), connection_id='subscriber_conn_id')
def test_publish_deltas(self): """Tests that a subscriber filtering on an address prefix receives only the changes in an event that match. """ mock_service = Mock() block_tree_manager = BlockTreeManager() database = DictDatabase() delta_store = StateDeltaStore(database) delta_processor = StateDeltaProcessor( service=mock_service, state_delta_store=delta_store, block_store=block_tree_manager.block_store) delta_processor.add_subscriber( 'test_conn_id', [block_tree_manager.chain_head.identifier], ['deadbeef']) next_block = block_tree_manager.generate_block() # State added during context squash for our block delta_store.save_state_deltas(next_block.header.state_root_hash, [ StateChange(address='deadbeef01', value='my_state_Value'.encode(), type=StateChange.SET), StateChange(address='a14ea01', value='some other state value'.encode(), type=StateChange.SET) ]) # call to publish deltas for that block to the subscribers delta_processor.publish_deltas(next_block) mock_service.send.assert_called_with( validator_pb2.Message.STATE_DELTA_EVENT, StateDeltaEvent( block_id=next_block.identifier, block_num=next_block.header.block_num, state_root_hash=next_block.header.state_root_hash, previous_block_id=next_block.header.previous_block_id, state_changes=[ StateChange(address='deadbeef01', value='my_state_Value'.encode(), type=StateChange.SET) ]).SerializeToString(), connection_id='test_conn_id')
def setUp(self): self.database_of_record = dict_database.DictDatabase() self.state_delta_store = StateDeltaStore(dict_database.DictDatabase()) self.context_manager = context_manager.ContextManager( self.database_of_record, self.state_delta_store) self.first_state_hash = self.context_manager.get_first_root() # used for replicating state hash through direct merkle tree updates self.database_results = dict_database.DictDatabase()
def test_publish_deltas_subscriber_matches_no_addresses(self): """Given a subscriber whose prefix filters don't match any addresses in the current state delta, it should still receive an event with the block change information. """ mock_service = Mock() block_tree_manager = BlockTreeManager() database = DictDatabase() delta_store = StateDeltaStore(database) delta_processor = StateDeltaProcessor( service=mock_service, state_delta_store=delta_store, block_store=block_tree_manager.block_store) delta_processor.add_subscriber( 'settings_conn_id', [block_tree_manager.chain_head.identifier], ['000000']) next_block = block_tree_manager.generate_block() # State added during context squash for our block delta_store.save_state_deltas( next_block.header.state_root_hash, [StateChange(address='deadbeef01', value='my_state_Value'.encode(), type=StateChange.SET), StateChange(address='a14ea01', value='some other state value'.encode(), type=StateChange.SET)]) # call to publish deltas for that block to the subscribers delta_processor.publish_deltas(next_block) mock_service.send.assert_called_with( validator_pb2.Message.STATE_DELTA_EVENT, StateDeltaEvent( block_id=next_block.identifier, block_num=next_block.header.block_num, state_root_hash=next_block.header.state_root_hash, state_changes=[] ).SerializeToString(), connection_id='settings_conn_id')
def test_get_events_ignore_bad_blocks(self): """Tests that the GetStateDeltaEventsHandler will return a response containing only the events for blocks that exists. """ block_tree_manager = BlockTreeManager() delta_store = StateDeltaStore(DictDatabase()) delta_store.save_state_deltas( block_tree_manager.chain_head.state_root_hash, [ StateChange(address='deadbeef0000000', value='my_genesis_value'.encode(), type=StateChange.SET), StateChange(address='a14ea01', value='some other state value'.encode(), type=StateChange.SET) ]) handler = GetStateDeltaEventsHandler(block_tree_manager.block_store, delta_store) request = GetStateDeltaEventsRequest( block_ids=[ block_tree_manager.chain_head.identifier, 'somebadblockid' ], address_prefixes=['deadbeef']).SerializeToString() response = handler.handle('test_conn_id', request) self.assertEqual(HandlerStatus.RETURN, response.status) self.assertEqual(GetStateDeltaEventsResponse.OK, response.message_out.status) chain_head = block_tree_manager.chain_head self.assertEqual([ StateDeltaEvent(block_id=chain_head.identifier, block_num=chain_head.block_num, state_root_hash=chain_head.state_root_hash, previous_block_id=chain_head.previous_block_id, state_changes=[ StateChange(address='deadbeef0000000', value='my_genesis_value'.encode(), type=StateChange.SET) ]) ], [event for event in response.message_out.events])
def test_is_valid_subscription_no_known_blocks(self): """Test that a check for a valid subscription with no known block ids returns True. """ mock_service = Mock() block_tree_manager = BlockTreeManager() delta_store = StateDeltaStore(DictDatabase()) delta_processor = StateDeltaProcessor( service=mock_service, state_delta_store=delta_store, block_store=block_tree_manager.block_store) self.assertTrue(delta_processor.is_valid_subscription([]))
def test_is_valid_subscription_known_chain_head(self): """Test that a check for a valid subscription with the known block id is the chain head returns True. """ mock_service = Mock() block_tree_manager = BlockTreeManager() delta_store = StateDeltaStore(DictDatabase()) delta_processor = StateDeltaProcessor( service=mock_service, state_delta_store=delta_store, block_store=block_tree_manager.block_store) self.assertTrue(delta_processor.is_valid_subscription([ block_tree_manager.chain_head.identifier]))
def test_is_valid_subscription_known_old_chain(self): """Test that a check for a valid subscription where the known block id is a block in the middle of the chain should return True. """ mock_service = Mock() block_tree_manager = BlockTreeManager() chain = block_tree_manager.generate_chain( block_tree_manager.chain_head, 5) # Grab an id from the middle of the chain known_id = chain[3].identifier block_tree_manager.block_store.update_chain(chain) delta_store = StateDeltaStore(DictDatabase()) delta_processor = StateDeltaProcessor( service=mock_service, state_delta_store=delta_store, block_store=block_tree_manager.block_store) self.assertTrue(delta_processor.is_valid_subscription([known_id]))
def test_get_events_no_valid_block_ids(self): """Tests that the GetStateDeltaEventsHandler will return a response with NO_VALID_BLOCKS_SPECIFIED error when no valid blocks are specified in the request. """ block_tree_manager = BlockTreeManager() delta_store = StateDeltaStore(DictDatabase()) handler = GetStateDeltaEventsHandler(block_tree_manager.block_store, delta_store) request = GetStateDeltaEventsRequest( block_ids=['somebadblockid'], address_prefixes=['deadbeef']).SerializeToString() response = handler.handle('test_conn_id', request) self.assertEqual(HandlerStatus.RETURN, response.status) self.assertEqual(GetStateDeltaEventsResponse.NO_VALID_BLOCKS_SPECIFIED, response.message_out.status)
class TestContextManager(unittest.TestCase): def setUp(self): self.database_of_record = dict_database.DictDatabase() self.state_delta_store = StateDeltaStore(dict_database.DictDatabase()) self.context_manager = context_manager.ContextManager( self.database_of_record, self.state_delta_store) self.first_state_hash = self.context_manager.get_first_root() # used for replicating state hash through direct merkle tree updates self.database_results = dict_database.DictDatabase() def tearDown(self): self.context_manager.stop() def _create_address(self, value=None): """ Args: value: (str) Returns: (str) sha512 of value or random """ if value is None: value = time.time().hex() return hashlib.sha512(value.encode()).hexdigest()[:70] def _setup_context(self): # 1) Create transaction data first_transaction = { 'inputs': [self._create_address(a) for a in ['aaaa', 'bbbb', 'cccc']], 'outputs': [self._create_address(a) for a in ['llaa', 'aall', 'nnnn']] } second_transaction = { 'inputs': [self._create_address(a) for a in ['aaaa', 'dddd']], 'outputs': [ self._create_address(a) for a in ['zzzz', 'yyyy', 'tttt', 'qqqq'] ] } third_transaction = { 'inputs': [self._create_address(a) for a in ['eeee', 'dddd', 'ffff']], 'outputs': [ self._create_address(a) for a in ['oooo', 'oozz', 'zzoo', 'ppoo', 'aeio'] ] } # 2) Create contexts based on that data context_id_1 = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=first_transaction['inputs'], outputs=first_transaction['outputs']) context_id_2 = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=second_transaction['inputs'], outputs=second_transaction['outputs']) context_id_3 = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=third_transaction['inputs'], outputs=third_transaction['outputs']) # 3) Set addresses with values self.context_manager.set(context_id_1, [{ self._create_address(a): v } for a, v in [('llaa', b'1'), ('aall', b'2'), ('nnnn', b'3')]]) self.context_manager.set(context_id_2, [{ self._create_address(a): v } for a, v in [('zzzz', b'9'), ('yyyy', b'11'), ('tttt', b'12'), ('qqqq', b'13')]]) self.context_manager.set(context_id_3, [{ self._create_address(a): v } for a, v in [('oooo', b'25'), ('oozz', b'26'), ( 'zzoo', b'27'), ('ppoo', b'28'), ('aeio', b'29')]]) # 4) context_id = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[context_id_1, context_id_2, context_id_3], inputs=[ self._create_address(a) for a in ['llaa', 'yyyy', 'tttt', 'zzoo'] ], outputs=[ self._create_address(a) for a in ['llaa', 'yyyy', 'tttt', 'zzoo', 'aeio'] ]) return context_id def _create_txn_inputs_outputs(self, start=None): """Create unique addresses that make up the inputs, outputs, reads, and writes that are involved in a context. Venn Diagram of relationship of disjoint sets that make up the inputs, outputs, reads, and writes. Knowledge of which disjoint set an address is a part of may give knowledge about a test failure in the context manager. Inputs Outputs +----------+--------------------------+-----------+ | | | | | i___ |Reads io__ Writes _o__ | | | | | | +-----------+-----------+---------------+ | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |i_r_ | ior_| iorw | io_w | _o_w | | | | | | | | | | | | | | | | | | | | | | | | | | | +-----------+-----------+---------------+ | | | | | | | | | +----------+--------------------------+-----------+ Args: start (int): An integer to start the sequence of integers being hashed to addresses. Returns (namedtuple): An object that holds inputs, outputs, reads, and writes. """ if start is None: start = 0 iorw = [self._create_address(str(i)) for i in range(start, start + 10)] i_r_ = [ self._create_address(str(i)) for i in range(start + 10, start + 20) ] ior_ = [ self._create_address(str(i)) for i in range(start + 20, start + 30) ] io__ = [ self._create_address(str(i)) for i in range(start + 30, start + 40) ] io_w = [ self._create_address(str(i)) for i in range(start + 40, start + 50) ] _o_w = [ self._create_address(str(i)) for i in range(start + 50, start + 60) ] _o__ = [ self._create_address(str(i)) for i in range(start + 60, start + 70) ] i___ = [ self._create_address(str(i)) for i in range(start + 70, start + 80) ] addresses = TestAddresses(inputs=iorw + ior_ + io__ + io_w + i___, outputs=ior_ + io__ + io_w + _o__ + _o_w, reads=i_r_ + ior_, writes=io_w + _o_w) return addresses def test_address_enforcement(self): """Tests that the ContextManager enforces address characteristics. Notes: 1. Call get and set on the ContextManager with an address that is under a namespace, but is an invalid address, and test that the methods raise an AuthorizationException. """ # 1) invalid_address1 = 'a' * 69 + 'n' invalid_address2 = 'b' * 69 + 'y' context_id1 = self.context_manager.create_context( state_hash=self.context_manager.get_first_root(), base_contexts=[], inputs=['aaaaaaaa', 'bbbbbbbb'], outputs=['aaaaaaaa', 'bbbbbbbb']) with self.assertRaises(context_manager.AuthorizationException): self.context_manager.get( context_id=context_id1, address_list=[invalid_address1, invalid_address2]) with self.assertRaises(context_manager.AuthorizationException): self.context_manager.set(context_id=context_id1, address_value_list=[{ invalid_address1: b'1' }, { invalid_address2: b'2' }]) def test_get_set_wrong_namespace(self): """Tests that getting and setting from outside the namespace will raise a AuthorizationException. Notes: 1. Assert that sets on a context with addresses that aren't under an output namespace raise an AuthorizationException. 2. Assert that gets on a context with addresses that aren't under an input namespace raise an AuthorizationException. """ wrong_namespace1 = self._create_address('a')[-10:] wrong_namespace2 = '00000000' ctx_1 = self.context_manager.create_context( state_hash=self.context_manager.get_first_root(), base_contexts=[], inputs=[wrong_namespace1, wrong_namespace2], outputs=[wrong_namespace1, wrong_namespace2]) # 1 with self.assertRaises(context_manager.AuthorizationException): self.context_manager.set(context_id=ctx_1, address_value_list=[{ self._create_address('a'): b'1' }]) with self.assertRaises(context_manager.AuthorizationException): self.context_manager.set(context_id=ctx_1, address_value_list=[{ self._create_address('c'): b'5' }]) # 2 with self.assertRaises(context_manager.AuthorizationException): self.context_manager.get(context_id=ctx_1, address_list=[self._create_address('a')]) with self.assertRaises(context_manager.AuthorizationException): self.context_manager.get(context_id=ctx_1, address_list=[self._create_address('c')]) def test_exception_on_invalid_input(self): """Tests that invalid inputs raise an exception. Tested with invalid characters, odd number of characters, and too long namespace; Notes: 1) Assert that inputs with a namespace with an odd number of characters raise a CreateContextException. 2) Assert that inputs with a 71 character namespace raise a CreateContextException. 3) Assert that inputs with a namespace with several invalid characters raise a CreateContextException. """ invalid_input_output1 = '0db7e8zc' # invalid character invalid_input_output2 = '7ef84ed' * 10 + '5' # too long, 71 chars invalid_input_output3 = 'yy76ftoph7465873ddde389f' # invalid chars valid_input_output1 = 'd8f533bbb74443222daad4' valid_input_output2 = '77465847465784757848ddddddf' state_hash = self.context_manager.get_first_root() # 1 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[invalid_input_output1, valid_input_output1], outputs=[valid_input_output2]) # 2 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[valid_input_output1, invalid_input_output2], outputs=[valid_input_output2]) # 3 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[invalid_input_output3, valid_input_output2], outputs=[valid_input_output2, valid_input_output1]) def test_exception_on_invalid_output(self): """Tests that invalid outputs raise an exception. Tested with invalid characters, odd number of characters, and too long namespace; Notes: 1) Assert that outputs with a namespace with an odd number of characters raise a CreateContextException. 2) Assert that outputs with a 71 character namespace raise a CreateContextException. 3) Assert that outputs with a namespace with several invalid characters raise a CreateContextException. """ invalid_input_output1 = '0db7e87' # Odd number of characters invalid_input_output2 = '7ef84ed' * 10 + '5' # too long, 71 chars invalid_input_output3 = 'yy76ftoph7465873ddde389f' # invalid chars valid_input_output1 = 'd8f533bbb74443222daad4' valid_input_output2 = '77465847465784757848ddddddff' state_hash = self.context_manager.get_first_root() # 1 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[valid_input_output2, valid_input_output1], outputs=[invalid_input_output1]) # 2 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[valid_input_output1, valid_input_output2], outputs=[invalid_input_output2]) # 3 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[valid_input_output1, valid_input_output2], outputs=[valid_input_output2, invalid_input_output3]) def test_namespace_gets(self): """Tests that gets for an address under a namespace will return the correct value. Notes: 1) Create ctx_1 and set 'b' to b'8'. 2) squash the previous context creating state_hash_1. 3) Create 2 contexts off of this state hash and assert that gets on these contexts retrieve the correct value for an address that is not fully specified in the inputs. 4) Set values to addresses in these contexts. 5) Create 1 context off of these prior 2 contexts and assert that gets from this context retrieve the correct values for addresses that are not fully specified in the inputs. 2 of the values are found in the chain of contexts, and 1 is not found and so must be retrieved from the merkle tree. """ # 1 ctx_1 = self.context_manager.create_context( state_hash=self.context_manager.get_first_root(), base_contexts=[], inputs=[self._create_address('a')], outputs=[self._create_address('b')]) self.context_manager.set(context_id=ctx_1, address_value_list=[{ self._create_address('b'): b'8' }]) # 2 squash = self.context_manager.get_squash_handler() state_hash_1 = squash(state_root=self.context_manager.get_first_root(), context_ids=[ctx_1], persist=True, clean_up=True) # 3 ctx_1a = self.context_manager.create_context( state_hash=state_hash_1, base_contexts=[], inputs=[self._create_address('a')[:10]], outputs=[self._create_address('c')]) self.assertEquals( self.context_manager.get(context_id=ctx_1a, address_list=[self._create_address('a')]), [(self._create_address('a'), None)]) ctx_1b = self.context_manager.create_context( state_hash=state_hash_1, base_contexts=[], inputs=[self._create_address('b')[:6]], outputs=[self._create_address('z')]) self.assertEquals( self.context_manager.get(context_id=ctx_1b, address_list=[self._create_address('b')]), [(self._create_address('b'), b'8')]) # 4 self.context_manager.set(context_id=ctx_1b, address_value_list=[{ self._create_address('z'): b'2' }]) self.context_manager.set(context_id=ctx_1a, address_value_list=[{ self._create_address('c'): b'1' }]) ctx_2 = self.context_manager.create_context( state_hash=state_hash_1, base_contexts=[ctx_1a, ctx_1b], inputs=[ self._create_address('z')[:10], self._create_address('c')[:10], self._create_address('b')[:10] ], outputs=[self._create_address('w')]) self.assertEquals( self.context_manager.get(context_id=ctx_2, address_list=[ self._create_address('z'), self._create_address('c'), self._create_address('b') ]), [(self._create_address('z'), b'2'), (self._create_address('c'), b'1'), (self._create_address('b'), b'8')]) def test_create_context_with_prior_state(self): """Tests context creation with prior state from base contexts. Notes: Set up the context: Create 3 prior contexts each with 3-5 addresses to set to. Make set calls to those addresses. Create 1 new context based on those three prior contexts. this test method: Test: Make a get call on addresses that are from prior state, making assertions about the correct values. """ context_id = self._setup_context() self.assertEqual( self.context_manager.get(context_id, [ self._create_address(a) for a in ['llaa', 'yyyy', 'tttt', 'zzoo'] ]), [(self._create_address(a), v) for a, v in [('llaa', b'1'), ('yyyy', b'11'), ('tttt', b'12'), ('zzoo', b'27')]]) def test_squash(self): """Tests that squashing a context based on state from other contexts will result in the same merkle hash as updating the merkle tree with the same data. Notes: Set up the context Test: 1) Make set calls on several of the addresses. 2) Squash the context to get a new state hash. 3) Apply all of the aggregate sets from all of the contexts, to another database with a merkle tree. 4) Assert that the state hashes are the same. 5) Assert that the state deltas have been stored """ # 1) context_id = self._setup_context() self.context_manager.set(context_id, [{ self._create_address(a): v } for a, v in [('yyyy', b'2'), ('tttt', b'4')]]) # 2) squash = self.context_manager.get_squash_handler() resulting_state_hash = squash(self.first_state_hash, [context_id], persist=True, clean_up=True) # 3) final_state_to_update = { self._create_address(a): v for a, v in [('llaa', b'1'), ('aall', b'2'), ( 'nnnn', b'3'), ('zzzz', b'9'), ('yyyy', b'2'), ( 'tttt', b'4'), ('qqqq', b'13'), ('oooo', b'25'), ( 'oozz', b'26'), ('zzoo', b'27'), ('ppoo', b'28'), ('aeio', b'29')] } test_merkle_tree = MerkleDatabase(self.database_results) test_resulting_state_hash = test_merkle_tree.update( final_state_to_update, virtual=False) # 4) self.assertEqual(resulting_state_hash, test_resulting_state_hash) state_changes = self.state_delta_store.get_state_deltas( resulting_state_hash) # 5) for addr, value in final_state_to_update.items(): expected_state_change = StateChange(address=addr, value=value, type=StateChange.SET) self.assertTrue(expected_state_change in state_changes) def test_squash_no_updates(self): """Tests that squashing a context that has no state updates will return the starting state root hash. Notes: Set up the context Test: 1) Squash the context. 2) Assert that the state hash is the same as the starting hash. 3) Assert that the state deltas have not been overwritten """ self.state_delta_store.save_state_deltas( self.first_state_hash, [StateChange(address='aaa', value=b'xyz', type=StateChange.SET)]) context_id = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=[], outputs=[]) # 1) squash = self.context_manager.get_squash_handler() resulting_state_hash = squash(self.first_state_hash, [context_id], persist=True, clean_up=True) # 2 self.assertIsNotNone(resulting_state_hash) self.assertEquals(resulting_state_hash, self.first_state_hash) # 3 changes = self.state_delta_store.get_state_deltas(resulting_state_hash) self.assertEqual( [StateChange(address='aaa', value=b'xyz', type=StateChange.SET)], [c for c in changes]) def test_reads_from_context_w_several_writes(self): """Tests that those context values that have been written to the Merkle tree, or that have been set to a base_context, will have the correct value at the address for a given context. ->context_id_a1 | | | | | | sh0-->context_id1-->sh1-->context_a----- -->context_id_b | | | | | | | | -->context_id_a2 Notes: Test: 1. From a Merkle Tree with only the root node in it, create a context and set several values, and then squash that context upon the first state hash. 2. Create a context with no base context, based on the merkle root computed from the first squash. Assert that gets from this context will provide values that were set in the first context. 3. Write to all of the available outputs 4. Create a new context based on context_a, from #2, 5. Assert that gets from this context equal the values set to Context A. 6. Create a new context based on context_a and set values to this context. 7. Create a new context based on the 2 contexts made in 4 and 6 8. From this context assert that gets equal the correct values set in the prior contexts. """ squash = self.context_manager.get_squash_handler() test_addresses = self._create_txn_inputs_outputs() # 1) context_id1 = self.context_manager.create_context( state_hash=self.first_state_hash, inputs=test_addresses.inputs, outputs=test_addresses.outputs, base_contexts=[]) values1 = [bytes(i) for i in range(len(test_addresses.writes))] self.context_manager.set(context_id1, [{ a: v } for a, v in zip(test_addresses.writes, values1)]) sh1 = squash(state_root=self.first_state_hash, context_ids=[context_id1], persist=True, clean_up=True) # 2) context_a = self.context_manager.create_context( state_hash=sh1, inputs=test_addresses.writes, # read from every address written to outputs=test_addresses.outputs, base_contexts=[]) address_values = self.context_manager.get(context_a, list(test_addresses.writes)) self.assertEquals(address_values, [(a, v) for a, v in zip(test_addresses.writes, values1)]) # 3) values2 = [bytes(v.encode()) for v in test_addresses.outputs] self.context_manager.set( context_id=context_a, address_value_list=[{ a: v } for a, v in zip(test_addresses.outputs, values2)]) # 4) context_id_a1 = self.context_manager.create_context( state_hash=sh1, inputs=test_addresses.outputs, outputs=test_addresses.outputs, base_contexts=[context_a]) # 5) c_ida1_address_values = self.context_manager.get( context_id=context_id_a1, address_list=list(test_addresses.outputs)) self.assertEquals(c_ida1_address_values, [(a, v) for a, v in zip(test_addresses.outputs, values2)]) # 6) test_addresses2 = self._create_txn_inputs_outputs(80) context_id_a2 = self.context_manager.create_context( state_hash=sh1, inputs=test_addresses2.inputs, outputs=test_addresses2.outputs, base_contexts=[context_a]) values3 = [bytes(v.encode()) for v in test_addresses2.writes] self.context_manager.set( context_id=context_id_a2, address_value_list=[{ a: v } for a, v in zip(test_addresses2.writes, values3)], ) # 7) context_id_b = self.context_manager.create_context( state_hash=sh1, inputs=test_addresses2.writes + test_addresses.outputs, outputs=[], base_contexts=[context_id_a1, context_id_a2]) # 8) self.assertEquals( self.context_manager.get( context_id_b, list(test_addresses2.writes + test_addresses.outputs)), [(a, v) for a, v in zip(test_addresses2.writes + test_addresses.outputs, values3 + values2)]) def test_state_root_after_parallel_ctx(self): """Tests that the correct state root is calculated after basing one context off of multiple contexts. i=abcd o=aaaa +>context_1+ | aaaa=1 | | | i=llll | i=bacd | i=bbbb,aaaa o=llll | o=bbbb | o=cccc,llll sh0--->ctx_0-->sh1>|-->context_2-+---->context_n---->sh2 llll=5 | bbbb=2 | cccc=4 | | llll=8 | i=abcd | | o=cccc | +>context_3+ cccc=3 Notes: Test: 1. Create a context, set a value in it and squash it into a new state hash. 2. Create 3 contexts based off of the state root from #1. 3. Set values at addresses to all three contexts. 4. Base another context off of the contexts from #2. 5. Set a value to an address in this context that has already been set to in the non-base context. 6. Squash the contexts producing a state hash and assert that it equals a state hash obtained by manually updating the merkle tree. """ sh0 = self.first_state_hash # 1) squash = self.context_manager.get_squash_handler() ctx_1 = self.context_manager.create_context( state_hash=sh0, base_contexts=[], inputs=[self._create_address('llll')], outputs=[self._create_address('llll')]) self.context_manager.set(context_id=ctx_1, address_value_list=[{ self._create_address('llll'): b'5' }]) sh1 = squash(state_root=sh0, context_ids=[ctx_1], persist=True, clean_up=True) # 2) context_1 = self.context_manager.create_context( state_hash=sh1, base_contexts=[], inputs=[self._create_address('abcd')], outputs=[self._create_address('aaaa')]) context_2 = self.context_manager.create_context( state_hash=sh1, base_contexts=[], inputs=[self._create_address('bacd')], outputs=[self._create_address('bbbb')]) context_3 = self.context_manager.create_context( state_hash=sh1, base_contexts=[], inputs=[self._create_address('abcd')], outputs=[ self._create_address('cccc'), self._create_address('dddd') ]) # 3) self.context_manager.set(context_id=context_1, address_value_list=[{ self._create_address('aaaa'): b'1' }]) self.context_manager.set(context_id=context_2, address_value_list=[{ self._create_address('bbbb'): b'2' }]) self.context_manager.set(context_id=context_3, address_value_list=[{ self._create_address('cccc'): b'3' }]) # 4) context_n = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_1, context_2, context_3], inputs=[ self._create_address('bbbb'), self._create_address('aaaa') ], outputs=[ self._create_address('cccc'), self._create_address('llll') ]) # 5) self.context_manager.set(context_id=context_n, address_value_list=[{ self._create_address('cccc'): b'4', self._create_address('llll'): b'8' }]) # 6) cm_state_root = squash(state_root=sh1, context_ids=[context_n], persist=False, clean_up=True) tree = MerkleDatabase(self.database_results) calc_state_root = tree.update({ self._create_address('aaaa'): b'1', self._create_address('bbbb'): b'2', self._create_address('cccc'): b'4', self._create_address('llll'): b'8' }) self.assertEquals(calc_state_root, cm_state_root) def test_complex_basecontext_squash(self): """Tests complex context basing and squashing. i=qq,dd dd=0 o=dd,pp pp=1 i=cc,aa +->context_3_2a_1+| o=dd,ll | | i=aa,ab +->context_2a| i=aa aa=0 | o=cc,ab | dd=10 | o=aa,ll ll=1 | sh0->context_1-->sh1| ll=11 +->context_3_2a_2+|->sh1 cc=0 | i=cc,aa +->context_3_2b_1+| ab=1 | o=nn,mm | i=nn,ba mm=0 | +->context_2b| o=mm,ba ba=1 | nn=0 | | mm=1 +->context_3_2b_2+| i=nn,oo ab=0 o=ab,oo oo=1 Notes: Test: 1. Create a context off of the first state hash, set addresses in it, and squash that context, getting a new merkle root. 2. Create 2 contexts with the context in # 1 as the base, and for each of these contexts set addresses to values where the outputs for each are disjoint. 3. For each of these 2 contexts create 2 more contexts each having one of the contexts in # 2 as the base context, and set addresses to values. 4. Squash the 4 contexts from #3 and assert the state hash is equal to a manually computed state hash. """ squash = self.context_manager.get_squash_handler() # 1) inputs_1 = [self._create_address('aa'), self._create_address('ab')] outputs_1 = [self._create_address('cc'), self._create_address('ab')] context_1 = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=inputs_1, outputs=outputs_1) self.context_manager.set( context_id=context_1, address_value_list=[{ a: v } for a, v in zip(outputs_1, [bytes(i) for i in range(len(outputs_1))])]) sh1 = squash(state_root=self.first_state_hash, context_ids=[context_1], persist=True, clean_up=True) # 2) inputs_2a = [self._create_address('cc'), self._create_address('aa')] outputs_2a = [self._create_address('dd'), self._create_address('ll')] context_2a = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=inputs_2a, outputs=outputs_2a) inputs_2b = [self._create_address('cc'), self._create_address('aa')] outputs_2b = [self._create_address('nn'), self._create_address('mm')] context_2b = self.context_manager.create_context(state_hash=sh1, base_contexts=[], inputs=inputs_2b, outputs=outputs_2b) self.context_manager.set( context_id=context_2a, address_value_list=[{ a: bytes(v) } for a, v in zip(outputs_2a, range(10, 10 + len(outputs_2a)))]) self.context_manager.set( context_id=context_2b, address_value_list=[{ a: bytes(v) } for a, v in zip(outputs_2b, range(len(outputs_2b)))]) # 3) inputs_3_2a_1 = [ self._create_address('qq'), self._create_address('dd') ] outputs_3_2a_1 = [ self._create_address('dd'), self._create_address('pp') ] context_3_2a_1 = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_2a], inputs=inputs_3_2a_1, outputs=outputs_3_2a_1) inputs_3_2a_2 = [self._create_address('aa')] outputs_3_2a_2 = [ self._create_address('aa'), self._create_address('ll') ] context_3_2a_2 = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_2a], inputs=inputs_3_2a_2, outputs=outputs_3_2a_2) inputs_3_2b_1 = [ self._create_address('nn'), self._create_address('ab') ] outputs_3_2b_1 = [ self._create_address('mm'), self._create_address('ba') ] context_3_2b_1 = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_2b], inputs=inputs_3_2b_1, outputs=outputs_3_2b_1) inputs_3_2b_2 = [ self._create_address('nn'), self._create_address('oo') ] outputs_3_2b_2 = [ self._create_address('ab'), self._create_address('oo') ] context_3_2b_2 = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_2b], inputs=inputs_3_2b_2, outputs=outputs_3_2b_2) self.context_manager.set( context_id=context_3_2a_1, address_value_list=[{ a: bytes(v) } for a, v in zip(outputs_3_2a_1, range(len(outputs_3_2a_1)))]) self.context_manager.set( context_id=context_3_2a_2, address_value_list=[{ a: bytes(v) } for a, v in zip(outputs_3_2a_2, range(len(outputs_3_2a_2)))]) self.context_manager.set( context_id=context_3_2b_1, address_value_list=[{ a: bytes(v) } for a, v in zip(outputs_3_2b_1, range(len(outputs_3_2b_1)))]) self.context_manager.set( context_id=context_3_2b_2, address_value_list=[{ a: bytes(v) } for a, v in zip(outputs_3_2b_2, range(len(outputs_3_2b_2)))]) # 4) sh2 = squash(state_root=sh1, context_ids=[ context_3_2a_1, context_3_2a_2, context_3_2b_1, context_3_2b_2 ], persist=False, clean_up=True) tree = MerkleDatabase(self.database_results) state_hash_from_1 = tree.update(set_items={ a: v for a, v in zip(outputs_1, [bytes(i) for i in range(len(outputs_1))]) }, virtual=False) self.assertEquals( state_hash_from_1, sh1, "The manually calculated state hash from the first " "context and the one calculated by squashing that " "state hash should be the same") tree.set_merkle_root(state_hash_from_1) test_sh2 = tree.update( set_items={ self._create_address('aa'): bytes(0), self._create_address('ab'): bytes(0), self._create_address('ba'): bytes(1), self._create_address('dd'): bytes(0), self._create_address('ll'): bytes(1), self._create_address('mm'): bytes(0), self._create_address('oo'): bytes(1), self._create_address('pp'): bytes(1), self._create_address('nn'): bytes(0), self._create_address('cc'): bytes(0) }) self.assertEquals( sh2, test_sh2, "Manually calculated and context " "manager calculated merkle hashes " "are the same") @unittest.skip("Necessary to catch scheduler bugs--Depth-first search") def test_check_for_bad_combination(self): """Tests that the context manager will raise an exception if asked to combine contexts, either via base contexts in create_context or via squash that shouldn't be combined because they share addresses that can't be determined by the scheduler to not have been parallel. This is a check on scheduler bugs. Examples where the context manager should raise an exception on duplicate addresses: 1. Success i=a o=b +>ctx_1+ dup|->b=3 | | | sh0| ----->state hash or context | i=q | | o=b | +>ctx_2+ dup-->b=2 2. i=b o=d +>ctx_1a_1+ | d=4 | i=a | | o=b | | +>ctx_1a| | | b=2 | i=b | | | o=c | sh0| +>ctx_1a_2| | dup-> c=7 |------>state hash or context | i=a +>ctx_1b_1| | o=c | | +>ctx_1b| | dup-->c=5 | i=t | | o=p | +>ctx_1b_2+ p=8 3. i=b o=d +>ctx_1a_1+ | d=4 | i=d,c i=a | | o=n o=b | <>ctx_3a+ +>ctx_1a| | n=5 | | b=2 | i=b | | | | o=c | | sh0| +>ctx_1a_2+ <----->state hash or context | dup--> c=7 | | i=a +>ctx_1b_1+ | | o=c | | i=c | +>ctx_1b| | o=q | c=5 | i=c <>ctx_3b+ | o=c | q=5 +>ctx_1b_2+ dup--> c=1 """ # 1. squash = self.context_manager.get_squash_handler() sh0 = self.first_state_hash inputs_1 = [self._create_address('a')] outputs_1 = [self._create_address('b')] ctx_1 = self.context_manager.create_context(state_hash=sh0, base_contexts=[], inputs=inputs_1, outputs=outputs_1) self.context_manager.set(context_id=ctx_1, address_value_list=[{ self._create_address('b'): b'3' }]) inputs_2 = [self._create_address('q')] outputs_2 = [self._create_address('b')] ctx_2 = self.context_manager.create_context(state_hash=sh0, base_contexts=[], inputs=inputs_2, outputs=outputs_2) self.context_manager.set(context_id=ctx_2, address_value_list=[{ self._create_address('b'): b'2' }]) try: sh1 = squash(state_root=sh0, context_ids=[ctx_1, ctx_2], persist=True, clean_up=True) self.fail("squash of two contexts with a duplicate address") except Exception: pass
class TestContextManager(unittest.TestCase): def setUp(self): self.database_of_record = dict_database.DictDatabase() self.state_delta_store = StateDeltaStore(dict_database.DictDatabase()) self.context_manager = context_manager.ContextManager( self.database_of_record, self.state_delta_store) self.first_state_hash = self.context_manager.get_first_root() # used for replicating state hash through direct merkle tree updates self.database_results = dict_database.DictDatabase() def tearDown(self): self.context_manager.stop() def _create_address(self, value=None): """ Args: value: (str) Returns: (str) sha512 of value or random """ if value is None: value = time.time().hex() return hashlib.sha512(value.encode()).hexdigest()[:70] def _setup_context(self): # 1) Create transaction data first_transaction = {'inputs': [self._create_address(a) for a in ['aaaa', 'bbbb', 'cccc']], 'outputs': [self._create_address(a) for a in ['llaa', 'aall', 'nnnn']]} second_transaction = { 'inputs': [self._create_address(a) for a in ['aaaa', 'dddd']], 'outputs': [self._create_address(a) for a in ['zzzz', 'yyyy', 'tttt', 'qqqq']] } third_transaction = { 'inputs': [self._create_address(a) for a in ['eeee', 'dddd', 'ffff']], 'outputs': [self._create_address(a) for a in ['oooo', 'oozz', 'zzoo', 'ppoo', 'aeio']] } # 2) Create contexts based on that data context_id_1 = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=first_transaction['inputs'], outputs=first_transaction['outputs']) context_id_2 = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=second_transaction['inputs'], outputs=second_transaction['outputs']) context_id_3 = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=third_transaction['inputs'], outputs=third_transaction['outputs']) # 3) Set addresses with values self.context_manager.set(context_id_1, [{self._create_address(a): v} for a, v in [('llaa', b'1'), ('aall', b'2'), ('nnnn', b'3')]]) self.context_manager.set(context_id_2, [{self._create_address(a): v} for a, v in [('zzzz', b'9'), ('yyyy', b'11'), ('tttt', b'12'), ('qqqq', b'13')]]) self.context_manager.set(context_id_3, [{self._create_address(a): v} for a, v in [('oooo', b'25'), ('oozz', b'26'), ('zzoo', b'27'), ('ppoo', b'28'), ('aeio', b'29')]]) # 4) context_id = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[context_id_1, context_id_2, context_id_3], inputs=[self._create_address(a) for a in ['llaa', 'yyyy', 'tttt', 'zzoo']], outputs=[self._create_address(a) for a in ['llaa', 'yyyy', 'tttt', 'zzoo', 'aeio']]) return context_id def _create_txn_inputs_outputs(self, start=None): """Create unique addresses that make up the inputs, outputs, reads, and writes that are involved in a context. Venn Diagram of relationship of disjoint sets that make up the inputs, outputs, reads, and writes. Knowledge of which disjoint set an address is a part of may give knowledge about a test failure in the context manager. Inputs Outputs +----------+--------------------------+-----------+ | | | | | i___ |Reads io__ Writes _o__ | | | | | | +-----------+-----------+---------------+ | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | | |i_r_ | ior_| iorw | io_w | _o_w | | | | | | | | | | | | | | | | | | | | | | | | | | | +-----------+-----------+---------------+ | | | | | | | | | +----------+--------------------------+-----------+ Args: start (int): An integer to start the sequence of integers being hashed to addresses. Returns (namedtuple): An object that holds inputs, outputs, reads, and writes. """ if start is None: start = 0 iorw = [self._create_address(str(i)) for i in range(start, start + 10)] i_r_ = [self._create_address(str(i)) for i in range(start + 10, start + 20)] ior_ = [self._create_address(str(i)) for i in range(start + 20, start + 30)] io__ = [self._create_address(str(i)) for i in range(start + 30, start + 40)] io_w = [self._create_address(str(i)) for i in range(start + 40, start + 50)] _o_w = [self._create_address(str(i)) for i in range(start + 50, start + 60)] _o__ = [self._create_address(str(i)) for i in range(start + 60, start + 70)] i___ = [self._create_address(str(i)) for i in range(start + 70, start + 80)] addresses = TestAddresses( inputs=iorw + ior_ + io__ + io_w + i___, outputs=ior_ + io__ + io_w + _o__ + _o_w, reads=i_r_ + ior_, writes=io_w + _o_w ) return addresses def test_address_enforcement(self): """Tests that the ContextManager enforces address characteristics. Notes: 1. Call get and set on the ContextManager with an address that is under a namespace, but is an invalid address, and test that the methods raise an AuthorizationException. """ # 1) invalid_address1 = 'a' * 69 + 'n' invalid_address2 = 'b' * 69 + 'y' context_id1 = self.context_manager.create_context( state_hash=self.context_manager.get_first_root(), base_contexts=[], inputs=['aaaaaaaa', 'bbbbbbbb'], outputs=['aaaaaaaa', 'bbbbbbbb']) with self.assertRaises(context_manager.AuthorizationException): self.context_manager.get( context_id=context_id1, address_list=[invalid_address1, invalid_address2]) with self.assertRaises(context_manager.AuthorizationException): self.context_manager.set( context_id=context_id1, address_value_list=[{invalid_address1: b'1'}, {invalid_address2: b'2'}]) def test_get_set_wrong_namespace(self): """Tests that getting and setting from outside the namespace will raise a AuthorizationException. Notes: 1. Assert that sets on a context with addresses that aren't under an output namespace raise an AuthorizationException. 2. Assert that gets on a context with addresses that aren't under an input namespace raise an AuthorizationException. """ wrong_namespace1 = self._create_address('a')[-10:] wrong_namespace2 = '00000000' ctx_1 = self.context_manager.create_context( state_hash=self.context_manager.get_first_root(), base_contexts=[], inputs=[wrong_namespace1, wrong_namespace2], outputs=[wrong_namespace1, wrong_namespace2]) # 1 with self.assertRaises(context_manager.AuthorizationException): self.context_manager.set( context_id=ctx_1, address_value_list=[{self._create_address('a'): b'1'}]) with self.assertRaises(context_manager.AuthorizationException): self.context_manager.set( context_id=ctx_1, address_value_list=[{self._create_address('c'): b'5'}]) # 2 with self.assertRaises(context_manager.AuthorizationException): self.context_manager.get( context_id=ctx_1, address_list=[self._create_address('a')]) with self.assertRaises(context_manager.AuthorizationException): self.context_manager.get( context_id=ctx_1, address_list=[self._create_address('c')]) def test_exception_on_invalid_input(self): """Tests that invalid inputs raise an exception. Tested with invalid characters, odd number of characters, and too long namespace; Notes: 1) Assert that inputs with a namespace with an odd number of characters raise a CreateContextException. 2) Assert that inputs with a 71 character namespace raise a CreateContextException. 3) Assert that inputs with a namespace with several invalid characters raise a CreateContextException. """ invalid_input_output1 = '0db7e8zc' # invalid character invalid_input_output2 = '7ef84ed' * 10 + '5' # too long, 71 chars invalid_input_output3 = 'yy76ftoph7465873ddde389f' # invalid chars valid_input_output1 = 'd8f533bbb74443222daad4' valid_input_output2 = '77465847465784757848ddddddf' state_hash = self.context_manager.get_first_root() # 1 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[invalid_input_output1, valid_input_output1], outputs=[valid_input_output2]) # 2 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[valid_input_output1, invalid_input_output2], outputs=[valid_input_output2]) # 3 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[invalid_input_output3, valid_input_output2], outputs=[valid_input_output2, valid_input_output1]) def test_exception_on_invalid_output(self): """Tests that invalid outputs raise an exception. Tested with invalid characters, odd number of characters, and too long namespace; Notes: 1) Assert that outputs with a namespace with an odd number of characters raise a CreateContextException. 2) Assert that outputs with a 71 character namespace raise a CreateContextException. 3) Assert that outputs with a namespace with several invalid characters raise a CreateContextException. """ invalid_input_output1 = '0db7e87' # Odd number of characters invalid_input_output2 = '7ef84ed' * 10 + '5' # too long, 71 chars invalid_input_output3 = 'yy76ftoph7465873ddde389f' # invalid chars valid_input_output1 = 'd8f533bbb74443222daad4' valid_input_output2 = '77465847465784757848ddddddff' state_hash = self.context_manager.get_first_root() # 1 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[valid_input_output2, valid_input_output1], outputs=[invalid_input_output1]) # 2 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[valid_input_output1, valid_input_output2], outputs=[invalid_input_output2]) # 3 with self.assertRaises(context_manager.CreateContextException): self.context_manager.create_context( state_hash=state_hash, base_contexts=[], inputs=[valid_input_output1, valid_input_output2], outputs=[valid_input_output2, invalid_input_output3]) def test_namespace_gets(self): """Tests that gets for an address under a namespace will return the correct value. Notes: 1) Create ctx_1 and set 'b' to b'8'. 2) squash the previous context creating state_hash_1. 3) Create 2 contexts off of this state hash and assert that gets on these contexts retrieve the correct value for an address that is not fully specified in the inputs. 4) Set values to addresses in these contexts. 5) Create 1 context off of these prior 2 contexts and assert that gets from this context retrieve the correct values for addresses that are not fully specified in the inputs. 2 of the values are found in the chain of contexts, and 1 is not found and so must be retrieved from the merkle tree. """ # 1 ctx_1 = self.context_manager.create_context( state_hash=self.context_manager.get_first_root(), base_contexts=[], inputs=[self._create_address('a')], outputs=[self._create_address('b')]) self.context_manager.set( context_id=ctx_1, address_value_list=[{self._create_address('b'): b'8'}]) # 2 squash = self.context_manager.get_squash_handler() state_hash_1 = squash( state_root=self.context_manager.get_first_root(), context_ids=[ctx_1], persist=True, clean_up=True) # 3 ctx_1a = self.context_manager.create_context( state_hash=state_hash_1, base_contexts=[], inputs=[self._create_address('a')[:10]], outputs=[self._create_address('c')]) self.assertEquals( self.context_manager.get( context_id=ctx_1a, address_list=[self._create_address('a')]), [(self._create_address('a'), None)]) ctx_1b = self.context_manager.create_context( state_hash=state_hash_1, base_contexts=[], inputs=[self._create_address('b')[:6]], outputs=[self._create_address('z')]) self.assertEquals( self.context_manager.get( context_id=ctx_1b, address_list=[self._create_address('b')]), [(self._create_address('b'), b'8')]) # 4 self.context_manager.set( context_id=ctx_1b, address_value_list=[{self._create_address('z'): b'2'}]) self.context_manager.set( context_id=ctx_1a, address_value_list=[{self._create_address('c'): b'1'}] ) ctx_2 = self.context_manager.create_context( state_hash=state_hash_1, base_contexts=[ctx_1a, ctx_1b], inputs=[self._create_address('z')[:10], self._create_address('c')[:10], self._create_address('b')[:10]], outputs=[self._create_address('w')]) self.assertEquals( self.context_manager.get( context_id=ctx_2, address_list=[self._create_address('z'), self._create_address('c'), self._create_address('b')]), [(self._create_address('z'), b'2'), (self._create_address('c'), b'1'), (self._create_address('b'), b'8')]) def test_create_context_with_prior_state(self): """Tests context creation with prior state from base contexts. Notes: Set up the context: Create 3 prior contexts each with 3-5 addresses to set to. Make set calls to those addresses. Create 1 new context based on those three prior contexts. this test method: Test: Make a get call on addresses that are from prior state, making assertions about the correct values. """ context_id = self._setup_context() self.assertEqual(self.context_manager.get( context_id, [self._create_address(a) for a in ['llaa', 'yyyy', 'tttt', 'zzoo']]), [(self._create_address(a), v) for a, v in [('llaa', b'1'), ('yyyy', b'11'), ('tttt', b'12'), ('zzoo', b'27')]]) def test_squash(self): """Tests that squashing a context based on state from other contexts will result in the same merkle hash as updating the merkle tree with the same data. Notes: Set up the context Test: 1) Make set calls on several of the addresses. 2) Squash the context to get a new state hash. 3) Apply all of the aggregate sets from all of the contexts, to another database with a merkle tree. 4) Assert that the state hashes are the same. 5) Assert that the state deltas have been stored """ # 1) context_id = self._setup_context() self.context_manager.set( context_id, [{self._create_address(a): v} for a, v in [('yyyy', b'2'), ('tttt', b'4')]]) # 2) squash = self.context_manager.get_squash_handler() resulting_state_hash = squash(self.first_state_hash, [context_id], persist=True, clean_up=True) # 3) final_state_to_update = {self._create_address(a): v for a, v in [('llaa', b'1'), ('aall', b'2'), ('nnnn', b'3'), ('zzzz', b'9'), ('yyyy', b'2'), ('tttt', b'4'), ('qqqq', b'13'), ('oooo', b'25'), ('oozz', b'26'), ('zzoo', b'27'), ('ppoo', b'28'), ('aeio', b'29')]} test_merkle_tree = MerkleDatabase(self.database_results) test_resulting_state_hash = test_merkle_tree.update( final_state_to_update, virtual=False) # 4) self.assertEqual(resulting_state_hash, test_resulting_state_hash) state_changes = self.state_delta_store.get_state_deltas( resulting_state_hash) # 5) for addr, value in final_state_to_update.items(): expected_state_change = StateChange( address=addr, value=value, type=StateChange.SET) self.assertTrue(expected_state_change in state_changes) def test_squash_no_updates(self): """Tests that squashing a context that has no state updates will return the starting state root hash. Notes: Set up the context Test: 1) Squash the context. 2) Assert that the state hash is the same as the starting hash. 3) Assert that the state deltas have not been overwritten """ self.state_delta_store.save_state_deltas( self.first_state_hash, [StateChange(address='aaa', value=b'xyz', type=StateChange.SET)]) context_id = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=[], outputs=[]) # 1) squash = self.context_manager.get_squash_handler() resulting_state_hash = squash(self.first_state_hash, [context_id], persist=True, clean_up=True) # 2 self.assertIsNotNone(resulting_state_hash) self.assertEquals(resulting_state_hash, self.first_state_hash) # 3 changes = self.state_delta_store.get_state_deltas(resulting_state_hash) self.assertEqual( [StateChange(address='aaa', value=b'xyz', type=StateChange.SET)], [c for c in changes]) def test_reads_from_context_w_several_writes(self): """Tests that those context values that have been written to the Merkle tree, or that have been set to a base_context, will have the correct value at the address for a given context. ->context_id_a1 | | | | | | sh0-->context_id1-->sh1-->context_a----- -->context_id_b | | | | | | | | -->context_id_a2 Notes: Test: 1. From a Merkle Tree with only the root node in it, create a context and set several values, and then squash that context upon the first state hash. 2. Create a context with no base context, based on the merkle root computed from the first squash. Assert that gets from this context will provide values that were set in the first context. 3. Write to all of the available outputs 4. Create a new context based on context_a, from #2, 5. Assert that gets from this context equal the values set to Context A. 6. Create a new context based on context_a and set values to this context. 7. Create a new context based on the 2 contexts made in 4 and 6 8. From this context assert that gets equal the correct values set in the prior contexts. """ squash = self.context_manager.get_squash_handler() test_addresses = self._create_txn_inputs_outputs() # 1) context_id1 = self.context_manager.create_context( state_hash=self.first_state_hash, inputs=test_addresses.inputs, outputs=test_addresses.outputs, base_contexts=[]) values1 = [bytes(i) for i in range(len(test_addresses.writes))] self.context_manager.set( context_id1, [{a: v} for a, v in zip(test_addresses.writes, values1)]) sh1 = squash( state_root=self.first_state_hash, context_ids=[context_id1], persist=True, clean_up=True) # 2) context_a = self.context_manager.create_context( state_hash=sh1, inputs=test_addresses.writes, # read from every address written to outputs=test_addresses.outputs, base_contexts=[] ) address_values = self.context_manager.get( context_a, list(test_addresses.writes) ) self.assertEquals( address_values, [(a, v) for a, v in zip(test_addresses.writes, values1)] ) # 3) values2 = [bytes(v.encode()) for v in test_addresses.outputs] self.context_manager.set( context_id=context_a, address_value_list=[{a: v} for a, v in zip(test_addresses.outputs, values2)]) # 4) context_id_a1 = self.context_manager.create_context( state_hash=sh1, inputs=test_addresses.outputs, outputs=test_addresses.outputs, base_contexts=[context_a] ) # 5) c_ida1_address_values = self.context_manager.get( context_id=context_id_a1, address_list=list(test_addresses.outputs) ) self.assertEquals( c_ida1_address_values, [(a, v) for a, v in zip(test_addresses.outputs, values2)] ) # 6) test_addresses2 = self._create_txn_inputs_outputs(80) context_id_a2 = self.context_manager.create_context( state_hash=sh1, inputs=test_addresses2.inputs, outputs=test_addresses2.outputs, base_contexts=[context_a] ) values3 = [bytes(v.encode()) for v in test_addresses2.writes] self.context_manager.set( context_id=context_id_a2, address_value_list=[{a: v} for a, v in zip(test_addresses2.writes, values3)], ) # 7) context_id_b = self.context_manager.create_context( state_hash=sh1, inputs=test_addresses2.writes + test_addresses.outputs, outputs=[], base_contexts=[context_id_a1, context_id_a2] ) # 8) self.assertEquals( self.context_manager.get( context_id_b, list(test_addresses2.writes + test_addresses.outputs) ), [(a, v) for a, v in zip( test_addresses2.writes + test_addresses.outputs, values3 + values2)] ) def test_state_root_after_parallel_ctx(self): """Tests that the correct state root is calculated after basing one context off of multiple contexts. i=abcd o=aaaa +>context_1+ | aaaa=1 | | | i=llll | i=bacd | i=bbbb,aaaa o=llll | o=bbbb | o=cccc,llll sh0--->ctx_0-->sh1>|-->context_2-+---->context_n---->sh2 llll=5 | bbbb=2 | cccc=4 | | llll=8 | i=abcd | | o=cccc | +>context_3+ cccc=3 Notes: Test: 1. Create a context, set a value in it and squash it into a new state hash. 2. Create 3 contexts based off of the state root from #1. 3. Set values at addresses to all three contexts. 4. Base another context off of the contexts from #2. 5. Set a value to an address in this context that has already been set to in the non-base context. 6. Squash the contexts producing a state hash and assert that it equals a state hash obtained by manually updating the merkle tree. """ sh0 = self.first_state_hash # 1) squash = self.context_manager.get_squash_handler() ctx_1 = self.context_manager.create_context( state_hash=sh0, base_contexts=[], inputs=[self._create_address('llll')], outputs=[self._create_address('llll')] ) self.context_manager.set( context_id=ctx_1, address_value_list=[{self._create_address('llll'): b'5'}] ) sh1 = squash( state_root=sh0, context_ids=[ctx_1], persist=True, clean_up=True) # 2) context_1 = self.context_manager.create_context( state_hash=sh1, base_contexts=[], inputs=[self._create_address('abcd')], outputs=[self._create_address('aaaa')] ) context_2 = self.context_manager.create_context( state_hash=sh1, base_contexts=[], inputs=[self._create_address('bacd')], outputs=[self._create_address('bbbb')] ) context_3 = self.context_manager.create_context( state_hash=sh1, base_contexts=[], inputs=[self._create_address('abcd')], outputs=[self._create_address('cccc'), self._create_address('dddd')] ) # 3) self.context_manager.set( context_id=context_1, address_value_list=[{self._create_address('aaaa'): b'1'}] ) self.context_manager.set( context_id=context_2, address_value_list=[{self._create_address('bbbb'): b'2'}] ) self.context_manager.set( context_id=context_3, address_value_list=[{self._create_address('cccc'): b'3'}] ) # 4) context_n = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_1, context_2, context_3], inputs=[self._create_address('bbbb'), self._create_address('aaaa')], outputs=[self._create_address('cccc'), self._create_address('llll')] ) # 5) self.context_manager.set( context_id=context_n, address_value_list=[{self._create_address('cccc'): b'4', self._create_address('llll'): b'8'}] ) # 6) cm_state_root = squash( state_root=sh1, context_ids=[context_n], persist=False, clean_up=True) tree = MerkleDatabase(self.database_results) calc_state_root = tree.update({self._create_address('aaaa'): b'1', self._create_address('bbbb'): b'2', self._create_address('cccc'): b'4', self._create_address('llll'): b'8'}) self.assertEquals(calc_state_root, cm_state_root) def test_complex_basecontext_squash(self): """Tests complex context basing and squashing. i=qq,dd dd=0 o=dd,pp pp=1 i=cc,aa +->context_3_2a_1+| o=dd,ll | | i=aa,ab +->context_2a| i=aa aa=0 | o=cc,ab | dd=10 | o=aa,ll ll=1 | sh0->context_1-->sh1| ll=11 +->context_3_2a_2+|->sh1 cc=0 | i=cc,aa +->context_3_2b_1+| ab=1 | o=nn,mm | i=nn,ba mm=0 | +->context_2b| o=mm,ba ba=1 | nn=0 | | mm=1 +->context_3_2b_2+| i=nn,oo ab=0 o=ab,oo oo=1 Notes: Test: 1. Create a context off of the first state hash, set addresses in it, and squash that context, getting a new merkle root. 2. Create 2 contexts with the context in # 1 as the base, and for each of these contexts set addresses to values where the outputs for each are disjoint. 3. For each of these 2 contexts create 2 more contexts each having one of the contexts in # 2 as the base context, and set addresses to values. 4. Squash the 4 contexts from #3 and assert the state hash is equal to a manually computed state hash. """ squash = self.context_manager.get_squash_handler() # 1) inputs_1 = [self._create_address('aa'), self._create_address('ab')] outputs_1 = [self._create_address('cc'), self._create_address('ab')] context_1 = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=inputs_1, outputs=outputs_1) self.context_manager.set( context_id=context_1, address_value_list=[{a: v} for a, v in zip( outputs_1, [bytes(i) for i in range(len(outputs_1))])]) sh1 = squash( state_root=self.first_state_hash, context_ids=[context_1], persist=True, clean_up=True) # 2) inputs_2a = [self._create_address('cc'), self._create_address('aa')] outputs_2a = [self._create_address('dd'), self._create_address('ll')] context_2a = self.context_manager.create_context( state_hash=self.first_state_hash, base_contexts=[], inputs=inputs_2a, outputs=outputs_2a) inputs_2b = [self._create_address('cc'), self._create_address('aa')] outputs_2b = [self._create_address('nn'), self._create_address('mm')] context_2b = self.context_manager.create_context( state_hash=sh1, base_contexts=[], inputs=inputs_2b, outputs=outputs_2b) self.context_manager.set( context_id=context_2a, address_value_list=[{a: bytes(v)} for a, v in zip(outputs_2a, range(10, 10 + len(outputs_2a)))] ) self.context_manager.set( context_id=context_2b, address_value_list=[{a: bytes(v)} for a, v in zip(outputs_2b, range(len(outputs_2b)))] ) # 3) inputs_3_2a_1 = [self._create_address('qq'), self._create_address('dd')] outputs_3_2a_1 = [self._create_address('dd'), self._create_address('pp')] context_3_2a_1 = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_2a], inputs=inputs_3_2a_1, outputs=outputs_3_2a_1 ) inputs_3_2a_2 = [self._create_address('aa')] outputs_3_2a_2 = [self._create_address('aa'), self._create_address('ll')] context_3_2a_2 = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_2a], inputs=inputs_3_2a_2, outputs=outputs_3_2a_2) inputs_3_2b_1 = [self._create_address('nn'), self._create_address('ab')] outputs_3_2b_1 = [self._create_address('mm'), self._create_address('ba')] context_3_2b_1 = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_2b], inputs=inputs_3_2b_1, outputs=outputs_3_2b_1) inputs_3_2b_2 = [self._create_address('nn'), self._create_address('oo')] outputs_3_2b_2 = [self._create_address('ab'), self._create_address('oo')] context_3_2b_2 = self.context_manager.create_context( state_hash=sh1, base_contexts=[context_2b], inputs=inputs_3_2b_2, outputs=outputs_3_2b_2) self.context_manager.set( context_id=context_3_2a_1, address_value_list=[{a: bytes(v)} for a, v in zip(outputs_3_2a_1, range(len(outputs_3_2a_1)))]) self.context_manager.set( context_id=context_3_2a_2, address_value_list=[{a: bytes(v)} for a, v in zip(outputs_3_2a_2, range(len(outputs_3_2a_2)))]) self.context_manager.set( context_id=context_3_2b_1, address_value_list=[{a: bytes(v)} for a, v in zip(outputs_3_2b_1, range(len(outputs_3_2b_1)))]) self.context_manager.set( context_id=context_3_2b_2, address_value_list=[{a: bytes(v)} for a, v in zip(outputs_3_2b_2, range(len(outputs_3_2b_2)))]) # 4) sh2 = squash( state_root=sh1, context_ids=[context_3_2a_1, context_3_2a_2, context_3_2b_1, context_3_2b_2], persist=False, clean_up=True) tree = MerkleDatabase(self.database_results) state_hash_from_1 = tree.update( set_items={a: v for a, v in zip(outputs_1, [bytes(i) for i in range(len(outputs_1))])}, virtual=False) self.assertEquals(state_hash_from_1, sh1, "The manually calculated state hash from the first " "context and the one calculated by squashing that " "state hash should be the same") tree.set_merkle_root(state_hash_from_1) test_sh2 = tree.update(set_items={self._create_address('aa'): bytes(0), self._create_address('ab'): bytes(0), self._create_address('ba'): bytes(1), self._create_address('dd'): bytes(0), self._create_address('ll'): bytes(1), self._create_address('mm'): bytes(0), self._create_address('oo'): bytes(1), self._create_address('pp'): bytes(1), self._create_address('nn'): bytes(0), self._create_address('cc'): bytes(0)}) self.assertEquals(sh2, test_sh2, "Manually calculated and context " "manager calculated merkle hashes " "are the same") @unittest.skip("Necessary to catch scheduler bugs--Depth-first search") def test_check_for_bad_combination(self): """Tests that the context manager will raise an exception if asked to combine contexts, either via base contexts in create_context or via squash that shouldn't be combined because they share addresses that can't be determined by the scheduler to not have been parallel. This is a check on scheduler bugs. Examples where the context manager should raise an exception on duplicate addresses: 1. Success i=a o=b +>ctx_1+ dup|->b=3 | | | sh0| ----->state hash or context | i=q | | o=b | +>ctx_2+ dup-->b=2 2. i=b o=d +>ctx_1a_1+ | d=4 | i=a | | o=b | | +>ctx_1a| | | b=2 | i=b | | | o=c | sh0| +>ctx_1a_2| | dup-> c=7 |------>state hash or context | i=a +>ctx_1b_1| | o=c | | +>ctx_1b| | dup-->c=5 | i=t | | o=p | +>ctx_1b_2+ p=8 3. i=b o=d +>ctx_1a_1+ | d=4 | i=d,c i=a | | o=n o=b | <>ctx_3a+ +>ctx_1a| | n=5 | | b=2 | i=b | | | | o=c | | sh0| +>ctx_1a_2+ <----->state hash or context | dup--> c=7 | | i=a +>ctx_1b_1+ | | o=c | | i=c | +>ctx_1b| | o=q | c=5 | i=c <>ctx_3b+ | o=c | q=5 +>ctx_1b_2+ dup--> c=1 """ # 1. squash = self.context_manager.get_squash_handler() sh0 = self.first_state_hash inputs_1 = [self._create_address('a')] outputs_1 = [self._create_address('b')] ctx_1 = self.context_manager.create_context( state_hash=sh0, base_contexts=[], inputs=inputs_1, outputs=outputs_1 ) self.context_manager.set( context_id=ctx_1, address_value_list=[{self._create_address('b'): b'3'}] ) inputs_2 = [self._create_address('q')] outputs_2 = [self._create_address('b')] ctx_2 = self.context_manager.create_context( state_hash=sh0, base_contexts=[], inputs=inputs_2, outputs=outputs_2 ) self.context_manager.set( context_id=ctx_2, address_value_list=[{self._create_address('b'): b'2'}] ) try: sh1 = squash( state_root=sh0, context_ids=[ctx_1, ctx_2], persist=True, clean_up=True) self.fail("squash of two contexts with a duplicate address") except Exception: pass
def __init__(self, network_endpoint, component_endpoint, public_uri, peering, join_list, peer_list, data_dir, config_dir, identity_signing_key): """Constructs a validator instance. Args: network_endpoint (str): the network endpoint component_endpoint (str): the component endpoint public_uri (str): the zmq-style URI of this validator's publically reachable endpoint peering (str): The type of peering approach. Either 'static' or 'dynamic'. In 'static' mode, no attempted topology buildout occurs -- the validator only attempts to initiate peering connections with endpoints specified in the peer_list. In 'dynamic' mode, the validator will first attempt to initiate peering connections with endpoints specified in the peer_list and then attempt to do a topology buildout starting with peer lists obtained from endpoints in the join_list. In either mode, the validator will accept incoming peer requests up to max_peers. join_list (list of str): a list of addresses to connect to in order to perform the initial topology buildout peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory config_dir (str): path to the config directory identity_signing_key (str): key validator uses for signing """ db_filename = os.path.join( data_dir, 'merkle-{}.lmdb'.format(network_endpoint[-2:])) LOGGER.debug('database file is %s', db_filename) merkle_db = LMDBNoLockDatabase(db_filename, 'c') delta_db_filename = os.path.join( data_dir, 'state-deltas-{}.lmdb'.format(network_endpoint[-2:])) LOGGER.debug('state delta store file is %s', delta_db_filename) state_delta_db = LMDBNoLockDatabase(delta_db_filename, 'c') state_delta_store = StateDeltaStore(state_delta_db) context_manager = ContextManager(merkle_db, state_delta_store) self._context_manager = context_manager state_view_factory = StateViewFactory(merkle_db) block_db_filename = os.path.join( data_dir, 'block-{}.lmdb'.format(network_endpoint[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = LMDBNoLockDatabase(block_db_filename, 'c') block_store = BlockStore(block_db) # setup network self._dispatcher = Dispatcher() thread_pool = ThreadPoolExecutor(max_workers=10) process_pool = ProcessPoolExecutor(max_workers=3) self._thread_pool = thread_pool self._process_pool = process_pool self._service = Interconnect(component_endpoint, self._dispatcher, secured=False, heartbeat=False, max_incoming_connections=20) executor = TransactionExecutor(service=self._service, context_manager=context_manager, config_view_factory=ConfigViewFactory( StateViewFactory(merkle_db))) self._executor = executor state_delta_processor = StateDeltaProcessor(self._service, state_delta_store, block_store) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] network_thread_pool = ThreadPoolExecutor(max_workers=10) self._network_thread_pool = network_thread_pool self._network_dispatcher = Dispatcher() # Server public and private keys are hardcoded here due to # the decision to avoid having separate identities for each # validator's server socket. This is appropriate for a public # network. For a permissioned network with requirements for # server endpoint authentication at the network level, this can # be augmented with a local lookup service for side-band provided # endpoint, public_key pairs and a local configuration option # for 'server' side private keys. self._network = Interconnect( network_endpoint, dispatcher=self._network_dispatcher, zmq_identity=zmq_identity, secured=True, server_public_key=b'wFMwoOt>yFqI/ek.G[tfMMILHWw#vXB[Sv}>l>i)', server_private_key=b'r&oJ5aQDj4+V]p2:Lz70Eu0x#m%IwzBdP(}&hWM*', heartbeat=True, public_uri=public_uri, connection_timeout=30, max_incoming_connections=100) self._gossip = Gossip(self._network, public_uri=public_uri, peering_mode=peering, initial_join_endpoints=join_list, initial_peer_endpoints=peer_list, minimum_peer_connectivity=3, maximum_peer_connectivity=10, topology_check_frequency=1) completer = Completer(block_store, self._gossip) block_sender = BroadcastBlockSender(completer, self._gossip) batch_sender = BroadcastBatchSender(completer, self._gossip) chain_id_manager = ChainIdManager(data_dir) # Create and configure journal self._journal = Journal( block_store=block_store, state_view_factory=StateViewFactory(merkle_db), block_sender=block_sender, batch_sender=batch_sender, transaction_executor=executor, squash_handler=context_manager.get_squash_handler(), identity_signing_key=identity_signing_key, chain_id_manager=chain_id_manager, state_delta_processor=state_delta_processor, data_dir=data_dir, config_dir=config_dir, check_publish_block_frequency=0.1, block_cache_purge_frequency=30, block_cache_keep_time=300) self._genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_key=identity_signing_key, data_dir=data_dir, config_dir=config_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender) responder = Responder(completer) completer.set_on_batch_received(self._journal.on_batch_received) completer.set_on_block_received(self._journal.on_block_received) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, processor_handlers.ProcessorRegisterHandler(executor.processors), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_UNREGISTER_REQUEST, processor_handlers.ProcessorUnRegisterHandler(executor.processors), thread_pool) # Set up base network handlers self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_PING, PingHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_CONNECT, ConnectHandler(network=self._network), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_DISCONNECT, DisconnectHandler(network=self._network), network_thread_pool) # Set up gossip handlers self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_REQUEST, GetPeersRequestHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_RESPONSE, GetPeersResponseHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(gossip=self._gossip), network_thread_pool) # GOSSIP_MESSAGE 1) Sends acknowledgement to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(), network_thread_pool) # GOSSIP_MESSAGE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, signature_verifier.GossipMessageSignatureVerifier(), process_pool) # GOSSIP_MESSAGE 3) Determines if we should broadcast the # message to our peers. It is important that this occur prior # to the sending of the message to the completer, as this step # relies on whether the gossip message has previously been # seen by the validator to determine whether or not forwarding # should occur self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipBroadcastHandler(gossip=self._gossip, completer=completer), network_thread_pool) # GOSSIP_MESSAGE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, CompleterGossipHandler(completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_REQUEST, BlockResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, GossipBlockResponseHandler(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, signature_verifier.GossipBlockResponseSignatureVerifier(), process_pool) # GOSSIP_BLOCK_RESPONSE 3) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, CompleterGossipBlockResponseHandler(completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, ResponderBlockResponseHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_BATCH_ID_REQUEST, BatchByBatchIdResponderHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_TRANSACTION_ID_REQUEST, BatchByTransactionIdResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BATCH_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, GossipBatchResponseHandler(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, signature_verifier.GossipBatchResponseSignatureVerifier(), process_pool) # GOSSIP_BATCH_RESPONSE 3) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, CompleterGossipBatchResponseHandler(completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, ResponderBatchResponseHandler(responder, self._gossip), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, signature_verifier.BatchListSignatureVerifier(), process_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, CompleterBatchListBroadcastHandler(completer, self._gossip), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, client_handlers.BatchSubmitFinisher( self._journal.get_block_store(), completer.batch_cache), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_STATUS_REQUEST, client_handlers.BatchStatusRequest(self._journal.get_block_store(), completer.batch_cache), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequest(merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequest(merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST, client_handlers.BlockListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST, client_handlers.BlockGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_LIST_REQUEST, client_handlers.BatchListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_GET_REQUEST, client_handlers.BatchGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_LIST_REQUEST, client_handlers.TransactionListRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_GET_REQUEST, client_handlers.TransactionGetRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, client_handlers.StateCurrentRequest( self._journal.get_current_root), thread_pool) # State Delta Subscription Handlers self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaSubscriberValidationHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaAddSubscriberHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_UNSUBSCRIBE_REQUEST, StateDeltaUnsubscriberHandler(state_delta_processor), thread_pool)
def __init__(self, bind_network, bind_component, endpoint, peering, seeds_list, peer_list, data_dir, config_dir, identity_signing_key, scheduler_type, permissions, network_public_key=None, network_private_key=None, roles=None, metrics_registry=None): """Constructs a validator instance. Args: bind_network (str): the network endpoint bind_component (str): the component endpoint endpoint (str): the zmq-style URI of this validator's publically reachable endpoint peering (str): The type of peering approach. Either 'static' or 'dynamic'. In 'static' mode, no attempted topology buildout occurs -- the validator only attempts to initiate peering connections with endpoints specified in the peer_list. In 'dynamic' mode, the validator will first attempt to initiate peering connections with endpoints specified in the peer_list and then attempt to do a topology buildout starting with peer lists obtained from endpoints in the seeds_list. In either mode, the validator will accept incoming peer requests up to max_peers. seeds_list (list of str): a list of addresses to connect to in order to perform the initial topology buildout peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory config_dir (str): path to the config directory identity_signing_key (str): key validator uses for signing """ # -- Setup Global State Database and Factory -- # global_state_db_filename = os.path.join( data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('global state database file is %s', global_state_db_filename) global_state_db = LMDBNoLockDatabase(global_state_db_filename, 'c') state_view_factory = StateViewFactory(global_state_db) # -- Setup State Delta Store -- # delta_db_filename = os.path.join( data_dir, 'state-deltas-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('state delta store file is %s', delta_db_filename) state_delta_db = LMDBNoLockDatabase(delta_db_filename, 'c') state_delta_store = StateDeltaStore(state_delta_db) # -- Setup Receipt Store -- # receipt_db_filename = os.path.join( data_dir, 'txn_receipts-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('txn receipt store file is %s', receipt_db_filename) receipt_db = LMDBNoLockDatabase(receipt_db_filename, 'c') receipt_store = TransactionReceiptStore(receipt_db) # -- Setup Block Store -- # block_db_filename = os.path.join( data_dir, 'block-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = LMDBNoLockDatabase(block_db_filename, 'c') block_store = BlockStore(block_db) block_cache = BlockCache(block_store, keep_time=300, purge_frequency=30) # -- Setup Thread Pools -- # component_thread_pool = InstrumentedThreadPoolExecutor( max_workers=10, name='Component') network_thread_pool = InstrumentedThreadPoolExecutor(max_workers=10, name='Network') sig_pool = InstrumentedThreadPoolExecutor(max_workers=3, name='Signature') # -- Setup Dispatchers -- # component_dispatcher = Dispatcher(metrics_registry=metrics_registry) network_dispatcher = Dispatcher(metrics_registry=metrics_registry) # -- Setup Services -- # component_service = Interconnect(bind_component, component_dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True, max_future_callback_workers=10, metrics_registry=metrics_registry) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] secure = False if network_public_key is not None and network_private_key is not None: secure = True network_service = Interconnect( bind_network, dispatcher=network_dispatcher, zmq_identity=zmq_identity, secured=secure, server_public_key=network_public_key, server_private_key=network_private_key, heartbeat=True, public_endpoint=endpoint, connection_timeout=30, max_incoming_connections=100, max_future_callback_workers=10, authorize=True, public_key=signing.generate_public_key(identity_signing_key), priv_key=identity_signing_key, roles=roles, metrics_registry=metrics_registry) # -- Setup Transaction Execution Platform -- # context_manager = ContextManager(global_state_db, state_delta_store) batch_tracker = BatchTracker(block_store) executor = TransactionExecutor( service=component_service, context_manager=context_manager, settings_view_factory=SettingsViewFactory(state_view_factory), scheduler_type=scheduler_type, invalid_observers=[batch_tracker], metrics_registry=metrics_registry) component_service.set_check_connections(executor.check_connections) state_delta_processor = StateDeltaProcessor(component_service, state_delta_store, block_store) event_broadcaster = EventBroadcaster(component_service, block_store, receipt_store) # -- Setup P2P Networking -- # gossip = Gossip(network_service, endpoint=endpoint, peering_mode=peering, initial_seed_endpoints=seeds_list, initial_peer_endpoints=peer_list, minimum_peer_connectivity=3, maximum_peer_connectivity=10, topology_check_frequency=1) completer = Completer(block_store, gossip) block_sender = BroadcastBlockSender(completer, gossip) batch_sender = BroadcastBatchSender(completer, gossip) chain_id_manager = ChainIdManager(data_dir) identity_view_factory = IdentityViewFactory( StateViewFactory(global_state_db)) id_cache = IdentityCache(identity_view_factory, block_store.chain_head_state_root) # -- Setup Permissioning -- # permission_verifier = PermissionVerifier( permissions, block_store.chain_head_state_root, id_cache) identity_observer = IdentityObserver(to_update=id_cache.invalidate, forked=id_cache.forked) # -- Setup Journal -- # batch_injector_factory = DefaultBatchInjectorFactory( block_store=block_store, state_view_factory=state_view_factory, signing_key=identity_signing_key) block_publisher = BlockPublisher( transaction_executor=executor, block_cache=block_cache, state_view_factory=state_view_factory, block_sender=block_sender, batch_sender=batch_sender, squash_handler=context_manager.get_squash_handler(), chain_head=block_store.chain_head, identity_signing_key=identity_signing_key, data_dir=data_dir, config_dir=config_dir, permission_verifier=permission_verifier, check_publish_block_frequency=0.1, batch_observers=[batch_tracker], batch_injector_factory=batch_injector_factory, metrics_registry=metrics_registry) chain_controller = ChainController( block_sender=block_sender, block_cache=block_cache, state_view_factory=state_view_factory, transaction_executor=executor, chain_head_lock=block_publisher.chain_head_lock, on_chain_updated=block_publisher.on_chain_updated, squash_handler=context_manager.get_squash_handler(), chain_id_manager=chain_id_manager, identity_signing_key=identity_signing_key, data_dir=data_dir, config_dir=config_dir, permission_verifier=permission_verifier, chain_observers=[ state_delta_processor, event_broadcaster, receipt_store, batch_tracker, identity_observer ], metrics_registry=metrics_registry) genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_key=identity_signing_key, data_dir=data_dir, config_dir=config_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender) responder = Responder(completer) completer.set_on_batch_received(block_publisher.queue_batch) completer.set_on_block_received(chain_controller.queue_block) # -- Register Message Handler -- # network_handlers.add(network_dispatcher, network_service, gossip, completer, responder, network_thread_pool, sig_pool, permission_verifier) component_handlers.add( component_dispatcher, gossip, context_manager, executor, completer, block_store, batch_tracker, global_state_db, self.get_chain_head_state_root_hash, receipt_store, state_delta_processor, state_delta_store, event_broadcaster, permission_verifier, component_thread_pool, sig_pool) # -- Store Object References -- # self._component_dispatcher = component_dispatcher self._component_service = component_service self._component_thread_pool = component_thread_pool self._network_dispatcher = network_dispatcher self._network_service = network_service self._network_thread_pool = network_thread_pool self._sig_pool = sig_pool self._context_manager = context_manager self._executor = executor self._genesis_controller = genesis_controller self._gossip = gossip self._block_publisher = block_publisher self._chain_controller = chain_controller
def __init__(self, bind_network, bind_component, endpoint, peering, seeds_list, peer_list, data_dir, config_dir, identity_signing_key, scheduler_type, network_public_key=None, network_private_key=None): """Constructs a validator instance. Args: bind_network (str): the network endpoint bind_component (str): the component endpoint endpoint (str): the zmq-style URI of this validator's publically reachable endpoint peering (str): The type of peering approach. Either 'static' or 'dynamic'. In 'static' mode, no attempted topology buildout occurs -- the validator only attempts to initiate peering connections with endpoints specified in the peer_list. In 'dynamic' mode, the validator will first attempt to initiate peering connections with endpoints specified in the peer_list and then attempt to do a topology buildout starting with peer lists obtained from endpoints in the seeds_list. In either mode, the validator will accept incoming peer requests up to max_peers. seeds_list (list of str): a list of addresses to connect to in order to perform the initial topology buildout peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory config_dir (str): path to the config directory identity_signing_key (str): key validator uses for signing """ db_filename = os.path.join(data_dir, 'merkle-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('database file is %s', db_filename) merkle_db = LMDBNoLockDatabase(db_filename, 'c') delta_db_filename = os.path.join(data_dir, 'state-deltas-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('state delta store file is %s', delta_db_filename) state_delta_db = LMDBNoLockDatabase(delta_db_filename, 'c') state_delta_store = StateDeltaStore(state_delta_db) context_manager = ContextManager(merkle_db, state_delta_store) self._context_manager = context_manager state_view_factory = StateViewFactory(merkle_db) block_db_filename = os.path.join(data_dir, 'block-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = LMDBNoLockDatabase(block_db_filename, 'c') block_store = BlockStore(block_db) batch_tracker = BatchTracker(block_store) block_store.add_update_observer(batch_tracker) # setup network self._dispatcher = Dispatcher() thread_pool = ThreadPoolExecutor(max_workers=10) sig_pool = ThreadPoolExecutor(max_workers=3) self._thread_pool = thread_pool self._sig_pool = sig_pool self._service = Interconnect(bind_component, self._dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True) config_file = os.path.join(config_dir, "validator.toml") validator_config = {} if os.path.exists(config_file): with open(config_file) as fd: raw_config = fd.read() validator_config = toml.loads(raw_config) if scheduler_type is None: scheduler_type = validator_config.get("scheduler", "serial") executor = TransactionExecutor( service=self._service, context_manager=context_manager, settings_view_factory=SettingsViewFactory( StateViewFactory(merkle_db)), scheduler_type=scheduler_type, invalid_observers=[batch_tracker]) self._executor = executor self._service.set_check_connections(executor.check_connections) state_delta_processor = StateDeltaProcessor(self._service, state_delta_store, block_store) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] network_thread_pool = ThreadPoolExecutor(max_workers=10) self._network_thread_pool = network_thread_pool self._network_dispatcher = Dispatcher() secure = False if network_public_key is not None and network_private_key is not None: secure = True self._network = Interconnect( bind_network, dispatcher=self._network_dispatcher, zmq_identity=zmq_identity, secured=secure, server_public_key=network_public_key, server_private_key=network_private_key, heartbeat=True, public_endpoint=endpoint, connection_timeout=30, max_incoming_connections=100) self._gossip = Gossip(self._network, endpoint=endpoint, peering_mode=peering, initial_seed_endpoints=seeds_list, initial_peer_endpoints=peer_list, minimum_peer_connectivity=3, maximum_peer_connectivity=10, topology_check_frequency=1) completer = Completer(block_store, self._gossip) block_sender = BroadcastBlockSender(completer, self._gossip) batch_sender = BroadcastBatchSender(completer, self._gossip) chain_id_manager = ChainIdManager(data_dir) # Create and configure journal self._journal = Journal( block_store=block_store, state_view_factory=StateViewFactory(merkle_db), block_sender=block_sender, batch_sender=batch_sender, transaction_executor=executor, squash_handler=context_manager.get_squash_handler(), identity_signing_key=identity_signing_key, chain_id_manager=chain_id_manager, state_delta_processor=state_delta_processor, data_dir=data_dir, config_dir=config_dir, check_publish_block_frequency=0.1, block_cache_purge_frequency=30, block_cache_keep_time=300, batch_observers=[batch_tracker] ) self._genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_key=identity_signing_key, data_dir=data_dir, config_dir=config_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender ) responder = Responder(completer) completer.set_on_batch_received(self._journal.on_batch_received) completer.set_on_block_received(self._journal.on_block_received) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, processor_handlers.ProcessorRegisterHandler(executor.processors), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_UNREGISTER_REQUEST, processor_handlers.ProcessorUnRegisterHandler(executor.processors), thread_pool) # Set up base network handlers self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_PING, PingHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_CONNECT, ConnectHandler(network=self._network), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_DISCONNECT, DisconnectHandler(network=self._network), network_thread_pool) # Set up gossip handlers self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_REQUEST, GetPeersRequestHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_RESPONSE, GetPeersResponseHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(gossip=self._gossip), network_thread_pool) # GOSSIP_MESSAGE 1) Sends acknowledgement to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(), network_thread_pool) # GOSSIP_MESSAGE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, signature_verifier.GossipMessageSignatureVerifier(), sig_pool) # GOSSIP_MESSAGE 3) Verifies batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, structure_verifier.GossipHandlerStructureVerifier(), network_thread_pool) # GOSSIP_MESSAGE 4) Determines if we should broadcast the # message to our peers. It is important that this occur prior # to the sending of the message to the completer, as this step # relies on whether the gossip message has previously been # seen by the validator to determine whether or not forwarding # should occur self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipBroadcastHandler( gossip=self._gossip, completer=completer), network_thread_pool) # GOSSIP_MESSAGE 5) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, CompleterGossipHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_REQUEST, BlockResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, GossipBlockResponseHandler(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, signature_verifier.GossipBlockResponseSignatureVerifier(), sig_pool) # GOSSIP_BLOCK_RESPONSE 3) Check batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, structure_verifier.GossipBlockResponseStructureVerifier(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, CompleterGossipBlockResponseHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, ResponderBlockResponseHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_BATCH_ID_REQUEST, BatchByBatchIdResponderHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_TRANSACTION_ID_REQUEST, BatchByTransactionIdResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BATCH_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, GossipBatchResponseHandler(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, signature_verifier.GossipBatchResponseSignatureVerifier(), sig_pool) # GOSSIP_BATCH_RESPONSE 3) Check batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, structure_verifier.GossipBatchResponseStructureVerifier(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, CompleterGossipBatchResponseHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, ResponderBatchResponseHandler(responder, self._gossip), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, BatchListPermissionVerifier( settings_view_factory=SettingsViewFactory( StateViewFactory(merkle_db)), current_root_func=self._journal.get_current_root, ), sig_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, signature_verifier.BatchListSignatureVerifier(), sig_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, structure_verifier.BatchListStructureVerifier(), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, CompleterBatchListBroadcastHandler( completer, self._gossip), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, client_handlers.BatchSubmitFinisher(batch_tracker), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_STATUS_REQUEST, client_handlers.BatchStatusRequest(batch_tracker), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST, client_handlers.BlockListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST, client_handlers.BlockGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_LIST_REQUEST, client_handlers.BatchListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_GET_REQUEST, client_handlers.BatchGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_LIST_REQUEST, client_handlers.TransactionListRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_GET_REQUEST, client_handlers.TransactionGetRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, client_handlers.StateCurrentRequest( self._journal.get_current_root), thread_pool) # State Delta Subscription Handlers self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaSubscriberValidationHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaAddSubscriberHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_UNSUBSCRIBE_REQUEST, StateDeltaUnsubscriberHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_GET_EVENTS_REQUEST, StateDeltaGetEventsHandler(block_store, state_delta_store), thread_pool)