def __init__(self, block_cache, state_view_factory, transaction_executor, squash_handler, identity_signer, data_dir, config_dir, permission_verifier, thread_pool=None): """Initialize the BlockValidator Args: block_cache: The cache of all recent blocks and the processing state associated with them. state_view_factory: A factory that can be used to create read- only views of state for a particular merkle root, in particular the state as it existed when a particular block was the chain head. transaction_executor: The transaction executor used to process transactions. squash_handler: A parameter passed when creating transaction schedulers. identity_signer: A cryptographic signer for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. permission_verifier: The delegate for handling permission validation on blocks. thread_pool: (Optional) Executor pool used to submit block validation jobs. If not specified, a default will be created. Returns: None """ self._block_cache = block_cache self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._squash_handler = squash_handler self._identity_signer = identity_signer self._public_key = identity_signer.get_public_key().as_hex() self._data_dir = data_dir self._config_dir = config_dir self._permission_verifier = permission_verifier self._settings_view_factory = SettingsViewFactory(state_view_factory) self._thread_pool = InstrumentedThreadPoolExecutor(1) \ if thread_pool is None else thread_pool self._moved_to_fork_count = COLLECTOR.counter( 'chain_head_moved_to_fork_count', instance=self) # Blocks that are currently being processed self._blocks_processing = ConcurrentSet() # Descendant blocks that are waiting for an in process block # to complete self._blocks_pending = ConcurrentMultiMap()
def __init__(self, block_cache, state_view_factory, transaction_executor, squash_handler, identity_signer, data_dir, config_dir, permission_verifier, metrics_registry=None, thread_pool=None): """Initialize the BlockValidator Args: block_cache: The cache of all recent blocks and the processing state associated with them. state_view_factory: A factory that can be used to create read- only views of state for a particular merkle root, in particular the state as it existed when a particular block was the chain head. transaction_executor: The transaction executor used to process transactions. squash_handler: A parameter passed when creating transaction schedulers. identity_signer: A cryptographic signer for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. permission_verifier: The delegate for handling permission validation on blocks. metrics_registry: (Optional) Pyformance metrics registry handle for creating new metrics. thread_pool: (Optional) Executor pool used to submit block validation jobs. If not specified, a default will be created. Returns: None """ self._block_cache = block_cache self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._squash_handler = squash_handler self._identity_signer = identity_signer self._data_dir = data_dir self._config_dir = config_dir self._permission_verifier = permission_verifier self._settings_view_factory = SettingsViewFactory(state_view_factory) self._thread_pool = InstrumentedThreadPoolExecutor(1) \ if thread_pool is None else thread_pool if metrics_registry: self._moved_to_fork_count = CounterWrapper( metrics_registry.counter('chain_head_moved_to_fork_count')) else: self._moved_to_fork_count = CounterWrapper()
def __init__(self, block_manager, block_store, state_view_factory, transaction_executor, identity_signer, data_dir, config_dir, permission_verifier, thread_pool=None): """Initialize the BlockValidator Args: block_manager: The component that stores blocks and maintains integrity of the predecessor relationship. state_view_factory: A factory that can be used to create read- only views of state for a particular merkle root, in particular the state as it existed when a particular block was the chain head. transaction_executor: The transaction executor used to process transactions. identity_signer: A cryptographic signer for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. permission_verifier: The delegate for handling permission validation on blocks. thread_pool: (Optional) Executor pool used to submit block validation jobs. If not specified, a default will be created. Returns: None """ self._block_manager = block_manager self._block_store = block_store self._block_validity_fn = None self._block_scheduler = BlockScheduler(block_manager) self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._identity_signer = identity_signer self._data_dir = data_dir self._config_dir = config_dir self._permission_verifier = permission_verifier self._settings_view_factory = SettingsViewFactory(state_view_factory) self._thread_pool = InstrumentedThreadPoolExecutor(1) \ if thread_pool is None else thread_pool
def do_batch_injection_start_block(): injected_batch = testBlockPublisher.make_batch() testBlockPublisher.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=testBlockPublisher.block_tree_manager.block_cache, state_view_factory=testBlockPublisher.state_view_factory, settings_cache=SettingsCache( SettingsViewFactory( testBlockPublisher.block_tree_manager.state_view_factory), ), block_sender=testBlockPublisher.block_sender, batch_sender=testBlockPublisher.batch_sender, squash_handler=None, chain_head=testBlockPublisher.block_tree_manager.chain_head, identity_signer=testBlockPublisher.block_tree_manager. identity_signer, data_dir=None, config_dir=None, permission_verifier=testBlockPublisher.permission_verifier, check_publish_block_frequency=0.1, batch_observers=[], batch_injector_factory=MockBatchInjectorFactory(injected_batch)) testBlockPublisher.receive_batches() testBlockPublisher.publish_block()
def setUp(self): database = DictDatabase() state_view_factory = StateViewFactory(database) self._settings_view_factory = SettingsViewFactory(state_view_factory) merkle_db = MerkleDatabase(database) self._current_root_hash = merkle_db.update( { TestSettingsView._address('my.setting'): TestSettingsView._setting_entry('my.setting', '10'), TestSettingsView._address('my.setting.list'): TestSettingsView._setting_entry('my.setting.list', '10,11,12'), TestSettingsView._address('my.other.list'): TestSettingsView._setting_entry('my.other.list', '13;14;15') }, virtual=False)
def __init__(self): self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.permission_verifier = MockPermissionVerifier() self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, settings_cache=SettingsCache( SettingsViewFactory( self.block_tree_manager.state_view_factory), ), block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, check_publish_block_frequency=0.1, batch_observers=[], permission_verifier=self.permission_verifier) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches()
def setUp(self): self._temp_dir = tempfile.mkdtemp() database = NativeLmdbDatabase( os.path.join(self._temp_dir, 'test_config_view.lmdb'), _size=10 * 1024 * 1024) state_view_factory = StateViewFactory(database) self._settings_view_factory = SettingsViewFactory(state_view_factory) merkle_db = MerkleDatabase(database) self._current_root_hash = merkle_db.update({ TestSettingsView._address('my.setting'): TestSettingsView._setting_entry('my.setting', '10'), TestSettingsView._address('my.setting.list'): TestSettingsView._setting_entry('my.setting.list', '10,11,12'), TestSettingsView._address('my.other.list'): TestSettingsView._setting_entry('my.other.list', '13;14;15') }, virtual=False)
def __init__(self, consensus_module, block_cache, new_block, state_view_factory, done_cb, executor, squash_handler, identity_signing_key, data_dir, config_dir, permission_verifier): """Initialize the BlockValidator Args: consensus_module: The consensus module that contains implementation of the consensus algorithm to use for block validation. block_cache: The cache of all recent blocks and the processing state associated with them. new_block: The block to validate. state_view_factory: The factory object to create. done_cb: The method to call when block validation completed executor: The thread pool to process block validations. squash_handler: A parameter passed when creating transaction schedulers. identity_signing_key: Private key for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. Returns: None """ self._consensus_module = consensus_module self._block_cache = block_cache self._chain_commit_state = ChainCommitState( self._block_cache.block_store, []) self._new_block = new_block # Set during execution of the of the BlockValidation to the current # chain_head at that time. self._chain_head = None self._state_view_factory = state_view_factory self._done_cb = done_cb self._executor = executor self._squash_handler = squash_handler self._identity_signing_key = identity_signing_key self._identity_public_key = \ signing.generate_public_key(self._identity_signing_key) self._data_dir = data_dir self._config_dir = config_dir self._result = { 'new_block': new_block, 'chain_head': None, 'new_chain': [], 'cur_chain': [], 'committed_batches': [], 'uncommitted_batches': [], 'execution_results': [], 'num_transactions': 0 } self._permission_verifier = permission_verifier self._validation_rule_enforcer = \ ValidationRuleEnforcer(SettingsViewFactory(state_view_factory))
def __init__(self, block_cache, state_view_factory, transaction_executor, identity_signer, data_dir, config_dir, permission_verifier, thread_pool=None): """Initialize the BlockValidator Args: block_cache: The cache of all recent blocks and the processing state associated with them. state_view_factory: A factory that can be used to create read- only views of state for a particular merkle root, in particular the state as it existed when a particular block was the chain head. transaction_executor: The transaction executor used to process transactions. identity_signer: A cryptographic signer for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. permission_verifier: The delegate for handling permission validation on blocks. thread_pool: (Optional) Executor pool used to submit block validation jobs. If not specified, a default will be created. Returns: None """ self._block_cache = block_cache self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._identity_signer = identity_signer self._data_dir = data_dir self._config_dir = config_dir self._permission_verifier = permission_verifier self._settings_view_factory = SettingsViewFactory(state_view_factory) self._thread_pool = InstrumentedThreadPoolExecutor(1) \ if thread_pool is None else thread_pool self._moved_to_fork_count = COLLECTOR.counter( 'chain_head_moved_to_fork_count', instance=self) # Blocks that are currently being processed self._blocks_processing = ConcurrentSet() # Descendant blocks that are waiting for an in process block # to complete self._blocks_pending = ConcurrentMultiMap()
def setUp(self): database = DictDatabase() state_view_factory = StateViewFactory(database) self._settings_view_factory = SettingsViewFactory(state_view_factory) merkle_db = MerkleDatabase(database) self._current_root_hash = merkle_db.update({ TestSettingsView._address('my.setting'): TestSettingsView._setting_entry('my.setting', '10'), TestSettingsView._address('my.setting.list'): TestSettingsView._setting_entry('my.setting.list', '10,11,12'), TestSettingsView._address('my.other.list'): TestSettingsView._setting_entry('my.other.list', '13;14;15') }, virtual=False)
def __init__(self, with_genesis=True): self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.block_store = BlockStore( DictDatabase(indexes=BlockStore.create_index_configuration())) self.block_cache = BlockCache(self.block_store) self.state_db = {} self.block_manager = BlockManager() self.block_manager.add_store("commit_store", self.block_store) # add the mock reference to the consensus consensus_setting_addr = SettingsView.setting_address( 'sawtooth.consensus.algorithm') self.state_db[consensus_setting_addr] = _setting_entry( 'sawtooth.consensus.algorithm', 'test_journal.mock_consensus') self.state_view_factory = MockStateViewFactory(self.state_db) context = create_context('secp256k1') private_key = context.new_random_private_key() crypto_factory = CryptoFactory(context) self.signer = crypto_factory.new_signer(private_key) identity_private_key = context.new_random_private_key() self.identity_signer = crypto_factory.new_signer(identity_private_key) chain_head = None if with_genesis: self.genesis_block = self.generate_genesis_block() chain_head = self.genesis_block self.block_manager.put([chain_head.block]) self.block_manager.persist(chain_head.block.header_signature, "commit_store") self.block_publisher = BlockPublisher( block_manager=self.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=self.block_store.has_transaction, batch_committed=self.block_store.has_batch, state_view_factory=self.state_view_factory, settings_cache=SettingsCache( SettingsViewFactory(self.state_view_factory), ), block_sender=self.block_sender, batch_sender=self.block_sender, chain_head=chain_head.block, identity_signer=self.identity_signer, data_dir=None, config_dir=None, permission_verifier=MockPermissionVerifier(), batch_observers=[])
def __init__(self, with_genesis=True): self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.block_store = BlockStore(DictDatabase( indexes=BlockStore.create_index_configuration())) self.block_cache = BlockCache(self.block_store) self.state_db = {} # add the mock reference to the consensus consensus_setting_addr = SettingsView.setting_address( 'sawtooth.consensus.algorithm') self.state_db[consensus_setting_addr] = _setting_entry( 'sawtooth.consensus.algorithm', 'test_journal.mock_consensus') self.state_view_factory = MockStateViewFactory(self.state_db) context = create_context('secp256k1') private_key = context.new_random_private_key() crypto_factory = CryptoFactory(context) self.signer = crypto_factory.new_signer(private_key) identity_private_key = context.new_random_private_key() self.identity_signer = crypto_factory.new_signer(identity_private_key) chain_head = None if with_genesis: self.genesis_block = self.generate_genesis_block() self.set_chain_head(self.genesis_block) chain_head = self.genesis_block self.block_publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_cache, state_view_factory=self.state_view_factory, settings_cache=SettingsCache( SettingsViewFactory(self.state_view_factory), ), block_sender=self.block_sender, batch_sender=self.block_sender, squash_handler=None, chain_head=chain_head, identity_signer=self.identity_signer, data_dir=None, config_dir=None, permission_verifier=MockPermissionVerifier(), check_publish_block_frequency=0.1, batch_observers=[])
def setUp(self): self._temp_dir = tempfile.mkdtemp() database = NativeLmdbDatabase( os.path.join(self._temp_dir, 'test_config_view.lmdb'), indexes=MerkleDatabase.create_index_configuration(), _size=10 * 1024 * 1024) state_view_factory = StateViewFactory(database) self._settings_view_factory = SettingsViewFactory(state_view_factory) merkle_db = MerkleDatabase(database) self._current_root_hash = merkle_db.update({ TestSettingsView._address('my.setting'): TestSettingsView._setting_entry('my.setting', '10'), TestSettingsView._address('my.setting.list'): TestSettingsView._setting_entry('my.setting.list', '10,11,12'), TestSettingsView._address('my.other.list'): TestSettingsView._setting_entry('my.other.list', '13;14;15') }, virtual=False)
def __init__(self, bind_network, bind_component, bind_consensus, endpoint, peering, seeds_list, peer_list, data_dir, config_dir, identity_signer, scheduler_type, permissions, minimum_peer_connectivity, maximum_peer_connectivity, state_pruning_block_depth, network_public_key=None, network_private_key=None, roles=None): """Constructs a validator instance. Args: bind_network (str): the network endpoint bind_component (str): the component endpoint endpoint (str): the zmq-style URI of this validator's publically reachable endpoint peering (str): The type of peering approach. Either 'static' or 'dynamic'. In 'static' mode, no attempted topology buildout occurs -- the validator only attempts to initiate peering connections with endpoints specified in the peer_list. In 'dynamic' mode, the validator will first attempt to initiate peering connections with endpoints specified in the peer_list and then attempt to do a topology buildout starting with peer lists obtained from endpoints in the seeds_list. In either mode, the validator will accept incoming peer requests up to max_peers. seeds_list (list of str): a list of addresses to connect to in order to perform the initial topology buildout peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory config_dir (str): path to the config directory identity_signer (str): cryptographic signer the validator uses for signing """ # -- Setup Global State Database and Factory -- # global_state_db_filename = os.path.join( data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('global state database file is %s', global_state_db_filename) global_state_db = NativeLmdbDatabase( global_state_db_filename, indexes=MerkleDatabase.create_index_configuration()) state_view_factory = StateViewFactory(global_state_db) # -- Setup Receipt Store -- # receipt_db_filename = os.path.join( data_dir, 'txn_receipts-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('txn receipt store file is %s', receipt_db_filename) receipt_db = LMDBNoLockDatabase(receipt_db_filename, 'c') receipt_store = TransactionReceiptStore(receipt_db) # -- Setup Block Store -- # block_db_filename = os.path.join( data_dir, 'block-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = IndexedDatabase( block_db_filename, BlockStore.serialize_block, BlockStore.deserialize_block, flag='c', indexes=BlockStore.create_index_configuration()) block_store = BlockStore(block_db) # The cache keep time for the journal's block cache must be greater # than the cache keep time used by the completer. base_keep_time = 1200 block_cache = BlockCache(block_store, keep_time=int(base_keep_time * 9 / 8), purge_frequency=30) # -- Setup Thread Pools -- # component_thread_pool = InstrumentedThreadPoolExecutor( max_workers=10, name='Component') network_thread_pool = InstrumentedThreadPoolExecutor(max_workers=10, name='Network') client_thread_pool = InstrumentedThreadPoolExecutor(max_workers=5, name='Client') sig_pool = InstrumentedThreadPoolExecutor(max_workers=3, name='Signature') # -- Setup Dispatchers -- # component_dispatcher = Dispatcher() network_dispatcher = Dispatcher() # -- Setup Services -- # component_service = Interconnect(bind_component, component_dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True, max_future_callback_workers=10) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] secure = False if network_public_key is not None and network_private_key is not None: secure = True network_service = Interconnect(bind_network, dispatcher=network_dispatcher, zmq_identity=zmq_identity, secured=secure, server_public_key=network_public_key, server_private_key=network_private_key, heartbeat=True, public_endpoint=endpoint, connection_timeout=120, max_incoming_connections=100, max_future_callback_workers=10, authorize=True, signer=identity_signer, roles=roles) # -- Setup Transaction Execution Platform -- # context_manager = ContextManager(global_state_db) batch_tracker = BatchTracker(block_store) settings_cache = SettingsCache( SettingsViewFactory(state_view_factory), ) transaction_executor = TransactionExecutor( service=component_service, context_manager=context_manager, settings_view_factory=SettingsViewFactory(state_view_factory), scheduler_type=scheduler_type, invalid_observers=[batch_tracker]) component_service.set_check_connections( transaction_executor.check_connections) event_broadcaster = EventBroadcaster(component_service, block_store, receipt_store) # -- Setup P2P Networking -- # gossip = Gossip(network_service, settings_cache, lambda: block_store.chain_head, block_store.chain_head_state_root, endpoint=endpoint, peering_mode=peering, initial_seed_endpoints=seeds_list, initial_peer_endpoints=peer_list, minimum_peer_connectivity=minimum_peer_connectivity, maximum_peer_connectivity=maximum_peer_connectivity, topology_check_frequency=1) completer = Completer(block_store, gossip, cache_keep_time=base_keep_time, cache_purge_frequency=30, requested_keep_time=300) block_sender = BroadcastBlockSender(completer, gossip) batch_sender = BroadcastBatchSender(completer, gossip) chain_id_manager = ChainIdManager(data_dir) identity_view_factory = IdentityViewFactory( StateViewFactory(global_state_db)) id_cache = IdentityCache(identity_view_factory) # -- Setup Permissioning -- # permission_verifier = PermissionVerifier( permissions, block_store.chain_head_state_root, id_cache) identity_observer = IdentityObserver(to_update=id_cache.invalidate, forked=id_cache.forked) settings_observer = SettingsObserver( to_update=settings_cache.invalidate, forked=settings_cache.forked) # -- Consensus Engine -- # consensus_thread_pool = InstrumentedThreadPoolExecutor( max_workers=3, name='Consensus') consensus_dispatcher = Dispatcher() consensus_service = Interconnect(bind_consensus, consensus_dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True, max_future_callback_workers=10) consensus_notifier = ConsensusNotifier(consensus_service) # -- Setup Journal -- # batch_injector_factory = DefaultBatchInjectorFactory( block_cache=block_cache, state_view_factory=state_view_factory, signer=identity_signer) block_publisher = BlockPublisher( transaction_executor=transaction_executor, block_cache=block_cache, state_view_factory=state_view_factory, settings_cache=settings_cache, block_sender=block_sender, batch_sender=batch_sender, chain_head=block_store.chain_head, identity_signer=identity_signer, data_dir=data_dir, config_dir=config_dir, permission_verifier=permission_verifier, check_publish_block_frequency=0.1, batch_observers=[batch_tracker], batch_injector_factory=batch_injector_factory) block_publisher_batch_sender = block_publisher.batch_sender() block_validator = BlockValidator( block_cache=block_cache, state_view_factory=state_view_factory, transaction_executor=transaction_executor, identity_signer=identity_signer, data_dir=data_dir, config_dir=config_dir, permission_verifier=permission_verifier) chain_controller = ChainController( block_store=block_store, block_cache=block_cache, block_validator=block_validator, state_database=global_state_db, chain_head_lock=block_publisher.chain_head_lock, state_pruning_block_depth=state_pruning_block_depth, data_dir=data_dir, observers=[ event_broadcaster, receipt_store, batch_tracker, identity_observer, settings_observer ]) genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=transaction_executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_signer=identity_signer, data_dir=data_dir, config_dir=config_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender) responder = Responder(completer) completer.set_on_batch_received(block_publisher_batch_sender.send) completer.set_on_block_received(chain_controller.queue_block) completer.set_chain_has_block(chain_controller.has_block) # -- Register Message Handler -- # network_handlers.add(network_dispatcher, network_service, gossip, completer, responder, network_thread_pool, sig_pool, chain_controller.has_block, block_publisher.has_batch, permission_verifier, block_publisher, consensus_notifier) component_handlers.add(component_dispatcher, gossip, context_manager, transaction_executor, completer, block_store, batch_tracker, global_state_db, self.get_chain_head_state_root_hash, receipt_store, event_broadcaster, permission_verifier, component_thread_pool, client_thread_pool, sig_pool, block_publisher) # -- Store Object References -- # self._component_dispatcher = component_dispatcher self._component_service = component_service self._component_thread_pool = component_thread_pool self._network_dispatcher = network_dispatcher self._network_service = network_service self._network_thread_pool = network_thread_pool consensus_proxy = ConsensusProxy( block_cache=block_cache, chain_controller=chain_controller, block_publisher=block_publisher, gossip=gossip, identity_signer=identity_signer, settings_view_factory=SettingsViewFactory(state_view_factory), state_view_factory=state_view_factory) consensus_handlers.add(consensus_dispatcher, consensus_thread_pool, consensus_proxy) self._consensus_dispatcher = consensus_dispatcher self._consensus_service = consensus_service self._consensus_thread_pool = consensus_thread_pool self._client_thread_pool = client_thread_pool self._sig_pool = sig_pool self._context_manager = context_manager self._transaction_executor = transaction_executor self._genesis_controller = genesis_controller self._gossip = gossip self._block_publisher = block_publisher self._chain_controller = chain_controller self._block_validator = block_validator
def do_publish_block(): btm = BlockTreeManager() block_publisher = None chain_controller = None try: block_publisher = BlockPublisher( transaction_executor=testJournal.txn_executor, block_cache=btm.block_cache, state_view_factory=MockStateViewFactory(btm.state_db), settings_cache=SettingsCache( SettingsViewFactory(btm.state_view_factory), ), block_sender=testJournal.block_sender, batch_sender=testJournal.batch_sender, squash_handler=None, chain_head=btm.block_store.chain_head, identity_signer=btm.identity_signer, data_dir=None, config_dir=None, permission_verifier=testJournal.permission_verifier, check_publish_block_frequency=0.1, batch_observers=[], batch_injector_factory=DefaultBatchInjectorFactory( block_store=btm.block_store, state_view_factory=MockStateViewFactory(btm.state_db), signer=btm.identity_signer)) block_validator = BlockValidator( state_view_factory=MockStateViewFactory(btm.state_db), block_cache=btm.block_cache, transaction_executor=testJournal.txn_executor, squash_handler=None, identity_signer=btm.identity_signer, data_dir=None, config_dir=None, permission_verifier=testJournal.permission_verifier) chain_controller = ChainController( block_cache=btm.block_cache, block_validator=block_validator, state_view_factory=MockStateViewFactory(btm.state_db), chain_head_lock=block_publisher.chain_head_lock, on_chain_updated=block_publisher.on_chain_updated, chain_id_manager=None, data_dir=None, config_dir=None, chain_observers=[]) testJournal.gossip.on_batch_received = block_publisher.queue_batch testJournal.gossip.on_block_received = chain_controller.queue_block block_publisher.start() chain_controller.start() # feed it a batch batch = Batch() block_publisher.queue_batch(batch) wait_until(lambda: testJournal.block_sender.new_block is not None, 2) assert testJournal.block_sender.new_block != None block = BlockWrapper(testJournal.block_sender.new_block) chain_controller.queue_block(block) # wait for the chain_head to be updated. wait_until(lambda: btm.chain_head.identifier == block.identifier, 2) # assert btm.chain_head.identifier == block.identifier finally: if block_publisher is not None: block_publisher.stop() if chain_controller is not None: chain_controller.stop() if block_validator is not None: block_validator.stop()
class TestSettingsView(unittest.TestCase): def __init__(self, test_name): super().__init__(test_name) self._settings_view_factory = None self._current_root_hash = None def setUp(self): database = DictDatabase() state_view_factory = StateViewFactory(database) self._settings_view_factory = SettingsViewFactory(state_view_factory) merkle_db = MerkleDatabase(database) self._current_root_hash = merkle_db.update( { TestSettingsView._address('my.setting'): TestSettingsView._setting_entry('my.setting', '10'), TestSettingsView._address('my.setting.list'): TestSettingsView._setting_entry('my.setting.list', '10,11,12'), TestSettingsView._address('my.other.list'): TestSettingsView._setting_entry('my.other.list', '13;14;15') }, virtual=False) def test_get_setting(self): """Verifies the correct operation of get_setting() by using it to get the config setting stored as "my.setting" and compare it to '10' (the value set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual('10', settings_view.get_setting('my.setting')) def test_get_setting_with_type_coercion(self): """Verifies the correct operation of get_setting() by using it to get the config setting stored as "my.setting" with a int type coercion function and compare it to the int 10 (the value set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual( 10, settings_view.get_setting('my.setting', value_type=int)) def test_get_setting_not_found(self): """Verifies the correct operation of get_setting() by using it to return None when an unknown setting is requested. """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertIsNone(settings_view.get_setting('non-existant.setting')) def test_get_setting_not_found_with_default(self): """Verifies the correct operation of get_setting() by using it to return a default value when an unknown setting is requested. """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual( 'default', settings_view.get_setting('non-existant.setting', default_value='default')) def test_get_setting_list(self): """Verifies the correct operation of get_setting_list() by using it to get the config setting stored as "my.setting.list" and compare it to ['10', '11', '12'] (the split value set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) # Verify we can still get the "raw" setting self.assertEqual('10,11,12', settings_view.get_setting('my.setting.list')) # And now the split setting self.assertEqual(['10', '11', '12'], settings_view.get_setting_list('my.setting.list')) def test_get_setting_list_not_found(self): """Verifies the correct operation of get_setting_list() by using it to return None when an unknown setting is requested. """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertIsNone( settings_view.get_setting_list('non-existant.setting.list')) def test_get_setting_list_not_found_with_default(self): """Verifies the correct operation of get_setting_list() by using it to return a default value when an unknown setting is requested. """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual([], settings_view.get_setting_list('non-existant.list', default_value=[])) def test_get_setting_list_alternate_delimiter(self): """Verifies the correct operation of get_setting_list() by using it to get the config setting stored as "my.other.list" and compare it to ['13', '14', '15'] (the value, split along an alternate delimiter, set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual(['13', '14', '15'], settings_view.get_setting_list('my.other.list', delimiter=';')) def test_get_setting_list_with_type_coercion(self): """Verifies the correct operation of get_setting_list() by using it to get the integer type-coerced config setting stored as "my.setting.list" and compare it to [10, 11, 12] (the split, type-coerced, value set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual([10, 11, 12], settings_view.get_setting_list('my.setting.list', value_type=int)) @staticmethod def _address(key): return '000000' + _key_to_address(key) @staticmethod def _setting_entry(key, value): return Setting( entries=[Setting.Entry(key=key, value=value)]).SerializeToString()
class BlockValidator: """ Responsible for validating a block. """ def __init__(self, block_cache, state_view_factory, transaction_executor, identity_signer, data_dir, config_dir, permission_verifier, thread_pool=None): """Initialize the BlockValidator Args: block_cache: The cache of all recent blocks and the processing state associated with them. state_view_factory: A factory that can be used to create read- only views of state for a particular merkle root, in particular the state as it existed when a particular block was the chain head. transaction_executor: The transaction executor used to process transactions. identity_signer: A cryptographic signer for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. permission_verifier: The delegate for handling permission validation on blocks. thread_pool: (Optional) Executor pool used to submit block validation jobs. If not specified, a default will be created. Returns: None """ self._block_cache = block_cache self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._identity_signer = identity_signer self._data_dir = data_dir self._config_dir = config_dir self._permission_verifier = permission_verifier self._settings_view_factory = SettingsViewFactory(state_view_factory) self._thread_pool = InstrumentedThreadPoolExecutor(1) \ if thread_pool is None else thread_pool self._block_scheduler = BlockScheduler(block_cache) def stop(self): self._thread_pool.shutdown(wait=True) def _get_previous_block_state_root(self, blkw): if blkw.previous_block_id == NULL_BLOCK_IDENTIFIER: return INIT_ROOT_KEY return self._block_cache[blkw.previous_block_id].state_root_hash def _validate_batches_in_block(self, blkw, prev_state_root): """ Validate all batches in the block. This includes: - Validating all transaction dependencies are met - Validating there are no duplicate batches or transactions - Validating execution of all batches in the block produces the correct state root hash Args: blkw: the block of batches to validate prev_state_root: the state root to execute transactions on top of Raises: BlockValidationFailure: If validation fails, raises this error with the reason. MissingDependency: Validation failed because of a missing dependency. DuplicateTransaction: Validation failed because of a duplicate transaction. DuplicateBatch: Validation failed because of a duplicate batch. """ if not blkw.block.batches: return scheduler = None try: while True: try: chain_head = self._block_cache.block_store.chain_head chain_commit_state = ChainCommitState( blkw.previous_block_id, self._block_cache, self._block_cache.block_store) chain_commit_state.check_for_duplicate_batches( blkw.block.batches) transactions = [] for batch in blkw.block.batches: transactions.extend(batch.transactions) chain_commit_state.check_for_duplicate_transactions( transactions) chain_commit_state.check_for_transaction_dependencies( transactions) if not self._check_chain_head_updated(chain_head, blkw): break except (DuplicateBatch, DuplicateTransaction, MissingDependency) as err: if not self._check_chain_head_updated(chain_head, blkw): raise BlockValidationFailure( "Block {} failed validation: {}".format(blkw, err)) scheduler = self._transaction_executor.create_scheduler( prev_state_root) for batch, has_more in look_ahead(blkw.block.batches): if has_more: scheduler.add_batch(batch) else: scheduler.add_batch(batch, blkw.state_root_hash) except Exception: if scheduler is not None: scheduler.cancel() raise scheduler.finalize() scheduler.complete(block=True) state_hash = None for batch in blkw.batches: batch_result = scheduler.get_batch_execution_result( batch.header_signature) if batch_result is not None and batch_result.is_valid: txn_results = \ scheduler.get_transaction_execution_results( batch.header_signature) blkw.execution_results.extend(txn_results) state_hash = batch_result.state_hash blkw.num_transactions += len(batch.transactions) else: raise BlockValidationFailure( "Block {} failed validation: Invalid batch " "{}".format(blkw, batch)) if blkw.state_root_hash != state_hash: raise BlockValidationFailure( "Block {} failed state root hash validation. Expected {}" " but got {}".format(blkw, blkw.state_root_hash, state_hash)) def _check_chain_head_updated(self, chain_head, block): # The validity of blocks depends partially on whether or not # there are any duplicate transactions or batches in the block. # This can only be checked accurately if the block store does # not update during validation. The current practice is the # assume this will not happen and, if it does, to reprocess the # validation. This has been experimentally proven to be more # performant than locking the chain head and block store around # duplicate checking. if chain_head is None: return False current_chain_head = self._block_cache.block_store.chain_head if chain_head.identifier != current_chain_head.identifier: LOGGER.warning( "Chain head updated from %s to %s while checking " "duplicates and dependencies in block %s. " "Reprocessing validation.", chain_head, current_chain_head, block) return True return False def _validate_permissions(self, blkw, prev_state_root): """ Validate that all of the batch signers and transaction signer for the batches in the block are permitted by the transactor permissioning roles stored in state as of the previous block. If a transactor is found to not be permitted, the block is invalid. """ if blkw.block_num != 0: for batch in blkw.batches: if not self._permission_verifier.is_batch_signer_authorized( batch, prev_state_root, from_state=True): return False return True def _validate_on_chain_rules(self, blkw, prev_state_root): """ Validate that the block conforms to all validation rules stored in state. If the block breaks any of the stored rules, the block is invalid. """ if blkw.block_num != 0: return enforce_validation_rules( self._settings_view_factory.create_settings_view( prev_state_root), blkw.header.signer_public_key, blkw.batches) return True def validate_block(self, blkw): if blkw.status == BlockStatus.Valid: return if blkw.status == BlockStatus.Invalid: raise BlockValidationFailure( 'Block {} is already invalid'.format(blkw)) # pylint: disable=broad-except try: try: prev_block = self._block_cache[blkw.previous_block_id] except KeyError: prev_block = None else: if prev_block.status == BlockStatus.Invalid: raise BlockValidationFailure( "Block {} rejected due to invalid predecessor" " {}".format(blkw, prev_block)) elif prev_block.status == BlockStatus.Unknown: raise BlockValidationError( "Attempted to validate block {} before its predecessor" " {}".format(blkw, prev_block)) try: prev_state_root = self._get_previous_block_state_root(blkw) except KeyError: raise BlockValidationError( 'Block {} rejected due to missing predecessor'.format( blkw)) if not self._validate_permissions(blkw, prev_state_root): raise BlockValidationFailure( 'Block {} failed permission validation'.format(blkw)) if not self._validate_on_chain_rules(blkw, prev_state_root): raise BlockValidationFailure( 'Block {} failed on-chain validation rules'.format(blkw)) self._validate_batches_in_block(blkw, prev_state_root) blkw.status = BlockStatus.Valid except BlockValidationFailure as err: blkw.status = BlockStatus.Invalid raise err except BlockValidationError as err: blkw.status = BlockStatus.Unknown raise err except Exception as e: LOGGER.exception( "Unhandled exception BlockValidator.validate_block()") raise e def submit_blocks_for_verification(self, blocks, callback): # This is a work-around for the fact that the blocks passed to this # function are both from the ChainController (in Rust) or itself. # This ensures that the blocks being operated on come from the cache blocks = [self._block_cache[b.identifier] for b in blocks] ready = self._block_scheduler.schedule(blocks) for block in ready: # Schedule the block for processing self._thread_pool.submit(self.process_block_verification, block, callback) def _release_pending(self, block): """Removes the block from processing and returns any blocks that should now be scheduled for processing, cleaning up the pending block trackers in the process. """ ready = [] if block.status == BlockStatus.Valid: ready.extend(self._block_scheduler.done(block)) elif block.status == BlockStatus.Invalid: # Mark all pending blocks as invalid invalid = self._block_scheduler.done(block, and_descendants=True) for blk in invalid: blk.status = BlockStatus.Invalid LOGGER.debug('Marking descendant block invalid: %s', blk) else: # An error occured during validation, something is wrong internally # and we need to abort validation of this block and all its # children without marking them as invalid. unknown = self._block_scheduler.done(block, and_descendants=True) for blk in unknown: LOGGER.debug( 'Removing block from cache and pending due to error ' 'during validation: %s', block) try: del self._block_cache[block.identifier] except KeyError: LOGGER.exception( "Tried to delete a descendant pending block from the" " block cache because of an error, but the descendant" " was not in the cache.") return ready def has_block(self, block_id): return block_id in self._block_scheduler def process_block_verification(self, block, callback): """ Main entry for Block Validation, Take a given candidate block and decide if it is valid then if it is valid determine if it should be the new head block. Returns the results to the ChainController so that the change over can be made if necessary. """ try: self.validate_block(block) LOGGER.info('Block %s passed validation', block) except BlockValidationFailure as err: LOGGER.warning('Block %s failed validation: %s', block, err) except BlockValidationError as err: LOGGER.error('Encountered an error while validating %s: %s', block, err) except Exception: # pylint: disable=broad-except LOGGER.exception( "Block validation failed with unexpected error: %s", block) else: callback(block) try: blocks_now_ready = self._release_pending(block) self.submit_blocks_for_verification(blocks_now_ready, callback) except Exception: # pylint: disable=broad-except LOGGER.exception( "Submitting pending blocks failed with unexpected error: %s", block)
class BlockValidator(object): """ Responsible for validating a block, handles both chain extensions and fork will determine if the new block should be the head of the chain and return the information necessary to do the switch if necessary. """ def __init__(self, block_cache, state_view_factory, transaction_executor, identity_signer, data_dir, config_dir, permission_verifier, thread_pool=None): """Initialize the BlockValidator Args: block_cache: The cache of all recent blocks and the processing state associated with them. state_view_factory: A factory that can be used to create read- only views of state for a particular merkle root, in particular the state as it existed when a particular block was the chain head. transaction_executor: The transaction executor used to process transactions. identity_signer: A cryptographic signer for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. permission_verifier: The delegate for handling permission validation on blocks. thread_pool: (Optional) Executor pool used to submit block validation jobs. If not specified, a default will be created. Returns: None """ self._block_cache = block_cache self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._identity_signer = identity_signer self._data_dir = data_dir self._config_dir = config_dir self._permission_verifier = permission_verifier self._settings_view_factory = SettingsViewFactory(state_view_factory) self._thread_pool = InstrumentedThreadPoolExecutor(1) \ if thread_pool is None else thread_pool self._moved_to_fork_count = COLLECTOR.counter( 'chain_head_moved_to_fork_count', instance=self) # Blocks that are currently being processed self._blocks_processing = ConcurrentSet() # Descendant blocks that are waiting for an in process block # to complete self._blocks_pending = ConcurrentMultiMap() def stop(self): self._thread_pool.shutdown(wait=True) def _get_previous_block_state_root(self, blkw): if blkw.previous_block_id == NULL_BLOCK_IDENTIFIER: return INIT_ROOT_KEY return self._block_cache[blkw.previous_block_id].state_root_hash def _validate_batches_in_block(self, blkw, prev_state_root): """ Validate all batches in the block. This includes: - Validating all transaction dependencies are met - Validating there are no duplicate batches or transactions - Validating execution of all batches in the block produces the correct state root hash Args: blkw: the block of batches to validate prev_state_root: the state root to execute transactions on top of Raises: BlockValidationFailure: If validation fails, raises this error with the reason. MissingDependency: Validation failed because of a missing dependency. DuplicateTransaction: Validation failed because of a duplicate transaction. DuplicateBatch: Validation failed because of a duplicate batch. """ if not blkw.block.batches: return try: chain_commit_state = ChainCommitState( blkw.previous_block_id, self._block_cache, self._block_cache.block_store) scheduler = self._transaction_executor.create_scheduler( prev_state_root) chain_commit_state.check_for_duplicate_batches( blkw.block.batches) transactions = [] for batch in blkw.block.batches: transactions.extend(batch.transactions) chain_commit_state.check_for_duplicate_transactions( transactions) chain_commit_state.check_for_transaction_dependencies( transactions) for batch, has_more in look_ahead(blkw.block.batches): if has_more: scheduler.add_batch(batch) else: scheduler.add_batch(batch, blkw.state_root_hash) except (DuplicateBatch, DuplicateTransaction, MissingDependency) as err: scheduler.cancel() raise BlockValidationFailure( "Block {} failed validation: {}".format(blkw, err)) except Exception: scheduler.cancel() raise scheduler.finalize() scheduler.complete(block=True) state_hash = None for batch in blkw.batches: batch_result = scheduler.get_batch_execution_result( batch.header_signature) if batch_result is not None and batch_result.is_valid: txn_results = \ scheduler.get_transaction_execution_results( batch.header_signature) blkw.execution_results.extend(txn_results) state_hash = batch_result.state_hash blkw.num_transactions += len(batch.transactions) else: raise BlockValidationFailure( "Block {} failed validation: Invalid batch " "{}".format(blkw, batch)) if blkw.state_root_hash != state_hash: raise BlockValidationFailure( "Block {} failed state root hash validation. Expected {}" " but got {}".format( blkw, blkw.state_root_hash, state_hash)) def _validate_permissions(self, blkw, prev_state_root): """ Validate that all of the batch signers and transaction signer for the batches in the block are permitted by the transactor permissioning roles stored in state as of the previous block. If a transactor is found to not be permitted, the block is invalid. """ if blkw.block_num != 0: for batch in blkw.batches: if not self._permission_verifier.is_batch_signer_authorized( batch, prev_state_root, from_state=True): return False return True def _validate_on_chain_rules(self, blkw, prev_state_root): """ Validate that the block conforms to all validation rules stored in state. If the block breaks any of the stored rules, the block is invalid. """ if blkw.block_num != 0: return enforce_validation_rules( self._settings_view_factory.create_settings_view( prev_state_root), blkw.header.signer_public_key, blkw.batches) return True def validate_block(self, blkw, chain_head=None): if blkw.status == BlockStatus.Valid: return elif blkw.status == BlockStatus.Invalid: raise BlockValidationFailure( 'Block {} is already invalid'.format(blkw)) # pylint: disable=broad-except try: if chain_head is None: # Try to get the chain head from the block store; note that the # block store may also return None for the chain head if a # genesis block hasn't been committed yet. chain_head = self._block_cache.block_store.chain_head try: prev_state_root = self._get_previous_block_state_root(blkw) except KeyError: raise BlockValidationError( 'Block {} rejected due to missing predecessor'.format( blkw)) if not self._validate_permissions(blkw, prev_state_root): raise BlockValidationFailure( 'Block {} failed permission validation'.format(blkw)) try: prev_block = self._block_cache[blkw.previous_block_id] except KeyError: prev_block = None consensus = self._load_consensus(prev_block) public_key = \ self._identity_signer.get_public_key().as_hex() consensus_block_verifier = consensus.BlockVerifier( block_cache=self._block_cache, state_view_factory=self._state_view_factory, data_dir=self._data_dir, config_dir=self._config_dir, validator_id=public_key) if not consensus_block_verifier.verify_block(blkw): raise BlockValidationFailure( 'Block {} failed {} consensus validation'.format( blkw, consensus)) if not self._validate_on_chain_rules(blkw, prev_state_root): raise BlockValidationFailure( 'Block {} failed on-chain validation rules'.format( blkw)) self._validate_batches_in_block(blkw, prev_state_root) # since changes to the chain-head can change the state of the # blocks in BlockStore we have to revalidate this block. block_store = self._block_cache.block_store # The chain_head is None when this is the genesis block or if the # block store has no chain_head. if chain_head is not None: if chain_head.identifier != block_store.chain_head.identifier: raise ChainHeadUpdated() blkw.status = BlockStatus.Valid except BlockValidationFailure as err: blkw.status = BlockStatus.Invalid raise err except BlockValidationError as err: blkw.status = BlockStatus.Unknown raise err except ChainHeadUpdated as e: raise e except Exception as e: LOGGER.exception( "Unhandled exception BlockValidator.validate_block()") raise e @staticmethod def _compare_chain_height(head_a, head_b): """Returns True if head_a is taller, False if head_b is taller, and True if the heights are the same.""" return head_a.block_num - head_b.block_num >= 0 def _build_fork_diff_to_common_height(self, head_long, head_short): """Returns a list of blocks on the longer chain since the greatest common height between the two chains. Note that the chains may not have the same block id at the greatest common height. Args: head_long (BlockWrapper) head_short (BlockWrapper) Returns: (list of BlockWrapper) All blocks in the longer chain since the last block in the shorter chain. Ordered newest to oldest. Raises: BlockValidationError The block is missing a predecessor. Note that normally this shouldn't happen because of the completer.""" fork_diff = [] last = head_short.block_num blk = head_long while blk.block_num > last: if blk.previous_block_id == NULL_BLOCK_IDENTIFIER: break fork_diff.append(blk) try: blk = self._block_cache[blk.previous_block_id] except KeyError: raise BlockValidationError( 'Failed to build fork diff: block {} missing predecessor' .format(blk)) return blk, fork_diff def _extend_fork_diff_to_common_ancestor( self, new_blkw, cur_blkw, new_chain, cur_chain ): """ Finds a common ancestor of the two chains. new_blkw and cur_blkw must be at the same height, or this will always fail. """ while cur_blkw.identifier != new_blkw.identifier: if (cur_blkw.previous_block_id == NULL_BLOCK_IDENTIFIER or new_blkw.previous_block_id == NULL_BLOCK_IDENTIFIER): # We are at a genesis block and the blocks are not the same for b in new_chain: b.status = BlockStatus.Invalid raise BlockValidationFailure( 'Block {} rejected due to wrong genesis {}'.format( cur_blkw, new_blkw)) new_chain.append(new_blkw) try: new_blkw = self._block_cache[new_blkw.previous_block_id] except KeyError: raise BlockValidationError( 'Block {} rejected due to missing predecessor {}'.format( new_blkw, new_blkw.previous_block_id)) cur_chain.append(cur_blkw) cur_blkw = self._block_cache[cur_blkw.previous_block_id] def _compare_forks_consensus(self, chain_head, new_block): """Ask the consensus module which fork to choose. """ public_key = self._identity_signer.get_public_key().as_hex() consensus = self._load_consensus(chain_head) fork_resolver = consensus.ForkResolver( block_cache=self._block_cache, state_view_factory=self._state_view_factory, data_dir=self._data_dir, config_dir=self._config_dir, validator_id=public_key) return fork_resolver.compare_forks(chain_head, new_block) def _load_consensus(self, block): """Load the consensus module using the state as of the given block.""" if block is not None: return ConsensusFactory.get_configured_consensus_module( block.header_signature, BlockWrapper.state_view_for_block( block, self._state_view_factory)) return ConsensusFactory.get_consensus_module('genesis') @staticmethod def _get_batch_commit_changes(new_chain, cur_chain): """ Get all the batches that should be committed from the new chain and all the batches that should be uncommitted from the current chain. """ committed_batches = [] for blkw in new_chain: for batch in blkw.batches: committed_batches.append(batch) uncommitted_batches = [] for blkw in cur_chain: for batch in blkw.batches: uncommitted_batches.append(batch) return (committed_batches, uncommitted_batches) def submit_blocks_for_verification(self, blocks, callback): for block in blocks: if self.in_process(block.header_signature): LOGGER.debug("Block already in process: %s", block) continue if self.in_process(block.previous_block_id): LOGGER.debug( "Previous block '%s' in process," " adding '%s' pending", block.previous_block_id, block) self._add_block_to_pending(block) continue if self.in_pending(block.previous_block_id): LOGGER.debug( "Previous block '%s' is pending," " adding '%s' pending", block.previous_block_id, block) self._add_block_to_pending(block) continue LOGGER.debug( "Adding block %s for processing", block.identifier) # Add the block to the set of blocks being processed self._blocks_processing.add(block.identifier) # Schedule the block for processing self._thread_pool.submit( self.process_block_verification, block, self._wrap_callback(block, callback)) def _wrap_callback(self, block, callback): # Internal cleanup after verification def wrapper(commit_new_block, result, chain_head_updated=False): block = result.block LOGGER.debug("Removing block from processing %s", block.identifier) try: self._blocks_processing.remove(block.identifier) except KeyError: LOGGER.warning( "Tried to remove block from in process but it" " wasn't in processes: %s", block.identifier) # If the block was valid, submit all pending blocks for validation if block.status == BlockStatus.Valid: blocks_now_ready = self._blocks_pending.pop( block.identifier, []) self.submit_blocks_for_verification(blocks_now_ready, callback) elif block.status == BlockStatus.Invalid: # If the block was invalid, mark all pending blocks as invalid blocks_now_invalid = self._blocks_pending.pop( block.identifier, []) while blocks_now_invalid: invalid_block = blocks_now_invalid.pop() invalid_block.status = BlockStatus.Invalid LOGGER.debug( 'Marking descendant block invalid: %s', invalid_block) # Get descendants of the descendant blocks_now_invalid.extend( self._blocks_pending.pop(invalid_block.identifier, [])) elif not chain_head_updated: # If an error occured during validation, something is wrong # internally and we need to abort validation of this block # and all its children without marking them as invalid. blocks_to_remove = self._blocks_pending.pop( block.identifier, []) while blocks_to_remove: block = blocks_to_remove.pop() LOGGER.error( 'Removing block from cache and pending due to error ' 'during validation: %s; status: %s', block, block.status) del self._block_cache[block.identifier] # Get descendants of the descendant blocks_to_remove.extend( self._blocks_pending.pop(block.identifier, [])) callback(commit_new_block, result) return wrapper def in_process(self, block_id): return block_id in self._blocks_processing def in_pending(self, block_id): return block_id in self._blocks_pending def _add_block_to_pending(self, block): previous = block.previous_block_id self._blocks_pending.append(previous, block) def process_block_verification(self, block, callback): """ Main entry for Block Validation, Take a given candidate block and decide if it is valid then if it is valid determine if it should be the new head block. Returns the results to the ChainController so that the change over can be made if necessary. """ try: result = BlockValidationResult(block) LOGGER.info("Starting block validation of : %s", block) # Get the current chain_head and store it in the result chain_head = self._block_cache.block_store.chain_head result.chain_head = chain_head # Create new local variables for current and new block, since # these variables get modified later current_block = chain_head new_block = block try: # Get all the blocks since the greatest common height from the # longer chain. if self._compare_chain_height(current_block, new_block): current_block, result.current_chain =\ self._build_fork_diff_to_common_height( current_block, new_block) else: new_block, result.new_chain =\ self._build_fork_diff_to_common_height( new_block, current_block) # Add blocks to the two chains until a common ancestor is found # or raise an exception if no common ancestor is found self._extend_fork_diff_to_common_ancestor( new_block, current_block, result.new_chain, result.current_chain) except BlockValidationFailure as err: LOGGER.warning( 'Block %s failed validation: %s', block, err) block.status = BlockStatus.Invalid except BlockValidationError as err: LOGGER.error( 'Encountered an error while validating %s: %s', block, err) callback(False, result) return valid = True for blk in reversed(result.new_chain): if valid: try: self.validate_block( blk, chain_head) except BlockValidationFailure as err: LOGGER.warning( 'Block %s failed validation: %s', blk, err) valid = False except BlockValidationError as err: LOGGER.error( 'Encountered an error while validating %s: %s', blk, err) callback(False, result) result.transaction_count += block.num_transactions else: LOGGER.info( "Block marked invalid (invalid predecessor): %s", blk) blk.status = BlockStatus.Invalid if not valid: callback(False, result) return # Ask consensus if the new chain should be committed LOGGER.info( "Comparing current chain head '%s' against new block '%s'", chain_head, new_block) for i in range(max( len(result.new_chain), len(result.current_chain) )): cur = new = num = "-" if i < len(result.current_chain): cur = result.current_chain[i].header_signature[:8] num = result.current_chain[i].block_num if i < len(result.new_chain): new = result.new_chain[i].header_signature[:8] num = result.new_chain[i].block_num LOGGER.info( "Fork comparison at height %s is between %s and %s", num, cur, new) commit_new_chain = self._compare_forks_consensus(chain_head, block) # If committing the new chain, get the list of committed batches # from the current chain that need to be uncommitted and the list # of uncommitted batches from the new chain that need to be # committed. if commit_new_chain: commit, uncommit =\ self._get_batch_commit_changes( result.new_chain, result.current_chain) result.committed_batches = commit result.uncommitted_batches = uncommit if result.new_chain[0].previous_block_id \ != chain_head.identifier: self._moved_to_fork_count.inc() # Pass the results to the callback function callback(commit_new_chain, result) LOGGER.info("Finished block validation of: %s", block) except ChainHeadUpdated: LOGGER.debug( "Block validation failed due to chain head update: %s", block) callback(False, result, chain_head_updated=True) return except Exception: # pylint: disable=broad-except LOGGER.exception( "Block validation failed with unexpected error: %s", block) # callback to clean up the block out of the processing list. callback(False, result)
def verify_state(global_state_db, blockstore, bind_component, scheduler_type): """ Verify the state root hash of all blocks is in state and if not, reconstruct the missing state. Assumes that there are no "holes" in state, ie starting from genesis, state is present for all blocks up to some point and then not at all. If persist is False, this recomputes state in memory for all blocks in the blockstore and verifies the state root hashes. Raises: InvalidChainError: The chain in the blockstore is not valid. ExecutionError: An unrecoverable error was encountered during batch execution. """ state_view_factory = StateViewFactory(global_state_db) # Check if we should do state verification start_block, prev_state_root = search_for_present_state_root( blockstore, state_view_factory) if start_block is None: LOGGER.info( "Skipping state verification: chain head's state root is present") return LOGGER.info( "Recomputing missing state from block %s with %s scheduler", start_block, scheduler_type) component_thread_pool = InstrumentedThreadPoolExecutor( max_workers=10, name='Component') component_dispatcher = Dispatcher() component_service = Interconnect( bind_component, component_dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True, max_future_callback_workers=10) context_manager = ContextManager(global_state_db) transaction_executor = TransactionExecutor( service=component_service, context_manager=context_manager, settings_view_factory=SettingsViewFactory(state_view_factory), scheduler_type=scheduler_type, invalid_observers=[]) component_service.set_check_connections( transaction_executor.check_connections) component_dispatcher.add_handler( validator_pb2.Message.TP_RECEIPT_ADD_DATA_REQUEST, tp_state_handlers.TpReceiptAddDataHandler(context_manager), component_thread_pool) component_dispatcher.add_handler( validator_pb2.Message.TP_EVENT_ADD_REQUEST, tp_state_handlers.TpEventAddHandler(context_manager), component_thread_pool) component_dispatcher.add_handler( validator_pb2.Message.TP_STATE_DELETE_REQUEST, tp_state_handlers.TpStateDeleteHandler(context_manager), component_thread_pool) component_dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), component_thread_pool) component_dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), component_thread_pool) component_dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, processor_handlers.ProcessorRegisterValidationHandler(), component_thread_pool) component_dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, processor_handlers.ProcessorRegisterHandler( transaction_executor.processor_manager), component_thread_pool) component_dispatcher.add_handler( validator_pb2.Message.TP_UNREGISTER_REQUEST, processor_handlers.ProcessorUnRegisterHandler( transaction_executor.processor_manager), component_thread_pool) component_dispatcher.start() component_service.start() process_blocks( initial_state_root=prev_state_root, blocks=blockstore.get_block_iter( start_block=start_block, reverse=False), transaction_executor=transaction_executor, context_manager=context_manager, state_view_factory=state_view_factory) component_dispatcher.stop() component_service.stop() component_thread_pool.shutdown(wait=True) transaction_executor.stop() context_manager.stop()
def __init__(self, bind_network, bind_component, endpoint, peering, seeds_list, peer_list, data_dir, config_dir, identity_signing_key, scheduler_type, network_public_key=None, network_private_key=None): """Constructs a validator instance. Args: bind_network (str): the network endpoint bind_component (str): the component endpoint endpoint (str): the zmq-style URI of this validator's publically reachable endpoint peering (str): The type of peering approach. Either 'static' or 'dynamic'. In 'static' mode, no attempted topology buildout occurs -- the validator only attempts to initiate peering connections with endpoints specified in the peer_list. In 'dynamic' mode, the validator will first attempt to initiate peering connections with endpoints specified in the peer_list and then attempt to do a topology buildout starting with peer lists obtained from endpoints in the seeds_list. In either mode, the validator will accept incoming peer requests up to max_peers. seeds_list (list of str): a list of addresses to connect to in order to perform the initial topology buildout peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory config_dir (str): path to the config directory identity_signing_key (str): key validator uses for signing """ db_filename = os.path.join(data_dir, 'merkle-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('database file is %s', db_filename) merkle_db = LMDBNoLockDatabase(db_filename, 'c') delta_db_filename = os.path.join(data_dir, 'state-deltas-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('state delta store file is %s', delta_db_filename) state_delta_db = LMDBNoLockDatabase(delta_db_filename, 'c') state_delta_store = StateDeltaStore(state_delta_db) context_manager = ContextManager(merkle_db, state_delta_store) self._context_manager = context_manager state_view_factory = StateViewFactory(merkle_db) block_db_filename = os.path.join(data_dir, 'block-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = LMDBNoLockDatabase(block_db_filename, 'c') block_store = BlockStore(block_db) batch_tracker = BatchTracker(block_store) block_store.add_update_observer(batch_tracker) # setup network self._dispatcher = Dispatcher() thread_pool = ThreadPoolExecutor(max_workers=10) sig_pool = ThreadPoolExecutor(max_workers=3) self._thread_pool = thread_pool self._sig_pool = sig_pool self._service = Interconnect(bind_component, self._dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True) config_file = os.path.join(config_dir, "validator.toml") validator_config = {} if os.path.exists(config_file): with open(config_file) as fd: raw_config = fd.read() validator_config = toml.loads(raw_config) if scheduler_type is None: scheduler_type = validator_config.get("scheduler", "serial") executor = TransactionExecutor( service=self._service, context_manager=context_manager, settings_view_factory=SettingsViewFactory( StateViewFactory(merkle_db)), scheduler_type=scheduler_type, invalid_observers=[batch_tracker]) self._executor = executor self._service.set_check_connections(executor.check_connections) state_delta_processor = StateDeltaProcessor(self._service, state_delta_store, block_store) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] network_thread_pool = ThreadPoolExecutor(max_workers=10) self._network_thread_pool = network_thread_pool self._network_dispatcher = Dispatcher() secure = False if network_public_key is not None and network_private_key is not None: secure = True self._network = Interconnect( bind_network, dispatcher=self._network_dispatcher, zmq_identity=zmq_identity, secured=secure, server_public_key=network_public_key, server_private_key=network_private_key, heartbeat=True, public_endpoint=endpoint, connection_timeout=30, max_incoming_connections=100) self._gossip = Gossip(self._network, endpoint=endpoint, peering_mode=peering, initial_seed_endpoints=seeds_list, initial_peer_endpoints=peer_list, minimum_peer_connectivity=3, maximum_peer_connectivity=10, topology_check_frequency=1) completer = Completer(block_store, self._gossip) block_sender = BroadcastBlockSender(completer, self._gossip) batch_sender = BroadcastBatchSender(completer, self._gossip) chain_id_manager = ChainIdManager(data_dir) # Create and configure journal self._journal = Journal( block_store=block_store, state_view_factory=StateViewFactory(merkle_db), block_sender=block_sender, batch_sender=batch_sender, transaction_executor=executor, squash_handler=context_manager.get_squash_handler(), identity_signing_key=identity_signing_key, chain_id_manager=chain_id_manager, state_delta_processor=state_delta_processor, data_dir=data_dir, config_dir=config_dir, check_publish_block_frequency=0.1, block_cache_purge_frequency=30, block_cache_keep_time=300, batch_observers=[batch_tracker] ) self._genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_key=identity_signing_key, data_dir=data_dir, config_dir=config_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender ) responder = Responder(completer) completer.set_on_batch_received(self._journal.on_batch_received) completer.set_on_block_received(self._journal.on_block_received) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, processor_handlers.ProcessorRegisterHandler(executor.processors), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_UNREGISTER_REQUEST, processor_handlers.ProcessorUnRegisterHandler(executor.processors), thread_pool) # Set up base network handlers self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_PING, PingHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_CONNECT, ConnectHandler(network=self._network), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_DISCONNECT, DisconnectHandler(network=self._network), network_thread_pool) # Set up gossip handlers self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_REQUEST, GetPeersRequestHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_RESPONSE, GetPeersResponseHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(gossip=self._gossip), network_thread_pool) # GOSSIP_MESSAGE 1) Sends acknowledgement to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(), network_thread_pool) # GOSSIP_MESSAGE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, signature_verifier.GossipMessageSignatureVerifier(), sig_pool) # GOSSIP_MESSAGE 3) Verifies batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, structure_verifier.GossipHandlerStructureVerifier(), network_thread_pool) # GOSSIP_MESSAGE 4) Determines if we should broadcast the # message to our peers. It is important that this occur prior # to the sending of the message to the completer, as this step # relies on whether the gossip message has previously been # seen by the validator to determine whether or not forwarding # should occur self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipBroadcastHandler( gossip=self._gossip, completer=completer), network_thread_pool) # GOSSIP_MESSAGE 5) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, CompleterGossipHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_REQUEST, BlockResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, GossipBlockResponseHandler(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, signature_verifier.GossipBlockResponseSignatureVerifier(), sig_pool) # GOSSIP_BLOCK_RESPONSE 3) Check batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, structure_verifier.GossipBlockResponseStructureVerifier(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, CompleterGossipBlockResponseHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, ResponderBlockResponseHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_BATCH_ID_REQUEST, BatchByBatchIdResponderHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_TRANSACTION_ID_REQUEST, BatchByTransactionIdResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BATCH_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, GossipBatchResponseHandler(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, signature_verifier.GossipBatchResponseSignatureVerifier(), sig_pool) # GOSSIP_BATCH_RESPONSE 3) Check batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, structure_verifier.GossipBatchResponseStructureVerifier(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, CompleterGossipBatchResponseHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, ResponderBatchResponseHandler(responder, self._gossip), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, BatchListPermissionVerifier( settings_view_factory=SettingsViewFactory( StateViewFactory(merkle_db)), current_root_func=self._journal.get_current_root, ), sig_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, signature_verifier.BatchListSignatureVerifier(), sig_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, structure_verifier.BatchListStructureVerifier(), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, CompleterBatchListBroadcastHandler( completer, self._gossip), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, client_handlers.BatchSubmitFinisher(batch_tracker), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_STATUS_REQUEST, client_handlers.BatchStatusRequest(batch_tracker), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST, client_handlers.BlockListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST, client_handlers.BlockGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_LIST_REQUEST, client_handlers.BatchListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_GET_REQUEST, client_handlers.BatchGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_LIST_REQUEST, client_handlers.TransactionListRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_GET_REQUEST, client_handlers.TransactionGetRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, client_handlers.StateCurrentRequest( self._journal.get_current_root), thread_pool) # State Delta Subscription Handlers self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaSubscriberValidationHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaAddSubscriberHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_UNSUBSCRIBE_REQUEST, StateDeltaUnsubscriberHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_GET_EVENTS_REQUEST, StateDeltaGetEventsHandler(block_store, state_delta_store), thread_pool)
class BlockValidator(object): """ Responsible for validating a block, handles both chain extensions and fork will determine if the new block should be the head of the chain and return the information necessary to do the switch if necessary. """ def __init__(self, block_cache, state_view_factory, transaction_executor, squash_handler, identity_signer, data_dir, config_dir, permission_verifier, metrics_registry=None, thread_pool=None): """Initialize the BlockValidator Args: block_cache: The cache of all recent blocks and the processing state associated with them. state_view_factory: A factory that can be used to create read- only views of state for a particular merkle root, in particular the state as it existed when a particular block was the chain head. transaction_executor: The transaction executor used to process transactions. squash_handler: A parameter passed when creating transaction schedulers. identity_signer: A cryptographic signer for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. permission_verifier: The delegate for handling permission validation on blocks. metrics_registry: (Optional) Pyformance metrics registry handle for creating new metrics. thread_pool: (Optional) Executor pool used to submit block validation jobs. If not specified, a default will be created. Returns: None """ self._block_cache = block_cache self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._squash_handler = squash_handler self._identity_signer = identity_signer self._data_dir = data_dir self._config_dir = config_dir self._permission_verifier = permission_verifier self._settings_view_factory = SettingsViewFactory(state_view_factory) self._thread_pool = InstrumentedThreadPoolExecutor(1) \ if thread_pool is None else thread_pool if metrics_registry: self._moved_to_fork_count = CounterWrapper( metrics_registry.counter('chain_head_moved_to_fork_count')) else: self._moved_to_fork_count = CounterWrapper() def stop(self): self._thread_pool.shutdown(wait=True) def _get_previous_block_state_root(self, blkw): if blkw.previous_block_id == NULL_BLOCK_IDENTIFIER: return INIT_ROOT_KEY return self._block_cache[blkw.previous_block_id].state_root_hash @staticmethod def _validate_transactions_in_batch(batch, chain_commit_state): """Verify that all transactions in this batch are unique and that all transaction dependencies in this batch have been satisfied. :param batch: the batch to verify :param chain_commit_state: the current chain commit state to verify the batch against :return: Boolean: True if all dependencies are present and all transactions are unique. """ for txn in batch.transactions: txn_hdr = TransactionHeader() txn_hdr.ParseFromString(txn.header) if chain_commit_state.has_transaction(txn.header_signature): LOGGER.debug( "Batch invalid due to duplicate transaction: %s", txn.header_signature[:8]) return False for dep in txn_hdr.dependencies: if not chain_commit_state.has_transaction(dep): LOGGER.debug( "Batch invalid due to missing transaction dependency;" " transaction %s depends on %s", txn.header_signature[:8], dep[:8]) return False return True def _validate_batches_in_block( self, blkw, prev_state_root, chain_commit_state ): if blkw.block.batches: scheduler = self._transaction_executor.create_scheduler( self._squash_handler, prev_state_root) self._transaction_executor.execute(scheduler) try: for batch, has_more in look_ahead(blkw.block.batches): if chain_commit_state.has_batch( batch.header_signature): LOGGER.debug("Block(%s) rejected due to duplicate " "batch, batch: %s", blkw, batch.header_signature[:8]) raise InvalidBatch() # Verify dependencies and uniqueness if self._validate_transactions_in_batch( batch, chain_commit_state ): # Only add transactions to commit state if all # transactions in the batch are good. chain_commit_state.add_batch( batch, add_transactions=True) else: raise InvalidBatch() if has_more: scheduler.add_batch(batch) else: scheduler.add_batch(batch, blkw.state_root_hash) except InvalidBatch: LOGGER.debug("Invalid batch %s encountered during " "verification of block %s", batch.header_signature[:8], blkw) scheduler.cancel() return False except Exception: scheduler.cancel() raise scheduler.finalize() scheduler.complete(block=True) state_hash = None for batch in blkw.batches: batch_result = scheduler.get_batch_execution_result( batch.header_signature) if batch_result is not None and batch_result.is_valid: txn_results = \ scheduler.get_transaction_execution_results( batch.header_signature) blkw.execution_results.extend(txn_results) state_hash = batch_result.state_hash blkw.num_transactions += len(batch.transactions) else: return False if blkw.state_root_hash != state_hash: LOGGER.debug("Block(%s) rejected due to state root hash " "mismatch: %s != %s", blkw, blkw.state_root_hash, state_hash) return False return True def _validate_permissions(self, blkw, prev_state_root): """ Validate that all of the batch signers and transaction signer for the batches in the block are permitted by the transactor permissioning roles stored in state as of the previous block. If a transactor is found to not be permitted, the block is invalid. """ if blkw.block_num != 0: for batch in blkw.batches: if not self._permission_verifier.is_batch_signer_authorized( batch, prev_state_root, from_state=True): return False return True def _validate_on_chain_rules(self, blkw, prev_state_root): """ Validate that the block conforms to all validation rules stored in state. If the block breaks any of the stored rules, the block is invalid. """ if blkw.block_num != 0: return enforce_validation_rules( self._settings_view_factory.create_settings_view( prev_state_root), blkw.header.signer_public_key, blkw.batches) return True def validate_block(self, blkw, consensus, chain_head=None, chain=None): if blkw.status == BlockStatus.Valid: return True elif blkw.status == BlockStatus.Invalid: return False # pylint: disable=broad-except try: if chain_head is None: # Try to get the chain head from the block store; note that the # block store may also return None for the chain head if a # genesis block hasn't been committed yet. chain_head = self._block_cache.block_store.chain_head if chain is None: chain = [] chain_commit_state = ChainCommitState( self._block_cache.block_store, chain) try: prev_state_root = self._get_previous_block_state_root(blkw) except KeyError: LOGGER.debug( "Block rejected due to missing predecessor: %s", blkw) return False if not self._validate_permissions(blkw, prev_state_root): blkw.status = BlockStatus.Invalid return False public_key = \ self._identity_signer.get_public_key().as_hex() consensus_block_verifier = consensus.BlockVerifier( block_cache=self._block_cache, state_view_factory=self._state_view_factory, data_dir=self._data_dir, config_dir=self._config_dir, validator_id=public_key) if not consensus_block_verifier.verify_block(blkw): blkw.status = BlockStatus.Invalid return False if not self._validate_on_chain_rules(blkw, prev_state_root): blkw.status = BlockStatus.Invalid return False if not self._validate_batches_in_block( blkw, prev_state_root, chain_commit_state ): blkw.status = BlockStatus.Invalid return False # since changes to the chain-head can change the state of the # blocks in BlockStore we have to revalidate this block. block_store = self._block_cache.block_store # The chain_head is None when this is the genesis block or if the # block store has no chain_head. if chain_head is not None: if chain_head.identifier != block_store.chain_head.identifier: raise ChainHeadUpdated() blkw.status = BlockStatus.Valid return True except ChainHeadUpdated as e: raise e except Exception: LOGGER.exception( "Unhandled exception BlockPublisher.validate_block()") return False @staticmethod def _compare_chain_height(head_a, head_b): """Returns True if head_a is taller, False if head_b is taller, and True if the heights are the same.""" return head_a.block_num - head_b.block_num >= 0 def _build_fork_diff_to_common_height(self, head_long, head_short): """Returns a list of blocks on the longer chain since the greatest common height between the two chains. Note that the chains may not have the same block id at the greatest common height. Args: head_long (BlockWrapper) head_short (BlockWrapper) Returns: (list of BlockWrapper) All blocks in the longer chain since the last block in the shorter chain. Ordered newest to oldest. Raises: BlockValidationAborted The block is missing a predecessor. Note that normally this shouldn't happen because of the completer.""" fork_diff = [] last = head_short.block_num blk = head_long while blk.block_num > last: if blk.previous_block_id == NULL_BLOCK_IDENTIFIER: break fork_diff.append(blk) try: blk = self._block_cache[blk.previous_block_id] except KeyError: LOGGER.debug( "Failed to build fork diff due to missing predecessor: %s", blk) # Mark all blocks in the longer chain since the invalid block # as invalid. for blk in fork_diff: blk.status = BlockStatus.Invalid raise BlockValidationAborted() return blk, fork_diff def _extend_fork_diff_to_common_ancestor( self, new_blkw, cur_blkw, new_chain, cur_chain ): """ Finds a common ancestor of the two chains. new_blkw and cur_blkw must be at the same height, or this will always fail. """ while cur_blkw.identifier != new_blkw.identifier: if (cur_blkw.previous_block_id == NULL_BLOCK_IDENTIFIER or new_blkw.previous_block_id == NULL_BLOCK_IDENTIFIER): # We are at a genesis block and the blocks are not the same LOGGER.info( "Block rejected due to wrong genesis: %s %s", cur_blkw, new_blkw) for b in new_chain: b.status = BlockStatus.Invalid raise BlockValidationAborted() new_chain.append(new_blkw) try: new_blkw = self._block_cache[new_blkw.previous_block_id] except KeyError: LOGGER.info( "Block %s rejected due to missing predecessor %s", new_blkw, new_blkw.previous_block_id) for b in new_chain: b.status = BlockStatus.Invalid raise BlockValidationAborted() cur_chain.append(cur_blkw) cur_blkw = self._block_cache[cur_blkw.previous_block_id] def _compare_forks_consensus(self, consensus, chain_head, new_block): """Ask the consensus module which fork to choose. """ public_key = self._identity_signer.get_public_key().as_hex() fork_resolver = consensus.ForkResolver( block_cache=self._block_cache, state_view_factory=self._state_view_factory, data_dir=self._data_dir, config_dir=self._config_dir, validator_id=public_key) return fork_resolver.compare_forks(chain_head, new_block) @staticmethod def _get_batch_commit_changes(new_chain, cur_chain): """ Get all the batches that should be committed from the new chain and all the batches that should be uncommitted from the current chain. """ committed_batches = [] for blkw in new_chain: for batch in blkw.batches: committed_batches.append(batch) uncommitted_batches = [] for blkw in cur_chain: for batch in blkw.batches: uncommitted_batches.append(batch) return (committed_batches, uncommitted_batches) def submit_blocks_for_verification( self, blocks, consensus, callback ): for block in blocks: self._thread_pool.submit( self.process_block_verification, block, consensus, callback) def process_block_verification(self, block, consensus, callback): """ Main entry for Block Validation, Take a given candidate block and decide if it is valid then if it is valid determine if it should be the new head block. Returns the results to the ChainController so that the change over can be made if necessary. """ try: result = BlockValidationResult(block) LOGGER.info("Starting block validation of : %s", block) # Get the current chain_head and store it in the result chain_head = self._block_cache.block_store.chain_head result.chain_head = chain_head # Create new local variables for current and new block, since # these variables get modified later current_block = chain_head new_block = block # Get all the blocks since the greatest common height from the # longer chain. if self._compare_chain_height(current_block, new_block): current_block, result.current_chain =\ self._build_fork_diff_to_common_height( current_block, new_block) else: new_block, result.new_chain =\ self._build_fork_diff_to_common_height( new_block, current_block) # Add blocks to the two chains until a common ancestor is found # or raise an exception if no common ancestor is found self._extend_fork_diff_to_common_ancestor( new_block, current_block, result.new_chain, result.current_chain) valid = True for blk in reversed(result.new_chain): if valid: if not self.validate_block( blk, consensus, chain_head, result.current_chain ): LOGGER.info("Block validation failed: %s", blk) valid = False result.transaction_count += block.num_transactions else: LOGGER.info( "Block marked invalid(invalid predecessor): %s", blk) blk.status = BlockStatus.Invalid if not valid: callback(False, result) return # Ask consensus if the new chain should be committed LOGGER.info( "Comparing current chain head '%s' against new block '%s'", chain_head, new_block) for i in range(max( len(result.new_chain), len(result.current_chain) )): cur = new = num = "-" if i < len(result.current_chain): cur = result.current_chain[i].header_signature[:8] num = result.current_chain[i].block_num if i < len(result.new_chain): new = result.new_chain[i].header_signature[:8] num = result.new_chain[i].block_num LOGGER.info( "Fork comparison at height %s is between %s and %s", num, cur, new) commit_new_chain = self._compare_forks_consensus( consensus, chain_head, block) # If committing the new chain, get the list of committed batches # from the current chain that need to be uncommitted and the list # of uncommitted batches from the new chain that need to be # committed. if commit_new_chain: commit, uncommit =\ self._get_batch_commit_changes( result.new_chain, result.current_chain) result.committed_batches = commit result.uncommitted_batches = uncommit if result.new_chain[0].previous_block_id \ != chain_head.identifier: self._moved_to_fork_count.inc() # Pass the results to the callback function callback(commit_new_chain, result) LOGGER.info("Finished block validation of: %s", block) except BlockValidationAborted: callback(False, result) return except ChainHeadUpdated: callback(False, result) return except Exception: # pylint: disable=broad-except LOGGER.exception( "Block validation failed with unexpected error: %s", block) # callback to clean up the block out of the processing list. callback(False, result)
class BlockValidator(object): """ Responsible for validating a block, handles both chain extensions and fork will determine if the new block should be the head of the chain and return the information necessary to do the switch if necessary. """ def __init__(self, block_cache, state_view_factory, transaction_executor, squash_handler, identity_signer, data_dir, config_dir, permission_verifier, metrics_registry=None, thread_pool=None): """Initialize the BlockValidator Args: block_cache: The cache of all recent blocks and the processing state associated with them. state_view_factory: A factory that can be used to create read- only views of state for a particular merkle root, in particular the state as it existed when a particular block was the chain head. transaction_executor: The transaction executor used to process transactions. squash_handler: A parameter passed when creating transaction schedulers. identity_signer: A cryptographic signer for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. permission_verifier: The delegate for handling permission validation on blocks. metrics_registry: (Optional) Pyformance metrics registry handle for creating new metrics. thread_pool: (Optional) Executor pool used to submit block validation jobs. If not specified, a default will be created. Returns: None """ self._block_cache = block_cache self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._squash_handler = squash_handler self._identity_signer = identity_signer self._data_dir = data_dir self._config_dir = config_dir self._permission_verifier = permission_verifier self._settings_view_factory = SettingsViewFactory(state_view_factory) self._thread_pool = InstrumentedThreadPoolExecutor(1) \ if thread_pool is None else thread_pool if metrics_registry: self._moved_to_fork_count = CounterWrapper( metrics_registry.counter('chain_head_moved_to_fork_count')) else: self._moved_to_fork_count = CounterWrapper() # Blocks that are currently being processed self._blocks_processing = ConcurrentSet() # Descendant blocks that are waiting for an in process block # to complete self._blocks_pending = ConcurrentMultiMap() def stop(self): self._thread_pool.shutdown(wait=True) def _get_previous_block_state_root(self, blkw): if blkw.previous_block_id == NULL_BLOCK_IDENTIFIER: return INIT_ROOT_KEY return self._block_cache[blkw.previous_block_id].state_root_hash def _validate_batches_in_block(self, blkw, prev_state_root): """ Validate all batches in the block. This includes: - Validating all transaction dependencies are met - Validating there are no duplicate batches or transactions - Validating execution of all batches in the block produces the correct state root hash Args: blkw: the block of batches to validate prev_state_root: the state root to execute transactions on top of Raises: BlockValidationError: If validation fails, raises this error with the reason. MissingDependency: Validation failed because of a missing dependency. DuplicateTransaction: Validation failed because of a duplicate transaction. DuplicateBatch: Validation failed because of a duplicate batch. """ if not blkw.block.batches: return try: chain_commit_state = ChainCommitState( blkw.previous_block_id, self._block_cache, self._block_cache.block_store) scheduler = self._transaction_executor.create_scheduler( self._squash_handler, prev_state_root) self._transaction_executor.execute(scheduler) chain_commit_state.check_for_duplicate_batches(blkw.block.batches) transactions = [] for batch in blkw.block.batches: transactions.extend(batch.transactions) chain_commit_state.check_for_duplicate_transactions(transactions) chain_commit_state.check_for_transaction_dependencies(transactions) for batch, has_more in look_ahead(blkw.block.batches): if has_more: scheduler.add_batch(batch) else: scheduler.add_batch(batch, blkw.state_root_hash) except (DuplicateBatch, DuplicateTransaction, MissingDependency) as err: scheduler.cancel() raise BlockValidationError("Block {} failed validation: {}".format( blkw, err)) except Exception: scheduler.cancel() raise scheduler.finalize() scheduler.complete(block=True) state_hash = None for batch in blkw.batches: batch_result = scheduler.get_batch_execution_result( batch.header_signature) if batch_result is not None and batch_result.is_valid: txn_results = \ scheduler.get_transaction_execution_results( batch.header_signature) blkw.execution_results.extend(txn_results) state_hash = batch_result.state_hash blkw.num_transactions += len(batch.transactions) else: raise BlockValidationError( "Block {} failed validation: Invalid batch " "{}".format(blkw, batch)) if blkw.state_root_hash != state_hash: raise BlockValidationError( "Block {} failed state root hash validation. Expected {}" " but got {}".format(blkw, blkw.state_root_hash, state_hash)) def _validate_permissions(self, blkw, prev_state_root): """ Validate that all of the batch signers and transaction signer for the batches in the block are permitted by the transactor permissioning roles stored in state as of the previous block. If a transactor is found to not be permitted, the block is invalid. """ if blkw.block_num != 0: for batch in blkw.batches: if not self._permission_verifier.is_batch_signer_authorized( batch, prev_state_root, from_state=True): return False return True def _validate_on_chain_rules(self, blkw, prev_state_root): """ Validate that the block conforms to all validation rules stored in state. If the block breaks any of the stored rules, the block is invalid. """ if blkw.block_num != 0: return enforce_validation_rules( self._settings_view_factory.create_settings_view( prev_state_root), blkw.header.signer_public_key, blkw.batches) return True def validate_block(self, blkw, chain_head=None): if blkw.status == BlockStatus.Valid: return elif blkw.status == BlockStatus.Invalid: raise BlockValidationError( 'Block {} is already invalid'.format(blkw)) # pylint: disable=broad-except try: if chain_head is None: # Try to get the chain head from the block store; note that the # block store may also return None for the chain head if a # genesis block hasn't been committed yet. chain_head = self._block_cache.block_store.chain_head try: prev_state_root = self._get_previous_block_state_root(blkw) except KeyError: raise BlockValidationError( 'Block {} rejected due to missing predecessor'.format( blkw)) if not self._validate_permissions(blkw, prev_state_root): raise BlockValidationError( 'Block {} failed permission validation'.format(blkw)) try: prev_block = self._block_cache[blkw.previous_block_id] except KeyError: prev_block = None consensus = self._load_consensus(prev_block) public_key = \ self._identity_signer.get_public_key().as_hex() consensus_block_verifier = consensus.BlockVerifier( block_cache=self._block_cache, state_view_factory=self._state_view_factory, data_dir=self._data_dir, config_dir=self._config_dir, validator_id=public_key) if not consensus_block_verifier.verify_block(blkw): raise BlockValidationError( 'Block {} failed {} consensus validation'.format( blkw, consensus)) if not self._validate_on_chain_rules(blkw, prev_state_root): raise BlockValidationError( 'Block {} failed on-chain validation rules'.format(blkw)) self._validate_batches_in_block(blkw, prev_state_root) # since changes to the chain-head can change the state of the # blocks in BlockStore we have to revalidate this block. block_store = self._block_cache.block_store # The chain_head is None when this is the genesis block or if the # block store has no chain_head. if chain_head is not None: if chain_head.identifier != block_store.chain_head.identifier: raise ChainHeadUpdated() blkw.status = BlockStatus.Valid except BlockValidationError as err: blkw.status = BlockStatus.Invalid raise err except ChainHeadUpdated as e: raise e except Exception as e: LOGGER.exception( "Unhandled exception BlockValidator.validate_block()") raise e @staticmethod def _compare_chain_height(head_a, head_b): """Returns True if head_a is taller, False if head_b is taller, and True if the heights are the same.""" return head_a.block_num - head_b.block_num >= 0 def _build_fork_diff_to_common_height(self, head_long, head_short): """Returns a list of blocks on the longer chain since the greatest common height between the two chains. Note that the chains may not have the same block id at the greatest common height. Args: head_long (BlockWrapper) head_short (BlockWrapper) Returns: (list of BlockWrapper) All blocks in the longer chain since the last block in the shorter chain. Ordered newest to oldest. Raises: BlockValidationError The block is missing a predecessor. Note that normally this shouldn't happen because of the completer.""" fork_diff = [] last = head_short.block_num blk = head_long while blk.block_num > last: if blk.previous_block_id == NULL_BLOCK_IDENTIFIER: break fork_diff.append(blk) try: blk = self._block_cache[blk.previous_block_id] except KeyError: LOGGER.debug( "Failed to build fork diff due to missing predecessor: %s", blk) # Mark all blocks in the longer chain since the invalid block # as invalid. for blk in fork_diff: blk.status = BlockStatus.Invalid raise BlockValidationError( 'Failed to build fork diff: block {} missing predecessor'. format(blk)) return blk, fork_diff def _extend_fork_diff_to_common_ancestor(self, new_blkw, cur_blkw, new_chain, cur_chain): """ Finds a common ancestor of the two chains. new_blkw and cur_blkw must be at the same height, or this will always fail. """ while cur_blkw.identifier != new_blkw.identifier: if (cur_blkw.previous_block_id == NULL_BLOCK_IDENTIFIER or new_blkw.previous_block_id == NULL_BLOCK_IDENTIFIER): # We are at a genesis block and the blocks are not the same for b in new_chain: b.status = BlockStatus.Invalid raise BlockValidationError( 'Block {} rejected due to wrong genesis {}'.format( cur_blkw, new_blkw)) new_chain.append(new_blkw) try: new_blkw = self._block_cache[new_blkw.previous_block_id] except KeyError: for b in new_chain: b.status = BlockStatus.Invalid raise BlockValidationError( 'Block {} rejected due to missing predecessor {}'.format( new_blkw, new_blkw.previous_block_id)) cur_chain.append(cur_blkw) cur_blkw = self._block_cache[cur_blkw.previous_block_id] def _compare_forks_consensus(self, chain_head, new_block): """Ask the consensus module which fork to choose. """ public_key = self._identity_signer.get_public_key().as_hex() consensus = self._load_consensus(chain_head) fork_resolver = consensus.ForkResolver( block_cache=self._block_cache, state_view_factory=self._state_view_factory, data_dir=self._data_dir, config_dir=self._config_dir, validator_id=public_key) return fork_resolver.compare_forks(chain_head, new_block) def _load_consensus(self, block): """Load the consensus module using the state as of the given block.""" if block is not None: return ConsensusFactory.get_configured_consensus_module( block.header_signature, BlockWrapper.state_view_for_block(block, self._state_view_factory)) return ConsensusFactory.get_consensus_module('genesis') @staticmethod def _get_batch_commit_changes(new_chain, cur_chain): """ Get all the batches that should be committed from the new chain and all the batches that should be uncommitted from the current chain. """ committed_batches = [] for blkw in new_chain: for batch in blkw.batches: committed_batches.append(batch) uncommitted_batches = [] for blkw in cur_chain: for batch in blkw.batches: uncommitted_batches.append(batch) return (committed_batches, uncommitted_batches) def submit_blocks_for_verification(self, blocks, callback): for block in blocks: if self.in_process(block.header_signature): LOGGER.debug("Block already in process: %s", block) continue if self.in_process(block.previous_block_id): LOGGER.debug( "Previous block '%s' in process," " adding '%s' pending", block.previous_block_id, block) self._add_block_to_pending(block) continue if self.in_pending(block.previous_block_id): LOGGER.debug( "Previous block '%s' is pending," " adding '%s' pending", block.previous_block_id, block) self._add_block_to_pending(block) continue LOGGER.debug("Adding block %s for processing", block.identifier) # Add the block to the set of blocks being processed self._blocks_processing.add(block.identifier) # Schedule the block for processing self._thread_pool.submit(self.process_block_verification, block, self._wrap_callback(block, callback)) def _wrap_callback(self, block, callback): # Internal cleanup after verification def wrapper(commit_new_block, result): LOGGER.debug("Removing block from processing %s", block.identifier[:6]) try: self._blocks_processing.remove(block.identifier) except KeyError: LOGGER.warning( "Tried to remove block from in process but it" " wasn't in processes: %s", block.identifier) # If the block is invalid, mark all descendant blocks as invalid # and remove from pending. if block.status == BlockStatus.Valid: blocks_now_ready = self._blocks_pending.pop( block.identifier, []) self.submit_blocks_for_verification(blocks_now_ready, callback) else: # Get all the pending blocks that can now be processed blocks_now_invalid = self._blocks_pending.pop( block.identifier, []) while blocks_now_invalid: invalid_block = blocks_now_invalid.pop() invalid_block.status = BlockStatus.Invalid LOGGER.debug('Marking descendant block invalid: %s', invalid_block) # Get descendants of the descendant blocks_now_invalid.extend( self._blocks_pending.pop(invalid_block.identifier, [])) callback(commit_new_block, result) return wrapper def in_process(self, block_id): return block_id in self._blocks_processing def in_pending(self, block_id): return block_id in self._blocks_pending def _add_block_to_pending(self, block): previous = block.previous_block_id self._blocks_pending.append(previous, block) def process_block_verification(self, block, callback): """ Main entry for Block Validation, Take a given candidate block and decide if it is valid then if it is valid determine if it should be the new head block. Returns the results to the ChainController so that the change over can be made if necessary. """ try: result = BlockValidationResult(block) LOGGER.info("Starting block validation of : %s", block) # Get the current chain_head and store it in the result chain_head = self._block_cache.block_store.chain_head result.chain_head = chain_head # Create new local variables for current and new block, since # these variables get modified later current_block = chain_head new_block = block try: # Get all the blocks since the greatest common height from the # longer chain. if self._compare_chain_height(current_block, new_block): current_block, result.current_chain =\ self._build_fork_diff_to_common_height( current_block, new_block) else: new_block, result.new_chain =\ self._build_fork_diff_to_common_height( new_block, current_block) # Add blocks to the two chains until a common ancestor is found # or raise an exception if no common ancestor is found self._extend_fork_diff_to_common_ancestor( new_block, current_block, result.new_chain, result.current_chain) except BlockValidationError as err: LOGGER.warning('%s', err) callback(False, result) return valid = True for blk in reversed(result.new_chain): if valid: try: self.validate_block(blk, chain_head) except BlockValidationError as err: LOGGER.warning('Block %s failed validation: %s', blk, err) valid = False result.transaction_count += block.num_transactions else: LOGGER.info( "Block marked invalid(invalid predecessor): %s", blk) blk.status = BlockStatus.Invalid if not valid: callback(False, result) return # Ask consensus if the new chain should be committed LOGGER.info( "Comparing current chain head '%s' against new block '%s'", chain_head, new_block) for i in range( max(len(result.new_chain), len(result.current_chain))): cur = new = num = "-" if i < len(result.current_chain): cur = result.current_chain[i].header_signature[:8] num = result.current_chain[i].block_num if i < len(result.new_chain): new = result.new_chain[i].header_signature[:8] num = result.new_chain[i].block_num LOGGER.info( "Fork comparison at height %s is between %s and %s", num, cur, new) commit_new_chain = self._compare_forks_consensus(chain_head, block) # If committing the new chain, get the list of committed batches # from the current chain that need to be uncommitted and the list # of uncommitted batches from the new chain that need to be # committed. if commit_new_chain: commit, uncommit =\ self._get_batch_commit_changes( result.new_chain, result.current_chain) result.committed_batches = commit result.uncommitted_batches = uncommit if result.new_chain[0].previous_block_id \ != chain_head.identifier: self._moved_to_fork_count.inc() # Pass the results to the callback function callback(commit_new_chain, result) LOGGER.info("Finished block validation of: %s", block) except ChainHeadUpdated: callback(False, result) return except Exception: # pylint: disable=broad-except LOGGER.exception( "Block validation failed with unexpected error: %s", block) # callback to clean up the block out of the processing list. callback(False, result)
class BlockValidator: """ Responsible for validating a block. """ def __init__(self, block_manager, block_store, state_view_factory, transaction_executor, identity_signer, data_dir, config_dir, permission_verifier, thread_pool=None): """Initialize the BlockValidator Args: block_manager: The component that stores blocks and maintains integrity of the predecessor relationship. state_view_factory: A factory that can be used to create read- only views of state for a particular merkle root, in particular the state as it existed when a particular block was the chain head. transaction_executor: The transaction executor used to process transactions. identity_signer: A cryptographic signer for signing blocks. data_dir: Path to location where persistent data for the consensus module can be stored. config_dir: Path to location where config data for the consensus module can be found. permission_verifier: The delegate for handling permission validation on blocks. thread_pool: (Optional) Executor pool used to submit block validation jobs. If not specified, a default will be created. Returns: None """ self._block_manager = block_manager self._block_store = block_store self._block_validity_fn = None self._block_scheduler = BlockScheduler(block_manager) self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._identity_signer = identity_signer self._data_dir = data_dir self._config_dir = config_dir self._permission_verifier = permission_verifier self._settings_view_factory = SettingsViewFactory(state_view_factory) self._thread_pool = InstrumentedThreadPoolExecutor(1) \ if thread_pool is None else thread_pool def stop(self): self._thread_pool.shutdown(wait=True) def set_block_validity_fn(self, func): self._block_validity_fn = func self._block_scheduler.set_block_validity_fn(func=func) def _block_validity(self, block_id): return self._block_validity_fn(block_id) def _get_previous_block_state_root(self, block): block_header = BlockHeader() block_header.ParseFromString(block.header) if block_header.previous_block_id == NULL_BLOCK_IDENTIFIER: return INIT_ROOT_KEY try: block = next( self._block_manager.get([block_header.previous_block_id])) except StopIteration: return None block_header = BlockHeader() block_header.ParseFromString(block.header) return block_header.state_root_hash def _validate_batches_in_block(self, block, prev_state_root): """ Validate all batches in the block. This includes: - Validating all transaction dependencies are met - Validating there are no duplicate batches or transactions - Validating execution of all batches in the block produces the correct state root hash Args: block: the block of batches to validate prev_state_root: the state root to execute transactions on top of Raises: BlockValidationFailure: If validation fails, raises this error with the reason. MissingDependency: Validation failed because of a missing dependency. DuplicateTransaction: Validation failed because of a duplicate transaction. DuplicateBatch: Validation failed because of a duplicate batch. """ if not block.batches: return scheduler = None try: while True: try: chain_head = self._block_store.chain_head block_header = BlockHeader() block_header.ParseFromString(block.header) chain_commit_state = ChainCommitState( block_header.previous_block_id, self._block_manager, self._block_store) chain_commit_state.check_for_duplicate_batches( block.batches) transactions = [] for batch in block.batches: transactions.extend(batch.transactions) chain_commit_state.check_for_duplicate_transactions( transactions) chain_commit_state.check_for_transaction_dependencies( transactions) if not self._check_chain_head_updated(chain_head, block): break except (DuplicateBatch, DuplicateTransaction, MissingDependency, BlockStoreUpdated) as err: if not self._check_chain_head_updated(chain_head, block): raise BlockValidationFailure( "Block {} failed validation: {}".format( block.header_signature, err)) scheduler = self._transaction_executor.create_scheduler( prev_state_root) for batch, has_more in look_ahead(block.batches): if has_more: scheduler.add_batch(batch) else: scheduler.add_batch(batch, block_header.state_root_hash) except Exception: if scheduler is not None: scheduler.cancel() raise scheduler.finalize() scheduler.complete(block=True) state_hash = None execution_results = [] num_transactions = 0 for batch in block.batches: batch_result = scheduler.get_batch_execution_result( batch.header_signature) if batch_result is not None and batch_result.is_valid: txn_results = \ scheduler.get_transaction_execution_results( batch.header_signature) execution_results.extend(txn_results) state_hash = batch_result.state_hash num_transactions += len(batch.transactions) else: raise BlockValidationFailure( "Block {} failed validation: Invalid batch " "{}".format(block.header_signature, batch)) if block_header.state_root_hash != state_hash: raise BlockValidationFailure( "Block {} failed state root hash validation. Expected {}" " but got {}".format(block.header_signature, block_header.state_root_hash, state_hash)) return execution_results, num_transactions def _check_chain_head_updated(self, chain_head, block): # The validity of blocks depends partially on whether or not # there are any duplicate transactions or batches in the block. # This can only be checked accurately if the block store does # not update during validation. The current practice is the # assume this will not happen and, if it does, to reprocess the # validation. This has been experimentally proven to be more # performant than locking the chain head and block store around # duplicate checking. if chain_head is None: return False current_chain_head = self._block_store.chain_head if chain_head.header_signature != current_chain_head.header_signature: LOGGER.warning( "Chain head updated from %s to %s while checking " "duplicates and dependencies in block %s. " "Reprocessing validation.", chain_head, current_chain_head, block) return True return False def _validate_permissions(self, block, prev_state_root): """ Validate that all of the batch signers and transaction signer for the batches in the block are permitted by the transactor permissioning roles stored in state as of the previous block. If a transactor is found to not be permitted, the block is invalid. """ block_header = BlockHeader() block_header.ParseFromString(block.header) if block_header.block_num != 0: for batch in block.batches: if not self._permission_verifier.is_batch_signer_authorized( batch, prev_state_root, from_state=True): return False return True def _validate_on_chain_rules(self, block, prev_state_root): """ Validate that the block conforms to all validation rules stored in state. If the block breaks any of the stored rules, the block is invalid. """ block_header = BlockHeader() block_header.ParseFromString(block.header) if block_header.block_num != 0: return enforce_validation_rules( self._settings_view_factory.create_settings_view( prev_state_root), block_header.signer_public_key, block.batches) return True def validate_block(self, block): block_header = BlockHeader() block_header.ParseFromString(block.header) # pylint: disable=broad-except try: try: prev_block = next( self._block_manager.get([block_header.previous_block_id])) except StopIteration: if block_header.previous_block_id != NULL_BLOCK_IDENTIFIER: raise else: if self._block_validity(prev_block.header_signature) == \ BlockStatus.Invalid: raise BlockValidationFailure( "Block {} rejected due to invalid predecessor" " {}".format(block.header_signature, prev_block.header_signature)) elif self._block_validity(prev_block.header_signature) == \ BlockStatus.Unknown: raise BlockValidationError( "Attempted to validate block {} before its predecessor" " {}".format(block.header_signature, prev_block.header_signature)) try: prev_state_root = self._get_previous_block_state_root(block) except KeyError: raise BlockValidationError( 'Block {} rejected due to missing predecessor'.format( block.header_signature)) if not self._validate_permissions(block, prev_state_root): raise BlockValidationFailure( 'Block {} failed permission validation'.format( block.header_signature)) if not self._validate_on_chain_rules(block, prev_state_root): raise BlockValidationFailure( 'Block {} failed on-chain validation rules'.format(block)) execution_results, num_txns = \ self._validate_batches_in_block(block, prev_state_root) return BlockValidationResult(execution_results=execution_results, block_id=block.header_signature, num_transactions=num_txns, status=BlockStatus.Valid) except BlockValidationFailure as err: raise err except BlockValidationError as err: raise err except Exception as e: LOGGER.exception( "Unhandled exception BlockValidator.validate_block()") raise e def submit_blocks_for_verification(self, blocks, callback): ready = self._block_scheduler.schedule(blocks) for block in ready: # Schedule the block for processing self._thread_pool.submit(self.process_block_verification, block, callback) def process_pending(self, block, callback): """Removes the block from processing if it is already valid, and returns pending blocks that were waiting on the valid block. """ ready = [] block_id = block.header_signature if self._block_validity(block_id) == BlockStatus.Valid: ready.extend(self._block_scheduler.done(block)) self.submit_blocks_for_verification(ready, callback) def has_block(self, block_id): return block_id in self._block_scheduler def return_block_failure(self, block): return BlockValidationResult(execution_results=[], num_transactions=0, status=BlockStatus.Invalid, block_id=block.header_signature) def process_block_verification(self, block, callback): """ Main entry for Block Validation, Take a given candidate block and decide if it is valid then if it is valid determine if it should be the new head block. Returns the results to the ChainController so that the change over can be made if necessary. """ result = None try: result = self.validate_block(block) LOGGER.info('Block %s passed validation', block.header_signature) except BlockValidationFailure as err: LOGGER.warning('Block %s failed validation: %s', block.header_signature, err) result = self.return_block_failure(block) except BlockValidationError as err: LOGGER.error('Encountered an error while validating %s: %s', block.header_signature, err) except Exception: # pylint: disable=broad-except LOGGER.exception( "Block validation failed with unexpected error: %s", block.header_signature) if result: callback(result)
class TestSettingsView(unittest.TestCase): def __init__(self, test_name): super().__init__(test_name) self._settings_view_factory = None self._current_root_hash = None def setUp(self): database = DictDatabase() state_view_factory = StateViewFactory(database) self._settings_view_factory = SettingsViewFactory(state_view_factory) merkle_db = MerkleDatabase(database) self._current_root_hash = merkle_db.update({ TestSettingsView._address('my.setting'): TestSettingsView._setting_entry('my.setting', '10'), TestSettingsView._address('my.setting.list'): TestSettingsView._setting_entry('my.setting.list', '10,11,12'), TestSettingsView._address('my.other.list'): TestSettingsView._setting_entry('my.other.list', '13;14;15') }, virtual=False) def test_get_setting(self): """Verifies the correct operation of get_setting() by using it to get the config setting stored as "my.setting" and compare it to '10' (the value set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual('10', settings_view.get_setting('my.setting')) def test_get_setting_with_type_coercion(self): """Verifies the correct operation of get_setting() by using it to get the config setting stored as "my.setting" with a int type coercion function and compare it to the int 10 (the value set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual(10, settings_view.get_setting('my.setting', value_type=int)) def test_get_setting_not_found(self): """Verifies the correct operation of get_setting() by using it to return None when an unknown setting is requested. """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertIsNone(settings_view.get_setting('non-existant.setting')) def test_get_setting_not_found_with_default(self): """Verifies the correct operation of get_setting() by using it to return a default value when an unknown setting is requested. """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual('default', settings_view.get_setting('non-existant.setting', default_value='default')) def test_get_setting_list(self): """Verifies the correct operation of get_setting_list() by using it to get the config setting stored as "my.setting.list" and compare it to ['10', '11', '12'] (the split value set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) # Verify we can still get the "raw" setting self.assertEqual('10,11,12', settings_view.get_setting('my.setting.list')) # And now the split setting self.assertEqual( ['10', '11', '12'], settings_view.get_setting_list('my.setting.list')) def test_get_setting_list_not_found(self): """Verifies the correct operation of get_setting_list() by using it to return None when an unknown setting is requested. """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertIsNone( settings_view.get_setting_list('non-existant.setting.list')) def test_get_setting_list_not_found_with_default(self): """Verifies the correct operation of get_setting_list() by using it to return a default value when an unknown setting is requested. """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual( [], settings_view.get_setting_list('non-existant.list', default_value=[])) def test_get_setting_list_alternate_delimiter(self): """Verifies the correct operation of get_setting_list() by using it to get the config setting stored as "my.other.list" and compare it to ['13', '14', '15'] (the value, split along an alternate delimiter, set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual( ['13', '14', '15'], settings_view.get_setting_list('my.other.list', delimiter=';')) def test_get_setting_list_with_type_coercion(self): """Verifies the correct operation of get_setting_list() by using it to get the integer type-coerced config setting stored as "my.setting.list" and compare it to [10, 11, 12] (the split, type-coerced, value set during setUp()). """ settings_view = self._settings_view_factory.create_settings_view( self._current_root_hash) self.assertEqual( [10, 11, 12], settings_view.get_setting_list('my.setting.list', value_type=int)) @staticmethod def _address(key): return '000000' + _key_to_address(key) @staticmethod def _setting_entry(key, value): return Setting( entries=[Setting.Entry(key=key, value=value)] ).SerializeToString()