def _build_block(self, chain_head): """ Build a candidate block and construct the consensus object to validate it. :param chain_head: The block to build on top of. :return: (BlockBuilder) - The candidate block in a BlockBuilder wrapper. """ state_view = BlockWrapper.state_view_for_block( chain_head, self._state_view_factory) consensus_module = ConsensusFactory.get_configured_consensus_module( chain_head.header_signature, state_view) self._consensus = consensus_module.\ BlockPublisher(block_cache=self._block_cache, state_view_factory=self._state_view_factory, batch_publisher=self._batch_publisher, data_dir=self._data_dir, validator_id=self._identity_public_key) block_header = BlockHeader( block_num=chain_head.block_num + 1, previous_block_id=chain_head.header_signature, signer_pubkey=self._identity_public_key) block_builder = BlockBuilder(block_header) if not self._consensus.initialize_block(block_builder.block_header): LOGGER.debug("Consensus not ready to build candidate block.") return None # Cancel the previous scheduler if it did not complete. if self._scheduler is not None \ and not self._scheduler.complete(block=False): self._scheduler.cancel() # create a new scheduler self._scheduler = self._transaction_executor.create_scheduler( self._squash_handler, chain_head.state_root_hash) # build the TransactionCache self._committed_txn_cache = TransactionCache( self._block_cache.block_store) if chain_head.header_signature not in self._block_cache.block_store: # if we opportunistically building a block # we need to check make sure we track that blocks transactions # as recorded. for batch in chain_head.block.batches: for txn in batch.transactions: self._committed_txn_cache.add_txn(txn.header_signature) self._transaction_executor.execute(self._scheduler) for batch in self._pending_batches: self._validate_batch(batch) return block_builder
def __init__(self, transaction_executor, block_cache, state_view_factory, block_sender, batch_sender, squash_handler, chain_head, identity_signing_key, data_dir): """ Initialize the BlockPublisher object Args: transaction_executor (:obj:`TransactionExecutor`): A TransactionExecutor instance. block_cache (:obj:`BlockCache`): A BlockCache instance. state_view_factory (:obj:`StateViewFactory`): StateViewFactory for read-only state views. block_sender (:obj:`BlockSender`): The BlockSender instance. batch_sender (:obj:`BatchSender`): The BatchSender instance. squash_handler (function): Squash handler function for merging contexts. chain_head (:obj:`BlockWrapper`): The initial chain head. identity_signing_key (str): Private key for signing blocks """ self._lock = RLock() self._candidate_block = None # the next block in potentia self._consensus = None self._block_cache = block_cache self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._block_sender = block_sender self._batch_publisher = BatchPublisher(identity_signing_key, batch_sender) self._pending_batches = [] # batches we are waiting for validation, # arranged in the order of batches received. self._committed_txn_cache = TransactionCache(self._block_cache. block_store) # Look-up cache for transactions that are committed in the current # chain. Cache is used here so that we can support opportunistically # building on top of a block we published. As well as hold the state # of the transactions already added to the candidate block. self._scheduler = None self._chain_head = chain_head # block (BlockWrapper) self._squash_handler = squash_handler self._identity_signing_key = identity_signing_key self._identity_public_key = \ signing.generate_pubkey(self._identity_signing_key) self._data_dir = data_dir
def on_chain_updated(self, chain_head, committed_batches=None, uncommitted_batches=None): """ The existing chain has been updated, the current head block has changed. :param chain_head: the new head of block_chain :param committed_batches: the set of batches that were committed as part of the new chain. :param uncommitted_batches: the list of transactions if any that are now de-committed when the new chain was selected. :return: None """ try: with self._lock: LOGGER.info('Now building on top of block: %s', chain_head) self._chain_head = chain_head if self._candidate_block is not None and \ chain_head is not None and \ chain_head.identifier == \ self._candidate_block.previous_block_id: # nothing to do. We are building of the current head. # This can happen after we publish a block and # opportunistically create a new block. return elif chain_head is None: # we don't have a chain head, we cannot build blocks self._candidate_block = None self._consensus = None self._committed_txn_cache =\ TransactionCache(self._block_cache.block_store) for batch in self._pending_batches: self._committed_txn_cache.add_batch(batch) else: self._rebuild_pending_batches(committed_batches, uncommitted_batches) self._candidate_block = self._build_block(chain_head) # pylint: disable=broad-except except Exception as exc: LOGGER.critical("on_chain_updated exception.") LOGGER.exception(exc)
def _build_block(self, chain_head): """ Build a candidate block and construct the consensus object to validate it. :param chain_head: The block to build on top of. :return: (BlockBuilder) - The candidate block in a BlockBuilder wrapper. """ prev_state = self._get_previous_block_root_state_hash(chain_head) state_view = self._state_view_factory. \ create_view(prev_state) consensus_module = ConsensusFactory.get_configured_consensus_module( state_view) self._consensus = consensus_module.\ BlockPublisher(block_cache=self._block_cache, state_view=state_view) block_header = BlockHeader( block_num=chain_head.block_num + 1, previous_block_id=chain_head.header_signature) block_builder = BlockBuilder(block_header) if not self._consensus.initialize_block(block_builder.block_header): LOGGER.debug("Consensus not ready to build candidate block.") # create a new scheduler self._scheduler = self._transaction_executor.create_scheduler( self._squash_handler, chain_head.state_root_hash) # build the TransactionCache self._committed_txn_cache = TransactionCache( self._block_cache.block_store) if chain_head.header_signature not in self._block_cache.block_store: # if we opportunistically building a block # we need to check make sure we track that blocks transactions # as recorded. for batch in chain_head.block.batches: for txn in batch.transactions: self._committed_txn_cache.add_txn(txn.header_signature) self._transaction_executor.execute(self._scheduler) for batch in self._pending_batches: self._validate_batch(batch) return block_builder
def _build_candidate_block(self, chain_head): """ Build a candidate block and construct the consensus object to validate it. :param chain_head: The block to build on top of. :return: (BlockBuilder) - The candidate block in a BlockBuilder wrapper. """ state_view = BlockWrapper.state_view_for_block( chain_head, self._state_view_factory) consensus_module = ConsensusFactory.get_configured_consensus_module( chain_head.header_signature, state_view) config_view = ConfigView(state_view) max_batches = config_view.get_setting( 'sawtooth.publisher.max_batches_per_block', default_value=0, value_type=int) consensus = consensus_module.\ BlockPublisher(block_cache=self._block_cache, state_view_factory=self._state_view_factory, batch_publisher=self._batch_publisher, data_dir=self._data_dir, config_dir=self._config_dir, validator_id=self._identity_public_key) block_header = BlockHeader( block_num=chain_head.block_num + 1, previous_block_id=chain_head.header_signature, signer_pubkey=self._identity_public_key) block_builder = BlockBuilder(block_header) if not consensus.initialize_block(block_builder.block_header): LOGGER.debug("Consensus not ready to build candidate block.") return None # create a new scheduler scheduler = self._transaction_executor.create_scheduler( self._squash_handler, chain_head.state_root_hash) # build the TransactionCache committed_txn_cache = TransactionCache(self._block_cache.block_store) self._transaction_executor.execute(scheduler) self._candidate_block = _CandidateBlock(self._block_cache.block_store, consensus, scheduler, committed_txn_cache, block_builder, max_batches) for batch in self._pending_batches: if self._candidate_block.can_add_batch: self._candidate_block.add_batch(batch) else: break
def _set_genesis(self, block): # This is used by a non-genesis journal when it has received the # genesis block from the genesis validator if block.previous_block_id == NULL_BLOCK_IDENTIFIER: chain_id = self._chain_id_manager.get_block_chain_id() if chain_id is not None and chain_id != block.identifier: LOGGER.warning( "Block id does not match block chain id %s. " "Cannot set initial chain head.: %s", chain_id[:8], block.identifier[:8]) else: state_view = self._state_view_factory.create_view() consensus_module = \ ConsensusFactory.get_configured_consensus_module( NULL_BLOCK_IDENTIFIER, state_view) committed_txn = TransactionCache(self._block_cache.block_store) validator = BlockValidator( consensus_module=consensus_module, new_block=block, chain_head=self._chain_head, block_cache=self._block_cache, state_view_factory=self._state_view_factory, done_cb=self.on_block_validated, executor=self._transaction_executor, squash_handler=self._squash_handler, identity_signing_key=self._identity_signing_key, data_dir=self._data_dir, config_dir=self._config_dir) valid = validator.validate_block(block, committed_txn) if valid: if chain_id is None: self._chain_id_manager.save_block_chain_id( block.identifier) self._block_store.update_chain([block]) self._chain_head = block self._notify_on_chain_updated(self._chain_head) else: LOGGER.warning( "The genesis block is not valid. Cannot " "set chain head: %s", block) else: LOGGER.warning( "Cannot set initial chain head, this is not a " "genesis block: %s", block)
def run(self): """ Main entry for Block Validation, Take a given candidate block and decide if it is valid then if it is valid determine if it should be the new head block. Returns the results to the ChainController so that the change over can be made if necessary. """ try: LOGGER.info("Starting block validation of : %s", self._new_block) cur_chain = self._result["cur_chain"] # ordered list of the # current chain blocks new_chain = self._result["new_chain"] # ordered list of the new # chain blocks # 1) Find the common ancestor block, the root of the fork. # walk back till both chains are the same height (new_blkw, cur_blkw) = self._find_common_height(new_chain, cur_chain) # 2) Walk back until we find the common ancestor self._find_common_ancestor(new_blkw, cur_blkw, new_chain, cur_chain) # 3) Determine the validity of the new fork # build the transaction cache to simulate the state of the # chain at the common root. committed_txn = TransactionCache(self._block_cache.block_store) for block in cur_chain: for batch in block.batches: committed_txn.uncommit_batch(batch) valid = True for block in reversed(new_chain): if valid: if not self.validate_block(block, committed_txn): LOGGER.info("Block validation failed: %s", block) valid = False else: LOGGER.info( "Block marked invalid(invalid predecessor): " + "%s", block) block.status = BlockStatus.Invalid if not valid: self._done_cb(False, self._result) return # 4) Evaluate the 2 chains to see if the new chain should be # committed commit_new_chain = self._test_commit_new_chain() # 5) Consensus to compute batch sets (only if we are switching). if commit_new_chain: (self._result["committed_batches"], self._result["uncommitted_batches"]) =\ self._compute_batch_change(new_chain, cur_chain) # 6) Tell the journal we are done. self._done_cb(commit_new_chain, self._result) LOGGER.info("Finished block validation of: %s", self._new_block) except BlockValidationAborted: return except Exception as exc: # pylint: disable=broad-except LOGGER.error("Block validation failed with unexpected error: %s", self._new_block) LOGGER.exception(exc) # callback to clean up the block out of the processing list. self._done_cb(False, self._result)
class BlockPublisher(object): """ Responsible for generating new blocks and publishing them when the Consensus deems it appropriate. """ def __init__(self, transaction_executor, block_cache, state_view_factory, block_sender, batch_sender, squash_handler, chain_head, identity_signing_key, data_dir): """ Initialize the BlockPublisher object Args: transaction_executor (:obj:`TransactionExecutor`): A TransactionExecutor instance. block_cache (:obj:`BlockCache`): A BlockCache instance. state_view_factory (:obj:`StateViewFactory`): StateViewFactory for read-only state views. block_sender (:obj:`BlockSender`): The BlockSender instance. batch_sender (:obj:`BatchSender`): The BatchSender instance. squash_handler (function): Squash handler function for merging contexts. chain_head (:obj:`BlockWrapper`): The initial chain head. identity_signing_key (str): Private key for signing blocks data_dir (str): path to location where persistent data for the consensus module can be stored. """ self._lock = RLock() self._candidate_block = None # the next block in potential chain self._consensus = None self._block_cache = block_cache self._state_view_factory = state_view_factory self._transaction_executor = transaction_executor self._block_sender = block_sender self._batch_publisher = BatchPublisher(identity_signing_key, batch_sender) self._pending_batches = [] # batches we are waiting for validation, # arranged in the order of batches received. self._committed_txn_cache = TransactionCache( self._block_cache.block_store) # Look-up cache for transactions that are committed in the current # chain. Cache is used here so that we can support opportunistically # building on top of a block we published. As well as hold the state # of the transactions already added to the candidate block. self._scheduler = None self._chain_head = chain_head # block (BlockWrapper) self._squash_handler = squash_handler self._identity_signing_key = identity_signing_key self._identity_public_key = \ signing.generate_pubkey(self._identity_signing_key) self._data_dir = data_dir def _build_block(self, chain_head): """ Build a candidate block and construct the consensus object to validate it. :param chain_head: The block to build on top of. :return: (BlockBuilder) - The candidate block in a BlockBuilder wrapper. """ state_view = BlockWrapper.state_view_for_block( chain_head, self._state_view_factory) consensus_module = ConsensusFactory.get_configured_consensus_module( chain_head.header_signature, state_view) self._consensus = consensus_module.\ BlockPublisher(block_cache=self._block_cache, state_view_factory=self._state_view_factory, batch_publisher=self._batch_publisher, data_dir=self._data_dir, validator_id=self._identity_public_key) block_header = BlockHeader( block_num=chain_head.block_num + 1, previous_block_id=chain_head.header_signature, signer_pubkey=self._identity_public_key) block_builder = BlockBuilder(block_header) if not self._consensus.initialize_block(block_builder.block_header): LOGGER.debug("Consensus not ready to build candidate block.") return None # Cancel the previous scheduler if it did not complete. if self._scheduler is not None \ and not self._scheduler.complete(block=False): self._scheduler.cancel() # create a new scheduler self._scheduler = self._transaction_executor.create_scheduler( self._squash_handler, chain_head.state_root_hash) # build the TransactionCache self._committed_txn_cache = TransactionCache( self._block_cache.block_store) if chain_head.header_signature not in self._block_cache.block_store: # if we opportunistically building a block # we need to check make sure we track that blocks transactions # as recorded. for batch in chain_head.block.batches: for txn in batch.transactions: self._committed_txn_cache.add_txn(txn.header_signature) self._transaction_executor.execute(self._scheduler) for batch in self._pending_batches: self._validate_batch(batch) return block_builder def _sign_block(self, block): """ The block should be complete and the final signature from the publishing validator(this validator) needs to be added. """ block_header = block.block_header header_bytes = block_header.SerializeToString() signature = signing.sign(header_bytes, self._identity_signing_key) block.set_signature(signature) return block def _check_batch_dependencies(self, batch, committed_txn_cache): """Check the dependencies for all transactions in this are present. If all are present the committed_txn is updated with all txn in this batch and True is returned. If they are not return failure and the committed_txn is not updated. :param batch: the batch to validate :param committed_txn(TransactionCache): Current set of committed transaction, updated during processing. :return: Boolean, True if dependencies checkout, False otherwise. """ for txn in batch.transactions: if not self._check_transaction_dependencies( txn, committed_txn_cache): # if any transaction in this batch fails the whole batch # fails. committed_txn_cache.remove_batch(batch) return False # update so any subsequent txn in the same batch can be dependent # on this transaction. committed_txn_cache.add_txn(txn.header_signature) return True def _check_transaction_dependencies(self, txn, committed_txn): """Check that all this transactions dependencies are present. :param tx: the transaction to check :param committed_txn(TransactionCache): Current set of committed :return: Boolean, True if dependencies checkout, False otherwise. """ txn_hdr = TransactionHeader() txn_hdr.ParseFromString(txn.header) for dep in txn_hdr.dependencies: if dep not in committed_txn: LOGGER.debug( "Transaction rejected due " + "missing dependency, transaction " + "%s depends on %s", txn.header_signature, dep) return False return True def _validate_batch(self, batch): """Schedule validation of a batch for inclusion in the new block :param batch: the batch to validate :return: None """ if self._scheduler: try: self._scheduler.add_batch(batch) except SchedulerError as err: LOGGER.debug("Scheduler error processing batch: %s", err) def is_batch_already_commited(self, batch): """ Test if a batch is already committed to the chain or is already in the pending queue. """ if self._block_cache.block_store.has_batch(batch.header_signature): return True else: for pending in self._pending_batches: if batch.header_signature == pending.header_signature: return True return False def on_batch_received(self, batch): """ A new batch is received, send it for validation :param batch: the new pending batch :return: None """ with self._lock: # first we check if the transaction dependencies are satisfied # The completer should have taken care of making sure all # Batches containing dependent transactions were sent to the # BlockPublisher prior to this Batch. So if there is a missing # dependency this is an error condition and the batch will be # dropped. if self.is_batch_already_commited(batch): # batch is already committed. LOGGER.debug("Dropping previously committed batch: %s", batch.header_signature) return elif self._check_batch_dependencies(batch, self._committed_txn_cache): self._pending_batches.append(batch) # if we are building a block then send schedule it for # execution. if self._chain_head is not None: self._validate_batch(batch) else: LOGGER.debug("Dropping batch due to missing dependencies: %s", batch.header_signature) def _rebuild_pending_batches(self, committed_batches, uncommitted_batches): """When the chain head is changed. This recomputes the list of pending transactions :param committed_batches: Batches committed in the current chain since the root of the fork switching from. :param uncommitted_batches: Batches that were committed in the old fork since the common root. """ if committed_batches is None: committed_batches = [] if uncommitted_batches is None: uncommitted_batches = [] committed_set = set([x.header_signature for x in committed_batches]) pending_batches = self._pending_batches self._pending_batches = [] # Uncommitted and pending disjoint sets # since batches can only be committed to a chain once. for batch in uncommitted_batches: if batch.header_signature not in committed_set: if self._check_batch_dependencies(batch, self._committed_txn_cache): self._pending_batches.append(batch) for batch in pending_batches: if batch.header_signature not in committed_set: if self._check_batch_dependencies(batch, self._committed_txn_cache): self._pending_batches.append(batch) def on_chain_updated(self, chain_head, committed_batches=None, uncommitted_batches=None): """ The existing chain has been updated, the current head block has changed. :param chain_head: the new head of block_chain :param committed_batches: the set of batches that were committed as part of the new chain. :param uncommitted_batches: the list of transactions if any that are now de-committed when the new chain was selected. :return: None """ try: with self._lock: LOGGER.info('Now building on top of block: %s', chain_head) self._chain_head = chain_head if self._candidate_block is not None and \ chain_head is not None and \ chain_head.identifier == \ self._candidate_block.previous_block_id: # nothing to do. We are building of the current head. # This can happen after we publish a block and # opportunistically create a new block. return elif chain_head is None: # we don't have a chain head, we cannot build blocks self._candidate_block = None self._consensus = None for batch in self._pending_batches: self._committed_txn_cache.add_batch(batch) else: self._rebuild_pending_batches(committed_batches, uncommitted_batches) self._candidate_block = self._build_block(chain_head) # pylint: disable=broad-except except Exception as exc: LOGGER.critical("on_chain_updated exception.") LOGGER.exception(exc) def _finalize_block(self, block): if self._scheduler: self._scheduler.finalize() self._scheduler.complete(block=True) # Read valid batches from self._scheduler pending_batches = self._pending_batches # this is a transaction cache to track the transactions committed # upto this batch. committed_txn_cache = TransactionCache(self._block_cache.block_store) self._pending_batches = [] self._committed_txn_cache = TransactionCache( self._block_cache.block_store) state_hash = None for batch in pending_batches: result = self._scheduler.get_batch_execution_result( batch.header_signature) # if a result is None, this means that the executor never # received the batch and it should be added to # the pending_batches if result is None: self._pending_batches.append(batch) self._committed_txn_cache.add_batch(batch) elif result.is_valid: # check if a dependent batch failed. This could be belt and # suspenders action here but it is logically possible that # a transaction has a dependency that fails it could # still succeed validation. In that case we do not want # to add it to the batch. if not self._check_batch_dependencies(batch, committed_txn_cache): LOGGER.debug( "Batch %s invalid, due to missing txn " "dependency.", batch.header_signature) LOGGER.debug( "Abandoning block %s:" + "root state hash has invalid txn applied", block) pending_batches.remove(batch) self._pending_batches = pending_batches self._committed_txn_cache = \ TransactionCache(self._block_cache.block_store) return False else: block.add_batch(batch) self._committed_txn_cache.add_batch(batch) state_hash = result.state_hash else: committed_txn_cache.uncommit_batch(batch) LOGGER.debug("Batch %s invalid, not added to block.", batch.header_signature) if state_hash is None: LOGGER.debug("Abandoning block %s no batches added", block) return False if not self._consensus.finalize_block(block.block_header): LOGGER.debug( "Abandoning block %s, consensus failed to finalize " "it", block) return False self._consensus = None block.set_state_hash(state_hash) self._sign_block(block) return True def on_check_publish_block(self, force=False): """Ask the consensus module if it is time to claim the candidate block if it is then, claim it and tell the world about it. :return: None """ try: with self._lock: if self._chain_head is not None and\ self._candidate_block is None and\ len(self._pending_batches) != 0: self._candidate_block = self._build_block(self._chain_head) if self._candidate_block and \ (force or len(self._pending_batches) != 0) and \ self._consensus.check_publish_block(self. _candidate_block. block_header): candidate = self._candidate_block self._candidate_block = None if not self._finalize_block(candidate): return block = BlockWrapper(candidate.build_block()) self._block_cache[block.identifier] = block # add the # block to the cache, so we can build on top of it. self._block_sender.send(block.block) LOGGER.info("Claimed Block: %s", block) # We built our candidate, disable processing until # the chain head is updated. self.on_chain_updated(None) # pylint: disable=broad-except except Exception as exc: LOGGER.critical("on_check_publish_block exception.") LOGGER.exception(exc)
def _finalize_block(self, block): if self._scheduler: self._scheduler.finalize() self._scheduler.complete(block=True) # Read valid batches from self._scheduler pending_batches = self._pending_batches # this is a transaction cache to track the transactions committed # upto this batch. committed_txn_cache = TransactionCache(self._block_cache.block_store) self._pending_batches = [] self._committed_txn_cache = TransactionCache( self._block_cache.block_store) state_hash = None for batch in pending_batches: result = self._scheduler.get_batch_execution_result( batch.header_signature) # if a result is None, this means that the executor never # received the batch and it should be added to # the pending_batches if result is None: self._pending_batches.append(batch) self._committed_txn_cache.add_batch(batch) elif result.is_valid: # check if a dependent batch failed. This could be belt and # suspenders action here but it is logically possible that # a transaction has a dependency that fails it could # still succeed validation. In that case we do not want # to add it to the batch. if not self._check_batch_dependencies(batch, committed_txn_cache): LOGGER.debug( "Batch %s invalid, due to missing txn " "dependency.", batch.header_signature) LOGGER.debug( "Abandoning block %s:" + "root state hash has invalid txn applied", block) pending_batches.remove(batch) self._pending_batches = pending_batches self._committed_txn_cache = \ TransactionCache(self._block_cache.block_store) return False else: block.add_batch(batch) self._committed_txn_cache.add_batch(batch) state_hash = result.state_hash else: committed_txn_cache.uncommit_batch(batch) LOGGER.debug("Batch %s invalid, not added to block.", batch.header_signature) if state_hash is None: LOGGER.debug("Abandoning block %s no batches added", block) return False if not self._consensus.finalize_block(block.block_header): LOGGER.debug( "Abandoning block %s, consensus failed to finalize " "it", block) return False self._consensus = None block.set_state_hash(state_hash) self._sign_block(block) return True
def finalize_block(self, identity_signing_key, pending_batches): """Compose the final Block to publish. This involves flushing the scheduler, having consensus bless the block, and signing the block. :param identity_signing_key: the key to sign the block with. :param pending_batches: list to receive any batches that were submitted to add to the block but were not validated before this call. :return: The generated Block, or None if Block failed to finalize. In both cases the pending_batches will contain the list of batches that need to be added to the next Block that is built. """ self._scheduler.finalize() self._scheduler.complete(block=True) # this is a transaction cache to track the transactions committed # up to this batch. Only valid transactions that were processed # by the scheduler are added. committed_txn_cache = TransactionCache(self._block_store) builder = self._block_builder bad_batches = [] # the list of batches that failed processing state_hash = None # Walk the pending batch list: # - find the state hash for the block, the block state_hash is # is randomly placed on one of the transactions, so must interogate # every batch to find it. If it is on a batch that failed processing # then this block will be abandoned. # - build three lists of batches: # 1) a lists of all valid transactions that will be included in the # block, these are added to the BlockBuilder to include in the Block # 2) all batches that were not executed, these are to be returned # in the pending_batches list # 3) all batches that failed processing. These will be discarded. # This list is needed in some case when the block is abandoned to # make sure they do not remain in the pending_batches list. for batch in self._pending_batches: result = self._scheduler.get_batch_execution_result( batch.header_signature) # if a result is None, this means that the executor never # received the batch and it should be added to # the pending_batches, to be added to the next # block if result is None: pending_batches.append(batch) elif result.is_valid: # check if a dependent batch failed. This could be belt and # suspenders action here but it is logically possible that # a transaction has a dependency that fails it could # still succeed validation. In which case we do not want # to add it to the batch. if not self._check_batch_dependencies(batch, committed_txn_cache): LOGGER.debug( "Batch %s invalid, due to missing txn " "dependency.", batch.header_signature) LOGGER.debug( "Abandoning block %s:" + "root state hash has invalid txn applied", builder) # Update the pending batch list to be all the # batches that passed validation to this point and # none of the ones that failed. It is possible that # this batch caused a future batch to fail so # we leave all of the batches that failed after this # one in the list. bad_batches.append(batch) pending_batches.clear() pending_batches.extend([ x for x in self._pending_batches if x not in bad_batches ]) return None else: builder.add_batch(batch) committed_txn_cache.add_batch(batch) if result.state_hash is not None: state_hash = result.state_hash else: bad_batches.append(batch) LOGGER.debug("Batch %s invalid, not added to block.", batch.header_signature) if state_hash is None or not builder.batches: LOGGER.debug("Abandoning block %s: no batches added", builder) return None if not self._consensus.finalize_block(builder.block_header): LOGGER.debug( "Abandoning block %s, consensus failed to finalize " "it", builder) # return all valid batches to the pending_batches list pending_batches.clear() pending_batches.extend( [x for x in self._pending_batches if x not in bad_batches]) return None builder.set_state_hash(state_hash) self._sign_block(builder, identity_signing_key) return builder.build_block()
def finalize_block(self, identity_signing_key, pending_batches): """Compose the final Block to publish. This involves flushing the scheduler, having consensus bless the block, and signing the block. :param identity_signing_key: the key to sign the block with. :param pending_batches: list to receive any batches that were submitted to add to the block but were not validated before this call. """ self._scheduler.finalize() self._scheduler.complete(block=True) # this is a transaction cache to track the transactions committed # up to this batch. Only valid transactions that were processed # by the scheduler are added. committed_txn_cache = TransactionCache(self._block_store) builder = self._block_builder state_hash = None for batch in self._pending_batches: result = self._scheduler.get_batch_execution_result( batch.header_signature) # if a result is None, this means that the executor never # received the batch and it should be added to # the pending_batches, to be added to the next # block if result is None: pending_batches.append(batch) elif result.is_valid: # check if a dependent batch failed. This could be belt and # suspenders action here but it is logically possible that # a transaction has a dependency that fails it could # still succeed validation. In which case we do not want # to add it to the batch. if not self._check_batch_dependencies(batch, committed_txn_cache): LOGGER.debug("Batch %s invalid, due to missing txn " "dependency.", batch.header_signature) LOGGER.debug("Abandoning block %s:" + "root state hash has invalid txn applied", builder) return None else: builder.add_batch(batch) committed_txn_cache.add_batch(batch) if result.state_hash is not None: state_hash = result.state_hash else: LOGGER.debug("Batch %s invalid, not added to block.", batch.header_signature) if state_hash is None: LOGGER.debug("Abandoning block %s no batches added", builder) return None if not self._consensus.finalize_block(builder.block_header): LOGGER.debug("Abandoning block %s, consensus failed to finalize " "it", builder) return False builder.set_state_hash(state_hash) self._sign_block(builder, identity_signing_key) return builder.build_block()
def finalize_block(self, identity_signing_key, pending_batches): """Compose the final Block to publish. This involves flushing the scheduler, having consensus bless the block, and signing the block. :param identity_signing_key: the key to sign the block with. :param pending_batches: list to receive any batches that were submitted to add to the block but were not validated before this call. :return: The generated Block, or None if Block failed to finalize. In both cases the pending_batches will contain the list of batches that need to be added to the next Block that is built. """ self._scheduler.finalize() self._scheduler.complete(block=True) # this is a transaction cache to track the transactions committed # up to this batch. Only valid transactions that were processed # by the scheduler are added. committed_txn_cache = TransactionCache(self._block_store) builder = self._block_builder bad_batches = [] # the list of batches that failed processing state_hash = None # Walk the pending batch list: # - find the state hash for the block, the block state_hash is # is randomly placed on one of the transactions, so must interogate # every batch to find it. If it is on a batch that failed processing # then this block will be abandoned. # - build three lists of batches: # 1) a lists of all valid transactions that will be included in the # block, these are added to the BlockBuilder to include in the Block # 2) all batches that were not executed, these are to be returned # in the pending_batches list # 3) all batches that failed processing. These will be discarded. # This list is needed in some case when the block is abandoned to # make sure they do not remain in the pending_batches list. for batch in self._pending_batches: result = self._scheduler.get_batch_execution_result( batch.header_signature) # if a result is None, this means that the executor never # received the batch and it should be added to # the pending_batches, to be added to the next # block if result is None: pending_batches.append(batch) elif result.is_valid: # check if a dependent batch failed. This could be belt and # suspenders action here but it is logically possible that # a transaction has a dependency that fails it could # still succeed validation. In which case we do not want # to add it to the batch. if not self._check_batch_dependencies(batch, committed_txn_cache): LOGGER.debug("Batch %s invalid, due to missing txn " "dependency.", batch.header_signature) LOGGER.debug("Abandoning block %s:" + "root state hash has invalid txn applied", builder) # Update the pending batch list to be all the # batches that passed validation to this point and # none of the ones that failed. It is possible that # this batch caused a future batch to fail so # we leave all of the batches that failed after this # one in the list. bad_batches.append(batch) pending_batches.clear() pending_batches.extend([x for x in self._pending_batches if x not in bad_batches]) return None else: builder.add_batch(batch) committed_txn_cache.add_batch(batch) if result.state_hash is not None: state_hash = result.state_hash else: bad_batches.append(batch) LOGGER.debug("Batch %s invalid, not added to block.", batch.header_signature) if state_hash is None or not builder.batches: LOGGER.debug("Abandoning block %s: no batches added", builder) return None if not self._consensus.finalize_block(builder.block_header): LOGGER.debug("Abandoning block %s, consensus failed to finalize " "it", builder) # return all valid batches to the pending_batches list pending_batches.clear() pending_batches.extend([x for x in self._pending_batches if x not in bad_batches]) return None builder.set_state_hash(state_hash) self._sign_block(builder, identity_signing_key) return builder.build_block()
def run(self): """ Main entry for Block Validation, Take a given candidate block and decide if it is valid then if it is valid determine if it should be the new head block. Returns the results to the ChainController so that the change over can be made if necessary. """ try: LOGGER.info("Starting block validation of : %s", self._new_block) cur_chain = self._result["cur_chain"] # ordered list of the # current chain blocks new_chain = self._result["new_chain"] # ordered list of the new # chain blocks # 1) Find the common ancestor block, the root of the fork. # walk back till both chains are the same height (new_blkw, cur_blkw) = self._find_common_height(new_chain, cur_chain) # 2) Walk back until we find the common ancestor self._find_common_ancestor(new_blkw, cur_blkw, new_chain, cur_chain) # 3) Determine the validity of the new fork # build the transaction cache to simulate the state of the # chain at the common root. committed_txn = TransactionCache(self._block_cache.block_store) for block in cur_chain: for batch in block.batches: committed_txn.uncommit_batch(batch) valid = True for block in reversed(new_chain): if valid: if not self.validate_block(block, committed_txn): LOGGER.info("Block validation failed: %s", block) valid = False else: LOGGER.info("Block marked invalid(invalid predecessor): " + "%s", block) block.status = BlockStatus.Invalid if not valid: self._done_cb(False, self._result) return # 4) Evaluate the 2 chains to see if the new chain should be # committed commit_new_chain = self._test_commit_new_chain() # 5) Consensus to compute batch sets (only if we are switching). if commit_new_chain: (self._result["committed_batches"], self._result["uncommitted_batches"]) =\ self._compute_batch_change(new_chain, cur_chain) # 6) Tell the journal we are done. self._done_cb(commit_new_chain, self._result) LOGGER.info("Finished block validation of: %s", self._new_block) except BlockValidationAborted: return except Exception as exc: # pylint: disable=broad-except LOGGER.error("Block validation failed with unexpected error: %s", self._new_block) LOGGER.exception(exc) # callback to clean up the block out of the processing list. self._done_cb(False, self._result)