def _build_block(self, chain_head):
""" Build a candidate block and construct the consensus object to
validate it.
:param chain_head: The block to build on top of.
:return: (BlockBuilder) - The candidate block in a BlockBuilder
wrapper.
"""
state_view = BlockWrapper.state_view_for_block(
chain_head, self._state_view_factory)
consensus_module = ConsensusFactory.get_configured_consensus_module(
chain_head.header_signature, state_view)
self._consensus = consensus_module.\
BlockPublisher(block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
batch_publisher=self._batch_publisher,
data_dir=self._data_dir,
validator_id=self._identity_public_key)
block_header = BlockHeader(
block_num=chain_head.block_num + 1,
previous_block_id=chain_head.header_signature,
signer_pubkey=self._identity_public_key)
block_builder = BlockBuilder(block_header)
if not self._consensus.initialize_block(block_builder.block_header):
LOGGER.debug("Consensus not ready to build candidate block.")
return None
# Cancel the previous scheduler if it did not complete.
if self._scheduler is not None \
and not self._scheduler.complete(block=False):
self._scheduler.cancel()
# create a new scheduler
self._scheduler = self._transaction_executor.create_scheduler(
self._squash_handler, chain_head.state_root_hash)
# build the TransactionCache
self._committed_txn_cache = TransactionCache(
self._block_cache.block_store)
if chain_head.header_signature not in self._block_cache.block_store:
# if we opportunistically building a block
# we need to check make sure we track that blocks transactions
# as recorded.
for batch in chain_head.block.batches:
for txn in batch.transactions:
self._committed_txn_cache.add_txn(txn.header_signature)
self._transaction_executor.execute(self._scheduler)
for batch in self._pending_batches:
self._validate_batch(batch)
return block_builder
def __init__(self,
transaction_executor,
block_cache,
state_view_factory,
block_sender,
batch_sender,
squash_handler,
chain_head,
identity_signing_key,
data_dir):
"""
Initialize the BlockPublisher object
Args:
transaction_executor (:obj:`TransactionExecutor`): A
TransactionExecutor instance.
block_cache (:obj:`BlockCache`): A BlockCache instance.
state_view_factory (:obj:`StateViewFactory`): StateViewFactory for
read-only state views.
block_sender (:obj:`BlockSender`): The BlockSender instance.
batch_sender (:obj:`BatchSender`): The BatchSender instance.
squash_handler (function): Squash handler function for merging
contexts.
chain_head (:obj:`BlockWrapper`): The initial chain head.
identity_signing_key (str): Private key for signing blocks
"""
self._lock = RLock()
self._candidate_block = None # the next block in potentia
self._consensus = None
self._block_cache = block_cache
self._state_view_factory = state_view_factory
self._transaction_executor = transaction_executor
self._block_sender = block_sender
self._batch_publisher = BatchPublisher(identity_signing_key,
batch_sender)
self._pending_batches = [] # batches we are waiting for validation,
# arranged in the order of batches received.
self._committed_txn_cache = TransactionCache(self._block_cache.
block_store)
# Look-up cache for transactions that are committed in the current
# chain. Cache is used here so that we can support opportunistically
# building on top of a block we published. As well as hold the state
# of the transactions already added to the candidate block.
self._scheduler = None
self._chain_head = chain_head # block (BlockWrapper)
self._squash_handler = squash_handler
self._identity_signing_key = identity_signing_key
self._identity_public_key = \
signing.generate_pubkey(self._identity_signing_key)
self._data_dir = data_dir
def on_chain_updated(self,
chain_head,
committed_batches=None,
uncommitted_batches=None):
"""
The existing chain has been updated, the current head block has
changed.
:param chain_head: the new head of block_chain
:param committed_batches: the set of batches that were committed
as part of the new chain.
:param uncommitted_batches: the list of transactions if any that are
now de-committed when the new chain was selected.
:return: None
"""
try:
with self._lock:
LOGGER.info('Now building on top of block: %s', chain_head)
self._chain_head = chain_head
if self._candidate_block is not None and \
chain_head is not None and \
chain_head.identifier == \
self._candidate_block.previous_block_id:
# nothing to do. We are building of the current head.
# This can happen after we publish a block and
# opportunistically create a new block.
return
elif chain_head is None:
# we don't have a chain head, we cannot build blocks
self._candidate_block = None
self._consensus = None
self._committed_txn_cache =\
TransactionCache(self._block_cache.block_store)
for batch in self._pending_batches:
self._committed_txn_cache.add_batch(batch)
else:
self._rebuild_pending_batches(committed_batches,
uncommitted_batches)
self._candidate_block = self._build_block(chain_head)
# pylint: disable=broad-except
except Exception as exc:
LOGGER.critical("on_chain_updated exception.")
LOGGER.exception(exc)
def _build_block(self, chain_head):
""" Build a candidate block and construct the consensus object to
validate it.
:param chain_head: The block to build on top of.
:return: (BlockBuilder) - The candidate block in a BlockBuilder
wrapper.
"""
prev_state = self._get_previous_block_root_state_hash(chain_head)
state_view = self._state_view_factory. \
create_view(prev_state)
consensus_module = ConsensusFactory.get_configured_consensus_module(
state_view)
self._consensus = consensus_module.\
BlockPublisher(block_cache=self._block_cache,
state_view=state_view)
block_header = BlockHeader(
block_num=chain_head.block_num + 1,
previous_block_id=chain_head.header_signature)
block_builder = BlockBuilder(block_header)
if not self._consensus.initialize_block(block_builder.block_header):
LOGGER.debug("Consensus not ready to build candidate block.")
# create a new scheduler
self._scheduler = self._transaction_executor.create_scheduler(
self._squash_handler, chain_head.state_root_hash)
# build the TransactionCache
self._committed_txn_cache = TransactionCache(
self._block_cache.block_store)
if chain_head.header_signature not in self._block_cache.block_store:
# if we opportunistically building a block
# we need to check make sure we track that blocks transactions
# as recorded.
for batch in chain_head.block.batches:
for txn in batch.transactions:
self._committed_txn_cache.add_txn(txn.header_signature)
self._transaction_executor.execute(self._scheduler)
for batch in self._pending_batches:
self._validate_batch(batch)
return block_builder
def _build_candidate_block(self, chain_head):
""" Build a candidate block and construct the consensus object to
validate it.
:param chain_head: The block to build on top of.
:return: (BlockBuilder) - The candidate block in a BlockBuilder
wrapper.
"""
state_view = BlockWrapper.state_view_for_block(
chain_head,
self._state_view_factory)
consensus_module = ConsensusFactory.get_configured_consensus_module(
chain_head.header_signature,
state_view)
config_view = ConfigView(state_view)
max_batches = config_view.get_setting(
'sawtooth.publisher.max_batches_per_block',
default_value=0, value_type=int)
consensus = consensus_module.\
BlockPublisher(block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
batch_publisher=self._batch_publisher,
data_dir=self._data_dir,
config_dir=self._config_dir,
validator_id=self._identity_public_key)
block_header = BlockHeader(
block_num=chain_head.block_num + 1,
previous_block_id=chain_head.header_signature,
signer_pubkey=self._identity_public_key)
block_builder = BlockBuilder(block_header)
if not consensus.initialize_block(block_builder.block_header):
LOGGER.debug("Consensus not ready to build candidate block.")
return None
# create a new scheduler
scheduler = self._transaction_executor.create_scheduler(
self._squash_handler, chain_head.state_root_hash)
# build the TransactionCache
committed_txn_cache = TransactionCache(self._block_cache.block_store)
self._transaction_executor.execute(scheduler)
self._candidate_block = _CandidateBlock(self._block_cache.block_store,
consensus, scheduler,
committed_txn_cache,
block_builder,
max_batches)
for batch in self._pending_batches:
if self._candidate_block.can_add_batch:
self._candidate_block.add_batch(batch)
else:
break
def _set_genesis(self, block):
# This is used by a non-genesis journal when it has received the
# genesis block from the genesis validator
if block.previous_block_id == NULL_BLOCK_IDENTIFIER:
chain_id = self._chain_id_manager.get_block_chain_id()
if chain_id is not None and chain_id != block.identifier:
LOGGER.warning(
"Block id does not match block chain id %s. "
"Cannot set initial chain head.: %s", chain_id[:8],
block.identifier[:8])
else:
state_view = self._state_view_factory.create_view()
consensus_module = \
ConsensusFactory.get_configured_consensus_module(
NULL_BLOCK_IDENTIFIER,
state_view)
committed_txn = TransactionCache(self._block_cache.block_store)
validator = BlockValidator(
consensus_module=consensus_module,
new_block=block,
chain_head=self._chain_head,
block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
done_cb=self.on_block_validated,
executor=self._transaction_executor,
squash_handler=self._squash_handler,
identity_signing_key=self._identity_signing_key,
data_dir=self._data_dir,
config_dir=self._config_dir)
valid = validator.validate_block(block, committed_txn)
if valid:
if chain_id is None:
self._chain_id_manager.save_block_chain_id(
block.identifier)
self._block_store.update_chain([block])
self._chain_head = block
self._notify_on_chain_updated(self._chain_head)
else:
LOGGER.warning(
"The genesis block is not valid. Cannot "
"set chain head: %s", block)
else:
LOGGER.warning(
"Cannot set initial chain head, this is not a "
"genesis block: %s", block)
def run(self):
"""
Main entry for Block Validation, Take a given candidate block
and decide if it is valid then if it is valid determine if it should
be the new head block. Returns the results to the ChainController
so that the change over can be made if necessary.
"""
try:
LOGGER.info("Starting block validation of : %s", self._new_block)
cur_chain = self._result["cur_chain"] # ordered list of the
# current chain blocks
new_chain = self._result["new_chain"] # ordered list of the new
# chain blocks
# 1) Find the common ancestor block, the root of the fork.
# walk back till both chains are the same height
(new_blkw,
cur_blkw) = self._find_common_height(new_chain, cur_chain)
# 2) Walk back until we find the common ancestor
self._find_common_ancestor(new_blkw, cur_blkw, new_chain,
cur_chain)
# 3) Determine the validity of the new fork
# build the transaction cache to simulate the state of the
# chain at the common root.
committed_txn = TransactionCache(self._block_cache.block_store)
for block in cur_chain:
for batch in block.batches:
committed_txn.uncommit_batch(batch)
valid = True
for block in reversed(new_chain):
if valid:
if not self.validate_block(block, committed_txn):
LOGGER.info("Block validation failed: %s", block)
valid = False
else:
LOGGER.info(
"Block marked invalid(invalid predecessor): " + "%s",
block)
block.status = BlockStatus.Invalid
if not valid:
self._done_cb(False, self._result)
return
# 4) Evaluate the 2 chains to see if the new chain should be
# committed
commit_new_chain = self._test_commit_new_chain()
# 5) Consensus to compute batch sets (only if we are switching).
if commit_new_chain:
(self._result["committed_batches"],
self._result["uncommitted_batches"]) =\
self._compute_batch_change(new_chain, cur_chain)
# 6) Tell the journal we are done.
self._done_cb(commit_new_chain, self._result)
LOGGER.info("Finished block validation of: %s", self._new_block)
except BlockValidationAborted:
return
except Exception as exc: # pylint: disable=broad-except
LOGGER.error("Block validation failed with unexpected error: %s",
self._new_block)
LOGGER.exception(exc)
# callback to clean up the block out of the processing list.
self._done_cb(False, self._result)
class BlockPublisher(object):
"""
Responsible for generating new blocks and publishing them when the
Consensus deems it appropriate.
"""
def __init__(self, transaction_executor, block_cache, state_view_factory,
block_sender, batch_sender, squash_handler, chain_head,
identity_signing_key, data_dir):
"""
Initialize the BlockPublisher object
Args:
transaction_executor (:obj:`TransactionExecutor`): A
TransactionExecutor instance.
block_cache (:obj:`BlockCache`): A BlockCache instance.
state_view_factory (:obj:`StateViewFactory`): StateViewFactory for
read-only state views.
block_sender (:obj:`BlockSender`): The BlockSender instance.
batch_sender (:obj:`BatchSender`): The BatchSender instance.
squash_handler (function): Squash handler function for merging
contexts.
chain_head (:obj:`BlockWrapper`): The initial chain head.
identity_signing_key (str): Private key for signing blocks
data_dir (str): path to location where persistent data for the
consensus module can be stored.
"""
self._lock = RLock()
self._candidate_block = None # the next block in potential chain
self._consensus = None
self._block_cache = block_cache
self._state_view_factory = state_view_factory
self._transaction_executor = transaction_executor
self._block_sender = block_sender
self._batch_publisher = BatchPublisher(identity_signing_key,
batch_sender)
self._pending_batches = [] # batches we are waiting for validation,
# arranged in the order of batches received.
self._committed_txn_cache = TransactionCache(
self._block_cache.block_store)
# Look-up cache for transactions that are committed in the current
# chain. Cache is used here so that we can support opportunistically
# building on top of a block we published. As well as hold the state
# of the transactions already added to the candidate block.
self._scheduler = None
self._chain_head = chain_head # block (BlockWrapper)
self._squash_handler = squash_handler
self._identity_signing_key = identity_signing_key
self._identity_public_key = \
signing.generate_pubkey(self._identity_signing_key)
self._data_dir = data_dir
def _build_block(self, chain_head):
""" Build a candidate block and construct the consensus object to
validate it.
:param chain_head: The block to build on top of.
:return: (BlockBuilder) - The candidate block in a BlockBuilder
wrapper.
"""
state_view = BlockWrapper.state_view_for_block(
chain_head, self._state_view_factory)
consensus_module = ConsensusFactory.get_configured_consensus_module(
chain_head.header_signature, state_view)
self._consensus = consensus_module.\
BlockPublisher(block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
batch_publisher=self._batch_publisher,
data_dir=self._data_dir,
validator_id=self._identity_public_key)
block_header = BlockHeader(
block_num=chain_head.block_num + 1,
previous_block_id=chain_head.header_signature,
signer_pubkey=self._identity_public_key)
block_builder = BlockBuilder(block_header)
if not self._consensus.initialize_block(block_builder.block_header):
LOGGER.debug("Consensus not ready to build candidate block.")
return None
# Cancel the previous scheduler if it did not complete.
if self._scheduler is not None \
and not self._scheduler.complete(block=False):
self._scheduler.cancel()
# create a new scheduler
self._scheduler = self._transaction_executor.create_scheduler(
self._squash_handler, chain_head.state_root_hash)
# build the TransactionCache
self._committed_txn_cache = TransactionCache(
self._block_cache.block_store)
if chain_head.header_signature not in self._block_cache.block_store:
# if we opportunistically building a block
# we need to check make sure we track that blocks transactions
# as recorded.
for batch in chain_head.block.batches:
for txn in batch.transactions:
self._committed_txn_cache.add_txn(txn.header_signature)
self._transaction_executor.execute(self._scheduler)
for batch in self._pending_batches:
self._validate_batch(batch)
return block_builder
def _sign_block(self, block):
""" The block should be complete and the final
signature from the publishing validator(this validator) needs to
be added.
"""
block_header = block.block_header
header_bytes = block_header.SerializeToString()
signature = signing.sign(header_bytes, self._identity_signing_key)
block.set_signature(signature)
return block
def _check_batch_dependencies(self, batch, committed_txn_cache):
"""Check the dependencies for all transactions in this are present.
If all are present the committed_txn is updated with all txn in this
batch and True is returned. If they are not return failure and the
committed_txn is not updated.
:param batch: the batch to validate
:param committed_txn(TransactionCache): Current set of committed
transaction, updated during processing.
:return: Boolean, True if dependencies checkout, False otherwise.
"""
for txn in batch.transactions:
if not self._check_transaction_dependencies(
txn, committed_txn_cache):
# if any transaction in this batch fails the whole batch
# fails.
committed_txn_cache.remove_batch(batch)
return False
# update so any subsequent txn in the same batch can be dependent
# on this transaction.
committed_txn_cache.add_txn(txn.header_signature)
return True
def _check_transaction_dependencies(self, txn, committed_txn):
"""Check that all this transactions dependencies are present.
:param tx: the transaction to check
:param committed_txn(TransactionCache): Current set of committed
:return: Boolean, True if dependencies checkout, False otherwise.
"""
txn_hdr = TransactionHeader()
txn_hdr.ParseFromString(txn.header)
for dep in txn_hdr.dependencies:
if dep not in committed_txn:
LOGGER.debug(
"Transaction rejected due " +
"missing dependency, transaction " + "%s depends on %s",
txn.header_signature, dep)
return False
return True
def _validate_batch(self, batch):
"""Schedule validation of a batch for inclusion in the new block
:param batch: the batch to validate
:return: None
"""
if self._scheduler:
try:
self._scheduler.add_batch(batch)
except SchedulerError as err:
LOGGER.debug("Scheduler error processing batch: %s", err)
def is_batch_already_commited(self, batch):
""" Test if a batch is already committed to the chain or
is already in the pending queue.
"""
if self._block_cache.block_store.has_batch(batch.header_signature):
return True
else:
for pending in self._pending_batches:
if batch.header_signature == pending.header_signature:
return True
return False
def on_batch_received(self, batch):
"""
A new batch is received, send it for validation
:param batch: the new pending batch
:return: None
"""
with self._lock:
# first we check if the transaction dependencies are satisfied
# The completer should have taken care of making sure all
# Batches containing dependent transactions were sent to the
# BlockPublisher prior to this Batch. So if there is a missing
# dependency this is an error condition and the batch will be
# dropped.
if self.is_batch_already_commited(batch):
# batch is already committed.
LOGGER.debug("Dropping previously committed batch: %s",
batch.header_signature)
return
elif self._check_batch_dependencies(batch,
self._committed_txn_cache):
self._pending_batches.append(batch)
# if we are building a block then send schedule it for
# execution.
if self._chain_head is not None:
self._validate_batch(batch)
else:
LOGGER.debug("Dropping batch due to missing dependencies: %s",
batch.header_signature)
def _rebuild_pending_batches(self, committed_batches, uncommitted_batches):
"""When the chain head is changed. This recomputes the list of pending
transactions
:param committed_batches: Batches committed in the current chain
since the root of the fork switching from.
:param uncommitted_batches: Batches that were committed in the old
fork since the common root.
"""
if committed_batches is None:
committed_batches = []
if uncommitted_batches is None:
uncommitted_batches = []
committed_set = set([x.header_signature for x in committed_batches])
pending_batches = self._pending_batches
self._pending_batches = []
# Uncommitted and pending disjoint sets
# since batches can only be committed to a chain once.
for batch in uncommitted_batches:
if batch.header_signature not in committed_set:
if self._check_batch_dependencies(batch,
self._committed_txn_cache):
self._pending_batches.append(batch)
for batch in pending_batches:
if batch.header_signature not in committed_set:
if self._check_batch_dependencies(batch,
self._committed_txn_cache):
self._pending_batches.append(batch)
def on_chain_updated(self,
chain_head,
committed_batches=None,
uncommitted_batches=None):
"""
The existing chain has been updated, the current head block has
changed.
:param chain_head: the new head of block_chain
:param committed_batches: the set of batches that were committed
as part of the new chain.
:param uncommitted_batches: the list of transactions if any that are
now de-committed when the new chain was selected.
:return: None
"""
try:
with self._lock:
LOGGER.info('Now building on top of block: %s', chain_head)
self._chain_head = chain_head
if self._candidate_block is not None and \
chain_head is not None and \
chain_head.identifier == \
self._candidate_block.previous_block_id:
# nothing to do. We are building of the current head.
# This can happen after we publish a block and
# opportunistically create a new block.
return
elif chain_head is None:
# we don't have a chain head, we cannot build blocks
self._candidate_block = None
self._consensus = None
for batch in self._pending_batches:
self._committed_txn_cache.add_batch(batch)
else:
self._rebuild_pending_batches(committed_batches,
uncommitted_batches)
self._candidate_block = self._build_block(chain_head)
# pylint: disable=broad-except
except Exception as exc:
LOGGER.critical("on_chain_updated exception.")
LOGGER.exception(exc)
def _finalize_block(self, block):
if self._scheduler:
self._scheduler.finalize()
self._scheduler.complete(block=True)
# Read valid batches from self._scheduler
pending_batches = self._pending_batches
# this is a transaction cache to track the transactions committed
# upto this batch.
committed_txn_cache = TransactionCache(self._block_cache.block_store)
self._pending_batches = []
self._committed_txn_cache = TransactionCache(
self._block_cache.block_store)
state_hash = None
for batch in pending_batches:
result = self._scheduler.get_batch_execution_result(
batch.header_signature)
# if a result is None, this means that the executor never
# received the batch and it should be added to
# the pending_batches
if result is None:
self._pending_batches.append(batch)
self._committed_txn_cache.add_batch(batch)
elif result.is_valid:
# check if a dependent batch failed. This could be belt and
# suspenders action here but it is logically possible that
# a transaction has a dependency that fails it could
# still succeed validation. In that case we do not want
# to add it to the batch.
if not self._check_batch_dependencies(batch,
committed_txn_cache):
LOGGER.debug(
"Batch %s invalid, due to missing txn "
"dependency.", batch.header_signature)
LOGGER.debug(
"Abandoning block %s:" +
"root state hash has invalid txn applied", block)
pending_batches.remove(batch)
self._pending_batches = pending_batches
self._committed_txn_cache = \
TransactionCache(self._block_cache.block_store)
return False
else:
block.add_batch(batch)
self._committed_txn_cache.add_batch(batch)
state_hash = result.state_hash
else:
committed_txn_cache.uncommit_batch(batch)
LOGGER.debug("Batch %s invalid, not added to block.",
batch.header_signature)
if state_hash is None:
LOGGER.debug("Abandoning block %s no batches added", block)
return False
if not self._consensus.finalize_block(block.block_header):
LOGGER.debug(
"Abandoning block %s, consensus failed to finalize "
"it", block)
return False
self._consensus = None
block.set_state_hash(state_hash)
self._sign_block(block)
return True
def on_check_publish_block(self, force=False):
"""Ask the consensus module if it is time to claim the candidate block
if it is then, claim it and tell the world about it.
:return:
None
"""
try:
with self._lock:
if self._chain_head is not None and\
self._candidate_block is None and\
len(self._pending_batches) != 0:
self._candidate_block = self._build_block(self._chain_head)
if self._candidate_block and \
(force or len(self._pending_batches) != 0) and \
self._consensus.check_publish_block(self.
_candidate_block.
block_header):
candidate = self._candidate_block
self._candidate_block = None
if not self._finalize_block(candidate):
return
block = BlockWrapper(candidate.build_block())
self._block_cache[block.identifier] = block # add the
# block to the cache, so we can build on top of it.
self._block_sender.send(block.block)
LOGGER.info("Claimed Block: %s", block)
# We built our candidate, disable processing until
# the chain head is updated.
self.on_chain_updated(None)
# pylint: disable=broad-except
except Exception as exc:
LOGGER.critical("on_check_publish_block exception.")
LOGGER.exception(exc)
def _finalize_block(self, block):
if self._scheduler:
self._scheduler.finalize()
self._scheduler.complete(block=True)
# Read valid batches from self._scheduler
pending_batches = self._pending_batches
# this is a transaction cache to track the transactions committed
# upto this batch.
committed_txn_cache = TransactionCache(self._block_cache.block_store)
self._pending_batches = []
self._committed_txn_cache = TransactionCache(
self._block_cache.block_store)
state_hash = None
for batch in pending_batches:
result = self._scheduler.get_batch_execution_result(
batch.header_signature)
# if a result is None, this means that the executor never
# received the batch and it should be added to
# the pending_batches
if result is None:
self._pending_batches.append(batch)
self._committed_txn_cache.add_batch(batch)
elif result.is_valid:
# check if a dependent batch failed. This could be belt and
# suspenders action here but it is logically possible that
# a transaction has a dependency that fails it could
# still succeed validation. In that case we do not want
# to add it to the batch.
if not self._check_batch_dependencies(batch,
committed_txn_cache):
LOGGER.debug(
"Batch %s invalid, due to missing txn "
"dependency.", batch.header_signature)
LOGGER.debug(
"Abandoning block %s:" +
"root state hash has invalid txn applied", block)
pending_batches.remove(batch)
self._pending_batches = pending_batches
self._committed_txn_cache = \
TransactionCache(self._block_cache.block_store)
return False
else:
block.add_batch(batch)
self._committed_txn_cache.add_batch(batch)
state_hash = result.state_hash
else:
committed_txn_cache.uncommit_batch(batch)
LOGGER.debug("Batch %s invalid, not added to block.",
batch.header_signature)
if state_hash is None:
LOGGER.debug("Abandoning block %s no batches added", block)
return False
if not self._consensus.finalize_block(block.block_header):
LOGGER.debug(
"Abandoning block %s, consensus failed to finalize "
"it", block)
return False
self._consensus = None
block.set_state_hash(state_hash)
self._sign_block(block)
return True
def finalize_block(self, identity_signing_key, pending_batches):
"""Compose the final Block to publish. This involves flushing
the scheduler, having consensus bless the block, and signing
the block.
:param identity_signing_key: the key to sign the block with.
:param pending_batches: list to receive any batches that were
submitted to add to the block but were not validated before this
call.
:return: The generated Block, or None if Block failed to finalize.
In both cases the pending_batches will contain the list of batches
that need to be added to the next Block that is built.
"""
self._scheduler.finalize()
self._scheduler.complete(block=True)
# this is a transaction cache to track the transactions committed
# up to this batch. Only valid transactions that were processed
# by the scheduler are added.
committed_txn_cache = TransactionCache(self._block_store)
builder = self._block_builder
bad_batches = [] # the list of batches that failed processing
state_hash = None
# Walk the pending batch list:
# - find the state hash for the block, the block state_hash is
# is randomly placed on one of the transactions, so must interogate
# every batch to find it. If it is on a batch that failed processing
# then this block will be abandoned.
# - build three lists of batches:
# 1) a lists of all valid transactions that will be included in the
# block, these are added to the BlockBuilder to include in the Block
# 2) all batches that were not executed, these are to be returned
# in the pending_batches list
# 3) all batches that failed processing. These will be discarded.
# This list is needed in some case when the block is abandoned to
# make sure they do not remain in the pending_batches list.
for batch in self._pending_batches:
result = self._scheduler.get_batch_execution_result(
batch.header_signature)
# if a result is None, this means that the executor never
# received the batch and it should be added to
# the pending_batches, to be added to the next
# block
if result is None:
pending_batches.append(batch)
elif result.is_valid:
# check if a dependent batch failed. This could be belt and
# suspenders action here but it is logically possible that
# a transaction has a dependency that fails it could
# still succeed validation. In which case we do not want
# to add it to the batch.
if not self._check_batch_dependencies(batch,
committed_txn_cache):
LOGGER.debug(
"Batch %s invalid, due to missing txn "
"dependency.", batch.header_signature)
LOGGER.debug(
"Abandoning block %s:" +
"root state hash has invalid txn applied", builder)
# Update the pending batch list to be all the
# batches that passed validation to this point and
# none of the ones that failed. It is possible that
# this batch caused a future batch to fail so
# we leave all of the batches that failed after this
# one in the list.
bad_batches.append(batch)
pending_batches.clear()
pending_batches.extend([
x for x in self._pending_batches
if x not in bad_batches
])
return None
else:
builder.add_batch(batch)
committed_txn_cache.add_batch(batch)
if result.state_hash is not None:
state_hash = result.state_hash
else:
bad_batches.append(batch)
LOGGER.debug("Batch %s invalid, not added to block.",
batch.header_signature)
if state_hash is None or not builder.batches:
LOGGER.debug("Abandoning block %s: no batches added", builder)
return None
if not self._consensus.finalize_block(builder.block_header):
LOGGER.debug(
"Abandoning block %s, consensus failed to finalize "
"it", builder)
# return all valid batches to the pending_batches list
pending_batches.clear()
pending_batches.extend(
[x for x in self._pending_batches if x not in bad_batches])
return None
builder.set_state_hash(state_hash)
self._sign_block(builder, identity_signing_key)
return builder.build_block()
def finalize_block(self, identity_signing_key, pending_batches):
"""Compose the final Block to publish. This involves flushing
the scheduler, having consensus bless the block, and signing
the block.
:param identity_signing_key: the key to sign the block with.
:param pending_batches: list to receive any batches that were
submitted to add to the block but were not validated before this
call.
"""
self._scheduler.finalize()
self._scheduler.complete(block=True)
# this is a transaction cache to track the transactions committed
# up to this batch. Only valid transactions that were processed
# by the scheduler are added.
committed_txn_cache = TransactionCache(self._block_store)
builder = self._block_builder
state_hash = None
for batch in self._pending_batches:
result = self._scheduler.get_batch_execution_result(
batch.header_signature)
# if a result is None, this means that the executor never
# received the batch and it should be added to
# the pending_batches, to be added to the next
# block
if result is None:
pending_batches.append(batch)
elif result.is_valid:
# check if a dependent batch failed. This could be belt and
# suspenders action here but it is logically possible that
# a transaction has a dependency that fails it could
# still succeed validation. In which case we do not want
# to add it to the batch.
if not self._check_batch_dependencies(batch,
committed_txn_cache):
LOGGER.debug("Batch %s invalid, due to missing txn "
"dependency.", batch.header_signature)
LOGGER.debug("Abandoning block %s:" +
"root state hash has invalid txn applied",
builder)
return None
else:
builder.add_batch(batch)
committed_txn_cache.add_batch(batch)
if result.state_hash is not None:
state_hash = result.state_hash
else:
LOGGER.debug("Batch %s invalid, not added to block.",
batch.header_signature)
if state_hash is None:
LOGGER.debug("Abandoning block %s no batches added", builder)
return None
if not self._consensus.finalize_block(builder.block_header):
LOGGER.debug("Abandoning block %s, consensus failed to finalize "
"it", builder)
return False
builder.set_state_hash(state_hash)
self._sign_block(builder, identity_signing_key)
return builder.build_block()
def finalize_block(self, identity_signing_key, pending_batches):
"""Compose the final Block to publish. This involves flushing
the scheduler, having consensus bless the block, and signing
the block.
:param identity_signing_key: the key to sign the block with.
:param pending_batches: list to receive any batches that were
submitted to add to the block but were not validated before this
call.
:return: The generated Block, or None if Block failed to finalize.
In both cases the pending_batches will contain the list of batches
that need to be added to the next Block that is built.
"""
self._scheduler.finalize()
self._scheduler.complete(block=True)
# this is a transaction cache to track the transactions committed
# up to this batch. Only valid transactions that were processed
# by the scheduler are added.
committed_txn_cache = TransactionCache(self._block_store)
builder = self._block_builder
bad_batches = [] # the list of batches that failed processing
state_hash = None
# Walk the pending batch list:
# - find the state hash for the block, the block state_hash is
# is randomly placed on one of the transactions, so must interogate
# every batch to find it. If it is on a batch that failed processing
# then this block will be abandoned.
# - build three lists of batches:
# 1) a lists of all valid transactions that will be included in the
# block, these are added to the BlockBuilder to include in the Block
# 2) all batches that were not executed, these are to be returned
# in the pending_batches list
# 3) all batches that failed processing. These will be discarded.
# This list is needed in some case when the block is abandoned to
# make sure they do not remain in the pending_batches list.
for batch in self._pending_batches:
result = self._scheduler.get_batch_execution_result(
batch.header_signature)
# if a result is None, this means that the executor never
# received the batch and it should be added to
# the pending_batches, to be added to the next
# block
if result is None:
pending_batches.append(batch)
elif result.is_valid:
# check if a dependent batch failed. This could be belt and
# suspenders action here but it is logically possible that
# a transaction has a dependency that fails it could
# still succeed validation. In which case we do not want
# to add it to the batch.
if not self._check_batch_dependencies(batch,
committed_txn_cache):
LOGGER.debug("Batch %s invalid, due to missing txn "
"dependency.", batch.header_signature)
LOGGER.debug("Abandoning block %s:" +
"root state hash has invalid txn applied",
builder)
# Update the pending batch list to be all the
# batches that passed validation to this point and
# none of the ones that failed. It is possible that
# this batch caused a future batch to fail so
# we leave all of the batches that failed after this
# one in the list.
bad_batches.append(batch)
pending_batches.clear()
pending_batches.extend([x for x in self._pending_batches
if x not in bad_batches])
return None
else:
builder.add_batch(batch)
committed_txn_cache.add_batch(batch)
if result.state_hash is not None:
state_hash = result.state_hash
else:
bad_batches.append(batch)
LOGGER.debug("Batch %s invalid, not added to block.",
batch.header_signature)
if state_hash is None or not builder.batches:
LOGGER.debug("Abandoning block %s: no batches added", builder)
return None
if not self._consensus.finalize_block(builder.block_header):
LOGGER.debug("Abandoning block %s, consensus failed to finalize "
"it", builder)
# return all valid batches to the pending_batches list
pending_batches.clear()
pending_batches.extend([x for x in self._pending_batches
if x not in bad_batches])
return None
builder.set_state_hash(state_hash)
self._sign_block(builder, identity_signing_key)
return builder.build_block()
def run(self):
"""
Main entry for Block Validation, Take a given candidate block
and decide if it is valid then if it is valid determine if it should
be the new head block. Returns the results to the ChainController
so that the change over can be made if necessary.
"""
try:
LOGGER.info("Starting block validation of : %s",
self._new_block)
cur_chain = self._result["cur_chain"] # ordered list of the
# current chain blocks
new_chain = self._result["new_chain"] # ordered list of the new
# chain blocks
# 1) Find the common ancestor block, the root of the fork.
# walk back till both chains are the same height
(new_blkw, cur_blkw) = self._find_common_height(new_chain,
cur_chain)
# 2) Walk back until we find the common ancestor
self._find_common_ancestor(new_blkw, cur_blkw,
new_chain, cur_chain)
# 3) Determine the validity of the new fork
# build the transaction cache to simulate the state of the
# chain at the common root.
committed_txn = TransactionCache(self._block_cache.block_store)
for block in cur_chain:
for batch in block.batches:
committed_txn.uncommit_batch(batch)
valid = True
for block in reversed(new_chain):
if valid:
if not self.validate_block(block, committed_txn):
LOGGER.info("Block validation failed: %s", block)
valid = False
else:
LOGGER.info("Block marked invalid(invalid predecessor): " +
"%s", block)
block.status = BlockStatus.Invalid
if not valid:
self._done_cb(False, self._result)
return
# 4) Evaluate the 2 chains to see if the new chain should be
# committed
commit_new_chain = self._test_commit_new_chain()
# 5) Consensus to compute batch sets (only if we are switching).
if commit_new_chain:
(self._result["committed_batches"],
self._result["uncommitted_batches"]) =\
self._compute_batch_change(new_chain, cur_chain)
# 6) Tell the journal we are done.
self._done_cb(commit_new_chain,
self._result)
LOGGER.info("Finished block validation of: %s",
self._new_block)
except BlockValidationAborted:
return
except Exception as exc: # pylint: disable=broad-except
LOGGER.error("Block validation failed with unexpected error: %s",
self._new_block)
LOGGER.exception(exc)
# callback to clean up the block out of the processing list.
self._done_cb(False, self._result)