class BlockPublisher(object):
"""
Responsible for generating new blocks and publishing them when the
Consensus deems it appropriate.
"""
def __init__(self, transaction_executor, block_cache, state_view_factory,
block_sender, batch_sender, squash_handler, chain_head,
identity_signing_key, data_dir):
"""
Initialize the BlockPublisher object
Args:
transaction_executor (:obj:`TransactionExecutor`): A
TransactionExecutor instance.
block_cache (:obj:`BlockCache`): A BlockCache instance.
state_view_factory (:obj:`StateViewFactory`): StateViewFactory for
read-only state views.
block_sender (:obj:`BlockSender`): The BlockSender instance.
batch_sender (:obj:`BatchSender`): The BatchSender instance.
squash_handler (function): Squash handler function for merging
contexts.
chain_head (:obj:`BlockWrapper`): The initial chain head.
identity_signing_key (str): Private key for signing blocks
data_dir (str): path to location where persistent data for the
consensus module can be stored.
"""
self._lock = RLock()
self._candidate_block = None # the next block in potential chain
self._consensus = None
self._block_cache = block_cache
self._state_view_factory = state_view_factory
self._transaction_executor = transaction_executor
self._block_sender = block_sender
self._batch_publisher = BatchPublisher(identity_signing_key,
batch_sender)
self._pending_batches = [] # batches we are waiting for validation,
# arranged in the order of batches received.
self._committed_txn_cache = TransactionCache(
self._block_cache.block_store)
# Look-up cache for transactions that are committed in the current
# chain. Cache is used here so that we can support opportunistically
# building on top of a block we published. As well as hold the state
# of the transactions already added to the candidate block.
self._scheduler = None
self._chain_head = chain_head # block (BlockWrapper)
self._squash_handler = squash_handler
self._identity_signing_key = identity_signing_key
self._identity_public_key = \
signing.generate_pubkey(self._identity_signing_key)
self._data_dir = data_dir
def _build_block(self, chain_head):
""" Build a candidate block and construct the consensus object to
validate it.
:param chain_head: The block to build on top of.
:return: (BlockBuilder) - The candidate block in a BlockBuilder
wrapper.
"""
state_view = BlockWrapper.state_view_for_block(
chain_head, self._state_view_factory)
consensus_module = ConsensusFactory.get_configured_consensus_module(
chain_head.header_signature, state_view)
self._consensus = consensus_module.\
BlockPublisher(block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
batch_publisher=self._batch_publisher,
data_dir=self._data_dir,
validator_id=self._identity_public_key)
block_header = BlockHeader(
block_num=chain_head.block_num + 1,
previous_block_id=chain_head.header_signature,
signer_pubkey=self._identity_public_key)
block_builder = BlockBuilder(block_header)
if not self._consensus.initialize_block(block_builder.block_header):
LOGGER.debug("Consensus not ready to build candidate block.")
return None
# Cancel the previous scheduler if it did not complete.
if self._scheduler is not None \
and not self._scheduler.complete(block=False):
self._scheduler.cancel()
# create a new scheduler
self._scheduler = self._transaction_executor.create_scheduler(
self._squash_handler, chain_head.state_root_hash)
# build the TransactionCache
self._committed_txn_cache = TransactionCache(
self._block_cache.block_store)
if chain_head.header_signature not in self._block_cache.block_store:
# if we opportunistically building a block
# we need to check make sure we track that blocks transactions
# as recorded.
for batch in chain_head.block.batches:
for txn in batch.transactions:
self._committed_txn_cache.add_txn(txn.header_signature)
self._transaction_executor.execute(self._scheduler)
for batch in self._pending_batches:
self._validate_batch(batch)
return block_builder
def _sign_block(self, block):
""" The block should be complete and the final
signature from the publishing validator(this validator) needs to
be added.
"""
block_header = block.block_header
header_bytes = block_header.SerializeToString()
signature = signing.sign(header_bytes, self._identity_signing_key)
block.set_signature(signature)
return block
def _check_batch_dependencies(self, batch, committed_txn_cache):
"""Check the dependencies for all transactions in this are present.
If all are present the committed_txn is updated with all txn in this
batch and True is returned. If they are not return failure and the
committed_txn is not updated.
:param batch: the batch to validate
:param committed_txn(TransactionCache): Current set of committed
transaction, updated during processing.
:return: Boolean, True if dependencies checkout, False otherwise.
"""
for txn in batch.transactions:
if not self._check_transaction_dependencies(
txn, committed_txn_cache):
# if any transaction in this batch fails the whole batch
# fails.
committed_txn_cache.remove_batch(batch)
return False
# update so any subsequent txn in the same batch can be dependent
# on this transaction.
committed_txn_cache.add_txn(txn.header_signature)
return True
def _check_transaction_dependencies(self, txn, committed_txn):
"""Check that all this transactions dependencies are present.
:param tx: the transaction to check
:param committed_txn(TransactionCache): Current set of committed
:return: Boolean, True if dependencies checkout, False otherwise.
"""
txn_hdr = TransactionHeader()
txn_hdr.ParseFromString(txn.header)
for dep in txn_hdr.dependencies:
if dep not in committed_txn:
LOGGER.debug(
"Transaction rejected due " +
"missing dependency, transaction " + "%s depends on %s",
txn.header_signature, dep)
return False
return True
def _validate_batch(self, batch):
"""Schedule validation of a batch for inclusion in the new block
:param batch: the batch to validate
:return: None
"""
if self._scheduler:
try:
self._scheduler.add_batch(batch)
except SchedulerError as err:
LOGGER.debug("Scheduler error processing batch: %s", err)
def is_batch_already_commited(self, batch):
""" Test if a batch is already committed to the chain or
is already in the pending queue.
"""
if self._block_cache.block_store.has_batch(batch.header_signature):
return True
else:
for pending in self._pending_batches:
if batch.header_signature == pending.header_signature:
return True
return False
def on_batch_received(self, batch):
"""
A new batch is received, send it for validation
:param batch: the new pending batch
:return: None
"""
with self._lock:
# first we check if the transaction dependencies are satisfied
# The completer should have taken care of making sure all
# Batches containing dependent transactions were sent to the
# BlockPublisher prior to this Batch. So if there is a missing
# dependency this is an error condition and the batch will be
# dropped.
if self.is_batch_already_commited(batch):
# batch is already committed.
LOGGER.debug("Dropping previously committed batch: %s",
batch.header_signature)
return
elif self._check_batch_dependencies(batch,
self._committed_txn_cache):
self._pending_batches.append(batch)
# if we are building a block then send schedule it for
# execution.
if self._chain_head is not None:
self._validate_batch(batch)
else:
LOGGER.debug("Dropping batch due to missing dependencies: %s",
batch.header_signature)
def _rebuild_pending_batches(self, committed_batches, uncommitted_batches):
"""When the chain head is changed. This recomputes the list of pending
transactions
:param committed_batches: Batches committed in the current chain
since the root of the fork switching from.
:param uncommitted_batches: Batches that were committed in the old
fork since the common root.
"""
if committed_batches is None:
committed_batches = []
if uncommitted_batches is None:
uncommitted_batches = []
committed_set = set([x.header_signature for x in committed_batches])
pending_batches = self._pending_batches
self._pending_batches = []
# Uncommitted and pending disjoint sets
# since batches can only be committed to a chain once.
for batch in uncommitted_batches:
if batch.header_signature not in committed_set:
if self._check_batch_dependencies(batch,
self._committed_txn_cache):
self._pending_batches.append(batch)
for batch in pending_batches:
if batch.header_signature not in committed_set:
if self._check_batch_dependencies(batch,
self._committed_txn_cache):
self._pending_batches.append(batch)
def on_chain_updated(self,
chain_head,
committed_batches=None,
uncommitted_batches=None):
"""
The existing chain has been updated, the current head block has
changed.
:param chain_head: the new head of block_chain
:param committed_batches: the set of batches that were committed
as part of the new chain.
:param uncommitted_batches: the list of transactions if any that are
now de-committed when the new chain was selected.
:return: None
"""
try:
with self._lock:
LOGGER.info('Now building on top of block: %s', chain_head)
self._chain_head = chain_head
if self._candidate_block is not None and \
chain_head is not None and \
chain_head.identifier == \
self._candidate_block.previous_block_id:
# nothing to do. We are building of the current head.
# This can happen after we publish a block and
# opportunistically create a new block.
return
elif chain_head is None:
# we don't have a chain head, we cannot build blocks
self._candidate_block = None
self._consensus = None
for batch in self._pending_batches:
self._committed_txn_cache.add_batch(batch)
else:
self._rebuild_pending_batches(committed_batches,
uncommitted_batches)
self._candidate_block = self._build_block(chain_head)
# pylint: disable=broad-except
except Exception as exc:
LOGGER.critical("on_chain_updated exception.")
LOGGER.exception(exc)
def _finalize_block(self, block):
if self._scheduler:
self._scheduler.finalize()
self._scheduler.complete(block=True)
# Read valid batches from self._scheduler
pending_batches = self._pending_batches
# this is a transaction cache to track the transactions committed
# upto this batch.
committed_txn_cache = TransactionCache(self._block_cache.block_store)
self._pending_batches = []
self._committed_txn_cache = TransactionCache(
self._block_cache.block_store)
state_hash = None
for batch in pending_batches:
result = self._scheduler.get_batch_execution_result(
batch.header_signature)
# if a result is None, this means that the executor never
# received the batch and it should be added to
# the pending_batches
if result is None:
self._pending_batches.append(batch)
self._committed_txn_cache.add_batch(batch)
elif result.is_valid:
# check if a dependent batch failed. This could be belt and
# suspenders action here but it is logically possible that
# a transaction has a dependency that fails it could
# still succeed validation. In that case we do not want
# to add it to the batch.
if not self._check_batch_dependencies(batch,
committed_txn_cache):
LOGGER.debug(
"Batch %s invalid, due to missing txn "
"dependency.", batch.header_signature)
LOGGER.debug(
"Abandoning block %s:" +
"root state hash has invalid txn applied", block)
pending_batches.remove(batch)
self._pending_batches = pending_batches
self._committed_txn_cache = \
TransactionCache(self._block_cache.block_store)
return False
else:
block.add_batch(batch)
self._committed_txn_cache.add_batch(batch)
state_hash = result.state_hash
else:
committed_txn_cache.uncommit_batch(batch)
LOGGER.debug("Batch %s invalid, not added to block.",
batch.header_signature)
if state_hash is None:
LOGGER.debug("Abandoning block %s no batches added", block)
return False
if not self._consensus.finalize_block(block.block_header):
LOGGER.debug(
"Abandoning block %s, consensus failed to finalize "
"it", block)
return False
self._consensus = None
block.set_state_hash(state_hash)
self._sign_block(block)
return True
def on_check_publish_block(self, force=False):
"""Ask the consensus module if it is time to claim the candidate block
if it is then, claim it and tell the world about it.
:return:
None
"""
try:
with self._lock:
if self._chain_head is not None and\
self._candidate_block is None and\
len(self._pending_batches) != 0:
self._candidate_block = self._build_block(self._chain_head)
if self._candidate_block and \
(force or len(self._pending_batches) != 0) and \
self._consensus.check_publish_block(self.
_candidate_block.
block_header):
candidate = self._candidate_block
self._candidate_block = None
if not self._finalize_block(candidate):
return
block = BlockWrapper(candidate.build_block())
self._block_cache[block.identifier] = block # add the
# block to the cache, so we can build on top of it.
self._block_sender.send(block.block)
LOGGER.info("Claimed Block: %s", block)
# We built our candidate, disable processing until
# the chain head is updated.
self.on_chain_updated(None)
# pylint: disable=broad-except
except Exception as exc:
LOGGER.critical("on_check_publish_block exception.")
LOGGER.exception(exc)
def finalize_block(self, identity_signing_key, pending_batches):
"""Compose the final Block to publish. This involves flushing
the scheduler, having consensus bless the block, and signing
the block.
:param identity_signing_key: the key to sign the block with.
:param pending_batches: list to receive any batches that were
submitted to add to the block but were not validated before this
call.
"""
self._scheduler.finalize()
self._scheduler.complete(block=True)
# this is a transaction cache to track the transactions committed
# up to this batch. Only valid transactions that were processed
# by the scheduler are added.
committed_txn_cache = TransactionCache(self._block_store)
builder = self._block_builder
state_hash = None
for batch in self._pending_batches:
result = self._scheduler.get_batch_execution_result(
batch.header_signature)
# if a result is None, this means that the executor never
# received the batch and it should be added to
# the pending_batches, to be added to the next
# block
if result is None:
pending_batches.append(batch)
elif result.is_valid:
# check if a dependent batch failed. This could be belt and
# suspenders action here but it is logically possible that
# a transaction has a dependency that fails it could
# still succeed validation. In which case we do not want
# to add it to the batch.
if not self._check_batch_dependencies(batch,
committed_txn_cache):
LOGGER.debug("Batch %s invalid, due to missing txn "
"dependency.", batch.header_signature)
LOGGER.debug("Abandoning block %s:" +
"root state hash has invalid txn applied",
builder)
return None
else:
builder.add_batch(batch)
committed_txn_cache.add_batch(batch)
if result.state_hash is not None:
state_hash = result.state_hash
else:
LOGGER.debug("Batch %s invalid, not added to block.",
batch.header_signature)
if state_hash is None:
LOGGER.debug("Abandoning block %s no batches added", builder)
return None
if not self._consensus.finalize_block(builder.block_header):
LOGGER.debug("Abandoning block %s, consensus failed to finalize "
"it", builder)
return False
builder.set_state_hash(state_hash)
self._sign_block(builder, identity_signing_key)
return builder.build_block()
def finalize_block(self, identity_signing_key, pending_batches):
"""Compose the final Block to publish. This involves flushing
the scheduler, having consensus bless the block, and signing
the block.
:param identity_signing_key: the key to sign the block with.
:param pending_batches: list to receive any batches that were
submitted to add to the block but were not validated before this
call.
:return: The generated Block, or None if Block failed to finalize.
In both cases the pending_batches will contain the list of batches
that need to be added to the next Block that is built.
"""
self._scheduler.finalize()
self._scheduler.complete(block=True)
# this is a transaction cache to track the transactions committed
# up to this batch. Only valid transactions that were processed
# by the scheduler are added.
committed_txn_cache = TransactionCache(self._block_store)
builder = self._block_builder
bad_batches = [] # the list of batches that failed processing
state_hash = None
# Walk the pending batch list:
# - find the state hash for the block, the block state_hash is
# is randomly placed on one of the transactions, so must interogate
# every batch to find it. If it is on a batch that failed processing
# then this block will be abandoned.
# - build three lists of batches:
# 1) a lists of all valid transactions that will be included in the
# block, these are added to the BlockBuilder to include in the Block
# 2) all batches that were not executed, these are to be returned
# in the pending_batches list
# 3) all batches that failed processing. These will be discarded.
# This list is needed in some case when the block is abandoned to
# make sure they do not remain in the pending_batches list.
for batch in self._pending_batches:
result = self._scheduler.get_batch_execution_result(
batch.header_signature)
# if a result is None, this means that the executor never
# received the batch and it should be added to
# the pending_batches, to be added to the next
# block
if result is None:
pending_batches.append(batch)
elif result.is_valid:
# check if a dependent batch failed. This could be belt and
# suspenders action here but it is logically possible that
# a transaction has a dependency that fails it could
# still succeed validation. In which case we do not want
# to add it to the batch.
if not self._check_batch_dependencies(batch,
committed_txn_cache):
LOGGER.debug(
"Batch %s invalid, due to missing txn "
"dependency.", batch.header_signature)
LOGGER.debug(
"Abandoning block %s:" +
"root state hash has invalid txn applied", builder)
# Update the pending batch list to be all the
# batches that passed validation to this point and
# none of the ones that failed. It is possible that
# this batch caused a future batch to fail so
# we leave all of the batches that failed after this
# one in the list.
bad_batches.append(batch)
pending_batches.clear()
pending_batches.extend([
x for x in self._pending_batches
if x not in bad_batches
])
return None
else:
builder.add_batch(batch)
committed_txn_cache.add_batch(batch)
if result.state_hash is not None:
state_hash = result.state_hash
else:
bad_batches.append(batch)
LOGGER.debug("Batch %s invalid, not added to block.",
batch.header_signature)
if state_hash is None or not builder.batches:
LOGGER.debug("Abandoning block %s: no batches added", builder)
return None
if not self._consensus.finalize_block(builder.block_header):
LOGGER.debug(
"Abandoning block %s, consensus failed to finalize "
"it", builder)
# return all valid batches to the pending_batches list
pending_batches.clear()
pending_batches.extend(
[x for x in self._pending_batches if x not in bad_batches])
return None
builder.set_state_hash(state_hash)
self._sign_block(builder, identity_signing_key)
return builder.build_block()
def finalize_block(self, identity_signing_key, pending_batches):
"""Compose the final Block to publish. This involves flushing
the scheduler, having consensus bless the block, and signing
the block.
:param identity_signing_key: the key to sign the block with.
:param pending_batches: list to receive any batches that were
submitted to add to the block but were not validated before this
call.
:return: The generated Block, or None if Block failed to finalize.
In both cases the pending_batches will contain the list of batches
that need to be added to the next Block that is built.
"""
self._scheduler.finalize()
self._scheduler.complete(block=True)
# this is a transaction cache to track the transactions committed
# up to this batch. Only valid transactions that were processed
# by the scheduler are added.
committed_txn_cache = TransactionCache(self._block_store)
builder = self._block_builder
bad_batches = [] # the list of batches that failed processing
state_hash = None
# Walk the pending batch list:
# - find the state hash for the block, the block state_hash is
# is randomly placed on one of the transactions, so must interogate
# every batch to find it. If it is on a batch that failed processing
# then this block will be abandoned.
# - build three lists of batches:
# 1) a lists of all valid transactions that will be included in the
# block, these are added to the BlockBuilder to include in the Block
# 2) all batches that were not executed, these are to be returned
# in the pending_batches list
# 3) all batches that failed processing. These will be discarded.
# This list is needed in some case when the block is abandoned to
# make sure they do not remain in the pending_batches list.
for batch in self._pending_batches:
result = self._scheduler.get_batch_execution_result(
batch.header_signature)
# if a result is None, this means that the executor never
# received the batch and it should be added to
# the pending_batches, to be added to the next
# block
if result is None:
pending_batches.append(batch)
elif result.is_valid:
# check if a dependent batch failed. This could be belt and
# suspenders action here but it is logically possible that
# a transaction has a dependency that fails it could
# still succeed validation. In which case we do not want
# to add it to the batch.
if not self._check_batch_dependencies(batch,
committed_txn_cache):
LOGGER.debug("Batch %s invalid, due to missing txn "
"dependency.", batch.header_signature)
LOGGER.debug("Abandoning block %s:" +
"root state hash has invalid txn applied",
builder)
# Update the pending batch list to be all the
# batches that passed validation to this point and
# none of the ones that failed. It is possible that
# this batch caused a future batch to fail so
# we leave all of the batches that failed after this
# one in the list.
bad_batches.append(batch)
pending_batches.clear()
pending_batches.extend([x for x in self._pending_batches
if x not in bad_batches])
return None
else:
builder.add_batch(batch)
committed_txn_cache.add_batch(batch)
if result.state_hash is not None:
state_hash = result.state_hash
else:
bad_batches.append(batch)
LOGGER.debug("Batch %s invalid, not added to block.",
batch.header_signature)
if state_hash is None or not builder.batches:
LOGGER.debug("Abandoning block %s: no batches added", builder)
return None
if not self._consensus.finalize_block(builder.block_header):
LOGGER.debug("Abandoning block %s, consensus failed to finalize "
"it", builder)
# return all valid batches to the pending_batches list
pending_batches.clear()
pending_batches.extend([x for x in self._pending_batches
if x not in bad_batches])
return None
builder.set_state_hash(state_hash)
self._sign_block(builder, identity_signing_key)
return builder.build_block()