def test_block_cache(self):
block_store = {}
cache = BlockCache(block_store=block_store, keep_time=1,
purge_frequency=1)
header1 = BlockHeader(previous_block_id="000")
block1 = BlockWrapper(Block(header=header1.SerializeToString(),
header_signature="ABC"))
header2 = BlockHeader(previous_block_id="ABC")
block2 = BlockWrapper(Block(header=header2.SerializeToString(),
header_signature="DEF"))
header3 = BlockHeader(previous_block_id="BCA")
block3 = BlockWrapper(Block(header=header3.SerializeToString(),
header_signature="FED"))
cache[block1.header_signature] = block1
cache[block2.header_signature] = block2
# Check that blocks are in the BlockCache
self.assertIn("ABC", cache)
self.assertIn("DEF", cache)
# Wait for purge time to expire
time.sleep(1)
# Add "FED"
cache[block3.header_signature] = block3
# Check that "ABC" is still in the cache even though the keep time has
# expired because it has a referecne count of 1 but "DEF" has been
# removed
self.assertIn("ABC", cache)
self.assertNotIn("DEF", cache)
self.assertIn("FED", cache)
def do_load_from_block_store():
bs = {}
block1 = Block(
header=BlockHeader(previous_block_id="000").SerializeToString(),
header_signature="test")
bs["test"] = BlockWrapper(block1)
block2 = Block(
header=BlockHeader(previous_block_id="000").SerializeToString(),
header_signature="test2")
blkw2 = BlockWrapper(block2)
bs["test2"] = blkw2
bc = BlockCache(bs)
return bc, blkw2
def _get_previous_block_state_root(self, block):
block_header = BlockHeader()
block_header.ParseFromString(block.header)
if block_header.previous_block_id == NULL_BLOCK_IDENTIFIER:
return INIT_ROOT_KEY
try:
block = next(
self._block_manager.get([block_header.previous_block_id]))
except StopIteration:
return None
block_header = BlockHeader()
block_header.ParseFromString(block.header)
return block_header.state_root_hash
def handle_request(self, request, response):
try:
LOGGER.debug('ConsensusChainHeadGetHandler: proxy parent_id=(%s) new_parent=(%s) is_new=%s',request.parent_id.hex()[:8],request.new_parent_id.hex()[:8],request.is_new)
chain_head = self._proxy.chain_head_get(request.parent_id,request.new_parent_id,request.is_new)
block_header = BlockHeader()
"""
chain_head.header for RUST
"""
block_header.ParseFromString(chain_head.block.header)
response.block.block_id = bytes.fromhex(
chain_head.header_signature)
response.block.previous_id =\
bytes.fromhex(block_header.previous_block_id)
response.block.signer_id =\
bytes.fromhex(block_header.signer_public_key)
response.block.block_num = block_header.block_num
response.block.payload = block_header.consensus
except TooManyBranch:
response.status = consensus_pb2.ConsensusChainHeadGetResponse.TOO_MANY_BRANCH
# change bgx.publisher.max_batches_per_block after nests were made
self._proxy.reset_max_batches_per_block()
except UnknownBlock:
response.status = consensus_pb2.ConsensusChainHeadGetResponse.NO_CHAIN_HEAD
except Exception: # pylint: disable=broad-except
LOGGER.exception("ConsensusChainHeadGet")
response.status = consensus_pb2.ConsensusChainHeadGetResponse.SERVICE_ERROR
def handle_request(self, request, response, connection_id):
try:
blocks = []
for block in self._proxy.blocks_get(request.block_ids):
block_header = BlockHeader()
block_header.ParseFromString(block.header)
blocks.append(
consensus_pb2.ConsensusBlock(
block_id=bytes.fromhex(block.header_signature),
previous_id=bytes.fromhex(
block_header.previous_block_id),
signer_id=bytes.fromhex(
block_header.signer_public_key),
block_num=block_header.block_num,
payload=block_header.consensus))
response.blocks.extend(blocks)
except UnknownBlock:
response.status =\
consensus_pb2.ConsensusBlocksGetResponse.UNKNOWN_BLOCK
except Exception: # pylint: disable=broad-except
LOGGER.exception("ConsensusBlocksGet")
response.status =\
consensus_pb2.ConsensusBlocksGetResponse.SERVICE_ERROR
return HandlerStatus.RETURN
def handle(self, identity, message_content):
helper = _ClientHelper(
message_content,
client_pb2.ClientBlockListRequest,
client_pb2.ClientBlockListResponse,
validator_pb2.Message.CLIENT_BLOCK_LIST_RESPONSE,
block_store=self._block_store)
blocks = [helper.get_head_block()]
if helper.has_response():
return helper.result
# Build block list
while True:
header = BlockHeader()
header.ParseFromString(blocks[-1].header)
previous_id = header.previous_block_id
if previous_id not in self._block_store:
break
blocks.append(self._block_store[previous_id].block)
helper.set_response(helper.status.OK,
head_id=helper.head_id,
blocks=blocks)
return helper.result
def handle(self, identity, message_content):
helper = _ClientHelper(
message_content,
client_pb2.ClientBlockListRequest,
client_pb2.ClientBlockListResponse,
validator_pb2.Message.CLIENT_BLOCK_LIST_RESPONSE,
block_store=self._block_store)
helper.set_head_id()
if helper.has_response():
return helper.result
# Build block list
current_id = helper.head_id
blocks = []
while current_id in self._block_store:
block = self._block_store[current_id].block
blocks.append(block)
header = BlockHeader()
header.ParseFromString(block.header)
current_id = header.previous_block_id
if blocks:
helper.set_response(
helper.status.OK,
head_id=helper.head_id,
blocks=blocks)
else:
helper.set_response(helper.status.NO_ROOT)
return helper.result
def get_configured_engine(block, settings_view_factory):
header = BlockHeader()
header.ParseFromString(block.header)
settings_view = settings_view_factory.create_settings_view(
header.state_root_hash)
conf_name = settings_view.get_setting('sawtooth.consensus.algorithm.name')
conf_version = settings_view.get_setting(
'sawtooth.consensus.algorithm.version')
# Fallback to devmode if nothing else is set
name = "Devmode"
version = "0.1"
# If name and version settings aren't set, check for PoET
if conf_name is None or conf_version is None:
algorithm = settings_view.get_setting('sawtooth.consensus.algorithm')
if algorithm and (algorithm.lower() == 'poet'):
name = "PoET"
# Otherwise use name and version settings
else:
name = conf_name
version = conf_version
return name, version
def _build_block(self, chain_head):
""" Build a candidate block and construct the consensus object to
validate it.
"""
prev_state = self._get_previous_block_root_state_hash(chain_head)
state_view = self._state_view_factory. \
create_view(prev_state)
self._consensus = self._consensus_module.\
BlockPublisher(block_cache=self._block_cache,
state_view=state_view)
block_header = BlockHeader(
block_num=chain_head.block_num + 1,
previous_block_id=chain_head.header_signature)
self._consensus.initialize_block(block_header)
# create a new scheduler
# TBD move factory in to executor for easier mocking --
# Yes I want to make fun of it.
self._scheduler = self._transaction_executor.create_scheduler(
self._squash_handler, chain_head.state_root_hash)
self._transaction_executor.execute(self._scheduler)
for batch in self._pending_batches:
self._scheduler.add_batch(batch)
self._pending_batches = []
return BlockBuilder(block_header)
def _set_root(self, request):
"""Sets the root of the merkle tree, returning any head id used.
Note:
This method will fail if `_tree` has not been set
Args:
request (object): The parsed protobuf request object
Returns:
None: if a merkle_root is specified directly, no id is returned
str: the id of the head block used to specify the root
Raises:
ResponseFailed: Failed to set the root if the merkle tree
"""
if request.merkle_root:
root = request.merkle_root
head_id = None
else:
head = self._get_head_block(request)
header = BlockHeader()
header.ParseFromString(head.header)
root = header.state_root_hash
head_id = head.header_signature
try:
self._tree.set_merkle_root(root)
except KeyError as e:
LOGGER.debug('Unable to find root "%s" in database', e)
raise _ResponseFailed(self._status.NO_ROOT)
return head_id
def _make_block(self, txns_family, signer_pubkey, same_pubkey=True):
transactions = []
for family in txns_family:
txn_header = TransactionHeader(
family_name=family,
signer_pubkey=signer_pubkey)
txn = Transaction(header=txn_header.SerializeToString())
transactions.append(txn)
batch = Batch(transactions=transactions)
if same_pubkey:
block_header = BlockHeader(signer_pubkey=signer_pubkey)
else:
block_header = BlockHeader(signer_pubkey="other")
block = Block(header=block_header.SerializeToString(), batches=[batch])
return BlockWrapper(block)
def state_get(self, block_id, addresses):
'''Returns a list of address/data pairs (str, bytes)'''
block = self._get_blocks([block_id.hex()])[0]
block_header = BlockHeader()
block_header.ParseFromString(block.header)
state_view = self._state_view_factory.create_view(
block_header.state_root_hash)
result = []
for address in addresses:
# a fully specified address
if len(address) == 70:
try:
value = state_view.get(address)
except KeyError:
# if the key is missing, leave it out of the response
continue
result.append((address, value))
continue
# an address prefix
leaves = state_view.leaves(address)
for leaf in leaves:
result.append(leaf)
return result
def activate_if_configured(self, engine_name, engine_version):
# Wait until chain head is committed
chain_head = None
while chain_head is None:
try:
chain_head = self.chain_head_get()
except UnknownBlock:
pass
header = BlockHeader()
header.ParseFromString(chain_head.header)
settings_view = self._settings_view_factory.create_settings_view(
header.state_root_hash)
conf_name = settings_view.get_setting(
'sawtooth.consensus.algorithm.name')
conf_version = settings_view.get_setting(
'sawtooth.consensus.algorithm.version')
if engine_name == conf_name and engine_version == conf_version:
self._consensus_registry.activate_engine(engine_name,
engine_version)
self._consensus_notifier.notify_engine_activated(chain_head)
LOGGER.info("Consensus engine activated: %s %s", engine_name,
engine_version)
def get_configured_engine(block, settings_view_factory):
header = BlockHeader()
header.ParseFromString(block.header)
settings_view = settings_view_factory.create_settings_view(
header.state_root_hash)
conf_name = settings_view.get_setting('sawtooth.consensus.algorithm.name')
conf_version = settings_view.get_setting(
'sawtooth.consensus.algorithm.version')
# For backwards compatibility with 1.0:
# - Use version "0.1" if sawtooth.consensus.algorithm.version is unset
# - Use sawtooth.consensus.algorithm if sawtooth.consensus.algorithm.name
# is unset
# - Use "Devmode" if sawtooth.consensus.algorithm is unset
if conf_version is not None:
version = conf_version
else:
version = "0.1"
if conf_name is not None:
name = conf_name
else:
algorithm = settings_view.get_setting('sawtooth.consensus.algorithm')
if algorithm is not None:
name = algorithm
else:
name = "Devmode"
return name, version
def gen_block(self, block_id, prev_id, num, batches):
return BlockWrapper(
Block(header_signature=block_id,
batches=batches,
header=BlockHeader(
block_num=num,
previous_block_id=prev_id).SerializeToString()))
def settings_get(self, block_id, settings):
'''Returns a list of key/value pairs (str, str).'''
block = self._get_blocks([block_id.hex()])[0]
block_header = BlockHeader()
block_header.ParseFromString(block.header)
try:
settings_view = self._settings_view_factory.create_settings_view(
block_header.state_root_hash)
except KeyError:
LOGGER.error(
'Settings from block %s requested, but root hash %s was '
'missing. Returning no setting values.', block_id.hex(),
block_header.state_root_hash)
# The state root does not exist, which may indicate a pruned root
# from a dropped fork or an invalid state.
return []
result = []
for setting in settings:
try:
value = settings_view.get_setting(setting)
except KeyError:
# if the key is missing, leave it out of the response
continue
result.append((setting, value))
return result
def handle(self, connection_id, message_content):
message = GossipMessage()
message.ParseFromString(message_content)
if message.content_type == "BLOCK":
public_key = \
self._network.connection_id_to_public_key(connection_id)
block = Block()
block.ParseFromString(message.content)
header = BlockHeader()
header.ParseFromString(block.header)
if header.signer_public_key == public_key:
permitted = \
self._permission_verifier.check_network_consensus_role(
public_key)
if not permitted:
LOGGER.debug(
"Public key is not permitted to publish block, "
"remove connection: %s", connection_id)
self._gossip.unregister_peer(connection_id)
violation = AuthorizationViolation(
violation=RoleType.Value("NETWORK"))
return HandlerResult(HandlerStatus.RETURN_AND_CLOSE,
message_out=violation,
message_type=validator_pb2.Message.
AUTHORIZATION_VIOLATION)
# if allowed pass message
return HandlerResult(HandlerStatus.PASS)
def _create_blocks(self,
block_count,
batch_count,
valid_block=True,
valid_batch=True):
block_list = []
for i in range(block_count):
batch_list = self._create_batches(batch_count,
2,
valid_batch=valid_batch)
batch_ids = [batch.header_signature for batch in batch_list]
block_header = BlockHeader(signer_pubkey=self.public_key,
batch_ids=batch_ids)
header_bytes = block_header.SerializeToString()
if valid_block:
signature = signing.sign(header_bytes, self.private_key)
else:
signature = "bad_signature"
block = Block(header=header_bytes,
batches=batch_list,
header_signature=signature)
block_list.append(block)
return block_list
def on_block_received(self, block):
with self._lock:
if block.header_signature in self._block_store:
# do we already have this block
return
header = BlockHeader()
header.ParseFromString(block.header)
block = BlockWrapper(header, block)
block_state = BlockState(block_wrapper=block, weight=0,
status=BlockStatus.Unknown)
self._block_store[block.header_signature] = block_state
self._blocks_pending[block.header_signature] = []
if block.header_signature in self._blocks_requested:
# is it a requested block
# route block to the validator that requested
validator = self._blocks_requested.pop(block.header_signature)
if validator.chain_head.block.header_signature != \
self._chain_head.block.header_signature:
# the head of the chain has changed start over
self._verify_block(validator.new_block)
else:
self._executor.submit(validator.run)
elif block.previous_block_id in self._blocks_processing:
# if the previous block is being processed...put it in a wait
# queue
pending_blocks = \
self._blocks_pending.get(block.previous_block_id,
[])
pending_blocks.append(block_state)
self._blocks_pending[block.previous_block_id] = \
pending_blocks
else:
# schedule this block for validation.
self._verify_block(block_state)
def notify_block_new(self, block):
"""
A new block was received and passed initial consensus validation
in federation mode - send only own cluster's nodes
"""
summary = hashlib.sha256()
for batch in block.batches:
summary.update(batch.header_signature.encode())
LOGGER.debug(
'ConsensusNotifier: notify_block_new BLOCK=%s SUMMARY=%s\n',
block.header_signature[:8],
summary.digest().hex()[:10])
block_header = BlockHeader()
block_header.ParseFromString(block.header)
self._notify(
validator_pb2.Message.CONSENSUS_NOTIFY_BLOCK_NEW,
consensus_pb2.ConsensusNotifyBlockNew(
block=consensus_pb2.ConsensusBlock(
block_id=bytes.fromhex(block.header_signature),
previous_id=bytes.fromhex(block_header.previous_block_id),
signer_id=bytes.fromhex(block_header.signer_public_key),
block_num=block_header.block_num,
payload=block_header.consensus,
summary=summary.digest())))
def _create_block(self):
return BlockWrapper.wrap(
Block(header_signature='some_block_id',
batches=[],
header=BlockHeader(block_num=0,
previous_block_id=NULL_BLOCK_IDENTIFIER).
SerializeToString()))
def add_block(self, base_id, root=b'merkle_root'.hex()):
block_id = 'b' * (128 - len(base_id)) + base_id
head = self.chain_head
if head:
previous_id = head.header_signature
num = head.header.block_num + 1
else:
previous_id = 'zzzzz'
num = 0
signer_public_key = b'public_key' + bytes(base_id, 'utf-8')
header = BlockHeader(
block_num=num,
previous_block_id=previous_id,
signer_public_key=signer_public_key.hex(),
batch_ids=[block_id],
consensus=b'consensus',
state_root_hash=root)
block = Block(
header=header.SerializeToString(),
header_signature=block_id,
batches=[make_mock_batch(base_id)])
self.put_blocks([block])
def set_root(self):
"""
Used by handlers that fetch data from the merkle tree. Sets the tree
with the proper root, and returns the chain head id if used.
"""
if self.has_response():
return
if self.request.merkle_root:
self._tree.set_merkle_root(self.request.merkle_root)
return
if self.request.head_id:
try:
head = self._block_store[self.request.head_id].block
except KeyError as e:
LOGGER.debug('Unable to find block "%s" in store', e)
self.set_response(self.status.NO_ROOT)
else:
head = self.get_genesis()
if self.has_response():
return
header = BlockHeader()
header.ParseFromString(head.header)
self._tree.set_merkle_root(header.state_root_hash)
self.head_id = head.header_signature
def handle_request(self, request, response):
try:
blocks = []
for block in self._proxy.blocks_get(request.block_ids):
LOGGER.debug('ConsensusBlocksGetHandler: block %s',type(block.header))
"""
block manager return blocks from store where header is string and we should decode it
in case of chain controller it is object
"""
if not isinstance(block.header,BlockHeader) :
block_header = BlockHeader()
block_header.ParseFromString(block.header)
else:
block_header = block.header
blocks.append(consensus_pb2.ConsensusBlock(
block_id=bytes.fromhex(block.header_signature),
previous_id=bytes.fromhex(block_header.previous_block_id),
signer_id=bytes.fromhex(block_header.signer_public_key),
block_num=block_header.block_num,
payload=block_header.consensus))
response.blocks.extend(blocks)
except UnknownBlock:
LOGGER.debug('ConsensusBlocksGetHandler:proxy UNKNOWN_BLOCK')
response.status = consensus_pb2.ConsensusBlocksGetResponse.UNKNOWN_BLOCK
except Exception: # pylint: disable=broad-except
LOGGER.exception("ConsensusBlocksGet")
response.status = consensus_pb2.ConsensusBlocksGetResponse.SERVICE_ERROR
def __init__(self, head_id, block_manager, block_store):
"""The constructor should be passed the previous block id of the block
being validated."""
uncommitted_block_ids = list()
uncommitted_batch_ids = set()
uncommitted_txn_ids = set()
# Find the most recent ancestor of this block that is in the block
# store. Batches and transactions that are in a block that is in the
# block store and that has a greater block number than this block must
# be ignored.
if head_id != NULL_BLOCK_IDENTIFIER:
head = next(block_manager.get([head_id]))
ancestor = head
while ancestor.header_signature not in block_store:
# For every block not in the block store, we need to track all
# its batch ids and transaction ids separately to ensure there
# are no duplicates.
for batch in ancestor.batches:
uncommitted_batch_ids.add(batch.header_signature)
for txn in batch.transactions:
uncommitted_txn_ids.add(txn.header_signature)
uncommitted_block_ids.append(ancestor.header_signature)
ancestor_header = BlockHeader()
ancestor_header.ParseFromString(ancestor.header)
previous_block_id = ancestor_header.previous_block_id
if previous_block_id == NULL_BLOCK_IDENTIFIER:
break
ancestor = next(block_manager.get([previous_block_id]))
else:
ancestor = None
self.block_store = block_store
ancestor_header = None
if ancestor:
ancestor_header = BlockHeader()
ancestor_header.ParseFromString(ancestor.header)
self.common_ancestor = ancestor_header
self.uncommitted_block_ids = uncommitted_block_ids
self.uncommitted_batch_ids = uncommitted_batch_ids
self.uncommitted_txn_ids = uncommitted_txn_ids
def _generate_genesis_block(self, header_sig="genesis"):
genesis_header = BlockHeader(previous_block_id="0000000000000000",
block_num=0)
# Small hack here not asking consensus if it is happy.
block = BlockWrapper(genesis_header)
block.set_signature(header_sig)
return block
def create_block(self, previous_block_id="0000000000000000"):
block_header = BlockHeader(block_num=85,
state_root_hash="0987654321fedcba",
previous_block_id=previous_block_id)
block = BlockWrapper(
Block(header_signature="abcdef1234567890",
header=block_header.SerializeToString()))
return block
def header(self):
"""
Returns the header of the block
"""
if self._block_header is None:
self._block_header = BlockHeader()
self._block_header.ParseFromString(self.block.header)
return self._block_header
def _generate_genesis_block():
genesis_header = BlockHeader(
previous_block_id="0000000000000000",
block_num=0)
block = BlockWrapper(genesis_header)
block.set_signature("genesis")
return block
def generate_genesis_block(self):
genesis_header = BlockHeader(previous_block_id=NULLIDENTIFIER,
block_num=0)
# Small hack here not asking consensus if it is happy.
self._candidate_block = \
self._finalize_block(BlockWrapper(genesis_header))
return self._candidate_block
