def test_batches_rejected_by_scheduler(self, mock_batch_injector_factory):
'''
Test that no block is published with
batches rejected by the scheduler
'''
mock_batch_injector_factory.create_injectors.return_value = []
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(
batch_execution_result=False),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.receive_batches()
# Block should be empty since all batches are rejected
with self.assertRaises(BlockEmpty, msg='Block should be empty'):
self.publish_block()
self.assert_no_block_published()
def test_batch_injection_start_block(self):
'''
Test that the batch is injected at the beginning of the block.
'''
injected_batch = self.make_batch()
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=self.permission_verifier,
batch_observers=[],
batch_injector_factory=MockBatchInjectorFactory(injected_batch))
self.receive_batches()
self.publish_block()
self.assert_batch_in_block(injected_batch)
def setUp(self, mock_batch_injector_factory):
mock_batch_injector_factory.create_injectors.return_value = []
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.permission_verifier = MockPermissionVerifier()
self.publisher = BlockPublisher(
block_store=self.block_tree_manager.block_store,
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def test_max_block_size(self):
'''
Test block publisher obeys the block size limits
'''
# Create a publisher that has a state view
# with a batch limit
addr, value = CreateSetting(
'sawtooth.publisher.max_batches_per_block', 1)
print('test', addr)
self.state_view_factory = MockStateViewFactory(
{addr: value})
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
# try to publish with no pending queue (failing)
for i in range(self.batch_count):
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block([self.batches[i]])
def setUp(self):
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def __init__(self):
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.permission_verifier = MockPermissionVerifier()
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
settings_cache=SettingsCache(
SettingsViewFactory(
self.block_tree_manager.state_view_factory), ),
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
check_publish_block_frequency=0.1,
batch_observers=[],
permission_verifier=self.permission_verifier)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def setUp(self):
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def test_batch_injection_start_block(self):
'''
Test that the batch is injected at the beginning of the block.
'''
injected_batch = self.make_batch()
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=self.permission_verifier,
batch_observers=[],
batch_injector_factory=MockBatchInjectorFactory(injected_batch))
self.receive_batches()
self.publish_block()
self.assert_batch_in_block(injected_batch)
def _init_subprocesses(self):
self._block_publisher = BlockPublisher(
transaction_executor=self._transaction_executor,
block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
block_sender=self._block_sender,
squash_handler=self._squash_handler,
chain_head=self._block_store.chain_head,
identity_signing_key=self._identity_signing_key
)
self._publisher_thread = self._PublisherThread(self._block_publisher,
self._batch_queue)
self._chain_controller = ChainController(
block_sender=self._block_sender,
block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
executor=ThreadPoolExecutor(1),
transaction_executor=self._transaction_executor,
on_chain_updated=self._block_publisher.on_chain_updated,
squash_handler=self._squash_handler,
chain_id_manager=self._chain_id_manager
)
self._chain_thread = self._ChainThread(self._chain_controller,
self._block_queue,
self._block_cache)
def _init_subprocesses(self):
self._block_publisher = BlockPublisher(
transaction_executor=self._transaction_executor,
block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
block_sender=self._block_sender,
batch_sender=self._batch_sender,
squash_handler=self._squash_handler,
chain_head=self._block_store.chain_head,
identity_signing_key=self._identity_signing_key,
data_dir=self._data_dir,
config_dir=self._config_dir)
self._publisher_thread = self._PublisherThread(
block_publisher=self._block_publisher,
batch_queue=self._batch_queue,
check_publish_block_frequency=self._check_publish_block_frequency)
self._chain_controller = ChainController(
block_sender=self._block_sender,
block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
executor=self._executor_threadpool,
transaction_executor=self._transaction_executor,
chain_head_lock=self._block_publisher.chain_head_lock,
on_chain_updated=self._block_publisher.on_chain_updated,
squash_handler=self._squash_handler,
chain_id_manager=self._chain_id_manager,
state_delta_processor=self._state_delta_processor,
identity_signing_key=self._identity_signing_key,
data_dir=self._data_dir,
config_dir=self._config_dir)
self._chain_thread = self._ChainThread(
chain_controller=self._chain_controller,
block_queue=self._block_queue,
block_cache=self._block_cache,
block_cache_purge_frequency=self._block_cache_purge_frequency)
def do_batch_injection_start_block():
injected_batch = testBlockPublisher.make_batch()
testBlockPublisher.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=testBlockPublisher.block_tree_manager.block_cache,
state_view_factory=testBlockPublisher.state_view_factory,
settings_cache=SettingsCache(
SettingsViewFactory(
testBlockPublisher.block_tree_manager.state_view_factory),
),
block_sender=testBlockPublisher.block_sender,
batch_sender=testBlockPublisher.batch_sender,
squash_handler=None,
chain_head=testBlockPublisher.block_tree_manager.chain_head,
identity_signer=testBlockPublisher.block_tree_manager.
identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=testBlockPublisher.permission_verifier,
check_publish_block_frequency=0.1,
batch_observers=[],
batch_injector_factory=MockBatchInjectorFactory(injected_batch))
testBlockPublisher.receive_batches()
testBlockPublisher.publish_block()
def test_max_block_size(self):
'''
Test block publisher obeys the block size limits
'''
# Create a publisher that has a state view
# with a batch limit
addr, value = CreateSetting(
'sawtooth.publisher.max_batches_per_block', 1)
print('test', addr)
self.state_view_factory = MockStateViewFactory(
{addr: value})
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
# try to publish with no pending queue (failing)
for i in range(self.batch_count):
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block([self.batches[i]])
def test_batches_rejected_by_scheduler(self, mock_batch_injector_factory):
'''
Test that no block is published with
batches rejected by the scheduler
'''
mock_batch_injector_factory.create_injectors.return_value = []
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(
batch_execution_result=False),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.receive_batches()
# Block should be empty since all batches are rejected
with self.assertRaises(BlockEmpty, msg='Block should be empty'):
self.publish_block()
self.assert_no_block_published()
def test_validation_rules_reject_batches(self,
mock_batch_injector_factory):
"""Test that a batch is not added to the block if it will violate the
block validation rules.
It does the following:
- Sets the block_validation_rules to limit the number of 'test'
transactions to 1
- creates two batches, limited to 1 transaction each, and receives
them
- verifies that only the first batch was committed to the block
"""
addr, value = CreateSetting(
'sawtooth.validator.block_validation_rules', 'NofX:1,test')
self.state_view_factory = MockStateViewFactory({addr: value})
mock_batch_injector_factory.create_injectors.return_value = []
batch1 = self.make_batch(txn_count=1)
batch2 = self.make_batch(txn_count=1)
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.receive_batches(batches=[batch1, batch2])
self.publish_block()
self.assert_block_batch_count(1)
self.assert_batch_in_block(batch1)
def __init__(self, consensus, block_store, send_message,
transaction_executor, squash_handler, first_state_root):
self._consensus = consensus
self._block_store = block_store
self._send_message = send_message
self._squash_handler = squash_handler
self._block_publisher = BlockPublisher(
consensus=consensus.BlockPublisher(),
transaction_executor=transaction_executor,
send_message=send_message,
squash_handler=squash_handler)
self._batch_queue = queue.Queue()
self._publisher_thread = self._PublisherThread(self._block_publisher,
self._batch_queue)
# HACK until genesis tool is working
if "chain_head_id" not in self._block_store:
genesis_block = BlockState(
block_wrapper=self._block_publisher.generate_genesis_block(),
weight=0,
status=BlockStatus.Valid)
genesis_block.block.set_state_hash(first_state_root)
self._block_store[genesis_block.block.header_signature] = \
genesis_block
self._block_store["chain_head_id"] = \
genesis_block.block.header_signature
self._block_publisher.on_chain_updated(genesis_block.block)
LOGGER.info("Journal created genesis block: %s",
genesis_block.block.header_signature)
self._chain_controller = ChainController(
consensus=consensus.BlockVerifier(),
block_store=block_store,
send_message=send_message,
executor=ThreadPoolExecutor(1),
transaction_executor=transaction_executor,
on_chain_updated=self._block_publisher.on_chain_updated,
squash_handler=self._squash_handler)
self._block_queue = queue.Queue()
self._chain_thread = self._ChainThread(self._chain_controller,
self._block_queue)
def test_max_block_size(self, mock_batch_injector_factory):
'''
Test block publisher obeys the block size limits
'''
mock_batch_injector_factory.create_injectors.return_value = []
# Create a publisher that has a state view
# with a batch limit
addr, value = CreateSetting('sawtooth.publisher.max_batches_per_block',
1)
self.state_view_factory = MockStateViewFactory({addr: value})
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
# try to publish with no pending queue (failing)
for i in range(self.batch_count):
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block([self.batches[i]])
def test_batches_rejected_by_scheduler(self):
'''
Test that no block is published with
batches rejected by the scheduler
'''
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(
batch_execution_result=False),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None)
self.receive_batches()
self.publish_block()
self.assert_no_block_published()
def __init__(self, with_genesis=True):
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.block_store = BlockStore(
DictDatabase(indexes=BlockStore.create_index_configuration()))
self.block_cache = BlockCache(self.block_store)
self.state_db = {}
self.block_manager = BlockManager()
self.block_manager.add_store("commit_store", self.block_store)
# add the mock reference to the consensus
consensus_setting_addr = SettingsView.setting_address(
'sawtooth.consensus.algorithm')
self.state_db[consensus_setting_addr] = _setting_entry(
'sawtooth.consensus.algorithm', 'test_journal.mock_consensus')
self.state_view_factory = MockStateViewFactory(self.state_db)
context = create_context('secp256k1')
private_key = context.new_random_private_key()
crypto_factory = CryptoFactory(context)
self.signer = crypto_factory.new_signer(private_key)
identity_private_key = context.new_random_private_key()
self.identity_signer = crypto_factory.new_signer(identity_private_key)
chain_head = None
if with_genesis:
self.genesis_block = self.generate_genesis_block()
chain_head = self.genesis_block
self.block_manager.put([chain_head.block])
self.block_manager.persist(chain_head.block.header_signature,
"commit_store")
self.block_publisher = BlockPublisher(
block_manager=self.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=self.block_store.has_transaction,
batch_committed=self.block_store.has_batch,
state_view_factory=self.state_view_factory,
settings_cache=SettingsCache(
SettingsViewFactory(self.state_view_factory), ),
block_sender=self.block_sender,
batch_sender=self.block_sender,
chain_head=chain_head.block,
identity_signer=self.identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=MockPermissionVerifier(),
batch_observers=[])
def test_no_chain_head(self):
publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.blocks.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
squash_handler=None,
chain_head=self.blocks.chain_head,
identity_signing_key=self.blocks.identity_signing_key)
# Test halting the BlockPublisher by setting the chain head to null
publisher.on_chain_updated(None)
batch = Batch()
publisher.on_batch_received(batch)
publisher.on_check_publish_block()
self.assertIsNone(self.block_sender.new_block)
def test_validation_rules_reject_batches(self,
mock_batch_injector_factory):
"""Test that a batch is not added to the block if it will violate the
block validation rules.
It does the following:
- Sets the block_validation_rules to limit the number of 'test'
transactions to 1
- creates two batches, limited to 1 transaction each, and receives
them
- verifies that only the first batch was committed to the block
"""
addr, value = CreateSetting(
'sawtooth.validator.block_validation_rules', 'NofX:1,test')
self.state_view_factory = MockStateViewFactory(
{addr: value})
mock_batch_injector_factory.create_injectors.return_value = []
batch1 = self.make_batch(txn_count=1)
batch2 = self.make_batch(txn_count=1)
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.receive_batches(batches=[batch1, batch2])
self.publish_block()
self.assert_block_batch_count(1)
self.assert_batch_in_block(batch1)
def __init__(self, with_genesis=True):
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.dir = tempfile.mkdtemp()
self.block_db = NativeLmdbDatabase(
os.path.join(self.dir, 'block.lmdb'),
BlockStore.create_index_configuration())
self.block_store = BlockStore(self.block_db)
self.block_cache = BlockCache(self.block_store)
self.state_db = NativeLmdbDatabase(
os.path.join(self.dir, "merkle.lmdb"),
MerkleDatabase.create_index_configuration())
self.state_view_factory = NativeStateViewFactory(self.state_db)
self.block_manager = BlockManager()
self.block_manager.add_commit_store(self.block_store)
context = create_context('secp256k1')
private_key = context.new_random_private_key()
crypto_factory = CryptoFactory(context)
self.signer = crypto_factory.new_signer(private_key)
identity_private_key = context.new_random_private_key()
self.identity_signer = crypto_factory.new_signer(identity_private_key)
chain_head = None
if with_genesis:
self.genesis_block = self.generate_genesis_block()
chain_head = self.genesis_block
self.block_manager.put([chain_head.block])
self.block_manager.persist(chain_head.block.header_signature,
"commit_store")
self.block_publisher = BlockPublisher(
block_manager=self.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=self.block_store.has_transaction,
batch_committed=self.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.block_sender,
chain_head=chain_head.block,
identity_signer=self.identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=MockPermissionVerifier(),
batch_observers=[])
def __init__(self, with_genesis=True):
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.block_store = BlockStore(DictDatabase(
indexes=BlockStore.create_index_configuration()))
self.block_cache = BlockCache(self.block_store)
self.state_db = {}
# add the mock reference to the consensus
consensus_setting_addr = SettingsView.setting_address(
'sawtooth.consensus.algorithm')
self.state_db[consensus_setting_addr] = _setting_entry(
'sawtooth.consensus.algorithm', 'test_journal.mock_consensus')
self.state_view_factory = MockStateViewFactory(self.state_db)
context = create_context('secp256k1')
private_key = context.new_random_private_key()
crypto_factory = CryptoFactory(context)
self.signer = crypto_factory.new_signer(private_key)
identity_private_key = context.new_random_private_key()
self.identity_signer = crypto_factory.new_signer(identity_private_key)
chain_head = None
if with_genesis:
self.genesis_block = self.generate_genesis_block()
self.set_chain_head(self.genesis_block)
chain_head = self.genesis_block
self.block_publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_cache,
state_view_factory=self.state_view_factory,
settings_cache=SettingsCache(
SettingsViewFactory(self.state_view_factory),
),
block_sender=self.block_sender,
batch_sender=self.block_sender,
squash_handler=None,
chain_head=chain_head,
identity_signer=self.identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=MockPermissionVerifier(),
check_publish_block_frequency=0.1,
batch_observers=[])
def _init_subprocesses(self):
batch_injector_factory = DefaultBatchInjectorFactory(
block_store=self._block_store,
state_view_factory=self._state_view_factory,
signing_key=self._identity_signing_key,
)
self._block_publisher = BlockPublisher(
transaction_executor=self._transaction_executor,
block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
block_sender=self._block_sender,
batch_sender=self._batch_sender,
squash_handler=self._squash_handler,
chain_head=self._block_store.chain_head,
identity_signing_key=self._identity_signing_key,
data_dir=self._data_dir,
config_dir=self._config_dir,
permission_verifier=self._permission_verifier,
batch_injector_factory=batch_injector_factory,
)
self._publisher_thread = self._PublisherThread(
block_publisher=self._block_publisher,
batch_queue=self._batch_queue,
check_publish_block_frequency=self._check_publish_block_frequency
)
self._chain_controller = ChainController(
block_sender=self._block_sender,
block_cache=self._block_cache,
state_view_factory=self._state_view_factory,
executor=self._executor_threadpool,
transaction_executor=self._transaction_executor,
chain_head_lock=self._block_publisher.chain_head_lock,
on_chain_updated=self._block_publisher.on_chain_updated,
squash_handler=self._squash_handler,
chain_id_manager=self._chain_id_manager,
identity_signing_key=self._identity_signing_key,
data_dir=self._data_dir,
config_dir=self._config_dir,
permission_verifier=self._permission_verifier,
chain_observers=self._chain_observers,
)
self._chain_thread = self._ChainThread(
chain_controller=self._chain_controller,
block_queue=self._block_queue,
block_cache=self._block_cache)
def test_publish(self):
publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.blocks.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
squash_handler=None,
chain_head=self.blocks.chain_head,
identity_signing_key=self.blocks.identity_signing_key)
# initial load of existing state
publisher.on_chain_updated(self.blocks.chain_head, [], [])
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
# current dev_mode consensus always claims blocks when asked.
# this will be called on a polling every so often or possibly triggered
# by events in the consensus it's self ... TBD
publisher.on_check_publish_block()
def test_max_block_size(self, mock_batch_injector_factory):
'''
Test block publisher obeys the block size limits
'''
mock_batch_injector_factory.create_injectors.return_value = []
# Create a publisher that has a state view
# with a batch limit
addr, value = CreateSetting(
'sawtooth.publisher.max_batches_per_block', 1)
self.state_view_factory = MockStateViewFactory(
{addr: value})
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
# try to publish with no pending queue (failing)
for i in range(self.batch_count):
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block([self.batches[i]])
def _init_subprocesses(self):
self._block_publisher = BlockPublisher(
consensus=self._consensus.BlockPublisher(),
transaction_executor=self._transaction_executor,
block_sender=self._block_sender,
squash_handler=self._squash_handler,
chain_head=self._block_store.chain_head)
self._publisher_thread = self._PublisherThread(self._block_publisher,
self._batch_queue)
self._chain_controller = ChainController(
consensus=self._consensus.BlockVerifier(),
block_sender=self._block_sender,
block_cache=self._block_cache,
executor=ThreadPoolExecutor(1),
transaction_executor=self._transaction_executor,
on_chain_updated=self._block_publisher.on_chain_updated,
squash_handler=self._squash_handler)
self._chain_thread = self._ChainThread(self._chain_controller,
self._block_queue,
self._block_cache)
def __init__(self, with_genesis=True):
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.block_store = BlockStore(DictDatabase())
self.block_cache = BlockCache(self.block_store)
self.state_db = {}
# add the mock reference to the consensus
consensus_setting_addr = SettingsView.setting_address(
'sawtooth.consensus.algorithm')
self.state_db[consensus_setting_addr] = _setting_entry(
'sawtooth.consensus.algorithm', 'test_journal.mock_consensus')
self.state_view_factory = MockStateViewFactory(self.state_db)
self.signing_key = signing.generate_private_key()
self.public_key = signing.generate_public_key(self.signing_key)
self.identity_signing_key = signing.generate_private_key()
chain_head = None
if with_genesis:
self.genesis_block = self.generate_genesis_block()
self.set_chain_head(self.genesis_block)
chain_head = self.genesis_block
self.block_publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.block_sender,
squash_handler=None,
chain_head=chain_head,
identity_signing_key=self.identity_signing_key,
data_dir=None,
config_dir=None,
permission_verifier=MockPermissionVerifier(),
check_publish_block_frequency=0.1,
batch_observers=[])
def test_batches_rejected_by_scheduler(self):
'''
Test that no block is published with
batches rejected by the scheduler
'''
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(
batch_execution_result=False),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.receive_batches()
self.publish_block()
self.assert_no_block_published()
def setUp(self, mock_batch_injector_factory):
mock_batch_injector_factory.create_injectors.return_value = []
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.permission_verifier = MockPermissionVerifier()
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
class TestBlockPublisher(unittest.TestCase):
'''
The block publisher has three main functions, and in these tests
those functions are given the following wrappers for convenience:
* on_batch_received -> receive_batches
* on_chain_updated -> update_chain_head
* on_check_publish_block -> publish_block
After publishing a block, publish_block sends its block to the
mock block sender, and that block is named result_block. This block
is what is checked by the test assertions.
The basic pattern for the publisher tests (with variations) is:
0) make a list of batches (usually in setUp);
1) receive the batches;
2) publish a block;
3) verify the block (checking that it contains the correct batches,
or checking that it doesn't exist, or whatever).
The publisher chain head might be updated several times in a test.
'''
def setUp(self):
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def test_publish(self):
'''
Publish a block with several batches
'''
self.receive_batches()
self.publish_block()
self.verify_block()
def test_reject_duplicate_batches_from_receive(self):
'''
Test that duplicate batches from on_batch_received are rejected
'''
for _ in range(5):
self.receive_batches()
self.publish_block()
self.verify_block()
def test_reject_duplicate_batches_from_store(self):
'''
Test that duplicate batches from block store are rejected
'''
self.update_chain_head(None)
self.update_chain_head(
head=self.init_chain_head,
uncommitted=self.batches)
self.receive_batches()
self.publish_block()
self.verify_block()
def test_no_chain_head(self):
'''
Test that nothing gets published with a null chain head,
then test that publishing resumes after updating
'''
self.update_chain_head(None)
self.receive_batches()
# try to publish block (failing)
self.publish_block()
self.assert_no_block_published()
# reset chain head several times,
# making sure batches remain queued
for _ in range(3):
self.update_chain_head(None)
self.update_chain_head(self.init_chain_head)
# try to publish block (succeeding)
self.publish_block()
self.verify_block()
def test_committed_batches(self):
'''
Test that batches committed upon updating the chain head
are not included in the next block.
'''
self.update_chain_head(None)
self.update_chain_head(
head=self.init_chain_head,
committed=self.batches)
new_batches = self.make_batches(batch_count=12)
self.receive_batches(new_batches)
self.publish_block()
self.verify_block(new_batches)
def test_uncommitted_batches(self):
'''
Test that batches uncommitted upon updating the chain head
are included in the next block.
'''
self.update_chain_head(None)
self.update_chain_head(
head=self.init_chain_head,
uncommitted=self.batches)
self.publish_block()
self.verify_block()
def test_empty_pending_queue(self):
'''
Test that no block is published if the pending queue is empty
'''
# try to publish with no pending queue (failing)
self.publish_block()
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
self.publish_block()
self.verify_block()
def test_missing_dependencies(self):
'''
Test that no block is published with missing dependencies
'''
self.batches = self.make_batches(
missing_deps=True)
self.receive_batches()
self.publish_block()
self.assert_no_block_published()
def test_batches_rejected_by_scheduler(self):
'''
Test that no block is published with
batches rejected by the scheduler
'''
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(
batch_execution_result=False),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.receive_batches()
self.publish_block()
self.assert_no_block_published()
def test_max_block_size(self):
'''
Test block publisher obeys the block size limits
'''
# Create a publisher that has a state view
# with a batch limit
addr, value = CreateSetting(
'sawtooth.publisher.max_batches_per_block', 1)
print('test', addr)
self.state_view_factory = MockStateViewFactory(
{addr: value})
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
# try to publish with no pending queue (failing)
for i in range(self.batch_count):
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block([self.batches[i]])
def test_duplicate_transactions(self):
'''
Test discards batches that have duplicate transactions in them.
'''
# receive batches, then try again (succeeding)
self.batches = self.batches[1:2]
self.receive_batches()
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block()
# build a new set of batches with the same transactions in them
self.batches = self.make_batches_with_duplicate_txn()
self.receive_batches()
self.publish_block()
self.assert_no_block_published() # block should be empty after batch
# with duplicate transaction is dropped.
# assertions
def assert_block_published(self):
self.assertIsNotNone(
self.result_block,
'Block should have been published')
def assert_no_block_published(self):
self.assertIsNone(
self.result_block,
'Block should not have been published')
def assert_batch_in_block(self, batch):
self.assertIn(
batch,
tuple(self.result_block.batches),
'Batch not in block')
def assert_batches_in_block(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.assert_batch_in_block(batch)
def assert_block_batch_count(self, batch_count=None):
if batch_count is None:
batch_count = self.batch_count
self.assertEqual(
len(self.result_block.batches),
batch_count,
'Wrong batch count in block')
def verify_block(self, batches=None):
if batches is None:
batches = self.batches
batch_count = None if batches is None else len(batches)
self.assert_block_published()
self.assert_batches_in_block(batches)
self.assert_block_batch_count(batch_count)
self.result_block = None
# publisher functions
def receive_batch(self, batch):
self.publisher.on_batch_received(batch)
def receive_batches(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.receive_batch(batch)
def publish_block(self):
self.publisher.on_check_publish_block()
self.result_block = self.block_sender.new_block
self.block_sender.new_block = None
def update_chain_head(self, head, committed=None, uncommitted=None):
if head:
self.block_tree_manager.block_store.update_chain([head])
self.publisher.on_chain_updated(
chain_head=head,
committed_batches=committed,
uncommitted_batches=uncommitted)
# batches
def make_batch(self, missing_deps=False):
return self.block_tree_manager.generate_batch(
missing_deps=missing_deps)
def make_batches(self, batch_count=None, missing_deps=False):
if batch_count is None:
batch_count = self.batch_count
return [self.make_batch(missing_deps=missing_deps)
for _ in range(batch_count)]
def make_batches_with_duplicate_txn(self):
txns = [self.batches[0].transactions[0],
self.block_tree_manager.generate_transaction("nonce")]
return [self.block_tree_manager.generate_batch(txns=txns)]
def do_publish_block():
btm = BlockTreeManager()
block_publisher = None
chain_controller = None
try:
block_publisher = BlockPublisher(
transaction_executor=testJournal.txn_executor,
block_cache=btm.block_cache,
state_view_factory=MockStateViewFactory(btm.state_db),
settings_cache=SettingsCache(
SettingsViewFactory(btm.state_view_factory), ),
block_sender=testJournal.block_sender,
batch_sender=testJournal.batch_sender,
squash_handler=None,
chain_head=btm.block_store.chain_head,
identity_signer=btm.identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=testJournal.permission_verifier,
check_publish_block_frequency=0.1,
batch_observers=[],
batch_injector_factory=DefaultBatchInjectorFactory(
block_store=btm.block_store,
state_view_factory=MockStateViewFactory(btm.state_db),
signer=btm.identity_signer))
block_validator = BlockValidator(
state_view_factory=MockStateViewFactory(btm.state_db),
block_cache=btm.block_cache,
transaction_executor=testJournal.txn_executor,
squash_handler=None,
identity_signer=btm.identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=testJournal.permission_verifier)
chain_controller = ChainController(
block_cache=btm.block_cache,
block_validator=block_validator,
state_view_factory=MockStateViewFactory(btm.state_db),
chain_head_lock=block_publisher.chain_head_lock,
on_chain_updated=block_publisher.on_chain_updated,
chain_id_manager=None,
data_dir=None,
config_dir=None,
chain_observers=[])
testJournal.gossip.on_batch_received = block_publisher.queue_batch
testJournal.gossip.on_block_received = chain_controller.queue_block
block_publisher.start()
chain_controller.start()
# feed it a batch
batch = Batch()
block_publisher.queue_batch(batch)
wait_until(lambda: testJournal.block_sender.new_block is not None,
2)
assert testJournal.block_sender.new_block != None
block = BlockWrapper(testJournal.block_sender.new_block)
chain_controller.queue_block(block)
# wait for the chain_head to be updated.
wait_until(lambda: btm.chain_head.identifier == block.identifier,
2)
# assert btm.chain_head.identifier == block.identifier
finally:
if block_publisher is not None:
block_publisher.stop()
if chain_controller is not None:
chain_controller.stop()
if block_validator is not None:
block_validator.stop()
class TestBlockPublisher(unittest.TestCase):
'''
The block publisher has five main functions, and in these tests
those functions are given the following wrappers for convenience:
* on_batch_received -> receive_batches
* on_chain_updated -> update_chain_head
* initialize_block -> initialize_block
* summarize_block -> summarize_block
* finalize_block -> finalize_block
Additionally, the publish_block is provided to call both initialize_block
and finalize_block.
After finalizing a block, finalize_block sends its block
to the mock block sender, and that block is named result_block. This block
is what is checked by the test assertions.
The basic pattern for the publisher tests (with variations) is:
0) make a list of batches (usually in setUp);
1) receive the batches;
2) initialize a block;
3) finalize a block;
4) verify the block (checking that it contains the correct batches,
or checking that it doesn't exist, etc.).
'''
@unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory')
def setUp(self, mock_batch_injector_factory):
mock_batch_injector_factory.create_injectors.return_value = []
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.permission_verifier = MockPermissionVerifier()
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def test_publish(self):
'''
Publish a block with several batches
'''
self.receive_batches()
self.publish_block()
self.verify_block()
def test_receive_after_initialize(self):
'''
Receive batches after initialization
'''
self.initialize_block()
self.receive_batches()
self.finalize_block()
self.verify_block()
def test_summarize_block(self):
'''
Initialize a block and summarize it
'''
self.receive_batches()
self.initialize_block()
self.assertIsNotNone(self.summarize_block(),
'Expected block summary')
def test_reject_double_initialization(self):
'''
Test that you can't initialize a candidate block twice
'''
self.initialize_block()
with self.assertRaises(
BlockInProgress,
msg='Second initialization should have rejected'):
self.initialize_block()
def test_reject_finalize_without_initialize(self):
'''
Test that no block is published if the block hasn't been initialized
'''
self.receive_batches()
with self.assertRaises(
BlockNotInitialized,
msg='Block should not be finalized'):
self.finalize_block()
def test_reject_duplicate_batches_from_receive(self):
'''
Test that duplicate batches from on_batch_received are rejected
'''
for _ in range(5):
self.receive_batches()
self.publish_block()
self.verify_block()
def test_reject_duplicate_batches_from_store(self):
'''
Test that duplicate batches from block store are rejected
'''
self.update_chain_head(
head=self.init_chain_head,
uncommitted=self.batches)
self.receive_batches()
self.publish_block()
self.verify_block()
def test_committed_batches(self):
'''
Test that batches committed upon updating the chain head
are not included in the next block.
'''
self.update_chain_head(
head=self.init_chain_head,
committed=self.batches)
new_batches = self.make_batches(batch_count=12)
self.receive_batches(new_batches)
self.publish_block()
self.verify_block(new_batches)
def test_uncommitted_batches(self):
'''
Test that batches uncommitted upon updating the chain head
are included in the next block.
'''
self.update_chain_head(
head=self.init_chain_head,
uncommitted=self.batches)
self.publish_block()
self.verify_block()
def test_empty_pending_queue(self):
'''
Test that no block is published if the pending queue is empty
'''
# try to publish with no pending queue (failing)
with self.assertRaises(
BlockEmpty, msg='Block should not be published'):
self.publish_block()
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
self.finalize_block()
self.verify_block()
def test_missing_dependencies(self):
'''
Test that no block is published with missing dependencies
'''
self.batches = self.make_batches(
missing_deps=True)
self.receive_batches()
# Block should be empty, since batches with missing deps aren't added
with self.assertRaises(BlockEmpty, msg='Block should be empty'):
self.publish_block()
self.assert_no_block_published()
@unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory')
def test_batches_rejected_by_scheduler(self, mock_batch_injector_factory):
'''
Test that no block is published with
batches rejected by the scheduler
'''
mock_batch_injector_factory.create_injectors.return_value = []
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(
batch_execution_result=False),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.receive_batches()
# Block should be empty since all batches are rejected
with self.assertRaises(BlockEmpty, msg='Block should be empty'):
self.publish_block()
self.assert_no_block_published()
@unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory')
def test_max_block_size(self, mock_batch_injector_factory):
'''
Test block publisher obeys the block size limits
'''
mock_batch_injector_factory.create_injectors.return_value = []
# Create a publisher that has a state view
# with a batch limit
addr, value = CreateSetting(
'sawtooth.publisher.max_batches_per_block', 1)
self.state_view_factory = MockStateViewFactory(
{addr: value})
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
# try to publish with no pending queue (failing)
for i in range(self.batch_count):
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block([self.batches[i]])
def test_duplicate_transactions(self):
'''
Test discards batches that have duplicate transactions in them.
'''
# receive batches, then try again (succeeding)
self.batches = self.batches[1:2]
self.receive_batches()
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block()
# build a new set of batches with the same transactions in them
self.batches = self.make_batches_with_duplicate_txn()
self.receive_batches()
with self.assertRaises(BlockEmpty, msg='Block should be empty'):
self.publish_block()
self.assert_no_block_published() # block should be empty after batch
# with duplicate transaction is dropped.
def test_batch_injection_start_block(self):
'''
Test that the batch is injected at the beginning of the block.
'''
injected_batch = self.make_batch()
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=self.permission_verifier,
batch_observers=[],
batch_injector_factory=MockBatchInjectorFactory(injected_batch))
self.receive_batches()
self.publish_block()
self.assert_batch_in_block(injected_batch)
@unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory')
def test_validation_rules_reject_batches(self,
mock_batch_injector_factory):
"""Test that a batch is not added to the block if it will violate the
block validation rules.
It does the following:
- Sets the block_validation_rules to limit the number of 'test'
transactions to 1
- creates two batches, limited to 1 transaction each, and receives
them
- verifies that only the first batch was committed to the block
"""
addr, value = CreateSetting(
'sawtooth.validator.block_validation_rules', 'NofX:1,test')
self.state_view_factory = MockStateViewFactory(
{addr: value})
mock_batch_injector_factory.create_injectors.return_value = []
batch1 = self.make_batch(txn_count=1)
batch2 = self.make_batch(txn_count=1)
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction
),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.receive_batches(batches=[batch1, batch2])
self.publish_block()
self.assert_block_batch_count(1)
self.assert_batch_in_block(batch1)
# assertions
def assert_block_published(self):
self.assertIsNotNone(
self.result_block,
'Block should have been published')
def assert_no_block_published(self):
self.assertIsNone(
self.result_block,
'Block should not have been published')
def assert_batch_in_block(self, batch):
self.assertIn(
batch,
tuple(self.result_block.batches),
'Batch not in block')
def assert_batches_in_block(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.assert_batch_in_block(batch)
def assert_block_batch_count(self, batch_count=None):
if batch_count is None:
batch_count = self.batch_count
self.assertEqual(
len(self.result_block.batches),
batch_count,
'Wrong batch count in block')
def verify_block(self, batches=None):
if batches is None:
batches = self.batches
batch_count = None if batches is None else len(batches)
self.assert_block_published()
self.assert_batches_in_block(batches)
self.assert_block_batch_count(batch_count)
self.result_block = None
# publisher functions
def receive_batch(self, batch):
self.publisher.on_batch_received(batch)
def receive_batches(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.receive_batch(batch)
def initialize_block(self):
self.publisher.initialize_block(self.block_tree_manager.chain_head)
def summarize_block(self):
return self.publisher.summarize_block()
def finalize_block(self):
self.publisher.finalize_block("")
self.result_block = self.block_sender.new_block
self.block_sender.new_block = None
def publish_block(self):
self.initialize_block()
self.finalize_block()
def update_chain_head(self, head, committed=None, uncommitted=None):
if head:
self.block_tree_manager.block_store.update_chain([head])
self.publisher.on_chain_updated(
chain_head=head,
committed_batches=committed,
uncommitted_batches=uncommitted)
# batches
def make_batch(self, missing_deps=False, txn_count=2):
return self.block_tree_manager.generate_batch(
txn_count=txn_count,
missing_deps=missing_deps)
def make_batches(self, batch_count=None, missing_deps=False):
if batch_count is None:
batch_count = self.batch_count
return [self.make_batch(missing_deps=missing_deps)
for _ in range(batch_count)]
def make_batches_with_duplicate_txn(self):
txns = [self.batches[0].transactions[0],
self.block_tree_manager.generate_transaction("nonce")]
return [self.block_tree_manager.generate_batch(txns=txns)]
def test_publish(self):
LOGGER.info(self.blocks)
publisher = BlockPublisher(
consensus=TestModePublisher(),
transaction_executor=MockTransactionExecutor(),
block_sender=self.block_sender,
squash_handler=None,
chain_head=self.blocks.chain_head)
# initial load of existing state
publisher.on_chain_updated(self.blocks.chain_head, [], [])
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
# current dev_mode consensus always claims blocks when asked.
# this will be called on a polling every so often or possibly triggered
# by events in the consensus it's self ... TBD
publisher.on_check_publish_block()
LOGGER.info(self.blocks)
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
publisher.on_check_publish_block()
LOGGER.info(self.blocks)
def __init__(self,
bind_network,
bind_component,
bind_consensus,
endpoint,
peering,
seeds_list,
peer_list,
data_dir,
config_dir,
identity_signer,
scheduler_type,
permissions,
minimum_peer_connectivity,
maximum_peer_connectivity,
state_pruning_block_depth,
network_public_key=None,
network_private_key=None,
roles=None):
"""Constructs a validator instance.
Args:
bind_network (str): the network endpoint
bind_component (str): the component endpoint
endpoint (str): the zmq-style URI of this validator's
publically reachable endpoint
peering (str): The type of peering approach. Either 'static'
or 'dynamic'. In 'static' mode, no attempted topology
buildout occurs -- the validator only attempts to initiate
peering connections with endpoints specified in the
peer_list. In 'dynamic' mode, the validator will first
attempt to initiate peering connections with endpoints
specified in the peer_list and then attempt to do a
topology buildout starting with peer lists obtained from
endpoints in the seeds_list. In either mode, the validator
will accept incoming peer requests up to max_peers.
seeds_list (list of str): a list of addresses to connect
to in order to perform the initial topology buildout
peer_list (list of str): a list of peer addresses
data_dir (str): path to the data directory
config_dir (str): path to the config directory
identity_signer (str): cryptographic signer the validator uses for
signing
"""
# -- Setup Global State Database and Factory -- #
global_state_db_filename = os.path.join(
data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug(
'global state database file is %s', global_state_db_filename)
global_state_db = NativeLmdbDatabase(
global_state_db_filename,
indexes=MerkleDatabase.create_index_configuration())
state_view_factory = StateViewFactory(global_state_db)
# -- Setup Receipt Store -- #
receipt_db_filename = os.path.join(
data_dir, 'txn_receipts-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('txn receipt store file is %s', receipt_db_filename)
receipt_db = LMDBNoLockDatabase(receipt_db_filename, 'c')
receipt_store = TransactionReceiptStore(receipt_db)
# -- Setup Block Store -- #
block_db_filename = os.path.join(
data_dir, 'block-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('block store file is %s', block_db_filename)
block_db = IndexedDatabase(
block_db_filename,
BlockStore.serialize_block,
BlockStore.deserialize_block,
flag='c',
indexes=BlockStore.create_index_configuration())
block_store = BlockStore(block_db)
# The cache keep time for the journal's block cache must be greater
# than the cache keep time used by the completer.
base_keep_time = 1200
block_cache = BlockCache(
block_store,
keep_time=int(base_keep_time * 9 / 8),
purge_frequency=30)
# -- Setup Thread Pools -- #
component_thread_pool = InstrumentedThreadPoolExecutor(
max_workers=10,
name='Component')
network_thread_pool = InstrumentedThreadPoolExecutor(
max_workers=10,
name='Network')
client_thread_pool = InstrumentedThreadPoolExecutor(
max_workers=5,
name='Client')
sig_pool = InstrumentedThreadPoolExecutor(
max_workers=3,
name='Signature')
# -- Setup Dispatchers -- #
component_dispatcher = Dispatcher()
network_dispatcher = Dispatcher()
# -- Setup Services -- #
component_service = Interconnect(
bind_component,
component_dispatcher,
secured=False,
heartbeat=False,
max_incoming_connections=20,
monitor=True,
max_future_callback_workers=10)
zmq_identity = hashlib.sha512(
time.time().hex().encode()).hexdigest()[:23]
secure = False
if network_public_key is not None and network_private_key is not None:
secure = True
network_service = Interconnect(
bind_network,
dispatcher=network_dispatcher,
zmq_identity=zmq_identity,
secured=secure,
server_public_key=network_public_key,
server_private_key=network_private_key,
heartbeat=True,
public_endpoint=endpoint,
connection_timeout=120,
max_incoming_connections=100,
max_future_callback_workers=10,
authorize=True,
signer=identity_signer,
roles=roles)
# -- Setup Transaction Execution Platform -- #
context_manager = ContextManager(global_state_db)
batch_tracker = BatchTracker(block_store)
settings_cache = SettingsCache(
SettingsViewFactory(state_view_factory),
)
transaction_executor = TransactionExecutor(
service=component_service,
context_manager=context_manager,
settings_view_factory=SettingsViewFactory(state_view_factory),
scheduler_type=scheduler_type,
invalid_observers=[batch_tracker])
component_service.set_check_connections(
transaction_executor.check_connections)
event_broadcaster = EventBroadcaster(
component_service, block_store, receipt_store)
# -- Setup P2P Networking -- #
gossip = Gossip(
network_service,
settings_cache,
lambda: block_store.chain_head,
block_store.chain_head_state_root,
endpoint=endpoint,
peering_mode=peering,
initial_seed_endpoints=seeds_list,
initial_peer_endpoints=peer_list,
minimum_peer_connectivity=minimum_peer_connectivity,
maximum_peer_connectivity=maximum_peer_connectivity,
topology_check_frequency=1
)
completer = Completer(
block_store,
gossip,
cache_keep_time=base_keep_time,
cache_purge_frequency=30,
requested_keep_time=300)
block_sender = BroadcastBlockSender(completer, gossip)
batch_sender = BroadcastBatchSender(completer, gossip)
chain_id_manager = ChainIdManager(data_dir)
identity_view_factory = IdentityViewFactory(
StateViewFactory(global_state_db))
id_cache = IdentityCache(identity_view_factory)
# -- Setup Permissioning -- #
permission_verifier = PermissionVerifier(
permissions,
block_store.chain_head_state_root,
id_cache)
identity_observer = IdentityObserver(
to_update=id_cache.invalidate,
forked=id_cache.forked)
settings_observer = SettingsObserver(
to_update=settings_cache.invalidate,
forked=settings_cache.forked)
# -- Consensus Engine -- #
consensus_thread_pool = InstrumentedThreadPoolExecutor(
max_workers=3,
name='Consensus')
consensus_dispatcher = Dispatcher()
consensus_service = Interconnect(
bind_consensus,
consensus_dispatcher,
secured=False,
heartbeat=False,
max_incoming_connections=20,
monitor=True,
max_future_callback_workers=10)
consensus_notifier = ConsensusNotifier(consensus_service)
# -- Setup Journal -- #
batch_injector_factory = DefaultBatchInjectorFactory(
block_cache=block_cache,
state_view_factory=state_view_factory,
signer=identity_signer)
block_publisher = BlockPublisher(
transaction_executor=transaction_executor,
block_cache=block_cache,
state_view_factory=state_view_factory,
settings_cache=settings_cache,
block_sender=block_sender,
batch_sender=batch_sender,
chain_head=block_store.chain_head,
identity_signer=identity_signer,
data_dir=data_dir,
config_dir=config_dir,
permission_verifier=permission_verifier,
check_publish_block_frequency=0.1,
batch_observers=[batch_tracker],
batch_injector_factory=batch_injector_factory)
block_publisher_batch_sender = block_publisher.batch_sender()
block_validator = BlockValidator(
block_cache=block_cache,
state_view_factory=state_view_factory,
transaction_executor=transaction_executor,
identity_signer=identity_signer,
data_dir=data_dir,
config_dir=config_dir,
permission_verifier=permission_verifier)
chain_controller = ChainController(
block_store=block_store,
block_cache=block_cache,
block_validator=block_validator,
state_database=global_state_db,
chain_head_lock=block_publisher.chain_head_lock,
state_pruning_block_depth=state_pruning_block_depth,
data_dir=data_dir,
observers=[
event_broadcaster,
receipt_store,
batch_tracker,
identity_observer,
settings_observer
])
genesis_controller = GenesisController(
context_manager=context_manager,
transaction_executor=transaction_executor,
completer=completer,
block_store=block_store,
state_view_factory=state_view_factory,
identity_signer=identity_signer,
data_dir=data_dir,
config_dir=config_dir,
chain_id_manager=chain_id_manager,
batch_sender=batch_sender)
responder = Responder(completer)
completer.set_on_batch_received(block_publisher_batch_sender.send)
completer.set_on_block_received(chain_controller.queue_block)
completer.set_chain_has_block(chain_controller.has_block)
# -- Register Message Handler -- #
network_handlers.add(
network_dispatcher, network_service, gossip, completer,
responder, network_thread_pool, sig_pool,
chain_controller.has_block, block_publisher.has_batch,
permission_verifier, block_publisher, consensus_notifier)
component_handlers.add(
component_dispatcher, gossip, context_manager,
transaction_executor, completer, block_store, batch_tracker,
global_state_db, self.get_chain_head_state_root_hash,
receipt_store, event_broadcaster, permission_verifier,
component_thread_pool, client_thread_pool,
sig_pool, block_publisher)
# -- Store Object References -- #
self._component_dispatcher = component_dispatcher
self._component_service = component_service
self._component_thread_pool = component_thread_pool
self._network_dispatcher = network_dispatcher
self._network_service = network_service
self._network_thread_pool = network_thread_pool
consensus_proxy = ConsensusProxy(
block_cache=block_cache,
chain_controller=chain_controller,
block_publisher=block_publisher,
gossip=gossip,
identity_signer=identity_signer,
settings_view_factory=SettingsViewFactory(state_view_factory),
state_view_factory=state_view_factory)
consensus_handlers.add(
consensus_dispatcher, consensus_thread_pool, consensus_proxy)
self._consensus_dispatcher = consensus_dispatcher
self._consensus_service = consensus_service
self._consensus_thread_pool = consensus_thread_pool
self._client_thread_pool = client_thread_pool
self._sig_pool = sig_pool
self._context_manager = context_manager
self._transaction_executor = transaction_executor
self._genesis_controller = genesis_controller
self._gossip = gossip
self._block_publisher = block_publisher
self._chain_controller = chain_controller
self._block_validator = block_validator
def test_publish(self):
LOGGER.info(self.blocks)
publisher = BlockPublisher(
consensus=TestModePublisher(),
transaction_executor=MockTransactionExecutor(),
block_sender=self.block_sender,
squash_handler=None,
chain_head=self.blocks.chain_head)
# initial load of existing state
publisher.on_chain_updated(self.blocks.chain_head, [], [])
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
# current dev_mode consensus always claims blocks when asked.
# this will be called on a polling every so often or possibly triggered
# by events in the consensus it's self ... TBD
publisher.on_check_publish_block()
LOGGER.info(self.blocks)
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
publisher.on_check_publish_block()
LOGGER.info(self.blocks)
class TestBlockPublisher():
'''
The block publisher has three main functions, and in these tests
those functions are given the following wrappers for convenience:
* on_batch_received -> receive_batches
* on_chain_updated -> update_chain_head
* on_check_publish_block -> publish_block
After publishing a block, publish_block sends its block to the
mock block sender, and that block is named result_block. This block
is what is checked by the test assertions.
The basic pattern for the publisher tests (with variations) is:
0) make a list of batches (usually in setUp);
1) receive the batches;
2) publish a block;
3) verify the block (checking that it contains the correct batches,
or checking that it doesn't exist, or whatever).
The publisher chain head might be updated several times in a test.
'''
def __init__(self):
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.permission_verifier = MockPermissionVerifier()
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
settings_cache=SettingsCache(
SettingsViewFactory(
self.block_tree_manager.state_view_factory), ),
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
check_publish_block_frequency=0.1,
batch_observers=[],
permission_verifier=self.permission_verifier)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def verify_block(self, batches=None):
if batches is None:
batches = self.batches
batch_count = None if batches is None else len(batches)
# publisher functions
def receive_batch(self, batch):
self.publisher.on_batch_received(batch)
def receive_batches(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.receive_batch(batch)
def publish_block(self):
self.publisher.on_check_publish_block()
self.result_block = self.block_sender.new_block
self.block_sender.new_block = None
def update_chain_head(self, head, committed=None, uncommitted=None):
if head:
self.block_tree_manager.block_store.update_chain([head])
self.publisher.on_chain_updated(chain_head=head,
committed_batches=committed,
uncommitted_batches=uncommitted)
# batches
def make_batch(self, missing_deps=False, txn_count=2):
return self.block_tree_manager.generate_batch(
txn_count=txn_count, missing_deps=missing_deps)
def make_batches(self, batch_count=None, missing_deps=False):
if batch_count is None:
batch_count = self.batch_count
return [
self.make_batch(missing_deps=missing_deps)
for _ in range(batch_count)
]
def make_batches_with_duplicate_txn(self):
txns = [
self.batches[0].transactions[0],
self.block_tree_manager.generate_transaction("nonce")
]
return [self.block_tree_manager.generate_batch(txns=txns)]
class TestBlockPublisher(unittest.TestCase):
'''
The block publisher has five main functions, and in these tests
those functions are given the following wrappers for convenience:
* on_batch_received -> receive_batches
* on_chain_updated -> update_chain_head
* initialize_block -> initialize_block
* summarize_block -> summarize_block
* finalize_block -> finalize_block
Additionally, the publish_block is provided to call both initialize_block
and finalize_block.
After finalizing a block, finalize_block sends its block
to the mock block sender, and that block is named result_block. This block
is what is checked by the test assertions.
The basic pattern for the publisher tests (with variations) is:
0) make a list of batches (usually in setUp);
1) receive the batches;
2) initialize a block;
3) finalize a block;
4) verify the block (checking that it contains the correct batches,
or checking that it doesn't exist, etc.).
'''
@unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory')
def setUp(self, mock_batch_injector_factory):
mock_batch_injector_factory.create_injectors.return_value = []
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.permission_verifier = MockPermissionVerifier()
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def test_publish(self):
'''
Publish a block with several batches
'''
self.receive_batches()
self.publish_block()
self.verify_block()
def test_receive_after_initialize(self):
'''
Receive batches after initialization
'''
self.initialize_block()
self.receive_batches()
self.finalize_block()
self.verify_block()
def test_summarize_block(self):
'''
Initialize a block and summarize it
'''
self.receive_batches()
self.initialize_block()
self.assertIsNotNone(self.summarize_block(), 'Expected block summary')
def test_reject_double_initialization(self):
'''
Test that you can't initialize a candidate block twice
'''
self.initialize_block()
with self.assertRaises(
BlockInProgress,
msg='Second initialization should have rejected'):
self.initialize_block()
def test_reject_finalize_without_initialize(self):
'''
Test that no block is published if the block hasn't been initialized
'''
self.receive_batches()
with self.assertRaises(BlockNotInitialized,
msg='Block should not be finalized'):
self.finalize_block()
def test_reject_duplicate_batches_from_receive(self):
'''
Test that duplicate batches from on_batch_received are rejected
'''
for _ in range(5):
self.receive_batches()
self.publish_block()
self.verify_block()
def test_reject_duplicate_batches_from_store(self):
'''
Test that duplicate batches from block store are rejected
'''
self.update_chain_head(head=self.init_chain_head,
uncommitted=self.batches)
self.receive_batches()
self.publish_block()
self.verify_block()
def test_committed_batches(self):
'''
Test that batches committed upon updating the chain head
are not included in the next block.
'''
self.update_chain_head(head=self.init_chain_head,
committed=self.batches)
new_batches = self.make_batches(batch_count=12)
self.receive_batches(new_batches)
self.publish_block()
self.verify_block(new_batches)
def test_uncommitted_batches(self):
'''
Test that batches uncommitted upon updating the chain head
are included in the next block.
'''
self.update_chain_head(head=self.init_chain_head,
uncommitted=self.batches)
self.publish_block()
self.verify_block()
def test_empty_pending_queue(self):
'''
Test that no block is published if the pending queue is empty
'''
# try to publish with no pending queue (failing)
with self.assertRaises(BlockEmpty,
msg='Block should not be published'):
self.publish_block()
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
self.finalize_block()
self.verify_block()
def test_missing_dependencies(self):
'''
Test that no block is published with missing dependencies
'''
self.batches = self.make_batches(missing_deps=True)
self.receive_batches()
# Block should be empty, since batches with missing deps aren't added
with self.assertRaises(BlockEmpty, msg='Block should be empty'):
self.publish_block()
self.assert_no_block_published()
@unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory')
def test_batches_rejected_by_scheduler(self, mock_batch_injector_factory):
'''
Test that no block is published with
batches rejected by the scheduler
'''
mock_batch_injector_factory.create_injectors.return_value = []
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(
batch_execution_result=False),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.receive_batches()
# Block should be empty since all batches are rejected
with self.assertRaises(BlockEmpty, msg='Block should be empty'):
self.publish_block()
self.assert_no_block_published()
@unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory')
def test_max_block_size(self, mock_batch_injector_factory):
'''
Test block publisher obeys the block size limits
'''
mock_batch_injector_factory.create_injectors.return_value = []
# Create a publisher that has a state view
# with a batch limit
addr, value = CreateSetting('sawtooth.publisher.max_batches_per_block',
1)
self.state_view_factory = MockStateViewFactory({addr: value})
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
# try to publish with no pending queue (failing)
for i in range(self.batch_count):
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block([self.batches[i]])
def test_duplicate_transactions(self):
'''
Test discards batches that have duplicate transactions in them.
'''
# receive batches, then try again (succeeding)
self.batches = self.batches[1:2]
self.receive_batches()
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block()
# build a new set of batches with the same transactions in them
self.batches = self.make_batches_with_duplicate_txn()
self.receive_batches()
with self.assertRaises(BlockEmpty, msg='Block should be empty'):
self.publish_block()
self.assert_no_block_published() # block should be empty after batch
# with duplicate transaction is dropped.
def test_batch_injection_start_block(self):
'''
Test that the batch is injected at the beginning of the block.
'''
injected_batch = self.make_batch()
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
permission_verifier=self.permission_verifier,
batch_observers=[],
batch_injector_factory=MockBatchInjectorFactory(injected_batch))
self.receive_batches()
self.publish_block()
self.assert_batch_in_block(injected_batch)
@unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory')
def test_validation_rules_reject_batches(self,
mock_batch_injector_factory):
"""Test that a batch is not added to the block if it will violate the
block validation rules.
It does the following:
- Sets the block_validation_rules to limit the number of 'test'
transactions to 1
- creates two batches, limited to 1 transaction each, and receives
them
- verifies that only the first batch was committed to the block
"""
addr, value = CreateSetting(
'sawtooth.validator.block_validation_rules', 'NofX:1,test')
self.state_view_factory = MockStateViewFactory({addr: value})
mock_batch_injector_factory.create_injectors.return_value = []
batch1 = self.make_batch(txn_count=1)
batch2 = self.make_batch(txn_count=1)
self.publisher = BlockPublisher(
block_manager=self.block_tree_manager.block_manager,
transaction_executor=MockTransactionExecutor(),
transaction_committed=(
self.block_tree_manager.block_store.has_transaction),
batch_committed=self.block_tree_manager.block_store.has_batch,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
chain_head=self.block_tree_manager.chain_head.block,
identity_signer=self.block_tree_manager.identity_signer,
data_dir=None,
config_dir=None,
batch_observers=[],
permission_verifier=self.permission_verifier,
batch_injector_factory=mock_batch_injector_factory)
self.receive_batches(batches=[batch1, batch2])
self.publish_block()
self.assert_block_batch_count(1)
self.assert_batch_in_block(batch1)
# assertions
def assert_block_published(self):
self.assertIsNotNone(self.result_block,
'Block should have been published')
def assert_no_block_published(self):
self.assertIsNone(self.result_block,
'Block should not have been published')
def assert_batch_in_block(self, batch):
self.assertIn(batch, tuple(self.result_block.batches),
'Batch not in block')
def assert_batches_in_block(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.assert_batch_in_block(batch)
def assert_block_batch_count(self, batch_count=None):
if batch_count is None:
batch_count = self.batch_count
self.assertEqual(len(self.result_block.batches), batch_count,
'Wrong batch count in block')
def verify_block(self, batches=None):
if batches is None:
batches = self.batches
batch_count = None if batches is None else len(batches)
self.assert_block_published()
self.assert_batches_in_block(batches)
self.assert_block_batch_count(batch_count)
self.result_block = None
# publisher functions
def receive_batch(self, batch):
self.publisher.on_batch_received(batch)
def receive_batches(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.receive_batch(batch)
def initialize_block(self):
self.publisher.initialize_block(self.block_tree_manager.chain_head)
def summarize_block(self):
return self.publisher.summarize_block()
def finalize_block(self):
self.publisher.finalize_block("")
self.result_block = self.block_sender.new_block
self.block_sender.new_block = None
def publish_block(self):
self.initialize_block()
self.finalize_block()
def update_chain_head(self, head, committed=None, uncommitted=None):
if head:
self.block_tree_manager.block_store.update_chain([head])
self.publisher.on_chain_updated(chain_head=head,
committed_batches=committed,
uncommitted_batches=uncommitted)
# batches
def make_batch(self, missing_deps=False, txn_count=2):
return self.block_tree_manager.generate_batch(
txn_count=txn_count, missing_deps=missing_deps)
def make_batches(self, batch_count=None, missing_deps=False):
if batch_count is None:
batch_count = self.batch_count
return [
self.make_batch(missing_deps=missing_deps)
for _ in range(batch_count)
]
def make_batches_with_duplicate_txn(self):
txns = [
self.batches[0].transactions[0],
self.block_tree_manager.generate_transaction("nonce")
]
return [self.block_tree_manager.generate_batch(txns=txns)]
def test_publish(self, args=sys.argv[1:]):
gossip = MockNetwork()
LOGGER.info(self.blocks)
publisher = BlockPublisher(
consensus=TestModePublisher(),
transaction_executor=MockTransactionExecutor(),
send_message=gossip.send_message,
squash_handler=None)
LOGGER.info("1")
# initial load of existing state
publisher.on_chain_updated(self.blocks.chain_head.block, [], [])
LOGGER.info("2")
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
LOGGER.info("3")
# current dev_mode consensus always claims blocks when asked.
# this will be called on a polling every so often or possibly triggered
# by events in the consensus it's self ... TBD
publisher.on_check_publish_block()
LOGGER.info("4")
LOGGER.info(self.blocks)
# repeat as necessary
batch = Batch()
publisher.on_batch_received(batch)
publisher.on_check_publish_block()
LOGGER.info(self.blocks)
class TestBlockPublisher(unittest.TestCase):
'''
The block publisher has three main functions, and in these tests
those functions are given the following wrappers for convenience:
* on_batch_received -> receive_batches
* on_chain_updated -> update_chain_head
* on_check_publish_block -> publish_block
After publishing a block, publish_block sends its block to the
mock block sender, and that block is named result_block. This block
is what is checked by the test assertions.
The basic pattern for the publisher tests (with variations) is:
0) make a list of batches (usually in setUp);
1) receive the batches;
2) publish a block;
3) verify the block (checking that it contains the correct batches,
or checking that it doesn't exist, or whatever).
The publisher chain head might be updated several times in a test.
'''
def setUp(self):
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def test_publish(self):
'''
Publish a block with several batches
'''
self.receive_batches()
self.publish_block()
self.verify_block()
def test_reject_duplicate_batches_from_receive(self):
'''
Test that duplicate batches from on_batch_received are rejected
'''
for _ in range(5):
self.receive_batches()
self.publish_block()
self.verify_block()
def test_reject_duplicate_batches_from_store(self):
'''
Test that duplicate batches from block store are rejected
'''
self.update_chain_head(None)
self.update_chain_head(
head=self.init_chain_head,
uncommitted=self.batches)
self.receive_batches()
self.publish_block()
self.verify_block()
def test_no_chain_head(self):
'''
Test that nothing gets published with a null chain head,
then test that publishing resumes after updating
'''
self.update_chain_head(None)
self.receive_batches()
# try to publish block (failing)
self.publish_block()
self.assert_no_block_published()
# reset chain head several times,
# making sure batches remain queued
for _ in range(3):
self.update_chain_head(None)
self.update_chain_head(self.init_chain_head)
# try to publish block (succeeding)
self.publish_block()
self.verify_block()
def test_committed_batches(self):
'''
Test that batches committed upon updating the chain head
are not included in the next block.
'''
self.update_chain_head(None)
self.update_chain_head(
head=self.init_chain_head,
committed=self.batches)
new_batches = self.make_batches(batch_count=12)
self.receive_batches(new_batches)
self.publish_block()
self.verify_block(new_batches)
def test_uncommitted_batches(self):
'''
Test that batches uncommitted upon updating the chain head
are included in the next block.
'''
self.update_chain_head(None)
self.update_chain_head(
head=self.init_chain_head,
uncommitted=self.batches)
self.publish_block()
self.verify_block()
def test_empty_pending_queue(self):
'''
Test that no block is published if the pending queue is empty
'''
# try to publish with no pending queue (failing)
self.publish_block()
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
self.publish_block()
self.verify_block()
def test_missing_dependencies(self):
'''
Test that no block is published with missing dependencies
'''
self.batches = self.make_batches(
missing_deps=True)
self.receive_batches()
self.publish_block()
self.assert_no_block_published()
def test_batches_rejected_by_scheduler(self):
'''
Test that no block is published with
batches rejected by the scheduler
'''
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(
batch_execution_result=False),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.receive_batches()
self.publish_block()
self.assert_no_block_published()
def test_max_block_size(self):
'''
Test block publisher obeys the block size limits
'''
# Create a publisher that has a state view
# with a batch limit
addr, value = CreateSetting(
'sawtooth.publisher.max_batches_per_block', 1)
print('test', addr)
self.state_view_factory = MockStateViewFactory(
{addr: value})
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None,
config_dir=None)
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
# try to publish with no pending queue (failing)
for i in range(self.batch_count):
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block([self.batches[i]])
def test_duplicate_transactions(self):
'''
Test discards batches that have duplicate transactions in them.
'''
# receive batches, then try again (succeeding)
self.batches = self.batches[1:2]
self.receive_batches()
self.publish_block()
self.assert_block_published()
self.update_chain_head(BlockWrapper(self.result_block))
self.verify_block()
# build a new set of batches with the same transactions in them
self.batches = self.make_batches_with_duplicate_txn()
self.receive_batches()
self.publish_block()
self.assert_no_block_published() # block should be empty after batch
# with duplicate transaction is dropped.
# assertions
def assert_block_published(self):
self.assertIsNotNone(
self.result_block,
'Block should have been published')
def assert_no_block_published(self):
self.assertIsNone(
self.result_block,
'Block should not have been published')
def assert_batch_in_block(self, batch):
self.assertIn(
batch,
tuple(self.result_block.batches),
'Batch not in block')
def assert_batches_in_block(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.assert_batch_in_block(batch)
def assert_block_batch_count(self, batch_count=None):
if batch_count is None:
batch_count = self.batch_count
self.assertEqual(
len(self.result_block.batches),
batch_count,
'Wrong batch count in block')
def verify_block(self, batches=None):
if batches is None:
batches = self.batches
batch_count = None if batches is None else len(batches)
self.assert_block_published()
self.assert_batches_in_block(batches)
self.assert_block_batch_count(batch_count)
self.result_block = None
# publisher functions
def receive_batch(self, batch):
self.publisher.on_batch_received(batch)
def receive_batches(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.receive_batch(batch)
def publish_block(self):
self.publisher.on_check_publish_block()
self.result_block = self.block_sender.new_block
self.block_sender.new_block = None
def update_chain_head(self, head, committed=None, uncommitted=None):
if head:
self.block_tree_manager.block_store.update_chain([head])
self.publisher.on_chain_updated(
chain_head=head,
committed_batches=committed,
uncommitted_batches=uncommitted)
# batches
def make_batch(self, missing_deps=False):
return self.block_tree_manager.generate_batch(
missing_deps=missing_deps)
def make_batches(self, batch_count=None, missing_deps=False):
if batch_count is None:
batch_count = self.batch_count
return [self.make_batch(missing_deps=missing_deps)
for _ in range(batch_count)]
def make_batches_with_duplicate_txn(self):
txns = [self.batches[0].transactions[0],
self.block_tree_manager.generate_transaction("nonce")]
return [self.block_tree_manager.generate_batch(txns=txns)]
class Journal(object):
"""
Manages the block chain, This responsibility boils down
1) to evaluating new blocks to determine if they should extend or replace
the current chain. Handled by the ChainController/
2) Claiming new blocks, handled by the BlockPublisher
This object provides the threading and event queue for the processors.
"""
class _ChainThread(Thread):
def __init__(self, chain_controller, block_queue):
Thread.__init__(self)
self._block_publisher = chain_controller
self._block_queue = block_queue
self._exit = False
def run(self):
while True:
try:
block = self._block_queue.get(timeout=0.1)
self._block_publisher.on_block_received(block)
except queue.Empty:
time.sleep(0.1)
if self._exit:
return
def stop(self):
self._exit = True
class _PublisherThread(Thread):
def __init__(self, block_publisher, batch_queue):
Thread.__init__(self)
self._block_publisher = block_publisher
self._batch_queue = batch_queue
self._exit = False
def run(self):
while True:
try:
batch = self._batch_queue.get(timeout=0.1)
self._block_publisher.on_batch_received(batch)
except queue.Empty:
time.sleep(0.1)
self._block_publisher.on_check_publish_block()
if self._exit:
return
def stop(self):
self._exit = True
def __init__(self, consensus, block_store, send_message,
transaction_executor, squash_handler, first_state_root):
self._consensus = consensus
self._block_store = block_store
self._send_message = send_message
self._squash_handler = squash_handler
self._block_publisher = BlockPublisher(
consensus=consensus.BlockPublisher(),
transaction_executor=transaction_executor,
send_message=send_message,
squash_handler=squash_handler)
self._batch_queue = queue.Queue()
self._publisher_thread = self._PublisherThread(self._block_publisher,
self._batch_queue)
# HACK until genesis tool is working
if "chain_head_id" not in self._block_store:
genesis_block = BlockState(
block_wrapper=self._block_publisher.generate_genesis_block(),
weight=0,
status=BlockStatus.Valid)
genesis_block.block.set_state_hash(first_state_root)
self._block_store[genesis_block.block.header_signature] = \
genesis_block
self._block_store["chain_head_id"] = \
genesis_block.block.header_signature
self._block_publisher.on_chain_updated(genesis_block.block)
LOGGER.info("Journal created genesis block: %s",
genesis_block.block.header_signature)
self._chain_controller = ChainController(
consensus=consensus.BlockVerifier(),
block_store=block_store,
send_message=send_message,
executor=ThreadPoolExecutor(1),
transaction_executor=transaction_executor,
on_chain_updated=self._block_publisher.on_chain_updated,
squash_handler=self._squash_handler)
self._block_queue = queue.Queue()
self._chain_thread = self._ChainThread(self._chain_controller,
self._block_queue)
def get_current_root(self):
# return self._block_publisher._chain_head.state_root_hash
return self._chain_controller.chain_head.block.state_root_hash
def start(self):
# TBD do load activities....
# TBD transfer activities - request chain-head from
# network
self._publisher_thread.start()
self._chain_thread.start()
def stop(self):
# time to murder the child threads. First ask politely for
# suicide
self._publisher_thread.stop()
self._chain_thread.stop()
def on_block_received(self, block):
self._block_queue.put(block)
def on_batch_received(self, batch):
self._batch_queue.put(batch)
def on_block_request(self, block_id):
if block_id in self._block_store:
self._send_message(self._block_store[block_id].block)
def __init__(self,
bind_network,
bind_component,
bind_consensus,
endpoint,
peering,
seeds_list,
peer_list,
data_dir,
config_dir,
identity_signer,
scheduler_type,
permissions,
minimum_peer_connectivity,
maximum_peer_connectivity,
state_pruning_block_depth,
network_public_key=None,
network_private_key=None,
roles=None):
"""Constructs a validator instance.
Args:
bind_network (str): the network endpoint
bind_component (str): the component endpoint
endpoint (str): the zmq-style URI of this validator's
publically reachable endpoint
peering (str): The type of peering approach. Either 'static'
or 'dynamic'. In 'static' mode, no attempted topology
buildout occurs -- the validator only attempts to initiate
peering connections with endpoints specified in the
peer_list. In 'dynamic' mode, the validator will first
attempt to initiate peering connections with endpoints
specified in the peer_list and then attempt to do a
topology buildout starting with peer lists obtained from
endpoints in the seeds_list. In either mode, the validator
will accept incoming peer requests up to max_peers.
seeds_list (list of str): a list of addresses to connect
to in order to perform the initial topology buildout
peer_list (list of str): a list of peer addresses
data_dir (str): path to the data directory
config_dir (str): path to the config directory
identity_signer (str): cryptographic signer the validator uses for
signing
"""
# -- Setup Global State Database and Factory -- #
global_state_db_filename = os.path.join(
data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('global state database file is %s',
global_state_db_filename)
global_state_db = NativeLmdbDatabase(
global_state_db_filename,
indexes=MerkleDatabase.create_index_configuration())
state_view_factory = StateViewFactory(global_state_db)
# -- Setup Receipt Store -- #
receipt_db_filename = os.path.join(
data_dir, 'txn_receipts-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('txn receipt store file is %s', receipt_db_filename)
receipt_db = LMDBNoLockDatabase(receipt_db_filename, 'c')
receipt_store = TransactionReceiptStore(receipt_db)
# -- Setup Block Store -- #
block_db_filename = os.path.join(
data_dir, 'block-{}.lmdb'.format(bind_network[-2:]))
LOGGER.debug('block store file is %s', block_db_filename)
block_db = IndexedDatabase(
block_db_filename,
BlockStore.serialize_block,
BlockStore.deserialize_block,
flag='c',
indexes=BlockStore.create_index_configuration())
block_store = BlockStore(block_db)
# The cache keep time for the journal's block cache must be greater
# than the cache keep time used by the completer.
base_keep_time = 1200
block_cache = BlockCache(block_store,
keep_time=int(base_keep_time * 9 / 8),
purge_frequency=30)
# -- Setup Thread Pools -- #
component_thread_pool = InstrumentedThreadPoolExecutor(
max_workers=10, name='Component')
network_thread_pool = InstrumentedThreadPoolExecutor(max_workers=10,
name='Network')
client_thread_pool = InstrumentedThreadPoolExecutor(max_workers=5,
name='Client')
sig_pool = InstrumentedThreadPoolExecutor(max_workers=3,
name='Signature')
# -- Setup Dispatchers -- #
component_dispatcher = Dispatcher()
network_dispatcher = Dispatcher()
# -- Setup Services -- #
component_service = Interconnect(bind_component,
component_dispatcher,
secured=False,
heartbeat=False,
max_incoming_connections=20,
monitor=True,
max_future_callback_workers=10)
zmq_identity = hashlib.sha512(
time.time().hex().encode()).hexdigest()[:23]
secure = False
if network_public_key is not None and network_private_key is not None:
secure = True
network_service = Interconnect(bind_network,
dispatcher=network_dispatcher,
zmq_identity=zmq_identity,
secured=secure,
server_public_key=network_public_key,
server_private_key=network_private_key,
heartbeat=True,
public_endpoint=endpoint,
connection_timeout=120,
max_incoming_connections=100,
max_future_callback_workers=10,
authorize=True,
signer=identity_signer,
roles=roles)
# -- Setup Transaction Execution Platform -- #
context_manager = ContextManager(global_state_db)
batch_tracker = BatchTracker(block_store)
settings_cache = SettingsCache(
SettingsViewFactory(state_view_factory), )
transaction_executor = TransactionExecutor(
service=component_service,
context_manager=context_manager,
settings_view_factory=SettingsViewFactory(state_view_factory),
scheduler_type=scheduler_type,
invalid_observers=[batch_tracker])
component_service.set_check_connections(
transaction_executor.check_connections)
event_broadcaster = EventBroadcaster(component_service, block_store,
receipt_store)
# -- Setup P2P Networking -- #
gossip = Gossip(network_service,
settings_cache,
lambda: block_store.chain_head,
block_store.chain_head_state_root,
endpoint=endpoint,
peering_mode=peering,
initial_seed_endpoints=seeds_list,
initial_peer_endpoints=peer_list,
minimum_peer_connectivity=minimum_peer_connectivity,
maximum_peer_connectivity=maximum_peer_connectivity,
topology_check_frequency=1)
completer = Completer(block_store,
gossip,
cache_keep_time=base_keep_time,
cache_purge_frequency=30,
requested_keep_time=300)
block_sender = BroadcastBlockSender(completer, gossip)
batch_sender = BroadcastBatchSender(completer, gossip)
chain_id_manager = ChainIdManager(data_dir)
identity_view_factory = IdentityViewFactory(
StateViewFactory(global_state_db))
id_cache = IdentityCache(identity_view_factory)
# -- Setup Permissioning -- #
permission_verifier = PermissionVerifier(
permissions, block_store.chain_head_state_root, id_cache)
identity_observer = IdentityObserver(to_update=id_cache.invalidate,
forked=id_cache.forked)
settings_observer = SettingsObserver(
to_update=settings_cache.invalidate, forked=settings_cache.forked)
# -- Consensus Engine -- #
consensus_thread_pool = InstrumentedThreadPoolExecutor(
max_workers=3, name='Consensus')
consensus_dispatcher = Dispatcher()
consensus_service = Interconnect(bind_consensus,
consensus_dispatcher,
secured=False,
heartbeat=False,
max_incoming_connections=20,
monitor=True,
max_future_callback_workers=10)
consensus_notifier = ConsensusNotifier(consensus_service)
# -- Setup Journal -- #
batch_injector_factory = DefaultBatchInjectorFactory(
block_cache=block_cache,
state_view_factory=state_view_factory,
signer=identity_signer)
block_publisher = BlockPublisher(
transaction_executor=transaction_executor,
block_cache=block_cache,
state_view_factory=state_view_factory,
settings_cache=settings_cache,
block_sender=block_sender,
batch_sender=batch_sender,
chain_head=block_store.chain_head,
identity_signer=identity_signer,
data_dir=data_dir,
config_dir=config_dir,
permission_verifier=permission_verifier,
check_publish_block_frequency=0.1,
batch_observers=[batch_tracker],
batch_injector_factory=batch_injector_factory)
block_publisher_batch_sender = block_publisher.batch_sender()
block_validator = BlockValidator(
block_cache=block_cache,
state_view_factory=state_view_factory,
transaction_executor=transaction_executor,
identity_signer=identity_signer,
data_dir=data_dir,
config_dir=config_dir,
permission_verifier=permission_verifier)
chain_controller = ChainController(
block_store=block_store,
block_cache=block_cache,
block_validator=block_validator,
state_database=global_state_db,
chain_head_lock=block_publisher.chain_head_lock,
state_pruning_block_depth=state_pruning_block_depth,
data_dir=data_dir,
observers=[
event_broadcaster, receipt_store, batch_tracker,
identity_observer, settings_observer
])
genesis_controller = GenesisController(
context_manager=context_manager,
transaction_executor=transaction_executor,
completer=completer,
block_store=block_store,
state_view_factory=state_view_factory,
identity_signer=identity_signer,
data_dir=data_dir,
config_dir=config_dir,
chain_id_manager=chain_id_manager,
batch_sender=batch_sender)
responder = Responder(completer)
completer.set_on_batch_received(block_publisher_batch_sender.send)
completer.set_on_block_received(chain_controller.queue_block)
completer.set_chain_has_block(chain_controller.has_block)
# -- Register Message Handler -- #
network_handlers.add(network_dispatcher, network_service, gossip,
completer, responder, network_thread_pool,
sig_pool, chain_controller.has_block,
block_publisher.has_batch, permission_verifier,
block_publisher, consensus_notifier)
component_handlers.add(component_dispatcher, gossip, context_manager,
transaction_executor, completer, block_store,
batch_tracker, global_state_db,
self.get_chain_head_state_root_hash,
receipt_store, event_broadcaster,
permission_verifier, component_thread_pool,
client_thread_pool, sig_pool, block_publisher)
# -- Store Object References -- #
self._component_dispatcher = component_dispatcher
self._component_service = component_service
self._component_thread_pool = component_thread_pool
self._network_dispatcher = network_dispatcher
self._network_service = network_service
self._network_thread_pool = network_thread_pool
consensus_proxy = ConsensusProxy(
block_cache=block_cache,
chain_controller=chain_controller,
block_publisher=block_publisher,
gossip=gossip,
identity_signer=identity_signer,
settings_view_factory=SettingsViewFactory(state_view_factory),
state_view_factory=state_view_factory)
consensus_handlers.add(consensus_dispatcher, consensus_thread_pool,
consensus_proxy)
self._consensus_dispatcher = consensus_dispatcher
self._consensus_service = consensus_service
self._consensus_thread_pool = consensus_thread_pool
self._client_thread_pool = client_thread_pool
self._sig_pool = sig_pool
self._context_manager = context_manager
self._transaction_executor = transaction_executor
self._genesis_controller = genesis_controller
self._gossip = gossip
self._block_publisher = block_publisher
self._chain_controller = chain_controller
self._block_validator = block_validator
class TestBlockPublisher(unittest.TestCase):
'''
The block publisher has three main functions, and in these tests
those functions are given the following wrappers for convenience:
* on_batch_received -> receive_batches
* on_chain_updated -> update_chain_head
* on_check_publish_block -> publish_block
After publishing a block, publish_block sends its block to the
mock block sender, and that block is named result_block. This block
is what is checked by the test assertions.
The basic pattern for the publisher tests (with variations) is:
0) make a list of batches (usually in setUp);
1) receive the batches;
2) publish a block;
3) verify the block (checking that it contains the correct batches,
or checking that it doesn't exist, or whatever).
The publisher chain head might be updated several times in a test.
'''
def setUp(self):
self.block_tree_manager = BlockTreeManager()
self.block_sender = MockBlockSender()
self.batch_sender = MockBatchSender()
self.state_view_factory = MockStateViewFactory({})
self.publisher = BlockPublisher(
transaction_executor=MockTransactionExecutor(),
block_cache=self.block_tree_manager.block_cache,
state_view_factory=self.state_view_factory,
block_sender=self.block_sender,
batch_sender=self.batch_sender,
squash_handler=None,
chain_head=self.block_tree_manager.chain_head,
identity_signing_key=self.block_tree_manager.identity_signing_key,
data_dir=None)
self.init_chain_head = self.block_tree_manager.chain_head
self.result_block = None
# A list of batches is created at the beginning of each test.
# The test assertions and the publisher function wrappers
# take these batches as a default argument.
self.batch_count = 8
self.batches = self.make_batches()
def test_publish(self):
'''
Publish a block with several batches
'''
self.receive_batches()
self.publish_block()
self.verify_block()
def test_reject_duplicate_batches_from_receive(self):
'''
Test that duplicate batches from on_batch_received are rejected
'''
for i in range(5):
self.receive_batches()
self.publish_block()
self.verify_block()
def test_reject_duplicate_batches_from_store(self):
'''
Test that duplicate batches from block store are rejected
'''
self.update_chain_head(None)
self.update_chain_head(head=self.init_chain_head,
uncommitted=self.batches)
self.receive_batches()
self.publish_block()
self.verify_block()
def test_no_chain_head(self):
'''
Test that nothing gets published with a null chain head,
then test that publishing resumes after updating
'''
self.update_chain_head(None)
self.receive_batches()
# try to publish block (failing)
self.publish_block()
self.assert_no_block_published()
# reset chain head several times,
# making sure batches remain queued
for i in range(3):
self.update_chain_head(None)
self.update_chain_head(self.init_chain_head)
# try to publish block (succeeding)
self.publish_block()
self.verify_block()
def test_committed_batches(self):
'''
Test that batches committed upon updating the chain head
are not included in the next block.
'''
self.update_chain_head(None)
self.update_chain_head(head=self.init_chain_head,
committed=self.batches)
new_batches = self.make_batches(batch_count=12)
self.receive_batches(new_batches)
self.publish_block()
self.verify_block(new_batches)
def test_uncommitted_batches(self):
'''
Test that batches uncommitted upon updating the chain head
are included in the next block.
'''
self.update_chain_head(None)
self.update_chain_head(head=self.init_chain_head,
uncommitted=self.batches)
self.publish_block()
self.verify_block()
def test_empty_pending_queue(self):
'''
Test that no block is published if the pending queue is empty
'''
# try to publish with no pending queue (failing)
self.publish_block()
self.assert_no_block_published()
# receive batches, then try again (succeeding)
self.receive_batches()
self.publish_block()
self.verify_block()
# assertions
def assert_block_published(self):
self.assertIsNotNone(self.result_block,
'Block should have been published')
def assert_no_block_published(self):
self.assertIsNone(self.result_block,
'Block should not have been published')
def assert_batch_in_block(self, batch):
self.assertIn(batch, tuple(self.result_block.batches),
'Batch not in block')
def assert_batches_in_block(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.assert_batch_in_block(batch)
def assert_block_batch_count(self, batch_count=None):
if batch_count is None:
batch_count = self.batch_count
self.assertEqual(len(self.result_block.batches), batch_count,
'Wrong batch count in block')
def verify_block(self, batches=None):
if batches is None:
batches = self.batches
batch_count = None if batches is None else len(batches)
self.assert_block_published()
self.assert_batches_in_block(batches)
self.assert_block_batch_count(batch_count)
self.result_block = None
# publisher functions
def receive_batch(self, batch):
self.publisher.on_batch_received(batch)
def receive_batches(self, batches=None):
if batches is None:
batches = self.batches
for batch in batches:
self.receive_batch(batch)
def publish_block(self):
self.publisher.on_check_publish_block()
self.result_block = self.block_sender.new_block
def update_chain_head(self, head, committed=None, uncommitted=None):
self.publisher.on_chain_updated(chain_head=head,
committed_batches=committed,
uncommitted_batches=uncommitted)
# batches
def make_batch(self, payload='batch'):
return self.block_tree_manager._generate_batch(payload)
def make_batches(self, batch_count=None):
if batch_count is None:
batch_count = self.batch_count
return [self.make_batch('batch_' + str(i)) for i in range(batch_count)]
