def test_batches_rejected_by_scheduler(self, mock_batch_injector_factory): ''' Test that no block is published with batches rejected by the scheduler ''' mock_batch_injector_factory.create_injectors.return_value = [] self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor( batch_execution_result=False), transaction_committed=( self.block_tree_manager.block_store.has_transaction), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.receive_batches() # Block should be empty since all batches are rejected with self.assertRaises(BlockEmpty, msg='Block should be empty'): self.publish_block() self.assert_no_block_published()
def test_batch_injection_start_block(self): ''' Test that the batch is injected at the beginning of the block. ''' injected_batch = self.make_batch() self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, permission_verifier=self.permission_verifier, batch_observers=[], batch_injector_factory=MockBatchInjectorFactory(injected_batch)) self.receive_batches() self.publish_block() self.assert_batch_in_block(injected_batch)
def setUp(self, mock_batch_injector_factory): mock_batch_injector_factory.create_injectors.return_value = [] self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.permission_verifier = MockPermissionVerifier() self.publisher = BlockPublisher( block_store=self.block_tree_manager.block_store, block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches()
def test_max_block_size(self): ''' Test block publisher obeys the block size limits ''' # Create a publisher that has a state view # with a batch limit addr, value = CreateSetting( 'sawtooth.publisher.max_batches_per_block', 1) print('test', addr) self.state_view_factory = MockStateViewFactory( {addr: value}) self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signing_key=self.block_tree_manager.identity_signing_key, data_dir=None, config_dir=None) self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() # try to publish with no pending queue (failing) for i in range(self.batch_count): self.publish_block() self.assert_block_published() self.update_chain_head(BlockWrapper(self.result_block)) self.verify_block([self.batches[i]])
def setUp(self): self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signing_key=self.block_tree_manager.identity_signing_key, data_dir=None, config_dir=None) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches()
def __init__(self): self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.permission_verifier = MockPermissionVerifier() self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, settings_cache=SettingsCache( SettingsViewFactory( self.block_tree_manager.state_view_factory), ), block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, check_publish_block_frequency=0.1, batch_observers=[], permission_verifier=self.permission_verifier) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches()
def test_batch_injection_start_block(self): ''' Test that the batch is injected at the beginning of the block. ''' injected_batch = self.make_batch() self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, permission_verifier=self.permission_verifier, batch_observers=[], batch_injector_factory=MockBatchInjectorFactory(injected_batch)) self.receive_batches() self.publish_block() self.assert_batch_in_block(injected_batch)
def _init_subprocesses(self): self._block_publisher = BlockPublisher( transaction_executor=self._transaction_executor, block_cache=self._block_cache, state_view_factory=self._state_view_factory, block_sender=self._block_sender, squash_handler=self._squash_handler, chain_head=self._block_store.chain_head, identity_signing_key=self._identity_signing_key ) self._publisher_thread = self._PublisherThread(self._block_publisher, self._batch_queue) self._chain_controller = ChainController( block_sender=self._block_sender, block_cache=self._block_cache, state_view_factory=self._state_view_factory, executor=ThreadPoolExecutor(1), transaction_executor=self._transaction_executor, on_chain_updated=self._block_publisher.on_chain_updated, squash_handler=self._squash_handler, chain_id_manager=self._chain_id_manager ) self._chain_thread = self._ChainThread(self._chain_controller, self._block_queue, self._block_cache)
def _init_subprocesses(self): self._block_publisher = BlockPublisher( transaction_executor=self._transaction_executor, block_cache=self._block_cache, state_view_factory=self._state_view_factory, block_sender=self._block_sender, batch_sender=self._batch_sender, squash_handler=self._squash_handler, chain_head=self._block_store.chain_head, identity_signing_key=self._identity_signing_key, data_dir=self._data_dir, config_dir=self._config_dir) self._publisher_thread = self._PublisherThread( block_publisher=self._block_publisher, batch_queue=self._batch_queue, check_publish_block_frequency=self._check_publish_block_frequency) self._chain_controller = ChainController( block_sender=self._block_sender, block_cache=self._block_cache, state_view_factory=self._state_view_factory, executor=self._executor_threadpool, transaction_executor=self._transaction_executor, chain_head_lock=self._block_publisher.chain_head_lock, on_chain_updated=self._block_publisher.on_chain_updated, squash_handler=self._squash_handler, chain_id_manager=self._chain_id_manager, state_delta_processor=self._state_delta_processor, identity_signing_key=self._identity_signing_key, data_dir=self._data_dir, config_dir=self._config_dir) self._chain_thread = self._ChainThread( chain_controller=self._chain_controller, block_queue=self._block_queue, block_cache=self._block_cache, block_cache_purge_frequency=self._block_cache_purge_frequency)
def do_batch_injection_start_block(): injected_batch = testBlockPublisher.make_batch() testBlockPublisher.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=testBlockPublisher.block_tree_manager.block_cache, state_view_factory=testBlockPublisher.state_view_factory, settings_cache=SettingsCache( SettingsViewFactory( testBlockPublisher.block_tree_manager.state_view_factory), ), block_sender=testBlockPublisher.block_sender, batch_sender=testBlockPublisher.batch_sender, squash_handler=None, chain_head=testBlockPublisher.block_tree_manager.chain_head, identity_signer=testBlockPublisher.block_tree_manager. identity_signer, data_dir=None, config_dir=None, permission_verifier=testBlockPublisher.permission_verifier, check_publish_block_frequency=0.1, batch_observers=[], batch_injector_factory=MockBatchInjectorFactory(injected_batch)) testBlockPublisher.receive_batches() testBlockPublisher.publish_block()
def test_batches_rejected_by_scheduler(self, mock_batch_injector_factory): ''' Test that no block is published with batches rejected by the scheduler ''' mock_batch_injector_factory.create_injectors.return_value = [] self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor( batch_execution_result=False), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.receive_batches() # Block should be empty since all batches are rejected with self.assertRaises(BlockEmpty, msg='Block should be empty'): self.publish_block() self.assert_no_block_published()
def test_validation_rules_reject_batches(self, mock_batch_injector_factory): """Test that a batch is not added to the block if it will violate the block validation rules. It does the following: - Sets the block_validation_rules to limit the number of 'test' transactions to 1 - creates two batches, limited to 1 transaction each, and receives them - verifies that only the first batch was committed to the block """ addr, value = CreateSetting( 'sawtooth.validator.block_validation_rules', 'NofX:1,test') self.state_view_factory = MockStateViewFactory({addr: value}) mock_batch_injector_factory.create_injectors.return_value = [] batch1 = self.make_batch(txn_count=1) batch2 = self.make_batch(txn_count=1) self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.receive_batches(batches=[batch1, batch2]) self.publish_block() self.assert_block_batch_count(1) self.assert_batch_in_block(batch1)
def __init__(self, consensus, block_store, send_message, transaction_executor, squash_handler, first_state_root): self._consensus = consensus self._block_store = block_store self._send_message = send_message self._squash_handler = squash_handler self._block_publisher = BlockPublisher( consensus=consensus.BlockPublisher(), transaction_executor=transaction_executor, send_message=send_message, squash_handler=squash_handler) self._batch_queue = queue.Queue() self._publisher_thread = self._PublisherThread(self._block_publisher, self._batch_queue) # HACK until genesis tool is working if "chain_head_id" not in self._block_store: genesis_block = BlockState( block_wrapper=self._block_publisher.generate_genesis_block(), weight=0, status=BlockStatus.Valid) genesis_block.block.set_state_hash(first_state_root) self._block_store[genesis_block.block.header_signature] = \ genesis_block self._block_store["chain_head_id"] = \ genesis_block.block.header_signature self._block_publisher.on_chain_updated(genesis_block.block) LOGGER.info("Journal created genesis block: %s", genesis_block.block.header_signature) self._chain_controller = ChainController( consensus=consensus.BlockVerifier(), block_store=block_store, send_message=send_message, executor=ThreadPoolExecutor(1), transaction_executor=transaction_executor, on_chain_updated=self._block_publisher.on_chain_updated, squash_handler=self._squash_handler) self._block_queue = queue.Queue() self._chain_thread = self._ChainThread(self._chain_controller, self._block_queue)
def test_max_block_size(self, mock_batch_injector_factory): ''' Test block publisher obeys the block size limits ''' mock_batch_injector_factory.create_injectors.return_value = [] # Create a publisher that has a state view # with a batch limit addr, value = CreateSetting('sawtooth.publisher.max_batches_per_block', 1) self.state_view_factory = MockStateViewFactory({addr: value}) self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() # try to publish with no pending queue (failing) for i in range(self.batch_count): self.publish_block() self.assert_block_published() self.update_chain_head(BlockWrapper(self.result_block)) self.verify_block([self.batches[i]])
def test_batches_rejected_by_scheduler(self): ''' Test that no block is published with batches rejected by the scheduler ''' self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor( batch_execution_result=False), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signing_key=self.block_tree_manager.identity_signing_key, data_dir=None) self.receive_batches() self.publish_block() self.assert_no_block_published()
def __init__(self, with_genesis=True): self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.block_store = BlockStore( DictDatabase(indexes=BlockStore.create_index_configuration())) self.block_cache = BlockCache(self.block_store) self.state_db = {} self.block_manager = BlockManager() self.block_manager.add_store("commit_store", self.block_store) # add the mock reference to the consensus consensus_setting_addr = SettingsView.setting_address( 'sawtooth.consensus.algorithm') self.state_db[consensus_setting_addr] = _setting_entry( 'sawtooth.consensus.algorithm', 'test_journal.mock_consensus') self.state_view_factory = MockStateViewFactory(self.state_db) context = create_context('secp256k1') private_key = context.new_random_private_key() crypto_factory = CryptoFactory(context) self.signer = crypto_factory.new_signer(private_key) identity_private_key = context.new_random_private_key() self.identity_signer = crypto_factory.new_signer(identity_private_key) chain_head = None if with_genesis: self.genesis_block = self.generate_genesis_block() chain_head = self.genesis_block self.block_manager.put([chain_head.block]) self.block_manager.persist(chain_head.block.header_signature, "commit_store") self.block_publisher = BlockPublisher( block_manager=self.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=self.block_store.has_transaction, batch_committed=self.block_store.has_batch, state_view_factory=self.state_view_factory, settings_cache=SettingsCache( SettingsViewFactory(self.state_view_factory), ), block_sender=self.block_sender, batch_sender=self.block_sender, chain_head=chain_head.block, identity_signer=self.identity_signer, data_dir=None, config_dir=None, permission_verifier=MockPermissionVerifier(), batch_observers=[])
def test_no_chain_head(self): publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.blocks.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, squash_handler=None, chain_head=self.blocks.chain_head, identity_signing_key=self.blocks.identity_signing_key) # Test halting the BlockPublisher by setting the chain head to null publisher.on_chain_updated(None) batch = Batch() publisher.on_batch_received(batch) publisher.on_check_publish_block() self.assertIsNone(self.block_sender.new_block)
def test_validation_rules_reject_batches(self, mock_batch_injector_factory): """Test that a batch is not added to the block if it will violate the block validation rules. It does the following: - Sets the block_validation_rules to limit the number of 'test' transactions to 1 - creates two batches, limited to 1 transaction each, and receives them - verifies that only the first batch was committed to the block """ addr, value = CreateSetting( 'sawtooth.validator.block_validation_rules', 'NofX:1,test') self.state_view_factory = MockStateViewFactory( {addr: value}) mock_batch_injector_factory.create_injectors.return_value = [] batch1 = self.make_batch(txn_count=1) batch2 = self.make_batch(txn_count=1) self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.receive_batches(batches=[batch1, batch2]) self.publish_block() self.assert_block_batch_count(1) self.assert_batch_in_block(batch1)
def __init__(self, with_genesis=True): self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.dir = tempfile.mkdtemp() self.block_db = NativeLmdbDatabase( os.path.join(self.dir, 'block.lmdb'), BlockStore.create_index_configuration()) self.block_store = BlockStore(self.block_db) self.block_cache = BlockCache(self.block_store) self.state_db = NativeLmdbDatabase( os.path.join(self.dir, "merkle.lmdb"), MerkleDatabase.create_index_configuration()) self.state_view_factory = NativeStateViewFactory(self.state_db) self.block_manager = BlockManager() self.block_manager.add_commit_store(self.block_store) context = create_context('secp256k1') private_key = context.new_random_private_key() crypto_factory = CryptoFactory(context) self.signer = crypto_factory.new_signer(private_key) identity_private_key = context.new_random_private_key() self.identity_signer = crypto_factory.new_signer(identity_private_key) chain_head = None if with_genesis: self.genesis_block = self.generate_genesis_block() chain_head = self.genesis_block self.block_manager.put([chain_head.block]) self.block_manager.persist(chain_head.block.header_signature, "commit_store") self.block_publisher = BlockPublisher( block_manager=self.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=self.block_store.has_transaction, batch_committed=self.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.block_sender, chain_head=chain_head.block, identity_signer=self.identity_signer, data_dir=None, config_dir=None, permission_verifier=MockPermissionVerifier(), batch_observers=[])
def __init__(self, with_genesis=True): self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.block_store = BlockStore(DictDatabase( indexes=BlockStore.create_index_configuration())) self.block_cache = BlockCache(self.block_store) self.state_db = {} # add the mock reference to the consensus consensus_setting_addr = SettingsView.setting_address( 'sawtooth.consensus.algorithm') self.state_db[consensus_setting_addr] = _setting_entry( 'sawtooth.consensus.algorithm', 'test_journal.mock_consensus') self.state_view_factory = MockStateViewFactory(self.state_db) context = create_context('secp256k1') private_key = context.new_random_private_key() crypto_factory = CryptoFactory(context) self.signer = crypto_factory.new_signer(private_key) identity_private_key = context.new_random_private_key() self.identity_signer = crypto_factory.new_signer(identity_private_key) chain_head = None if with_genesis: self.genesis_block = self.generate_genesis_block() self.set_chain_head(self.genesis_block) chain_head = self.genesis_block self.block_publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_cache, state_view_factory=self.state_view_factory, settings_cache=SettingsCache( SettingsViewFactory(self.state_view_factory), ), block_sender=self.block_sender, batch_sender=self.block_sender, squash_handler=None, chain_head=chain_head, identity_signer=self.identity_signer, data_dir=None, config_dir=None, permission_verifier=MockPermissionVerifier(), check_publish_block_frequency=0.1, batch_observers=[])
def _init_subprocesses(self): batch_injector_factory = DefaultBatchInjectorFactory( block_store=self._block_store, state_view_factory=self._state_view_factory, signing_key=self._identity_signing_key, ) self._block_publisher = BlockPublisher( transaction_executor=self._transaction_executor, block_cache=self._block_cache, state_view_factory=self._state_view_factory, block_sender=self._block_sender, batch_sender=self._batch_sender, squash_handler=self._squash_handler, chain_head=self._block_store.chain_head, identity_signing_key=self._identity_signing_key, data_dir=self._data_dir, config_dir=self._config_dir, permission_verifier=self._permission_verifier, batch_injector_factory=batch_injector_factory, ) self._publisher_thread = self._PublisherThread( block_publisher=self._block_publisher, batch_queue=self._batch_queue, check_publish_block_frequency=self._check_publish_block_frequency ) self._chain_controller = ChainController( block_sender=self._block_sender, block_cache=self._block_cache, state_view_factory=self._state_view_factory, executor=self._executor_threadpool, transaction_executor=self._transaction_executor, chain_head_lock=self._block_publisher.chain_head_lock, on_chain_updated=self._block_publisher.on_chain_updated, squash_handler=self._squash_handler, chain_id_manager=self._chain_id_manager, identity_signing_key=self._identity_signing_key, data_dir=self._data_dir, config_dir=self._config_dir, permission_verifier=self._permission_verifier, chain_observers=self._chain_observers, ) self._chain_thread = self._ChainThread( chain_controller=self._chain_controller, block_queue=self._block_queue, block_cache=self._block_cache)
def test_publish(self): publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.blocks.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, squash_handler=None, chain_head=self.blocks.chain_head, identity_signing_key=self.blocks.identity_signing_key) # initial load of existing state publisher.on_chain_updated(self.blocks.chain_head, [], []) # repeat as necessary batch = Batch() publisher.on_batch_received(batch) # current dev_mode consensus always claims blocks when asked. # this will be called on a polling every so often or possibly triggered # by events in the consensus it's self ... TBD publisher.on_check_publish_block()
def test_max_block_size(self, mock_batch_injector_factory): ''' Test block publisher obeys the block size limits ''' mock_batch_injector_factory.create_injectors.return_value = [] # Create a publisher that has a state view # with a batch limit addr, value = CreateSetting( 'sawtooth.publisher.max_batches_per_block', 1) self.state_view_factory = MockStateViewFactory( {addr: value}) self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() # try to publish with no pending queue (failing) for i in range(self.batch_count): self.publish_block() self.assert_block_published() self.update_chain_head(BlockWrapper(self.result_block)) self.verify_block([self.batches[i]])
def _init_subprocesses(self): self._block_publisher = BlockPublisher( consensus=self._consensus.BlockPublisher(), transaction_executor=self._transaction_executor, block_sender=self._block_sender, squash_handler=self._squash_handler, chain_head=self._block_store.chain_head) self._publisher_thread = self._PublisherThread(self._block_publisher, self._batch_queue) self._chain_controller = ChainController( consensus=self._consensus.BlockVerifier(), block_sender=self._block_sender, block_cache=self._block_cache, executor=ThreadPoolExecutor(1), transaction_executor=self._transaction_executor, on_chain_updated=self._block_publisher.on_chain_updated, squash_handler=self._squash_handler) self._chain_thread = self._ChainThread(self._chain_controller, self._block_queue, self._block_cache)
def __init__(self, with_genesis=True): self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.block_store = BlockStore(DictDatabase()) self.block_cache = BlockCache(self.block_store) self.state_db = {} # add the mock reference to the consensus consensus_setting_addr = SettingsView.setting_address( 'sawtooth.consensus.algorithm') self.state_db[consensus_setting_addr] = _setting_entry( 'sawtooth.consensus.algorithm', 'test_journal.mock_consensus') self.state_view_factory = MockStateViewFactory(self.state_db) self.signing_key = signing.generate_private_key() self.public_key = signing.generate_public_key(self.signing_key) self.identity_signing_key = signing.generate_private_key() chain_head = None if with_genesis: self.genesis_block = self.generate_genesis_block() self.set_chain_head(self.genesis_block) chain_head = self.genesis_block self.block_publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.block_sender, squash_handler=None, chain_head=chain_head, identity_signing_key=self.identity_signing_key, data_dir=None, config_dir=None, permission_verifier=MockPermissionVerifier(), check_publish_block_frequency=0.1, batch_observers=[])
def test_batches_rejected_by_scheduler(self): ''' Test that no block is published with batches rejected by the scheduler ''' self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor( batch_execution_result=False), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signing_key=self.block_tree_manager.identity_signing_key, data_dir=None, config_dir=None) self.receive_batches() self.publish_block() self.assert_no_block_published()
def setUp(self, mock_batch_injector_factory): mock_batch_injector_factory.create_injectors.return_value = [] self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.permission_verifier = MockPermissionVerifier() self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches()
class TestBlockPublisher(unittest.TestCase): ''' The block publisher has three main functions, and in these tests those functions are given the following wrappers for convenience: * on_batch_received -> receive_batches * on_chain_updated -> update_chain_head * on_check_publish_block -> publish_block After publishing a block, publish_block sends its block to the mock block sender, and that block is named result_block. This block is what is checked by the test assertions. The basic pattern for the publisher tests (with variations) is: 0) make a list of batches (usually in setUp); 1) receive the batches; 2) publish a block; 3) verify the block (checking that it contains the correct batches, or checking that it doesn't exist, or whatever). The publisher chain head might be updated several times in a test. ''' def setUp(self): self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signing_key=self.block_tree_manager.identity_signing_key, data_dir=None, config_dir=None) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches() def test_publish(self): ''' Publish a block with several batches ''' self.receive_batches() self.publish_block() self.verify_block() def test_reject_duplicate_batches_from_receive(self): ''' Test that duplicate batches from on_batch_received are rejected ''' for _ in range(5): self.receive_batches() self.publish_block() self.verify_block() def test_reject_duplicate_batches_from_store(self): ''' Test that duplicate batches from block store are rejected ''' self.update_chain_head(None) self.update_chain_head( head=self.init_chain_head, uncommitted=self.batches) self.receive_batches() self.publish_block() self.verify_block() def test_no_chain_head(self): ''' Test that nothing gets published with a null chain head, then test that publishing resumes after updating ''' self.update_chain_head(None) self.receive_batches() # try to publish block (failing) self.publish_block() self.assert_no_block_published() # reset chain head several times, # making sure batches remain queued for _ in range(3): self.update_chain_head(None) self.update_chain_head(self.init_chain_head) # try to publish block (succeeding) self.publish_block() self.verify_block() def test_committed_batches(self): ''' Test that batches committed upon updating the chain head are not included in the next block. ''' self.update_chain_head(None) self.update_chain_head( head=self.init_chain_head, committed=self.batches) new_batches = self.make_batches(batch_count=12) self.receive_batches(new_batches) self.publish_block() self.verify_block(new_batches) def test_uncommitted_batches(self): ''' Test that batches uncommitted upon updating the chain head are included in the next block. ''' self.update_chain_head(None) self.update_chain_head( head=self.init_chain_head, uncommitted=self.batches) self.publish_block() self.verify_block() def test_empty_pending_queue(self): ''' Test that no block is published if the pending queue is empty ''' # try to publish with no pending queue (failing) self.publish_block() self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() self.publish_block() self.verify_block() def test_missing_dependencies(self): ''' Test that no block is published with missing dependencies ''' self.batches = self.make_batches( missing_deps=True) self.receive_batches() self.publish_block() self.assert_no_block_published() def test_batches_rejected_by_scheduler(self): ''' Test that no block is published with batches rejected by the scheduler ''' self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor( batch_execution_result=False), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signing_key=self.block_tree_manager.identity_signing_key, data_dir=None, config_dir=None) self.receive_batches() self.publish_block() self.assert_no_block_published() def test_max_block_size(self): ''' Test block publisher obeys the block size limits ''' # Create a publisher that has a state view # with a batch limit addr, value = CreateSetting( 'sawtooth.publisher.max_batches_per_block', 1) print('test', addr) self.state_view_factory = MockStateViewFactory( {addr: value}) self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signing_key=self.block_tree_manager.identity_signing_key, data_dir=None, config_dir=None) self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() # try to publish with no pending queue (failing) for i in range(self.batch_count): self.publish_block() self.assert_block_published() self.update_chain_head(BlockWrapper(self.result_block)) self.verify_block([self.batches[i]]) def test_duplicate_transactions(self): ''' Test discards batches that have duplicate transactions in them. ''' # receive batches, then try again (succeeding) self.batches = self.batches[1:2] self.receive_batches() self.publish_block() self.assert_block_published() self.update_chain_head(BlockWrapper(self.result_block)) self.verify_block() # build a new set of batches with the same transactions in them self.batches = self.make_batches_with_duplicate_txn() self.receive_batches() self.publish_block() self.assert_no_block_published() # block should be empty after batch # with duplicate transaction is dropped. # assertions def assert_block_published(self): self.assertIsNotNone( self.result_block, 'Block should have been published') def assert_no_block_published(self): self.assertIsNone( self.result_block, 'Block should not have been published') def assert_batch_in_block(self, batch): self.assertIn( batch, tuple(self.result_block.batches), 'Batch not in block') def assert_batches_in_block(self, batches=None): if batches is None: batches = self.batches for batch in batches: self.assert_batch_in_block(batch) def assert_block_batch_count(self, batch_count=None): if batch_count is None: batch_count = self.batch_count self.assertEqual( len(self.result_block.batches), batch_count, 'Wrong batch count in block') def verify_block(self, batches=None): if batches is None: batches = self.batches batch_count = None if batches is None else len(batches) self.assert_block_published() self.assert_batches_in_block(batches) self.assert_block_batch_count(batch_count) self.result_block = None # publisher functions def receive_batch(self, batch): self.publisher.on_batch_received(batch) def receive_batches(self, batches=None): if batches is None: batches = self.batches for batch in batches: self.receive_batch(batch) def publish_block(self): self.publisher.on_check_publish_block() self.result_block = self.block_sender.new_block self.block_sender.new_block = None def update_chain_head(self, head, committed=None, uncommitted=None): if head: self.block_tree_manager.block_store.update_chain([head]) self.publisher.on_chain_updated( chain_head=head, committed_batches=committed, uncommitted_batches=uncommitted) # batches def make_batch(self, missing_deps=False): return self.block_tree_manager.generate_batch( missing_deps=missing_deps) def make_batches(self, batch_count=None, missing_deps=False): if batch_count is None: batch_count = self.batch_count return [self.make_batch(missing_deps=missing_deps) for _ in range(batch_count)] def make_batches_with_duplicate_txn(self): txns = [self.batches[0].transactions[0], self.block_tree_manager.generate_transaction("nonce")] return [self.block_tree_manager.generate_batch(txns=txns)]
def do_publish_block(): btm = BlockTreeManager() block_publisher = None chain_controller = None try: block_publisher = BlockPublisher( transaction_executor=testJournal.txn_executor, block_cache=btm.block_cache, state_view_factory=MockStateViewFactory(btm.state_db), settings_cache=SettingsCache( SettingsViewFactory(btm.state_view_factory), ), block_sender=testJournal.block_sender, batch_sender=testJournal.batch_sender, squash_handler=None, chain_head=btm.block_store.chain_head, identity_signer=btm.identity_signer, data_dir=None, config_dir=None, permission_verifier=testJournal.permission_verifier, check_publish_block_frequency=0.1, batch_observers=[], batch_injector_factory=DefaultBatchInjectorFactory( block_store=btm.block_store, state_view_factory=MockStateViewFactory(btm.state_db), signer=btm.identity_signer)) block_validator = BlockValidator( state_view_factory=MockStateViewFactory(btm.state_db), block_cache=btm.block_cache, transaction_executor=testJournal.txn_executor, squash_handler=None, identity_signer=btm.identity_signer, data_dir=None, config_dir=None, permission_verifier=testJournal.permission_verifier) chain_controller = ChainController( block_cache=btm.block_cache, block_validator=block_validator, state_view_factory=MockStateViewFactory(btm.state_db), chain_head_lock=block_publisher.chain_head_lock, on_chain_updated=block_publisher.on_chain_updated, chain_id_manager=None, data_dir=None, config_dir=None, chain_observers=[]) testJournal.gossip.on_batch_received = block_publisher.queue_batch testJournal.gossip.on_block_received = chain_controller.queue_block block_publisher.start() chain_controller.start() # feed it a batch batch = Batch() block_publisher.queue_batch(batch) wait_until(lambda: testJournal.block_sender.new_block is not None, 2) assert testJournal.block_sender.new_block != None block = BlockWrapper(testJournal.block_sender.new_block) chain_controller.queue_block(block) # wait for the chain_head to be updated. wait_until(lambda: btm.chain_head.identifier == block.identifier, 2) # assert btm.chain_head.identifier == block.identifier finally: if block_publisher is not None: block_publisher.stop() if chain_controller is not None: chain_controller.stop() if block_validator is not None: block_validator.stop()
class TestBlockPublisher(unittest.TestCase): ''' The block publisher has five main functions, and in these tests those functions are given the following wrappers for convenience: * on_batch_received -> receive_batches * on_chain_updated -> update_chain_head * initialize_block -> initialize_block * summarize_block -> summarize_block * finalize_block -> finalize_block Additionally, the publish_block is provided to call both initialize_block and finalize_block. After finalizing a block, finalize_block sends its block to the mock block sender, and that block is named result_block. This block is what is checked by the test assertions. The basic pattern for the publisher tests (with variations) is: 0) make a list of batches (usually in setUp); 1) receive the batches; 2) initialize a block; 3) finalize a block; 4) verify the block (checking that it contains the correct batches, or checking that it doesn't exist, etc.). ''' @unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory') def setUp(self, mock_batch_injector_factory): mock_batch_injector_factory.create_injectors.return_value = [] self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.permission_verifier = MockPermissionVerifier() self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches() def test_publish(self): ''' Publish a block with several batches ''' self.receive_batches() self.publish_block() self.verify_block() def test_receive_after_initialize(self): ''' Receive batches after initialization ''' self.initialize_block() self.receive_batches() self.finalize_block() self.verify_block() def test_summarize_block(self): ''' Initialize a block and summarize it ''' self.receive_batches() self.initialize_block() self.assertIsNotNone(self.summarize_block(), 'Expected block summary') def test_reject_double_initialization(self): ''' Test that you can't initialize a candidate block twice ''' self.initialize_block() with self.assertRaises( BlockInProgress, msg='Second initialization should have rejected'): self.initialize_block() def test_reject_finalize_without_initialize(self): ''' Test that no block is published if the block hasn't been initialized ''' self.receive_batches() with self.assertRaises( BlockNotInitialized, msg='Block should not be finalized'): self.finalize_block() def test_reject_duplicate_batches_from_receive(self): ''' Test that duplicate batches from on_batch_received are rejected ''' for _ in range(5): self.receive_batches() self.publish_block() self.verify_block() def test_reject_duplicate_batches_from_store(self): ''' Test that duplicate batches from block store are rejected ''' self.update_chain_head( head=self.init_chain_head, uncommitted=self.batches) self.receive_batches() self.publish_block() self.verify_block() def test_committed_batches(self): ''' Test that batches committed upon updating the chain head are not included in the next block. ''' self.update_chain_head( head=self.init_chain_head, committed=self.batches) new_batches = self.make_batches(batch_count=12) self.receive_batches(new_batches) self.publish_block() self.verify_block(new_batches) def test_uncommitted_batches(self): ''' Test that batches uncommitted upon updating the chain head are included in the next block. ''' self.update_chain_head( head=self.init_chain_head, uncommitted=self.batches) self.publish_block() self.verify_block() def test_empty_pending_queue(self): ''' Test that no block is published if the pending queue is empty ''' # try to publish with no pending queue (failing) with self.assertRaises( BlockEmpty, msg='Block should not be published'): self.publish_block() self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() self.finalize_block() self.verify_block() def test_missing_dependencies(self): ''' Test that no block is published with missing dependencies ''' self.batches = self.make_batches( missing_deps=True) self.receive_batches() # Block should be empty, since batches with missing deps aren't added with self.assertRaises(BlockEmpty, msg='Block should be empty'): self.publish_block() self.assert_no_block_published() @unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory') def test_batches_rejected_by_scheduler(self, mock_batch_injector_factory): ''' Test that no block is published with batches rejected by the scheduler ''' mock_batch_injector_factory.create_injectors.return_value = [] self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor( batch_execution_result=False), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.receive_batches() # Block should be empty since all batches are rejected with self.assertRaises(BlockEmpty, msg='Block should be empty'): self.publish_block() self.assert_no_block_published() @unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory') def test_max_block_size(self, mock_batch_injector_factory): ''' Test block publisher obeys the block size limits ''' mock_batch_injector_factory.create_injectors.return_value = [] # Create a publisher that has a state view # with a batch limit addr, value = CreateSetting( 'sawtooth.publisher.max_batches_per_block', 1) self.state_view_factory = MockStateViewFactory( {addr: value}) self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() # try to publish with no pending queue (failing) for i in range(self.batch_count): self.publish_block() self.assert_block_published() self.update_chain_head(BlockWrapper(self.result_block)) self.verify_block([self.batches[i]]) def test_duplicate_transactions(self): ''' Test discards batches that have duplicate transactions in them. ''' # receive batches, then try again (succeeding) self.batches = self.batches[1:2] self.receive_batches() self.publish_block() self.assert_block_published() self.update_chain_head(BlockWrapper(self.result_block)) self.verify_block() # build a new set of batches with the same transactions in them self.batches = self.make_batches_with_duplicate_txn() self.receive_batches() with self.assertRaises(BlockEmpty, msg='Block should be empty'): self.publish_block() self.assert_no_block_published() # block should be empty after batch # with duplicate transaction is dropped. def test_batch_injection_start_block(self): ''' Test that the batch is injected at the beginning of the block. ''' injected_batch = self.make_batch() self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, permission_verifier=self.permission_verifier, batch_observers=[], batch_injector_factory=MockBatchInjectorFactory(injected_batch)) self.receive_batches() self.publish_block() self.assert_batch_in_block(injected_batch) @unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory') def test_validation_rules_reject_batches(self, mock_batch_injector_factory): """Test that a batch is not added to the block if it will violate the block validation rules. It does the following: - Sets the block_validation_rules to limit the number of 'test' transactions to 1 - creates two batches, limited to 1 transaction each, and receives them - verifies that only the first batch was committed to the block """ addr, value = CreateSetting( 'sawtooth.validator.block_validation_rules', 'NofX:1,test') self.state_view_factory = MockStateViewFactory( {addr: value}) mock_batch_injector_factory.create_injectors.return_value = [] batch1 = self.make_batch(txn_count=1) batch2 = self.make_batch(txn_count=1) self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction ), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.receive_batches(batches=[batch1, batch2]) self.publish_block() self.assert_block_batch_count(1) self.assert_batch_in_block(batch1) # assertions def assert_block_published(self): self.assertIsNotNone( self.result_block, 'Block should have been published') def assert_no_block_published(self): self.assertIsNone( self.result_block, 'Block should not have been published') def assert_batch_in_block(self, batch): self.assertIn( batch, tuple(self.result_block.batches), 'Batch not in block') def assert_batches_in_block(self, batches=None): if batches is None: batches = self.batches for batch in batches: self.assert_batch_in_block(batch) def assert_block_batch_count(self, batch_count=None): if batch_count is None: batch_count = self.batch_count self.assertEqual( len(self.result_block.batches), batch_count, 'Wrong batch count in block') def verify_block(self, batches=None): if batches is None: batches = self.batches batch_count = None if batches is None else len(batches) self.assert_block_published() self.assert_batches_in_block(batches) self.assert_block_batch_count(batch_count) self.result_block = None # publisher functions def receive_batch(self, batch): self.publisher.on_batch_received(batch) def receive_batches(self, batches=None): if batches is None: batches = self.batches for batch in batches: self.receive_batch(batch) def initialize_block(self): self.publisher.initialize_block(self.block_tree_manager.chain_head) def summarize_block(self): return self.publisher.summarize_block() def finalize_block(self): self.publisher.finalize_block("") self.result_block = self.block_sender.new_block self.block_sender.new_block = None def publish_block(self): self.initialize_block() self.finalize_block() def update_chain_head(self, head, committed=None, uncommitted=None): if head: self.block_tree_manager.block_store.update_chain([head]) self.publisher.on_chain_updated( chain_head=head, committed_batches=committed, uncommitted_batches=uncommitted) # batches def make_batch(self, missing_deps=False, txn_count=2): return self.block_tree_manager.generate_batch( txn_count=txn_count, missing_deps=missing_deps) def make_batches(self, batch_count=None, missing_deps=False): if batch_count is None: batch_count = self.batch_count return [self.make_batch(missing_deps=missing_deps) for _ in range(batch_count)] def make_batches_with_duplicate_txn(self): txns = [self.batches[0].transactions[0], self.block_tree_manager.generate_transaction("nonce")] return [self.block_tree_manager.generate_batch(txns=txns)]
def test_publish(self): LOGGER.info(self.blocks) publisher = BlockPublisher( consensus=TestModePublisher(), transaction_executor=MockTransactionExecutor(), block_sender=self.block_sender, squash_handler=None, chain_head=self.blocks.chain_head) # initial load of existing state publisher.on_chain_updated(self.blocks.chain_head, [], []) # repeat as necessary batch = Batch() publisher.on_batch_received(batch) # current dev_mode consensus always claims blocks when asked. # this will be called on a polling every so often or possibly triggered # by events in the consensus it's self ... TBD publisher.on_check_publish_block() LOGGER.info(self.blocks) # repeat as necessary batch = Batch() publisher.on_batch_received(batch) publisher.on_check_publish_block() LOGGER.info(self.blocks)
def __init__(self, bind_network, bind_component, bind_consensus, endpoint, peering, seeds_list, peer_list, data_dir, config_dir, identity_signer, scheduler_type, permissions, minimum_peer_connectivity, maximum_peer_connectivity, state_pruning_block_depth, network_public_key=None, network_private_key=None, roles=None): """Constructs a validator instance. Args: bind_network (str): the network endpoint bind_component (str): the component endpoint endpoint (str): the zmq-style URI of this validator's publically reachable endpoint peering (str): The type of peering approach. Either 'static' or 'dynamic'. In 'static' mode, no attempted topology buildout occurs -- the validator only attempts to initiate peering connections with endpoints specified in the peer_list. In 'dynamic' mode, the validator will first attempt to initiate peering connections with endpoints specified in the peer_list and then attempt to do a topology buildout starting with peer lists obtained from endpoints in the seeds_list. In either mode, the validator will accept incoming peer requests up to max_peers. seeds_list (list of str): a list of addresses to connect to in order to perform the initial topology buildout peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory config_dir (str): path to the config directory identity_signer (str): cryptographic signer the validator uses for signing """ # -- Setup Global State Database and Factory -- # global_state_db_filename = os.path.join( data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug( 'global state database file is %s', global_state_db_filename) global_state_db = NativeLmdbDatabase( global_state_db_filename, indexes=MerkleDatabase.create_index_configuration()) state_view_factory = StateViewFactory(global_state_db) # -- Setup Receipt Store -- # receipt_db_filename = os.path.join( data_dir, 'txn_receipts-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('txn receipt store file is %s', receipt_db_filename) receipt_db = LMDBNoLockDatabase(receipt_db_filename, 'c') receipt_store = TransactionReceiptStore(receipt_db) # -- Setup Block Store -- # block_db_filename = os.path.join( data_dir, 'block-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = IndexedDatabase( block_db_filename, BlockStore.serialize_block, BlockStore.deserialize_block, flag='c', indexes=BlockStore.create_index_configuration()) block_store = BlockStore(block_db) # The cache keep time for the journal's block cache must be greater # than the cache keep time used by the completer. base_keep_time = 1200 block_cache = BlockCache( block_store, keep_time=int(base_keep_time * 9 / 8), purge_frequency=30) # -- Setup Thread Pools -- # component_thread_pool = InstrumentedThreadPoolExecutor( max_workers=10, name='Component') network_thread_pool = InstrumentedThreadPoolExecutor( max_workers=10, name='Network') client_thread_pool = InstrumentedThreadPoolExecutor( max_workers=5, name='Client') sig_pool = InstrumentedThreadPoolExecutor( max_workers=3, name='Signature') # -- Setup Dispatchers -- # component_dispatcher = Dispatcher() network_dispatcher = Dispatcher() # -- Setup Services -- # component_service = Interconnect( bind_component, component_dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True, max_future_callback_workers=10) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] secure = False if network_public_key is not None and network_private_key is not None: secure = True network_service = Interconnect( bind_network, dispatcher=network_dispatcher, zmq_identity=zmq_identity, secured=secure, server_public_key=network_public_key, server_private_key=network_private_key, heartbeat=True, public_endpoint=endpoint, connection_timeout=120, max_incoming_connections=100, max_future_callback_workers=10, authorize=True, signer=identity_signer, roles=roles) # -- Setup Transaction Execution Platform -- # context_manager = ContextManager(global_state_db) batch_tracker = BatchTracker(block_store) settings_cache = SettingsCache( SettingsViewFactory(state_view_factory), ) transaction_executor = TransactionExecutor( service=component_service, context_manager=context_manager, settings_view_factory=SettingsViewFactory(state_view_factory), scheduler_type=scheduler_type, invalid_observers=[batch_tracker]) component_service.set_check_connections( transaction_executor.check_connections) event_broadcaster = EventBroadcaster( component_service, block_store, receipt_store) # -- Setup P2P Networking -- # gossip = Gossip( network_service, settings_cache, lambda: block_store.chain_head, block_store.chain_head_state_root, endpoint=endpoint, peering_mode=peering, initial_seed_endpoints=seeds_list, initial_peer_endpoints=peer_list, minimum_peer_connectivity=minimum_peer_connectivity, maximum_peer_connectivity=maximum_peer_connectivity, topology_check_frequency=1 ) completer = Completer( block_store, gossip, cache_keep_time=base_keep_time, cache_purge_frequency=30, requested_keep_time=300) block_sender = BroadcastBlockSender(completer, gossip) batch_sender = BroadcastBatchSender(completer, gossip) chain_id_manager = ChainIdManager(data_dir) identity_view_factory = IdentityViewFactory( StateViewFactory(global_state_db)) id_cache = IdentityCache(identity_view_factory) # -- Setup Permissioning -- # permission_verifier = PermissionVerifier( permissions, block_store.chain_head_state_root, id_cache) identity_observer = IdentityObserver( to_update=id_cache.invalidate, forked=id_cache.forked) settings_observer = SettingsObserver( to_update=settings_cache.invalidate, forked=settings_cache.forked) # -- Consensus Engine -- # consensus_thread_pool = InstrumentedThreadPoolExecutor( max_workers=3, name='Consensus') consensus_dispatcher = Dispatcher() consensus_service = Interconnect( bind_consensus, consensus_dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True, max_future_callback_workers=10) consensus_notifier = ConsensusNotifier(consensus_service) # -- Setup Journal -- # batch_injector_factory = DefaultBatchInjectorFactory( block_cache=block_cache, state_view_factory=state_view_factory, signer=identity_signer) block_publisher = BlockPublisher( transaction_executor=transaction_executor, block_cache=block_cache, state_view_factory=state_view_factory, settings_cache=settings_cache, block_sender=block_sender, batch_sender=batch_sender, chain_head=block_store.chain_head, identity_signer=identity_signer, data_dir=data_dir, config_dir=config_dir, permission_verifier=permission_verifier, check_publish_block_frequency=0.1, batch_observers=[batch_tracker], batch_injector_factory=batch_injector_factory) block_publisher_batch_sender = block_publisher.batch_sender() block_validator = BlockValidator( block_cache=block_cache, state_view_factory=state_view_factory, transaction_executor=transaction_executor, identity_signer=identity_signer, data_dir=data_dir, config_dir=config_dir, permission_verifier=permission_verifier) chain_controller = ChainController( block_store=block_store, block_cache=block_cache, block_validator=block_validator, state_database=global_state_db, chain_head_lock=block_publisher.chain_head_lock, state_pruning_block_depth=state_pruning_block_depth, data_dir=data_dir, observers=[ event_broadcaster, receipt_store, batch_tracker, identity_observer, settings_observer ]) genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=transaction_executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_signer=identity_signer, data_dir=data_dir, config_dir=config_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender) responder = Responder(completer) completer.set_on_batch_received(block_publisher_batch_sender.send) completer.set_on_block_received(chain_controller.queue_block) completer.set_chain_has_block(chain_controller.has_block) # -- Register Message Handler -- # network_handlers.add( network_dispatcher, network_service, gossip, completer, responder, network_thread_pool, sig_pool, chain_controller.has_block, block_publisher.has_batch, permission_verifier, block_publisher, consensus_notifier) component_handlers.add( component_dispatcher, gossip, context_manager, transaction_executor, completer, block_store, batch_tracker, global_state_db, self.get_chain_head_state_root_hash, receipt_store, event_broadcaster, permission_verifier, component_thread_pool, client_thread_pool, sig_pool, block_publisher) # -- Store Object References -- # self._component_dispatcher = component_dispatcher self._component_service = component_service self._component_thread_pool = component_thread_pool self._network_dispatcher = network_dispatcher self._network_service = network_service self._network_thread_pool = network_thread_pool consensus_proxy = ConsensusProxy( block_cache=block_cache, chain_controller=chain_controller, block_publisher=block_publisher, gossip=gossip, identity_signer=identity_signer, settings_view_factory=SettingsViewFactory(state_view_factory), state_view_factory=state_view_factory) consensus_handlers.add( consensus_dispatcher, consensus_thread_pool, consensus_proxy) self._consensus_dispatcher = consensus_dispatcher self._consensus_service = consensus_service self._consensus_thread_pool = consensus_thread_pool self._client_thread_pool = client_thread_pool self._sig_pool = sig_pool self._context_manager = context_manager self._transaction_executor = transaction_executor self._genesis_controller = genesis_controller self._gossip = gossip self._block_publisher = block_publisher self._chain_controller = chain_controller self._block_validator = block_validator
class TestBlockPublisher(): ''' The block publisher has three main functions, and in these tests those functions are given the following wrappers for convenience: * on_batch_received -> receive_batches * on_chain_updated -> update_chain_head * on_check_publish_block -> publish_block After publishing a block, publish_block sends its block to the mock block sender, and that block is named result_block. This block is what is checked by the test assertions. The basic pattern for the publisher tests (with variations) is: 0) make a list of batches (usually in setUp); 1) receive the batches; 2) publish a block; 3) verify the block (checking that it contains the correct batches, or checking that it doesn't exist, or whatever). The publisher chain head might be updated several times in a test. ''' def __init__(self): self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.permission_verifier = MockPermissionVerifier() self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, settings_cache=SettingsCache( SettingsViewFactory( self.block_tree_manager.state_view_factory), ), block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, check_publish_block_frequency=0.1, batch_observers=[], permission_verifier=self.permission_verifier) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches() def verify_block(self, batches=None): if batches is None: batches = self.batches batch_count = None if batches is None else len(batches) # publisher functions def receive_batch(self, batch): self.publisher.on_batch_received(batch) def receive_batches(self, batches=None): if batches is None: batches = self.batches for batch in batches: self.receive_batch(batch) def publish_block(self): self.publisher.on_check_publish_block() self.result_block = self.block_sender.new_block self.block_sender.new_block = None def update_chain_head(self, head, committed=None, uncommitted=None): if head: self.block_tree_manager.block_store.update_chain([head]) self.publisher.on_chain_updated(chain_head=head, committed_batches=committed, uncommitted_batches=uncommitted) # batches def make_batch(self, missing_deps=False, txn_count=2): return self.block_tree_manager.generate_batch( txn_count=txn_count, missing_deps=missing_deps) def make_batches(self, batch_count=None, missing_deps=False): if batch_count is None: batch_count = self.batch_count return [ self.make_batch(missing_deps=missing_deps) for _ in range(batch_count) ] def make_batches_with_duplicate_txn(self): txns = [ self.batches[0].transactions[0], self.block_tree_manager.generate_transaction("nonce") ] return [self.block_tree_manager.generate_batch(txns=txns)]
class TestBlockPublisher(unittest.TestCase): ''' The block publisher has five main functions, and in these tests those functions are given the following wrappers for convenience: * on_batch_received -> receive_batches * on_chain_updated -> update_chain_head * initialize_block -> initialize_block * summarize_block -> summarize_block * finalize_block -> finalize_block Additionally, the publish_block is provided to call both initialize_block and finalize_block. After finalizing a block, finalize_block sends its block to the mock block sender, and that block is named result_block. This block is what is checked by the test assertions. The basic pattern for the publisher tests (with variations) is: 0) make a list of batches (usually in setUp); 1) receive the batches; 2) initialize a block; 3) finalize a block; 4) verify the block (checking that it contains the correct batches, or checking that it doesn't exist, etc.). ''' @unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory') def setUp(self, mock_batch_injector_factory): mock_batch_injector_factory.create_injectors.return_value = [] self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.permission_verifier = MockPermissionVerifier() self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches() def test_publish(self): ''' Publish a block with several batches ''' self.receive_batches() self.publish_block() self.verify_block() def test_receive_after_initialize(self): ''' Receive batches after initialization ''' self.initialize_block() self.receive_batches() self.finalize_block() self.verify_block() def test_summarize_block(self): ''' Initialize a block and summarize it ''' self.receive_batches() self.initialize_block() self.assertIsNotNone(self.summarize_block(), 'Expected block summary') def test_reject_double_initialization(self): ''' Test that you can't initialize a candidate block twice ''' self.initialize_block() with self.assertRaises( BlockInProgress, msg='Second initialization should have rejected'): self.initialize_block() def test_reject_finalize_without_initialize(self): ''' Test that no block is published if the block hasn't been initialized ''' self.receive_batches() with self.assertRaises(BlockNotInitialized, msg='Block should not be finalized'): self.finalize_block() def test_reject_duplicate_batches_from_receive(self): ''' Test that duplicate batches from on_batch_received are rejected ''' for _ in range(5): self.receive_batches() self.publish_block() self.verify_block() def test_reject_duplicate_batches_from_store(self): ''' Test that duplicate batches from block store are rejected ''' self.update_chain_head(head=self.init_chain_head, uncommitted=self.batches) self.receive_batches() self.publish_block() self.verify_block() def test_committed_batches(self): ''' Test that batches committed upon updating the chain head are not included in the next block. ''' self.update_chain_head(head=self.init_chain_head, committed=self.batches) new_batches = self.make_batches(batch_count=12) self.receive_batches(new_batches) self.publish_block() self.verify_block(new_batches) def test_uncommitted_batches(self): ''' Test that batches uncommitted upon updating the chain head are included in the next block. ''' self.update_chain_head(head=self.init_chain_head, uncommitted=self.batches) self.publish_block() self.verify_block() def test_empty_pending_queue(self): ''' Test that no block is published if the pending queue is empty ''' # try to publish with no pending queue (failing) with self.assertRaises(BlockEmpty, msg='Block should not be published'): self.publish_block() self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() self.finalize_block() self.verify_block() def test_missing_dependencies(self): ''' Test that no block is published with missing dependencies ''' self.batches = self.make_batches(missing_deps=True) self.receive_batches() # Block should be empty, since batches with missing deps aren't added with self.assertRaises(BlockEmpty, msg='Block should be empty'): self.publish_block() self.assert_no_block_published() @unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory') def test_batches_rejected_by_scheduler(self, mock_batch_injector_factory): ''' Test that no block is published with batches rejected by the scheduler ''' mock_batch_injector_factory.create_injectors.return_value = [] self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor( batch_execution_result=False), transaction_committed=( self.block_tree_manager.block_store.has_transaction), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.receive_batches() # Block should be empty since all batches are rejected with self.assertRaises(BlockEmpty, msg='Block should be empty'): self.publish_block() self.assert_no_block_published() @unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory') def test_max_block_size(self, mock_batch_injector_factory): ''' Test block publisher obeys the block size limits ''' mock_batch_injector_factory.create_injectors.return_value = [] # Create a publisher that has a state view # with a batch limit addr, value = CreateSetting('sawtooth.publisher.max_batches_per_block', 1) self.state_view_factory = MockStateViewFactory({addr: value}) self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() # try to publish with no pending queue (failing) for i in range(self.batch_count): self.publish_block() self.assert_block_published() self.update_chain_head(BlockWrapper(self.result_block)) self.verify_block([self.batches[i]]) def test_duplicate_transactions(self): ''' Test discards batches that have duplicate transactions in them. ''' # receive batches, then try again (succeeding) self.batches = self.batches[1:2] self.receive_batches() self.publish_block() self.assert_block_published() self.update_chain_head(BlockWrapper(self.result_block)) self.verify_block() # build a new set of batches with the same transactions in them self.batches = self.make_batches_with_duplicate_txn() self.receive_batches() with self.assertRaises(BlockEmpty, msg='Block should be empty'): self.publish_block() self.assert_no_block_published() # block should be empty after batch # with duplicate transaction is dropped. def test_batch_injection_start_block(self): ''' Test that the batch is injected at the beginning of the block. ''' injected_batch = self.make_batch() self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, permission_verifier=self.permission_verifier, batch_observers=[], batch_injector_factory=MockBatchInjectorFactory(injected_batch)) self.receive_batches() self.publish_block() self.assert_batch_in_block(injected_batch) @unittest.mock.patch('test_journal.mock.MockBatchInjectorFactory') def test_validation_rules_reject_batches(self, mock_batch_injector_factory): """Test that a batch is not added to the block if it will violate the block validation rules. It does the following: - Sets the block_validation_rules to limit the number of 'test' transactions to 1 - creates two batches, limited to 1 transaction each, and receives them - verifies that only the first batch was committed to the block """ addr, value = CreateSetting( 'sawtooth.validator.block_validation_rules', 'NofX:1,test') self.state_view_factory = MockStateViewFactory({addr: value}) mock_batch_injector_factory.create_injectors.return_value = [] batch1 = self.make_batch(txn_count=1) batch2 = self.make_batch(txn_count=1) self.publisher = BlockPublisher( block_manager=self.block_tree_manager.block_manager, transaction_executor=MockTransactionExecutor(), transaction_committed=( self.block_tree_manager.block_store.has_transaction), batch_committed=self.block_tree_manager.block_store.has_batch, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, chain_head=self.block_tree_manager.chain_head.block, identity_signer=self.block_tree_manager.identity_signer, data_dir=None, config_dir=None, batch_observers=[], permission_verifier=self.permission_verifier, batch_injector_factory=mock_batch_injector_factory) self.receive_batches(batches=[batch1, batch2]) self.publish_block() self.assert_block_batch_count(1) self.assert_batch_in_block(batch1) # assertions def assert_block_published(self): self.assertIsNotNone(self.result_block, 'Block should have been published') def assert_no_block_published(self): self.assertIsNone(self.result_block, 'Block should not have been published') def assert_batch_in_block(self, batch): self.assertIn(batch, tuple(self.result_block.batches), 'Batch not in block') def assert_batches_in_block(self, batches=None): if batches is None: batches = self.batches for batch in batches: self.assert_batch_in_block(batch) def assert_block_batch_count(self, batch_count=None): if batch_count is None: batch_count = self.batch_count self.assertEqual(len(self.result_block.batches), batch_count, 'Wrong batch count in block') def verify_block(self, batches=None): if batches is None: batches = self.batches batch_count = None if batches is None else len(batches) self.assert_block_published() self.assert_batches_in_block(batches) self.assert_block_batch_count(batch_count) self.result_block = None # publisher functions def receive_batch(self, batch): self.publisher.on_batch_received(batch) def receive_batches(self, batches=None): if batches is None: batches = self.batches for batch in batches: self.receive_batch(batch) def initialize_block(self): self.publisher.initialize_block(self.block_tree_manager.chain_head) def summarize_block(self): return self.publisher.summarize_block() def finalize_block(self): self.publisher.finalize_block("") self.result_block = self.block_sender.new_block self.block_sender.new_block = None def publish_block(self): self.initialize_block() self.finalize_block() def update_chain_head(self, head, committed=None, uncommitted=None): if head: self.block_tree_manager.block_store.update_chain([head]) self.publisher.on_chain_updated(chain_head=head, committed_batches=committed, uncommitted_batches=uncommitted) # batches def make_batch(self, missing_deps=False, txn_count=2): return self.block_tree_manager.generate_batch( txn_count=txn_count, missing_deps=missing_deps) def make_batches(self, batch_count=None, missing_deps=False): if batch_count is None: batch_count = self.batch_count return [ self.make_batch(missing_deps=missing_deps) for _ in range(batch_count) ] def make_batches_with_duplicate_txn(self): txns = [ self.batches[0].transactions[0], self.block_tree_manager.generate_transaction("nonce") ] return [self.block_tree_manager.generate_batch(txns=txns)]
def test_publish(self, args=sys.argv[1:]): gossip = MockNetwork() LOGGER.info(self.blocks) publisher = BlockPublisher( consensus=TestModePublisher(), transaction_executor=MockTransactionExecutor(), send_message=gossip.send_message, squash_handler=None) LOGGER.info("1") # initial load of existing state publisher.on_chain_updated(self.blocks.chain_head.block, [], []) LOGGER.info("2") # repeat as necessary batch = Batch() publisher.on_batch_received(batch) LOGGER.info("3") # current dev_mode consensus always claims blocks when asked. # this will be called on a polling every so often or possibly triggered # by events in the consensus it's self ... TBD publisher.on_check_publish_block() LOGGER.info("4") LOGGER.info(self.blocks) # repeat as necessary batch = Batch() publisher.on_batch_received(batch) publisher.on_check_publish_block() LOGGER.info(self.blocks)
class Journal(object): """ Manages the block chain, This responsibility boils down 1) to evaluating new blocks to determine if they should extend or replace the current chain. Handled by the ChainController/ 2) Claiming new blocks, handled by the BlockPublisher This object provides the threading and event queue for the processors. """ class _ChainThread(Thread): def __init__(self, chain_controller, block_queue): Thread.__init__(self) self._block_publisher = chain_controller self._block_queue = block_queue self._exit = False def run(self): while True: try: block = self._block_queue.get(timeout=0.1) self._block_publisher.on_block_received(block) except queue.Empty: time.sleep(0.1) if self._exit: return def stop(self): self._exit = True class _PublisherThread(Thread): def __init__(self, block_publisher, batch_queue): Thread.__init__(self) self._block_publisher = block_publisher self._batch_queue = batch_queue self._exit = False def run(self): while True: try: batch = self._batch_queue.get(timeout=0.1) self._block_publisher.on_batch_received(batch) except queue.Empty: time.sleep(0.1) self._block_publisher.on_check_publish_block() if self._exit: return def stop(self): self._exit = True def __init__(self, consensus, block_store, send_message, transaction_executor, squash_handler, first_state_root): self._consensus = consensus self._block_store = block_store self._send_message = send_message self._squash_handler = squash_handler self._block_publisher = BlockPublisher( consensus=consensus.BlockPublisher(), transaction_executor=transaction_executor, send_message=send_message, squash_handler=squash_handler) self._batch_queue = queue.Queue() self._publisher_thread = self._PublisherThread(self._block_publisher, self._batch_queue) # HACK until genesis tool is working if "chain_head_id" not in self._block_store: genesis_block = BlockState( block_wrapper=self._block_publisher.generate_genesis_block(), weight=0, status=BlockStatus.Valid) genesis_block.block.set_state_hash(first_state_root) self._block_store[genesis_block.block.header_signature] = \ genesis_block self._block_store["chain_head_id"] = \ genesis_block.block.header_signature self._block_publisher.on_chain_updated(genesis_block.block) LOGGER.info("Journal created genesis block: %s", genesis_block.block.header_signature) self._chain_controller = ChainController( consensus=consensus.BlockVerifier(), block_store=block_store, send_message=send_message, executor=ThreadPoolExecutor(1), transaction_executor=transaction_executor, on_chain_updated=self._block_publisher.on_chain_updated, squash_handler=self._squash_handler) self._block_queue = queue.Queue() self._chain_thread = self._ChainThread(self._chain_controller, self._block_queue) def get_current_root(self): # return self._block_publisher._chain_head.state_root_hash return self._chain_controller.chain_head.block.state_root_hash def start(self): # TBD do load activities.... # TBD transfer activities - request chain-head from # network self._publisher_thread.start() self._chain_thread.start() def stop(self): # time to murder the child threads. First ask politely for # suicide self._publisher_thread.stop() self._chain_thread.stop() def on_block_received(self, block): self._block_queue.put(block) def on_batch_received(self, batch): self._batch_queue.put(batch) def on_block_request(self, block_id): if block_id in self._block_store: self._send_message(self._block_store[block_id].block)
def __init__(self, bind_network, bind_component, bind_consensus, endpoint, peering, seeds_list, peer_list, data_dir, config_dir, identity_signer, scheduler_type, permissions, minimum_peer_connectivity, maximum_peer_connectivity, state_pruning_block_depth, network_public_key=None, network_private_key=None, roles=None): """Constructs a validator instance. Args: bind_network (str): the network endpoint bind_component (str): the component endpoint endpoint (str): the zmq-style URI of this validator's publically reachable endpoint peering (str): The type of peering approach. Either 'static' or 'dynamic'. In 'static' mode, no attempted topology buildout occurs -- the validator only attempts to initiate peering connections with endpoints specified in the peer_list. In 'dynamic' mode, the validator will first attempt to initiate peering connections with endpoints specified in the peer_list and then attempt to do a topology buildout starting with peer lists obtained from endpoints in the seeds_list. In either mode, the validator will accept incoming peer requests up to max_peers. seeds_list (list of str): a list of addresses to connect to in order to perform the initial topology buildout peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory config_dir (str): path to the config directory identity_signer (str): cryptographic signer the validator uses for signing """ # -- Setup Global State Database and Factory -- # global_state_db_filename = os.path.join( data_dir, 'merkle-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('global state database file is %s', global_state_db_filename) global_state_db = NativeLmdbDatabase( global_state_db_filename, indexes=MerkleDatabase.create_index_configuration()) state_view_factory = StateViewFactory(global_state_db) # -- Setup Receipt Store -- # receipt_db_filename = os.path.join( data_dir, 'txn_receipts-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('txn receipt store file is %s', receipt_db_filename) receipt_db = LMDBNoLockDatabase(receipt_db_filename, 'c') receipt_store = TransactionReceiptStore(receipt_db) # -- Setup Block Store -- # block_db_filename = os.path.join( data_dir, 'block-{}.lmdb'.format(bind_network[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = IndexedDatabase( block_db_filename, BlockStore.serialize_block, BlockStore.deserialize_block, flag='c', indexes=BlockStore.create_index_configuration()) block_store = BlockStore(block_db) # The cache keep time for the journal's block cache must be greater # than the cache keep time used by the completer. base_keep_time = 1200 block_cache = BlockCache(block_store, keep_time=int(base_keep_time * 9 / 8), purge_frequency=30) # -- Setup Thread Pools -- # component_thread_pool = InstrumentedThreadPoolExecutor( max_workers=10, name='Component') network_thread_pool = InstrumentedThreadPoolExecutor(max_workers=10, name='Network') client_thread_pool = InstrumentedThreadPoolExecutor(max_workers=5, name='Client') sig_pool = InstrumentedThreadPoolExecutor(max_workers=3, name='Signature') # -- Setup Dispatchers -- # component_dispatcher = Dispatcher() network_dispatcher = Dispatcher() # -- Setup Services -- # component_service = Interconnect(bind_component, component_dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True, max_future_callback_workers=10) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] secure = False if network_public_key is not None and network_private_key is not None: secure = True network_service = Interconnect(bind_network, dispatcher=network_dispatcher, zmq_identity=zmq_identity, secured=secure, server_public_key=network_public_key, server_private_key=network_private_key, heartbeat=True, public_endpoint=endpoint, connection_timeout=120, max_incoming_connections=100, max_future_callback_workers=10, authorize=True, signer=identity_signer, roles=roles) # -- Setup Transaction Execution Platform -- # context_manager = ContextManager(global_state_db) batch_tracker = BatchTracker(block_store) settings_cache = SettingsCache( SettingsViewFactory(state_view_factory), ) transaction_executor = TransactionExecutor( service=component_service, context_manager=context_manager, settings_view_factory=SettingsViewFactory(state_view_factory), scheduler_type=scheduler_type, invalid_observers=[batch_tracker]) component_service.set_check_connections( transaction_executor.check_connections) event_broadcaster = EventBroadcaster(component_service, block_store, receipt_store) # -- Setup P2P Networking -- # gossip = Gossip(network_service, settings_cache, lambda: block_store.chain_head, block_store.chain_head_state_root, endpoint=endpoint, peering_mode=peering, initial_seed_endpoints=seeds_list, initial_peer_endpoints=peer_list, minimum_peer_connectivity=minimum_peer_connectivity, maximum_peer_connectivity=maximum_peer_connectivity, topology_check_frequency=1) completer = Completer(block_store, gossip, cache_keep_time=base_keep_time, cache_purge_frequency=30, requested_keep_time=300) block_sender = BroadcastBlockSender(completer, gossip) batch_sender = BroadcastBatchSender(completer, gossip) chain_id_manager = ChainIdManager(data_dir) identity_view_factory = IdentityViewFactory( StateViewFactory(global_state_db)) id_cache = IdentityCache(identity_view_factory) # -- Setup Permissioning -- # permission_verifier = PermissionVerifier( permissions, block_store.chain_head_state_root, id_cache) identity_observer = IdentityObserver(to_update=id_cache.invalidate, forked=id_cache.forked) settings_observer = SettingsObserver( to_update=settings_cache.invalidate, forked=settings_cache.forked) # -- Consensus Engine -- # consensus_thread_pool = InstrumentedThreadPoolExecutor( max_workers=3, name='Consensus') consensus_dispatcher = Dispatcher() consensus_service = Interconnect(bind_consensus, consensus_dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True, max_future_callback_workers=10) consensus_notifier = ConsensusNotifier(consensus_service) # -- Setup Journal -- # batch_injector_factory = DefaultBatchInjectorFactory( block_cache=block_cache, state_view_factory=state_view_factory, signer=identity_signer) block_publisher = BlockPublisher( transaction_executor=transaction_executor, block_cache=block_cache, state_view_factory=state_view_factory, settings_cache=settings_cache, block_sender=block_sender, batch_sender=batch_sender, chain_head=block_store.chain_head, identity_signer=identity_signer, data_dir=data_dir, config_dir=config_dir, permission_verifier=permission_verifier, check_publish_block_frequency=0.1, batch_observers=[batch_tracker], batch_injector_factory=batch_injector_factory) block_publisher_batch_sender = block_publisher.batch_sender() block_validator = BlockValidator( block_cache=block_cache, state_view_factory=state_view_factory, transaction_executor=transaction_executor, identity_signer=identity_signer, data_dir=data_dir, config_dir=config_dir, permission_verifier=permission_verifier) chain_controller = ChainController( block_store=block_store, block_cache=block_cache, block_validator=block_validator, state_database=global_state_db, chain_head_lock=block_publisher.chain_head_lock, state_pruning_block_depth=state_pruning_block_depth, data_dir=data_dir, observers=[ event_broadcaster, receipt_store, batch_tracker, identity_observer, settings_observer ]) genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=transaction_executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_signer=identity_signer, data_dir=data_dir, config_dir=config_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender) responder = Responder(completer) completer.set_on_batch_received(block_publisher_batch_sender.send) completer.set_on_block_received(chain_controller.queue_block) completer.set_chain_has_block(chain_controller.has_block) # -- Register Message Handler -- # network_handlers.add(network_dispatcher, network_service, gossip, completer, responder, network_thread_pool, sig_pool, chain_controller.has_block, block_publisher.has_batch, permission_verifier, block_publisher, consensus_notifier) component_handlers.add(component_dispatcher, gossip, context_manager, transaction_executor, completer, block_store, batch_tracker, global_state_db, self.get_chain_head_state_root_hash, receipt_store, event_broadcaster, permission_verifier, component_thread_pool, client_thread_pool, sig_pool, block_publisher) # -- Store Object References -- # self._component_dispatcher = component_dispatcher self._component_service = component_service self._component_thread_pool = component_thread_pool self._network_dispatcher = network_dispatcher self._network_service = network_service self._network_thread_pool = network_thread_pool consensus_proxy = ConsensusProxy( block_cache=block_cache, chain_controller=chain_controller, block_publisher=block_publisher, gossip=gossip, identity_signer=identity_signer, settings_view_factory=SettingsViewFactory(state_view_factory), state_view_factory=state_view_factory) consensus_handlers.add(consensus_dispatcher, consensus_thread_pool, consensus_proxy) self._consensus_dispatcher = consensus_dispatcher self._consensus_service = consensus_service self._consensus_thread_pool = consensus_thread_pool self._client_thread_pool = client_thread_pool self._sig_pool = sig_pool self._context_manager = context_manager self._transaction_executor = transaction_executor self._genesis_controller = genesis_controller self._gossip = gossip self._block_publisher = block_publisher self._chain_controller = chain_controller self._block_validator = block_validator
class TestBlockPublisher(unittest.TestCase): ''' The block publisher has three main functions, and in these tests those functions are given the following wrappers for convenience: * on_batch_received -> receive_batches * on_chain_updated -> update_chain_head * on_check_publish_block -> publish_block After publishing a block, publish_block sends its block to the mock block sender, and that block is named result_block. This block is what is checked by the test assertions. The basic pattern for the publisher tests (with variations) is: 0) make a list of batches (usually in setUp); 1) receive the batches; 2) publish a block; 3) verify the block (checking that it contains the correct batches, or checking that it doesn't exist, or whatever). The publisher chain head might be updated several times in a test. ''' def setUp(self): self.block_tree_manager = BlockTreeManager() self.block_sender = MockBlockSender() self.batch_sender = MockBatchSender() self.state_view_factory = MockStateViewFactory({}) self.publisher = BlockPublisher( transaction_executor=MockTransactionExecutor(), block_cache=self.block_tree_manager.block_cache, state_view_factory=self.state_view_factory, block_sender=self.block_sender, batch_sender=self.batch_sender, squash_handler=None, chain_head=self.block_tree_manager.chain_head, identity_signing_key=self.block_tree_manager.identity_signing_key, data_dir=None) self.init_chain_head = self.block_tree_manager.chain_head self.result_block = None # A list of batches is created at the beginning of each test. # The test assertions and the publisher function wrappers # take these batches as a default argument. self.batch_count = 8 self.batches = self.make_batches() def test_publish(self): ''' Publish a block with several batches ''' self.receive_batches() self.publish_block() self.verify_block() def test_reject_duplicate_batches_from_receive(self): ''' Test that duplicate batches from on_batch_received are rejected ''' for i in range(5): self.receive_batches() self.publish_block() self.verify_block() def test_reject_duplicate_batches_from_store(self): ''' Test that duplicate batches from block store are rejected ''' self.update_chain_head(None) self.update_chain_head(head=self.init_chain_head, uncommitted=self.batches) self.receive_batches() self.publish_block() self.verify_block() def test_no_chain_head(self): ''' Test that nothing gets published with a null chain head, then test that publishing resumes after updating ''' self.update_chain_head(None) self.receive_batches() # try to publish block (failing) self.publish_block() self.assert_no_block_published() # reset chain head several times, # making sure batches remain queued for i in range(3): self.update_chain_head(None) self.update_chain_head(self.init_chain_head) # try to publish block (succeeding) self.publish_block() self.verify_block() def test_committed_batches(self): ''' Test that batches committed upon updating the chain head are not included in the next block. ''' self.update_chain_head(None) self.update_chain_head(head=self.init_chain_head, committed=self.batches) new_batches = self.make_batches(batch_count=12) self.receive_batches(new_batches) self.publish_block() self.verify_block(new_batches) def test_uncommitted_batches(self): ''' Test that batches uncommitted upon updating the chain head are included in the next block. ''' self.update_chain_head(None) self.update_chain_head(head=self.init_chain_head, uncommitted=self.batches) self.publish_block() self.verify_block() def test_empty_pending_queue(self): ''' Test that no block is published if the pending queue is empty ''' # try to publish with no pending queue (failing) self.publish_block() self.assert_no_block_published() # receive batches, then try again (succeeding) self.receive_batches() self.publish_block() self.verify_block() # assertions def assert_block_published(self): self.assertIsNotNone(self.result_block, 'Block should have been published') def assert_no_block_published(self): self.assertIsNone(self.result_block, 'Block should not have been published') def assert_batch_in_block(self, batch): self.assertIn(batch, tuple(self.result_block.batches), 'Batch not in block') def assert_batches_in_block(self, batches=None): if batches is None: batches = self.batches for batch in batches: self.assert_batch_in_block(batch) def assert_block_batch_count(self, batch_count=None): if batch_count is None: batch_count = self.batch_count self.assertEqual(len(self.result_block.batches), batch_count, 'Wrong batch count in block') def verify_block(self, batches=None): if batches is None: batches = self.batches batch_count = None if batches is None else len(batches) self.assert_block_published() self.assert_batches_in_block(batches) self.assert_block_batch_count(batch_count) self.result_block = None # publisher functions def receive_batch(self, batch): self.publisher.on_batch_received(batch) def receive_batches(self, batches=None): if batches is None: batches = self.batches for batch in batches: self.receive_batch(batch) def publish_block(self): self.publisher.on_check_publish_block() self.result_block = self.block_sender.new_block def update_chain_head(self, head, committed=None, uncommitted=None): self.publisher.on_chain_updated(chain_head=head, committed_batches=committed, uncommitted_batches=uncommitted) # batches def make_batch(self, payload='batch'): return self.block_tree_manager._generate_batch(payload) def make_batches(self, batch_count=None): if batch_count is None: batch_count = self.batch_count return [self.make_batch('batch_' + str(i)) for i in range(batch_count)]