class Validator(object): def __init__(self, network_endpoint, component_endpoint, peer_list): db_filename = os.path.join(os.path.expanduser('~'), 'merkle.lmdb') LOGGER.debug('database file is %s', db_filename) lmdb = LMDBNoLockDatabase(db_filename, 'n') context_manager = ContextManager(lmdb) block_db_filename = os.path.join(os.path.expanduser('~'), 'block.lmdb') LOGGER.debug('block store file is %s', block_db_filename) block_store = LMDBNoLockDatabase(block_db_filename, 'n') self._service = ValidatorService(component_endpoint) # setup network dispatcher = Dispatcher() faux_network = FauxNetwork(dispatcher=dispatcher) identity = "{}-{}".format(socket.gethostname(), os.getpid()).encode('ascii') self._network = Network(identity, network_endpoint, peer_list, dispatcher=dispatcher) # Create and configure journal executor = TransactionExecutor(self._service, context_manager) self._journal = Journal( consensus=dev_mode_consensus, block_store={}, # FIXME - block_store=block_store # -- need to serialize blocks to dicts send_message=faux_network.send_message, transaction_executor=executor, squash_handler=context_manager.get_squash_handler(), first_state_root=context_manager.get_first_root()) dispatcher.on_batch_received = \ self._journal.on_batch_received dispatcher.on_block_received = \ self._journal.on_block_received dispatcher.on_block_request = \ self._journal.on_block_request self._service.add_handler('default', DefaultHandler()) self._service.add_handler('state/getrequest', state.GetHandler(context_manager)) self._service.add_handler('state/setrequest', state.SetHandler(context_manager)) self._service.add_handler('tp/register', ProcessorRegisterHandler(self._service)) self._service.add_handler('system/load', SystemLoadHandler(faux_network)) def start(self): self._service.start() self._journal.start() def stop(self): self._service.stop() self._journal.stop()
def test_publish_block(self): """ Test that the Journal will produce blocks and consume those blocks to extend the chain. :return: """ # construction and wire the journal to the # gossip layer. LOGGER.info("test_publish_block") block_store = {} journal = None try: journal = Journal( consensus=test_mode_consensus, block_store=block_store, send_message=self.gossip.send_message, transaction_executor=self.txn_executor, squash_handler=None, first_state_root="000000") self.gossip.on_batch_received = \ journal.on_batch_received self.gossip.on_block_received = \ journal.on_block_received self.gossip.on_block_request = \ journal.on_block_request journal.start() # feed it a batch batch = Batch() journal.on_batch_received(batch) # wait for a block message to arrive should be soon to = TimeOut(2) while len(self.gossip.messages) == 0: time.sleep(0.1) LOGGER.info("Batches: %s", self.gossip.messages) self.assertTrue(len(self.gossip.messages) != 0) block = self.gossip.messages[0] # dispatch the message self.gossip.dispatch_messages() # wait for the chain_head to be updated. to = TimeOut(2) while block_store['chain_head_id'] != block.header_signature: time.sleep(0.1) self.assertTrue(block_store['chain_head_id'] == block.header_signature) finally: if journal is not None: journal.stop()
def test_publish_block(self): """ Test that the Journal will produce blocks and consume those blocks to extend the chain. :return: """ # construction and wire the journal to the # gossip layer. btm = BlockTreeManager() journal = None try: journal = Journal( block_store=btm.block_store, block_cache=btm.block_cache, state_view_factory=StateViewFactory(DictDatabase()), block_sender=self.block_sender, batch_sender=self.batch_sender, transaction_executor=self.txn_executor, squash_handler=None, identity_signing_key=btm.identity_signing_key, chain_id_manager=None, state_delta_processor=self.state_delta_processor, data_dir=None, config_dir=None ) self.gossip.on_batch_received = journal.on_batch_received self.gossip.on_block_received = journal.on_block_received journal.start() # feed it a batch batch = Batch() journal.on_batch_received(batch) wait_until(lambda: self.block_sender.new_block is not None, 2) self.assertTrue(self.block_sender.new_block is not None) block = BlockWrapper(self.block_sender.new_block) journal.on_block_received(block) # wait for the chain_head to be updated. wait_until(lambda: btm.chain_head.identifier == block.identifier, 2) self.assertTrue(btm.chain_head.identifier == block.identifier) finally: if journal is not None: journal.stop()
def test_publish_block(self): """ Test that the Journal will produce blocks and consume those blocks to extend the chain. :return: """ # construction and wire the journal to the # gossip layer. LOGGER.info("test_publish_block") btm = BlockTreeManager() journal = None try: journal = Journal( consensus=test_mode_consensus, block_store=btm.block_store.store, block_cache=btm.block_cache, block_sender=self.block_sender, transaction_executor=self.txn_executor, squash_handler=None ) self.gossip.on_batch_received = \ journal.on_batch_received self.gossip.on_block_received = \ journal.on_block_received journal.start() # feed it a batch batch = Batch() journal.on_batch_received(batch) wait_until(lambda: self.block_sender.new_block is not None, 2) self.assertTrue(self.block_sender.new_block is not None) block = BlockWrapper(self.block_sender.new_block) journal.on_block_received(block) # wait for the chain_head to be updated. wait_until(lambda: btm.chain_head.identifier == block.identifier, 2) self.assertTrue(btm.chain_head.identifier == block.identifier) finally: if journal is not None: journal.stop()
def test_publish_block(self): """ Test that the Journal will produce blocks and consume those blocks to extend the chain. :return: """ # construction and wire the journal to the # gossip layer. btm = BlockTreeManager() journal = None try: journal = Journal(consensus=test_mode_consensus, block_store=btm.block_store.store, block_cache=btm.block_cache, block_sender=self.block_sender, transaction_executor=self.txn_executor, squash_handler=None) self.gossip.on_batch_received = journal.on_batch_received self.gossip.on_block_received = journal.on_block_received journal.start() # feed it a batch batch = Batch() journal.on_batch_received(batch) wait_until(lambda: self.block_sender.new_block is not None, 2) self.assertTrue(self.block_sender.new_block is not None) block = BlockWrapper(self.block_sender.new_block) journal.on_block_received(block) # wait for the chain_head to be updated. wait_until(lambda: btm.chain_head.identifier == block.identifier, 2) self.assertTrue(btm.chain_head.identifier == block.identifier) finally: if journal is not None: journal.stop()
class Validator(object): def __init__(self, network_endpoint, component_endpoint, peer_list, data_dir, identity_signing_key): """Constructs a validator instance. Args: network_endpoint (str): the network endpoint component_endpoint (str): the component endpoint peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory key_dir (str): path to the key directory """ db_filename = os.path.join( data_dir, 'merkle-{}.lmdb'.format(network_endpoint[-2:])) LOGGER.debug('database file is %s', db_filename) merkle_db = LMDBNoLockDatabase(db_filename, 'c') context_manager = ContextManager(merkle_db) self._context_manager = context_manager state_view_factory = StateViewFactory(merkle_db) block_db_filename = os.path.join( data_dir, 'block-{}.lmdb'.format(network_endpoint[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = LMDBNoLockDatabase(block_db_filename, 'c') block_store = BlockStore(block_db) # setup network self._dispatcher = Dispatcher() thread_pool = ThreadPoolExecutor(max_workers=10) process_pool = ProcessPoolExecutor(max_workers=3) self._thread_pool = thread_pool self._process_pool = process_pool self._service = Interconnect(component_endpoint, self._dispatcher, secured=False, heartbeat=False, max_incoming_connections=20) executor = TransactionExecutor(service=self._service, context_manager=context_manager, config_view_factory=ConfigViewFactory( StateViewFactory(merkle_db))) self._executor = executor zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] network_thread_pool = ThreadPoolExecutor(max_workers=10) self._network_thread_pool = network_thread_pool self._network_dispatcher = Dispatcher() # Server public and private keys are hardcoded here due to # the decision to avoid having separate identities for each # validator's server socket. This is appropriate for a public # network. For a permissioned network with requirements for # server endpoint authentication at the network level, this can # be augmented with a local lookup service for side-band provided # endpoint, public_key pairs and a local configuration option # for 'server' side private keys. self._network = Interconnect( network_endpoint, dispatcher=self._network_dispatcher, zmq_identity=zmq_identity, secured=True, server_public_key=b'wFMwoOt>yFqI/ek.G[tfMMILHWw#vXB[Sv}>l>i)', server_private_key=b'r&oJ5aQDj4+V]p2:Lz70Eu0x#m%IwzBdP(}&hWM*', heartbeat=True, connection_timeout=30, max_incoming_connections=100) self._gossip = Gossip(self._network, initial_peer_endpoints=peer_list) completer = Completer(block_store, self._gossip) block_sender = BroadcastBlockSender(completer, self._gossip) batch_sender = BroadcastBatchSender(completer, self._gossip) chain_id_manager = ChainIdManager(data_dir) # Create and configure journal self._journal = Journal( block_store=block_store, state_view_factory=StateViewFactory(merkle_db), block_sender=block_sender, batch_sender=batch_sender, transaction_executor=executor, squash_handler=context_manager.get_squash_handler(), identity_signing_key=identity_signing_key, chain_id_manager=chain_id_manager, data_dir=data_dir, check_publish_block_frequency=0.1, block_cache_purge_frequency=30, block_cache_keep_time=300) self._genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_key=identity_signing_key, data_dir=data_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender) responder = Responder(completer) completer.set_on_batch_received(self._journal.on_batch_received) completer.set_on_block_received(self._journal.on_block_received) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, processor_handlers.ProcessorRegisterHandler(executor.processors), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_UNREGISTER_REQUEST, processor_handlers.ProcessorUnRegisterHandler(executor.processors), thread_pool) # Set up base network handlers self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_PING, PingHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_CONNECT, ConnectHandler(network=self._network), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_DISCONNECT, DisconnectHandler(network=self._network), network_thread_pool) # Set up gossip handlers self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(gossip=self._gossip), network_thread_pool) # GOSSIP_MESSAGE 1) Sends acknowledgement to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(), network_thread_pool) # GOSSIP_MESSAGE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, signature_verifier.GossipMessageSignatureVerifier(), process_pool) # GOSSIP_MESSAGE 3) Determines if we should broadcast the # message to our peers. It is important that this occur prior # to the sending of the message to the completer, as this step # relies on whether the gossip message has previously been # seen by the validator to determine whether or not forwarding # should occur self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipBroadcastHandler(gossip=self._gossip, completer=completer), network_thread_pool) # GOSSIP_MESSAGE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, CompleterGossipHandler(completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_REQUEST, BlockResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, GossipBlockResponseHandler(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, signature_verifier.GossipBlockResponseSignatureVerifier(), process_pool) # GOSSIP_BLOCK_RESPONSE 3) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, CompleterGossipBlockResponseHandler(completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_BATCH_ID_REQUEST, BatchByBatchIdResponderHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_TRANSACTION_ID_REQUEST, BatchByTransactionIdResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BATCH_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, GossipBatchResponseHandler(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, signature_verifier.GossipBatchResponseSignatureVerifier(), process_pool) # GOSSIP_BATCH_RESPONSE 3) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, CompleterGossipBatchResponseHandler(completer), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, signature_verifier.BatchListSignatureVerifier(), process_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, CompleterBatchListBroadcastHandler(completer, self._gossip), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, client_handlers.BatchSubmitFinisher( self._journal.get_block_store(), completer.batch_cache), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_STATUS_REQUEST, client_handlers.BatchStatusRequest(self._journal.get_block_store(), completer.batch_cache), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequest(merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequest(merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST, client_handlers.BlockListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST, client_handlers.BlockGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_LIST_REQUEST, client_handlers.BatchListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_GET_REQUEST, client_handlers.BatchGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, client_handlers.StateCurrentRequest( self._journal.get_current_root), thread_pool) def start(self): self._dispatcher.start() self._service.start() if self._genesis_controller.requires_genesis(): self._genesis_controller.start(self._start) else: self._start() def _start(self): self._network_dispatcher.start() self._network.start() self._gossip.start() self._journal.start() signal_event = threading.Event() signal.signal(signal.SIGTERM, lambda sig, fr: signal_event.set()) # This is where the main thread will be during the bulk of the # validator's life. while not signal_event.is_set(): signal_event.wait(timeout=20) def stop(self): self._network.stop() self._service.stop() self._process_pool.shutdown(wait=True) self._network_thread_pool.shutdown(wait=True) self._thread_pool.shutdown(wait=True) self._executor.stop() self._context_manager.stop() self._journal.stop() threads = threading.enumerate() # This will remove the MainThread, which will exit when we exit with # a sys.exit() or exit of main(). threads.remove(threading.current_thread()) # Several Thread subclasses have a stop method not defined # in superclass. for t in threads: if hasattr(t, 'stop'): t.stop() while len(threads) > 0: if len(threads) < 4: LOGGER.info( "remaining threads: %s", ", ".join([ "{} ({})".format(x.name, x.__class__.__name__) for x in threads ])) for t in threads.copy(): if not t.is_alive(): t.join() threads.remove(t) if len(threads) > 0: time.sleep(1) LOGGER.info("All threads have been stopped and joined")
class Validator(object): def __init__(self, network_endpoint, component_endpoint, peer_list, data_dir, identity_signing_key): """Constructs a validator instance. Args: network_endpoint (str): the network endpoint component_endpoint (str): the component endpoint peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory key_dir (str): path to the key directory """ db_filename = os.path.join(data_dir, 'merkle-{}.lmdb'.format( network_endpoint[-2:])) LOGGER.debug('database file is %s', db_filename) merkle_db = LMDBNoLockDatabase(db_filename, 'n') context_manager = ContextManager(merkle_db) state_view_factory = StateViewFactory(merkle_db) block_db_filename = os.path.join(data_dir, 'block-{}.lmdb'.format( network_endpoint[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = LMDBNoLockDatabase(block_db_filename, 'n') block_store = BlockStore(block_db) # setup network self._dispatcher = Dispatcher() thread_pool = ThreadPoolExecutor(max_workers=10) process_pool = ProcessPoolExecutor(max_workers=3) self._service = Interconnect(component_endpoint, self._dispatcher, secured=False, heartbeat=False, max_incoming_connections=20) executor = TransactionExecutor(service=self._service, context_manager=context_manager, config_view_factory=ConfigViewFactory( StateViewFactory(merkle_db))) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] network_thread_pool = ThreadPoolExecutor(max_workers=10) self._network_dispatcher = Dispatcher() # Server public and private keys are hardcoded here due to # the decision to avoid having separate identities for each # validator's server socket. This is appropriate for a public # network. For a permissioned network with requirements for # server endpoint authentication at the network level, this can # be augmented with a local lookup service for side-band provided # endpoint, public_key pairs and a local configuration option # for 'server' side private keys. self._network = Interconnect( network_endpoint, dispatcher=self._network_dispatcher, zmq_identity=zmq_identity, secured=True, server_public_key=b'wFMwoOt>yFqI/ek.G[tfMMILHWw#vXB[Sv}>l>i)', server_private_key=b'r&oJ5aQDj4+V]p2:Lz70Eu0x#m%IwzBdP(}&hWM*', heartbeat=True, connection_timeout=30, max_incoming_connections=100) self._gossip = Gossip(self._network, initial_peer_endpoints=peer_list) completer = Completer(block_store, self._gossip) block_sender = BroadcastBlockSender(completer, self._gossip) batch_sender = BroadcastBatchSender(completer, self._gossip) chain_id_manager = ChainIdManager(data_dir) # Create and configure journal self._journal = Journal( block_store=block_store, state_view_factory=StateViewFactory(merkle_db), block_sender=block_sender, batch_sender=batch_sender, transaction_executor=executor, squash_handler=context_manager.get_squash_handler(), identity_signing_key=identity_signing_key, chain_id_manager=chain_id_manager, data_dir=data_dir, ) self._genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_key=identity_signing_key, data_dir=data_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender ) responder = Responder(completer) completer.set_on_batch_received(self._journal.on_batch_received) completer.set_on_block_received(self._journal.on_block_received) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, processor_handlers.ProcessorRegisterHandler(executor.processors), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_UNREGISTER_REQUEST, processor_handlers.ProcessorUnRegisterHandler(executor.processors), thread_pool) # Set up base network handlers self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_PING, PingHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_CONNECT, ConnectHandler(network=self._network), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_DISCONNECT, DisconnectHandler(network=self._network), network_thread_pool) # Set up gossip handlers self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, signature_verifier.GossipMessageSignatureVerifier(), process_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipBroadcastHandler( gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, CompleterGossipHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_REQUEST, BlockResponderHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_BATCH_ID_REQUEST, BatchByBatchIdResponderHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_TRANSACTION_ID_REQUEST, BatchByTransactionIdResponderHandler(responder, self._gossip), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, signature_verifier.BatchListSignatureVerifier(), process_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, CompleterBatchListBroadcastHandler( completer, self._gossip), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, client_handlers.BatchSubmitFinisher( self._journal.get_block_store(), completer.batch_cache), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_STATUS_REQUEST, client_handlers.BatchStatusRequest( self._journal.get_block_store(), completer.batch_cache), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST, client_handlers.BlockGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST, client_handlers.BlockListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, client_handlers.StateCurrentRequest( self._journal.get_current_root), thread_pool) def start(self): self._dispatcher.start() self._service.start() if self._genesis_controller.requires_genesis(): self._genesis_controller.start(self._start) else: self._start() def _start(self): self._network_dispatcher.start() self._network.start(daemon=True) self._gossip.start() self._journal.start() def stop(self): self._service.stop() self._network.stop() self._journal.stop()
class Validator(object): def __init__(self, network_endpoint, component_endpoint, peer_list, data_dir): """Constructs a validator instance. Args: network_endpoint (str): the network endpoint component_endpoint (str): the component endpoint peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory """ db_filename = os.path.join( data_dir, 'merkle-{}.lmdb'.format(network_endpoint[-2:])) LOGGER.debug('database file is %s', db_filename) lmdb = LMDBNoLockDatabase(db_filename, 'n') context_manager = ContextManager(lmdb) block_db_filename = os.path.join(data_dir, 'block.lmdb') LOGGER.debug('block store file is %s', block_db_filename) # block_store = LMDBNoLockDatabase(block_db_filename, 'n') block_store = {} # this is not currently being used but will be something like this # in the future, when Journal takes a block_store that isn't a dict # setup network self._dispatcher = Dispatcher() completer = Completer(block_store) thread_pool = ThreadPoolExecutor(max_workers=10) process_pool = ProcessPoolExecutor(max_workers=3) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), thread_pool) self._service = Interconnect(component_endpoint, self._dispatcher, secured=False) executor = TransactionExecutor(self._service, context_manager) self._dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, ProcessorRegisterHandler(executor.processors), thread_pool) identity = hashlib.sha512(time.time().hex().encode()).hexdigest()[:23] network_thread_pool = ThreadPoolExecutor(max_workers=10) self._network_dispatcher = Dispatcher() # Server public and private keys are hardcoded here due to # the decision to avoid having separate identities for each # validator's server socket. This is appropriate for a public # network. For a permissioned network with requirements for # server endpoint authentication at the network level, this can # be augmented with a local lookup service for side-band provided # endpoint, public_key pairs and a local configuration option # for 'server' side private keys. self._network = Interconnect( network_endpoint, dispatcher=self._network_dispatcher, identity=identity, peer_connections=peer_list, secured=True, server_public_key=b'wFMwoOt>yFqI/ek.G[tfMMILHWw#vXB[Sv}>l>i)', server_private_key=b'r&oJ5aQDj4+V]p2:Lz70Eu0x#m%IwzBdP(}&hWM*') self._gossip = Gossip(self._network) block_sender = BroadcastBlockSender(completer, self._gossip) self._network_dispatcher.add_handler(validator_pb2.Message.GOSSIP_PING, PingHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, signature_verifier.GossipMessageSignatureVerifier(), process_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipBroadcastHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, CompleterGossipHandler(completer), network_thread_pool) # Create and configure journal self._journal = Journal( consensus=dev_mode_consensus, block_store=block_store, block_sender=block_sender, transaction_executor=executor, squash_handler=context_manager.get_squash_handler()) self._genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, data_dir=data_dir) completer.set_on_batch_received(self._journal.on_batch_received) completer.set_on_block_received(self._journal.on_block_received) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, signature_verifier.BatchListSignatureVerifier(), process_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, CompleterBatchListBroadcastHandler(completer, self._gossip), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequest(lmdb, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequest(lmdb, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST, client_handlers.BlockGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST, client_handlers.BlockListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, client_handlers.StateCurrentRequest( self._journal.get_current_root), thread_pool) def start(self): self._dispatcher.start() self._service.start() if self._genesis_controller.requires_genesis(): self._genesis_controller.start(self._start) else: self._start() def _start(self): self._network_dispatcher.start() self._network.start(daemon=True) self._gossip.start() self._journal.start() def stop(self): self._service.stop() self._network.stop() self._journal.stop()
class Validator(object): def __init__(self, network_endpoint, component_endpoint, peer_list): data_dir = os.path.expanduser('~') db_filename = os.path.join(data_dir, 'merkle-{}.lmdb'.format( network_endpoint[-2:])) LOGGER.debug('database file is %s', db_filename) lmdb = LMDBNoLockDatabase(db_filename, 'n') context_manager = ContextManager(lmdb) block_db_filename = os.path.join(data_dir, 'block.lmdb') LOGGER.debug('block store file is %s', block_db_filename) block_store = {} # block_store = LMDBNoLockDatabase(block_db_filename, 'n') # this is not currently being used but will be something like this # in the future, when Journal takes a block_store that isn't a dict # setup network self._dispatcher = Dispatcher() completer = Completer() thread_pool = ThreadPoolExecutor(max_workers=10) process_pool = ProcessPoolExecutor(max_workers=3) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequestHandler(lmdb), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequestHandler(lmdb), thread_pool) self._service = Interconnect(component_endpoint, self._dispatcher) executor = TransactionExecutor(self._service, context_manager) self._dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, ProcessorRegisterHandler(executor.processors), thread_pool) identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] network_thread_pool = ThreadPoolExecutor(max_workers=10) self._network_dispatcher = Dispatcher() self._network = Interconnect( network_endpoint, dispatcher=self._network_dispatcher, identity=identity, peer_connections=peer_list) self._gossip = Gossip(self._network) block_sender = BroadcastBlockSender(completer, self._gossip) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_PING, PingHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, signature_verifier.GossipMessageSignatureVerifier(), process_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipBroadcastHandler( gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, CompleterGossipHandler( completer), network_thread_pool) # Create and configure journal self._journal = Journal( consensus=dev_mode_consensus, block_store=block_store, block_sender=block_sender, transaction_executor=executor, squash_handler=context_manager.get_squash_handler()) self._genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, data_dir=data_dir ) completer.set_on_batch_received(self._journal.on_batch_received) completer.set_on_block_received(self._journal.on_block_received) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, signature_verifier.BatchListSignatureVerifier(), process_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, CompleterBatchListBroadcastHandler( completer, self._gossip), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequestHandler(lmdb), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequestHandler(lmdb), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST, client_handlers.BlockGetRequestHandler( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST, client_handlers.BlockListRequestHandler( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, client_handlers.StateCurrentRequestHandler( self._journal.get_current_root), thread_pool) def start(self): self._dispatcher.start() self._service.start() if self._genesis_controller.requires_genesis(): self._genesis_controller.start(self._start) else: self._start() def _start(self): self._network_dispatcher.start() self._network.start(daemon=True) self._gossip.start() self._journal.start() def stop(self): self._service.stop() self._network.stop() self._journal.stop()
class Validator(object): def __init__(self, bind_network, bind_component, endpoint, peering, seeds_list, peer_list, data_dir, config_dir, identity_signing_key, scheduler_type, network_public_key=None, network_private_key=None): """Constructs a validator instance. Args: bind_network (str): the network endpoint bind_component (str): the component endpoint endpoint (str): the zmq-style URI of this validator's publically reachable endpoint peering (str): The type of peering approach. Either 'static' or 'dynamic'. In 'static' mode, no attempted topology buildout occurs -- the validator only attempts to initiate peering connections with endpoints specified in the peer_list. In 'dynamic' mode, the validator will first attempt to initiate peering connections with endpoints specified in the peer_list and then attempt to do a topology buildout starting with peer lists obtained from endpoints in the seeds_list. In either mode, the validator will accept incoming peer requests up to max_peers. seeds_list (list of str): a list of addresses to connect to in order to perform the initial topology buildout peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory config_dir (str): path to the config directory identity_signing_key (str): key validator uses for signing """ db_filename = os.path.join(data_dir, 'merkle-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('database file is %s', db_filename) merkle_db = LMDBNoLockDatabase(db_filename, 'c') delta_db_filename = os.path.join(data_dir, 'state-deltas-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('state delta store file is %s', delta_db_filename) state_delta_db = LMDBNoLockDatabase(delta_db_filename, 'c') state_delta_store = StateDeltaStore(state_delta_db) context_manager = ContextManager(merkle_db, state_delta_store) self._context_manager = context_manager state_view_factory = StateViewFactory(merkle_db) block_db_filename = os.path.join(data_dir, 'block-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = LMDBNoLockDatabase(block_db_filename, 'c') block_store = BlockStore(block_db) batch_tracker = BatchTracker(block_store) block_store.add_update_observer(batch_tracker) # setup network self._dispatcher = Dispatcher() thread_pool = ThreadPoolExecutor(max_workers=10) sig_pool = ThreadPoolExecutor(max_workers=3) self._thread_pool = thread_pool self._sig_pool = sig_pool self._service = Interconnect(bind_component, self._dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True) config_file = os.path.join(config_dir, "validator.toml") validator_config = {} if os.path.exists(config_file): with open(config_file) as fd: raw_config = fd.read() validator_config = toml.loads(raw_config) if scheduler_type is None: scheduler_type = validator_config.get("scheduler", "serial") executor = TransactionExecutor( service=self._service, context_manager=context_manager, settings_view_factory=SettingsViewFactory( StateViewFactory(merkle_db)), scheduler_type=scheduler_type, invalid_observers=[batch_tracker]) self._executor = executor self._service.set_check_connections(executor.check_connections) state_delta_processor = StateDeltaProcessor(self._service, state_delta_store, block_store) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] network_thread_pool = ThreadPoolExecutor(max_workers=10) self._network_thread_pool = network_thread_pool self._network_dispatcher = Dispatcher() secure = False if network_public_key is not None and network_private_key is not None: secure = True self._network = Interconnect( bind_network, dispatcher=self._network_dispatcher, zmq_identity=zmq_identity, secured=secure, server_public_key=network_public_key, server_private_key=network_private_key, heartbeat=True, public_endpoint=endpoint, connection_timeout=30, max_incoming_connections=100) self._gossip = Gossip(self._network, endpoint=endpoint, peering_mode=peering, initial_seed_endpoints=seeds_list, initial_peer_endpoints=peer_list, minimum_peer_connectivity=3, maximum_peer_connectivity=10, topology_check_frequency=1) completer = Completer(block_store, self._gossip) block_sender = BroadcastBlockSender(completer, self._gossip) batch_sender = BroadcastBatchSender(completer, self._gossip) chain_id_manager = ChainIdManager(data_dir) # Create and configure journal self._journal = Journal( block_store=block_store, state_view_factory=StateViewFactory(merkle_db), block_sender=block_sender, batch_sender=batch_sender, transaction_executor=executor, squash_handler=context_manager.get_squash_handler(), identity_signing_key=identity_signing_key, chain_id_manager=chain_id_manager, state_delta_processor=state_delta_processor, data_dir=data_dir, config_dir=config_dir, check_publish_block_frequency=0.1, block_cache_purge_frequency=30, block_cache_keep_time=300, batch_observers=[batch_tracker] ) self._genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_key=identity_signing_key, data_dir=data_dir, config_dir=config_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender ) responder = Responder(completer) completer.set_on_batch_received(self._journal.on_batch_received) completer.set_on_block_received(self._journal.on_block_received) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, processor_handlers.ProcessorRegisterHandler(executor.processors), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_UNREGISTER_REQUEST, processor_handlers.ProcessorUnRegisterHandler(executor.processors), thread_pool) # Set up base network handlers self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_PING, PingHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_CONNECT, ConnectHandler(network=self._network), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_DISCONNECT, DisconnectHandler(network=self._network), network_thread_pool) # Set up gossip handlers self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_REQUEST, GetPeersRequestHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_RESPONSE, GetPeersResponseHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(gossip=self._gossip), network_thread_pool) # GOSSIP_MESSAGE 1) Sends acknowledgement to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(), network_thread_pool) # GOSSIP_MESSAGE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, signature_verifier.GossipMessageSignatureVerifier(), sig_pool) # GOSSIP_MESSAGE 3) Verifies batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, structure_verifier.GossipHandlerStructureVerifier(), network_thread_pool) # GOSSIP_MESSAGE 4) Determines if we should broadcast the # message to our peers. It is important that this occur prior # to the sending of the message to the completer, as this step # relies on whether the gossip message has previously been # seen by the validator to determine whether or not forwarding # should occur self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipBroadcastHandler( gossip=self._gossip, completer=completer), network_thread_pool) # GOSSIP_MESSAGE 5) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, CompleterGossipHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_REQUEST, BlockResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, GossipBlockResponseHandler(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, signature_verifier.GossipBlockResponseSignatureVerifier(), sig_pool) # GOSSIP_BLOCK_RESPONSE 3) Check batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, structure_verifier.GossipBlockResponseStructureVerifier(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, CompleterGossipBlockResponseHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, ResponderBlockResponseHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_BATCH_ID_REQUEST, BatchByBatchIdResponderHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_TRANSACTION_ID_REQUEST, BatchByTransactionIdResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BATCH_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, GossipBatchResponseHandler(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, signature_verifier.GossipBatchResponseSignatureVerifier(), sig_pool) # GOSSIP_BATCH_RESPONSE 3) Check batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, structure_verifier.GossipBatchResponseStructureVerifier(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, CompleterGossipBatchResponseHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, ResponderBatchResponseHandler(responder, self._gossip), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, BatchListPermissionVerifier( settings_view_factory=SettingsViewFactory( StateViewFactory(merkle_db)), current_root_func=self._journal.get_current_root, ), sig_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, signature_verifier.BatchListSignatureVerifier(), sig_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, structure_verifier.BatchListStructureVerifier(), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, CompleterBatchListBroadcastHandler( completer, self._gossip), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, client_handlers.BatchSubmitFinisher(batch_tracker), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_STATUS_REQUEST, client_handlers.BatchStatusRequest(batch_tracker), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST, client_handlers.BlockListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST, client_handlers.BlockGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_LIST_REQUEST, client_handlers.BatchListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_GET_REQUEST, client_handlers.BatchGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_LIST_REQUEST, client_handlers.TransactionListRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_GET_REQUEST, client_handlers.TransactionGetRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, client_handlers.StateCurrentRequest( self._journal.get_current_root), thread_pool) # State Delta Subscription Handlers self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaSubscriberValidationHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaAddSubscriberHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_UNSUBSCRIBE_REQUEST, StateDeltaUnsubscriberHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_GET_EVENTS_REQUEST, StateDeltaGetEventsHandler(block_store, state_delta_store), thread_pool) def start(self): self._dispatcher.start() self._service.start() if self._genesis_controller.requires_genesis(): self._genesis_controller.start(self._start) else: self._start() def _start(self): self._network_dispatcher.start() self._network.start() self._gossip.start() self._journal.start() signal_event = threading.Event() signal.signal(signal.SIGTERM, lambda sig, fr: signal_event.set()) # This is where the main thread will be during the bulk of the # validator's life. while not signal_event.is_set(): signal_event.wait(timeout=20) def stop(self): self._gossip.stop() self._dispatcher.stop() self._network_dispatcher.stop() self._network.stop() self._service.stop() self._network_thread_pool.shutdown(wait=True) self._thread_pool.shutdown(wait=True) self._sig_pool.shutdown(wait=True) self._executor.stop() self._context_manager.stop() self._journal.stop() threads = threading.enumerate() # This will remove the MainThread, which will exit when we exit with # a sys.exit() or exit of main(). threads.remove(threading.current_thread()) while threads: if len(threads) < 4: LOGGER.info( "remaining threads: %s", ", ".join( ["{} ({})".format(x.name, x.__class__.__name__) for x in threads])) for t in threads.copy(): if not t.is_alive(): t.join() threads.remove(t) if threads: time.sleep(1) LOGGER.info("All threads have been stopped and joined")
def test_publish_block(self): """ Test that the Journal will produce blocks and consume those blocks to extend the chain. :return: """ # construction and wire the journal to the # gossip layer. LOGGER.info("test_publish_block") block_store = { 'chain_head_id': 'genesis', 'genesis': BlockState( block_wrapper=_generate_genesis_block(), weight=0, status=BlockStatus.Valid) } journal = None try: journal = Journal( consensus=test_mode_consensus, block_store=block_store, block_sender=self.block_sender, transaction_executor=self.txn_executor, squash_handler=None ) self.gossip.on_batch_received = \ journal.on_batch_received self.gossip.on_block_received = \ journal.on_block_received self.gossip.on_block_request = \ journal.on_block_request journal.start() # feed it a batch batch = Batch() journal.on_batch_received(batch) # wait for a block message to arrive should be soon to = TimeOut(2) while (self.block_sender.new_block is None): time.sleep(0.1) self.assertTrue(self.block_sender.new_block is not None) self.gossip.messages.append(self.block_sender.new_block) block = self.gossip.messages[0] # dispatch the message self.gossip.dispatch_messages() # wait for the chain_head to be updated. to = TimeOut(2) while block_store['chain_head_id'] != block.header_signature: time.sleep(0.1) self.assertTrue(block_store['chain_head_id'] == block.header_signature) finally: if journal is not None: journal.stop()
class Validator(object): def __init__(self, network_endpoint, component_endpoint, peer_list): db_filename = os.path.join(os.path.expanduser('~'), 'merkle.lmdb') LOGGER.debug('database file is %s', db_filename) lmdb = LMDBNoLockDatabase(db_filename, 'n') context_manager = ContextManager(lmdb) block_db_filename = os.path.join(os.path.expanduser('~'), 'block.lmdb') LOGGER.debug('block store file is %s', block_db_filename) # block_store = LMDBNoLockDatabase(block_db_filename, 'n') # this is not currently being used but will be something like this # in the future, when Journal takes a block_store that isn't a dict self._service = Interconnect(component_endpoint) # setup network dispatcher = Dispatcher() faux_network = FauxNetwork(dispatcher=dispatcher) identity = "{}-{}".format(socket.gethostname(), os.getpid()).encode('ascii') self._network = Network(identity, network_endpoint, peer_list, dispatcher=dispatcher) # Create and configure journal executor = TransactionExecutor(self._service, context_manager) self._journal = Journal( consensus=dev_mode_consensus, block_store={}, # -- need to serialize blocks to dicts send_message=faux_network.send_message, transaction_executor=executor, squash_handler=context_manager.get_squash_handler(), first_state_root=context_manager.get_first_root()) dispatcher.on_batch_received = \ self._journal.on_batch_received dispatcher.on_block_received = \ self._journal.on_block_received dispatcher.on_block_request = \ self._journal.on_block_request self._service.add_handler( validator_pb2.Message.DEFAULT, DefaultHandler()) self._service.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, state.GetHandler(context_manager)) self._service.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, state.SetHandler(context_manager)) self._service.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, ProcessorRegisterHandler(self._service)) self._service.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, SystemLoadHandler(faux_network)) self._service.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, ClientStateCurrentRequestHandler(self._journal.get_current_root)) self._service.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, ClientStateGetRequestHandler(lmdb)) self._service.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, ClientStateListRequestHandler(lmdb)) def start(self): self._service.start() self._journal.start() def stop(self): self._service.stop() self._journal.stop()
class Validator(object): def __init__(self, bind_network, bind_component, endpoint, peering, seeds_list, peer_list, data_dir, config_dir, identity_signing_key, network_public_key=None, network_private_key=None): """Constructs a validator instance. Args: bind_network (str): the network endpoint bind_component (str): the component endpoint endpoint (str): the zmq-style URI of this validator's publically reachable endpoint peering (str): The type of peering approach. Either 'static' or 'dynamic'. In 'static' mode, no attempted topology buildout occurs -- the validator only attempts to initiate peering connections with endpoints specified in the peer_list. In 'dynamic' mode, the validator will first attempt to initiate peering connections with endpoints specified in the peer_list and then attempt to do a topology buildout starting with peer lists obtained from endpoints in the seeds_list. In either mode, the validator will accept incoming peer requests up to max_peers. seeds_list (list of str): a list of addresses to connect to in order to perform the initial topology buildout peer_list (list of str): a list of peer addresses data_dir (str): path to the data directory config_dir (str): path to the config directory identity_signing_key (str): key validator uses for signing """ db_filename = os.path.join(data_dir, 'merkle-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('database file is %s', db_filename) merkle_db = LMDBNoLockDatabase(db_filename, 'c') delta_db_filename = os.path.join(data_dir, 'state-deltas-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('state delta store file is %s', delta_db_filename) state_delta_db = LMDBNoLockDatabase(delta_db_filename, 'c') state_delta_store = StateDeltaStore(state_delta_db) context_manager = ContextManager(merkle_db, state_delta_store) self._context_manager = context_manager state_view_factory = StateViewFactory(merkle_db) block_db_filename = os.path.join(data_dir, 'block-{}.lmdb'.format( bind_network[-2:])) LOGGER.debug('block store file is %s', block_db_filename) block_db = LMDBNoLockDatabase(block_db_filename, 'c') block_store = BlockStore(block_db) batch_tracker = BatchTracker(block_store) block_store.add_update_observer(batch_tracker) # setup network self._dispatcher = Dispatcher() thread_pool = ThreadPoolExecutor(max_workers=10) sig_pool = ThreadPoolExecutor(max_workers=3) self._thread_pool = thread_pool self._sig_pool = sig_pool self._service = Interconnect(bind_component, self._dispatcher, secured=False, heartbeat=False, max_incoming_connections=20, monitor=True) executor = TransactionExecutor( service=self._service, context_manager=context_manager, settings_view_factory=SettingsViewFactory( StateViewFactory(merkle_db)), invalid_observers=[batch_tracker]) self._executor = executor self._service.set_check_connections(executor.check_connections) state_delta_processor = StateDeltaProcessor(self._service, state_delta_store, block_store) zmq_identity = hashlib.sha512( time.time().hex().encode()).hexdigest()[:23] network_thread_pool = ThreadPoolExecutor(max_workers=10) self._network_thread_pool = network_thread_pool self._network_dispatcher = Dispatcher() secure = False if network_public_key is not None and network_private_key is not None: secure = True self._network = Interconnect( bind_network, dispatcher=self._network_dispatcher, zmq_identity=zmq_identity, secured=secure, server_public_key=network_public_key, server_private_key=network_private_key, heartbeat=True, public_endpoint=endpoint, connection_timeout=30, max_incoming_connections=100) self._gossip = Gossip(self._network, endpoint=endpoint, peering_mode=peering, initial_seed_endpoints=seeds_list, initial_peer_endpoints=peer_list, minimum_peer_connectivity=3, maximum_peer_connectivity=10, topology_check_frequency=1) completer = Completer(block_store, self._gossip) block_sender = BroadcastBlockSender(completer, self._gossip) batch_sender = BroadcastBatchSender(completer, self._gossip) chain_id_manager = ChainIdManager(data_dir) # Create and configure journal self._journal = Journal( block_store=block_store, state_view_factory=StateViewFactory(merkle_db), block_sender=block_sender, batch_sender=batch_sender, transaction_executor=executor, squash_handler=context_manager.get_squash_handler(), identity_signing_key=identity_signing_key, chain_id_manager=chain_id_manager, state_delta_processor=state_delta_processor, data_dir=data_dir, config_dir=config_dir, check_publish_block_frequency=0.1, block_cache_purge_frequency=30, block_cache_keep_time=300, batch_observers=[batch_tracker] ) self._genesis_controller = GenesisController( context_manager=context_manager, transaction_executor=executor, completer=completer, block_store=block_store, state_view_factory=state_view_factory, identity_key=identity_signing_key, data_dir=data_dir, config_dir=config_dir, chain_id_manager=chain_id_manager, batch_sender=batch_sender ) responder = Responder(completer) completer.set_on_batch_received(self._journal.on_batch_received) completer.set_on_block_received(self._journal.on_block_received) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_GET_REQUEST, tp_state_handlers.TpStateGetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_STATE_SET_REQUEST, tp_state_handlers.TpStateSetHandler(context_manager), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_REGISTER_REQUEST, processor_handlers.ProcessorRegisterHandler(executor.processors), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.TP_UNREGISTER_REQUEST, processor_handlers.ProcessorUnRegisterHandler(executor.processors), thread_pool) # Set up base network handlers self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_PING, PingHandler(), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_CONNECT, ConnectHandler(network=self._network), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.NETWORK_DISCONNECT, DisconnectHandler(network=self._network), network_thread_pool) # Set up gossip handlers self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_REQUEST, GetPeersRequestHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_GET_PEERS_RESPONSE, GetPeersResponseHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_REGISTER, PeerRegisterHandler(gossip=self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_UNREGISTER, PeerUnregisterHandler(gossip=self._gossip), network_thread_pool) # GOSSIP_MESSAGE 1) Sends acknowledgement to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipMessageHandler(), network_thread_pool) # GOSSIP_MESSAGE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, signature_verifier.GossipMessageSignatureVerifier(), sig_pool) # GOSSIP_MESSAGE 3) Verifies batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, structure_verifier.GossipHandlerStructureVerifier(), network_thread_pool) # GOSSIP_MESSAGE 4) Determines if we should broadcast the # message to our peers. It is important that this occur prior # to the sending of the message to the completer, as this step # relies on whether the gossip message has previously been # seen by the validator to determine whether or not forwarding # should occur self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, GossipBroadcastHandler( gossip=self._gossip, completer=completer), network_thread_pool) # GOSSIP_MESSAGE 5) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_MESSAGE, CompleterGossipHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_REQUEST, BlockResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, GossipBlockResponseHandler(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, signature_verifier.GossipBlockResponseSignatureVerifier(), sig_pool) # GOSSIP_BLOCK_RESPONSE 3) Check batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, structure_verifier.GossipBlockResponseStructureVerifier(), network_thread_pool) # GOSSIP_BLOCK_RESPONSE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, CompleterGossipBlockResponseHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BLOCK_RESPONSE, ResponderBlockResponseHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_BATCH_ID_REQUEST, BatchByBatchIdResponderHandler(responder, self._gossip), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_BY_TRANSACTION_ID_REQUEST, BatchByTransactionIdResponderHandler(responder, self._gossip), network_thread_pool) # GOSSIP_BATCH_RESPONSE 1) Sends ack to the sender self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, GossipBatchResponseHandler(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 2) Verifies signature self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, signature_verifier.GossipBatchResponseSignatureVerifier(), sig_pool) # GOSSIP_BATCH_RESPONSE 3) Check batch structure self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, structure_verifier.GossipBatchResponseStructureVerifier(), network_thread_pool) # GOSSIP_BATCH_RESPONSE 4) Send message to completer self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, CompleterGossipBatchResponseHandler( completer), network_thread_pool) self._network_dispatcher.add_handler( validator_pb2.Message.GOSSIP_BATCH_RESPONSE, ResponderBatchResponseHandler(responder, self._gossip), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, BatchListPermissionVerifier( settings_view_factory=SettingsViewFactory( StateViewFactory(merkle_db)), current_root_func=self._journal.get_current_root, ), sig_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, signature_verifier.BatchListSignatureVerifier(), sig_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, structure_verifier.BatchListStructureVerifier(), network_thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, CompleterBatchListBroadcastHandler( completer, self._gossip), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_SUBMIT_REQUEST, client_handlers.BatchSubmitFinisher(batch_tracker), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_STATUS_REQUEST, client_handlers.BatchStatusRequest(batch_tracker), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_LIST_REQUEST, client_handlers.StateListRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_GET_REQUEST, client_handlers.StateGetRequest( merkle_db, self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_LIST_REQUEST, client_handlers.BlockListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BLOCK_GET_REQUEST, client_handlers.BlockGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_LIST_REQUEST, client_handlers.BatchListRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_BATCH_GET_REQUEST, client_handlers.BatchGetRequest(self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_LIST_REQUEST, client_handlers.TransactionListRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_TRANSACTION_GET_REQUEST, client_handlers.TransactionGetRequest( self._journal.get_block_store()), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.CLIENT_STATE_CURRENT_REQUEST, client_handlers.StateCurrentRequest( self._journal.get_current_root), thread_pool) # State Delta Subscription Handlers self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaSubscriberValidationHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_SUBSCRIBE_REQUEST, StateDeltaAddSubscriberHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_UNSUBSCRIBE_REQUEST, StateDeltaUnsubscriberHandler(state_delta_processor), thread_pool) self._dispatcher.add_handler( validator_pb2.Message.STATE_DELTA_GET_EVENTS_REQUEST, GetStateDeltaEventsHandler(block_store, state_delta_store), thread_pool) def start(self): self._dispatcher.start() self._service.start() if self._genesis_controller.requires_genesis(): self._genesis_controller.start(self._start) else: self._start() def _start(self): self._network_dispatcher.start() self._network.start() self._gossip.start() self._journal.start() signal_event = threading.Event() signal.signal(signal.SIGTERM, lambda sig, fr: signal_event.set()) # This is where the main thread will be during the bulk of the # validator's life. while not signal_event.is_set(): signal_event.wait(timeout=20) def stop(self): self._gossip.stop() self._dispatcher.stop() self._network_dispatcher.stop() self._network.stop() self._service.stop() self._network_thread_pool.shutdown(wait=True) self._thread_pool.shutdown(wait=True) self._sig_pool.shutdown(wait=True) self._executor.stop() self._context_manager.stop() self._journal.stop() threads = threading.enumerate() # This will remove the MainThread, which will exit when we exit with # a sys.exit() or exit of main(). threads.remove(threading.current_thread()) while threads: if len(threads) < 4: LOGGER.info( "remaining threads: %s", ", ".join( ["{} ({})".format(x.name, x.__class__.__name__) for x in threads])) for t in threads.copy(): if not t.is_alive(): t.join() threads.remove(t) if threads: time.sleep(1) LOGGER.info("All threads have been stopped and joined")