def get_chain_context(base_db, privkey): chain = FakeAsyncTestnetChain(base_db, privkey.public_key.to_canonical_address(), privkey) chaindb = FakeAsyncChainDB(base_db) chain_head_db = FakeAsyncChainHeadDB.load_from_saved_root_hash(base_db) consensus_db = FakeAsyncConsensusDB(chaindb) chain_config = ChainConfig(network_id=TESTNET_NETWORK_ID) chain_config._node_private_helios_key = privkey chain_config.num_chain_processes = 1 network_id = TESTNET_NETWORK_ID vm_configuration = tuple() chain_context = ChainContext( base_db=base_db, chains=[chain], chaindb=chaindb, chain_head_db=chain_head_db, consensus_db=consensus_db, chain_config=chain_config, network_id=network_id, vm_configuration=vm_configuration, ) return chain_context
def launch_node(args: Namespace, chain_config: ChainConfig, endpoint: Endpoint) -> None: with chain_config.process_id_file('networking'): endpoint.connect() NodeClass = chain_config.node_class # Temporary hack: We setup a second instance of the PluginManager. # The first instance was only to configure the ArgumentParser whereas # for now, the second instance that lives inside the networking process # performs the bulk of the work. In the future, the PluginManager # should probably live in its own process and manage whether plugins # run in the shared plugin process or spawn their own. plugin_manager = setup_plugins(SharedProcessScope(endpoint)) plugin_manager.prepare(args, chain_config) plugin_manager.broadcast(HeliosStartupEvent( args, chain_config )) node = NodeClass(plugin_manager, chain_config) loop = node.get_event_loop() asyncio.ensure_future(handle_networking_exit(node, plugin_manager, endpoint), loop=loop) asyncio.ensure_future(node.run(), loop=loop) loop.run_forever() loop.close()
def test_chain_config_explictely_provided_nodekey(nodekey_bytes, as_bytes): chain_config = ChainConfig( network_id=1, nodekey=nodekey_bytes if as_bytes else keys.PrivateKey(nodekey_bytes), ) assert chain_config.nodekey.to_bytes() == nodekey_bytes
def test_chain_config_explicit_properties(): chain_config = ChainConfig(network_id=1, data_dir='./data-dir', nodekey_path='./nodekey') assert chain_config.data_dir == Path('./data-dir').resolve() assert chain_config.nodekey_path == Path('./nodekey').resolve()
def test_chain_config_nodekey_loading(nodekey_bytes, nodekey_path): chain_config = ChainConfig( network_id=1, nodekey_path=nodekey_path, ) assert chain_config.nodekey.to_bytes() == nodekey_bytes
def run_database_process(chain_config: ChainConfig, db_class: Type[BaseDB]) -> None: with chain_config.process_id_file('database'): if chain_config.report_memory_usage: from threading import Thread memory_logger = logging.getLogger('hvm.memoryLogger') t = Thread(target=sync_periodically_report_memory_stats, args=(chain_config.memory_usage_report_interval, memory_logger)) t.start() base_db = db_class(db_path=chain_config.database_dir) # TODO:remove #base_db = JournalDB(base_db) manager = get_chaindb_manager(chain_config, base_db) server = manager.get_server() # type: ignore def _sigint_handler(*args: Any) -> None: server.stop_event.set() signal.signal(signal.SIGINT, _sigint_handler) try: server.serve_forever() except SystemExit: server.stop_event.set() raise
def get_chain_manager(chain_config: ChainConfig, base_db: AsyncBaseDB, instance=0) -> BaseManager: # TODO: think about using async chian here. Depends which process we would like the threaded work to happen in. # There might be a performance savings by doing the threaded work in this process to avoid one process hop. if chain_config.network_id == MAINNET_NETWORK_ID: chain_class = MainnetChain else: raise NotImplementedError( "Only the mainnet chain is currently supported") chain = chain_class(base_db, chain_config.node_wallet_address, chain_config.node_private_helios_key) # type: ignore class ChainManager(BaseManager): pass ChainManager.register( # type: ignore 'get_chain', callable=lambda: TracebackRecorder(chain), proxytype=ChainProxy) manager = ChainManager(address=str( chain_config.get_chain_ipc_path(instance))) # type: ignore return manager
def test_chain_config_computed_properties(xdg_helios_root): data_dir = get_local_data_dir('muffin', xdg_helios_root) chain_config = ChainConfig(network_id=1234, data_dir=data_dir) assert chain_config.network_id == 1234 assert chain_config.data_dir == data_dir assert chain_config.database_dir == data_dir / DATABASE_DIR_NAME / "full" assert chain_config.nodekey_path == get_nodekey_path(data_dir)
def test_chain_config_computed_properties_custom_xdg(tmpdir, xdg_helios_root): alt_xdg_root = tmpdir.mkdir('helios-custom') assert not is_under_path(alt_xdg_root, xdg_helios_root) data_dir = get_data_dir_for_network_id(1, alt_xdg_root) chain_config = ChainConfig(helios_root_dir=alt_xdg_root, network_id=1) assert chain_config.network_id == 1 assert chain_config.data_dir == data_dir assert chain_config.database_dir == data_dir / DATABASE_DIR_NAME / "full" assert chain_config.nodekey_path == get_nodekey_path(data_dir)
def __init__(self, plugin_manager: PluginManager, chain_config: ChainConfig) -> None: super().__init__() self.chain_config: ChainConfig = chain_config self.private_helios_key = chain_config.node_private_helios_key self.wallet_address = chain_config.node_wallet_address self._plugin_manager = plugin_manager self._db_manager = create_db_manager(chain_config.database_ipc_path) self._db_manager.connect() # type: ignore for i in range(chain_config.num_chain_processes): chain_manager = create_chain_manager( chain_config.get_chain_ipc_path(i)) chain_manager.connect() self._chain_managers.append(chain_manager) self._chain_head_db = self._db_manager.get_chain_head_db( ) # type: ignore self._jsonrpc_ipc_path: Path = chain_config.jsonrpc_ipc_path
def run_chain_process(chain_config: ChainConfig, instance = 0) -> None: with chain_config.process_id_file('database_{}'.format(instance)): # connect with database process db_manager = create_db_manager(chain_config.database_ipc_path) db_manager.connect() base_db = db_manager.get_db() # start chain process manager = get_chain_manager(chain_config, base_db, instance) server = manager.get_server() # type: ignore def _sigint_handler(*args: Any) -> None: server.stop_event.set() signal.signal(signal.SIGINT, _sigint_handler) try: server.serve_forever() except SystemExit: server.stop_event.set() raise
def database_server_ipc_path(): core_db = AtomicDB() core_db[b'key-a'] = b'value-a' chaindb = ChainDB(core_db) # TODO: use a custom chain class only for testing. chaindb.persist_header(ROPSTEN_GENESIS_HEADER) with tempfile.TemporaryDirectory() as temp_dir: chain_config = ChainConfig(network_id=ROPSTEN_NETWORK_ID, max_peers=1, data_dir=temp_dir) manager = get_chaindb_manager(chain_config, core_db) chaindb_server_process = multiprocessing.Process( target=serve_chaindb, args=(manager,), ) chaindb_server_process.start() wait_for_ipc(chain_config.database_ipc_path) try: yield chain_config.database_ipc_path finally: kill_process_gracefully(chaindb_server_process, logging.getLogger())
def chain_config(): return ChainConfig(network_id=1, max_peers=1)
def chain_config(): _chain_config = ChainConfig(network_id=1, max_peers=1) initialize_data_dir(_chain_config) return _chain_config
def main() -> None: event_bus = EventBus(ctx) main_endpoint = event_bus.create_endpoint(MAIN_EVENTBUS_ENDPOINT) main_endpoint.connect() plugin_manager = setup_plugins( MainAndIsolatedProcessScope(event_bus, main_endpoint) ) plugin_manager.amend_argparser_config(parser, subparser) args = parser.parse_args() # # Dev testing stuff # if args.start_memory_profile: os.environ["PYTHONTRACEMALLOC"] = '1' if args.rand_db: os.environ["GENERATE_RANDOM_DATABASE"] = 'true' if args.instance is not None: from helios.utils.xdg import get_xdg_helios_root args.port = args.port + args.instance * 2 if args.instance != 0: args.do_rpc_http_server = False subdir = 'instance_' + str(args.instance) absolute_path = get_xdg_helios_root() / subdir absolute_dir = os.path.dirname(os.path.realpath(__file__)) absolute_keystore_path = absolute_dir + '/keystore/' args.keystore_path = absolute_keystore_path + subdir args.keystore_password = '******' os.environ["HELIOS_DATA_DIR"] = str(absolute_path.resolve()) os.environ["INSTANCE_NUMBER"] = str(args.instance) # # # if not args.keystore_password and not hasattr(args, 'func'): password = getpass.getpass(prompt='Keystore Password: '******'default'] = TRACE_LEVEL_NUM log_levels['hvm'] = TRACE_LEVEL_NUM log_levels['hp2p'] = TRACE_LEVEL_NUM log_levels['helios'] = TRACE_LEVEL_NUM log_levels['urllib3'] = TRACE_LEVEL_NUM log_levels['ssdp'] = TRACE_LEVEL_NUM log_levels['Service'] = TRACE_LEVEL_NUM log_levels['Action'] = TRACE_LEVEL_NUM log_levels['Device'] = TRACE_LEVEL_NUM log_levels['helios.extensibility'] = TRACE_LEVEL_NUM else: log_levels['default'] = logging.INFO log_levels['urllib3'] = logging.INFO log_levels['ssdp'] = logging.INFO log_levels['Service'] = logging.INFO log_levels['hvm'] = logging.DEBUG #sets all of hvm log_levels['hvm.db.account.AccountDB'] = logging.DEBUG log_levels['hvm.vm.base.VM.VM'] = logging.DEBUG log_levels['hvm.chain'] = logging.DEBUG #log_levels['hvm.chain.chain.Chain'] = logging.DEBUG log_levels['hvm.db.chain_head.ChainHeadDB'] = logging.DEBUG log_levels['hvm.db.chain_db.ChainDB'] = logging.DEBUG log_levels['hvm.db.consensus'] = logging.DEBUG log_levels['hvm.memoryLogger'] = logging.DEBUG #log_levels['hp2p'] = logging.INFO log_levels['hp2p.peer'] = logging.DEBUG log_levels['hp2p.peer.PeerPool'] = logging.DEBUG log_levels['hp2p.consensus.Consensus'] = logging.DEBUG log_levels['hp2p.SmartContractChainManager'] = logging.DEBUG log_levels['hp2p.kademlia.KademliaProtocol'] = logging.DEBUG log_levels['hp2p.discovery.DiscoveryProtocol'] = logging.INFO log_levels['hp2p.discovery.DiscoveryService'] = logging.INFO log_levels['hp2p.nat.UPnPService'] = logging.CRITICAL log_levels['connectionpool'] = logging.CRITICAL log_levels['hp2p.protocol'] = logging.DEBUG log_levels['hp2p.protocol.Protocol'] = logging.DEBUG #log_levels['helios'] = logging.INFO log_levels['helios.rpc.ipc'] = logging.INFO log_levels['helios.Node'] = logging.INFO log_levels['helios.sync'] = logging.DEBUG log_levels['helios.protocol'] = logging.INFO log_levels['helios.protocol.common'] = logging.DEBUG log_levels['helios.protocol.hls.peer.HLSPeer'] = 5 log_levels['helios.memoryLogger'] = logging.DEBUG log_levels['hp2p.hls'] = logging.INFO log_levels['helios.server.FullServer'] = logging.DEBUG log_levels['Action'] = logging.INFO log_levels['Device'] = logging.INFO log_levels['helios.extensibility'] = logging.INFO setup_log_levels(log_levels = log_levels) try: chain_config = ChainConfig.from_parser_args(args) except AmbigiousFileSystem: parser.error(HELIOS_AMBIGIOUS_FILESYSTEM_INFO) if not is_data_dir_initialized(chain_config): # TODO: this will only work as is for chains with known genesis # parameters. Need to flesh out how genesis parameters for custom # chains are defined and passed around. try: initialize_data_dir(chain_config) except AmbigiousFileSystem: parser.error(HELIOS_AMBIGIOUS_FILESYSTEM_INFO) except MissingPath as e: parser.error( "\n" f"It appears that {e.path} does not exist. " "Helios does not attempt to create directories outside of its root path. " "Either manually create the path or ensure you are using a data directory " "inside the XDG_HELIOS_ROOT path" ) file_logger, log_queue, listener = setup_helios_file_and_queue_logging( stderr_logger, formatter, handler_stream, chain_config, args.file_log_level, ) display_launch_logs(chain_config) # compute the minimum configured log level across all configured loggers. min_configured_log_level = min( stderr_logger.level, file_logger.level, *(args.log_levels or {}).values(), *(log_levels or {}).values() ) extra_kwargs = { 'log_queue': log_queue, 'log_level': min_configured_log_level, 'log_levels': log_levels, 'profile': args.profile, } # Plugins can provide a subcommand with a `func` which does then control # the entire process from here. if hasattr(args, 'func'): args.func(args, chain_config) else: helios_boot( args, chain_config, extra_kwargs, plugin_manager, listener, event_bus, main_endpoint, stderr_logger, )
def fix_unclean_shutdown(self, args: Namespace, chain_config: ChainConfig) -> None: self.logger.info("Cleaning up unclean shutdown...") self.logger.info("Searching for process id files in %s..." % chain_config.data_dir) pidfiles = tuple(chain_config.data_dir.glob('*.pid')) if len(pidfiles) > 1: self.logger.info( 'Found %d processes from a previous run. Closing...' % len(pidfiles)) elif len(pidfiles) == 1: self.logger.info('Found 1 process from a previous run. Closing...') else: self.logger.info( 'Found 0 processes from a previous run. No processes to kill.') for pidfile in pidfiles: process_id = int(pidfile.read_text()) kill_process_id_gracefully(process_id, time.sleep, self.logger) try: pidfile.unlink() self.logger.info( 'Manually removed %s after killing process id %d' % (pidfile, process_id)) except FileNotFoundError: self.logger.debug( 'pidfile %s was gone after killing process id %d' % (pidfile, process_id)) db_ipc = chain_config.database_ipc_path try: db_ipc.unlink() self.logger.info( 'Removed a dangling IPC socket file for database connections at %s', db_ipc) except FileNotFoundError: self.logger.debug( 'The IPC socket file for database connections at %s was already gone', db_ipc) for i in range(chain_config.num_chain_processes): chain_ipc = chain_config.get_chain_ipc_path(i) try: chain_ipc.unlink() self.logger.info( 'Removed a dangling IPC socket file for chain instance {} process at {}' .format(i, chain_ipc)) except FileNotFoundError: self.logger.debug( 'The IPC socket file for chain instance {} process at {} was already gone' .format(i, chain_ipc)) jsonrpc_ipc = chain_config.jsonrpc_ipc_path try: jsonrpc_ipc.unlink() self.logger.info( 'Removed a dangling IPC socket file for JSON-RPC connections at %s', jsonrpc_ipc, ) except FileNotFoundError: self.logger.debug( 'The IPC socket file for JSON-RPC connections at %s was already gone', jsonrpc_ipc, )
def helios_boot(args: Namespace, chain_config: ChainConfig, extra_kwargs: Dict[str, Any], plugin_manager: PluginManager, listener: logging.handlers.QueueListener, event_bus: EventBus, main_endpoint: Endpoint, logger: logging.Logger) -> None: # start the listener thread to handle logs produced by other processes in # the local logger. listener.start() networking_endpoint = event_bus.create_endpoint(NETWORKING_EVENTBUS_ENDPOINT) event_bus.start() # First initialize the database process. database_server_process = ctx.Process( target=run_database_process, args=( chain_config, LevelDB, ), kwargs=extra_kwargs, ) chain_processes = [] for i in range(chain_config.num_chain_processes): chain_process = ctx.Process( target=run_chain_process, args=( chain_config, i ), kwargs=extra_kwargs, ) chain_processes.append(chain_process) networking_process = ctx.Process( target=launch_node, args=(args, chain_config, networking_endpoint,), kwargs=extra_kwargs, ) # start the processes database_server_process.start() logger.info("Started DB server process (pid=%d)", database_server_process.pid) # networking process needs the IPC socket file provided by the database process try: wait_for_ipc(chain_config.database_ipc_path) except TimeoutError as e: logger.error("Timeout waiting for database to start. Exiting...") kill_process_gracefully(database_server_process, logger) ArgumentParser().error(message="Timed out waiting for database start") for i in range(chain_config.num_chain_processes): chain_process = chain_processes[i] chain_process.start() logger.info("Started chain instance {} process (pid={})".format(i,database_server_process.pid)) try: wait_for_ipc(chain_config.get_chain_ipc_path(i)) except TimeoutError as e: logger.error("Timeout waiting for chain instance {} to start. Exiting...".format(i)) kill_process_gracefully(database_server_process, logger) for j in range(i+1): kill_process_gracefully(chain_processes[j], logger) ArgumentParser().error(message="Timed out waiting for chain instance {} start".format(i)) networking_process.start() logger.info("Started networking process (pid=%d)", networking_process.pid) main_endpoint.subscribe( ShutdownRequest, lambda ev: kill_helios_gracefully( logger, database_server_process, chain_processes, networking_process, plugin_manager, main_endpoint, event_bus ) ) plugin_manager.prepare(args, chain_config, extra_kwargs) plugin_manager.broadcast(HeliosStartupEvent( args, chain_config )) try: loop = asyncio.get_event_loop() loop.run_forever() loop.close() except KeyboardInterrupt: kill_helios_gracefully( logger, database_server_process, chain_processes, networking_process, plugin_manager, main_endpoint, event_bus )