def main(): config = load_config(extra_args_funcs=(gw_args, Logger.update_log_args)) logger = Logger(config) logger.add_pkg('aiotools') logger.add_pkg('aiopg') logger.add_pkg('ai.backend') with logger: log.info(f'Backend.AI Gateway {__version__}') log.info(f'runtime: {env_info()}') log_config = logging.getLogger('ai.backend.gateway.config') log_config.debug('debug mode enabled.') if config.debug: aiohttp.log.server_logger.setLevel('DEBUG') aiohttp.log.access_logger.setLevel('DEBUG') else: aiohttp.log.server_logger.setLevel('WARNING') aiohttp.log.access_logger.setLevel('WARNING') asyncio.set_event_loop_policy(uvloop.EventLoopPolicy()) try: aiotools.start_server(server_main, num_workers=config.num_proc, extra_procs=[event_router], args=(config, )) finally: log.info('terminated.')
def main(ctx, config_path, debug): cfg = load_config(config_path, debug) if ctx.invoked_subcommand is None: cfg['manager']['pid-file'].write_text(str(os.getpid())) try: logger = Logger(cfg['logging']) with logger: ns = cfg['etcd']['namespace'] setproctitle(f"backend.ai: manager {ns}") log.info('Backend.AI Gateway {0}', __version__) log.info('runtime: {0}', env_info()) log_config = logging.getLogger('ai.backend.gateway.config') log_config.debug('debug mode enabled.') if cfg['manager']['event-loop'] == 'uvloop': uvloop.install() log.info('Using uvloop as the event loop backend') try: aiotools.start_server( server_main, num_workers=cfg['manager']['num-proc'], extra_procs=[event_router], args=(cfg, )) finally: log.info('terminated.') finally: if cfg['manager']['pid-file'].is_file(): # check is_file() to prevent deleting /dev/null! cfg['manager']['pid-file'].unlink() else: # Click is going to invoke a subcommand. pass
async def server_main_logwrapper(loop, pidx, _args): setproctitle(f"backend.ai: storage-proxy worker-{pidx}") log_endpoint = _args[1] logger = Logger(_args[0]['logging'], is_master=False, log_endpoint=log_endpoint) with logger: async with server_main(loop, pidx, _args): yield
def main(ctx, config_path, debug): cfg = load_config(config_path) setproctitle(f"backend.ai: manager.cli {cfg['etcd']['namespace']}") if 'file' in cfg['logging']['drivers']: cfg['logging']['drivers'].remove('file') logger = Logger(cfg['logging']) ctx.obj = CLIContext( logger=logger, config=cfg, )
async def server_main_logwrapper( loop: asyncio.AbstractEventLoop, pidx: int, _args: Tuple[Any, ...], ) -> AsyncGenerator[None, signal.Signals]: setproctitle(f"backend.ai: agent worker-{pidx}") log_endpoint = _args[1] logger = Logger(_args[0]['logging'], is_master=False, log_endpoint=log_endpoint) with logger: async with server_main(loop, pidx, _args): yield
async def server_main_logwrapper(loop, pidx, _args): setproctitle(f"backend.ai: storage-proxy worker-{pidx}") try: asyncio.get_child_watcher() except (AttributeError, NotImplementedError): pass log_endpoint = _args[1] logger = Logger(_args[0]["logging"], is_master=False, log_endpoint=log_endpoint) with logger: async with server_main(loop, pidx, _args): yield
async def server_main_logwrapper(loop: asyncio.AbstractEventLoop, pidx: int, _args: List[Any]) -> AsyncIterator[None]: setproctitle(f"backend.ai: manager worker-{pidx}") log_endpoint = _args[1] logger = Logger(_args[0]['logging'], is_master=False, log_endpoint=log_endpoint) try: with logger: async with server_main(loop, pidx, _args): yield except Exception: traceback.print_exc()
def generate_keypair(): logger = Logger({ 'level': 'INFO', 'drivers': ['console'], 'pkg-ns': { 'ai.backend': 'INFO' }, 'console': { 'colored': True, 'format': 'verbose' }, }) with logger: log.info('generating keypair...') ak, sk = _gen_keypair() print(f'Access Key: {ak} ({len(ak)} bytes)') print(f'Secret Key: {sk} ({len(sk)} bytes)')
def main(ctx: click.Context, config_path: Path, debug: bool) -> None: cfg = load_config(config_path, debug) if ctx.invoked_subcommand is None: cfg['manager']['pid-file'].write_text(str(os.getpid())) log_sockpath = Path( f'/tmp/backend.ai/ipc/manager-logger-{os.getpid()}.sock') log_sockpath.parent.mkdir(parents=True, exist_ok=True) log_endpoint = f'ipc://{log_sockpath}' try: logger = Logger(cfg['logging'], is_master=True, log_endpoint=log_endpoint) with logger: ns = cfg['etcd']['namespace'] setproctitle(f"backend.ai: manager {ns}") log.info('Backend.AI Gateway {0}', __version__) log.info('runtime: {0}', env_info()) log_config = logging.getLogger('ai.backend.gateway.config') log_config.debug('debug mode enabled.') if cfg['manager']['event-loop'] == 'uvloop': import uvloop uvloop.install() log.info('Using uvloop as the event loop backend') try: aiotools.start_server( server_main_logwrapper, num_workers=cfg['manager']['num-proc'], args=(cfg, log_endpoint), ) finally: log.info('terminated.') finally: if cfg['manager']['pid-file'].is_file(): # check is_file() to prevent deleting /dev/null! cfg['manager']['pid-file'].unlink() else: # Click is going to invoke a subcommand. pass
def main(ctx, config_path, debug): cfg = load_config(config_path) setproctitle(f"backend.ai: manager.cli {cfg['etcd']['namespace']}") if 'file' in cfg['logging']['drivers']: cfg['logging']['drivers'].remove('file') # log_endpoint = f'tcp://127.0.0.1:{find_free_port()}' log_sockpath = Path(f'/tmp/backend.ai/ipc/manager-cli-{os.getpid()}.sock') log_sockpath.parent.mkdir(parents=True, exist_ok=True) log_endpoint = f'ipc://{log_sockpath}' logger = Logger(cfg['logging'], is_master=True, log_endpoint=log_endpoint) ctx.obj = CLIContext( logger=logger, config=cfg, ) def _clean_logger(): try: os.unlink(log_sockpath) except FileNotFoundError: pass atexit.register(_clean_logger)
def main(cli_ctx, config_path, debug): volume_config_iv = t.Dict({ t.Key('etcd'): t.Dict({ t.Key('namespace'): t.String, t.Key('addr'): tx.HostPortPair(allow_blank_host=False) }).allow_extra('*'), t.Key('logging'): t.Any, # checked in ai.backend.common.logging t.Key('agent'): t.Dict({ t.Key('mode'): t.Enum('scratch', 'vfolder'), t.Key('rpc-listen-addr'): tx.HostPortPair(allow_blank_host=True), t.Key('user-uid'): t.Int, t.Key('user-gid'): t.Int }), t.Key('storage'): t.Dict({ t.Key('mode'): t.Enum('xfs', 'btrfs'), t.Key('path'): t.String }) }).allow_extra('*') # Determine where to read configuration. raw_cfg, cfg_src_path = config.read_from_file(config_path, 'agent') config.override_with_env(raw_cfg, ('etcd', 'namespace'), 'BACKEND_NAMESPACE') config.override_with_env(raw_cfg, ('etcd', 'addr'), 'BACKEND_ETCD_ADDR') config.override_with_env(raw_cfg, ('etcd', 'user'), 'BACKEND_ETCD_USER') config.override_with_env(raw_cfg, ('etcd', 'password'), 'BACKEND_ETCD_PASSWORD') config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'host'), 'BACKEND_AGENT_HOST_OVERRIDE') config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'port'), 'BACKEND_AGENT_PORT') if debug: config.override_key(raw_cfg, ('debug', 'enabled'), True) config.override_key(raw_cfg, ('logging', 'level'), 'DEBUG') config.override_key(raw_cfg, ('logging', 'pkg-ns', 'ai.backend'), 'DEBUG') try: cfg = config.check(raw_cfg, volume_config_iv) cfg['_src'] = cfg_src_path except config.ConfigurationError as e: print( 'ConfigurationError: Validation of agent configuration has failed:', file=sys.stderr) print(pformat(e.invalid_data), file=sys.stderr) raise click.Abort() rpc_host = cfg['agent']['rpc-listen-addr'].host if (isinstance(rpc_host, BaseIPAddress) and (rpc_host.is_unspecified or rpc_host.is_link_local)): print( 'ConfigurationError: ' 'Cannot use link-local or unspecified IP address as the RPC listening host.', file=sys.stderr) raise click.Abort() if os.getuid() != 0: print('Storage agent can only be run as root', file=sys.stderr) raise click.Abort() if cli_ctx.invoked_subcommand is None: setproctitle('Backend.AI: Storage Agent') logger = Logger(cfg['logging']) with logger: log.info('Backend.AI Storage Agent', VERSION) log_config = logging.getLogger('ai.backend.agent.config') if debug: log_config.debug('debug mode enabled.') if 'debug' in cfg and cfg['debug']['enabled']: print('== Agent configuration ==') pprint(cfg) aiotools.start_server(server_main, num_workers=1, use_threading=True, args=(cfg, )) log.info('exit.') return 0
break time.sleep(1.0) loop.run_until_complete(docker.close()) except (KeyboardInterrupt, SystemExit): sys.exit(1) else: sys.exit(0) finally: signal_sock.close() os.unlink(signal_path) log.info('terminated statistics collection for {}', args.cid) if __name__ == '__main__': # The entry point for stat collector daemon parser = argparse.ArgumentParser() parser.add_argument('sockaddr', type=str) parser.add_argument('cid', type=str) parser.add_argument('-t', '--type', choices=['cgroup', 'api'], default='cgroup') args = parser.parse_args() setproctitle(f'backend.ai: stat-collector {args.cid[:7]}') log_config = argparse.Namespace() log_config.log_file = None log_config.debug = False logger = Logger(log_config) with logger: main(args)
def main(cli_ctx, config_path, debug): watcher_config_iv = t.Dict({ t.Key('watcher'): t.Dict({ t.Key('service-addr', default=('0.0.0.0', 6009)): tx.HostPortPair, t.Key('ssl-enabled', default=False): t.Bool, t.Key('ssl-cert', default=None): t.Null | tx.Path(type='file'), t.Key('ssl-key', default=None): t.Null | tx.Path(type='file'), t.Key('target-service', default='backendai-agent.service'): t.String, t.Key('soft-reset-available', default=False): t.Bool, }).allow_extra('*'), t.Key('logging'): t.Any, # checked in ai.backend.common.logging t.Key('debug'): t.Dict({ t.Key('enabled', default=False): t.Bool, }).allow_extra('*'), }).merge(config.etcd_config_iv).allow_extra('*') raw_cfg, cfg_src_path = config.read_from_file(config_path, 'agent') config.override_with_env(raw_cfg, ('etcd', 'namespace'), 'BACKEND_NAMESPACE') config.override_with_env(raw_cfg, ('etcd', 'addr'), 'BACKEND_ETCD_ADDR') config.override_with_env(raw_cfg, ('etcd', 'user'), 'BACKEND_ETCD_USER') config.override_with_env(raw_cfg, ('etcd', 'password'), 'BACKEND_ETCD_PASSWORD') config.override_with_env(raw_cfg, ('watcher', 'service-addr', 'host'), 'BACKEND_WATCHER_SERVICE_IP') config.override_with_env(raw_cfg, ('watcher', 'service-addr', 'port'), 'BACKEND_WATCHER_SERVICE_PORT') if debug: config.override_key(raw_cfg, ('debug', 'enabled'), True) try: cfg = config.check(raw_cfg, watcher_config_iv) if 'debug' in cfg and cfg['debug']['enabled']: print('== Watcher configuration ==') pprint(cfg) cfg['_src'] = cfg_src_path except config.ConfigurationError as e: print('Validation of watcher configuration has failed:', file=sys.stderr) print(pformat(e.invalid_data), file=sys.stderr) raise click.Abort() # Change the filename from the logging config's file section. log_sockpath = Path(f'/tmp/backend.ai/ipc/watcher-logger-{os.getpid()}.sock') log_sockpath.parent.mkdir(parents=True, exist_ok=True) log_endpoint = f'ipc://{log_sockpath}' cfg['logging']['endpoint'] = log_endpoint logger = Logger(cfg['logging'], is_master=True, log_endpoint=log_endpoint) if 'file' in cfg['logging']['drivers']: fn = Path(cfg['logging']['file']['filename']) cfg['logging']['file']['filename'] = f"{fn.stem}-watcher{fn.suffix}" setproctitle(f"backend.ai: watcher {cfg['etcd']['namespace']}") with logger: log.info('Backend.AI Agent Watcher {0}', VERSION) log.info('runtime: {0}', utils.env_info()) log_config = logging.getLogger('ai.backend.agent.config') log_config.debug('debug mode enabled.') aiotools.start_server( watcher_server, num_workers=1, args=(cfg, ), stop_signals={signal.SIGINT, signal.SIGTERM, signal.SIGALRM}, ) log.info('exit.') return 0
def main( cli_ctx: click.Context, config_path: Path, debug: bool, ) -> int: # Determine where to read configuration. raw_cfg, cfg_src_path = config.read_from_file(config_path, 'agent') # Override the read config with environment variables (for legacy). config.override_with_env(raw_cfg, ('etcd', 'namespace'), 'BACKEND_NAMESPACE') config.override_with_env(raw_cfg, ('etcd', 'addr'), 'BACKEND_ETCD_ADDR') config.override_with_env(raw_cfg, ('etcd', 'user'), 'BACKEND_ETCD_USER') config.override_with_env(raw_cfg, ('etcd', 'password'), 'BACKEND_ETCD_PASSWORD') config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'host'), 'BACKEND_AGENT_HOST_OVERRIDE') config.override_with_env(raw_cfg, ('agent', 'rpc-listen-addr', 'port'), 'BACKEND_AGENT_PORT') config.override_with_env(raw_cfg, ('agent', 'pid-file'), 'BACKEND_PID_FILE') config.override_with_env(raw_cfg, ('container', 'port-range'), 'BACKEND_CONTAINER_PORT_RANGE') config.override_with_env(raw_cfg, ('container', 'kernel-host'), 'BACKEND_KERNEL_HOST_OVERRIDE') config.override_with_env(raw_cfg, ('container', 'sandbox-type'), 'BACKEND_SANDBOX_TYPE') config.override_with_env(raw_cfg, ('container', 'scratch-root'), 'BACKEND_SCRATCH_ROOT') if debug: config.override_key(raw_cfg, ('debug', 'enabled'), True) config.override_key(raw_cfg, ('logging', 'level'), 'DEBUG') config.override_key(raw_cfg, ('logging', 'pkg-ns', 'ai.backend'), 'DEBUG') # Validate and fill configurations # (allow_extra will make configs to be forward-copmatible) try: cfg = config.check(raw_cfg, agent_local_config_iv) if cfg['agent']['backend'] == AgentBackend.KUBERNETES: cfg = config.check(raw_cfg, k8s_extra_config_iv) if cfg['registry']['type'] == 'local': registry_target_config_iv = registry_local_config_iv elif cfg['registry']['type'] == 'ecr': registry_target_config_iv = registry_ecr_config_iv else: print('Validation of agent configuration has failed: registry type {} not supported' .format(cfg['registry']['type']), file=sys.stderr) raise click.Abort() registry_cfg = config.check(cfg['registry'], registry_target_config_iv) cfg['registry'] = registry_cfg if cfg['agent']['backend'] == AgentBackend.DOCKER: config.check(raw_cfg, docker_extra_config_iv) if 'debug' in cfg and cfg['debug']['enabled']: print('== Agent configuration ==') pprint(cfg) cfg['_src'] = cfg_src_path except config.ConfigurationError as e: print('ConfigurationError: Validation of agent configuration has failed:', file=sys.stderr) print(pformat(e.invalid_data), file=sys.stderr) raise click.Abort() rpc_host = cfg['agent']['rpc-listen-addr'].host if (isinstance(rpc_host, BaseIPAddress) and (rpc_host.is_unspecified or rpc_host.is_link_local)): print('ConfigurationError: ' 'Cannot use link-local or unspecified IP address as the RPC listening host.', file=sys.stderr) raise click.Abort() if os.getuid() != 0 and cfg['container']['stats-type'] == 'cgroup': print('Cannot use cgroup statistics collection mode unless the agent runs as root.', file=sys.stderr) raise click.Abort() if cli_ctx.invoked_subcommand is None: if cfg['debug']['coredump']['enabled']: if not sys.platform.startswith('linux'): print('ConfigurationError: ' 'Storing container coredumps is only supported in Linux.', file=sys.stderr) raise click.Abort() core_pattern = Path('/proc/sys/kernel/core_pattern').read_text().strip() if core_pattern.startswith('|') or not core_pattern.startswith('/'): print('ConfigurationError: ' '/proc/sys/kernel/core_pattern must be an absolute path ' 'to enable container coredumps.', file=sys.stderr) raise click.Abort() cfg['debug']['coredump']['core_path'] = Path(core_pattern).parent cfg['agent']['pid-file'].write_text(str(os.getpid())) log_sockpath = Path(f'/tmp/backend.ai/ipc/agent-logger-{os.getpid()}.sock') log_sockpath.parent.mkdir(parents=True, exist_ok=True) log_endpoint = f'ipc://{log_sockpath}' cfg['logging']['endpoint'] = log_endpoint try: logger = Logger(cfg['logging'], is_master=True, log_endpoint=log_endpoint) with logger: ns = cfg['etcd']['namespace'] setproctitle(f"backend.ai: agent {ns}") log.info('Backend.AI Agent {0}', VERSION) log.info('runtime: {0}', utils.env_info()) log_config = logging.getLogger('ai.backend.agent.config') if debug: log_config.debug('debug mode enabled.') if cfg['agent']['event-loop'] == 'uvloop': import uvloop uvloop.install() log.info('Using uvloop as the event loop backend') aiotools.start_server( server_main_logwrapper, num_workers=1, args=(cfg, log_endpoint), ) log.info('exit.') finally: if cfg['agent']['pid-file'].is_file(): # check is_file() to prevent deleting /dev/null! cfg['agent']['pid-file'].unlink() else: # Click is going to invoke a subcommand. pass return 0
def main(cli_ctx, config_path, debug): # Determine where to read configuration. raw_cfg, cfg_src_path = config.read_from_file(config_path, "storage-proxy") config.override_with_env(raw_cfg, ("etcd", "namespace"), "BACKEND_NAMESPACE") config.override_with_env(raw_cfg, ("etcd", "addr"), "BACKEND_ETCD_ADDR") config.override_with_env(raw_cfg, ("etcd", "user"), "BACKEND_ETCD_USER") config.override_with_env(raw_cfg, ("etcd", "password"), "BACKEND_ETCD_PASSWORD") if debug: config.override_key(raw_cfg, ("debug", "enabled"), True) try: local_config = config.check(raw_cfg, local_config_iv) local_config["_src"] = cfg_src_path except config.ConfigurationError as e: print( "ConfigurationError: Validation of agent configuration has failed:", file=sys.stderr, ) print(pformat(e.invalid_data), file=sys.stderr) raise click.Abort() if local_config["debug"]["enabled"]: config.override_key(local_config, ("logging", "level"), "DEBUG") config.override_key(local_config, ("logging", "pkg-ns", "ai.backend"), "DEBUG") # if os.getuid() != 0: # print('Storage agent can only be run as root', file=sys.stderr) # raise click.Abort() multiprocessing.set_start_method("spawn") if cli_ctx.invoked_subcommand is None: local_config["storage-proxy"]["pid-file"].write_text(str(os.getpid())) log_sockpath = Path( f"/tmp/backend.ai/ipc/storage-proxy-logger-{os.getpid()}.sock", ) log_sockpath.parent.mkdir(parents=True, exist_ok=True) log_endpoint = f"ipc://{log_sockpath}" local_config["logging"]["endpoint"] = log_endpoint try: logger = Logger( local_config["logging"], is_master=True, log_endpoint=log_endpoint, ) with logger: setproctitle("backend.ai: storage-proxy") log.info("Backend.AI Storage Proxy", VERSION) log.info("Runtime: {0}", env_info()) log.info("Node ID: {0}", local_config["storage-proxy"]["node-id"]) log_config = logging.getLogger("ai.backend.agent.config") if local_config["debug"]["enabled"]: log_config.debug("debug mode enabled.") if "debug" in local_config and local_config["debug"]["enabled"]: print("== Storage proxy configuration ==") pprint(local_config) if local_config["storage-proxy"]["event-loop"] == "uvloop": import uvloop uvloop.install() log.info("Using uvloop as the event loop backend") aiotools.start_server( server_main_logwrapper, num_workers=local_config["storage-proxy"]["num-proc"], args=(local_config, log_endpoint), ) log.info("exit.") finally: if local_config["storage-proxy"]["pid-file"].is_file(): # check is_file() to prevent deleting /dev/null! local_config["storage-proxy"]["pid-file"].unlink() return 0
def main(cli_ctx, config_path, debug): # Determine where to read configuration. raw_cfg, cfg_src_path = config.read_from_file(config_path, 'storage-proxy') config.override_with_env(raw_cfg, ('etcd', 'namespace'), 'BACKEND_NAMESPACE') config.override_with_env(raw_cfg, ('etcd', 'addr'), 'BACKEND_ETCD_ADDR') config.override_with_env(raw_cfg, ('etcd', 'user'), 'BACKEND_ETCD_USER') config.override_with_env(raw_cfg, ('etcd', 'password'), 'BACKEND_ETCD_PASSWORD') if debug: config.override_key(raw_cfg, ('debug', 'enabled'), True) try: local_config = config.check(raw_cfg, local_config_iv) local_config['_src'] = cfg_src_path except config.ConfigurationError as e: print( 'ConfigurationError: Validation of agent configuration has failed:', file=sys.stderr) print(pformat(e.invalid_data), file=sys.stderr) raise click.Abort() if local_config['debug']['enabled']: config.override_key(local_config, ('logging', 'level'), 'DEBUG') config.override_key(local_config, ('logging', 'pkg-ns', 'ai.backend'), 'DEBUG') # if os.getuid() != 0: # print('Storage agent can only be run as root', file=sys.stderr) # raise click.Abort() multiprocessing.set_start_method('spawn') if cli_ctx.invoked_subcommand is None: local_config['storage-proxy']['pid-file'].write_text(str(os.getpid())) log_sockpath = Path( f'/tmp/backend.ai/ipc/storage-proxy-logger-{os.getpid()}.sock') log_sockpath.parent.mkdir(parents=True, exist_ok=True) log_endpoint = f'ipc://{log_sockpath}' local_config['logging']['endpoint'] = log_endpoint try: logger = Logger(local_config['logging'], is_master=True, log_endpoint=log_endpoint) with logger: setproctitle('backend.ai: storage-proxy') log.info('Backend.AI Storage Proxy', VERSION) log.info('Runtime: {0}', env_info()) log.info('Node ID: {0}', local_config['storage-proxy']['node-id']) log_config = logging.getLogger('ai.backend.agent.config') if local_config['debug']['enabled']: log_config.debug('debug mode enabled.') if 'debug' in local_config and local_config['debug']['enabled']: print('== Storage proxy configuration ==') pprint(local_config) if local_config['storage-proxy']['event-loop'] == 'uvloop': import uvloop uvloop.install() log.info('Using uvloop as the event loop backend') aiotools.start_server( server_main_logwrapper, use_threading=False, num_workers=local_config['storage-proxy']['num-proc'], args=(local_config, log_endpoint), ) log.info('exit.') finally: if local_config['storage-proxy']['pid-file'].is_file(): # check is_file() to prevent deleting /dev/null! local_config['storage-proxy']['pid-file'].unlink() return 0
ak = 'AKIA' + base64.b32encode(secrets.token_bytes(10)).decode('ascii') sk = secrets.token_urlsafe(30) return ak, sk def create_app(): app = web.Application() app['prefix'] = 'auth' # slashed to distinguish with "/vN/authorize" app['api_versions'] = (1, 2, 3, 4) res = app.router.add_resource(r'') res.add_route('GET', auth_test) res.add_route('POST', auth_test) return app, [auth_middleware] if __name__ == '__main__': def auth_args(parser): parser.add('--generate-keypair', action='store_true', default=False, help='Generate a pair of access key and secret key.') config = load_config(extra_args_funcs=(auth_args, Logger.update_log_args)) logger = Logger(config) logger.add_pkg('ai.backend') with logger: if config.generate_keypair: ak, sk = generate_keypair() print(f'Access Key: {ak} ({len(ak)} bytes)') print(f'Secret Key: {sk} ({len(sk)} bytes)')