def test_context(self): ''' Ensure the context manager results in an update when changes are made, works with existing configs, etc. ''' with tempfile.TemporaryDirectory() as root: root = pathlib.Path(root) path = pathlib.Path(root / 'hypergolix.yml') config = Config(path) other_cfg = Config(path) self.assertEqual(config, other_cfg) with Config(path) as config: config.instrumentation.debug = True self.assertNotEqual(config, other_cfg) other_cfg.reload() self.assertEqual(config, other_cfg) with config: config.instrumentation.debug = False self.assertNotEqual(config, other_cfg) other_cfg.reload() self.assertEqual(config, other_cfg)
def start(namespace=None): ''' Starts a Hypergolix daemon. ''' with Daemonizer() as (is_setup, daemonizer): # Need these so that the second time around doesn't NameError user_id = None password = None pid_file = None parent_port = 7771 homedir = None if is_setup: with Config() as config: user_id = config.user_id password = config.password # Convert the path to a str pid_file = str(config.pid_file) homedir = str(config.home_dir) if password is None: password = _request_password(user_id) print('Starting Hypergolix...') # Daemonize. Don't strip cmd-line arguments, or we won't know to # continue with startup is_parent, user_id, password = daemonizer(pid_file, user_id, password, chdir=homedir) if is_parent: # Set up a logging server that we can print() to the terminal _startup_listener(port=parent_port, timeout=60) ##################### # PARENT EXITS HERE # ##################### # Daemonized child only from here on out. with _StartupReporter(parent_port) as startup_logger: # We need to set up a signal handler ASAP with Config() as config: pid_file = str(config.pid_file) sighandler = SignalHandler1(pid_file) sighandler.start() core = app_core(user_id, password, startup_logger) startup_logger.info('Hypergolix startup complete.') # Wait indefinitely until signal caught. # TODO: literally anything smarter than this. try: while True: time.sleep(.5) except SIGTERM: logger.info('Caught SIGTERM. Exiting.') del core
def test_decode(self): ''' Test decoding both as round-trip and against a test vector. ''' predecoded = obj_cfg decoded = Config(pathlib.Path()) decoded.decode(vec_cfg) freshdump_1 = predecoded.encode() freshdump_2 = decoded.encode() self.assertEqual(freshdump_1, freshdump_2) self.assertEqual(freshdump_2, vec_cfg)
def stop(namespace=None): ''' Stops the Hypergolix daemon. ''' with Config.find() as config: pid_file = str(config.process.pid_file) daemoniker.send(pid_file, SIGTERM)
def stop(namespace=None): ''' Stops the Hypergolix daemon. ''' with Config() as config: pid_file = str(config.pid_file) daemoniker.send(pid_file, SIGTERM)
async def setup(self): ''' Extend setup to also close the boot logger and, if desired, save the config. ''' await super().setup() try: if self._save_cfg: user_id = self.account._user_id fingerprint = self.account._fingerprint logger.critical('Account created. Record these values in ' + 'case you need to log in to Hypergolix from ' + 'another machine, or in case your config ' + 'file is corrupted or lost:') logger.critical('User ID:\n' + user_id.as_str()) logger.critical('Fingerprint:\n' + fingerprint.as_str()) with Config.load(self._cfg_path) as config: config.user.fingerprint = fingerprint config.user.user_id = user_id finally: logger.critical('Hypergolix boot complete.') self._boot_logger.stop()
def stop(namespace=None): ''' Stops the Hypergolix daemon. ''' if namespace.pidfile is not None: raise RuntimeError('Server pidfile specification through CLI is no ' + 'longer supported. Edit hypergolix.yml ' + 'configuration file instead.') config = Config.find() daemoniker.send(str(config.server.pid_file), SIGTERM)
def make_fixtures(debug, hgx_root_1, hgx_root_2): ''' Makes fixtures for the test. hgx_root_# is the root app directory, used by config. It contains the cache directory. ''' server_port = 6022 aengel = Aengel() with Config(hgx_root_1) as config: config.set_remote('127.0.0.1', server_port, False) config.ipc_port = 6023 with Config(hgx_root_2) as config: config.set_remote('127.0.0.1', server_port, False) config.ipc_port = 6024 hgxserver = _hgx_server( host='127.0.0.1', port=server_port, cache_dir=None, debug=debug, traceur=False, aengel=aengel, ) # localhost:6023, no tls hgxraz = app_core(user_id=None, password='******', startup_logger=None, aengel=aengel, _scrypt_hardness=1024, hgx_root=hgx_root_1, enable_logs=False) # localhost:6024, no tls hgxdes = app_core(user_id=None, password='******', startup_logger=None, aengel=aengel, _scrypt_hardness=1024, hgx_root=hgx_root_2, enable_logs=False) return hgxserver, hgxraz, hgxdes, aengel
def test_find_cfg_from_env(self): with tempfile.TemporaryDirectory() as root: os.environ['HYPERGOLIX_HOME'] = root try: fake_config = pathlib.Path(root) / 'hypergolix.yml' # Create a fake file to pick up its existence fake_config.touch() config = Config.find() self.assertEqual(config.path, fake_config) finally: del os.environ['HYPERGOLIX_HOME']
def test_context(self): ''' Ensure the context manager results in an update when changes are made, works with existing configs, etc. ''' with tempfile.TemporaryDirectory() as root: root = pathlib.Path(root) other_cfg = _make_blank_cfg() with Config(root) as config: self.assertEqual(config._cfg, other_cfg) config.debug_mode = True other_cfg['instrumentation'].debug = True self.assertEqual(config._cfg, other_cfg) with Config(root) as config: self.assertEqual(config._cfg, other_cfg) config.debug_mode = False other_cfg['instrumentation'].debug = False self.assertEqual(config._cfg, other_cfg) with Config(root) as config: self.assertEqual(config._cfg, other_cfg)
def test_upgrade(self): ''' Test loading old configs is equivalent to loading new ones. ''' new = Config(pathlib.Path()) old = Config(pathlib.Path()) new.decode(vec_cfg) old.decode(vec_cfg_depr) self.assertEqual(new, old)
def test_stuffs(self): ''' Tests attribute manipulation. ''' with tempfile.TemporaryDirectory() as root: root = pathlib.Path(root) homedir = _ensure_hgx_homedir(root) with Config(root) as config: self.assertEqual(config.home_dir, homedir) self.assertEqual(config.remotes, tuple()) self.assertEqual(config.fingerprint, None) self.assertEqual(config.user_id, None) self.assertEqual(config.password, None) self.assertEqual(config.log_verbosity, 'warning') self.assertEqual(config.debug_mode, False) config.set_remote('host', 123, True) self.assertIn(('host', 123, True), config.remotes) config.remove_remote('host', 123) self.assertNotIn(('host', 123, True), config.remotes) # Test fingerprints and user_id fingerprint = make_random_ghid() user_id = make_random_ghid() config.fingerprint = fingerprint self.assertEqual(config.fingerprint, fingerprint) config.user_id = user_id self.assertEqual(config.user_id, user_id) # Test modification updates appropriately fingerprint = make_random_ghid() user_id = make_random_ghid() config.fingerprint = fingerprint self.assertEqual(config.fingerprint, fingerprint) config.user_id = user_id self.assertEqual(config.user_id, user_id) # Now everything else config.log_verbosity = 'info' self.assertEqual(config.log_verbosity, 'info') config.debug_mode = True self.assertEqual(config.debug_mode, True)
def test_encode(self): ''' Test encoding both as round-trip and against a test vector. ''' preencoded = vec_cfg encoded = obj_cfg.encode() freshconfig_1 = Config(pathlib.Path()) freshconfig_2 = Config(pathlib.Path()) freshconfig_1.decode(preencoded) freshconfig_2.decode(encoded) self.assertEqual(freshconfig_1, freshconfig_2) self.assertEqual(freshconfig_2, obj_cfg)
def test_find_cfg_from_appdata(self): with tempfile.TemporaryDirectory() as root: try: oldappdata = os.environ['LOCALAPPDATA'] except KeyError: oldappdata = None os.environ['LOCALAPPDATA'] = root try: fake_config = pathlib.Path(root) / 'hypergolix.yml' # Create a fake file to pick up its existence fake_config.touch() config = Config.find() self.assertEqual(config.path, fake_config) finally: if oldappdata is None: del os.environ['LOCALAPPDATA'] else: os.environ['LOCALAPPDATA'] = oldappdata
def run_daemon(cfg_path, pid_file, parent_port, account_entity, root_secret): ''' Start the actual Hypergolix daemon. ''' # Start reporting to our parent about how stuff is going. parent_signaller = _StartupReporter(parent_port) startup_logger = parent_signaller.start() try: config = Config.load(cfg_path) # Convert paths to strs and make sure the dirs exist cache_dir = str(config.process.ghidcache) log_dir = str(config.process.logdir) _ensure_dir_exists(config.process.ghidcache) _ensure_dir_exists(config.process.logdir) debug = _default_to(config.instrumentation.debug, False) verbosity = _default_to(config.instrumentation.verbosity, 'info') logutils.autoconfig( tofile = True, logdirname = log_dir, loglevel = verbosity, logname = 'hgxapp' ) ipc_port = config.process.ipc_port remotes = config.remotes # Look to see if we have an existing user_id to determine behavior save_cfg = not bool(config.user.user_id) hgxcore = _DaemonCore( cache_dir = cache_dir, ipc_port = ipc_port, reusable_loop = False, threaded = False, debug = debug, cfg_path = cfg_path, save_cfg = save_cfg, boot_logger = parent_signaller ) for remote in remotes: hgxcore.add_remote( connection_cls = WSBeatingConn, host = remote.host, port = remote.port, tls = remote.tls ) account = Account( user_id = account_entity, root_secret = root_secret, hgxcore = hgxcore ) hgxcore.account = account # We need a signal handler for that. def signal_handler(signum): logger.info('Caught signal. Exiting.') hgxcore.stop_threadsafe_nowait() # Normally I'd do this within daemonization, but in this case, we need # to wait to have access to the handler. sighandler = SignalHandler1( pid_file, sigint = signal_handler, sigterm = signal_handler, sigabrt = signal_handler ) sighandler.start() startup_logger.info('Booting Hypergolix...') hgxcore.start() except Exception: startup_logger.error('Failed to start Hypergolix:\n' + ''.join(traceback.format_exc())) raise finally: # This is idempotent, so no worries if we already called it parent_signaller.stop()
def test_full(self): ''' Test a full command chain for everything. ''' with tempfile.TemporaryDirectory() as root: root = pathlib.Path(root) blank = Config(root / 'hypergolix.yml') blank.coerce_defaults() debug = Config(root / 'hypergolix.yml') debug.coerce_defaults() debug.instrumentation.debug = True nodebug = Config(root / 'hypergolix.yml') nodebug.coerce_defaults() nodebug.instrumentation.debug = False loud = Config(root / 'hypergolix.yml') loud.coerce_defaults() loud.instrumentation.verbosity = 'info' loud.instrumentation.debug = False normal = Config(root / 'hypergolix.yml') normal.coerce_defaults() normal.instrumentation.verbosity = 'warning' normal.instrumentation.debug = False host1 = Config(root / 'hypergolix.yml') host1.coerce_defaults() host1.remotes.append(Remote('host1', 123, True)) host1.instrumentation.verbosity = 'warning' host1.instrumentation.debug = False host1hgx = Config(root / 'hypergolix.yml') host1hgx.coerce_defaults() host1hgx.remotes.append(Remote('host1', 123, True)) host1hgx.remotes.append(Remote('hgx.hypergolix.com', 443, True)) host1hgx.instrumentation.verbosity = 'warning' host1hgx.instrumentation.debug = False host1host2f = Config(root / 'hypergolix.yml') host1host2f.coerce_defaults() host1host2f.remotes.append(Remote('host1', 123, True)) host1host2f.remotes.append(Remote('host2', 123, False)) host1host2f.instrumentation.verbosity = 'warning' host1host2f.instrumentation.debug = False host1host2 = Config(root / 'hypergolix.yml') host1host2.coerce_defaults() host1host2.remotes.append(Remote('host1', 123, True)) host1host2.remotes.append(Remote('host2', 123, True)) host1host2.instrumentation.verbosity = 'warning' host1host2.instrumentation.debug = False # Definitely want to control the order of execution for this. valid_commands = [ ('config', blank), ('config --whoami', blank) ] deprecated_commands = [ ('config --debug', debug), ('config --no-debug', nodebug), ('config --verbosity loud', loud), ('config --verbosity normal', normal), ('config -ah host1 123 t', host1), ('config --addhost host1 123 t', host1), ('config -a hgx', host1hgx), ('config --add hgx', host1hgx), ('config -r hgx', host1), ('config --remove hgx', host1), # Note switch of TLS flag ('config -ah host2 123 f', host1host2f), # Note return of TLS flag ('config --addhost host2 123 t', host1host2), ('config -rh host2 123', host1), ('config --removehost host2 123', host1), ('config -o local', normal), ('config --only local', normal), ('config -ah host1 123 t -ah host2 123 t', host1host2), ] failing_commands = [ 'config -zz top', 'config --verbosity XXXTREEEEEEEME', 'config --debug --no-debug', 'config -o local -a hgx', ] for cmd_str, cmd_result in valid_commands: with self.subTest(cmd_str): cfg_path = root / 'hypergolix.yml' # NOTE THAT THESE TESTS ARE CUMULATIVE! We definitely DON'T # want to start with a fresh config each time around, or # the tests will fail! argv = cmd_str.split() argv.append('--root') argv.append(str(cfg_path)) with _NoSTDOUT(): ingest_args(argv) config = Config.load(cfg_path) # THE PROBLEM HERE IS NOT JUST COERCE DEFAULTS! config.py, # in its handle_args section, is passing in default values # that are interfering with everything else. self.assertEqual(config, cmd_result) for cmd_str, cmd_result in deprecated_commands: with self.subTest(cmd_str): cfg_path = root / 'hypergolix.yml' # NOTE THAT THESE TESTS ARE CUMULATIVE! We definitely DON'T # want to start with a fresh config each time around, or # the tests will fail! argv = cmd_str.split() argv.append('--root') argv.append(str(cfg_path)) with _NoSTDOUT(), self.assertWarns(DeprecationWarning): ingest_args(argv) config = Config.load(cfg_path) # THE PROBLEM HERE IS NOT JUST COERCE DEFAULTS! config.py, # in its handle_args section, is passing in default values # that are interfering with everything else. self.assertEqual(config, cmd_result) # Don't need the temp dir for this; un-context to escape permissions for cmd_str in failing_commands: with self.subTest(cmd_str): argv = cmd_str.split() argv.append('--root') argv.append(str(root)) # Note that argparse will always push usage to stderr in a # suuuuuuper annoying way if we don't suppress it. with self.assertRaises(SystemExit), _NoSTDERR(), _NoSTDOUT(): ingest_args(argv)
def test_manipulate_remotes(self): config = Config(pathlib.Path()) rem1 = Remote('host1', 123, False) rem2 = Remote('host2', 123, False) rem2a = Remote('host2', 123, True) rem3 = Remote('host3', 123, True) self.assertIsNone(config.index_remote(rem1)) self.assertIsNone(config.index_remote(rem2)) self.assertIsNone(config.index_remote(rem2a)) self.assertIsNone(config.index_remote(rem3)) config.set_remote(rem1.host, rem1.port, rem1.tls) self.assertEqual(config.index_remote(rem1), 0) config.set_remote(rem2.host, rem2.port, rem2.tls) self.assertEqual(config.index_remote(rem1), 0) self.assertEqual(config.index_remote(rem2), 1) self.assertEqual(config.index_remote(rem2a), 1) config.set_remote(rem3.host, rem3.port, rem3.tls) self.assertEqual(config.index_remote(rem1), 0) self.assertEqual(config.index_remote(rem2), 1) self.assertEqual(config.index_remote(rem3), 2) config.set_remote(rem2a.host, rem2a.port, rem2a.tls) self.assertEqual(config.index_remote(rem1), 0) self.assertEqual(config.index_remote(rem2), 1) self.assertEqual(config.index_remote(rem3), 2) config.remove_remote(rem2.host, rem2.port) self.assertEqual(config.index_remote(rem1), 0) self.assertIsNone(config.index_remote(rem2)) self.assertEqual(config.index_remote(rem3), 1)
def do_setup(): ''' Does initial setup of the daemon BEFORE daemonizing. ''' try: config = Config.find() except ConfigMissing: print('Welcome to Hypergolix!') print('No existing configuration found; creating a new one.') config = Config.wherever() # We do need to wrap this, so that we actually store a new config (and # so that we actually coerce the defaults) with config: user_id = config.user.user_id fingerprint = config.user.fingerprint root_secret = config.user.root_secret # Convert the path to a str pid_file = str(config.process.pid_file) if bool(user_id) ^ bool(fingerprint): raise ConfigIncomplete('Invalid config. Config must declare both ' + 'user_id and fingerprint, or neither.') # We have no root secret, so we need to get a password and then inflate # it. if not root_secret: # We have an existing account, so do a single prompt. if user_id: account_entity = user_id password = _enter_password() # We have no existing account, so do a double prompt (and then # generate keys) and then inflate the password. else: password = _create_password() print('Generating a new set of private keys. This may take ' + 'a while.') account_entity = FirstParty() fingerprint = account_entity.ghid account_entity = account_entity._serialize() print('Private keys generated.') print('Expanding password using scrypt. This may take a while.') root_secret = _expand_password( salt_ghid = fingerprint, password = password ) # We have a root secret... else: print('Using stored secret.') # ...and an existing account if user_id: account_entity = user_id # ...but need a new account else: print('Generating a new set of private keys. This may take ' + 'a while.') account_entity = FirstParty() print('Private keys generated.') account_entity = account_entity._serialize() return config.path, pid_file, account_entity, root_secret
def app_core(user_id, password, startup_logger, aengel=None, _scrypt_hardness=None, hgx_root=None, enable_logs=True): ''' This is where all of the UX goes for the service itself. From here, we build a credential, then a bootstrap, and then persisters, IPC, etc. Expected defaults: host: 'localhost' port: 7770 tls: True ipc_port: 7772 debug: False logfile: None verbosity: 'warning' traceur: False ''' if startup_logger is not None: # At some point, this will need to restore the module logger, but for # now it really doesn't make any difference whatsoever effective_logger = startup_logger else: effective_logger = logger with Config(hgx_root) as config: # Convert paths to strs cache_dir = str(config.cache_dir) log_dir = str(config.log_dir) if user_id is None: user_id = config.user_id debug = config.debug_mode verbosity = config.log_verbosity ipc_port = config.ipc_port remotes = config.remotes if enable_logs: logutils.autoconfig( tofile = True, logdirname = log_dir, loglevel = verbosity, logname = 'hgxapp' ) if not aengel: aengel = Aengel() core = AgentBootstrap(aengel=aengel, debug=debug, cache_dir=cache_dir) core.assemble() # In this case, we have no existing user_id. if user_id is None: user_id = core.bootstrap_zero( password = password, _scrypt_hardness = _scrypt_hardness ) effective_logger.critical( 'Identity created. Your user ID is ' + str(user_id) + '. You ' + 'will need your user ID to log in to Hypergolix from another ' + 'machine, or if your Hypergolix configuration file is corrupted ' + 'or lost.' ) with Config(hgx_root) as config: config.fingerprint = core.whoami config.user_id = user_id # Hey look, we have an existing user. else: core.bootstrap( user_id = user_id, password = password, _scrypt_hardness = _scrypt_hardness, ) effective_logger.info('Login successful.') # Add all of the remotes to a namespace preserver persisters = [] for remote in remotes: try: persister = Autocomms( autoresponder_name = 'remrecli', autoresponder_class = PersisterBridgeClient, connector_name = 'remwscli', connector_class = WSBasicClient, connector_kwargs = { 'host': remote.host, 'port': remote.port, 'tls': remote.tls, }, debug = debug, aengel = aengel, ) except CancelledError: effective_logger.error( 'Error while connecting to upstream remote at ' + remote.host + ':' + str(remote.port) + '. Connection will ' + 'only be reattempted after restarting Hypergolix.' ) else: core.salmonator.add_upstream_remote(persister) persisters.append(persister) # Finally, add the ipc system core.ipccore.add_ipc_server( 'wslocal', WSBasicServer, host = 'localhost', port = ipc_port, tls = False, debug = debug, aengel = aengel, threaded = True, thread_name = _generate_threadnames('ipc-ws')[0], ) return persisters, core, aengel
def start(namespace=None): ''' Starts a Hypergolix daemon. ''' # Command arg support is deprecated. if namespace is not None: # Gigantic error trap if ((namespace.host is not None) | (namespace.port is not None) | (namespace.debug is not None) | (namespace.traceur is not None) | (namespace.pidfile is not None) | (namespace.logdir is not None) | (namespace.cachedir is not None) | (namespace.chdir is not None) | (namespace.verbosity is not None)): raise RuntimeError('Server configuration through CLI is no ' + 'longer supported. Edit hypergolix.yml ' + 'configuration file instead.') with Daemonizer() as (is_setup, daemonizer): # Get our config path in setup, so that we error out before attempting # to daemonize (if anything is wrong). if is_setup: config = Config.find() config_path = config.path chdir = config_path.parent pid_file = config.server.pid_file else: config_path = None pid_file = None chdir = None # Daemonize. is_parent, config_path = daemonizer( str(pid_file), config_path, chdir = str(chdir), explicit_rescript = '-m hypergolix.service' ) ##################### # PARENT EXITS HERE # ##################### config = Config.load(config_path) _ensure_dir_exists(config.server.ghidcache) _ensure_dir_exists(config.server.logdir) debug = _default_to(config.server.debug, False) verbosity = _default_to(config.server.verbosity, 'info') logutils.autoconfig( tofile = True, logdirname = config.server.logdir, logname = 'hgxserver', loglevel = verbosity ) logger.debug('Parsing config...') host = _cast_host(config.server.host) rps = RemotePersistenceServer( config.server.ghidcache, host, config.server.port, reusable_loop = False, threaded = False, debug = debug ) logger.debug('Starting health check...') # Start a health check healthcheck_server, healthcheck_thread = _serve_healthcheck() healthcheck_thread.start() logger.debug('Starting signal handler...') def signal_handler(signum): logger.info('Caught signal. Exiting.') healthcheck_server.shutdown() rps.stop_threadsafe_nowait() # Normally I'd do this within daemonization, but in this case, we need to # wait to have access to the handler. sighandler = SignalHandler1( str(config.server.pid_file), sigint = signal_handler, sigterm = signal_handler, sigabrt = signal_handler ) sighandler.start() logger.info('Starting remote persistence server...') rps.start()
def test_full(self): ''' Test a full command chain for everything. ''' blank = _make_blank_cfg() debug = _make_blank_cfg() debug['instrumentation'].debug = True loud = _make_blank_cfg() loud['instrumentation'].verbosity = 'info' host1 = _make_blank_cfg() host1['remotes'].append(('host1', 123, True)) host1hgxtest = _make_blank_cfg() host1hgxtest['remotes'].append(('host1', 123, True)) host1hgxtest['remotes'].append(('hgxtest.hypergolix.com', 443, True)) host1host2f = _make_blank_cfg() host1host2f['remotes'].append(('host1', 123, True)) host1host2f['remotes'].append(('host2', 123, False)) host1host2 = _make_blank_cfg() host1host2['remotes'].append(('host1', 123, True)) host1host2['remotes'].append(('host2', 123, True)) # Definitely want to control the order of execution for this. valid_commands = [ ('--debug', debug), ('--no-debug', blank), ('--verbosity loud', loud), ('--verbosity normal', blank), ('-ah host1 123 t', host1), ('--addhost host1 123 t', host1), ('-a hgxtest', host1hgxtest), ('--add hgxtest', host1hgxtest), ('-r hgxtest', host1), ('--remove hgxtest', host1), # Note switch of TLS flag ('-ah host2 123 f', host1host2f), # Note return of TLS flag ('--addhost host2 123 t', host1host2), ('-rh host2 123', host1), ('--removehost host2 123', host1), ('-o local', blank), ('--only local', blank), ('-ah host1 123 t -ah host2 123 t', host1host2), ] failing_commands = [ '-zz top', '--verbosity XXXTREEEEEEEME', '--debug --no-debug', '-o local -a hgxtest', ] with tempfile.TemporaryDirectory() as root: root = pathlib.Path(root) for cmd_str, cmd_result in valid_commands: with self.subTest(cmd_str): argv = cmd_str.split() args = _ingest_args(argv) _handle_args(args, root) with Config(root) as config: self.assertEqual(config._cfg, cmd_result) # Don't need the temp dir for this. for cmd_str in failing_commands: with self.subTest(cmd_str): argv = cmd_str.split() # Note that argparse will always push usage to stderr in a # suuuuuuper annoying way if we don't suppress it. with self.assertRaises(SystemExit), _SuppressSTDERR(): args = _ingest_args(argv)