def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None): """Scrape list of suites from the filesystem. Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for active suites. Yields: tuple - (reg, host, port) """ srv_files_mgr = SuiteSrvFilesManager() if owner_pattern is None: # Run directory of current user only run_dirs = [(glbl_cfg().get_host_item('run directory'), None)] else: # Run directory of all users matching "owner_pattern". # But skip those with /nologin or /false shells run_dirs = [] skips = ('/false', '/nologin') for pwent in getpwall(): if any(pwent.pw_shell.endswith(s) for s in (skips)): continue if owner_pattern.match(pwent.pw_name): run_dirs.append(( glbl_cfg().get_host_item( 'run directory', owner=pwent.pw_name, owner_home=pwent.pw_dir), pwent.pw_name)) if cylc.flow.flags.debug: sys.stderr.write('Listing suites:%s%s\n' % ( DEBUG_DELIM, DEBUG_DELIM.join(item[1] for item in run_dirs if item[1] is not None))) for run_d, owner in run_dirs: for dirpath, dnames, _ in os.walk(run_d, followlinks=True): # Always descend for top directory, but # don't descend further if it has a .service/ or log/ dir if dirpath != run_d and ( srv_files_mgr.DIR_BASE_SRV in dnames or 'log' in dnames): dnames[:] = [] # Filter suites by name reg = os.path.relpath(dirpath, run_d) if reg_pattern and not reg_pattern.match(reg): continue # Choose only suites with .service and matching filter try: contact_data = srv_files_mgr.load_contact_file(reg, owner) except (SuiteServiceFileError, IOError, TypeError, ValueError): continue else: yield ( reg, contact_data[srv_files_mgr.KEY_HOST], contact_data[srv_files_mgr.KEY_PORT] )
class TestSuiteSrvFilesManager(unittest.TestCase): def setUp(self): self.suite_srv_files_mgr = SuiteSrvFilesManager() @mock.patch('cylc.flow.suite_srv_files_mgr.os') def test_register(self, mocked_os): """Test the SuiteSrvFilesManager register function.""" def mkdirs_standin(_, exist_ok=False): return True # we do not need to mock these functions mocked_os.path.basename = os.path.basename mocked_os.path.join = os.path.join mocked_os.path.normpath = os.path.normpath mocked_os.path.dirname = os.path.dirname mocked_os.makedirs = mkdirs_standin mocked_os.path.abspath = lambda x: x for reg, source, redirect, cwd, isabs, isfile, \ suite_srv_dir, readlink, expected_symlink, \ expected, e_expected, e_message \ in get_register_test_cases(): mocked_os.getcwd = lambda: cwd mocked_os.path.isabs = lambda x: isabs mocked_os.path.isfile = lambda x: isfile self.suite_srv_files_mgr.get_suite_srv_dir = mock.MagicMock( return_value=suite_srv_dir ) if readlink == OSError: mocked_os.readlink.side_effect = readlink else: mocked_os.readlink.side_effect = lambda x: readlink if e_expected is None: reg = self.suite_srv_files_mgr.register(reg, source, redirect) self.assertEqual(expected, reg) if mocked_os.symlink.call_count > 0: # first argument, of the first call arg0 = mocked_os.symlink.call_args[0][0] self.assertEqual(expected_symlink, arg0) else: with self.assertRaises(e_expected) as cm: self.suite_srv_files_mgr.register(reg, source, redirect) if e_message is not None: the_exception = cm.exception self.assertTrue(e_message in str(the_exception), str(the_exception))
class TestSuiteSrvFilesManager(unittest.TestCase): def setUp(self): self.suite_srv_files_mgr = SuiteSrvFilesManager() @mock.patch('cylc.flow.suite_srv_files_mgr.os') def test_register(self, mocked_os): """Test the SuiteSrvFilesManager register function.""" def mkdirs_standin(_, exist_ok=False): return True # we do not need to mock these functions mocked_os.path.basename.side_effect = os.path.basename mocked_os.path.join = os.path.join mocked_os.path.normpath = os.path.normpath mocked_os.path.dirname = os.path.dirname mocked_os.makedirs.side_effect = mkdirs_standin mocked_os.path.abspath.side_effect = lambda x: x for reg, source, redirect, cwd, isabs, isfile, \ suite_srv_dir, readlink, expected_symlink, \ expected, e_expected, e_message \ in get_register_test_cases(): mocked_os.getcwd.side_effect = lambda: cwd mocked_os.path.isabs.side_effect = lambda x: isabs mocked_os.path.isfile = lambda x: isfile self.suite_srv_files_mgr.get_suite_srv_dir = mock.MagicMock( return_value=suite_srv_dir ) if readlink == OSError: mocked_os.readlink.side_effect = readlink else: mocked_os.readlink.side_effect = lambda x: readlink if e_expected is None: reg = self.suite_srv_files_mgr.register(reg, source, redirect) self.assertEqual(expected, reg) if mocked_os.symlink.call_count > 0: # first argument, of the first call arg0 = mocked_os.symlink.call_args[0][0] self.assertEqual(expected_symlink, arg0) else: with self.assertRaises(e_expected) as cm: self.suite_srv_files_mgr.register(reg, source, redirect) if e_message is not None: the_exception = cm.exception self.assertTrue(e_message in str(the_exception), str(the_exception))
def get_location(cls, suite: str, owner: str, host: str): """Extract host and port from a suite's contact file. NB: if it fails to load the suite contact file, it will exit. Args: suite (str): suite name owner (str): owner of the suite host (str): host name Returns: Tuple[str, int]: tuple with the host name and port number. Raises: ClientError: if the suite is not running. """ try: contact = SuiteSrvFilesManager().load_contact_file( suite, owner, host) except SuiteServiceFileError: raise ClientError(f'Contact info not found for suite ' f'"{suite}", suite not running?') if not host: host = contact[SuiteSrvFilesManager.KEY_HOST] host = get_fqdn_by_host(host) port = int(contact[SuiteSrvFilesManager.KEY_PORT]) return host, port
def scheduler_cli(parser, options, args, is_restart=False): """CLI main.""" reg = args[0] # Check suite is not already running before start of host selection. try: SuiteSrvFilesManager().detect_old_contact_file(reg) except SuiteServiceFileError as exc: sys.exit(exc) # Create auth files if needed. SuiteSrvFilesManager().create_auth_files(reg) # Extract job.sh from library, for use in job scripts. extract_resources(SuiteSrvFilesManager().get_suite_srv_dir(reg), ['etc/job.sh']) # Check whether a run host is explicitly specified, else select one. if not options.host: try: host = HostAppointer().appoint_host() except EmptyHostList as exc: if cylc.flow.flags.debug: raise else: sys.exit(str(exc)) if is_remote_host(host): if is_restart: base_cmd = ["restart"] + sys.argv[1:] else: base_cmd = ["run"] + sys.argv[1:] # Prevent recursive host selection base_cmd.append("--host=localhost") return remote_cylc_cmd(base_cmd, host=host) if remrun(set_rel_local=True): # State localhost as above. sys.exit() try: SuiteSrvFilesManager().get_suite_source_dir(args[0], options.owner) except SuiteServiceFileError: # Source path is assumed to be the run directory SuiteSrvFilesManager().register(args[0], get_suite_run_dir(args[0])) try: scheduler = Scheduler(is_restart, options, args) except SuiteServiceFileError as exc: sys.exit(exc) scheduler.start()
def _auto_register(): """Register a suite installed in the cylc-run directory.""" try: reg = SuiteSrvFilesManager().register() except SuiteServiceFileError as exc: sys.exit(exc) # Replace this process with "cylc run REG ..." for 'ps -f'. os.execv(sys.argv[0], [sys.argv[0]] + [reg] + sys.argv[1:])
def get_secret(suite): """Return the secret used for encrypting messages. Currently this is the suite passphrase. This means we are sending many messages all encrypted with the same hash which isn't great. TODO: Upgrade the secret to add foreword security. """ return SuiteSrvFilesManager().get_auth_item( SuiteSrvFilesManager.FILE_BASE_PASSPHRASE, suite, content=True)
def _timeout_handler(suite: str, host: str, port: Union[int, str]): """Handle the eventuality of a communication timeout with the suite. Args: suite (str): suite name host (str): host name port (Union[int, str]): port number Raises: ClientError: if the suite has already stopped. """ if suite is None: return # Cannot connect, perhaps suite is no longer running and is leaving # behind a contact file? try: SuiteSrvFilesManager().detect_old_contact_file(suite, (host, port)) except (AssertionError, SuiteServiceFileError): # * contact file not have matching (host, port) to suite proc # * old contact file exists and the suite process still alive return else: # the suite has stopped raise ClientError('Suite "%s" already stopped' % suite)
def setUp(self): self.suite_srv_files_mgr = SuiteSrvFilesManager()
def setUp(self): self.suite_srv_files_mgr = SuiteSrvFilesManager()