def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None):
"""Scrape list of suites from the filesystem.
Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for
active suites.
Yields:
tuple - (reg, host, port)
"""
srv_files_mgr = SuiteSrvFilesManager()
if owner_pattern is None:
# Run directory of current user only
run_dirs = [(glbl_cfg().get_host_item('run directory'), None)]
else:
# Run directory of all users matching "owner_pattern".
# But skip those with /nologin or /false shells
run_dirs = []
skips = ('/false', '/nologin')
for pwent in getpwall():
if any(pwent.pw_shell.endswith(s) for s in (skips)):
continue
if owner_pattern.match(pwent.pw_name):
run_dirs.append((
glbl_cfg().get_host_item(
'run directory',
owner=pwent.pw_name,
owner_home=pwent.pw_dir),
pwent.pw_name))
if cylc.flow.flags.debug:
sys.stderr.write('Listing suites:%s%s\n' % (
DEBUG_DELIM, DEBUG_DELIM.join(item[1] for item in run_dirs if
item[1] is not None)))
for run_d, owner in run_dirs:
for dirpath, dnames, _ in os.walk(run_d, followlinks=True):
# Always descend for top directory, but
# don't descend further if it has a .service/ or log/ dir
if dirpath != run_d and (
srv_files_mgr.DIR_BASE_SRV in dnames or 'log' in dnames):
dnames[:] = []
# Filter suites by name
reg = os.path.relpath(dirpath, run_d)
if reg_pattern and not reg_pattern.match(reg):
continue
# Choose only suites with .service and matching filter
try:
contact_data = srv_files_mgr.load_contact_file(reg, owner)
except (SuiteServiceFileError, IOError, TypeError, ValueError):
continue
else:
yield (
reg,
contact_data[srv_files_mgr.KEY_HOST],
contact_data[srv_files_mgr.KEY_PORT]
)
class TestSuiteSrvFilesManager(unittest.TestCase):
def setUp(self):
self.suite_srv_files_mgr = SuiteSrvFilesManager()
@mock.patch('cylc.flow.suite_srv_files_mgr.os')
def test_register(self, mocked_os):
"""Test the SuiteSrvFilesManager register function."""
def mkdirs_standin(_, exist_ok=False):
return True
# we do not need to mock these functions
mocked_os.path.basename = os.path.basename
mocked_os.path.join = os.path.join
mocked_os.path.normpath = os.path.normpath
mocked_os.path.dirname = os.path.dirname
mocked_os.makedirs = mkdirs_standin
mocked_os.path.abspath = lambda x: x
for reg, source, redirect, cwd, isabs, isfile, \
suite_srv_dir, readlink, expected_symlink, \
expected, e_expected, e_message \
in get_register_test_cases():
mocked_os.getcwd = lambda: cwd
mocked_os.path.isabs = lambda x: isabs
mocked_os.path.isfile = lambda x: isfile
self.suite_srv_files_mgr.get_suite_srv_dir = mock.MagicMock(
return_value=suite_srv_dir
)
if readlink == OSError:
mocked_os.readlink.side_effect = readlink
else:
mocked_os.readlink.side_effect = lambda x: readlink
if e_expected is None:
reg = self.suite_srv_files_mgr.register(reg, source, redirect)
self.assertEqual(expected, reg)
if mocked_os.symlink.call_count > 0:
# first argument, of the first call
arg0 = mocked_os.symlink.call_args[0][0]
self.assertEqual(expected_symlink, arg0)
else:
with self.assertRaises(e_expected) as cm:
self.suite_srv_files_mgr.register(reg, source, redirect)
if e_message is not None:
the_exception = cm.exception
self.assertTrue(e_message in str(the_exception),
str(the_exception))
class TestSuiteSrvFilesManager(unittest.TestCase):
def setUp(self):
self.suite_srv_files_mgr = SuiteSrvFilesManager()
@mock.patch('cylc.flow.suite_srv_files_mgr.os')
def test_register(self, mocked_os):
"""Test the SuiteSrvFilesManager register function."""
def mkdirs_standin(_, exist_ok=False):
return True
# we do not need to mock these functions
mocked_os.path.basename.side_effect = os.path.basename
mocked_os.path.join = os.path.join
mocked_os.path.normpath = os.path.normpath
mocked_os.path.dirname = os.path.dirname
mocked_os.makedirs.side_effect = mkdirs_standin
mocked_os.path.abspath.side_effect = lambda x: x
for reg, source, redirect, cwd, isabs, isfile, \
suite_srv_dir, readlink, expected_symlink, \
expected, e_expected, e_message \
in get_register_test_cases():
mocked_os.getcwd.side_effect = lambda: cwd
mocked_os.path.isabs.side_effect = lambda x: isabs
mocked_os.path.isfile = lambda x: isfile
self.suite_srv_files_mgr.get_suite_srv_dir = mock.MagicMock(
return_value=suite_srv_dir
)
if readlink == OSError:
mocked_os.readlink.side_effect = readlink
else:
mocked_os.readlink.side_effect = lambda x: readlink
if e_expected is None:
reg = self.suite_srv_files_mgr.register(reg, source, redirect)
self.assertEqual(expected, reg)
if mocked_os.symlink.call_count > 0:
# first argument, of the first call
arg0 = mocked_os.symlink.call_args[0][0]
self.assertEqual(expected_symlink, arg0)
else:
with self.assertRaises(e_expected) as cm:
self.suite_srv_files_mgr.register(reg, source, redirect)
if e_message is not None:
the_exception = cm.exception
self.assertTrue(e_message in str(the_exception),
str(the_exception))
def get_location(cls, suite: str, owner: str, host: str):
"""Extract host and port from a suite's contact file.
NB: if it fails to load the suite contact file, it will exit.
Args:
suite (str): suite name
owner (str): owner of the suite
host (str): host name
Returns:
Tuple[str, int]: tuple with the host name and port number.
Raises:
ClientError: if the suite is not running.
"""
try:
contact = SuiteSrvFilesManager().load_contact_file(
suite, owner, host)
except SuiteServiceFileError:
raise ClientError(f'Contact info not found for suite '
f'"{suite}", suite not running?')
if not host:
host = contact[SuiteSrvFilesManager.KEY_HOST]
host = get_fqdn_by_host(host)
port = int(contact[SuiteSrvFilesManager.KEY_PORT])
return host, port
def scheduler_cli(parser, options, args, is_restart=False):
"""CLI main."""
reg = args[0]
# Check suite is not already running before start of host selection.
try:
SuiteSrvFilesManager().detect_old_contact_file(reg)
except SuiteServiceFileError as exc:
sys.exit(exc)
# Create auth files if needed.
SuiteSrvFilesManager().create_auth_files(reg)
# Extract job.sh from library, for use in job scripts.
extract_resources(SuiteSrvFilesManager().get_suite_srv_dir(reg),
['etc/job.sh'])
# Check whether a run host is explicitly specified, else select one.
if not options.host:
try:
host = HostAppointer().appoint_host()
except EmptyHostList as exc:
if cylc.flow.flags.debug:
raise
else:
sys.exit(str(exc))
if is_remote_host(host):
if is_restart:
base_cmd = ["restart"] + sys.argv[1:]
else:
base_cmd = ["run"] + sys.argv[1:]
# Prevent recursive host selection
base_cmd.append("--host=localhost")
return remote_cylc_cmd(base_cmd, host=host)
if remrun(set_rel_local=True): # State localhost as above.
sys.exit()
try:
SuiteSrvFilesManager().get_suite_source_dir(args[0], options.owner)
except SuiteServiceFileError:
# Source path is assumed to be the run directory
SuiteSrvFilesManager().register(args[0], get_suite_run_dir(args[0]))
try:
scheduler = Scheduler(is_restart, options, args)
except SuiteServiceFileError as exc:
sys.exit(exc)
scheduler.start()
def _auto_register():
"""Register a suite installed in the cylc-run directory."""
try:
reg = SuiteSrvFilesManager().register()
except SuiteServiceFileError as exc:
sys.exit(exc)
# Replace this process with "cylc run REG ..." for 'ps -f'.
os.execv(sys.argv[0], [sys.argv[0]] + [reg] + sys.argv[1:])
def get_secret(suite):
"""Return the secret used for encrypting messages.
Currently this is the suite passphrase. This means we are sending
many messages all encrypted with the same hash which isn't great.
TODO: Upgrade the secret to add foreword security.
"""
return SuiteSrvFilesManager().get_auth_item(
SuiteSrvFilesManager.FILE_BASE_PASSPHRASE, suite, content=True)
def _timeout_handler(suite: str, host: str, port: Union[int, str]):
"""Handle the eventuality of a communication timeout with the suite.
Args:
suite (str): suite name
host (str): host name
port (Union[int, str]): port number
Raises:
ClientError: if the suite has already stopped.
"""
if suite is None:
return
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
SuiteSrvFilesManager().detect_old_contact_file(suite, (host, port))
except (AssertionError, SuiteServiceFileError):
# * contact file not have matching (host, port) to suite proc
# * old contact file exists and the suite process still alive
return
else:
# the suite has stopped
raise ClientError('Suite "%s" already stopped' % suite)
def setUp(self):
self.suite_srv_files_mgr = SuiteSrvFilesManager()
def setUp(self):
self.suite_srv_files_mgr = SuiteSrvFilesManager()