def main(is_restart=False):
"""CLI main."""
options, args = parse_commandline(is_restart)
if not args:
# Auto-registration: "cylc run" (no args) in source dir.
try:
reg = SuiteSrvFilesManager().register()
except SuiteServiceFileError as exc:
sys.exit(exc)
# Replace this process with "cylc run REG ..." for 'ps -f'.
os.execv(sys.argv[0], [sys.argv[0]] + [reg] + sys.argv[1:])
# Check suite is not already running before start of host selection.
try:
SuiteSrvFilesManager().detect_old_contact_file(args[0])
except SuiteServiceFileError as exc:
sys.exit(exc)
# Create auth files if needed. On a shared FS if the suite host changes
# this may (will?) renew the ssl.cert to reflect the change in host name.
SuiteSrvFilesManager().create_auth_files(args[0])
# Check whether a run host is explicitly specified, else select one.
if not options.host:
try:
host = HostAppointer().appoint_host()
except EmptyHostList as exc:
if cylc.flags.debug:
raise
else:
sys.exit(str(exc))
if is_remote_host(host):
if is_restart:
base_cmd = ["restart"] + sys.argv[1:]
else:
base_cmd = ["run"] + sys.argv[1:]
# Prevent recursive host selection
base_cmd.append("--host=localhost")
return remote_cylc_cmd(base_cmd, host=host)
if remrun(set_rel_local=True): # State localhost as above.
sys.exit()
try:
SuiteSrvFilesManager().get_suite_source_dir(args[0], options.owner)
except SuiteServiceFileError:
# Source path is assumed to be the run directory
SuiteSrvFilesManager().register(
args[0],
glbl_cfg().get_derived_host_item(args[0], 'suite run directory'))
try:
scheduler = Scheduler(is_restart, options, args)
except SuiteServiceFileError as exc:
sys.exit(exc)
scheduler.start()
def __init__(self,
suite,
owner=None,
host=None,
port=None,
timeout=None,
my_uuid=None,
print_uuid=False,
comms_protocol=None,
auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_protocol = comms_protocol
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
def __init__(
self, suite, owner=None, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False, auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.comms1 = {} # content in primary contact file
self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def get_scan_items_from_fs(owner_pattern=None, updater=None):
"""Get list of host:port available to scan using the file system.
Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for
active suites.
Return (list): List of (host, port) available for scan.
"""
srv_files_mgr = SuiteSrvFilesManager()
if owner_pattern is None:
# Run directory of current user only
run_dirs = [(GLOBAL_CFG.get_host_item('run directory'), None)]
else:
# Run directory of all users matching "owner_pattern".
# But skip those with /nologin or /false shells
run_dirs = []
skips = ('/false', '/nologin')
for pwent in getpwall():
if any(pwent.pw_shell.endswith(s) for s in (skips)):
continue
if owner_pattern.match(pwent.pw_name):
run_dirs.append(
(GLOBAL_CFG.get_host_item('run directory',
owner=pwent.pw_name,
owner_home=pwent.pw_dir),
pwent.pw_name))
if cylc.flags.debug:
sys.stderr.write(
'Listing suites:%s%s\n' %
(DEBUG_DELIM,
DEBUG_DELIM.join(item[1]
for item in run_dirs if item[1] is not None)))
items = []
for run_d, owner in run_dirs:
for dirpath, dnames, fnames in os.walk(run_d, followlinks=True):
if updater and updater.quit:
return
# Always descend for top directory, but
# don't descend further if it has a:
# * .service/ or log/
# * cylc-suite.db: (pre-cylc-7 suites don't have ".service/").
if dirpath != run_d and (srv_files_mgr.DIR_BASE_SRV in dnames
or 'log' in dnames
or 'cylc-suite.db' in fnames):
dnames[:] = []
# Choose only suites with .service and matching filter
reg = os.path.relpath(dirpath, run_d)
try:
contact_data = srv_files_mgr.load_contact_file(reg, owner)
except (SuiteServiceFileError, IOError, TypeError, ValueError):
continue
else:
items.append((contact_data[srv_files_mgr.KEY_HOST],
contact_data[srv_files_mgr.KEY_PORT]))
return items
def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None):
"""Scrape list of suites from the filesystem.
Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for
active suites.
Yields:
tuple - (reg, host, port)
"""
srv_files_mgr = SuiteSrvFilesManager()
if owner_pattern is None:
# Run directory of current user only
run_dirs = [(glbl_cfg().get_host_item('run directory'), None)]
else:
# Run directory of all users matching "owner_pattern".
# But skip those with /nologin or /false shells
run_dirs = []
skips = ('/false', '/nologin')
for pwent in getpwall():
if any(pwent.pw_shell.endswith(s) for s in (skips)):
continue
if owner_pattern.match(pwent.pw_name):
run_dirs.append(
(glbl_cfg().get_host_item('run directory',
owner=pwent.pw_name,
owner_home=pwent.pw_dir),
pwent.pw_name))
if cylc.flags.debug:
sys.stderr.write(
'Listing suites:%s%s\n' %
(DEBUG_DELIM,
DEBUG_DELIM.join(item[1]
for item in run_dirs if item[1] is not None)))
for run_d, owner in run_dirs:
for dirpath, dnames, _ in os.walk(run_d, followlinks=True):
# Always descend for top directory, but
# don't descend further if it has a .service/ or log/ dir
if dirpath != run_d and (srv_files_mgr.DIR_BASE_SRV in dnames
or 'log' in dnames):
dnames[:] = []
# Filter suites by name
reg = os.path.relpath(dirpath, run_d)
if reg_pattern and not reg_pattern.match(reg):
continue
# Choose only suites with .service and matching filter
try:
contact_data = srv_files_mgr.load_contact_file(reg, owner)
except (SuiteServiceFileError, IOError, TypeError, ValueError):
continue
else:
yield (reg, contact_data[srv_files_mgr.KEY_HOST],
contact_data[srv_files_mgr.KEY_PORT])
def get_secret(suite):
"""Return the secret used for encrypting messages.
Currently this is the suite passphrase. This means we are sending
many messages all encrypted with the same hash which isn't great.
TODO: Upgrade the secret to add foreword security.
"""
return SuiteSrvFilesManager().get_auth_item(
SuiteSrvFilesManager.FILE_BASE_PASSPHRASE,
suite, content=True
)
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
self.engine = None
self.port = None
# Figure out the ports we are allowed to use.
base_port = glbl_cfg().get(['communication', 'base port'])
max_ports = glbl_cfg().get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = glbl_cfg().get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_method = glbl_cfg().get(['communication', 'method'])
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc':
self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite,
content=True),
'anon':
NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
if self.comms_method == 'http':
self.cert = None
self.pkey = None
else: # if self.comms_method in [None, 'https']:
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
ERR.error("no HTTPS/OpenSSL support. Aborting...")
raise CylcError("No HTTPS support. "
"Configure user's global.rc to use HTTP.")
self.start()
def _scan_worker(conn, timeout, my_uuid):
"""Port scan worker."""
srv_files_mgr = SuiteSrvFilesManager()
while True:
try:
if not conn.poll(SLEEP_INTERVAL):
continue
item = conn.recv()
if item == MSG_QUIT:
break
conn.send(_scan_item(timeout, my_uuid, srv_files_mgr, item))
except KeyboardInterrupt:
break
conn.close()
def __init__(self, regd_treestore, filtr=None, timeout=None):
self.db = SuiteSrvFilesManager()
self.quit = False
if timeout:
self.timeout = float(timeout)
else:
self.timeout = None
self.regd_treestore = regd_treestore
super(db_updater, self).__init__()
self.next_scan_time = None
self.running_choices = None
self.newtree = {}
self.regd_choices = self.db.list_suites(filtr)
def _timeout_handler(suite, host, port):
"""Handle the eventuality of a communication timeout with the suite."""
if suite is None:
return
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
SuiteSrvFilesManager().detect_old_contact_file(suite, (host, port))
except (AssertionError, SuiteServiceFileError):
# * contact file not have matching (host, port) to suite proc
# * old contact file exists and the suite process still alive
return
else:
# the suite has stopped
raise ClientError( # TODO: SuiteStoppedError?
{'message': 'Suite "%s" already stopped' % suite})
def send(self, messages):
"""Send messages back to the suite."""
self._print_messages(messages)
self._update_job_status_file(messages)
try:
self.env_map.update(SuiteSrvFilesManager().load_contact_file(
self.suite))
except (IOError, ValueError, SuiteServiceFileError) as exc:
# No suite to communicate with, just print to stdout.
if cylc.flags.debug:
print >> sys.stderr, exc
return
if (self.env_map.get('CYLC_TASK_COMMS_METHOD') == 'ssh'
and self._send_by_ssh()):
return
self._send_by_remote_port(messages)
def __init__(self, suite, owner=USER, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False):
self.suite = suite
self.owner = owner
self.host = host
self.port = port
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.srv_files_mgr = SuiteSrvFilesManager()
self.prog_name = os.path.basename(sys.argv[0])
self.server_cert = None
self.auth = None
def get_location(cls, suite, owner, host):
"""Extract host and port from a suite's contact file."""
try:
contact = SuiteSrvFilesManager().load_contact_file(
suite, owner, host)
except SuiteServiceFileError:
sys.exit(cls.NOT_RUNNING % suite)
# monkey-patch the error message to make it more informative.
# exc.args = (cls.NOT_RUNNING % suite,)
# raise
if not host:
host = contact[SuiteSrvFilesManager.KEY_HOST]
host = get_fqdn_by_host(host)
port = int(contact[SuiteSrvFilesManager.KEY_PORT])
return host, port
def send(self, messages):
"""Send messages back to the suite."""
self._print_messages(messages)
self._update_job_status_file(messages)
messages = [msg + ' at ' + self.true_event_time for msg in messages]
try:
self.env_map.update(SuiteSrvFilesManager().load_contact_file(
self.suite))
except (IOError, ValueError, SuiteServiceFileError):
# No suite to communicate with, just print to stdout.
if cylc.flags.debug:
import traceback
traceback.print_exc()
return
if self.env_map.get('CYLC_TASK_COMMS_METHOD') == 'ssh':
self._send_by_ssh()
else:
self._send_by_remote_port(messages)
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
# Figure out the ports we are allowed to use.
base_port = GLOBAL_CFG.get(['communication', 'base port'])
max_ports = GLOBAL_CFG.get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = GLOBAL_CFG.get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc':
self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite,
content=True),
'anon':
NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
self.cert = None
self.pkey = None
self.client_reporter = CommsClientReporter.get_inst()
self.start()
def setUp(self):
self.suite_srv_files_mgr = SuiteSrvFilesManager()
def _scan1_impl(conn, timeout, my_uuid):
"""Connect to host:port to get suite identify."""
srv_files_mgr = SuiteSrvFilesManager()
while True:
if not conn.poll(SLEEP_INTERVAL):
continue
item = conn.recv()
if item == MSG_QUIT:
break
host, port = item
host_anon = host
if is_remote_host(host):
host_anon = get_host_ip_by_name(host) # IP reduces DNS traffic
client = SuiteIdClientAnon(None,
host=host_anon,
port=port,
my_uuid=my_uuid,
timeout=timeout)
try:
result = client.identify()
except ConnectionTimeout as exc:
conn.send((host, port, MSG_TIMEOUT))
except (ConnectionError, SuiteStillInitialisingError) as exc:
conn.send((host, port, None))
else:
owner = result.get('owner')
name = result.get('name')
states = result.get('states', None)
if cylc.flags.debug:
print >> sys.stderr, ' suite:', name, owner
if states is None:
# This suite keeps its state info private.
# Try again with the passphrase if I have it.
try:
pphrase = srv_files_mgr.get_auth_item(
srv_files_mgr.FILE_BASE_PASSPHRASE,
name,
owner,
host,
content=True)
except SuiteServiceFileError:
pass
else:
if pphrase:
client = SuiteIdClient(name,
owner=owner,
host=host,
port=port,
my_uuid=my_uuid,
timeout=timeout)
try:
result = client.identify()
except SuiteStillInitialisingError as exc:
if cylc.flags.debug:
print >> sys.stderr, (
' (connected with passphrase,' +
' suite initialising)')
except ConnectionError as exc:
# Nope (private suite, wrong passphrase).
if cylc.flags.debug:
print >> sys.stderr, ' (wrong passphrase)'
else:
if cylc.flags.debug:
print >> sys.stderr, (
' (got states with passphrase)')
conn.send((host, port, result))
conn.close()
def update_suites_info(updater, full_mode=False):
"""Return mapping of suite info by host, owner and suite name.
Args:
updater (object): gscan or gpanel updater:
Compulsory attributes from updater:
hosts: hosts to scan
owner_pattern: re to filter results by owners
suite_info_map: previous results returned by this function
Optional attributes from updater:
timeout: communication timeout
full_mode (boolean): update in full mode?
Return:
dict: {(host, owner, name): suite_info, ...}
where each "suite_info" is a dict with keys:
KEY_GROUP: group name of suite
KEY_OWNER: suite owner name
KEY_PORT: suite port, for running suites only
KEY_STATES: suite state
KEY_TASKS_BY_STATE: tasks by state
KEY_TITLE: suite title
KEY_UPDATE_TIME: last update time of suite
"""
# Compulsory attributes from updater
# hosts - hosts to scan, or the default set in the site/user global.rc
# owner_pattern - return only suites with owners matching this compiled re
# suite_info_map - previous results returned by this function
# Optional attributes from updater
# timeout - communication timeout
owner_pattern = updater.owner_pattern
timeout = getattr(updater, "comms_timeout", None)
# name_pattern - return only suites with names matching this compiled re
name_pattern = getattr(updater, "name_pattern", None)
# Determine items to scan
results = {}
items = []
if full_mode and not updater.hosts:
# Scan users suites. Walk "~/cylc-run/" to get (host, port) from
# ".service/contact" for active suites
suite_srv_files_mgr = SuiteSrvFilesManager()
if owner_pattern is None:
# Run directory of current user only
run_dirs = [GLOBAL_CFG.get_host_item('run directory')]
else:
# Run directory of all users matching "owner_pattern".
# But skip those with /nologin or /false shells
run_dirs = []
skips = ('/false', '/nologin')
for pwent in getpwall():
if any(pwent.pw_shell.endswith(s) for s in (skips)):
continue
if owner_pattern.match(pwent.pw_name):
run_dirs.append(
GLOBAL_CFG.get_host_item('run directory',
owner=pwent.pw_name,
owner_home=pwent.pw_dir))
if cylc.flags.debug:
sys.stderr.write(
'Listing suites:%s%s\n' %
(_UPDATE_DEBUG_DELIM, _UPDATE_DEBUG_DELIM.join(run_dirs)))
for run_d in run_dirs:
for dirpath, dnames, fnames in os.walk(run_d, followlinks=True):
if updater.quit:
return
# Always descend for top directory, but
# don't descend further if it has a:
# * .service/
# * cylc-suite.db: (pre-cylc-7 suites don't have ".service/").
if dirpath != run_d and (suite_srv_files_mgr.DIR_BASE_SRV
in dnames
or 'cylc-suite.db' in fnames):
dnames[:] = []
# Choose only suites with .service and matching filter
reg = os.path.relpath(dirpath, run_d)
try:
contact_data = suite_srv_files_mgr.load_contact_file(reg)
except (SuiteServiceFileError, IOError, TypeError, ValueError):
continue
else:
items.append((contact_data[suite_srv_files_mgr.KEY_HOST],
contact_data[suite_srv_files_mgr.KEY_PORT]))
elif full_mode:
# Scan full port range on all hosts
items.extend(updater.hosts)
else:
# Scan suites in previous results only
for (host, owner, name), prev_result in updater.suite_info_map.items():
port = prev_result.get(KEY_PORT)
if port:
items.append((host, port))
else:
results[(host, owner, name)] = prev_result
if not items:
return results
if cylc.flags.debug:
sys.stderr.write(
'Scan items:%s%s\n' %
(_UPDATE_DEBUG_DELIM,
_UPDATE_DEBUG_DELIM.join(str(item) for item in items)))
# Scan
for host, port, result in scan_many(items,
timeout=timeout,
updater=updater):
if updater.quit:
return
if (name_pattern and not name_pattern.match(result[KEY_NAME]) or
owner_pattern and not owner_pattern.match(result[KEY_OWNER])):
continue
try:
result[KEY_PORT] = port
results[(host, result[KEY_OWNER], result[KEY_NAME])] = result
result[KEY_UPDATE_TIME] = int(float(result[KEY_UPDATE_TIME]))
except (KeyError, TypeError, ValueError):
pass
expire_threshold = time() - DURATION_EXPIRE_STOPPED
for (host, owner, name), prev_result in updater.suite_info_map.items():
if updater.quit:
return
if ((host, owner, name) in results
or owner_pattern and not owner_pattern.match(owner)
or name_pattern and not name_pattern.match(name)):
# OK if suite already in current results set.
# Don't bother if:
# * previous owner does not match current owner pattern
# * previous suite name does not match current name pattern
continue
if prev_result.get(KEY_PORT):
# A previously running suite is no longer running.
# Get suite info with "cat-state", if possible, and include in the
# results set.
try:
prev_result = _update_stopped_suite_info((host, owner, name))
except (IndexError, TypeError, ValueError):
continue
if prev_result.get(KEY_UPDATE_TIME, 0) > expire_threshold:
results[(host, owner, name)] = prev_result
return results
def __init__(self):
self.initialised = {} # {(user, host): should_unlink, ...}
self.single_task_mode = False
self.suite_srv_files_mgr = SuiteSrvFilesManager()
