def __init__(self,
suite,
owner=None,
host=None,
port=None,
timeout=None,
my_uuid=None,
print_uuid=False,
comms_protocol=None,
auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_protocol = comms_protocol
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
def __init__(
self, suite, owner=None, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False, auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.comms1 = {} # content in primary contact file
self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None):
"""Scrape list of suites from the filesystem.
Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for
active suites.
Yields:
tuple - (reg, host, port)
"""
srv_files_mgr = SuiteSrvFilesManager()
if owner_pattern is None:
# Run directory of current user only
run_dirs = [(glbl_cfg().get_host_item('run directory'), None)]
else:
# Run directory of all users matching "owner_pattern".
# But skip those with /nologin or /false shells
run_dirs = []
skips = ('/false', '/nologin')
for pwent in getpwall():
if any(pwent.pw_shell.endswith(s) for s in (skips)):
continue
if owner_pattern.match(pwent.pw_name):
run_dirs.append((
glbl_cfg().get_host_item(
'run directory',
owner=pwent.pw_name,
owner_home=pwent.pw_dir),
pwent.pw_name))
if cylc.flags.debug:
sys.stderr.write('Listing suites:%s%s\n' % (
DEBUG_DELIM, DEBUG_DELIM.join(item[1] for item in run_dirs if
item[1] is not None)))
for run_d, owner in run_dirs:
for dirpath, dnames, _ in os.walk(run_d, followlinks=True):
# Always descend for top directory, but
# don't descend further if it has a .service/ or log/ dir
if dirpath != run_d and (
srv_files_mgr.DIR_BASE_SRV in dnames or 'log' in dnames):
dnames[:] = []
# Filter suites by name
reg = os.path.relpath(dirpath, run_d)
if reg_pattern and not reg_pattern.match(reg):
continue
# Choose only suites with .service and matching filter
try:
contact_data = srv_files_mgr.load_contact_file(reg, owner)
except (SuiteServiceFileError, IOError, TypeError, ValueError):
continue
else:
yield (
reg,
contact_data[srv_files_mgr.KEY_HOST],
contact_data[srv_files_mgr.KEY_PORT]
)
def get_scan_items_from_fs(owner_pattern=None, updater=None):
"""Get list of host:port available to scan using the file system.
Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for
active suites.
Return (list): List of (host, port) available for scan.
"""
srv_files_mgr = SuiteSrvFilesManager()
if owner_pattern is None:
# Run directory of current user only
run_dirs = [(glbl_cfg().get_host_item('run directory'), None)]
else:
# Run directory of all users matching "owner_pattern".
# But skip those with /nologin or /false shells
run_dirs = []
skips = ('/false', '/nologin')
for pwent in getpwall():
if any(pwent.pw_shell.endswith(s) for s in (skips)):
continue
if owner_pattern.match(pwent.pw_name):
run_dirs.append((
glbl_cfg().get_host_item(
'run directory',
owner=pwent.pw_name,
owner_home=pwent.pw_dir),
pwent.pw_name))
if cylc.flags.debug:
sys.stderr.write('Listing suites:%s%s\n' % (
DEBUG_DELIM, DEBUG_DELIM.join(item[1] for item in run_dirs if
item[1] is not None)))
items = []
for run_d, owner in run_dirs:
for dirpath, dnames, fnames in os.walk(run_d, followlinks=True):
if updater and updater.quit:
return
# Always descend for top directory, but
# don't descend further if it has a:
# * .service/ or log/
# * cylc-suite.db: (pre-cylc-7 suites don't have ".service/").
if dirpath != run_d and (
srv_files_mgr.DIR_BASE_SRV in dnames or 'log' in dnames or
'cylc-suite.db' in fnames):
dnames[:] = []
# Choose only suites with .service and matching filter
reg = os.path.relpath(dirpath, run_d)
try:
contact_data = srv_files_mgr.load_contact_file(reg, owner)
except (SuiteServiceFileError, IOError, TypeError, ValueError):
continue
else:
items.append((
contact_data[srv_files_mgr.KEY_HOST],
contact_data[srv_files_mgr.KEY_PORT]))
return items
def get_scan_items_from_fs(owner_pattern=None, reg_pattern=None):
"""Scrape list of suites from the filesystem.
Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for
active suites.
Yields:
tuple - (reg, host, port)
"""
srv_files_mgr = SuiteSrvFilesManager()
if owner_pattern is None:
# Run directory of current user only
run_dirs = [(glbl_cfg().get_host_item('run directory'), None)]
else:
# Run directory of all users matching "owner_pattern".
# But skip those with /nologin or /false shells
run_dirs = []
skips = ('/false', '/nologin')
for pwent in getpwall():
if any(pwent.pw_shell.endswith(s) for s in (skips)):
continue
if owner_pattern.match(pwent.pw_name):
run_dirs.append(
(glbl_cfg().get_host_item('run directory',
owner=pwent.pw_name,
owner_home=pwent.pw_dir),
pwent.pw_name))
if cylc.flags.debug:
sys.stderr.write(
'Listing suites:%s%s\n' %
(DEBUG_DELIM,
DEBUG_DELIM.join(item[1]
for item in run_dirs if item[1] is not None)))
for run_d, owner in run_dirs:
for dirpath, dnames, _ in os.walk(run_d, followlinks=True):
# Always descend for top directory, but
# don't descend further if it has a .service/ or log/ dir
if dirpath != run_d and (srv_files_mgr.DIR_BASE_SRV in dnames
or 'log' in dnames):
dnames[:] = []
# Filter suites by name
reg = os.path.relpath(dirpath, run_d)
if reg_pattern and not reg_pattern.match(reg):
continue
# Choose only suites with .service and matching filter
try:
contact_data = srv_files_mgr.load_contact_file(reg, owner)
except (SuiteServiceFileError, IOError, TypeError, ValueError):
continue
else:
yield (reg, contact_data[srv_files_mgr.KEY_HOST],
contact_data[srv_files_mgr.KEY_PORT])
def main(is_restart=False):
"""CLI main."""
options, args = parse_commandline(is_restart)
if not args:
# Auto-registration: "cylc run" (no args) in source dir.
try:
reg = SuiteSrvFilesManager().register()
except SuiteServiceFileError as exc:
sys.exit(exc)
# Replace this process with "cylc run REG ..." for 'ps -f'.
os.execv(sys.argv[0], [sys.argv[0]] + [reg] + sys.argv[1:])
# Check suite is not already running before start of host selection.
try:
SuiteSrvFilesManager().detect_old_contact_file(args[0])
except SuiteServiceFileError as exc:
sys.exit(exc)
# Create auth files if needed. On a shared FS if the suite host changes
# this may (will?) renew the ssl.cert to reflect the change in host name.
SuiteSrvFilesManager().create_auth_files(args[0])
# Check whether a run host is explicitly specified, else select one.
if not options.host:
try:
host = HostAppointer().appoint_host()
except EmptyHostList as exc:
if cylc.flags.debug:
raise
else:
sys.exit(str(exc))
if is_remote_host(host):
if is_restart:
base_cmd = ["restart"] + sys.argv[1:]
else:
base_cmd = ["run"] + sys.argv[1:]
# Prevent recursive host selection
base_cmd.append("--host=localhost")
return remote_cylc_cmd(base_cmd, host=host)
if remrun(set_rel_local=True): # State localhost as above.
sys.exit()
try:
SuiteSrvFilesManager().get_suite_source_dir(args[0], options.owner)
except SuiteServiceFileError:
# Source path is assumed to be the run directory
SuiteSrvFilesManager().register(
args[0],
glbl_cfg().get_derived_host_item(args[0], 'suite run directory'))
try:
scheduler = Scheduler(is_restart, options, args)
except SuiteServiceFileError as exc:
sys.exit(exc)
scheduler.start()
def get_scan_items_from_fs(owner_pattern=None, updater=None):
"""Get list of host:port available to scan using the file system.
Walk users' "~/cylc-run/" to get (host, port) from ".service/contact" for
active suites.
Return (list): List of (host, port) available for scan.
"""
srv_files_mgr = SuiteSrvFilesManager()
if owner_pattern is None:
# Run directory of current user only
run_dirs = [(GLOBAL_CFG.get_host_item('run directory'), None)]
else:
# Run directory of all users matching "owner_pattern".
# But skip those with /nologin or /false shells
run_dirs = []
skips = ('/false', '/nologin')
for pwent in getpwall():
if any(pwent.pw_shell.endswith(s) for s in (skips)):
continue
if owner_pattern.match(pwent.pw_name):
run_dirs.append(
(GLOBAL_CFG.get_host_item('run directory',
owner=pwent.pw_name,
owner_home=pwent.pw_dir),
pwent.pw_name))
if cylc.flags.debug:
sys.stderr.write(
'Listing suites:%s%s\n' %
(DEBUG_DELIM,
DEBUG_DELIM.join(item[1]
for item in run_dirs if item[1] is not None)))
items = []
for run_d, owner in run_dirs:
for dirpath, dnames, fnames in os.walk(run_d, followlinks=True):
if updater and updater.quit:
return
# Always descend for top directory, but
# don't descend further if it has a:
# * .service/ or log/
# * cylc-suite.db: (pre-cylc-7 suites don't have ".service/").
if dirpath != run_d and (srv_files_mgr.DIR_BASE_SRV in dnames
or 'log' in dnames
or 'cylc-suite.db' in fnames):
dnames[:] = []
# Choose only suites with .service and matching filter
reg = os.path.relpath(dirpath, run_d)
try:
contact_data = srv_files_mgr.load_contact_file(reg, owner)
except (SuiteServiceFileError, IOError, TypeError, ValueError):
continue
else:
items.append((contact_data[srv_files_mgr.KEY_HOST],
contact_data[srv_files_mgr.KEY_PORT]))
return items
def _scan1_impl(conn, timeout, my_uuid):
"""Connect to host:port to get suite identify."""
srv_files_mgr = SuiteSrvFilesManager()
while True:
if not conn.poll(SLEEP_INTERVAL):
continue
item = conn.recv()
if item == MSG_QUIT:
break
host, port = item
host_anon = host
if is_remote_host(host):
host_anon = get_host_ip_by_name(host) # IP reduces DNS traffic
client = SuiteIdClientAnon(
None, host=host_anon, port=port, my_uuid=my_uuid, timeout=timeout)
try:
result = client.identify()
except ConnectionTimeout as exc:
conn.send((host, port, MSG_TIMEOUT))
except ConnectionError as exc:
conn.send((host, port, None))
else:
owner = result.get('owner')
name = result.get('name')
states = result.get('states', None)
if cylc.flags.debug:
print >> sys.stderr, ' suite:', name, owner
if states is None:
# This suite keeps its state info private.
# Try again with the passphrase if I have it.
try:
pphrase = srv_files_mgr.get_auth_item(
srv_files_mgr.FILE_BASE_PASSPHRASE, name, owner, host,
content=True)
except SuiteServiceFileError:
pass
else:
if pphrase:
client = SuiteIdClient(
name, owner=owner, host=host, port=port,
my_uuid=my_uuid, timeout=timeout)
try:
result = client.identify()
except ConnectionError as exc:
# Nope (private suite, wrong passphrase).
if cylc.flags.debug:
print >> sys.stderr, ' (wrong passphrase)'
else:
if cylc.flags.debug:
print >> sys.stderr, (
' (got states with passphrase)')
conn.send((host, port, result))
conn.close()
def _load_suite_contact_file(self):
"""Load contact environment file."""
from cylc.suite_srv_files_mgr import SuiteSrvFilesManager
srv_files_mgr = SuiteSrvFilesManager()
contact_data = srv_files_mgr.load_contact_file(self.suite)
for attr, key in (
('owner', srv_files_mgr.KEY_OWNER),
('host', srv_files_mgr.KEY_HOST),
('port', srv_files_mgr.KEY_PORT)):
try:
self.env_map[key] = contact_data[key]
setattr(self, attr, contact_data[key])
except KeyError:
pass
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
self.engine = None
self.port = None
# Figure out the ports we are allowed to use.
base_port = glbl_cfg().get(['communication', 'base port'])
max_ports = glbl_cfg().get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = glbl_cfg().get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_method = glbl_cfg().get(['communication', 'method'])
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc':
self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite,
content=True),
'anon':
NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
if self.comms_method == 'http':
self.cert = None
self.pkey = None
else: # if self.comms_method in [None, 'https']:
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
ERR.error("no HTTPS/OpenSSL support. Aborting...")
raise CylcError("No HTTPS support. "
"Configure user's global.rc to use HTTP.")
self.start()
class TestSuiteSrvFilesManager(unittest.TestCase):
def setUp(self):
self.suite_srv_files_mgr = SuiteSrvFilesManager()
@mock.patch('cylc.suite_srv_files_mgr.os')
def test_register(self, mocked_os):
"""Test the SuiteSrvFilesManager register function."""
def mkdirs_standin(_, exist_ok=False):
return True
# we do not need to mock these functions
mocked_os.path.basename.side_effect = os.path.basename
mocked_os.path.join = os.path.join
mocked_os.path.normpath = os.path.normpath
mocked_os.path.dirname = os.path.dirname
mocked_os.makedirs.side_effect = mkdirs_standin
mocked_os.path.abspath.side_effect = lambda x: x
for reg, source, redirect, cwd, isabs, isfile, \
suite_srv_dir, readlink, expected_symlink, \
expected, e_expected, e_message \
in get_register_test_cases():
mocked_os.getcwd.side_effect = lambda: cwd
mocked_os.path.isabs.side_effect = lambda x: isabs
mocked_os.path.isfile = lambda x: isfile
self.suite_srv_files_mgr.get_suite_srv_dir = mock.MagicMock(
return_value=suite_srv_dir
)
if readlink == OSError:
mocked_os.readlink.side_effect = readlink
else:
mocked_os.readlink.side_effect = lambda x: readlink
if e_expected is None:
reg = self.suite_srv_files_mgr.register(reg, source, redirect)
self.assertEqual(expected, reg)
if mocked_os.symlink.call_count > 0:
# first argument, of the first call
arg0 = mocked_os.symlink.call_args[0][0]
self.assertEqual(expected_symlink, arg0)
else:
with self.assertRaises(e_expected) as cm:
self.suite_srv_files_mgr.register(reg, source, redirect)
if e_message is not None:
the_exception = cm.exception
self.assertTrue(e_message in str(the_exception),
str(the_exception))
def __init__(self, suite, owner=USER, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False):
self.suite = suite
self.owner = owner
self.host = host
self.port = port
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.srv_files_mgr = SuiteSrvFilesManager()
self.prog_name = os.path.basename(sys.argv[0])
self.server_cert = None
self.auth = None
def __init__(
self, suite, owner=None, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False, auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.comms1 = {} # content in primary contact file
self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def __init__(self, regd_treestore, filtr=None, timeout=None):
self.db = SuiteSrvFilesManager()
self.quit = False
if timeout:
self.timeout = float(timeout)
else:
self.timeout = None
self.regd_treestore = regd_treestore
super(db_updater, self).__init__()
self.next_scan_time = None
self.running_choices = None
self.newtree = {}
self.regd_choices = self.db.list_suites(filtr)
class TestSuiteSrvFilesManager(unittest.TestCase):
def setUp(self):
self.suite_srv_files_mgr = SuiteSrvFilesManager()
@mock.patch('cylc.suite_srv_files_mgr.os')
def test_register(self, mocked_os):
"""Test the SuiteSrvFilesManager register function."""
def mkdirs_standin(_, exist_ok=False):
return True
# we do not need to mock these functions
mocked_os.path.basename.side_effect = os.path.basename
mocked_os.path.join = os.path.join
mocked_os.path.normpath = os.path.normpath
mocked_os.path.dirname = os.path.dirname
mocked_os.makedirs.side_effect = mkdirs_standin
mocked_os.path.abspath.side_effect = lambda x: x
for reg, source, redirect, cwd, isabs, isfile, \
suite_srv_dir, readlink, expected_symlink, \
expected, e_expected, e_message \
in get_register_test_cases():
mocked_os.getcwd.side_effect = lambda: cwd
mocked_os.path.isabs.side_effect = lambda x: isabs
mocked_os.path.isfile = lambda x: isfile
self.suite_srv_files_mgr.get_suite_srv_dir = mock.MagicMock(
return_value=suite_srv_dir)
if readlink == OSError:
mocked_os.readlink.side_effect = readlink
else:
mocked_os.readlink.side_effect = lambda x: readlink
if e_expected is None:
reg = self.suite_srv_files_mgr.register(reg, source, redirect)
self.assertEqual(expected, reg)
if mocked_os.symlink.call_count > 0:
# first argument, of the first call
arg0 = mocked_os.symlink.call_args[0][0]
self.assertEqual(expected_symlink, arg0)
else:
with self.assertRaises(e_expected) as cm:
self.suite_srv_files_mgr.register(reg, source, redirect)
if e_message is not None:
the_exception = cm.exception
self.assertTrue(e_message in str(the_exception),
str(the_exception))
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
# Figure out the ports we are allowed to use.
base_port = GLOBAL_CFG.get(['communication', 'base port'])
max_ports = GLOBAL_CFG.get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = GLOBAL_CFG.get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc':
self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite,
content=True),
'anon':
NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
self.cert = None
self.pkey = None
self.client_reporter = CommsClientReporter.get_inst()
self.start()
def get_secret(suite):
"""Return the secret used for encrypting messages.
Currently this is the suite passphrase. This means we are sending
many messages all encrypted with the same hash which isn't great.
TODO: Upgrade the secret to add foreword security.
"""
return SuiteSrvFilesManager().get_auth_item(
SuiteSrvFilesManager.FILE_BASE_PASSPHRASE,
suite, content=True
)
def _scan_worker(conn, timeout, my_uuid):
"""Port scan worker."""
srv_files_mgr = SuiteSrvFilesManager()
while True:
try:
if not conn.poll(SLEEP_INTERVAL):
continue
item = conn.recv()
if item == MSG_QUIT:
break
conn.send(_scan_item(timeout, my_uuid, srv_files_mgr, item))
except KeyboardInterrupt:
break
conn.close()
def __init__(self, regd_treestore, filtr=None, timeout=None):
self.db = SuiteSrvFilesManager()
self.quit = False
if timeout:
self.timeout = float(timeout)
else:
self.timeout = None
self.regd_treestore = regd_treestore
super(db_updater, self).__init__()
self.next_scan_time = None
self.running_choices = None
self.newtree = {}
self.regd_choices = self.db.list_suites(filtr)
def _timeout_handler(suite, host, port):
"""Handle the eventuality of a communication timeout with the suite."""
if suite is None:
return
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
SuiteSrvFilesManager().detect_old_contact_file(suite, (host, port))
except (AssertionError, SuiteServiceFileError):
# * contact file not have matching (host, port) to suite proc
# * old contact file exists and the suite process still alive
return
else:
# the suite has stopped
raise ClientError( # TODO: SuiteStoppedError?
{'message': 'Suite "%s" already stopped' % suite})
def __init__(self, suite, owner=USER, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False):
self.suite = suite
self.owner = owner
self.host = host
self.port = port
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.srv_files_mgr = SuiteSrvFilesManager()
self.prog_name = os.path.basename(sys.argv[0])
self.server_cert = None
self.auth = None
def send(self, messages):
"""Send messages back to the suite."""
self._print_messages(messages)
self._update_job_status_file(messages)
try:
self.env_map.update(SuiteSrvFilesManager().load_contact_file(
self.suite))
except (IOError, ValueError, SuiteServiceFileError) as exc:
# No suite to communicate with, just print to stdout.
if cylc.flags.debug:
print >> sys.stderr, exc
return
if (self.env_map.get('CYLC_TASK_COMMS_METHOD') == 'ssh'
and self._send_by_ssh()):
return
self._send_by_remote_port(messages)
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
self.engine = None
self.port = None
# Figure out the ports we are allowed to use.
base_port = glbl_cfg().get(['communication', 'base port'])
max_ports = glbl_cfg().get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = glbl_cfg().get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_method = glbl_cfg().get(['communication', 'method'])
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc': self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite, content=True),
'anon': NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
if self.comms_method == 'http':
self.cert = None
self.pkey = None
else: # if self.comms_method in [None, 'https']:
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
ERR.error("no HTTPS/OpenSSL support. Aborting...")
raise CylcError("No HTTPS support. "
"Configure user's global.rc to use HTTP.")
self.start()
def get_location(cls, suite, owner, host):
"""Extract host and port from a suite's contact file."""
try:
contact = SuiteSrvFilesManager().load_contact_file(
suite, owner, host)
except SuiteServiceFileError:
sys.exit(cls.NOT_RUNNING % suite)
# monkey-patch the error message to make it more informative.
# exc.args = (cls.NOT_RUNNING % suite,)
# raise
if not host:
host = contact[SuiteSrvFilesManager.KEY_HOST]
host = get_fqdn_by_host(host)
port = int(contact[SuiteSrvFilesManager.KEY_PORT])
return host, port
def send(self, messages):
"""Send messages back to the suite."""
self._print_messages(messages)
self._update_job_status_file(messages)
messages = [msg + ' at ' + self.true_event_time for msg in messages]
try:
self.env_map.update(SuiteSrvFilesManager().load_contact_file(
self.suite))
except (IOError, ValueError, SuiteServiceFileError):
# No suite to communicate with, just print to stdout.
if cylc.flags.debug:
import traceback
traceback.print_exc()
return
if self.env_map.get('CYLC_TASK_COMMS_METHOD') == 'ssh':
self._send_by_ssh()
else:
self._send_by_remote_port(messages)
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
# Figure out the ports we are allowed to use.
base_port = GLOBAL_CFG.get(['communication', 'base port'])
max_ports = GLOBAL_CFG.get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = GLOBAL_CFG.get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc': self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite, content=True),
'anon': NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
self.cert = None
self.pkey = None
self.client_reporter = CommsClientReporter.get_inst()
self.start()
class HTTPServer(object):
"""HTTP(S) server by cherrypy, for serving suite runtime API."""
API = 1
LOG_CONNECT_DENIED_TMPL = "[client-connect] DENIED %s@%s:%s %s"
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
self.engine = None
self.port = None
# Figure out the ports we are allowed to use.
base_port = GLOBAL_CFG.get(['communication', 'base port'])
max_ports = GLOBAL_CFG.get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = GLOBAL_CFG.get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_method = GLOBAL_CFG.get(['communication', 'method'])
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc': self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite, content=True),
'anon': NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
if self.comms_method == 'http':
self.cert = None
self.pkey = None
else: # if self.comms_method in [None, 'https']:
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
ERR.error("no HTTPS/OpenSSL support. Aborting...")
raise CylcError("No HTTPS support. "
"Configure user's global.rc to use HTTP.")
self.start()
@cherrypy.expose
def apiversion(self):
"""Return API version."""
return str(self.API)
@staticmethod
def connect(schd):
"""Mount suite schedular object to the web server."""
cherrypy.tree.mount(SuiteRuntimeService(schd), '/')
# For back-compat with "scan"
cherrypy.tree.mount(SuiteRuntimeService(schd), '/id')
@staticmethod
def disconnect(schd):
"""Disconnect obj from the web server."""
del cherrypy.tree.apps['/%s/%s' % (schd.owner, schd.suite)]
def get_port(self):
"""Return the web server port."""
return self.port
def shutdown(self):
"""Shutdown the web server."""
if hasattr(self, "engine"):
self.engine.exit()
self.engine.block()
def start(self):
"""Start quick web service."""
# cherrypy.config["tools.encode.on"] = True
# cherrypy.config["tools.encode.encoding"] = "utf-8"
cherrypy.config["server.socket_host"] = get_host()
cherrypy.config["engine.autoreload.on"] = False
if self.comms_method == "https":
# Setup SSL etc. Otherwise fail and exit.
# Require connection method to be the same e.g HTTP/HTTPS matching.
cherrypy.config['server.ssl_module'] = 'pyopenSSL'
cherrypy.config['server.ssl_certificate'] = self.cert
cherrypy.config['server.ssl_private_key'] = self.pkey
cherrypy.config['log.screen'] = None
key = binascii.hexlify(os.urandom(16))
cherrypy.config.update({
'tools.auth_digest.on': True,
'tools.auth_digest.realm': self.suite,
'tools.auth_digest.get_ha1': self.get_ha1,
'tools.auth_digest.key': key,
'tools.auth_digest.algorithm': self.hash_algorithm
})
cherrypy.tools.connect_log = cherrypy.Tool(
'on_end_resource', self._report_connection_if_denied)
cherrypy.config['tools.connect_log.on'] = True
self.engine = cherrypy.engine
for port in self.ok_ports:
cherrypy.config["server.socket_port"] = port
try:
cherrypy.engine.start()
cherrypy.engine.wait(cherrypy.engine.states.STARTED)
except cherrypy.process.wspbus.ChannelFailures:
if cylc.flags.debug:
traceback.print_exc()
# We need to reinitialise the httpserver for each port attempt.
cherrypy.server.httpserver = None
else:
if cherrypy.engine.state == cherrypy.engine.states.STARTED:
self.port = port
return
raise Exception("No available ports")
@staticmethod
def _get_client_connection_denied():
"""Return whether a connection was denied."""
if "Authorization" not in cherrypy.request.headers:
# Probably just the initial HTTPS handshake.
return False
status = cherrypy.response.status
if isinstance(status, basestring):
return cherrypy.response.status.split()[0] in ["401", "403"]
return cherrypy.response.status in [401, 403]
def _report_connection_if_denied(self):
"""Log an (un?)successful connection attempt."""
prog_name, user, host, uuid = _get_client_info()[1:]
connection_denied = self._get_client_connection_denied()
if connection_denied:
LOG.warning(self.__class__.LOG_CONNECT_DENIED_TMPL % (
user, host, prog_name, uuid))
class CommsDaemon(object):
"""Wrap HTTPS daemon for a suite."""
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
# Figure out the ports we are allowed to use.
base_port = GLOBAL_CFG.get(['communication', 'base port'])
max_ports = GLOBAL_CFG.get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = GLOBAL_CFG.get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc': self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite, content=True),
'anon': NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
self.cert = None
self.pkey = None
self.client_reporter = CommsClientReporter.get_inst()
self.start()
def shutdown(self):
"""Shutdown the daemon."""
if hasattr(self, "engine"):
self.engine.exit()
self.engine.block()
def connect(self, obj, name):
"""Connect obj and name to the daemon."""
cherrypy.tree.mount(obj, "/" + name)
def disconnect(self, obj):
"""Disconnect obj from the daemon."""
pass
def get_port(self):
"""Return the daemon port."""
return self.port
def report_connection_if_denied(self):
"""Report connection if denied."""
self.client_reporter.report_connection_if_denied()
def start(self):
"""Start quick web service."""
# cherrypy.config["tools.encode.on"] = True
# cherrypy.config["tools.encode.encoding"] = "utf-8"
cherrypy.config["server.socket_host"] = '0.0.0.0'
cherrypy.config["engine.autoreload.on"] = False
try:
from OpenSSL import SSL, crypto
cherrypy.config['server.ssl_module'] = 'pyopenSSL'
cherrypy.config['server.ssl_certificate'] = self.cert
cherrypy.config['server.ssl_private_key'] = self.pkey
except ImportError:
ERR.warning("no HTTPS/OpenSSL support")
cherrypy.config['log.screen'] = None
key = binascii.hexlify(os.urandom(16))
cherrypy.config.update({
'tools.auth_digest.on': True,
'tools.auth_digest.realm': self.suite,
'tools.auth_digest.get_ha1': self.get_ha1,
'tools.auth_digest.key': key,
'tools.auth_digest.algorithm': self.hash_algorithm
})
cherrypy.tools.connect_log = cherrypy.Tool(
'on_end_resource', self.report_connection_if_denied)
cherrypy.config['tools.connect_log.on'] = True
self.engine = cherrypy.engine
for port in self.ok_ports:
cherrypy.config["server.socket_port"] = port
try:
cherrypy.engine.start()
cherrypy.engine.wait(cherrypy.engine.states.STARTED)
except Exception:
if cylc.flags.debug:
traceback.print_exc()
# We need to reinitialise the httpserver for each port attempt.
cherrypy.server.httpserver = None
else:
if cherrypy.engine.state == cherrypy.engine.states.STARTED:
self.port = port
return
raise Exception("No available ports")
class db_updater(threading.Thread):
SCAN_INTERVAL = 60.0
def __init__(self, regd_treestore, filtr=None, timeout=None):
self.db = SuiteSrvFilesManager()
self.quit = False
if timeout:
self.timeout = float(timeout)
else:
self.timeout = None
self.regd_treestore = regd_treestore
super(db_updater, self).__init__()
self.next_scan_time = None
self.running_choices = None
self.newtree = {}
self.regd_choices = self.db.list_suites(filtr)
def construct_newtree(self):
"""construct self.newtree[one][two]...[nnn] = [auth, descr, dir ]"""
regd_choices = {}
for suite, suite_dir, descr in sorted(self.regd_choices):
regd_choices[suite] = (suite, suite_dir, descr)
self.newtree = {}
for suite, auth in self.running_choices:
if suite in regd_choices:
if is_remote_host(auth.split(':', 1)[0]):
descr, suite_dir = (None, None)
else:
# local suite
_, suite_dir, descr = regd_choices[suite]
del regd_choices[suite]
nest2 = self.newtree
regp = suite.split(SuiteSrvFilesManager.DELIM)
for key in regp[:-1]:
if key not in nest2:
nest2[key] = {}
nest2 = nest2[key]
nest2[(regp[-1], suite, auth)] = [auth, descr, suite_dir]
for suite, suite_dir, descr in regd_choices.values():
suite_dir = re.sub('^' + os.environ['HOME'], '~', suite_dir)
nest2 = self.newtree
regp = suite.split(SuiteSrvFilesManager.DELIM)
for key in regp[:-1]:
if key not in nest2:
nest2[key] = {}
nest2 = nest2[key]
nest2[(regp[-1], suite, '-')] = ['-', descr, suite_dir]
def build_treestore(self, data, piter=None):
for key, value in sorted(data.items()):
if isinstance(key, tuple):
item = key[0]
else:
item = key
if isinstance(value, dict):
# final three items are colours
iter_ = self.regd_treestore.append(
piter, [item, None, None, None, None, None, None])
self.build_treestore(value, iter_)
else:
state, descr, suite_dir = value
self.regd_treestore.append(
piter, [item, state, descr, suite_dir, None, None, None])
def update(self):
"""Update tree, if necessary."""
if self.next_scan_time is not None and self.next_scan_time > time():
return
# Scan for running suites
choices = []
for host, port, suite_identity in scan_all(timeout=self.timeout):
name = suite_identity['name']
owner = suite_identity['owner']
if is_remote_user(owner):
continue # current user only
auth = "%s:%d" % (host, port)
choices.append((name, auth))
choices.sort()
self.next_scan_time = time() + self.SCAN_INTERVAL
if choices == self.running_choices:
return
# Update tree if running suites changed
self.running_choices = choices
self.construct_newtree()
self.update_treestore(
self.newtree, self.regd_treestore.get_iter_first())
def update_treestore(self, new, iter_):
# iter_ is None for an empty treestore (no suites registered)
ts = self.regd_treestore
if iter_:
opath = ts.get_path(iter_)
# get parent iter_ before pruning in case we prune last item at
# this level
piter = ts.iter_parent(iter_)
else:
opath = None
piter = None
def my_get_iter(item):
# find the TreeIter pointing at item at this level
if not opath:
return None
iter_ = ts.get_iter(opath)
while iter_:
val, = ts.get(iter_, 0)
if val == item:
return iter_
iter_ = ts.iter_next(iter_)
return None
# new items at this level
new_items = new.keys()
old_items = []
prune = []
while iter_:
# iterate through old items at this level
item, state, descr, dir = ts.get(iter_, 0, 1, 2, 3)
if item not in new_items:
# old item is not in new - prune it
res = ts.remove(iter_)
if not res: # Nec?
iter_ = None
else:
# old item is in new - update it in case it changed
old_items.append(item)
# update old items that do appear in new
chiter = ts.iter_children(iter_)
if not isinstance(new[item], dict):
# new item is not a group - update title etc.
state, descr, dir = new[item]
sc = self.statecol(state)
ni = new[item]
ts.set(iter_, 0, item, 1, ni[0], 2, ni[1], 3, ni[2],
4, sc[0], 5, sc[1], 6, sc[2])
if chiter:
# old item was a group - kill its children
while chiter:
res = ts.remove(chiter)
if not res:
chiter = None
else:
# new item is a group
if not chiter:
# old item was not a group
ts.set(
iter_, 0, item, 1, None, 2, None, 3, None, 4,
None, 5, None, 6, None)
self.build_treestore(new[item], iter_)
# continue
iter_ = ts.iter_next(iter_)
# return to original iter_
if opath:
try:
iter_ = ts.get_iter(opath)
except ValueError:
# removed the item pointed to
# TODO - NEED TO WORRY ABOUT OTHERS AT THIS LEVEL?
iter_ = None
else:
iter_ = None
# add new items at this level
for key in sorted(new_items):
if isinstance(key, tuple):
item = key[0]
else:
item = key
if item not in old_items:
# new data wasn't in old - add it
if isinstance(new[key], dict):
xiter = ts.append(
piter, [item] + [None, None, None, None, None, None])
self.build_treestore(new[key], xiter)
else:
state, descr, dir = new[key]
yiter = ts.append(
piter, [item] + new[key] + list(self.statecol(state)))
else:
# new data was already in old
if isinstance(new[key], dict):
# check lower levels
niter = my_get_iter(key)
if niter:
chiter = ts.iter_children(niter)
if chiter:
self.update_treestore(new[key], chiter)
def run(self):
"""Main loop."""
while not self.quit:
gobject.idle_add(self.update)
sleep(0.1)
def statecol(self, state):
bg = '#19ae0a'
fg = '#030'
if state == '-':
return (None, None, None)
else:
return (fg, bg, bg)
def search_level(self, model, iter_, func, data):
while iter_:
if func(model, iter_, data):
return iter_
iter_ = model.iter_next(iter_)
return None
def search_treemodel(self, model, iter_, func, data):
while iter_:
if func(model, iter_, data):
return iter_
result = self.search_treemodel(
model, model.iter_children(iter_), func, data)
if result:
return result
iter_ = model.iter_next(iter_)
return None
def match_func(self, model, iter_, data):
column, key = data
value = model.get_value(iter_, column)
return value == key
class SuiteRuntimeServiceClient(object):
"""Client for calling the HTTP(S) API of running suites."""
ANON_AUTH = ('anon', NO_PASSPHRASE, False)
COMPAT_MAP = { # Limited pre-7.5.0 API compat mapping
'clear_broadcast': {0: 'broadcast/clear'},
'expire_broadcast': {0: 'broadcast/expire'},
'get_broadcast': {0: 'broadcast/get'},
'get_info': {0: 'info/'},
'get_suite_state_summary': {0: 'state/get_state_summary'},
'put_broadcast': {0: 'broadcast/put'},
'put_command': {0: 'command/'},
'put_ext_trigger': {0: 'ext-trigger/put'},
'put_messages': {0: 'message/put', 1: 'put_message'},
}
ERROR_NO_HTTPS_SUPPORT = (
"ERROR: server has no HTTPS support," +
" configure your global.rc file to use HTTP : {0}\n"
)
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
MSG_RETRY_INTVL = 5.0
MSG_MAX_TRIES = 7
MSG_TIMEOUT = 30.0
def __init__(
self, suite, owner=None, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False, auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.comms1 = {} # content in primary contact file
self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def _compat(self, name, default=None):
"""Return server function name.
Handle back-compat for pre-7.5.0 if relevant.
"""
# Need to load contact info here to get API version.
self._load_contact_info()
if default is None:
default = name
return self.COMPAT_MAP[name].get(
self.comms1.get(self.srv_files_mgr.KEY_API), default)
def clear_broadcast(self, payload):
"""Clear broadcast runtime task settings."""
return self._call_server(
self._compat('clear_broadcast'), payload=payload)
def expire_broadcast(self, **kwargs):
"""Expire broadcast runtime task settings."""
return self._call_server(self._compat('expire_broadcast'), **kwargs)
def get_broadcast(self, **kwargs):
"""Return broadcast settings."""
return self._call_server(
self._compat('get_broadcast'), method=self.METHOD_GET, **kwargs)
def get_info(self, command, **kwargs):
"""Return suite info."""
return self._call_server(
self._compat('get_info', default='') + command,
method=self.METHOD_GET, **kwargs)
def get_latest_state(self, full_mode=False):
"""Return latest state of the suite (for the GUI)."""
self._load_contact_info()
if self.comms1.get(self.srv_files_mgr.KEY_API) == 0:
# Basic compat for pre-7.5.0 suites
# Full mode only.
# Error content/size not supported.
# Report made-up main loop interval of 5.0 seconds.
return {
'cylc_version': self.get_info('get_cylc_version'),
'full_mode': full_mode,
'summary': self.get_suite_state_summary(),
'ancestors': self.get_info('get_first_parent_ancestors'),
'ancestors_pruned': self.get_info(
'get_first_parent_ancestors', pruned=True),
'descendants': self.get_info('get_first_parent_descendants'),
'err_content': '',
'err_size': 0,
'mean_main_loop_interval': 5.0}
else:
return self._call_server(
'get_latest_state',
method=self.METHOD_GET, full_mode=full_mode)
def get_suite_state_summary(self):
"""Return the global, task, and family summary data structures."""
return utf8_enforce(self._call_server(
self._compat('get_suite_state_summary'), method=self.METHOD_GET))
def identify(self):
"""Return suite identity."""
# Note on compat: Suites on 7.6.0 or above can just call "identify",
# but has compat for "id/identity".
return self._call_server('id/identify', method=self.METHOD_GET)
def put_broadcast(self, payload):
"""Put/set broadcast runtime task settings."""
return self._call_server(
self._compat('put_broadcast'), payload=payload)
def put_command(self, command, **kwargs):
"""Invoke suite command."""
return self._call_server(
self._compat('put_command', default='') + command, **kwargs)
def put_ext_trigger(self, event_message, event_id):
"""Put external trigger."""
return self._call_server(
self._compat('put_ext_trigger'),
event_message=event_message, event_id=event_id)
def put_messages(self, payload):
"""Send task messages to suite server program.
Arguments:
payload (dict):
task_job (str): Task job as "CYCLE/TASK_NAME/SUBMIT_NUM".
event_time (str): Event time as string.
messages (list): List in the form [[severity, message], ...].
"""
retry_intvl = float(self.comms1.get(
self.srv_files_mgr.KEY_TASK_MSG_RETRY_INTVL,
self.MSG_RETRY_INTVL))
max_tries = int(self.comms1.get(
self.srv_files_mgr.KEY_TASK_MSG_MAX_TRIES,
self.MSG_MAX_TRIES))
for i in range(1, max_tries + 1): # 1..max_tries inclusive
orig_timeout = self.timeout
if self.timeout is None:
self.timeout = self.MSG_TIMEOUT
try:
func_name = self._compat('put_messages')
if func_name == 'put_messages':
results = self._call_server(func_name, payload=payload)
elif func_name == 'put_message': # API 1, 7.5.0 compat
cycle, name = payload['task_job'].split('/')[0:2]
for severity, message in payload['messages']:
results.append(self._call_server(
func_name, task_id='%s.%s' % (name, cycle),
severity=severity, message=message))
else: # API 0, pre-7.5.0 compat, priority instead of severity
cycle, name = payload['task_job'].split('/')[0:2]
for severity, message in payload['messages']:
results.append(self._call_server(
func_name, task_id='%s.%s' % (name, cycle),
priority=severity, message=message))
except ClientInfoError:
# Contact info file not found, suite probably not running.
# Don't bother with retry, suite restart will poll any way.
raise
except ClientError as exc:
now = get_current_time_string()
sys.stderr.write(
"%s WARNING - Message send failed, try %s of %s: %s\n" % (
now, i, max_tries, exc))
if i < max_tries:
sys.stderr.write(
" retry in %s seconds, timeout is %s\n" % (
retry_intvl, self.timeout))
sleep(retry_intvl)
# Reset in case contact info or passphrase change
self.comms1 = {}
self.host = None
self.port = None
self.auth = None
else:
if i > 1:
# Continue to write to STDERR, so users can easily see that
# it has recovered from previous failures.
sys.stderr.write(
"%s INFO - Send message: try %s of %s succeeded\n" % (
get_current_time_string(), i, max_tries))
return results
finally:
self.timeout = orig_timeout
def reset(self):
"""Compat method, does nothing."""
pass
def signout(self):
"""Tell server to forget this client."""
return self._call_server('signout')
def _call_server(self, function, method=METHOD, payload=None, **kwargs):
"""Build server URL + call it"""
if self.comms2:
return self._call_server_via_comms2(function, payload, **kwargs)
url = self._call_server_get_url(function, **kwargs)
# Remove proxy settings from environment for now
environ = {}
for key in ("http_proxy", "https_proxy"):
val = os.environ.pop(key, None)
if val:
environ[key] = val
try:
return self.call_server_impl(url, method, payload)
finally:
os.environ.update(environ)
def _call_server_get_url(self, function, **kwargs):
"""Build request URL."""
scheme = self.comms1.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
if scheme is None:
# Use standard setting from global configuration
scheme = glbl_cfg().get(['communication', 'method'])
url = '%s://%s:%s/%s' % (
scheme, self.host, self.port, function)
# If there are any parameters left in the dict after popping,
# append them to the url.
if kwargs:
import urllib
params = urllib.urlencode(kwargs, doseq=True)
url += "?" + params
return url
def call_server_impl(self, url, method, payload):
"""Determine whether to use requests or urllib2 to call suite API."""
impl = self._call_server_impl_urllib2
try:
import requests
except ImportError:
pass
else:
if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]:
impl = self._call_server_impl_requests
try:
return impl(url, method, payload)
except ClientConnectError as exc:
if self.suite is None:
raise
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
self.srv_files_mgr.detect_old_contact_file(
self.suite, (self.host, self.port))
except (AssertionError, SuiteServiceFileError):
raise exc
else:
# self.srv_files_mgr.detect_old_contact_file should delete left
# behind contact file if the old suite process no longer
# exists. Should be safe to report that the suite has stopped.
raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite)
def _call_server_impl_requests(self, url, method, payload):
"""Call server with "requests" library."""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter("ignore", InsecureRequestWarning)
if self.session is None:
self.session = requests.Session()
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
scheme = url.split(':', 1)[0] # Can use urlparse?
username, password, verify = self._get_auth(scheme)
try:
ret = session_method(
url,
json=payload,
verify=verify,
proxies={},
headers=self._get_headers(),
auth=requests.auth.HTTPDigestAuth(username, password),
timeout=self.timeout
)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
if ret.status_code == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
if ret.status_code >= 400:
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectedError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return ret.json()
except ValueError:
return ret.text
def _call_server_impl_urllib2(self, url, method, payload):
"""Call server with "urllib2" library."""
import json
import urllib2
import ssl
unverified_context = getattr(ssl, '_create_unverified_context', None)
if unverified_context is not None:
ssl._create_default_https_context = unverified_context
scheme = url.split(':', 1)[0] # Can use urlparse?
username, password = self._get_auth(scheme)[0:2]
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if payload:
payload = json.dumps(payload)
headers_list.append(('Accept', 'application/json'))
json_headers = {'Content-Type': 'application/json',
'Content-Length': len(payload)}
else:
payload = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, payload, json_headers)
# This is an unpleasant monkey patch, but there isn't an
# alternative. urllib2 uses POST if there is a data payload
# but that is not the correct criterion.
# The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
if "timed out" in str(exc):
raise ClientTimeout(url, exc)
else:
raise ClientConnectError(url, exc)
except Exception as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientError(url, exc)
if response.getcode() == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
response_text = response.read()
if response.getcode() >= 400:
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ClientConnectedError(
url,
"%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return json.loads(response_text)
except ValueError:
return response_text
def _call_server_via_comms2(self, function, payload, **kwargs):
"""Call server via "cylc client --use-ssh".
Call "cylc client --use-ssh" using `subprocess.Popen`. Payload and
arguments of the API method are serialized as JSON and are written to a
temporary file, which is then used as the STDIN of the "cylc client"
command. The external call here should be even safer than a direct
HTTP(S) call, as it can be blocked by SSH before it even gets a chance
to make the subsequent HTTP(S) call.
Arguments:
function (str): name of API method, argument 1 of "cylc client".
payload (str): extra data or information for the API method.
**kwargs (dict): arguments for the API method.
"""
import json
from cylc.remote import remote_cylc_cmd
command = ["client", function, self.suite]
if payload:
kwargs["payload"] = payload
if kwargs:
from tempfile import TemporaryFile
stdin = TemporaryFile()
json.dump(kwargs, stdin)
stdin.seek(0)
else:
# With stdin=None, `remote_cylc_cmd` will:
# * Set stdin to open(os.devnull)
# * Add `-n` to the SSH command
stdin = None
proc = remote_cylc_cmd(
command, self.owner, self.host, capture=True,
ssh_login_shell=(self.comms1.get(
self.srv_files_mgr.KEY_SSH_USE_LOGIN_SHELL
) in ['True', 'true']),
ssh_cylc=(r'%s/bin/cylc' % self.comms1.get(
self.srv_files_mgr.KEY_DIR_ON_SUITE_HOST)
),
stdin=stdin,
)
out = proc.communicate()[0]
return_code = proc.wait()
if return_code:
from pipes import quote
command_str = " ".join(quote(item) for item in command)
raise ClientError(command_str, "return-code=%d" % return_code)
return json.loads(out)
def _get_auth(self, protocol):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = self.ANON_AUTH
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite, self.owner, self.host, content=True)
if protocol == 'https':
verify = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT,
self.suite, self.owner, self.host)
else:
verify = False
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase, verify)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = (
"cylc/%s prog_name/%s uuid/%s" % (
CYLC_VERSION, self.prog_name, self.my_uuid
)
)
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string,
"From": auth_info}
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
try:
# Always trust the values in the contact file otherwise.
self.comms1 = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
# Port inside "try" block, as it needs a type conversion
self.port = int(self.comms1.get(self.srv_files_mgr.KEY_PORT))
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
else:
# Check mismatch suite UUID
env_suite = os.getenv(self.srv_files_mgr.KEY_NAME)
env_uuid = os.getenv(self.srv_files_mgr.KEY_UUID)
if (self.suite and env_suite and env_suite == self.suite and
env_uuid and
env_uuid != self.comms1.get(self.srv_files_mgr.KEY_UUID)):
raise ClientInfoUUIDError(
env_uuid, self.comms1[self.srv_files_mgr.KEY_UUID])
# All good
self.host = self.comms1.get(self.srv_files_mgr.KEY_HOST)
self.owner = self.comms1.get(self.srv_files_mgr.KEY_OWNER)
if self.srv_files_mgr.KEY_API not in self.comms1:
self.comms1[self.srv_files_mgr.KEY_API] = 0 # <=7.5.0 compat
# Indirect comms settings
self.comms2.clear()
try:
self.comms2.update(self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host,
SuiteSrvFilesManager.FILE_BASE_CONTACT2))
except SuiteServiceFileError:
pass
def _scan1_impl(conn, timeout, my_uuid):
"""Connect to host:port to get suite identify."""
srv_files_mgr = SuiteSrvFilesManager()
while True:
if not conn.poll(SLEEP_INTERVAL):
continue
item = conn.recv()
if item == MSG_QUIT:
break
host, port = item
host_anon = host
if is_remote_host(host):
host_anon = get_host_ip_by_name(host) # IP reduces DNS traffic
client = SuiteIdClientAnon(None,
host=host_anon,
port=port,
my_uuid=my_uuid,
timeout=timeout)
try:
result = client.identify()
except ConnectionTimeout as exc:
conn.send((host, port, MSG_TIMEOUT))
except (ConnectionError, SuiteStillInitialisingError) as exc:
conn.send((host, port, None))
else:
owner = result.get('owner')
name = result.get('name')
states = result.get('states', None)
if cylc.flags.debug:
print >> sys.stderr, ' suite:', name, owner
if states is None:
# This suite keeps its state info private.
# Try again with the passphrase if I have it.
try:
pphrase = srv_files_mgr.get_auth_item(
srv_files_mgr.FILE_BASE_PASSPHRASE,
name,
owner,
host,
content=True)
except SuiteServiceFileError:
pass
else:
if pphrase:
client = SuiteIdClient(name,
owner=owner,
host=host,
port=port,
my_uuid=my_uuid,
timeout=timeout)
try:
result = client.identify()
except SuiteStillInitialisingError as exc:
if cylc.flags.debug:
print >> sys.stderr, (
' (connected with passphrase,' +
' suite initialising)')
except ConnectionError as exc:
# Nope (private suite, wrong passphrase).
if cylc.flags.debug:
print >> sys.stderr, ' (wrong passphrase)'
else:
if cylc.flags.debug:
print >> sys.stderr, (
' (got states with passphrase)')
conn.send((host, port, result))
conn.close()
class SuiteRuntimeServiceClient(object):
"""Client for calling the HTTP(S) API of running suites."""
ANON_AUTH = ('anon', NO_PASSPHRASE, False)
ERROR_NO_HTTPS_SUPPORT = (
"ERROR: server has no HTTPS support," +
" configure your global.rc file to use HTTP : {0}\n")
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
def __init__(self,
suite,
owner=None,
host=None,
port=None,
timeout=None,
my_uuid=None,
print_uuid=False,
comms_protocol=None,
auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_protocol = comms_protocol
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
def clear_broadcast(self, **kwargs):
"""Clear broadcast runtime task settings."""
return self._call_server('clear_broadcast', payload=kwargs)
def expire_broadcast(self, **kwargs):
"""Expire broadcast runtime task settings."""
return self._call_server('expire_broadcast', **kwargs)
def get_broadcast(self, **kwargs):
"""Return broadcast settings."""
return self._call_server('get_broadcast',
method=self.METHOD_GET,
**kwargs)
def get_info(self, command, *args, **kwargs):
"""Return suite info."""
kwargs['method'] = self.METHOD_GET
return self._call_server(command, *args, **kwargs)
def get_latest_state(self, full_mode):
"""Return latest state of the suite (for the GUI)."""
return self._call_server('get_latest_state',
method=self.METHOD_GET,
full_mode=full_mode)
def get_suite_state_summary(self):
"""Return the global, task, and family summary data structures."""
return utf8_enforce(
self._call_server('get_suite_state_summary',
method=self.METHOD_GET))
def get_tasks_by_state(self):
"""Returns a dict containing lists of tasks by state.
Result in the form:
{state: [(most_recent_time_string, task_name, point_string), ...]}
"""
return self._call_server('get_tasks_by_state', method=self.METHOD_GET)
def identify(self):
"""Return suite identity."""
return self._call_server('identify', method=self.METHOD_GET)
def put_broadcast(self, **kwargs):
"""Put/set broadcast runtime task settings."""
return self._call_server('put_broadcast', payload=kwargs)
def put_command(self, command, **kwargs):
"""Invoke suite command."""
return self._call_server(command, **kwargs)
def put_ext_trigger(self, event_message, event_id):
"""Put external trigger."""
return self._call_server('put_ext_trigger',
event_message=event_message,
event_id=event_id)
def put_message(self, task_id, priority, message):
"""Send task message."""
return self._call_server('put_message',
task_id=task_id,
priority=priority,
message=message)
def reset(self, *args, **kwargs):
"""Compat method, does nothing."""
pass
def signout(self, *args, **kwargs):
"""Tell server to forget this client."""
return self._call_server('signout')
def _compile_request(self, func_dict, host, comms_protocol=None):
"""Build request URL."""
payload = func_dict.pop("payload", None)
method = func_dict.pop("method", self.METHOD)
function = func_dict.pop("function", None)
if comms_protocol is None:
# Use standard setting from global configuration
from cylc.cfgspec.globalcfg import GLOBAL_CFG
comms_protocol = GLOBAL_CFG.get(['communication', 'method'])
url = '%s://%s:%s/%s' % (comms_protocol, host, self.port, function)
# If there are any parameters left in the dict after popping,
# append them to the url.
if func_dict:
import urllib
params = urllib.urlencode(func_dict, doseq=True)
url += "?" + params
return (method, url, payload, comms_protocol)
def _call_server(self, *func_dicts, **fargs):
"""func_dict is a dictionary of command names (fnames)
and arguments to that command"""
# Deal with the case of one func_dict/function name passed
# by converting them to the generic case: a dictionary of
# a single function and its function arguments.
if isinstance(func_dicts[0], str):
func_dict = {"function": func_dicts[0]}
func_dict.update(fargs)
else:
func_dict = None
try:
self._load_contact_info()
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
http_request_items = []
try:
# dictionary containing: url, payload, method
http_request_items.append(
self._compile_request(func_dict, self.host,
self.comms_protocol))
except (IndexError, ValueError, AttributeError):
for f_dict in func_dicts:
http_request_items.append(
self._compile_request(f_dict, self.host,
self.comms_protocol))
# Remove proxy settings from environment for now
environ = {}
for key in ("http_proxy", "https_proxy"):
val = os.environ.pop(key, None)
if val:
environ[key] = val
# Returns a list of http returns from the requests
try:
return self.call_server_impl(http_request_items)
finally:
os.environ.update(environ)
def call_server_impl(self, http_request_items):
"""Determine whether to use requests or urllib2 to call suite API."""
method = self._call_server_impl_urllib2
try:
import requests
except ImportError:
pass
else:
if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]:
method = self._call_server_impl_requests
try:
return method(http_request_items)
except ClientConnectError as exc:
if self.suite is None:
raise
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
self.srv_files_mgr.detect_old_contact_file(
self.suite, (self.host, self.port))
except (AssertionError, SuiteServiceFileError):
raise exc
else:
# self.srv_files_mgr.detect_old_contact_file should delete left
# behind contact file if the old suite process no longer
# exists. Should be safe to report that the suite has stopped.
raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite)
def _call_server_impl_requests(self, http_request_items):
"""Call server with "requests" library."""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter("ignore", InsecureRequestWarning)
if not hasattr(self, "session"):
self.session = requests.Session()
http_return_items = []
for method, url, payload, comms_protocol in http_request_items:
if method is None:
method = self.METHOD
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
username, password, verify = self._get_auth(comms_protocol)
try:
ret = session_method(url,
json=payload,
verify=verify,
proxies={},
headers=self._get_headers(),
auth=requests.auth.HTTPDigestAuth(
username, password),
timeout=self.timeout)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
if ret.status_code == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
if ret.status_code >= 400:
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectedError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(self.suite, self.owner,
self.host, self.auth[1])
try:
ret = ret.json()
http_return_items.append(ret)
except ValueError:
ret = ret.text
http_return_items.append(ret)
# Return a single http return or a list of them if multiple
return (http_return_items
if len(http_return_items) > 1 else http_return_items[0])
def _call_server_impl_urllib2(self, http_request_items):
"""Call server with "urllib2" library."""
import json
import urllib2
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
http_return_items = []
for method, url, payload, comms_protocol in http_request_items:
if method is None:
method = self.METHOD
username, password = self._get_auth(comms_protocol)[0:2]
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if payload:
payload = json.dumps(payload)
headers_list.append(('Accept', 'application/json'))
json_headers = {
'Content-Type': 'application/json',
'Content-Length': len(payload)
}
else:
payload = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, payload, json_headers)
# This is an unpleasant monkey patch, but there isn't an
# alternative. urllib2 uses POST if there is a data payload
# but that is not the correct criterion.
# The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
if "timed out" in str(exc):
raise ClientTimeout(url, exc)
else:
raise ClientConnectError(url, exc)
except Exception as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientError(url, exc)
if response.getcode() == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
response_text = response.read()
if response.getcode() >= 400:
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ClientConnectedError(
url, "%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(self.suite, self.owner,
self.host, self.auth[1])
try:
http_return_items.append(json.loads(response_text))
except ValueError:
http_return_items.append(response_text)
# Return a single http return or a list of them if multiple
return (http_return_items
if len(http_return_items) > 1 else http_return_items[0])
def _get_auth(self, protocol):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = self.ANON_AUTH
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite,
self.owner,
self.host,
content=True)
if protocol == 'https':
verify = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite,
self.owner, self.host)
else:
verify = False
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase, verify)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = ("cylc/%s prog_name/%s uuid/%s" %
(CYLC_VERSION, self.prog_name, self.my_uuid))
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string, "From": auth_info}
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
# Always trust the values in the contact file otherwise.
data = self.srv_files_mgr.load_contact_file(self.suite, self.owner,
self.host)
self.host = data.get(self.srv_files_mgr.KEY_HOST)
self.port = int(data.get(self.srv_files_mgr.KEY_PORT))
self.owner = data.get(self.srv_files_mgr.KEY_OWNER)
self.comms_protocol = data.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
class HTTPServer(object):
"""HTTP(S) server by cherrypy, for serving suite runtime API."""
API = 2
LOG_CONNECT_DENIED_TMPL = "[client-connect] DENIED %s@%s:%s %s"
RE_MESSAGE_TIME = re.compile(
r'\A(.+) at (' + RE_DATE_TIME_FORMAT_EXTENDED + r')\Z', re.DOTALL)
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
self.engine = None
self.port = None
# Figure out the ports we are allowed to use.
base_port = glbl_cfg().get(['communication', 'base port'])
max_ports = glbl_cfg().get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = glbl_cfg().get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_method = glbl_cfg().get(['communication', 'method'])
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc': self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite, content=True),
'anon': NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
if self.comms_method == 'http':
self.cert = None
self.pkey = None
else: # if self.comms_method in [None, 'https']:
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
ERR.error("no HTTPS/OpenSSL support. Aborting...")
raise CylcError("No HTTPS support. "
"Configure user's global.rc to use HTTP.")
self.start()
@cherrypy.expose
def apiversion(self):
"""Return API version."""
return str(self.API)
@staticmethod
def connect(schd):
"""Mount suite schedular object to the web server."""
cherrypy.tree.mount(SuiteRuntimeService(schd), '/')
# For back-compat with "scan"
cherrypy.tree.mount(SuiteRuntimeService(schd), '/id')
@staticmethod
def disconnect(schd):
"""Disconnect obj from the web server."""
del cherrypy.tree.apps['/%s/%s' % (schd.owner, schd.suite)]
def get_port(self):
"""Return the web server port."""
return self.port
def shutdown(self):
"""Shutdown the web server."""
if hasattr(self, "engine"):
self.engine.exit()
self.engine.block()
def start(self):
"""Start quick web service."""
# cherrypy.config["tools.encode.on"] = True
# cherrypy.config["tools.encode.encoding"] = "utf-8"
cherrypy.config["server.socket_host"] = get_host()
cherrypy.config["engine.autoreload.on"] = False
if self.comms_method == "https":
# Setup SSL etc. Otherwise fail and exit.
# Require connection method to be the same e.g HTTP/HTTPS matching.
cherrypy.config['server.ssl_module'] = 'pyopenSSL'
cherrypy.config['server.ssl_certificate'] = self.cert
cherrypy.config['server.ssl_private_key'] = self.pkey
cherrypy.config['log.screen'] = None
key = binascii.hexlify(os.urandom(16))
cherrypy.config.update({
'tools.auth_digest.on': True,
'tools.auth_digest.realm': self.suite,
'tools.auth_digest.get_ha1': self.get_ha1,
'tools.auth_digest.key': key,
'tools.auth_digest.algorithm': self.hash_algorithm
})
cherrypy.tools.connect_log = cherrypy.Tool(
'on_end_resource', self._report_connection_if_denied)
cherrypy.config['tools.connect_log.on'] = True
self.engine = cherrypy.engine
for port in self.ok_ports:
cherrypy.config["server.socket_port"] = port
try:
cherrypy.engine.start()
cherrypy.engine.wait(cherrypy.engine.states.STARTED)
except cherrypy.process.wspbus.ChannelFailures:
if cylc.flags.debug:
traceback.print_exc()
# We need to reinitialise the httpserver for each port attempt.
cherrypy.server.httpserver = None
else:
if cherrypy.engine.state == cherrypy.engine.states.STARTED:
self.port = port
return
raise Exception("No available ports")
@staticmethod
def _get_client_connection_denied():
"""Return whether a connection was denied."""
if "Authorization" not in cherrypy.request.headers:
# Probably just the initial HTTPS handshake.
return False
status = cherrypy.response.status
if isinstance(status, basestring):
return cherrypy.response.status.split()[0] in ["401", "403"]
return cherrypy.response.status in [401, 403]
def _report_connection_if_denied(self):
"""Log an (un?)successful connection attempt."""
prog_name, user, host, uuid = _get_client_info()[1:]
connection_denied = self._get_client_connection_denied()
if connection_denied:
LOG.warning(self.__class__.LOG_CONNECT_DENIED_TMPL % (
user, host, prog_name, uuid))
def __init__(self):
self.initialised = {} # {(user, host): should_unlink, ...}
self.single_task_mode = False
self.suite_srv_files_mgr = SuiteSrvFilesManager()
class BaseCommsClient(object):
"""Base class for client-side suite object interfaces."""
ACCESS_DESCRIPTION = 'private'
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
def __init__(self, suite, owner=USER, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False):
self.suite = suite
self.owner = owner
self.host = host
self.port = port
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.srv_files_mgr = SuiteSrvFilesManager()
self.prog_name = os.path.basename(sys.argv[0])
self.server_cert = None
self.auth = None
def call_server_func(self, category, fname, **fargs):
"""Call server_object.fname(*fargs, **fargs)."""
if self.host is None and self.port is not None:
self.host = get_hostname()
try:
self._load_contact_info()
except (IOError, ValueError, SuiteServiceFileError):
raise ConnectionInfoError(self.suite)
handle_proxies()
payload = fargs.pop("payload", None)
method = fargs.pop("method", self.METHOD)
host = self.host
if host == "localhost":
host = get_hostname().split(".")[0]
url = 'https://%s:%s/%s/%s' % (host, self.port, category, fname)
if fargs:
import urllib
params = urllib.urlencode(fargs, doseq=True)
url += "?" + params
return self._get_data_from_url(url, payload, method=method)
def _get_data_from_url(self, url, json_data, method=None):
requests_ok = True
try:
import requests
except ImportError:
requests_ok = False
else:
version = [int(_) for _ in requests.__version__.split(".")]
if version < [2, 4, 2]:
requests_ok = False
if requests_ok:
return self._get_data_from_url_with_requests(
url, json_data, method=method)
return self._get_data_from_url_with_urllib2(
url, json_data, method=method)
def _get_data_from_url_with_requests(self, url, json_data, method=None):
import requests
username, password = self._get_auth()
auth = requests.auth.HTTPDigestAuth(username, password)
if not hasattr(self, "session"):
self.session = requests.Session()
if method is None:
method = self.METHOD
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
try:
ret = session_method(
url,
json=json_data,
verify=self._get_verify(),
proxies={},
headers=self._get_headers(),
auth=auth,
timeout=self.timeout
)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc))
return self._get_data_from_url_with_requests(
url.replace("https:", "http:", 1), json_data)
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if ret.status_code == 401:
raise ConnectionDeniedError(url, self.prog_name,
self.ACCESS_DESCRIPTION)
if ret.status_code >= 400:
from cylc.network.https.util import get_exception_from_html
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return ret.json()
except ValueError:
return ret.text
def _get_data_from_url_with_urllib2(self, url, json_data, method=None):
import json
import urllib2
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if method is None:
method = self.METHOD
orig_json_data = json_data
username, password = self._get_auth()
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if json_data:
json_data = json.dumps(json_data)
headers_list.append(('Accept', 'application/json'))
json_headers = {'Content-Type': 'application/json',
'Content-Length': len(json_data)}
else:
json_data = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, json_data, json_headers)
# This is an unpleasant monkey patch, but there isn't an alternative.
# urllib2 uses POST if there is a data payload, but that is not the
# correct criterion. The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc))
return self._get_data_from_url_with_urllib2(
url.replace("https:", "http:", 1), orig_json_data)
if cylc.flags.debug:
import traceback
traceback.print_exc()
if "timed out" in str(exc):
raise ConnectionTimeout(url, exc)
else:
raise ConnectionError(url, exc)
except Exception as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if response.getcode() == 401:
raise ConnectionDeniedError(url, self.prog_name,
self.ACCESS_DESCRIPTION)
response_text = response.read()
if response.getcode() >= 400:
from cylc.network.https.util import get_exception_from_html
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ConnectionError(url,
"%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return json.loads(response_text)
except ValueError:
return response_text
def _get_auth(self):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = ('anon', NO_PASSPHRASE)
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite, self.owner, self.host, content=True)
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = (
"cylc/%s prog_name/%s uuid/%s" % (
CYLC_VERSION, self.prog_name, self.my_uuid
)
)
auth_info = "%s@%s" % (USER, get_hostname())
return {"User-Agent": user_agent_string,
"From": auth_info}
def _get_verify(self):
"""Return the server certificate if possible."""
if self.server_cert is None:
try:
self.server_cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT,
self.suite, self.owner, self.host)
except SuiteServiceFileError:
self.server_cert = False
return self.server_cert
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
data = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
if not self.host:
self.host = data.get(self.srv_files_mgr.KEY_HOST)
if not self.port:
self.port = int(data.get(self.srv_files_mgr.KEY_PORT))
if not self.owner:
self.owner = data.get(self.srv_files_mgr.KEY_OWNER)
def reset(self, *args, **kwargs):
pass
def signout(self, *args, **kwargs):
pass
class BaseCommsClient(object):
"""Base class for client-side suite object interfaces."""
ACCESS_DESCRIPTION = 'private'
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
def __init__(self, suite, owner=USER, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False):
self.suite = suite
self.owner = owner
self.host = host
self.port = port
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.srv_files_mgr = SuiteSrvFilesManager()
self.prog_name = os.path.basename(sys.argv[0])
self.server_cert = None
self.auth = None
def call_server_func(self, category, fname, **fargs):
"""Call server_object.fname(*fargs, **fargs)."""
if self.host is None and self.port is not None:
self.host = get_hostname()
if self.host is None or self.port is None:
try:
self._load_contact_info()
except (IOError, ValueError, SuiteServiceFileError):
raise ConnectionInfoError(self.suite)
handle_proxies()
payload = fargs.pop("payload", None)
method = fargs.pop("method", self.METHOD)
host = self.host
if not self.host.split(".")[0].isdigit():
host = self.host.split(".")[0]
if host == "localhost":
host = get_hostname().split(".")[0]
url = 'https://%s:%s/%s/%s' % (host, self.port, category, fname)
if fargs:
import urllib
params = urllib.urlencode(fargs, doseq=True)
url += "?" + params
return self._get_data_from_url(url, payload, method=method)
def _get_data_from_url(self, url, json_data, method=None):
requests_ok = True
try:
import requests
except ImportError:
requests_ok = False
else:
version = [int(_) for _ in requests.__version__.split(".")]
if version < [2, 4, 2]:
requests_ok = False
if requests_ok:
return self._get_data_from_url_with_requests(
url, json_data, method=method)
return self._get_data_from_url_with_urllib2(
url, json_data, method=method)
def _get_data_from_url_with_requests(self, url, json_data, method=None):
import requests
username, password = self._get_auth()
auth = requests.auth.HTTPDigestAuth(username, password)
if not hasattr(self, "session"):
self.session = requests.Session()
if method is None:
method = self.METHOD
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
try:
ret = session_method(
url,
json=json_data,
verify=self._get_verify(),
proxies={},
headers=self._get_headers(),
auth=auth,
timeout=self.timeout
)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc))
return self._get_data_from_url_with_requests(
url.replace("https:", "http:", 1), json_data)
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if ret.status_code == 401:
raise ConnectionDeniedError(url, self.prog_name,
self.ACCESS_DESCRIPTION)
if ret.status_code >= 400:
from cylc.network.https.util import get_exception_from_html
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return ret.json()
except ValueError:
return ret.text
def _get_data_from_url_with_urllib2(self, url, json_data, method=None):
import json
import urllib2
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
if method is None:
method = self.METHOD
orig_json_data = json_data
username, password = self._get_auth()
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if json_data:
json_data = json.dumps(json_data)
headers_list.append(('Accept', 'application/json'))
json_headers = {'Content-Type': 'application/json',
'Content-Length': len(json_data)}
else:
json_data = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, json_data, json_headers)
# This is an unpleasant monkey patch, but there isn't an alternative.
# urllib2 uses POST iff there is a data payload, but that is not the
# correct criterion. The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(WARNING_NO_HTTPS_SUPPORT.format(exc))
return self._get_data_from_url_with_urllib2(
url.replace("https:", "http:", 1), orig_json_data)
if cylc.flags.debug:
import traceback
traceback.print_exc()
if "timed out" in str(exc):
raise ConnectionTimeout(url, exc)
else:
raise ConnectionError(url, exc)
except Exception as exc:
if cylc.flags.debug:
import traceback
traceback.print_exc()
raise ConnectionError(url, exc)
if response.getcode() == 401:
raise ConnectionDeniedError(url, self.prog_name,
self.ACCESS_DESCRIPTION)
response_text = response.read()
if response.getcode() >= 400:
from cylc.network.https.util import get_exception_from_html
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ConnectionError(url,
"%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return json.loads(response_text)
except ValueError:
return response_text
def _get_auth(self):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = ('anon', NO_PASSPHRASE)
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite, self.owner, self.host, content=True)
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = (
"cylc/%s prog_name/%s uuid/%s" % (
CYLC_VERSION, self.prog_name, self.my_uuid
)
)
auth_info = "%s@%s" % (USER, get_hostname())
return {"User-Agent": user_agent_string,
"From": auth_info}
def _get_verify(self):
"""Return the server certificate if possible."""
if self.server_cert is None:
try:
self.server_cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT,
self.suite, self.owner, self.host)
except SuiteServiceFileError:
self.server_cert = False
return self.server_cert
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
data = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
if not self.host:
self.host = data.get(self.srv_files_mgr.KEY_HOST)
if not self.port:
self.port = int(data.get(self.srv_files_mgr.KEY_PORT))
if not self.owner:
self.owner = data.get(self.srv_files_mgr.KEY_OWNER)
def reset(self, *args, **kwargs):
pass
def signout(self, *args, **kwargs):
pass
class db_updater(threading.Thread):
SCAN_INTERVAL = 60.0
def __init__(self, regd_treestore, filtr=None, timeout=None):
self.db = SuiteSrvFilesManager()
self.quit = False
if timeout:
self.timeout = float(timeout)
else:
self.timeout = None
self.regd_treestore = regd_treestore
super(db_updater, self).__init__()
self.next_scan_time = None
self.running_choices = None
self.newtree = {}
self.regd_choices = self.db.list_suites(filtr)
def construct_newtree(self):
"""construct self.newtree[one][two]...[nnn] = [auth, descr, dir ]"""
regd_choices = {}
for suite, suite_dir, descr in sorted(self.regd_choices):
regd_choices[suite] = (suite, suite_dir, descr)
self.newtree = {}
for suite, auth in self.running_choices:
if suite in regd_choices:
if is_remote_host(auth.split(':', 1)[0]):
descr, suite_dir = (None, None)
else:
# local suite
suite_dir, descr = regd_choices[suite][1:3]
del regd_choices[suite]
nest2 = self.newtree
regp = suite.split(SuiteSrvFilesManager.DELIM)
for key in regp[:-1]:
if key not in nest2:
nest2[key] = {}
nest2 = nest2[key]
nest2[(regp[-1], suite, auth)] = [auth, descr, suite_dir]
for suite, suite_dir, descr in regd_choices.values():
suite_dir = re.sub('^' + os.environ['HOME'], '~', suite_dir)
nest2 = self.newtree
regp = suite.split(SuiteSrvFilesManager.DELIM)
for key in regp[:-1]:
if key not in nest2:
nest2[key] = {}
nest2 = nest2[key]
nest2[(regp[-1], suite, '-')] = ['-', descr, suite_dir]
def build_treestore(self, data, piter=None):
for key, value in sorted(data.items()):
if isinstance(key, tuple):
item = key[0]
else:
item = key
if isinstance(value, dict):
# final three items are colours
iter_ = self.regd_treestore.append(
piter, [item, None, None, None, None, None, None])
self.build_treestore(value, iter_)
else:
state, descr, suite_dir = value
self.regd_treestore.append(
piter, [item, state, descr, suite_dir, None, None, None])
def update(self):
"""Update tree, if necessary."""
if self.next_scan_time is not None and self.next_scan_time > time():
return
# Scan for running suites
choices = []
for host, port, suite_identity in scan_many(get_scan_items_from_fs(),
timeout=self.timeout):
name = suite_identity[KEY_NAME]
owner = suite_identity[KEY_OWNER]
if is_remote_user(owner):
continue # current user only
auth = "%s:%s" % (host, port)
choices.append((name, auth))
choices.sort()
self.next_scan_time = time() + self.SCAN_INTERVAL
if choices == self.running_choices:
return
# Update tree if running suites changed
self.running_choices = choices
self.construct_newtree()
self.update_treestore(self.newtree,
self.regd_treestore.get_iter_first())
def update_treestore(self, new, iter_):
# iter_ is None for an empty treestore (no suites registered)
ts = self.regd_treestore
if iter_:
opath = ts.get_path(iter_)
# get parent iter_ before pruning in case we prune last item at
# this level
piter = ts.iter_parent(iter_)
else:
opath = None
piter = None
def my_get_iter(item):
# find the TreeIter pointing at item at this level
if not opath:
return None
iter_ = ts.get_iter(opath)
while iter_:
val, = ts.get(iter_, 0)
if val == item:
return iter_
iter_ = ts.iter_next(iter_)
return None
# new items at this level
new_items = new.keys()
old_items = []
while iter_:
# iterate through old items at this level
item, state = ts.get(iter_, 0, 1, 2, 3)[0:2]
if item not in new_items:
# old item is not in new - prune it
res = ts.remove(iter_)
if not res: # Nec?
iter_ = None
else:
# old item is in new - update it in case it changed
old_items.append(item)
# update old items that do appear in new
chiter = ts.iter_children(iter_)
if not isinstance(new[item], dict):
# new item is not a group - update title etc.
state = new[item][0]
sc = self.statecol(state)
ni = new[item]
ts.set(iter_, 0, item, 1, ni[0], 2, ni[1], 3, ni[2], 4,
sc[0], 5, sc[1], 6, sc[2])
if chiter:
# old item was a group - kill its children
while chiter:
res = ts.remove(chiter)
if not res:
chiter = None
else:
# new item is a group
if not chiter:
# old item was not a group
ts.set(iter_, 0, item, 1, None, 2, None, 3, None, 4,
None, 5, None, 6, None)
self.build_treestore(new[item], iter_)
# continue
iter_ = ts.iter_next(iter_)
# return to original iter_
if opath:
try:
iter_ = ts.get_iter(opath)
except ValueError:
# removed the item pointed to
# TODO - NEED TO WORRY ABOUT OTHERS AT THIS LEVEL?
iter_ = None
else:
iter_ = None
# add new items at this level
for key in sorted(new_items):
if isinstance(key, tuple):
item = key[0]
else:
item = key
if item not in old_items:
# new data wasn't in old - add it
if isinstance(new[key], dict):
xiter = ts.append(piter, [item] +
[None, None, None, None, None, None])
self.build_treestore(new[key], xiter)
else:
state = new[key][0]
ts.append(piter,
[item] + new[key] + list(self.statecol(state)))
else:
# new data was already in old
if isinstance(new[key], dict):
# check lower levels
niter = my_get_iter(key)
if niter:
chiter = ts.iter_children(niter)
if chiter:
self.update_treestore(new[key], chiter)
def run(self):
"""Main loop."""
while not self.quit:
gobject.idle_add(self.update)
sleep(0.1)
@staticmethod
def statecol(state):
bg_ = '#19ae0a'
fg_ = '#030'
if state == '-':
return (None, None, None)
else:
return (fg_, bg_, bg_)
@staticmethod
def search_level(model, iter_, func, data):
while iter_:
if func(model, iter_, data):
return iter_
iter_ = model.iter_next(iter_)
return None
def search_treemodel(self, model, iter_, func, data):
while iter_:
if func(model, iter_, data):
return iter_
result = self.search_treemodel(model, model.iter_children(iter_),
func, data)
if result:
return result
iter_ = model.iter_next(iter_)
return None
@staticmethod
def match_func(model, iter_, data):
column, key = data
value = model.get_value(iter_, column)
return value == key
class SuiteRuntimeServiceClient(object):
"""Client for calling the HTTP(S) API of running suites."""
ANON_AUTH = ('anon', NO_PASSPHRASE, False)
COMPAT_MAP = { # Limited pre-7.5.0 API compat mapping
'clear_broadcast': {0: 'broadcast/clear'},
'expire_broadcast': {0: 'broadcast/expire'},
'get_broadcast': {0: 'broadcast/get'},
'get_info': {0: 'info/'},
'get_suite_state_summary': {0: 'state/get_state_summary'},
'get_tasks_by_state': {0: 'state/get_tasks_by_state'},
'put_broadcast': {0: 'broadcast/put'},
'put_command': {0: 'command/'},
'put_ext_trigger': {0: 'ext-trigger/put'},
'put_message': {0: 'message/put'},
}
ERROR_NO_HTTPS_SUPPORT = (
"ERROR: server has no HTTPS support," +
" configure your global.rc file to use HTTP : {0}\n")
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
def __init__(self,
suite,
owner=None,
host=None,
port=None,
timeout=None,
my_uuid=None,
print_uuid=False,
comms_protocol=None,
auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_protocol = comms_protocol
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.api = None
def _compat(self, name, default=None):
"""Return server function name.
Handle back-compat for pre-7.5.0 if relevant.
"""
# Need to load contact info here to get API version.
self._load_contact_info()
if default is None:
default = name
return self.COMPAT_MAP[name].get(self.api, default)
def clear_broadcast(self, **kwargs):
"""Clear broadcast runtime task settings."""
return self._call_server(self._compat('clear_broadcast'),
payload=kwargs)
def expire_broadcast(self, **kwargs):
"""Expire broadcast runtime task settings."""
return self._call_server(self._compat('expire_broadcast'), **kwargs)
def get_broadcast(self, **kwargs):
"""Return broadcast settings."""
return self._call_server(self._compat('get_broadcast'),
method=self.METHOD_GET,
**kwargs)
def get_info(self, command, **kwargs):
"""Return suite info."""
return self._call_server(self._compat('get_info', default='') +
command,
method=self.METHOD_GET,
**kwargs)
def get_latest_state(self, full_mode):
"""Return latest state of the suite (for the GUI)."""
self._load_contact_info()
if self.api == 0:
# Basic compat for pre-7.5.0 suites
# Full mode only.
# Error content/size not supported.
# Report made-up main loop interval of 5.0 seconds.
return {
'cylc_version':
self.get_info('get_cylc_version'),
'full_mode':
full_mode,
'summary':
self.get_suite_state_summary(),
'ancestors':
self.get_info('get_first_parent_ancestors'),
'ancestors_pruned':
self.get_info('get_first_parent_ancestors', pruned=True),
'descendants':
self.get_info('get_first_parent_descendants'),
'err_content':
'',
'err_size':
0,
'mean_main_loop_interval':
5.0
}
else:
return self._call_server('get_latest_state',
method=self.METHOD_GET,
full_mode=full_mode)
def get_suite_state_summary(self):
"""Return the global, task, and family summary data structures."""
return utf8_enforce(
self._call_server(self._compat('get_suite_state_summary'),
method=self.METHOD_GET))
def get_tasks_by_state(self):
"""Returns a dict containing lists of tasks by state.
Result in the form:
{state: [(most_recent_time_string, task_name, point_string), ...]}
"""
return self._call_server(self._compat('get_tasks_by_state'),
method=self.METHOD_GET)
def identify(self):
"""Return suite identity."""
# Note on compat: Suites on 7.6.0 or above can just call "identify",
# but has compat for "id/identity".
return self._call_server('id/identify', method=self.METHOD_GET)
def put_broadcast(self, **kwargs):
"""Put/set broadcast runtime task settings."""
return self._call_server(self._compat('put_broadcast'), payload=kwargs)
def put_command(self, command, **kwargs):
"""Invoke suite command."""
return self._call_server(
self._compat('put_command', default='') + command, **kwargs)
def put_ext_trigger(self, event_message, event_id):
"""Put external trigger."""
return self._call_server(self._compat('put_ext_trigger'),
event_message=event_message,
event_id=event_id)
def put_message(self, task_id, severity, message):
"""Send task message."""
func_name = self._compat('put_message')
if func_name == 'put_message':
return self._call_server(func_name,
task_id=task_id,
severity=severity,
message=message)
else: # pre-7.5.0 API compat
return self._call_server(func_name,
task_id=task_id,
priority=severity,
message=message)
def reset(self):
"""Compat method, does nothing."""
pass
def signout(self):
"""Tell server to forget this client."""
return self._call_server('signout')
def _call_server(self, function, method=METHOD, payload=None, **kwargs):
"""Build server URL + call it"""
url = self._call_server_get_url(function, **kwargs)
# Remove proxy settings from environment for now
environ = {}
for key in ("http_proxy", "https_proxy"):
val = os.environ.pop(key, None)
if val:
environ[key] = val
try:
return self.call_server_impl(url, method, payload)
finally:
os.environ.update(environ)
def _call_server_get_url(self, function, **kwargs):
"""Build request URL."""
comms_protocol = self.comms_protocol
if comms_protocol is None:
# Use standard setting from global configuration
comms_protocol = glbl_cfg().get(['communication', 'method'])
url = '%s://%s:%s/%s' % (comms_protocol, self.host, self.port,
function)
# If there are any parameters left in the dict after popping,
# append them to the url.
if kwargs:
import urllib
params = urllib.urlencode(kwargs, doseq=True)
url += "?" + params
return url
def call_server_impl(self, url, method, payload):
"""Determine whether to use requests or urllib2 to call suite API."""
impl = self._call_server_impl_urllib2
try:
import requests
except ImportError:
pass
else:
if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]:
impl = self._call_server_impl_requests
try:
return impl(url, method, payload)
except ClientConnectError as exc:
if self.suite is None:
raise
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
self.srv_files_mgr.detect_old_contact_file(
self.suite, (self.host, self.port))
except (AssertionError, SuiteServiceFileError):
raise exc
else:
# self.srv_files_mgr.detect_old_contact_file should delete left
# behind contact file if the old suite process no longer
# exists. Should be safe to report that the suite has stopped.
raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite)
def _call_server_impl_requests(self, url, method, payload):
"""Call server with "requests" library."""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter("ignore", InsecureRequestWarning)
if self.session is None:
self.session = requests.Session()
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
comms_protocol = url.split(':', 1)[0] # Can use urlparse?
username, password, verify = self._get_auth(comms_protocol)
try:
ret = session_method(url,
json=payload,
verify=verify,
proxies={},
headers=self._get_headers(),
auth=requests.auth.HTTPDigestAuth(
username, password),
timeout=self.timeout)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError("Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
if ret.status_code == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
if ret.status_code >= 400:
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectedError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(self.suite, self.owner,
self.host, self.auth[1])
try:
return ret.json()
except ValueError:
return ret.text
def _call_server_impl_urllib2(self, url, method, payload):
"""Call server with "urllib2" library."""
import json
import urllib2
import ssl
unverified_context = getattr(ssl, '_create_unverified_context', None)
if unverified_context is not None:
ssl._create_default_https_context = unverified_context
comms_protocol = url.split(':', 1)[0] # Can use urlparse?
username, password = self._get_auth(comms_protocol)[0:2]
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if payload:
payload = json.dumps(payload)
headers_list.append(('Accept', 'application/json'))
json_headers = {
'Content-Type': 'application/json',
'Content-Length': len(payload)
}
else:
payload = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, payload, json_headers)
# This is an unpleasant monkey patch, but there isn't an
# alternative. urllib2 uses POST if there is a data payload
# but that is not the correct criterion.
# The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError("Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
if "timed out" in str(exc):
raise ClientTimeout(url, exc)
else:
raise ClientConnectError(url, exc)
except Exception as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientError(url, exc)
if response.getcode() == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
response_text = response.read()
if response.getcode() >= 400:
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ClientConnectedError(
url, "%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(self.suite, self.owner,
self.host, self.auth[1])
try:
return json.loads(response_text)
except ValueError:
return response_text
def _get_auth(self, protocol):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = self.ANON_AUTH
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite,
self.owner,
self.host,
content=True)
if protocol == 'https':
verify = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite,
self.owner, self.host)
else:
verify = False
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase, verify)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = ("cylc/%s prog_name/%s uuid/%s" %
(CYLC_VERSION, self.prog_name, self.my_uuid))
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string, "From": auth_info}
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
try:
# Always trust the values in the contact file otherwise.
data = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
# Port inside "try" block, as it needs a type conversion
self.port = int(data.get(self.srv_files_mgr.KEY_PORT))
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
self.host = data.get(self.srv_files_mgr.KEY_HOST)
self.owner = data.get(self.srv_files_mgr.KEY_OWNER)
self.comms_protocol = data.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
try:
self.api = int(data.get(self.srv_files_mgr.KEY_API))
except (TypeError, ValueError):
self.api = 0 # Assume cylc-7.5.0 or before
def update_suites_info(updater, full_mode=False):
"""Return mapping of suite info by host, owner and suite name.
Args:
updater (object): gscan or gpanel updater:
Compulsory attributes from updater:
hosts: hosts to scan
owner_pattern: re to filter results by owners
suite_info_map: previous results returned by this function
Optional attributes from updater:
timeout: communication timeout
full_mode (boolean): update in full mode?
Return:
dict: {(host, owner, name): suite_info, ...}
where each "suite_info" is a dict with keys:
KEY_GROUP: group name of suite
KEY_OWNER: suite owner name
KEY_PORT: suite port, for running suites only
KEY_STATES: suite state
KEY_TASKS_BY_STATE: tasks by state
KEY_TITLE: suite title
KEY_UPDATE_TIME: last update time of suite
"""
# Compulsory attributes from updater
# hosts - hosts to scan, or the default set in the site/user global.rc
# owner_pattern - return only suites with owners matching this compiled re
# suite_info_map - previous results returned by this function
# Optional attributes from updater
# timeout - communication timeout
owner_pattern = updater.owner_pattern
timeout = getattr(updater, "comms_timeout", None)
# name_pattern - return only suites with names matching this compiled re
name_pattern = getattr(updater, "name_pattern", None)
# Determine items to scan
results = {}
items = []
if full_mode and not updater.hosts:
# Scan users suites. Walk "~/cylc-run/" to get (host, port) from
# ".service/contact" for active suites
suite_srv_files_mgr = SuiteSrvFilesManager()
if owner_pattern is None:
# Run directory of current user only
run_dirs = [GLOBAL_CFG.get_host_item('run directory')]
else:
# Run directory of all users matching "owner_pattern".
# But skip those with /nologin or /false shells
run_dirs = []
skips = ('/false', '/nologin')
for pwent in getpwall():
if any(pwent.pw_shell.endswith(s) for s in (skips)):
continue
if owner_pattern.match(pwent.pw_name):
run_dirs.append(
GLOBAL_CFG.get_host_item('run directory',
owner=pwent.pw_name,
owner_home=pwent.pw_dir))
if cylc.flags.debug:
sys.stderr.write(
'Listing suites:%s%s\n' %
(_UPDATE_DEBUG_DELIM, _UPDATE_DEBUG_DELIM.join(run_dirs)))
for run_d in run_dirs:
for dirpath, dnames, fnames in os.walk(run_d, followlinks=True):
if updater.quit:
return
# Always descend for top directory, but
# don't descend further if it has a:
# * .service/
# * cylc-suite.db: (pre-cylc-7 suites don't have ".service/").
if dirpath != run_d and (suite_srv_files_mgr.DIR_BASE_SRV
in dnames
or 'cylc-suite.db' in fnames):
dnames[:] = []
# Choose only suites with .service and matching filter
reg = os.path.relpath(dirpath, run_d)
try:
contact_data = suite_srv_files_mgr.load_contact_file(reg)
except (SuiteServiceFileError, IOError, TypeError, ValueError):
continue
else:
items.append((contact_data[suite_srv_files_mgr.KEY_HOST],
contact_data[suite_srv_files_mgr.KEY_PORT]))
elif full_mode:
# Scan full port range on all hosts
items.extend(updater.hosts)
else:
# Scan suites in previous results only
for (host, owner, name), prev_result in updater.suite_info_map.items():
port = prev_result.get(KEY_PORT)
if port:
items.append((host, port))
else:
results[(host, owner, name)] = prev_result
if not items:
return results
if cylc.flags.debug:
sys.stderr.write(
'Scan items:%s%s\n' %
(_UPDATE_DEBUG_DELIM,
_UPDATE_DEBUG_DELIM.join(str(item) for item in items)))
# Scan
for host, port, result in scan_many(items,
timeout=timeout,
updater=updater):
if updater.quit:
return
if (name_pattern and not name_pattern.match(result[KEY_NAME]) or
owner_pattern and not owner_pattern.match(result[KEY_OWNER])):
continue
try:
result[KEY_PORT] = port
results[(host, result[KEY_OWNER], result[KEY_NAME])] = result
result[KEY_UPDATE_TIME] = int(float(result[KEY_UPDATE_TIME]))
except (KeyError, TypeError, ValueError):
pass
expire_threshold = time() - DURATION_EXPIRE_STOPPED
for (host, owner, name), prev_result in updater.suite_info_map.items():
if updater.quit:
return
if ((host, owner, name) in results
or owner_pattern and not owner_pattern.match(owner)
or name_pattern and not name_pattern.match(name)):
# OK if suite already in current results set.
# Don't bother if:
# * previous owner does not match current owner pattern
# * previous suite name does not match current name pattern
continue
if prev_result.get(KEY_PORT):
# A previously running suite is no longer running.
# Get suite info with "cat-state", if possible, and include in the
# results set.
try:
prev_result = _update_stopped_suite_info((host, owner, name))
except (IndexError, TypeError, ValueError):
continue
if prev_result.get(KEY_UPDATE_TIME, 0) > expire_threshold:
results[(host, owner, name)] = prev_result
return results
class RemoteJobHostManager(object):
"""Manage a remote job host."""
_INSTANCE = None
@classmethod
def get_inst(cls):
"""Return a singleton instance of this class."""
if cls._INSTANCE is None:
cls._INSTANCE = cls()
return cls._INSTANCE
def __init__(self):
self.initialised = {} # {(user, host): should_unlink, ...}
self.single_task_mode = False
self.suite_srv_files_mgr = SuiteSrvFilesManager()
def init_suite_run_dir(self, reg, host, owner):
"""Initialise suite run dir on a user@host.
Create SUITE_RUN_DIR/log/job/ if necessary.
Install suite contact environment file.
Install suite python modules.
Raise RemoteJobHostInitError if initialisation cannot complete.
"""
if host is None:
host = 'localhost'
if ((host, owner) in [('localhost', None), ('localhost', USER)]
or (host, owner) in self.initialised or self.single_task_mode):
return
user_at_host = host
if owner:
user_at_host = owner + '@' + host
r_suite_run_dir = GLOBAL_CFG.get_derived_host_item(
reg, 'suite run directory', host, owner)
r_log_job_dir = GLOBAL_CFG.get_derived_host_item(
reg, 'suite job log directory', host, owner)
r_suite_srv_dir = os.path.join(r_suite_run_dir,
self.suite_srv_files_mgr.DIR_BASE_SRV)
# Create a UUID file in the service directory.
# If remote host has the file in its service directory, we can assume
# that the remote host has a shared file system with the suite host.
ssh_tmpl = GLOBAL_CFG.get_host_item('remote shell template', host,
owner)
uuid_str = str(uuid4())
uuid_fname = os.path.join(
self.suite_srv_files_mgr.get_suite_srv_dir(reg), uuid_str)
try:
open(uuid_fname, 'wb').close()
proc = Popen(shlex.split(ssh_tmpl) + [
'-n', user_at_host, 'test', '-e',
os.path.join(r_suite_srv_dir, uuid_str)
],
stdout=PIPE,
stderr=PIPE)
if proc.wait() == 0:
# Initialised, but no need to tidy up
self.initialised[(host, owner)] = False
return
finally:
try:
os.unlink(uuid_fname)
except OSError:
pass
cmds = []
# Command to create suite directory structure on remote host.
cmds.append(
shlex.split(ssh_tmpl) + [
'-n', user_at_host, 'mkdir', '-p', r_suite_run_dir,
r_log_job_dir, r_suite_srv_dir
])
# Command to copy contact and authentication files to remote host.
# Note: no need to do this if task communication method is "poll".
should_unlink = GLOBAL_CFG.get_host_item('task communication method',
host, owner) != "poll"
if should_unlink:
scp_tmpl = GLOBAL_CFG.get_host_item('remote copy template', host,
owner)
cmds.append(
shlex.split(scp_tmpl) + [
'-p',
self.suite_srv_files_mgr.get_contact_file(reg),
self.suite_srv_files_mgr.get_auth_item(
self.suite_srv_files_mgr.FILE_BASE_PASSPHRASE, reg),
self.suite_srv_files_mgr.get_auth_item(
self.suite_srv_files_mgr.FILE_BASE_SSL_CERT, reg),
user_at_host + ':' + r_suite_srv_dir + '/'
])
# Command to copy python library to remote host.
suite_run_py = os.path.join(
GLOBAL_CFG.get_derived_host_item(reg, 'suite run directory'),
'python')
if os.path.isdir(suite_run_py):
cmds.append(
shlex.split(scp_tmpl) + [
'-pr', suite_run_py, user_at_host + ':' + r_suite_run_dir +
'/'
])
# Run commands in sequence.
for cmd in cmds:
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
out, err = proc.communicate()
if proc.wait():
raise RemoteJobHostInitError(
RemoteJobHostInitError.MSG_INIT, user_at_host,
' '.join([quote(item) for item in cmd]), proc.returncode,
out, err)
self.initialised[(host, owner)] = should_unlink
LOG.info('Initialised %s:%s' % (user_at_host, r_suite_run_dir))
def unlink_suite_contact_files(self, reg):
"""Remove suite contact files from initialised hosts.
This is called on shutdown, so we don't want anything to hang.
Terminate any incomplete SSH commands after 10 seconds.
"""
# Issue all SSH commands in parallel
procs = {}
for (host, owner), should_unlink in self.initialised.items():
if not should_unlink:
continue
user_at_host = host
if owner:
user_at_host = owner + '@' + host
ssh_tmpl = GLOBAL_CFG.get_host_item('remote shell template', host,
owner)
r_suite_contact_file = os.path.join(
GLOBAL_CFG.get_derived_host_item(reg, 'suite run directory',
host, owner),
SuiteSrvFilesManager.DIR_BASE_SRV,
SuiteSrvFilesManager.FILE_BASE_CONTACT)
cmd = shlex.split(ssh_tmpl) + [
'-n', user_at_host, 'rm', '-f', r_suite_contact_file
]
procs[user_at_host] = (cmd, Popen(cmd, stdout=PIPE, stderr=PIPE))
# Wait for commands to complete for a max of 10 seconds
timeout = time() + 10.0
while procs and time() < timeout:
for user_at_host, (cmd, proc) in procs.copy().items():
if proc.poll() is None:
continue
del procs[user_at_host]
out, err = proc.communicate()
if proc.wait():
ERR.warning(
RemoteJobHostInitError(
RemoteJobHostInitError.MSG_TIDY, user_at_host,
' '.join([quote(item) for item in cmd]),
proc.returncode, out, err))
# Terminate any remaining commands
for user_at_host, (cmd, proc) in procs.items():
try:
proc.terminate()
except OSError:
pass
out, err = proc.communicate()
if proc.wait():
ERR.warning(
RemoteJobHostInitError(
RemoteJobHostInitError.MSG_TIDY, user_at_host,
' '.join([quote(item) for item in cmd]),
proc.returncode, out, err))
class SuiteRuntimeServiceClient(object):
"""Client for calling the HTTP(S) API of running suites."""
ANON_AUTH = ('anon', NO_PASSPHRASE, False)
COMPAT_MAP = { # Limited pre-7.5.0 API compat mapping
'clear_broadcast': {0: 'broadcast/clear'},
'expire_broadcast': {0: 'broadcast/expire'},
'get_broadcast': {0: 'broadcast/get'},
'get_info': {0: 'info/'},
'get_suite_state_summary': {0: 'state/get_state_summary'},
'put_broadcast': {0: 'broadcast/put'},
'put_command': {0: 'command/'},
'put_ext_trigger': {0: 'ext-trigger/put'},
'put_messages': {0: 'message/put', 1: 'put_message'},
}
ERROR_NO_HTTPS_SUPPORT = (
"ERROR: server has no HTTPS support," +
" configure your global.rc file to use HTTP : {0}\n"
)
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
MSG_RETRY_INTVL = 5.0
MSG_MAX_TRIES = 7
MSG_TIMEOUT = 30.0
def __init__(
self, suite, owner=None, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False, auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.comms1 = {} # content in primary contact file
self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def _compat(self, name, default=None):
"""Return server function name.
Handle back-compat for pre-7.5.0 if relevant.
"""
# Need to load contact info here to get API version.
self._load_contact_info()
if default is None:
default = name
return self.COMPAT_MAP[name].get(
self.comms1.get(self.srv_files_mgr.KEY_API), default)
def clear_broadcast(self, payload):
"""Clear broadcast runtime task settings."""
return self._call_server(
self._compat('clear_broadcast'), payload=payload)
def expire_broadcast(self, **kwargs):
"""Expire broadcast runtime task settings."""
return self._call_server(self._compat('expire_broadcast'), **kwargs)
def get_broadcast(self, **kwargs):
"""Return broadcast settings."""
return self._call_server(
self._compat('get_broadcast'), method=self.METHOD_GET, **kwargs)
def get_info(self, command, **kwargs):
"""Return suite info."""
return self._call_server(
self._compat('get_info', default='') + command,
method=self.METHOD_GET, **kwargs)
def get_latest_state(self, full_mode=False):
"""Return latest state of the suite (for the GUI)."""
self._load_contact_info()
if self.comms1.get(self.srv_files_mgr.KEY_API) == 0:
# Basic compat for pre-7.5.0 suites
# Full mode only.
# Error content/size not supported.
# Report made-up main loop interval of 5.0 seconds.
return {
'cylc_version': self.get_info('get_cylc_version'),
'full_mode': full_mode,
'summary': self.get_suite_state_summary(),
'ancestors': self.get_info('get_first_parent_ancestors'),
'ancestors_pruned': self.get_info(
'get_first_parent_ancestors', pruned=True),
'descendants': self.get_info('get_first_parent_descendants'),
'err_content': '',
'err_size': 0,
'mean_main_loop_interval': 5.0}
else:
return self._call_server(
'get_latest_state',
method=self.METHOD_GET, full_mode=full_mode)
def get_suite_state_summary(self):
"""Return the global, task, and family summary data structures."""
return utf8_enforce(self._call_server(
self._compat('get_suite_state_summary'), method=self.METHOD_GET))
def identify(self):
"""Return suite identity."""
# Note on compat: Suites on 7.6.0 or above can just call "identify",
# but has compat for "id/identity".
return self._call_server('id/identify', method=self.METHOD_GET)
def put_broadcast(self, payload):
"""Put/set broadcast runtime task settings."""
return self._call_server(
self._compat('put_broadcast'), payload=payload)
def put_command(self, command, **kwargs):
"""Invoke suite command."""
return self._call_server(
self._compat('put_command', default='') + command, **kwargs)
def put_ext_trigger(self, event_message, event_id):
"""Put external trigger."""
return self._call_server(
self._compat('put_ext_trigger'),
event_message=event_message, event_id=event_id)
def put_messages(self, payload):
"""Send task messages to suite server program.
Arguments:
payload (dict):
task_job (str): Task job as "CYCLE/TASK_NAME/SUBMIT_NUM".
event_time (str): Event time as string.
messages (list): List in the form [[severity, message], ...].
"""
retry_intvl = float(self.comms1.get(
self.srv_files_mgr.KEY_TASK_MSG_RETRY_INTVL,
self.MSG_RETRY_INTVL))
max_tries = int(self.comms1.get(
self.srv_files_mgr.KEY_TASK_MSG_MAX_TRIES,
self.MSG_MAX_TRIES))
for i in range(1, max_tries + 1): # 1..max_tries inclusive
orig_timeout = self.timeout
if self.timeout is None:
self.timeout = self.MSG_TIMEOUT
try:
func_name = self._compat('put_messages')
if func_name == 'put_messages':
results = self._call_server(func_name, payload=payload)
elif func_name == 'put_message': # API 1, 7.5.0 compat
cycle, name = payload['task_job'].split('/')[0:2]
for severity, message in payload['messages']:
results.append(self._call_server(
func_name, task_id='%s.%s' % (name, cycle),
severity=severity, message=message))
else: # API 0, pre-7.5.0 compat, priority instead of severity
cycle, name = payload['task_job'].split('/')[0:2]
for severity, message in payload['messages']:
results.append(self._call_server(
func_name, task_id='%s.%s' % (name, cycle),
priority=severity, message=message))
except ClientInfoError:
# Contact info file not found, suite probably not running.
# Don't bother with retry, suite restart will poll any way.
raise
except ClientError as exc:
now = get_current_time_string()
sys.stderr.write(
"%s WARNING - Message send failed, try %s of %s: %s\n" % (
now, i, max_tries, exc))
if i < max_tries:
sys.stderr.write(
" retry in %s seconds, timeout is %s\n" % (
retry_intvl, self.timeout))
sleep(retry_intvl)
# Reset in case contact info or passphrase change
self.comms1 = {}
self.host = None
self.port = None
self.auth = None
else:
if i > 1:
# Continue to write to STDERR, so users can easily see that
# it has recovered from previous failures.
sys.stderr.write(
"%s INFO - Send message: try %s of %s succeeded\n" % (
get_current_time_string(), i, max_tries))
return results
finally:
self.timeout = orig_timeout
def reset(self):
"""Compat method, does nothing."""
pass
def signout(self):
"""Tell server to forget this client."""
return self._call_server('signout')
def _call_server(self, function, method=METHOD, payload=None, **kwargs):
"""Build server URL + call it"""
if self.comms2:
return self._call_server_via_comms2(function, payload, **kwargs)
url = self._call_server_get_url(function, **kwargs)
# Remove proxy settings from environment for now
environ = {}
for key in ("http_proxy", "https_proxy"):
val = os.environ.pop(key, None)
if val:
environ[key] = val
try:
return self.call_server_impl(url, method, payload)
finally:
os.environ.update(environ)
def _call_server_get_url(self, function, **kwargs):
"""Build request URL."""
scheme = self.comms1.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
if scheme is None:
# Use standard setting from global configuration
scheme = glbl_cfg().get(['communication', 'method'])
url = '%s://%s:%s/%s' % (
scheme, self.host, self.port, function)
# If there are any parameters left in the dict after popping,
# append them to the url.
if kwargs:
import urllib
params = urllib.urlencode(kwargs, doseq=True)
url += "?" + params
return url
def call_server_impl(self, url, method, payload):
"""Determine whether to use requests or urllib2 to call suite API."""
impl = self._call_server_impl_urllib2
try:
import requests
except ImportError:
pass
else:
if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]:
impl = self._call_server_impl_requests
try:
return impl(url, method, payload)
except ClientConnectError as exc:
if self.suite is None:
raise
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
self.srv_files_mgr.detect_old_contact_file(
self.suite, (self.host, self.port))
except (AssertionError, SuiteServiceFileError):
raise exc
else:
# self.srv_files_mgr.detect_old_contact_file should delete left
# behind contact file if the old suite process no longer
# exists. Should be safe to report that the suite has stopped.
raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite)
def _call_server_impl_requests(self, url, method, payload):
"""Call server with "requests" library."""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter("ignore", InsecureRequestWarning)
if self.session is None:
self.session = requests.Session()
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
scheme = url.split(':', 1)[0] # Can use urlparse?
username, password, verify = self._get_auth(scheme)
try:
ret = session_method(
url,
json=payload,
verify=verify,
proxies={},
headers=self._get_headers(),
auth=requests.auth.HTTPDigestAuth(username, password),
timeout=self.timeout
)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
if ret.status_code == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
if ret.status_code >= 400:
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectedError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return ret.json()
except ValueError:
return ret.text
def _call_server_impl_urllib2(self, url, method, payload):
"""Call server with "urllib2" library."""
import json
import urllib2
import ssl
unverified_context = getattr(ssl, '_create_unverified_context', None)
if unverified_context is not None:
ssl._create_default_https_context = unverified_context
scheme = url.split(':', 1)[0] # Can use urlparse?
username, password = self._get_auth(scheme)[0:2]
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if payload:
payload = json.dumps(payload)
headers_list.append(('Accept', 'application/json'))
json_headers = {'Content-Type': 'application/json',
'Content-Length': len(payload)}
else:
payload = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, payload, json_headers)
# This is an unpleasant monkey patch, but there isn't an
# alternative. urllib2 uses POST if there is a data payload
# but that is not the correct criterion.
# The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
if "timed out" in str(exc):
raise ClientTimeout(url, exc)
else:
raise ClientConnectError(url, exc)
except Exception as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientError(url, exc)
if response.getcode() == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
response_text = response.read()
if response.getcode() >= 400:
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ClientConnectedError(
url,
"%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return json.loads(response_text)
except ValueError:
return response_text
def _call_server_via_comms2(self, function, payload, **kwargs):
"""Call server via "cylc client --use-ssh".
Call "cylc client --use-ssh" using `subprocess.Popen`. Payload and
arguments of the API method are serialized as JSON and are written to a
temporary file, which is then used as the STDIN of the "cylc client"
command. The external call here should be even safer than a direct
HTTP(S) call, as it can be blocked by SSH before it even gets a chance
to make the subsequent HTTP(S) call.
Arguments:
function (str): name of API method, argument 1 of "cylc client".
payload (str): extra data or information for the API method.
**kwargs (dict): arguments for the API method.
"""
import json
from cylc.remote import remote_cylc_cmd
command = ["client", function, self.suite]
if payload:
kwargs["payload"] = payload
if kwargs:
from tempfile import TemporaryFile
stdin = TemporaryFile()
json.dump(kwargs, stdin)
stdin.seek(0)
else:
# With stdin=None, `remote_cylc_cmd` will:
# * Set stdin to open(os.devnull)
# * Add `-n` to the SSH command
stdin = None
proc = remote_cylc_cmd(
command, self.owner, self.host, capture=True,
ssh_login_shell=(self.comms1.get(
self.srv_files_mgr.KEY_SSH_USE_LOGIN_SHELL
) in ['True', 'true']),
ssh_cylc=(r'%s/bin/cylc' % self.comms1.get(
self.srv_files_mgr.KEY_DIR_ON_SUITE_HOST)
),
stdin=stdin,
)
out = proc.communicate()[0]
return_code = proc.wait()
if return_code:
from pipes import quote
command_str = " ".join(quote(item) for item in command)
raise ClientError(command_str, "return-code=%d" % return_code)
return json.loads(out)
def _get_auth(self, protocol):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = self.ANON_AUTH
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite, self.owner, self.host, content=True)
if protocol == 'https':
verify = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT,
self.suite, self.owner, self.host)
else:
verify = False
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase, verify)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = (
"cylc/%s prog_name/%s uuid/%s" % (
CYLC_VERSION, self.prog_name, self.my_uuid
)
)
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string,
"From": auth_info}
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
try:
# Always trust the values in the contact file otherwise.
self.comms1 = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
# Port inside "try" block, as it needs a type conversion
self.port = int(self.comms1.get(self.srv_files_mgr.KEY_PORT))
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
else:
# Check mismatch suite UUID
env_suite = os.getenv(self.srv_files_mgr.KEY_NAME)
env_uuid = os.getenv(self.srv_files_mgr.KEY_UUID)
if (self.suite and env_suite and env_suite == self.suite and
env_uuid and
env_uuid != self.comms1.get(self.srv_files_mgr.KEY_UUID)):
raise ClientInfoUUIDError(
env_uuid, self.comms1[self.srv_files_mgr.KEY_UUID])
# All good
self.host = self.comms1.get(self.srv_files_mgr.KEY_HOST)
self.owner = self.comms1.get(self.srv_files_mgr.KEY_OWNER)
if self.srv_files_mgr.KEY_API not in self.comms1:
self.comms1[self.srv_files_mgr.KEY_API] = 0 # <=7.5.0 compat
# Indirect comms settings
self.comms2.clear()
try:
self.comms2.update(self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host,
SuiteSrvFilesManager.FILE_BASE_CONTACT2))
except SuiteServiceFileError:
pass
def __init__(self):
self.initialised_hosts = {} # user_at_host: should_unlink
self.single_task_mode = False
self.suite_srv_files_mgr = SuiteSrvFilesManager()
def setUp(self):
self.suite_srv_files_mgr = SuiteSrvFilesManager()
def setUp(self):
self.suite_srv_files_mgr = SuiteSrvFilesManager()
class CommsDaemon(object):
"""Wrap HTTPS daemon for a suite."""
def __init__(self, suite):
# Suite only needed for back-compat with old clients (see below):
self.suite = suite
# Figure out the ports we are allowed to use.
base_port = GLOBAL_CFG.get(['communication', 'base port'])
max_ports = GLOBAL_CFG.get(
['communication', 'maximum number of ports'])
self.ok_ports = range(int(base_port), int(base_port) + int(max_ports))
random.shuffle(self.ok_ports)
comms_options = GLOBAL_CFG.get(['communication', 'options'])
# HTTP Digest Auth uses MD5 - pretty secure in this use case.
# Extending it with extra algorithms is allowed, but won't be
# supported by most browsers. requests and urllib2 are OK though.
self.hash_algorithm = "MD5"
if "SHA1" in comms_options:
# Note 'SHA' rather than 'SHA1'.
self.hash_algorithm = "SHA"
self.srv_files_mgr = SuiteSrvFilesManager()
self.get_ha1 = cherrypy.lib.auth_digest.get_ha1_dict_plain(
{
'cylc':
self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
suite,
content=True),
'anon':
NO_PASSPHRASE
},
algorithm=self.hash_algorithm)
try:
self.cert = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, suite)
self.pkey = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_PEM, suite)
except SuiteServiceFileError:
self.cert = None
self.pkey = None
self.client_reporter = CommsClientReporter.get_inst()
self.start()
def shutdown(self):
"""Shutdown the daemon."""
if hasattr(self, "engine"):
self.engine.exit()
self.engine.block()
def connect(self, obj, name):
"""Connect obj and name to the daemon."""
cherrypy.tree.mount(obj, "/" + name)
def disconnect(self, obj):
"""Disconnect obj from the daemon."""
pass
def get_port(self):
"""Return the daemon port."""
return self.port
def report_connection_if_denied(self):
"""Report connection if denied."""
self.client_reporter.report_connection_if_denied()
def start(self):
"""Start quick web service."""
# cherrypy.config["tools.encode.on"] = True
# cherrypy.config["tools.encode.encoding"] = "utf-8"
cherrypy.config["server.socket_host"] = '0.0.0.0'
cherrypy.config["engine.autoreload.on"] = False
try:
from OpenSSL import SSL, crypto
cherrypy.config['server.ssl_module'] = 'pyopenSSL'
cherrypy.config['server.ssl_certificate'] = self.cert
cherrypy.config['server.ssl_private_key'] = self.pkey
except ImportError:
ERR.warning("no HTTPS/OpenSSL support")
cherrypy.config['log.screen'] = None
key = binascii.hexlify(os.urandom(16))
cherrypy.config.update({
'tools.auth_digest.on':
True,
'tools.auth_digest.realm':
self.suite,
'tools.auth_digest.get_ha1':
self.get_ha1,
'tools.auth_digest.key':
key,
'tools.auth_digest.algorithm':
self.hash_algorithm
})
cherrypy.tools.connect_log = cherrypy.Tool(
'on_end_resource', self.report_connection_if_denied)
cherrypy.config['tools.connect_log.on'] = True
self.engine = cherrypy.engine
for port in self.ok_ports:
cherrypy.config["server.socket_port"] = port
try:
cherrypy.engine.start()
cherrypy.engine.wait(cherrypy.engine.states.STARTED)
except Exception:
if cylc.flags.debug:
traceback.print_exc()
# We need to reinitialise the httpserver for each port attempt.
cherrypy.server.httpserver = None
else:
if cherrypy.engine.state == cherrypy.engine.states.STARTED:
self.port = port
return
raise Exception("No available ports")
class RemoteJobHostManager(object):
"""Manage a remote job host."""
_INSTANCE = None
@classmethod
def get_inst(cls):
"""Return a singleton instance of this class."""
if cls._INSTANCE is None:
cls._INSTANCE = cls()
return cls._INSTANCE
def __init__(self):
self.initialised_hosts = {} # user_at_host: should_unlink
self.single_task_mode = False
self.suite_srv_files_mgr = SuiteSrvFilesManager()
def init_suite_run_dir(self, reg, user_at_host):
"""Initialise suite run dir on a user@host.
Create SUITE_RUN_DIR/log/job/ if necessary.
Install suite contact environment file.
Install suite python modules.
Raise RemoteJobHostInitError if initialisation cannot complete.
"""
if "@" in user_at_host:
owner, host = user_at_host.split("@", 1)
else:
owner, host = None, user_at_host
if (
(owner, host) in [(None, "localhost"), (USER, "localhost")]
or host in self.initialised_hosts
or self.single_task_mode
):
return
r_suite_run_dir = GLOBAL_CFG.get_derived_host_item(reg, "suite run directory", host, owner)
r_log_job_dir = GLOBAL_CFG.get_derived_host_item(reg, "suite job log directory", host, owner)
r_suite_srv_dir = os.path.join(r_suite_run_dir, self.suite_srv_files_mgr.DIR_BASE_SRV)
# Create a UUID file in the service directory.
# If remote host has the file in its service directory, we can assume
# that the remote host has a shared file system with the suite host.
ssh_tmpl = GLOBAL_CFG.get_host_item("remote shell template", host, owner)
uuid_str = str(uuid4())
uuid_fname = os.path.join(self.suite_srv_files_mgr.get_suite_srv_dir(reg), uuid_str)
try:
open(uuid_fname, "wb").close()
proc = Popen(
shlex.split(ssh_tmpl) + ["-n", user_at_host, "test", "-e", os.path.join(r_suite_srv_dir, uuid_str)],
stdout=PIPE,
stderr=PIPE,
)
if proc.wait() == 0:
# Initialised, but no need to tidy up
self.initialised_hosts[user_at_host] = False
return
finally:
try:
os.unlink(uuid_fname)
except OSError:
pass
cmds = []
# Command to create suite directory structure on remote host.
cmds.append(
shlex.split(ssh_tmpl) + ["-n", user_at_host, "mkdir", "-p", r_suite_run_dir, r_log_job_dir, r_suite_srv_dir]
)
# Command to copy contact and authentication files to remote host.
# Note: no need to do this if task communication method is "poll".
should_unlink = GLOBAL_CFG.get_host_item("task communication method", host, owner) != "poll"
if should_unlink:
scp_tmpl = GLOBAL_CFG.get_host_item("remote copy template", host, owner)
cmds.append(
shlex.split(scp_tmpl)
+ [
"-p",
self.suite_srv_files_mgr.get_contact_file(reg),
self.suite_srv_files_mgr.get_auth_item(self.suite_srv_files_mgr.FILE_BASE_PASSPHRASE, reg),
self.suite_srv_files_mgr.get_auth_item(self.suite_srv_files_mgr.FILE_BASE_SSL_CERT, reg),
user_at_host + ":" + r_suite_srv_dir + "/",
]
)
# Command to copy python library to remote host.
suite_run_py = os.path.join(GLOBAL_CFG.get_derived_host_item(reg, "suite run directory"), "python")
if os.path.isdir(suite_run_py):
cmds.append(shlex.split(scp_tmpl) + ["-pr", suite_run_py, user_at_host + ":" + r_suite_run_dir + "/"])
# Run commands in sequence.
for cmd in cmds:
proc = Popen(cmd, stdout=PIPE, stderr=PIPE)
out, err = proc.communicate()
if proc.wait():
raise RemoteJobHostInitError(
RemoteJobHostInitError.MSG_INIT,
user_at_host,
" ".join([quote(item) for item in cmd]),
proc.returncode,
out,
err,
)
self.initialised_hosts[user_at_host] = should_unlink
LOG.info("Initialised %s:%s" % (user_at_host, r_suite_run_dir))
def unlink_suite_contact_files(self, reg):
"""Remove suite contact files from initialised hosts.
This is called on shutdown, so we don't want anything to hang.
Terminate any incomplete SSH commands after 10 seconds.
"""
# Issue all SSH commands in parallel
procs = {}
for user_at_host, should_unlink in self.initialised_hosts.items():
if not should_unlink:
continue
if "@" in user_at_host:
owner, host = user_at_host.split("@", 1)
else:
owner, host = None, user_at_host
ssh_tmpl = GLOBAL_CFG.get_host_item("remote shell template", host, owner)
r_suite_contact_file = os.path.join(
GLOBAL_CFG.get_derived_host_item(reg, "suite run directory", host, owner),
SuiteSrvFilesManager.DIR_BASE_SRV,
SuiteSrvFilesManager.FILE_BASE_CONTACT,
)
cmd = shlex.split(ssh_tmpl) + ["-n", user_at_host, "rm", "-f", r_suite_contact_file]
procs[user_at_host] = (cmd, Popen(cmd, stdout=PIPE, stderr=PIPE))
# Wait for commands to complete for a max of 10 seconds
timeout = time() + 10.0
while procs and time() < timeout:
for user_at_host, (cmd, proc) in procs.items():
if not proc.poll():
continue
del procs[user_at_host]
out, err = proc.communicate()
if proc.wait():
ERR.warning(
RemoteJobHostInitError(
RemoteJobHostInitError.MSG_TIDY,
user_at_host,
" ".join([quote(item) for item in cmd]),
proc.returncode,
out,
err,
)
)
# Terminate any remaining commands
for user_at_host, (cmd, proc) in procs.items():
try:
proc.terminate()
except OSError:
pass
out, err = proc.communicate()
proc.wait()
ERR.warning(
RemoteJobHostInitError(
RemoteJobHostInitError.MSG_TIDY,
user_at_host,
" ".join([quote(item) for item in cmd]),
proc.returncode,
out,
err,
)
)
