def test_compile_url_compiler_http(self):
"""Test that the url compiler produces a http request when
http is specified."""
myclient = SuiteRuntimeServiceClient(
"test-suite", host=get_host(), port=80)
myclient.comms1[SuiteSrvFilesManager.KEY_COMMS_PROTOCOL] = 'http'
self.assertEqual(
'http://%s:80/test_command?apples=False&oranges=True' %
get_host(),
myclient._call_server_get_url(
"test_command", apples="False", oranges="True"))
def test_url_compiler_https(self):
"""Tests that the url parser works for a single url and command
using https"""
myclient = SuiteRuntimeServiceClient(
"test-suite", host=get_host(), port=80)
myclient.comms1[SuiteSrvFilesManager.KEY_COMMS_PROTOCOL] = 'https'
self.assertEqual(
'https://%s:80/test_command?apples=False&oranges=True' %
get_host(),
myclient._call_server_get_url(
"test_command", apples="False", oranges="True"))
def test_url_compiler_https(self):
"""Tests that the url parser works for a single url and command
using https"""
myclient = SuiteRuntimeServiceClient(
"test-suite", host=get_host(), port=80)
myclient.comms1[SuiteSrvFilesManager.KEY_COMMS_PROTOCOL] = 'https'
self.assertEqual(
'https://%s:80/test_command?apples=False&oranges=True' %
get_host(),
myclient._call_server_get_url(
"test_command", apples="False", oranges="True"))
def test_compile_url_compiler_http(self):
"""Test that the url compiler produces a http request when
http is specified."""
myclient = SuiteRuntimeServiceClient(
"test-suite", host=get_host(), port=80,
comms_protocol="http")
self.assertEqual(
'http://%s:80/test_command?apples=False&oranges=True' %
get_host(),
myclient._call_server_get_url(
"test_command", apples="False", oranges="True"))
def test_url_compiler_https(self):
"""Tests that the url parser works for a single url and command
using https"""
myclient = SuiteRuntimeServiceClient(
"test-suite", host=get_host(), port=80,
comms_protocol="https")
self.assertEqual(
'https://%s:80/test_command?apples=False&oranges=True' %
get_host(),
myclient._call_server_get_url(
"test_command", apples="False", oranges="True"))
def test_compile_url_compiler_http(self):
"""Test that the url compiler produces a http request when
http is specified."""
myclient = SuiteRuntimeServiceClient(
"test-suite", host=get_host(), port=80)
myclient.comms1[SuiteSrvFilesManager.KEY_COMMS_PROTOCOL] = 'http'
self.assertEqual(
'http://%s:80/test_command?apples=False&oranges=True' %
get_host(),
myclient._call_server_get_url(
"test_command", apples="False", oranges="True"))
def __init__(
self, suite, owner=None, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False, auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.comms1 = {} # content in primary contact file
self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def test_remove_bad_hosts(self):
"""Test the '_remove_bad_hosts' method.
Test using 'localhost' only since remote host functionality is
contained only inside remote_cylc_cmd() so is outside of the scope
of HostAppointer.
"""
self.mock_global_config(set_hosts=['localhost'])
self.assertTrue(self.app._remove_bad_hosts().get('localhost', False))
# Test 'localhost' true identifier is treated properly too.
self.mock_global_config(set_hosts=[get_host()])
self.assertTrue(self.app._remove_bad_hosts().get('localhost', False))
self.mock_global_config(set_hosts=['localhost', 'FAKE_HOST'])
# Check for no exceptions and 'localhost' but not 'FAKE_HOST' data
# Difficult to unittest for specific stderr string; this is sufficient.
self.assertTrue(self.app._remove_bad_hosts().get('localhost', False))
self.assertTrue(self.app._remove_bad_hosts().get('FAKE_HOST', True))
# Apply thresholds impossible to pass; check results in host removal.
self.mock_global_config(set_hosts=['localhost'],
set_thresholds='load:15 0.0')
self.assertEqual(self.app._remove_bad_hosts(), {})
self.mock_global_config(set_hosts=['localhost'],
set_thresholds='memory 1000000000')
self.assertEqual(self.app._remove_bad_hosts(), {})
def _setup_event_mail(self, itask, event):
"""Set up task event notification, by email."""
if event in self.NON_UNIQUE_EVENTS:
key1 = (self.HANDLER_MAIL,
'%s-%d' % (event, itask.non_unique_events.get(event, 1)))
else:
key1 = (self.HANDLER_MAIL, event)
id_key = (key1, str(itask.point), itask.tdef.name, itask.submit_num)
if (id_key in self.event_timers or event not in self._get_events_conf(
itask, "mail events", [])):
return
retry_delays = self._get_events_conf(itask, "mail retry delays")
if not retry_delays:
retry_delays = [0]
self.event_timers[id_key] = TaskActionTimer(
TaskEventMailContext(
self.HANDLER_MAIL, # key
self.HANDLER_MAIL, # ctx_type
self._get_events_conf( # mail_from
itask,
"mail from",
"notifications@" + get_host(),
),
self._get_events_conf(itask, "mail to", get_user()), # mail_to
self._get_events_conf(itask, "mail smtp"), # mail_smtp
),
retry_delays)
def test_remove_bad_hosts(self):
"""Test the '_remove_bad_hosts' method.
Test using 'localhost' only since remote host functionality is
contained only inside remote_cylc_cmd() so is outside of the scope
of HostAppointer.
"""
self.mock_global_config(set_hosts=['localhost'])
self.assertTrue(self.app._remove_bad_hosts().get('localhost', False))
# Test 'localhost' true identifier is treated properly too.
self.mock_global_config(set_hosts=[get_host()])
self.assertTrue(self.app._remove_bad_hosts().get('localhost', False))
self.mock_global_config(set_hosts=['localhost', 'FAKE_HOST'])
# Check for no exceptions and 'localhost' but not 'FAKE_HOST' data
# Difficult to unittest for specific stderr string; this is sufficient.
self.assertTrue(self.app._remove_bad_hosts().get('localhost', False))
self.assertTrue(self.app._remove_bad_hosts().get('FAKE_HOST', True))
# Apply thresholds impossible to pass; check results in host removal.
self.mock_global_config(
set_hosts=['localhost'], set_thresholds='load:15 0.0')
self.assertEqual(self.app._remove_bad_hosts(), {})
self.mock_global_config(
set_hosts=['localhost'], set_thresholds='memory 1000000000')
self.assertEqual(self.app._remove_bad_hosts(), {})
def __init__(self,
suite,
owner=None,
host=None,
port=None,
timeout=None,
my_uuid=None,
print_uuid=False,
comms_protocol=None,
auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_protocol = comms_protocol
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
try:
# Always trust the values in the contact file otherwise.
data = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
# Port inside "try" block, as it needs a type conversion
self.port = int(data.get(self.srv_files_mgr.KEY_PORT))
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
self.host = data.get(self.srv_files_mgr.KEY_HOST)
self.owner = data.get(self.srv_files_mgr.KEY_OWNER)
self.comms_protocol = data.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
try:
self.api = int(data.get(self.srv_files_mgr.KEY_API))
except (TypeError, ValueError):
self.api = 0 # Assume cylc-7.5.0 or before
def __init__(
self, suite, owner=None, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False, auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.comms1 = {} # content in primary contact file
self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def _setup_event_mail(self, itask, event):
"""Set up task event notification, by email."""
if event in self.NON_UNIQUE_EVENTS:
key1 = (
self.HANDLER_MAIL,
'%s-%d' % (event, itask.non_unique_events.get(event, 1)))
else:
key1 = (self.HANDLER_MAIL, event)
id_key = (key1, str(itask.point), itask.tdef.name, itask.submit_num)
if (id_key in self.event_timers or
event not in self._get_events_conf(itask, "mail events", [])):
return
retry_delays = self._get_events_conf(itask, "mail retry delays")
if not retry_delays:
retry_delays = [0]
self.event_timers[id_key] = TaskActionTimer(
TaskEventMailContext(
self.HANDLER_MAIL, # key
self.HANDLER_MAIL, # ctx_type
self._get_events_conf( # mail_from
itask,
"mail from",
"notifications@" + get_host(),
),
self._get_events_conf(itask, "mail to", get_user()), # mail_to
self._get_events_conf(itask, "mail smtp"), # mail_smtp
),
retry_delays)
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = (
"cylc/%s prog_name/%s uuid/%s" % (
CYLC_VERSION, self.prog_name, self.my_uuid
)
)
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string,
"From": auth_info}
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = (
"cylc/%s prog_name/%s uuid/%s" % (
CYLC_VERSION, self.prog_name, self.my_uuid
)
)
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string,
"From": auth_info}
def test_compile_url_compiler_none_specified(self):
"""Test that the url compiler produces a http request when
none is specified. This should retrieve it from the
global config."""
myclient = SuiteRuntimeServiceClient(
"test-suite", host=get_host(), port=80)
url = myclient._call_server_get_url(
"test_command", apples="False", oranges="True")
# Check that the url has had http (or https) appended
# to it. (If it does not start with "http*" then something
# has gone wrong.)
self.assertTrue(url.startswith("http"))
def test_compile_url_compiler_none_specified(self):
"""Test that the url compiler produces a http request when
none is specified. This should retrieve it from the
global config."""
myclient = SuiteRuntimeServiceClient(
"test-suite", host=get_host(), port=80)
url = myclient._call_server_get_url(
"test_command", apples="False", oranges="True")
# Check that the url has had http (or https) appended
# to it. (If it does not start with "http*" then something
# has gone wrong.)
self.assertTrue(url.startswith("http"))
def _get_ssl_cert(self, path, pkey_obj):
"""Load or create ssl.cert file for suite in path.
Self-signed SSL certificate file.
"""
try:
from OpenSSL import crypto
except ImportError:
# OpenSSL not installed, so we can't use HTTPS anyway.
return
# Use suite host as the 'common name', but no more than 64 chars.
host = get_host()
common_name = host
if len(common_name) > 64:
common_name = common_name[:61] + "..."
# See https://github.com/kennethreitz/requests/issues/2621
ext = crypto.X509Extension(
"subjectAltName",
False,
"DNS:%(dns)s, IP:%(ip)s, DNS:%(ip)s" % {
"dns": host, "ip": get_local_ip_address(host)})
file_name = self._locate_item(self.FILE_BASE_SSL_CERT, path)
if file_name:
cert_obj = crypto.load_certificate(
crypto.FILETYPE_PEM, open(file_name).read())
try:
prev_ext = cert_obj.get_extension(0)
except (AttributeError, IndexError):
pass
else:
if (cert_obj.get_subject().CN == common_name and
not cert_obj.has_expired() and
str(prev_ext) == str(ext)):
return # certificate good for the same suite and host
# Generate a new certificate
cert_obj = crypto.X509()
# cert_obj.get_subject().O = "Cylc"
cert_obj.get_subject().CN = common_name
cert_obj.gmtime_adj_notBefore(0)
cert_obj.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) # 10 years.
cert_obj.set_issuer(cert_obj.get_subject())
cert_obj.set_pubkey(pkey_obj)
# Set a random serial number to avoid browser error
# SEC_ERROR_REUSED_ISSUER_AND_SERIAL.
cert_obj.set_serial_number(uuid4().int)
cert_obj.add_extensions([ext])
cert_obj.sign(pkey_obj, 'sha256')
self._dump_item(
path, self.FILE_BASE_SSL_CERT,
crypto.dump_certificate(crypto.FILETYPE_PEM, cert_obj))
def __init__(self, *args, **kwargs):
self.exposed = True
self.suite_dao = CylcReviewDAO()
self.logo = os.path.basename(
get_util_home("doc", "src", "cylc-logo.png"))
self.title = self.TITLE
self.host_name = get_host()
if self.host_name and "." in self.host_name:
self.host_name = self.host_name.split(".", 1)[0]
self.cylc_version = CYLC_VERSION
template_env = jinja2.Environment(loader=jinja2.FileSystemLoader(
get_util_home("lib", "cylc", "cylc-review", "template")))
template_env.filters['urlise'] = self.url2hyperlink
self.template_env = template_env
def _get_ssl_cert(self, path, pkey_obj):
"""Load or create ssl.cert file for suite in path.
Self-signed SSL certificate file.
"""
try:
from OpenSSL import crypto
except ImportError:
# OpenSSL not installed, so we can't use HTTPS anyway.
return
# Use suite host as the 'common name', but no more than 64 chars.
host = get_host()
common_name = host
if len(common_name) > 64:
common_name = common_name[:61] + "..."
# See https://github.com/kennethreitz/requests/issues/2621
ext = crypto.X509Extension(
"subjectAltName",
False,
"DNS:%(dns)s, IP:%(ip)s, DNS:%(ip)s" % {
"dns": host, "ip": get_local_ip_address(host)})
file_name = self._locate_item(self.FILE_BASE_SSL_CERT, path)
if file_name:
cert_obj = crypto.load_certificate(
crypto.FILETYPE_PEM, open(file_name).read())
try:
prev_ext = cert_obj.get_extension(0)
except (AttributeError, IndexError):
pass
else:
if (cert_obj.get_subject().CN == common_name and
not cert_obj.has_expired() and
str(prev_ext) == str(ext)):
return # certificate good for the same suite and host
# Generate a new certificate
cert_obj = crypto.X509()
# cert_obj.get_subject().O = "Cylc"
cert_obj.get_subject().CN = common_name
cert_obj.gmtime_adj_notBefore(0)
cert_obj.gmtime_adj_notAfter(10 * 365 * 24 * 60 * 60) # 10 years.
cert_obj.set_issuer(cert_obj.get_subject())
cert_obj.set_pubkey(pkey_obj)
cert_obj.set_serial_number(1)
cert_obj.add_extensions([ext])
cert_obj.sign(pkey_obj, 'sha256')
self._dump_item(
path, self.FILE_BASE_SSL_CERT,
crypto.dump_certificate(crypto.FILETYPE_PEM, cert_obj))
def start(self):
"""Start quick web service."""
# cherrypy.config["tools.encode.on"] = True
# cherrypy.config["tools.encode.encoding"] = "utf-8"
cherrypy.config["server.socket_host"] = get_host()
cherrypy.config["engine.autoreload.on"] = False
if self.comms_method == "https":
# Setup SSL etc. Otherwise fail and exit.
# Require connection method to be the same e.g HTTP/HTTPS matching.
cherrypy.config['server.ssl_module'] = 'pyopenSSL'
cherrypy.config['server.ssl_certificate'] = self.cert
cherrypy.config['server.ssl_private_key'] = self.pkey
cherrypy.config['log.screen'] = None
key = binascii.hexlify(os.urandom(16))
cherrypy.config.update({
'tools.auth_digest.on':
True,
'tools.auth_digest.realm':
self.suite,
'tools.auth_digest.get_ha1':
self.get_ha1,
'tools.auth_digest.key':
key,
'tools.auth_digest.algorithm':
self.hash_algorithm
})
cherrypy.tools.connect_log = cherrypy.Tool(
'on_end_resource', self._report_connection_if_denied)
cherrypy.config['tools.connect_log.on'] = True
self.engine = cherrypy.engine
for port in self.ok_ports:
cherrypy.config["server.socket_port"] = port
try:
cherrypy.engine.start()
cherrypy.engine.wait(cherrypy.engine.states.STARTED)
except cherrypy.process.wspbus.ChannelFailures:
if cylc.flags.debug:
traceback.print_exc()
# We need to reinitialise the httpserver for each port attempt.
cherrypy.server.httpserver = None
else:
if cherrypy.engine.state == cherrypy.engine.states.STARTED:
self.port = port
return
raise Exception("No available ports")
def _run_event_mail(self, config, ctx):
"""Helper for "run_event_handlers", do mail notification."""
if ctx.event in self.get_events_conf(config, 'mail events', []):
# SMTP server
env = dict(os.environ)
mail_smtp = self.get_events_conf(config, 'mail smtp')
if mail_smtp:
env['smtp'] = mail_smtp
subject = '[suite %(event)s] %(suite)s' % {
'suite': ctx.suite, 'event': ctx.event}
stdin_str = ''
for name, value in [
('suite event', ctx.event),
('reason', ctx.reason),
('suite', ctx.suite),
('host', ctx.host),
('port', ctx.port),
('owner', ctx.owner)]:
if value:
stdin_str += '%s: %s\n' % (name, value)
mail_footer_tmpl = self.get_events_conf(config, 'mail footer')
if mail_footer_tmpl:
stdin_str += (mail_footer_tmpl + '\n') % {
'host': ctx.host,
'port': ctx.port,
'owner': ctx.owner,
'suite': ctx.suite}
proc_ctx = SuiteProcContext(
(self.SUITE_EVENT_HANDLER, ctx.event),
[
'mail',
'-s', subject,
'-r', self.get_events_conf(
config,
'mail from', 'notifications@' + get_host()),
self.get_events_conf(config, 'mail to', get_user()),
],
env=env,
stdin_str=stdin_str)
if self.proc_pool.closed:
# Run command in foreground if process pool is closed
self.proc_pool.run_command(proc_ctx)
self._run_event_handlers_callback(proc_ctx)
else:
# Run command using process pool otherwise
self.proc_pool.put_command(
proc_ctx, self._run_event_mail_callback)
def _run_event_mail(self, config, ctx):
"""Helper for "run_event_handlers", do mail notification."""
if ctx.event in self.get_events_conf(config, 'mail events', []):
# SMTP server
env = dict(os.environ)
mail_smtp = self.get_events_conf(config, 'mail smtp')
if mail_smtp:
env['smtp'] = mail_smtp
subject = '[suite %(event)s] %(suite)s' % {
'suite': ctx.suite,
'event': ctx.event
}
stdin_str = ''
for name, value in [('suite event', ctx.event),
('reason', ctx.reason), ('suite', ctx.suite),
('host', ctx.host), ('port', ctx.port),
('owner', ctx.owner)]:
if value:
stdin_str += '%s: %s\n' % (name, value)
mail_footer_tmpl = self.get_events_conf(config, 'mail footer')
if mail_footer_tmpl:
stdin_str += (mail_footer_tmpl + '\n') % {
'host': ctx.host,
'port': ctx.port,
'owner': ctx.owner,
'suite': ctx.suite
}
proc_ctx = SuiteProcContext(
(self.SUITE_EVENT_HANDLER, ctx.event), [
'mail',
'-s',
subject,
'-r',
self.get_events_conf(config, 'mail from',
'notifications@' + get_host()),
self.get_events_conf(config, 'mail to', get_user()),
],
env=env,
stdin_str=stdin_str)
if self.proc_pool.is_closed():
# Run command in foreground if process pool is closed
self.proc_pool.run_command(proc_ctx)
self._run_event_handlers_callback(proc_ctx)
else:
# Run command using process pool otherwise
self.proc_pool.put_command(proc_ctx,
self._run_event_mail_callback)
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
try:
# Always trust the values in the contact file otherwise.
self.comms1 = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
# Port inside "try" block, as it needs a type conversion
self.port = int(self.comms1.get(self.srv_files_mgr.KEY_PORT))
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
else:
# Check mismatch suite UUID
env_suite = os.getenv(self.srv_files_mgr.KEY_NAME)
env_uuid = os.getenv(self.srv_files_mgr.KEY_UUID)
if (self.suite and env_suite and env_suite == self.suite
and env_uuid and
env_uuid != self.comms1.get(self.srv_files_mgr.KEY_UUID)):
raise ClientInfoUUIDError(
env_uuid, self.comms1[self.srv_files_mgr.KEY_UUID])
# All good
self.host = self.comms1.get(self.srv_files_mgr.KEY_HOST)
self.owner = self.comms1.get(self.srv_files_mgr.KEY_OWNER)
if self.srv_files_mgr.KEY_API not in self.comms1:
self.comms1[self.srv_files_mgr.KEY_API] = 0 # <=7.5.0 compat
# Indirect comms settings
self.comms2.clear()
try:
self.comms2.update(
self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host,
SuiteSrvFilesManager.FILE_BASE_CONTACT2))
except SuiteServiceFileError:
pass
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
# Always trust the values in the contact file otherwise.
data = self.srv_files_mgr.load_contact_file(self.suite, self.owner,
self.host)
self.host = data.get(self.srv_files_mgr.KEY_HOST)
self.port = int(data.get(self.srv_files_mgr.KEY_PORT))
self.owner = data.get(self.srv_files_mgr.KEY_OWNER)
self.comms_protocol = data.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
try:
# Always trust the values in the contact file otherwise.
self.comms1 = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
# Port inside "try" block, as it needs a type conversion
self.port = int(self.comms1.get(self.srv_files_mgr.KEY_PORT))
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
else:
# Check mismatch suite UUID
env_suite = os.getenv(self.srv_files_mgr.KEY_NAME)
env_uuid = os.getenv(self.srv_files_mgr.KEY_UUID)
if (self.suite and env_suite and env_suite == self.suite and
env_uuid and
env_uuid != self.comms1.get(self.srv_files_mgr.KEY_UUID)):
raise ClientInfoUUIDError(
env_uuid, self.comms1[self.srv_files_mgr.KEY_UUID])
# All good
self.host = self.comms1.get(self.srv_files_mgr.KEY_HOST)
self.owner = self.comms1.get(self.srv_files_mgr.KEY_OWNER)
if self.srv_files_mgr.KEY_API not in self.comms1:
self.comms1[self.srv_files_mgr.KEY_API] = 0 # <=7.5.0 compat
# Indirect comms settings
self.comms2.clear()
try:
self.comms2.update(self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host,
SuiteSrvFilesManager.FILE_BASE_CONTACT2))
except SuiteServiceFileError:
pass
def start(self):
"""Start quick web service."""
# cherrypy.config["tools.encode.on"] = True
# cherrypy.config["tools.encode.encoding"] = "utf-8"
cherrypy.config["server.socket_host"] = get_host()
cherrypy.config["engine.autoreload.on"] = False
if self.comms_method == "https":
# Setup SSL etc. Otherwise fail and exit.
# Require connection method to be the same e.g HTTP/HTTPS matching.
cherrypy.config['server.ssl_module'] = 'pyopenSSL'
cherrypy.config['server.ssl_certificate'] = self.cert
cherrypy.config['server.ssl_private_key'] = self.pkey
cherrypy.config['log.screen'] = None
key = binascii.hexlify(os.urandom(16))
cherrypy.config.update({
'tools.auth_digest.on': True,
'tools.auth_digest.realm': self.suite,
'tools.auth_digest.get_ha1': self.get_ha1,
'tools.auth_digest.key': key,
'tools.auth_digest.algorithm': self.hash_algorithm
})
cherrypy.tools.connect_log = cherrypy.Tool(
'on_end_resource', self._report_connection_if_denied)
cherrypy.config['tools.connect_log.on'] = True
self.engine = cherrypy.engine
for port in self.ok_ports:
cherrypy.config["server.socket_port"] = port
try:
cherrypy.engine.start()
cherrypy.engine.wait(cherrypy.engine.states.STARTED)
except cherrypy.process.wspbus.ChannelFailures:
if cylc.flags.debug:
traceback.print_exc()
# We need to reinitialise the httpserver for each port attempt.
cherrypy.server.httpserver = None
else:
if cherrypy.engine.state == cherrypy.engine.states.STARTED:
self.port = port
return
raise Exception("No available ports")
def cache_passphrase(self, reg, owner, host, value):
"""Cache and dump passphrase for a remote suite in standard location.
Save passphrase to ~/.cylc/auth/owner@host/reg if possible.
This is normally called on a successful authentication, and will cache
the remote passphrase in memory as well.
"""
if owner is None:
owner = get_user()
if host is None:
host = get_host()
path = self._get_cache_dir(reg, owner, host)
self.cache[self.FILE_BASE_PASSPHRASE][(reg, owner, host)] = value
# Dump to a file only for remote suites loaded via SSH.
if self.can_disk_cache_passphrases.get((reg, owner, host)):
# Although not desirable, failing to dump the passphrase to a file
# is not disastrous.
try:
self._dump_item(path, self.FILE_BASE_PASSPHRASE, value)
except (IOError, OSError):
if cylc.flags.debug:
import traceback
traceback.print_exc()
def cache_passphrase(self, reg, owner, host, value):
"""Cache and dump passphrase for a remote suite in standard location.
Save passphrase to ~/.cylc/auth/owner@host/reg if possible.
This is normally called on a successful authentication, and will cache
the remote passphrase in memory as well.
"""
if owner is None:
owner = get_user()
if host is None:
host = get_host()
path = self._get_cache_dir(reg, owner, host)
self.cache[self.FILE_BASE_PASSPHRASE][(reg, owner, host)] = value
# Dump to a file only for remote suites loaded via SSH.
if self.can_disk_cache_passphrases.get((reg, owner, host)):
# Although not desirable, failing to dump the passphrase to a file
# is not disastrous.
try:
self._dump_item(path, self.FILE_BASE_PASSPHRASE, value)
except (IOError, OSError):
if cylc.flags.debug:
import traceback
traceback.print_exc()
def _is_local_auth_ok(self, reg, owner, host):
"""Return True if it is OK to use local passphrase file.
Use values in ~/cylc-run/REG/.service/contact to make a judgement.
Cache results in self.can_use_load_auths.
"""
if (reg, owner, host) not in self.can_use_load_auths:
if is_remote(host, owner):
fname = os.path.join(
self.get_suite_srv_dir(reg), self.FILE_BASE_CONTACT)
data = {}
try:
for line in open(fname):
key, value = (
[item.strip() for item in line.split("=", 1)])
data[key] = value
except (IOError, ValueError):
# No contact file
self.can_use_load_auths[(reg, owner, host)] = False
else:
# Contact file exists, check values match
if owner is None:
owner = get_user()
if host is None:
host = get_host()
host_value = data.get(self.KEY_HOST, "")
self.can_use_load_auths[(reg, owner, host)] = (
reg == data.get(self.KEY_NAME) and
owner == data.get(self.KEY_OWNER) and
(
host == host_value or
host == host_value.split(".", 1)[0] # no domain
)
)
else:
self.can_use_load_auths[(reg, owner, host)] = True
return self.can_use_load_auths[(reg, owner, host)]
def _is_local_auth_ok(self, reg, owner, host):
"""Return True if it is OK to use local passphrase, ssl.* files.
Use values in ~/cylc-run/REG/.service/contact to make a judgement.
Cache results in self.can_use_load_auths.
"""
if (reg, owner, host) not in self.can_use_load_auths:
if is_remote(host, owner):
fname = os.path.join(
self.get_suite_srv_dir(reg), self.FILE_BASE_CONTACT)
data = {}
try:
for line in open(fname):
key, value = (
[item.strip() for item in line.split("=", 1)])
data[key] = value
except (IOError, ValueError):
# No contact file
self.can_use_load_auths[(reg, owner, host)] = False
else:
# Contact file exists, check values match
if owner is None:
owner = get_user()
if host is None:
host = get_host()
host_value = data.get(self.KEY_HOST, "")
self.can_use_load_auths[(reg, owner, host)] = (
reg == data.get(self.KEY_NAME) and
owner == data.get(self.KEY_OWNER) and
(
host == host_value or
host == host_value.split(".", 1)[0] # no domain
)
)
else:
self.can_use_load_auths[(reg, owner, host)] = True
return self.can_use_load_auths[(reg, owner, host)]
def get_auth_item(self, item, reg, owner=None, host=None, content=False):
"""Locate/load passphrase, SSL private key, SSL certificate, etc.
Return file name, or content of file if content=True is set.
Files are searched from these locations in order:
1/ For running task jobs, service directory under:
a/ $CYLC_SUITE_RUN_DIR for remote jobs.
b/ $CYLC_SUITE_RUN_DIR_ON_SUITE_HOST for local jobs or remote jobs
with SSH messaging.
2/ (Passphrases only) From memory cache, for remote suite passphrases.
Don't use if content=False.
3/ For suite on local user@host. The suite service directory.
4/ Location under $HOME/.cylc/ for remote suite control from accounts
that do not actually need the suite definition directory to be
installed:
$HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/
5/ For remote suites, try locating the file from the suite service
directory on remote owner@host via SSH. If content=False, the value
of the located file will be dumped under:
$HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/
"""
if item not in [
self.FILE_BASE_PASSPHRASE, self.FILE_BASE_CONTACT,
self.FILE_BASE_CONTACT2]:
raise ValueError("%s: item not recognised" % item)
if item == self.FILE_BASE_PASSPHRASE:
self.can_disk_cache_passphrases[(reg, owner, host)] = False
if reg == os.getenv('CYLC_SUITE_NAME'):
env_keys = []
if 'CYLC_SUITE_RUN_DIR' in os.environ:
# 1(a)/ Task messaging call.
env_keys.append('CYLC_SUITE_RUN_DIR')
elif self.KEY_SUITE_RUN_DIR_ON_SUITE_HOST in os.environ:
# 1(b)/ Task messaging call via ssh messaging.
env_keys.append(self.KEY_SUITE_RUN_DIR_ON_SUITE_HOST)
for key in env_keys:
path = os.path.join(os.environ[key], self.DIR_BASE_SRV)
if content:
value = self._load_local_item(item, path)
else:
value = self._locate_item(item, path)
if value:
return value
# 2/ From memory cache
if item in self.cache:
my_owner = owner
my_host = host
if my_owner is None:
my_owner = get_user()
if my_host is None:
my_host = get_host()
try:
return self.cache[item][(reg, my_owner, my_host)]
except KeyError:
pass
# 3/ Local suite service directory
if self._is_local_auth_ok(reg, owner, host):
path = self.get_suite_srv_dir(reg)
if content:
value = self._load_local_item(item, path)
else:
value = self._locate_item(item, path)
if value:
return value
# 4/ Disk cache for remote suites
if owner is not None and host is not None:
paths = [self._get_cache_dir(reg, owner, host)]
short_host = host.split('.', 1)[0]
if short_host != host:
paths.append(self._get_cache_dir(reg, owner, short_host))
for path in paths:
if content:
value = self._load_local_item(item, path)
else:
value = self._locate_item(item, path)
if value:
return value
# 5/ Use SSH to load content from remote owner@host
# Note: It is not possible to find ".service/contact2" on the suite
# host, because it is installed on task host by "cylc remote-init" on
# demand.
if item != self.FILE_BASE_CONTACT2:
value = self._load_remote_item(item, reg, owner, host)
if value:
if item == self.FILE_BASE_PASSPHRASE:
self.can_disk_cache_passphrases[(reg, owner, host)] = True
if not content:
path = self._get_cache_dir(reg, owner, host)
self._dump_item(path, item, value)
value = os.path.join(path, item)
return value
raise SuiteServiceFileError("Couldn't get %s" % item)
def get_auth_item(self, item, reg, owner=None, host=None, content=False):
"""Locate/load passphrase, SSL private key, SSL certificate, etc.
Return file name, or content of file if content=True is set.
Files are searched from these locations in order:
1/ For running task jobs, service directory under:
a/ $CYLC_SUITE_RUN_DIR for remote jobs.
b/ $CYLC_SUITE_RUN_DIR_ON_SUITE_HOST for local jobs or remote jobs
with SSH messaging.
2/ (Passphrases only) From memory cache, for remote suite passphrases.
Don't use if content=False.
3/ For suite on local user@host. The suite service directory.
4/ Location under $HOME/.cylc/ for remote suite control from accounts
that do not actually need the suite definition directory to be
installed:
$HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/
5/ For remote suites, try locating the file from the suite service
directory on remote owner@host via SSH. If content=False, the value
of the located file will be dumped under:
$HOME/.cylc/auth/SUITE_OWNER@SUITE_HOST/SUITE_NAME/
"""
if item not in [
self.FILE_BASE_SSL_CERT, self.FILE_BASE_SSL_PEM,
self.FILE_BASE_PASSPHRASE, self.FILE_BASE_CONTACT,
self.FILE_BASE_CONTACT2
]:
raise ValueError("%s: item not recognised" % item)
if item == self.FILE_BASE_PASSPHRASE:
self.can_disk_cache_passphrases[(reg, owner, host)] = False
if reg == os.getenv('CYLC_SUITE_NAME'):
env_keys = []
if 'CYLC_SUITE_RUN_DIR' in os.environ:
# 1(a)/ Task messaging call.
env_keys.append('CYLC_SUITE_RUN_DIR')
elif self.KEY_SUITE_RUN_DIR_ON_SUITE_HOST in os.environ:
# 1(b)/ Task messaging call via ssh messaging.
env_keys.append(self.KEY_SUITE_RUN_DIR_ON_SUITE_HOST)
for key in env_keys:
path = os.path.join(os.environ[key], self.DIR_BASE_SRV)
if content:
value = self._load_local_item(item, path)
else:
value = self._locate_item(item, path)
if value:
return value
# 2/ From memory cache
if item in self.cache:
my_owner = owner
my_host = host
if my_owner is None:
my_owner = get_user()
if my_host is None:
my_host = get_host()
try:
return self.cache[item][(reg, my_owner, my_host)]
except KeyError:
pass
# 3/ Local suite service directory
if self._is_local_auth_ok(reg, owner, host):
path = self.get_suite_srv_dir(reg)
if content:
value = self._load_local_item(item, path)
else:
value = self._locate_item(item, path)
if value:
return value
# 4/ Disk cache for remote suites
if owner is not None and host is not None:
paths = [self._get_cache_dir(reg, owner, host)]
short_host = host.split('.', 1)[0]
if short_host != host:
paths.append(self._get_cache_dir(reg, owner, short_host))
for path in paths:
if content:
value = self._load_local_item(item, path)
else:
value = self._locate_item(item, path)
if value:
return value
# 5/ Use SSH to load content from remote owner@host
# Note: It is not possible to find ".service/contact2" on the suite
# host, because it is installed on task host by "cylc remote-init" on
# demand.
if item != self.FILE_BASE_CONTACT2:
value = self._load_remote_item(item, reg, owner, host)
if value:
if item == self.FILE_BASE_PASSPHRASE:
self.can_disk_cache_passphrases[(reg, owner, host)] = True
if not content:
path = self._get_cache_dir(reg, owner, host)
self._dump_item(path, item, value)
value = os.path.join(path, item)
return value
raise SuiteServiceFileError("Couldn't get %s" % item)
def test_is_remote_host_on_localhost(self):
"""is_remote_host with localhost."""
self.assertFalse(is_remote_host(None))
self.assertFalse(is_remote_host('localhost'))
self.assertFalse(is_remote_host(os.getenv('HOSTNAME')))
self.assertFalse(is_remote_host(get_host()))
def test_is_remote_host_on_localhost(self):
"""is_remote_host with localhost."""
self.assertFalse(is_remote_host(None))
self.assertFalse(is_remote_host('localhost'))
self.assertFalse(is_remote_host(os.getenv('HOSTNAME')))
self.assertFalse(is_remote_host(get_host()))
def submit_task_jobs(self, suite, itasks, is_simulation=False):
"""Prepare and submit task jobs.
Submit tasks where possible. Ignore tasks that are waiting for host
select command to complete, or tasks that are waiting for remote
initialisation. Bad host select command, error writing to a job file or
bad remote initialisation will cause a bad task - leading to submission
failure.
This method uses prep_submit_task_job() as helper.
Return (list): list of tasks that attempted submission.
"""
if is_simulation:
return self._simulation_submit_task_jobs(itasks)
# Prepare tasks for job submission
prepared_tasks, bad_tasks = self.prep_submit_task_jobs(suite, itasks)
# Reset consumed host selection results
self.task_remote_mgr.remote_host_select_reset()
if not prepared_tasks:
return bad_tasks
# Group task jobs by (host, owner)
auth_itasks = {} # {(host, owner): [itask, ...], ...}
for itask in prepared_tasks:
auth_itasks.setdefault((itask.task_host, itask.task_owner), [])
auth_itasks[(itask.task_host, itask.task_owner)].append(itask)
# Submit task jobs for each (host, owner) group
done_tasks = bad_tasks
for (host, owner), itasks in sorted(auth_itasks.items()):
is_init = self.task_remote_mgr.remote_init(host, owner)
if is_init is None:
# Remote is waiting to be initialised
for itask in itasks:
itask.set_summary_message(self.REMOTE_INIT_MSG)
continue
# Ensure that localhost background/at jobs are recorded as running
# on the host name of the current suite host, rather than just
# "localhost". On suite restart on a different suite host, this
# allows the restart logic to correctly poll the status of the
# background/at jobs that may still be running on the previous
# suite host.
if (
self.batch_sys_mgr.is_job_local_to_host(
itask.summary['batch_sys_name']) and
not is_remote_host(host)
):
owner_at_host = get_host()
else:
owner_at_host = host
# Persist
if owner:
owner_at_host = owner + '@' + owner_at_host
now_str = get_current_time_string()
done_tasks.extend(itasks)
for itask in itasks:
# Log and persist
LOG.info(
'[%s] -submit-num=%d, owner@host=%s',
itask, itask.submit_num, owner_at_host)
self.suite_db_mgr.put_insert_task_jobs(itask, {
'is_manual_submit': itask.is_manual_submit,
'try_num': itask.get_try_num(),
'time_submit': now_str,
'user_at_host': owner_at_host,
'batch_sys_name': itask.summary['batch_sys_name'],
})
itask.is_manual_submit = False
if is_init == REMOTE_INIT_FAILED:
# Remote has failed to initialise
# Set submit-failed for all affected tasks
for itask in itasks:
itask.local_job_file_path = None # reset for retry
log_task_job_activity(
SubProcContext(
self.JOBS_SUBMIT,
'(init %s)' % owner_at_host,
err=REMOTE_INIT_FAILED,
ret_code=1),
suite, itask.point, itask.tdef.name)
self.task_events_mgr.process_message(
itask, CRITICAL,
self.task_events_mgr.EVENT_SUBMIT_FAILED)
continue
# Build the "cylc jobs-submit" command
cmd = ['cylc', self.JOBS_SUBMIT]
if LOG.isEnabledFor(DEBUG):
cmd.append('--debug')
if get_utc_mode():
cmd.append('--utc-mode')
remote_mode = False
kwargs = {}
for key, value, test_func in [
('host', host, is_remote_host),
('user', owner, is_remote_user)]:
if test_func(value):
cmd.append('--%s=%s' % (key, value))
remote_mode = True
kwargs[key] = value
if remote_mode:
cmd.append('--remote-mode')
cmd.append('--')
cmd.append(glbl_cfg().get_derived_host_item(
suite, 'suite job log directory', host, owner))
# Chop itasks into a series of shorter lists if it's very big
# to prevent overloading of stdout and stderr pipes.
itasks = sorted(itasks, key=lambda itask: itask.identity)
chunk_size = len(itasks) // ((len(itasks) // 100) + 1) + 1
itasks_batches = [
itasks[i:i + chunk_size] for i in range(0,
len(itasks),
chunk_size)]
LOG.debug(
'%s ... # will invoke in batches, sizes=%s',
cmd, [len(b) for b in itasks_batches])
for i, itasks_batch in enumerate(itasks_batches):
stdin_files = []
job_log_dirs = []
for itask in itasks_batch:
if remote_mode:
stdin_files.append(
get_task_job_job_log(
suite, itask.point, itask.tdef.name,
itask.submit_num))
job_log_dirs.append(get_task_job_id(
itask.point, itask.tdef.name, itask.submit_num))
# The job file is now (about to be) used: reset the file
# write flag so that subsequent manual retrigger will
# generate a new job file.
itask.local_job_file_path = None
itask.state.reset_state(TASK_STATUS_READY)
if itask.state.outputs.has_custom_triggers():
self.suite_db_mgr.put_update_task_outputs(itask)
self.proc_pool.put_command(
SubProcContext(
self.JOBS_SUBMIT,
cmd + job_log_dirs,
stdin_files=stdin_files,
job_log_dirs=job_log_dirs,
**kwargs
),
self._submit_task_jobs_callback, [suite, itasks_batch])
return done_tasks
def submit_task_jobs(self, suite, itasks, is_simulation=False):
"""Prepare and submit task jobs.
Submit tasks where possible. Ignore tasks that are waiting for host
select command to complete, or tasks that are waiting for remote
initialisation. Bad host select command, error writing to a job file or
bad remote initialisation will cause a bad task - leading to submission
failure.
This method uses prep_submit_task_job() as helper.
Return (list): list of tasks that attempted submission.
"""
if is_simulation:
return self._simulation_submit_task_jobs(itasks)
# Prepare tasks for job submission
prepared_tasks, bad_tasks = self.prep_submit_task_jobs(suite, itasks)
# Reset consumed host selection results
self.task_remote_mgr.remote_host_select_reset()
if not prepared_tasks:
return bad_tasks
# Group task jobs by (host, owner)
auth_itasks = {} # {(host, owner): [itask, ...], ...}
for itask in prepared_tasks:
auth_itasks.setdefault((itask.task_host, itask.task_owner), [])
auth_itasks[(itask.task_host, itask.task_owner)].append(itask)
# Submit task jobs for each (host, owner) group
done_tasks = bad_tasks
for (host, owner), itasks in sorted(auth_itasks.items()):
is_init = self.task_remote_mgr.remote_init(host, owner)
if is_init is None:
# Remote is waiting to be initialised
for itask in itasks:
itask.set_summary_message(self.REMOTE_INIT_MSG)
continue
# Ensure that localhost background/at jobs are recorded as running
# on the host name of the current suite host, rather than just
# "localhost". On suite restart on a different suite host, this
# allows the restart logic to correctly poll the status of the
# background/at jobs that may still be running on the previous
# suite host.
if (
self.batch_sys_mgr.is_job_local_to_host(
itask.summary['batch_sys_name']) and
not is_remote_host(host)
):
owner_at_host = get_host()
else:
owner_at_host = host
# Persist
if owner:
owner_at_host = owner + '@' + owner_at_host
now_str = get_current_time_string()
done_tasks.extend(itasks)
for itask in itasks:
# Log and persist
LOG.info(
'[%s] -submit-num=%d, owner@host=%s',
itask, itask.submit_num, owner_at_host)
self.suite_db_mgr.put_insert_task_jobs(itask, {
'is_manual_submit': itask.is_manual_submit,
'try_num': itask.get_try_num(),
'time_submit': now_str,
'user_at_host': owner_at_host,
'batch_sys_name': itask.summary['batch_sys_name'],
})
itask.is_manual_submit = False
if is_init == REMOTE_INIT_FAILED:
# Remote has failed to initialise
# Set submit-failed for all affected tasks
for itask in itasks:
itask.local_job_file_path = None # reset for retry
log_task_job_activity(
SubProcContext(
self.JOBS_SUBMIT,
'(init %s)' % owner_at_host,
err=REMOTE_INIT_FAILED,
ret_code=1),
suite, itask.point, itask.tdef.name)
self.task_events_mgr.process_message(
itask, CRITICAL,
self.task_events_mgr.EVENT_SUBMIT_FAILED)
continue
# Build the "cylc jobs-submit" command
cmd = ['cylc', self.JOBS_SUBMIT]
if LOG.isEnabledFor(DEBUG):
cmd.append('--debug')
if get_utc_mode():
cmd.append('--utc-mode')
remote_mode = False
kwargs = {}
for key, value, test_func in [
('host', host, is_remote_host),
('user', owner, is_remote_user)]:
if test_func(value):
cmd.append('--%s=%s' % (key, value))
remote_mode = True
kwargs[key] = value
if remote_mode:
cmd.append('--remote-mode')
cmd.append('--')
cmd.append(glbl_cfg().get_derived_host_item(
suite, 'suite job log directory', host, owner))
# Chop itasks into a series of shorter lists if it's very big
# to prevent overloading of stdout and stderr pipes.
itasks = sorted(itasks, key=lambda itask: itask.identity)
chunk_size = len(itasks) // ((len(itasks) // 100) + 1) + 1
itasks_batches = [
itasks[i:i + chunk_size] for i in range(0,
len(itasks),
chunk_size)]
LOG.debug(
'%s ... # will invoke in batches, sizes=%s',
cmd, [len(b) for b in itasks_batches])
for i, itasks_batch in enumerate(itasks_batches):
stdin_files = []
job_log_dirs = []
for itask in itasks_batch:
if remote_mode:
stdin_files.append(
get_task_job_job_log(
suite, itask.point, itask.tdef.name,
itask.submit_num))
job_log_dirs.append(get_task_job_id(
itask.point, itask.tdef.name, itask.submit_num))
# The job file is now (about to be) used: reset the file
# write flag so that subsequent manual retrigger will
# generate a new job file.
itask.local_job_file_path = None
itask.state.reset_state(TASK_STATUS_READY)
if itask.state.outputs.has_custom_triggers():
self.suite_db_mgr.put_update_task_outputs(itask)
self.proc_pool.put_command(
SubProcContext(
self.JOBS_SUBMIT,
cmd + job_log_dirs,
stdin_files=stdin_files,
job_log_dirs=job_log_dirs,
**kwargs
),
self._submit_task_jobs_callback, [suite, itasks_batch])
return done_tasks