class SuiteRuntimeServiceClient(object):
"""Client for calling the HTTP(S) API of running suites."""
ANON_AUTH = ('anon', NO_PASSPHRASE, False)
COMPAT_MAP = { # Limited pre-7.5.0 API compat mapping
'clear_broadcast': {0: 'broadcast/clear'},
'expire_broadcast': {0: 'broadcast/expire'},
'get_broadcast': {0: 'broadcast/get'},
'get_info': {0: 'info/'},
'get_suite_state_summary': {0: 'state/get_state_summary'},
'put_broadcast': {0: 'broadcast/put'},
'put_command': {0: 'command/'},
'put_ext_trigger': {0: 'ext-trigger/put'},
'put_messages': {0: 'message/put', 1: 'put_message'},
}
ERROR_NO_HTTPS_SUPPORT = (
"ERROR: server has no HTTPS support," +
" configure your global.rc file to use HTTP : {0}\n"
)
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
MSG_RETRY_INTVL = 5.0
MSG_MAX_TRIES = 7
MSG_TIMEOUT = 30.0
def __init__(
self, suite, owner=None, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False, auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.comms1 = {} # content in primary contact file
self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def _compat(self, name, default=None):
"""Return server function name.
Handle back-compat for pre-7.5.0 if relevant.
"""
# Need to load contact info here to get API version.
self._load_contact_info()
if default is None:
default = name
return self.COMPAT_MAP[name].get(
self.comms1.get(self.srv_files_mgr.KEY_API), default)
def clear_broadcast(self, payload):
"""Clear broadcast runtime task settings."""
return self._call_server(
self._compat('clear_broadcast'), payload=payload)
def expire_broadcast(self, **kwargs):
"""Expire broadcast runtime task settings."""
return self._call_server(self._compat('expire_broadcast'), **kwargs)
def get_broadcast(self, **kwargs):
"""Return broadcast settings."""
return self._call_server(
self._compat('get_broadcast'), method=self.METHOD_GET, **kwargs)
def get_info(self, command, **kwargs):
"""Return suite info."""
return self._call_server(
self._compat('get_info', default='') + command,
method=self.METHOD_GET, **kwargs)
def get_latest_state(self, full_mode=False):
"""Return latest state of the suite (for the GUI)."""
self._load_contact_info()
if self.comms1.get(self.srv_files_mgr.KEY_API) == 0:
# Basic compat for pre-7.5.0 suites
# Full mode only.
# Error content/size not supported.
# Report made-up main loop interval of 5.0 seconds.
return {
'cylc_version': self.get_info('get_cylc_version'),
'full_mode': full_mode,
'summary': self.get_suite_state_summary(),
'ancestors': self.get_info('get_first_parent_ancestors'),
'ancestors_pruned': self.get_info(
'get_first_parent_ancestors', pruned=True),
'descendants': self.get_info('get_first_parent_descendants'),
'err_content': '',
'err_size': 0,
'mean_main_loop_interval': 5.0}
else:
return self._call_server(
'get_latest_state',
method=self.METHOD_GET, full_mode=full_mode)
def get_suite_state_summary(self):
"""Return the global, task, and family summary data structures."""
return utf8_enforce(self._call_server(
self._compat('get_suite_state_summary'), method=self.METHOD_GET))
def identify(self):
"""Return suite identity."""
# Note on compat: Suites on 7.6.0 or above can just call "identify",
# but has compat for "id/identity".
return self._call_server('id/identify', method=self.METHOD_GET)
def put_broadcast(self, payload):
"""Put/set broadcast runtime task settings."""
return self._call_server(
self._compat('put_broadcast'), payload=payload)
def put_command(self, command, **kwargs):
"""Invoke suite command."""
return self._call_server(
self._compat('put_command', default='') + command, **kwargs)
def put_ext_trigger(self, event_message, event_id):
"""Put external trigger."""
return self._call_server(
self._compat('put_ext_trigger'),
event_message=event_message, event_id=event_id)
def put_messages(self, payload):
"""Send task messages to suite server program.
Arguments:
payload (dict):
task_job (str): Task job as "CYCLE/TASK_NAME/SUBMIT_NUM".
event_time (str): Event time as string.
messages (list): List in the form [[severity, message], ...].
"""
retry_intvl = float(self.comms1.get(
self.srv_files_mgr.KEY_TASK_MSG_RETRY_INTVL,
self.MSG_RETRY_INTVL))
max_tries = int(self.comms1.get(
self.srv_files_mgr.KEY_TASK_MSG_MAX_TRIES,
self.MSG_MAX_TRIES))
for i in range(1, max_tries + 1): # 1..max_tries inclusive
orig_timeout = self.timeout
if self.timeout is None:
self.timeout = self.MSG_TIMEOUT
try:
func_name = self._compat('put_messages')
if func_name == 'put_messages':
results = self._call_server(func_name, payload=payload)
elif func_name == 'put_message': # API 1, 7.5.0 compat
cycle, name = payload['task_job'].split('/')[0:2]
for severity, message in payload['messages']:
results.append(self._call_server(
func_name, task_id='%s.%s' % (name, cycle),
severity=severity, message=message))
else: # API 0, pre-7.5.0 compat, priority instead of severity
cycle, name = payload['task_job'].split('/')[0:2]
for severity, message in payload['messages']:
results.append(self._call_server(
func_name, task_id='%s.%s' % (name, cycle),
priority=severity, message=message))
except ClientInfoError:
# Contact info file not found, suite probably not running.
# Don't bother with retry, suite restart will poll any way.
raise
except ClientError as exc:
now = get_current_time_string()
sys.stderr.write(
"%s WARNING - Message send failed, try %s of %s: %s\n" % (
now, i, max_tries, exc))
if i < max_tries:
sys.stderr.write(
" retry in %s seconds, timeout is %s\n" % (
retry_intvl, self.timeout))
sleep(retry_intvl)
# Reset in case contact info or passphrase change
self.comms1 = {}
self.host = None
self.port = None
self.auth = None
else:
if i > 1:
# Continue to write to STDERR, so users can easily see that
# it has recovered from previous failures.
sys.stderr.write(
"%s INFO - Send message: try %s of %s succeeded\n" % (
get_current_time_string(), i, max_tries))
return results
finally:
self.timeout = orig_timeout
def reset(self):
"""Compat method, does nothing."""
pass
def signout(self):
"""Tell server to forget this client."""
return self._call_server('signout')
def _call_server(self, function, method=METHOD, payload=None, **kwargs):
"""Build server URL + call it"""
if self.comms2:
return self._call_server_via_comms2(function, payload, **kwargs)
url = self._call_server_get_url(function, **kwargs)
# Remove proxy settings from environment for now
environ = {}
for key in ("http_proxy", "https_proxy"):
val = os.environ.pop(key, None)
if val:
environ[key] = val
try:
return self.call_server_impl(url, method, payload)
finally:
os.environ.update(environ)
def _call_server_get_url(self, function, **kwargs):
"""Build request URL."""
scheme = self.comms1.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
if scheme is None:
# Use standard setting from global configuration
scheme = glbl_cfg().get(['communication', 'method'])
url = '%s://%s:%s/%s' % (
scheme, self.host, self.port, function)
# If there are any parameters left in the dict after popping,
# append them to the url.
if kwargs:
import urllib
params = urllib.urlencode(kwargs, doseq=True)
url += "?" + params
return url
def call_server_impl(self, url, method, payload):
"""Determine whether to use requests or urllib2 to call suite API."""
impl = self._call_server_impl_urllib2
try:
import requests
except ImportError:
pass
else:
if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]:
impl = self._call_server_impl_requests
try:
return impl(url, method, payload)
except ClientConnectError as exc:
if self.suite is None:
raise
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
self.srv_files_mgr.detect_old_contact_file(
self.suite, (self.host, self.port))
except (AssertionError, SuiteServiceFileError):
raise exc
else:
# self.srv_files_mgr.detect_old_contact_file should delete left
# behind contact file if the old suite process no longer
# exists. Should be safe to report that the suite has stopped.
raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite)
def _call_server_impl_requests(self, url, method, payload):
"""Call server with "requests" library."""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter("ignore", InsecureRequestWarning)
if self.session is None:
self.session = requests.Session()
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
scheme = url.split(':', 1)[0] # Can use urlparse?
username, password, verify = self._get_auth(scheme)
try:
ret = session_method(
url,
json=payload,
verify=verify,
proxies={},
headers=self._get_headers(),
auth=requests.auth.HTTPDigestAuth(username, password),
timeout=self.timeout
)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
if ret.status_code == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
if ret.status_code >= 400:
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectedError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return ret.json()
except ValueError:
return ret.text
def _call_server_impl_urllib2(self, url, method, payload):
"""Call server with "urllib2" library."""
import json
import urllib2
import ssl
unverified_context = getattr(ssl, '_create_unverified_context', None)
if unverified_context is not None:
ssl._create_default_https_context = unverified_context
scheme = url.split(':', 1)[0] # Can use urlparse?
username, password = self._get_auth(scheme)[0:2]
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if payload:
payload = json.dumps(payload)
headers_list.append(('Accept', 'application/json'))
json_headers = {'Content-Type': 'application/json',
'Content-Length': len(payload)}
else:
payload = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, payload, json_headers)
# This is an unpleasant monkey patch, but there isn't an
# alternative. urllib2 uses POST if there is a data payload
# but that is not the correct criterion.
# The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
if "timed out" in str(exc):
raise ClientTimeout(url, exc)
else:
raise ClientConnectError(url, exc)
except Exception as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientError(url, exc)
if response.getcode() == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
response_text = response.read()
if response.getcode() >= 400:
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ClientConnectedError(
url,
"%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return json.loads(response_text)
except ValueError:
return response_text
def _call_server_via_comms2(self, function, payload, **kwargs):
"""Call server via "cylc client --use-ssh".
Call "cylc client --use-ssh" using `subprocess.Popen`. Payload and
arguments of the API method are serialized as JSON and are written to a
temporary file, which is then used as the STDIN of the "cylc client"
command. The external call here should be even safer than a direct
HTTP(S) call, as it can be blocked by SSH before it even gets a chance
to make the subsequent HTTP(S) call.
Arguments:
function (str): name of API method, argument 1 of "cylc client".
payload (str): extra data or information for the API method.
**kwargs (dict): arguments for the API method.
"""
import json
from cylc.remote import remote_cylc_cmd
command = ["client", function, self.suite]
if payload:
kwargs["payload"] = payload
if kwargs:
from tempfile import TemporaryFile
stdin = TemporaryFile()
json.dump(kwargs, stdin)
stdin.seek(0)
else:
# With stdin=None, `remote_cylc_cmd` will:
# * Set stdin to open(os.devnull)
# * Add `-n` to the SSH command
stdin = None
proc = remote_cylc_cmd(
command, self.owner, self.host, capture=True,
ssh_login_shell=(self.comms1.get(
self.srv_files_mgr.KEY_SSH_USE_LOGIN_SHELL
) in ['True', 'true']),
ssh_cylc=(r'%s/bin/cylc' % self.comms1.get(
self.srv_files_mgr.KEY_DIR_ON_SUITE_HOST)
),
stdin=stdin,
)
out = proc.communicate()[0]
return_code = proc.wait()
if return_code:
from pipes import quote
command_str = " ".join(quote(item) for item in command)
raise ClientError(command_str, "return-code=%d" % return_code)
return json.loads(out)
def _get_auth(self, protocol):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = self.ANON_AUTH
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite, self.owner, self.host, content=True)
if protocol == 'https':
verify = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT,
self.suite, self.owner, self.host)
else:
verify = False
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase, verify)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = (
"cylc/%s prog_name/%s uuid/%s" % (
CYLC_VERSION, self.prog_name, self.my_uuid
)
)
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string,
"From": auth_info}
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
try:
# Always trust the values in the contact file otherwise.
self.comms1 = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
# Port inside "try" block, as it needs a type conversion
self.port = int(self.comms1.get(self.srv_files_mgr.KEY_PORT))
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
else:
# Check mismatch suite UUID
env_suite = os.getenv(self.srv_files_mgr.KEY_NAME)
env_uuid = os.getenv(self.srv_files_mgr.KEY_UUID)
if (self.suite and env_suite and env_suite == self.suite and
env_uuid and
env_uuid != self.comms1.get(self.srv_files_mgr.KEY_UUID)):
raise ClientInfoUUIDError(
env_uuid, self.comms1[self.srv_files_mgr.KEY_UUID])
# All good
self.host = self.comms1.get(self.srv_files_mgr.KEY_HOST)
self.owner = self.comms1.get(self.srv_files_mgr.KEY_OWNER)
if self.srv_files_mgr.KEY_API not in self.comms1:
self.comms1[self.srv_files_mgr.KEY_API] = 0 # <=7.5.0 compat
# Indirect comms settings
self.comms2.clear()
try:
self.comms2.update(self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host,
SuiteSrvFilesManager.FILE_BASE_CONTACT2))
except SuiteServiceFileError:
pass
class SuiteRuntimeServiceClient(object):
"""Client for calling the HTTP(S) API of running suites."""
ANON_AUTH = ('anon', NO_PASSPHRASE, False)
ERROR_NO_HTTPS_SUPPORT = (
"ERROR: server has no HTTPS support," +
" configure your global.rc file to use HTTP : {0}\n")
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
def __init__(self,
suite,
owner=None,
host=None,
port=None,
timeout=None,
my_uuid=None,
print_uuid=False,
comms_protocol=None,
auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_protocol = comms_protocol
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
print >> sys.stderr, '%s' % self.my_uuid
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
def clear_broadcast(self, **kwargs):
"""Clear broadcast runtime task settings."""
return self._call_server('clear_broadcast', payload=kwargs)
def expire_broadcast(self, **kwargs):
"""Expire broadcast runtime task settings."""
return self._call_server('expire_broadcast', **kwargs)
def get_broadcast(self, **kwargs):
"""Return broadcast settings."""
return self._call_server('get_broadcast',
method=self.METHOD_GET,
**kwargs)
def get_info(self, command, *args, **kwargs):
"""Return suite info."""
kwargs['method'] = self.METHOD_GET
return self._call_server(command, *args, **kwargs)
def get_latest_state(self, full_mode):
"""Return latest state of the suite (for the GUI)."""
return self._call_server('get_latest_state',
method=self.METHOD_GET,
full_mode=full_mode)
def get_suite_state_summary(self):
"""Return the global, task, and family summary data structures."""
return utf8_enforce(
self._call_server('get_suite_state_summary',
method=self.METHOD_GET))
def get_tasks_by_state(self):
"""Returns a dict containing lists of tasks by state.
Result in the form:
{state: [(most_recent_time_string, task_name, point_string), ...]}
"""
return self._call_server('get_tasks_by_state', method=self.METHOD_GET)
def identify(self):
"""Return suite identity."""
return self._call_server('identify', method=self.METHOD_GET)
def put_broadcast(self, **kwargs):
"""Put/set broadcast runtime task settings."""
return self._call_server('put_broadcast', payload=kwargs)
def put_command(self, command, **kwargs):
"""Invoke suite command."""
return self._call_server(command, **kwargs)
def put_ext_trigger(self, event_message, event_id):
"""Put external trigger."""
return self._call_server('put_ext_trigger',
event_message=event_message,
event_id=event_id)
def put_message(self, task_id, priority, message):
"""Send task message."""
return self._call_server('put_message',
task_id=task_id,
priority=priority,
message=message)
def reset(self, *args, **kwargs):
"""Compat method, does nothing."""
pass
def signout(self, *args, **kwargs):
"""Tell server to forget this client."""
return self._call_server('signout')
def _compile_request(self, func_dict, host, comms_protocol=None):
"""Build request URL."""
payload = func_dict.pop("payload", None)
method = func_dict.pop("method", self.METHOD)
function = func_dict.pop("function", None)
if comms_protocol is None:
# Use standard setting from global configuration
from cylc.cfgspec.globalcfg import GLOBAL_CFG
comms_protocol = GLOBAL_CFG.get(['communication', 'method'])
url = '%s://%s:%s/%s' % (comms_protocol, host, self.port, function)
# If there are any parameters left in the dict after popping,
# append them to the url.
if func_dict:
import urllib
params = urllib.urlencode(func_dict, doseq=True)
url += "?" + params
return (method, url, payload, comms_protocol)
def _call_server(self, *func_dicts, **fargs):
"""func_dict is a dictionary of command names (fnames)
and arguments to that command"""
# Deal with the case of one func_dict/function name passed
# by converting them to the generic case: a dictionary of
# a single function and its function arguments.
if isinstance(func_dicts[0], str):
func_dict = {"function": func_dicts[0]}
func_dict.update(fargs)
else:
func_dict = None
try:
self._load_contact_info()
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
http_request_items = []
try:
# dictionary containing: url, payload, method
http_request_items.append(
self._compile_request(func_dict, self.host,
self.comms_protocol))
except (IndexError, ValueError, AttributeError):
for f_dict in func_dicts:
http_request_items.append(
self._compile_request(f_dict, self.host,
self.comms_protocol))
# Remove proxy settings from environment for now
environ = {}
for key in ("http_proxy", "https_proxy"):
val = os.environ.pop(key, None)
if val:
environ[key] = val
# Returns a list of http returns from the requests
try:
return self.call_server_impl(http_request_items)
finally:
os.environ.update(environ)
def call_server_impl(self, http_request_items):
"""Determine whether to use requests or urllib2 to call suite API."""
method = self._call_server_impl_urllib2
try:
import requests
except ImportError:
pass
else:
if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]:
method = self._call_server_impl_requests
try:
return method(http_request_items)
except ClientConnectError as exc:
if self.suite is None:
raise
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
self.srv_files_mgr.detect_old_contact_file(
self.suite, (self.host, self.port))
except (AssertionError, SuiteServiceFileError):
raise exc
else:
# self.srv_files_mgr.detect_old_contact_file should delete left
# behind contact file if the old suite process no longer
# exists. Should be safe to report that the suite has stopped.
raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite)
def _call_server_impl_requests(self, http_request_items):
"""Call server with "requests" library."""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter("ignore", InsecureRequestWarning)
if not hasattr(self, "session"):
self.session = requests.Session()
http_return_items = []
for method, url, payload, comms_protocol in http_request_items:
if method is None:
method = self.METHOD
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
username, password, verify = self._get_auth(comms_protocol)
try:
ret = session_method(url,
json=payload,
verify=verify,
proxies={},
headers=self._get_headers(),
auth=requests.auth.HTTPDigestAuth(
username, password),
timeout=self.timeout)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
if ret.status_code == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
if ret.status_code >= 400:
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectedError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(self.suite, self.owner,
self.host, self.auth[1])
try:
ret = ret.json()
http_return_items.append(ret)
except ValueError:
ret = ret.text
http_return_items.append(ret)
# Return a single http return or a list of them if multiple
return (http_return_items
if len(http_return_items) > 1 else http_return_items[0])
def _call_server_impl_urllib2(self, http_request_items):
"""Call server with "urllib2" library."""
import json
import urllib2
import ssl
if hasattr(ssl, '_create_unverified_context'):
ssl._create_default_https_context = ssl._create_unverified_context
http_return_items = []
for method, url, payload, comms_protocol in http_request_items:
if method is None:
method = self.METHOD
username, password = self._get_auth(comms_protocol)[0:2]
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if payload:
payload = json.dumps(payload)
headers_list.append(('Accept', 'application/json'))
json_headers = {
'Content-Type': 'application/json',
'Content-Length': len(payload)
}
else:
payload = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, payload, json_headers)
# This is an unpleasant monkey patch, but there isn't an
# alternative. urllib2 uses POST if there is a data payload
# but that is not the correct criterion.
# The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
if "timed out" in str(exc):
raise ClientTimeout(url, exc)
else:
raise ClientConnectError(url, exc)
except Exception as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientError(url, exc)
if response.getcode() == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
response_text = response.read()
if response.getcode() >= 400:
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ClientConnectedError(
url, "%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(self.suite, self.owner,
self.host, self.auth[1])
try:
http_return_items.append(json.loads(response_text))
except ValueError:
http_return_items.append(response_text)
# Return a single http return or a list of them if multiple
return (http_return_items
if len(http_return_items) > 1 else http_return_items[0])
def _get_auth(self, protocol):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = self.ANON_AUTH
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite,
self.owner,
self.host,
content=True)
if protocol == 'https':
verify = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite,
self.owner, self.host)
else:
verify = False
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase, verify)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = ("cylc/%s prog_name/%s uuid/%s" %
(CYLC_VERSION, self.prog_name, self.my_uuid))
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string, "From": auth_info}
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
# Always trust the values in the contact file otherwise.
data = self.srv_files_mgr.load_contact_file(self.suite, self.owner,
self.host)
self.host = data.get(self.srv_files_mgr.KEY_HOST)
self.port = int(data.get(self.srv_files_mgr.KEY_PORT))
self.owner = data.get(self.srv_files_mgr.KEY_OWNER)
self.comms_protocol = data.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
class SuiteRuntimeServiceClient(object):
"""Client for calling the HTTP(S) API of running suites."""
ANON_AUTH = ('anon', NO_PASSPHRASE, False)
COMPAT_MAP = { # Limited pre-7.5.0 API compat mapping
'clear_broadcast': {0: 'broadcast/clear'},
'expire_broadcast': {0: 'broadcast/expire'},
'get_broadcast': {0: 'broadcast/get'},
'get_info': {0: 'info/'},
'get_suite_state_summary': {0: 'state/get_state_summary'},
'get_tasks_by_state': {0: 'state/get_tasks_by_state'},
'put_broadcast': {0: 'broadcast/put'},
'put_command': {0: 'command/'},
'put_ext_trigger': {0: 'ext-trigger/put'},
'put_message': {0: 'message/put'},
}
ERROR_NO_HTTPS_SUPPORT = (
"ERROR: server has no HTTPS support," +
" configure your global.rc file to use HTTP : {0}\n")
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
def __init__(self,
suite,
owner=None,
host=None,
port=None,
timeout=None,
my_uuid=None,
print_uuid=False,
comms_protocol=None,
auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
self.comms_protocol = comms_protocol
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.api = None
def _compat(self, name, default=None):
"""Return server function name.
Handle back-compat for pre-7.5.0 if relevant.
"""
# Need to load contact info here to get API version.
self._load_contact_info()
if default is None:
default = name
return self.COMPAT_MAP[name].get(self.api, default)
def clear_broadcast(self, **kwargs):
"""Clear broadcast runtime task settings."""
return self._call_server(self._compat('clear_broadcast'),
payload=kwargs)
def expire_broadcast(self, **kwargs):
"""Expire broadcast runtime task settings."""
return self._call_server(self._compat('expire_broadcast'), **kwargs)
def get_broadcast(self, **kwargs):
"""Return broadcast settings."""
return self._call_server(self._compat('get_broadcast'),
method=self.METHOD_GET,
**kwargs)
def get_info(self, command, **kwargs):
"""Return suite info."""
return self._call_server(self._compat('get_info', default='') +
command,
method=self.METHOD_GET,
**kwargs)
def get_latest_state(self, full_mode):
"""Return latest state of the suite (for the GUI)."""
self._load_contact_info()
if self.api == 0:
# Basic compat for pre-7.5.0 suites
# Full mode only.
# Error content/size not supported.
# Report made-up main loop interval of 5.0 seconds.
return {
'cylc_version':
self.get_info('get_cylc_version'),
'full_mode':
full_mode,
'summary':
self.get_suite_state_summary(),
'ancestors':
self.get_info('get_first_parent_ancestors'),
'ancestors_pruned':
self.get_info('get_first_parent_ancestors', pruned=True),
'descendants':
self.get_info('get_first_parent_descendants'),
'err_content':
'',
'err_size':
0,
'mean_main_loop_interval':
5.0
}
else:
return self._call_server('get_latest_state',
method=self.METHOD_GET,
full_mode=full_mode)
def get_suite_state_summary(self):
"""Return the global, task, and family summary data structures."""
return utf8_enforce(
self._call_server(self._compat('get_suite_state_summary'),
method=self.METHOD_GET))
def get_tasks_by_state(self):
"""Returns a dict containing lists of tasks by state.
Result in the form:
{state: [(most_recent_time_string, task_name, point_string), ...]}
"""
return self._call_server(self._compat('get_tasks_by_state'),
method=self.METHOD_GET)
def identify(self):
"""Return suite identity."""
# Note on compat: Suites on 7.6.0 or above can just call "identify",
# but has compat for "id/identity".
return self._call_server('id/identify', method=self.METHOD_GET)
def put_broadcast(self, **kwargs):
"""Put/set broadcast runtime task settings."""
return self._call_server(self._compat('put_broadcast'), payload=kwargs)
def put_command(self, command, **kwargs):
"""Invoke suite command."""
return self._call_server(
self._compat('put_command', default='') + command, **kwargs)
def put_ext_trigger(self, event_message, event_id):
"""Put external trigger."""
return self._call_server(self._compat('put_ext_trigger'),
event_message=event_message,
event_id=event_id)
def put_message(self, task_id, severity, message):
"""Send task message."""
func_name = self._compat('put_message')
if func_name == 'put_message':
return self._call_server(func_name,
task_id=task_id,
severity=severity,
message=message)
else: # pre-7.5.0 API compat
return self._call_server(func_name,
task_id=task_id,
priority=severity,
message=message)
def reset(self):
"""Compat method, does nothing."""
pass
def signout(self):
"""Tell server to forget this client."""
return self._call_server('signout')
def _call_server(self, function, method=METHOD, payload=None, **kwargs):
"""Build server URL + call it"""
url = self._call_server_get_url(function, **kwargs)
# Remove proxy settings from environment for now
environ = {}
for key in ("http_proxy", "https_proxy"):
val = os.environ.pop(key, None)
if val:
environ[key] = val
try:
return self.call_server_impl(url, method, payload)
finally:
os.environ.update(environ)
def _call_server_get_url(self, function, **kwargs):
"""Build request URL."""
comms_protocol = self.comms_protocol
if comms_protocol is None:
# Use standard setting from global configuration
comms_protocol = glbl_cfg().get(['communication', 'method'])
url = '%s://%s:%s/%s' % (comms_protocol, self.host, self.port,
function)
# If there are any parameters left in the dict after popping,
# append them to the url.
if kwargs:
import urllib
params = urllib.urlencode(kwargs, doseq=True)
url += "?" + params
return url
def call_server_impl(self, url, method, payload):
"""Determine whether to use requests or urllib2 to call suite API."""
impl = self._call_server_impl_urllib2
try:
import requests
except ImportError:
pass
else:
if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]:
impl = self._call_server_impl_requests
try:
return impl(url, method, payload)
except ClientConnectError as exc:
if self.suite is None:
raise
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
self.srv_files_mgr.detect_old_contact_file(
self.suite, (self.host, self.port))
except (AssertionError, SuiteServiceFileError):
raise exc
else:
# self.srv_files_mgr.detect_old_contact_file should delete left
# behind contact file if the old suite process no longer
# exists. Should be safe to report that the suite has stopped.
raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite)
def _call_server_impl_requests(self, url, method, payload):
"""Call server with "requests" library."""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter("ignore", InsecureRequestWarning)
if self.session is None:
self.session = requests.Session()
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
comms_protocol = url.split(':', 1)[0] # Can use urlparse?
username, password, verify = self._get_auth(comms_protocol)
try:
ret = session_method(url,
json=payload,
verify=verify,
proxies={},
headers=self._get_headers(),
auth=requests.auth.HTTPDigestAuth(
username, password),
timeout=self.timeout)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError("Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
if ret.status_code == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
if ret.status_code >= 400:
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectedError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(self.suite, self.owner,
self.host, self.auth[1])
try:
return ret.json()
except ValueError:
return ret.text
def _call_server_impl_urllib2(self, url, method, payload):
"""Call server with "urllib2" library."""
import json
import urllib2
import ssl
unverified_context = getattr(ssl, '_create_unverified_context', None)
if unverified_context is not None:
ssl._create_default_https_context = unverified_context
comms_protocol = url.split(':', 1)[0] # Can use urlparse?
username, password = self._get_auth(comms_protocol)[0:2]
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if payload:
payload = json.dumps(payload)
headers_list.append(('Accept', 'application/json'))
json_headers = {
'Content-Type': 'application/json',
'Content-Length': len(payload)
}
else:
payload = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, payload, json_headers)
# This is an unpleasant monkey patch, but there isn't an
# alternative. urllib2 uses POST if there is a data payload
# but that is not the correct criterion.
# The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError("Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
if "timed out" in str(exc):
raise ClientTimeout(url, exc)
else:
raise ClientConnectError(url, exc)
except Exception as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientError(url, exc)
if response.getcode() == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
response_text = response.read()
if response.getcode() >= 400:
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ClientConnectedError(
url, "%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(self.suite, self.owner,
self.host, self.auth[1])
try:
return json.loads(response_text)
except ValueError:
return response_text
def _get_auth(self, protocol):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = self.ANON_AUTH
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite,
self.owner,
self.host,
content=True)
if protocol == 'https':
verify = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT, self.suite,
self.owner, self.host)
else:
verify = False
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase, verify)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = ("cylc/%s prog_name/%s uuid/%s" %
(CYLC_VERSION, self.prog_name, self.my_uuid))
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string, "From": auth_info}
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
try:
# Always trust the values in the contact file otherwise.
data = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
# Port inside "try" block, as it needs a type conversion
self.port = int(data.get(self.srv_files_mgr.KEY_PORT))
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
self.host = data.get(self.srv_files_mgr.KEY_HOST)
self.owner = data.get(self.srv_files_mgr.KEY_OWNER)
self.comms_protocol = data.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
try:
self.api = int(data.get(self.srv_files_mgr.KEY_API))
except (TypeError, ValueError):
self.api = 0 # Assume cylc-7.5.0 or before
class SuiteRuntimeServiceClient(object):
"""Client for calling the HTTP(S) API of running suites."""
ANON_AUTH = ('anon', NO_PASSPHRASE, False)
COMPAT_MAP = { # Limited pre-7.5.0 API compat mapping
'clear_broadcast': {0: 'broadcast/clear'},
'expire_broadcast': {0: 'broadcast/expire'},
'get_broadcast': {0: 'broadcast/get'},
'get_info': {0: 'info/'},
'get_suite_state_summary': {0: 'state/get_state_summary'},
'put_broadcast': {0: 'broadcast/put'},
'put_command': {0: 'command/'},
'put_ext_trigger': {0: 'ext-trigger/put'},
'put_messages': {0: 'message/put', 1: 'put_message'},
}
ERROR_NO_HTTPS_SUPPORT = (
"ERROR: server has no HTTPS support," +
" configure your global.rc file to use HTTP : {0}\n"
)
METHOD = 'POST'
METHOD_POST = 'POST'
METHOD_GET = 'GET'
MSG_RETRY_INTVL = 5.0
MSG_MAX_TRIES = 7
MSG_TIMEOUT = 30.0
def __init__(
self, suite, owner=None, host=None, port=None, timeout=None,
my_uuid=None, print_uuid=False, auth=None):
self.suite = suite
if not owner:
owner = get_user()
self.owner = owner
self.host = host
if self.host and self.host.split('.')[0] == 'localhost':
self.host = get_host()
elif self.host and '.' not in self.host: # Not IP and no domain
self.host = get_fqdn_by_host(self.host)
self.port = port
self.srv_files_mgr = SuiteSrvFilesManager()
if timeout is not None:
timeout = float(timeout)
self.timeout = timeout
self.my_uuid = my_uuid or uuid4()
if print_uuid:
sys.stderr.write('%s\n' % self.my_uuid)
self.prog_name = os.path.basename(sys.argv[0])
self.auth = auth
self.session = None
self.comms1 = {} # content in primary contact file
self.comms2 = {} # content in extra contact file, e.g. contact via ssh
def _compat(self, name, default=None):
"""Return server function name.
Handle back-compat for pre-7.5.0 if relevant.
"""
# Need to load contact info here to get API version.
self._load_contact_info()
if default is None:
default = name
return self.COMPAT_MAP[name].get(
self.comms1.get(self.srv_files_mgr.KEY_API), default)
def clear_broadcast(self, payload):
"""Clear broadcast runtime task settings."""
return self._call_server(
self._compat('clear_broadcast'), payload=payload)
def expire_broadcast(self, **kwargs):
"""Expire broadcast runtime task settings."""
return self._call_server(self._compat('expire_broadcast'), **kwargs)
def get_broadcast(self, **kwargs):
"""Return broadcast settings."""
return self._call_server(
self._compat('get_broadcast'), method=self.METHOD_GET, **kwargs)
def get_info(self, command, **kwargs):
"""Return suite info."""
return self._call_server(
self._compat('get_info', default='') + command,
method=self.METHOD_GET, **kwargs)
def get_latest_state(self, full_mode=False):
"""Return latest state of the suite (for the GUI)."""
self._load_contact_info()
if self.comms1.get(self.srv_files_mgr.KEY_API) == 0:
# Basic compat for pre-7.5.0 suites
# Full mode only.
# Error content/size not supported.
# Report made-up main loop interval of 5.0 seconds.
return {
'cylc_version': self.get_info('get_cylc_version'),
'full_mode': full_mode,
'summary': self.get_suite_state_summary(),
'ancestors': self.get_info('get_first_parent_ancestors'),
'ancestors_pruned': self.get_info(
'get_first_parent_ancestors', pruned=True),
'descendants': self.get_info('get_first_parent_descendants'),
'err_content': '',
'err_size': 0,
'mean_main_loop_interval': 5.0}
else:
return self._call_server(
'get_latest_state',
method=self.METHOD_GET, full_mode=full_mode)
def get_suite_state_summary(self):
"""Return the global, task, and family summary data structures."""
return utf8_enforce(self._call_server(
self._compat('get_suite_state_summary'), method=self.METHOD_GET))
def identify(self):
"""Return suite identity."""
# Note on compat: Suites on 7.6.0 or above can just call "identify",
# but has compat for "id/identity".
return self._call_server('id/identify', method=self.METHOD_GET)
def put_broadcast(self, payload):
"""Put/set broadcast runtime task settings."""
return self._call_server(
self._compat('put_broadcast'), payload=payload)
def put_command(self, command, **kwargs):
"""Invoke suite command."""
return self._call_server(
self._compat('put_command', default='') + command, **kwargs)
def put_ext_trigger(self, event_message, event_id):
"""Put external trigger."""
return self._call_server(
self._compat('put_ext_trigger'),
event_message=event_message, event_id=event_id)
def put_messages(self, payload):
"""Send task messages to suite server program.
Arguments:
payload (dict):
task_job (str): Task job as "CYCLE/TASK_NAME/SUBMIT_NUM".
event_time (str): Event time as string.
messages (list): List in the form [[severity, message], ...].
"""
retry_intvl = float(self.comms1.get(
self.srv_files_mgr.KEY_TASK_MSG_RETRY_INTVL,
self.MSG_RETRY_INTVL))
max_tries = int(self.comms1.get(
self.srv_files_mgr.KEY_TASK_MSG_MAX_TRIES,
self.MSG_MAX_TRIES))
for i in range(1, max_tries + 1): # 1..max_tries inclusive
orig_timeout = self.timeout
if self.timeout is None:
self.timeout = self.MSG_TIMEOUT
try:
func_name = self._compat('put_messages')
if func_name == 'put_messages':
results = self._call_server(func_name, payload=payload)
elif func_name == 'put_message': # API 1, 7.5.0 compat
cycle, name = payload['task_job'].split('/')[0:2]
for severity, message in payload['messages']:
results.append(self._call_server(
func_name, task_id='%s.%s' % (name, cycle),
severity=severity, message=message))
else: # API 0, pre-7.5.0 compat, priority instead of severity
cycle, name = payload['task_job'].split('/')[0:2]
for severity, message in payload['messages']:
results.append(self._call_server(
func_name, task_id='%s.%s' % (name, cycle),
priority=severity, message=message))
except ClientInfoError:
# Contact info file not found, suite probably not running.
# Don't bother with retry, suite restart will poll any way.
raise
except ClientError as exc:
now = get_current_time_string()
sys.stderr.write(
"%s WARNING - Message send failed, try %s of %s: %s\n" % (
now, i, max_tries, exc))
if i < max_tries:
sys.stderr.write(
" retry in %s seconds, timeout is %s\n" % (
retry_intvl, self.timeout))
sleep(retry_intvl)
# Reset in case contact info or passphrase change
self.comms1 = {}
self.host = None
self.port = None
self.auth = None
else:
if i > 1:
# Continue to write to STDERR, so users can easily see that
# it has recovered from previous failures.
sys.stderr.write(
"%s INFO - Send message: try %s of %s succeeded\n" % (
get_current_time_string(), i, max_tries))
return results
finally:
self.timeout = orig_timeout
def reset(self):
"""Compat method, does nothing."""
pass
def signout(self):
"""Tell server to forget this client."""
return self._call_server('signout')
def _call_server(self, function, method=METHOD, payload=None, **kwargs):
"""Build server URL + call it"""
if self.comms2:
return self._call_server_via_comms2(function, payload, **kwargs)
url = self._call_server_get_url(function, **kwargs)
# Remove proxy settings from environment for now
environ = {}
for key in ("http_proxy", "https_proxy"):
val = os.environ.pop(key, None)
if val:
environ[key] = val
try:
return self.call_server_impl(url, method, payload)
finally:
os.environ.update(environ)
def _call_server_get_url(self, function, **kwargs):
"""Build request URL."""
scheme = self.comms1.get(self.srv_files_mgr.KEY_COMMS_PROTOCOL)
if scheme is None:
# Use standard setting from global configuration
scheme = glbl_cfg().get(['communication', 'method'])
url = '%s://%s:%s/%s' % (
scheme, self.host, self.port, function)
# If there are any parameters left in the dict after popping,
# append them to the url.
if kwargs:
import urllib
params = urllib.urlencode(kwargs, doseq=True)
url += "?" + params
return url
def call_server_impl(self, url, method, payload):
"""Determine whether to use requests or urllib2 to call suite API."""
impl = self._call_server_impl_urllib2
try:
import requests
except ImportError:
pass
else:
if [int(_) for _ in requests.__version__.split(".")] >= [2, 4, 2]:
impl = self._call_server_impl_requests
try:
return impl(url, method, payload)
except ClientConnectError as exc:
if self.suite is None:
raise
# Cannot connect, perhaps suite is no longer running and is leaving
# behind a contact file?
try:
self.srv_files_mgr.detect_old_contact_file(
self.suite, (self.host, self.port))
except (AssertionError, SuiteServiceFileError):
raise exc
else:
# self.srv_files_mgr.detect_old_contact_file should delete left
# behind contact file if the old suite process no longer
# exists. Should be safe to report that the suite has stopped.
raise ClientConnectError(exc.args[0], exc.STOPPED % self.suite)
def _call_server_impl_requests(self, url, method, payload):
"""Call server with "requests" library."""
import requests
from requests.packages.urllib3.exceptions import InsecureRequestWarning
warnings.simplefilter("ignore", InsecureRequestWarning)
if self.session is None:
self.session = requests.Session()
if method == self.METHOD_POST:
session_method = self.session.post
else:
session_method = self.session.get
scheme = url.split(':', 1)[0] # Can use urlparse?
username, password, verify = self._get_auth(scheme)
try:
ret = session_method(
url,
json=payload,
verify=verify,
proxies={},
headers=self._get_headers(),
auth=requests.auth.HTTPDigestAuth(username, password),
timeout=self.timeout
)
except requests.exceptions.SSLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
except requests.exceptions.Timeout as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientTimeout(url, exc)
except requests.exceptions.RequestException as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectError(url, exc)
if ret.status_code == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
if ret.status_code >= 400:
exception_text = get_exception_from_html(ret.text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(ret.text)
try:
ret.raise_for_status()
except requests.exceptions.HTTPError as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientConnectedError(url, exc)
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return ret.json()
except ValueError:
return ret.text
def _call_server_impl_urllib2(self, url, method, payload):
"""Call server with "urllib2" library."""
import json
import urllib2
import ssl
unverified_context = getattr(ssl, '_create_unverified_context', None)
if unverified_context is not None:
ssl._create_default_https_context = unverified_context
scheme = url.split(':', 1)[0] # Can use urlparse?
username, password = self._get_auth(scheme)[0:2]
auth_manager = urllib2.HTTPPasswordMgrWithDefaultRealm()
auth_manager.add_password(None, url, username, password)
auth = urllib2.HTTPDigestAuthHandler(auth_manager)
opener = urllib2.build_opener(auth, urllib2.HTTPSHandler())
headers_list = self._get_headers().items()
if payload:
payload = json.dumps(payload)
headers_list.append(('Accept', 'application/json'))
json_headers = {'Content-Type': 'application/json',
'Content-Length': len(payload)}
else:
payload = None
json_headers = {'Content-Length': 0}
opener.addheaders = headers_list
req = urllib2.Request(url, payload, json_headers)
# This is an unpleasant monkey patch, but there isn't an
# alternative. urllib2 uses POST if there is a data payload
# but that is not the correct criterion.
# The difference is basically that POST changes
# server state and GET doesn't.
req.get_method = lambda: method
try:
response = opener.open(req, timeout=self.timeout)
except urllib2.URLError as exc:
if "unknown protocol" in str(exc) and url.startswith("https:"):
# Server is using http rather than https, for some reason.
sys.stderr.write(self.ERROR_NO_HTTPS_SUPPORT.format(exc))
raise CylcError(
"Cannot issue commands over unsecured http.")
if cylc.flags.debug:
traceback.print_exc()
if "timed out" in str(exc):
raise ClientTimeout(url, exc)
else:
raise ClientConnectError(url, exc)
except Exception as exc:
if cylc.flags.debug:
traceback.print_exc()
raise ClientError(url, exc)
if response.getcode() == 401:
access_desc = 'private'
if self.auth == self.ANON_AUTH:
access_desc = 'public'
raise ClientDeniedError(url, self.prog_name, access_desc)
response_text = response.read()
if response.getcode() >= 400:
exception_text = get_exception_from_html(response_text)
if exception_text:
sys.stderr.write(exception_text)
else:
sys.stderr.write(response_text)
raise ClientConnectedError(
url,
"%s HTTP return code" % response.getcode())
if self.auth and self.auth[1] != NO_PASSPHRASE:
self.srv_files_mgr.cache_passphrase(
self.suite, self.owner, self.host, self.auth[1])
try:
return json.loads(response_text)
except ValueError:
return response_text
def _call_server_via_comms2(self, function, payload, **kwargs):
"""Call server via "cylc client --use-ssh".
Call "cylc client --use-ssh" using `subprocess.Popen`. Payload and
arguments of the API method are serialized as JSON and are written to a
temporary file, which is then used as the STDIN of the "cylc client"
command. The external call here should be even safer than a direct
HTTP(S) call, as it can be blocked by SSH before it even gets a chance
to make the subsequent HTTP(S) call.
Arguments:
function (str): name of API method, argument 1 of "cylc client".
payload (str): extra data or information for the API method.
**kwargs (dict): arguments for the API method.
"""
import json
from cylc.remote import remote_cylc_cmd
command = ["client", function, self.suite]
if payload:
kwargs["payload"] = payload
if kwargs:
from tempfile import TemporaryFile
stdin = TemporaryFile()
json.dump(kwargs, stdin)
stdin.seek(0)
else:
# With stdin=None, `remote_cylc_cmd` will:
# * Set stdin to open(os.devnull)
# * Add `-n` to the SSH command
stdin = None
proc = remote_cylc_cmd(
command, self.owner, self.host, capture=True,
ssh_login_shell=(self.comms1.get(
self.srv_files_mgr.KEY_SSH_USE_LOGIN_SHELL
) in ['True', 'true']),
ssh_cylc=(r'%s/bin/cylc' % self.comms1.get(
self.srv_files_mgr.KEY_DIR_ON_SUITE_HOST)
),
stdin=stdin,
)
out = proc.communicate()[0]
return_code = proc.wait()
if return_code:
from pipes import quote
command_str = " ".join(quote(item) for item in command)
raise ClientError(command_str, "return-code=%d" % return_code)
return json.loads(out)
def _get_auth(self, protocol):
"""Return a user/password Digest Auth."""
if self.auth is None:
self.auth = self.ANON_AUTH
try:
pphrase = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_PASSPHRASE,
self.suite, self.owner, self.host, content=True)
if protocol == 'https':
verify = self.srv_files_mgr.get_auth_item(
self.srv_files_mgr.FILE_BASE_SSL_CERT,
self.suite, self.owner, self.host)
else:
verify = False
except SuiteServiceFileError:
pass
else:
if pphrase and pphrase != NO_PASSPHRASE:
self.auth = ('cylc', pphrase, verify)
return self.auth
def _get_headers(self):
"""Return HTTP headers identifying the client."""
user_agent_string = (
"cylc/%s prog_name/%s uuid/%s" % (
CYLC_VERSION, self.prog_name, self.my_uuid
)
)
auth_info = "%s@%s" % (get_user(), get_host())
return {"User-Agent": user_agent_string,
"From": auth_info}
def _load_contact_info(self):
"""Obtain suite owner, host, port info.
Determine host and port using content in port file, unless already
specified.
"""
if self.host and self.port:
return
if self.port:
# In case the contact file is corrupted, user can specify the port.
self.host = get_host()
return
try:
# Always trust the values in the contact file otherwise.
self.comms1 = self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host)
# Port inside "try" block, as it needs a type conversion
self.port = int(self.comms1.get(self.srv_files_mgr.KEY_PORT))
except (IOError, ValueError, SuiteServiceFileError):
raise ClientInfoError(self.suite)
else:
# Check mismatch suite UUID
env_suite = os.getenv(self.srv_files_mgr.KEY_NAME)
env_uuid = os.getenv(self.srv_files_mgr.KEY_UUID)
if (self.suite and env_suite and env_suite == self.suite and
env_uuid and
env_uuid != self.comms1.get(self.srv_files_mgr.KEY_UUID)):
raise ClientInfoUUIDError(
env_uuid, self.comms1[self.srv_files_mgr.KEY_UUID])
# All good
self.host = self.comms1.get(self.srv_files_mgr.KEY_HOST)
self.owner = self.comms1.get(self.srv_files_mgr.KEY_OWNER)
if self.srv_files_mgr.KEY_API not in self.comms1:
self.comms1[self.srv_files_mgr.KEY_API] = 0 # <=7.5.0 compat
# Indirect comms settings
self.comms2.clear()
try:
self.comms2.update(self.srv_files_mgr.load_contact_file(
self.suite, self.owner, self.host,
SuiteSrvFilesManager.FILE_BASE_CONTACT2))
except SuiteServiceFileError:
pass
