def _get_conf(self, conf_tree, key, max_args=0):
"""Get a list of cycles from a configuration setting.
key -- An option key in self.SECTION to locate the setting.
max_args -- Maximum number of extra arguments for an item in the list.
The value of the setting is expected to be split by shlex.split into a
list of items. If max_args == 0, an item should be a string
representing a cycle or an cycle offset. If max_args > 0, the cycle
or cycle offset string can, optionally, have arguments. The arguments
are delimited by colons ":".
E.g.:
prune-remote-logs-at=-6h -12h
prune-server-logs-at=-7d
prune-datac-at=-6h:foo/* -12h:'bar/* baz/*' -1d
prune-work-at=-6h:t1*:*.tar -12h:t1*: -12h:*.gz -1d
If max_args == 0, return a list of cycles.
If max_args > 0, return a list of (cycle, [arg, ...])
"""
items_str = conf_tree.node.get_value([self.SECTION, key])
if items_str is None:
return []
try:
items_str = env_var_process(items_str)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
items = []
ref_time_point = os.getenv(
RoseDateTimeOperator.TASK_CYCLE_TIME_MODE_ENV)
date_time_oper = RoseDateTimeOperator(ref_time_point=ref_time_point)
for item_str in shlex.split(items_str):
args = item_str.split(":", max_args)
offset = args.pop(0)
cycle = offset
if ref_time_point:
if os.getenv("ROSE_CYCLING_MODE") == "integer":
try:
cycle = str(int(ref_time_point) +
int(offset.replace("P","")))
except ValueError:
pass
else:
try:
time_point, parse_format = date_time_oper.date_parse()
time_point = date_time_oper.date_shift(time_point, offset)
cycle = date_time_oper.date_format(
parse_format,
time_point)
except ValueError:
pass
if max_args:
items.append((cycle, args))
else:
items.append(cycle)
return items
class SuiteEngineProcessor(object):
"""An abstract suite engine processor."""
TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
SCHEME = None
SCHEME_HANDLER_MANAGER = None
SCHEME_DEFAULT = "cylc" # TODO: site configuration?
TIMEOUT = 5 # seconds
@classmethod
def get_processor(cls, key=None, event_handler=None, popen=None,
fs_util=None, host_selector=None):
"""Return a processor for the suite engine named by "key"."""
if cls.SCHEME_HANDLER_MANAGER is None:
path = os.path.dirname(
os.path.dirname(sys.modules["rose"].__file__))
cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
[path], ns="rose.suite_engine_procs", attrs=["SCHEME"],
can_handle=None, event_handler=event_handler, popen=popen,
fs_util=fs_util, host_selector=host_selector)
if key is None:
key = cls.SCHEME_DEFAULT
return cls.SCHEME_HANDLER_MANAGER.get_handler(key)
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **_):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def check_global_conf_compat(self):
"""Raise exception on suite engine specific incompatibity.
Should raise SuiteEngineGlobalConfCompatError.
"""
raise NotImplementedError()
def check_suite_not_running(self, suite_name):
"""Check that suite is not running.
This method is not implemented. Sub-class should override.
Arguments:
suite_name: name of suite to check.
Raise:
SuiteStillRunningError:
Should raise SuiteStillRunningError if suite is still running.
"""
raise NotImplementedError()
def cmp_suite_conf(
self, suite_name, run_mode, strict_mode=False, debug_mode=False):
"""Compare current suite configuration with that in the previous run.
An implementation of this method should:
* Raise an exception on failure.
* Return True if suite configuration is unmodified c.f. previous run.
* Return False otherwise.
"""
raise NotImplementedError()
def get_suite_contact(self, suite_name):
"""Return suite contact information for a user suite.
Return (dict): suite contact information.
"""
raise NotImplementedError()
def get_suite_dir(self, suite_name, *paths):
"""Return the path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
return os.path.join(os.path.expanduser("~"),
self.get_suite_dir_rel(suite_name, *paths))
def get_suite_dir_rel(self, suite_name, *paths):
"""Return the relative path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
raise NotImplementedError()
def get_suite_log_url(self, user_name, suite_name):
"""Return the "rose bush" URL for a user's suite."""
prefix = "~"
if user_name:
prefix += user_name
suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
suite_d = os.path.expanduser(suite_d)
if not os.path.isdir(suite_d):
raise NoSuiteLogError(user_name, suite_name)
rose_bush_url = None
for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")):
status = {}
for line in open(f_name):
key, value = line.strip().split("=", 1)
status[key] = value
if status.get("host"):
rose_bush_url = "http://" + status["host"]
if status.get("port"):
rose_bush_url += ":" + status["port"]
rose_bush_url += "/"
break
if not rose_bush_url:
conf = ResourceLocator.default().get_conf()
rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
if not rose_bush_url:
return "file://" + suite_d
if not rose_bush_url.endswith("/"):
rose_bush_url += "/"
if not user_name:
user_name = pwd.getpwuid(os.getuid()).pw_name
return rose_bush_url + "/".join(["taskjobs", user_name, suite_name])
def get_task_auth(self, suite_name, task_name):
"""Return [user@]host for a remote task in a suite."""
raise NotImplementedError()
def get_tasks_auths(self, suite_name):
"""Return a list of [user@]host for remote tasks in a suite."""
raise NotImplementedError()
def get_task_props(self, *args, **kwargs):
"""Return a TaskProps object containing suite task's attributes."""
calendar_mode = self.date_time_oper.get_calendar_mode()
try:
return self._get_task_props(*args, **kwargs)
finally:
# Restore calendar mode if changed
self.date_time_oper.set_calendar_mode(calendar_mode)
def _get_task_props(self, *_, **kwargs):
"""Helper for get_task_props."""
tprops = TaskProps()
# If suite_name and task_id are defined, we can assume that the rest
# are defined as well.
if tprops.suite_name is not None and tprops.task_id is not None:
return tprops
tprops = self.get_task_props_from_env()
# Modify calendar mode, if possible
self.date_time_oper.set_calendar_mode(tprops.cycling_mode)
if kwargs["cycle"] is not None:
try:
cycle_offset = get_cycle_offset(kwargs["cycle"])
except ISO8601SyntaxError:
tprops.task_cycle_time = kwargs["cycle"]
else:
if tprops.task_cycle_time:
tprops.task_cycle_time = self._get_offset_cycle_time(
tprops.task_cycle_time, cycle_offset)
else:
tprops.task_cycle_time = kwargs["cycle"]
# Etc directory
if os.path.exists(os.path.join(tprops.suite_dir, "etc")):
tprops.dir_etc = os.path.join(tprops.suite_dir, "etc")
# Data directory: generic, current cycle, and previous cycle
tprops.dir_data = os.path.join(tprops.suite_dir, "share", "data")
if tprops.task_cycle_time is not None:
task_cycle_time = tprops.task_cycle_time
tprops.dir_data_cycle = os.path.join(
tprops.suite_dir, "share", "cycle", str(task_cycle_time))
# Offset cycles
if kwargs.get("cycle_offsets"):
cycle_offset_strings = []
for value in kwargs.get("cycle_offsets"):
cycle_offset_strings.extend(value.split(","))
for value in cycle_offset_strings:
if tprops.cycling_mode == "integer":
cycle_offset = value
if cycle_offset.startswith("__"):
sign_factor = 1
else:
sign_factor = -1
offset_val = cycle_offset.replace("__", "")
cycle_time = str(
int(task_cycle_time) +
sign_factor * int(offset_val.replace("P", "")))
else:
cycle_offset = get_cycle_offset(value)
cycle_time = self._get_offset_cycle_time(
task_cycle_time, cycle_offset)
tprops.dir_data_cycle_offsets[str(cycle_offset)] = (
os.path.join(
tprops.suite_dir, "share", "cycle", cycle_time))
# Create data directories if necessary
# Note: should we create the offsets directories?
for dir_ in (
[tprops.dir_data, tprops.dir_data_cycle] +
tprops.dir_data_cycle_offsets.values()):
if dir_ is None:
continue
if os.path.exists(dir_) and not os.path.isdir(dir_):
self.fs_util.delete(dir_)
self.fs_util.makedirs(dir_)
# Task prefix and suffix
for key, split, index in [("prefix", str.split, 0),
("suffix", str.rsplit, 1)]:
delim = self.TASK_NAME_DELIM[key]
if kwargs.get(key + "_delim"):
delim = kwargs.get(key + "_delim")
if delim in tprops.task_name:
res = split(tprops.task_name, delim, 1)
setattr(tprops, "task_" + key, res[index])
return tprops
def get_task_props_from_env(self):
"""Return a TaskProps object.
This method should not be used directly. Call get_task_props() instead.
"""
raise NotImplementedError()
def get_version(self):
"""Return the version string of the suite engine."""
raise NotImplementedError()
def get_version_env_name(self):
"""Return the name of the suite engine version environment variable."""
return self.SCHEME.upper() + "_VERSION"
def handle_event(self, *args, **kwargs):
"""Call self.event_handler if it is callable."""
if callable(self.event_handler):
return self.event_handler(*args, **kwargs)
def gcontrol(self, suite_name, args=None):
"""Launch control GUI for a suite_name."""
raise NotImplementedError()
def gscan(self, args=None):
"""Launch suites scan GUI."""
raise NotImplementedError()
def is_suite_registered(self, suite_name):
"""Return whether or not a suite is registered."""
raise NotImplementedError()
def job_logs_archive(self, suite_name, items):
"""Archive cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def job_logs_pull_remote(self, suite_name, items,
prune_remote_mode=False, force_mode=False):
"""Pull and housekeep the job logs on remote task hosts.
suite_name -- The name of a suite.
items -- A list of relevant items.
prune_remote_mode -- Remove remote job logs after pulling them.
force_mode -- Force retrieval, even if it may not be necessary.
"""
raise NotImplementedError()
def job_logs_remove_on_server(self, suite_name, items):
"""Remove cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def launch_suite_log_browser(self, user_name, suite_name):
"""Launch web browser to view suite log.
Return URL of suite log on success, None otherwise.
"""
url = self.get_suite_log_url(user_name, suite_name)
browser = webbrowser.get()
browser.open(url, new=True, autoraise=True)
self.handle_event(WebBrowserEvent(browser.name, url))
return url
def parse_job_log_rel_path(self, f_name):
"""Return (cycle, task, submit_num, ext) for a job log rel path."""
raise NotImplementedError()
def run(self, suite_name, host=None, run_mode=None, args=None):
"""Start a suite (in a specified host)."""
raise NotImplementedError()
def shutdown(self, suite_name, args=None, stderr=None, stdout=None):
"""Shut down the suite."""
raise NotImplementedError()
def _get_offset_cycle_time(self, cycle, cycle_offset):
"""Return the actual date time of an BaseCycleOffset against cycle.
cycle: a YYYYmmddHH or ISO 8601 date/time string.
cycle_offset: an instance of BaseCycleOffset.
Return date time in the same format as cycle.
Note: It would be desirable to switch to a ISO 8601 format,
but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
at the moment.
"""
offset_str = str(cycle_offset.to_duration())
try:
time_point, parse_format = self.date_time_oper.date_parse(cycle)
time_point = self.date_time_oper.date_shift(time_point, offset_str)
return self.date_time_oper.date_format(parse_format, time_point)
except OffsetValueError:
raise
except ValueError:
raise CycleTimeError(cycle)
class SuiteEngineProcessor(object):
"""An abstract suite engine processor."""
TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
SCHEME_HANDLER_MANAGER = None
SCHEME_DEFAULT = "cylc" # TODO: site configuration?
TIMEOUT = 5 # seconds
@classmethod
def get_processor(cls, key=None, event_handler=None, popen=None,
fs_util=None, host_selector=None):
"""Return a processor for the suite engine named by "key"."""
if cls.SCHEME_HANDLER_MANAGER is None:
p = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
[p], ns="rose.suite_engine_procs", attrs=["SCHEME"],
can_handle=None, event_handler=event_handler, popen=popen,
fs_util=fs_util, host_selector=host_selector)
if key is None:
key = cls.SCHEME_DEFAULT
return cls.SCHEME_HANDLER_MANAGER.get_handler(key)
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **kwargs):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def check_global_conf_compat(self):
"""Raise exception on suite engine specific incompatibity.
Should raise SuiteEngineGlobalConfCompatError.
"""
raise NotImplementedError()
def check_suite_not_running(self, suite_name, hosts=None):
"""Raise SuiteStillRunningError if suite is still running."""
reasons = self.is_suite_running(
None, suite_name, self.get_suite_hosts(suite_name, hosts))
if reasons:
raise SuiteStillRunningError(suite_name, reasons)
def clean_hook(self, suite_name=None):
"""Run suite engine dependent logic (at end of "rose suite-clean")."""
raise NotImplementedError()
def cmp_suite_conf(self, suite_name, strict_mode=False, debug_mode=False):
"""Compare current suite configuration with that in the previous run.
An implementation of this method should:
* Raise an exception on failure.
* Return True if suite configuration is unmodified c.f. previous run.
* Return False otherwise.
"""
raise NotImplementedError()
def get_cycle_items_globs(self, name, cycle):
"""Return a glob to match named items created for a given cycle.
E.g.:
suite_engine_proc.get_cycle_items_globs("datac", "20130101T0000Z")
# return "share/data/20130101T0000Z"
Return None if named item not supported.
"""
raise NotImplementedError()
def get_suite_dir(self, suite_name, *paths):
"""Return the path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
return os.path.join(os.path.expanduser("~"),
self.get_suite_dir_rel(suite_name, *paths))
def get_suite_dir_rel(self, suite_name, *paths):
"""Return the relative path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
raise NotImplementedError()
def get_suite_hosts(self, suite_name=None, hosts=None):
"""Return names of potential suite hosts.
If "suite_name" is specified, return all possible hosts including what
is written in the "log/rose-suite-run.host" file.
If "suite_name" is not specified and if "[rose-suite-run]hosts" is
defined, split and return the setting.
Otherwise, return ["localhost"].
"""
conf = ResourceLocator.default().get_conf()
hostnames = []
if hosts:
hostnames += hosts
if suite_name:
hostnames.append("localhost")
# Get host name from "log/rose-suite-run.host" file
host_file_path = self.get_suite_dir(
suite_name, "log", "rose-suite-run.host")
try:
for line in open(host_file_path):
hostnames.append(line.strip())
except IOError:
pass
# Scan-able list
suite_hosts = conf.get_value(
["rose-suite-run", "scan-hosts"], "").split()
if suite_hosts:
hostnames += self.host_selector.expand(suite_hosts)[0]
# Normal list
suite_hosts = conf.get_value(["rose-suite-run", "hosts"], "").split()
if suite_hosts:
hostnames += self.host_selector.expand(suite_hosts)[0]
hostnames = list(set(hostnames))
return hostnames
def get_suite_job_events(self, user_name, suite_name, cycles, tasks,
no_statuses, order, limit, offset):
"""Return suite job events.
user -- A string containing a valid user ID
suite -- A string containing a valid suite ID
cycles -- Display only task jobs matching these cycles. A value in the
list can be a cycle, the string "before|after CYCLE", or a
glob to match cycles.
tasks -- Display only jobs for task names matching these names. Values
can be a valid task name or a glob like pattern for matching
valid task names.
no_statues -- Do not display jobs with these statuses. Valid values are
the keys of CylcProcessor.STATUSES.
order -- Order search in a predetermined way. A valid value is one of
the keys in CylcProcessor.ORDERS.
limit -- Limit number of returned entries
offset -- Offset entry number
Return (entries, of_n_entries) where:
entries -- A list of matching entries
of_n_entries -- Total number of entries matching query
Each entry is a dict:
{"cycle": cycle, "name": name, "submit_num": submit_num,
"events": [time_submit, time_init, time_exit],
"status": None|"submit|fail(submit)|init|success|fail|fail(%s)",
"logs": {"script": {"path": path, "path_in_tar", path_in_tar,
"size": size, "mtime": mtime},
"out": {...},
"err": {...},
...}}
"""
raise NotImplementedError()
def get_suite_log_url(self, user_name, suite_name):
"""Return the "rose bush" URL for a user's suite."""
prefix = "~"
if user_name:
prefix += user_name
suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
suite_d = os.path.expanduser(suite_d)
if not os.path.isdir(suite_d):
raise NoSuiteLogError(user_name, suite_name)
rose_bush_status_f_name = os.path.expanduser(
"~/.metomi/rose-bush.status")
rose_bush_url = None
if os.path.isfile(rose_bush_status_f_name):
status = {}
for line in open(rose_bush_status_f_name):
k, v = line.strip().split("=", 1)
status[k] = v
if status.get("host"):
rose_bush_url = "http://" + status["host"]
if status.get("port"):
rose_bush_url += ":" + status["port"]
rose_bush_url += "/"
if not rose_bush_url:
conf = ResourceLocator.default().get_conf()
rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
if not rose_bush_url:
return "file://" + suite_d
if not rose_bush_url.endswith("/"):
rose_bush_url += "/"
if not user_name:
user_name = pwd.getpwuid(os.getuid()).pw_name
return rose_bush_url + "/".join(["list", user_name, suite_name])
def get_suite_logs_info(self, user_name, suite_name):
"""Return the information of the suite logs.
Return a tuple that looks like:
("cylc-run",
{"err": {"path": "log/suite/err", "mtime": mtime, "size": size},
"log": {"path": "log/suite/log", "mtime": mtime, "size": size},
"out": {"path": "log/suite/out", "mtime": mtime, "size": size}})
"""
raise NotImplementedError()
def get_suite_state_summary(self, user_name, suite_name):
"""Return a the state summary of a user's suite.
Return {"last_activity_time": s, "is_running": b, "is_failed": b}
where:
* last_activity_time is a string in %Y-%m-%dT%H:%M:%S format,
the time of the latest activity in the suite
* is_running is a boolean to indicate if the suite is running
* is_failed: a boolean to indicate if any tasks (submit) failed
"""
raise NotImplementedError()
def get_task_auth(self, suite_name, task_name):
"""Return [user@]host for a remote task in a suite."""
raise NotImplementedError()
def get_tasks_auths(self, suite_name):
"""Return a list of [user@]host for remote tasks in a suite."""
raise NotImplementedError()
def get_task_props(self, *args, **kwargs):
"""Return a TaskProps object containing the attributes of a suite task.
"""
t = TaskProps()
# If suite_name and task_id are defined, we can assume that the rest
# are defined as well.
if t.suite_name is not None and t.task_id is not None:
return t
t = self.get_task_props_from_env()
if kwargs["cycle"] is not None:
try:
cycle_offset = get_cycle_offset(kwargs["cycle"])
except Exception:
t.task_cycle_time = kwargs["cycle"]
else:
if t.task_cycle_time:
t.task_cycle_time = self._get_offset_cycle_time(
t.task_cycle_time, cycle_offset)
else:
t.task_cycle_time = kwargs["cycle"]
# Etc directory
if os.path.exists(os.path.join(t.suite_dir, "etc")):
t.dir_etc = os.path.join(t.suite_dir, "etc")
# Data directory: generic, current cycle, and previous cycle
t.dir_data = os.path.join(t.suite_dir, "share", "data")
if t.task_cycle_time is not None:
task_cycle_time = t.task_cycle_time
t.dir_data_cycle = os.path.join(t.dir_data, str(task_cycle_time))
# Offset cycles
if kwargs.get("cycle_offsets"):
cycle_offset_strings = []
for v in kwargs.get("cycle_offsets"):
cycle_offset_strings.extend(v.split(","))
for v in cycle_offset_strings:
if t.cycling_mode == "integer":
cycle_offset = v
if cycle_offset.startswith("__"):
sign_factor = 1
else:
sign_factor = -1
offset_val = cycle_offset.replace("__", "")
cycle_time = str(int(task_cycle_time)
+ sign_factor * int(offset_val.replace("P", "")))
else:
cycle_offset = get_cycle_offset(v)
cycle_time = self._get_offset_cycle_time(
task_cycle_time, cycle_offset)
t.dir_data_cycle_offsets[str(cycle_offset)] = os.path.join(
t.dir_data, cycle_time)
# Create data directories if necessary
# Note: should we create the offsets directories?
for d in ([t.dir_data, t.dir_data_cycle] +
t.dir_data_cycle_offsets.values()):
if d is None:
continue
if os.path.exists(d) and not os.path.isdir(d):
self.fs_util.delete(d)
self.fs_util.makedirs(d)
# Task prefix and suffix
for key, split, index in [("prefix", str.split, 0),
("suffix", str.rsplit, 1)]:
delim = self.TASK_NAME_DELIM[key]
if kwargs.get(key + "_delim"):
delim = kwargs.get(key + "_delim")
if delim in t.task_name:
res = split(t.task_name, delim, 1)
setattr(t, "task_" + key, res[index])
return t
def get_task_props_from_env(self):
"""Return a TaskProps object.
This method should not be used directly. Call get_task_props() instead.
"""
raise NotImplementedError()
def get_version(self):
"""Return the version string of the suite engine."""
raise NotImplementedError()
def get_version_env_name(self):
"""Return the name of the suite engine version environment variable."""
return self.SCHEME.upper() + "_VERSION"
def handle_event(self, *args, **kwargs):
"""Call self.event_handler if it is callable."""
if callable(self.event_handler):
return self.event_handler(*args, **kwargs)
def gcontrol(self, suite_name, host=None, engine_version=None, args=None):
"""Launch control GUI for a suite_name running at a host."""
raise NotImplementedError()
def is_conf(self, path):
"""Return the file type if path is a config of this suite engine."""
raise NotImplementedError()
def is_suite_registered(self, suite_name):
"""Return whether or not a suite is registered."""
raise NotImplementedError()
def is_suite_running(self, user_name, suite_name, hosts=None):
"""Return a list of reasons if it looks like suite is running.
Each reason should be a dict with the following keys:
* "host": the host name where the suite appears to be running on.
* "reason_key": a key, such as "process-id", "port-file", etc.
* "reason_value": the value of the reason, e.g. the process ID, the
path to a port file, etc.
"""
raise NotImplementedError()
def job_logs_archive(self, suite_name, items):
"""Archive cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def job_logs_db_create(self, suite_name, close=False):
"""Create the job logs database."""
raise NotImplementedError()
def job_logs_pull_remote(self, suite_name, items, prune_remote_mode=False):
"""Pull and housekeep the job logs on remote task hosts.
suite_name -- The name of a suite.
items -- A list of relevant items.
prune_remote_mode -- Remove remote job logs after pulling them.
"""
raise NotImplementedError()
def job_logs_remove_on_server(self, suite_name, items):
"""Remove cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def launch_suite_log_browser(self, user_name, suite_name):
"""Launch web browser to view suite log.
Return URL of suite log on success, None otherwise.
"""
url = self.get_suite_log_url(user_name, suite_name)
w = webbrowser.get()
w.open(url, new=True, autoraise=True)
self.handle_event(WebBrowserEvent(w.name, url))
return url
def ping(self, suite_name, hosts=None, timeout=10):
"""Return a list of host names where suite_name is running."""
raise NotImplementedError()
def run(self, suite_name, host=None, host_environ=None, restart_mode=False,
args=None):
"""Start a suite (in a specified host)."""
raise NotImplementedError()
def scan(self, host_names=None, timeout=TIMEOUT):
"""Scan for running suites (in hosts).
Return (suite_scan_results, exceptions) where
suite_scan_results is a list of SuiteScanResult instances and
exceptions is a list of exceptions resulting from any failed scans
Default timeout for SSH and "cylc scan" command is 5 seconds.
"""
raise NotImplementedError()
def shutdown(self, suite_name, host=None, engine_version=None, args=None,
stderr=None, stdout=None):
"""Shut down the suite."""
raise NotImplementedError()
def _get_offset_cycle_time(self, cycle, cycle_offset):
"""Return the actual date time of an BaseCycleOffset against cycle.
cycle: a YYYYmmddHH or ISO 8601 date/time string.
cycle_offset: an instance of BaseCycleOffset.
The returned date time would be an YYYYmmdd[HH[MM]] string.
Note: It would be desirable to switch to a ISO 8601 format,
but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
at the moment.
"""
offset_str = str(cycle_offset.to_duration())
try:
time_point, parse_format = self.date_time_oper.date_parse(cycle)
time_point = self.date_time_oper.date_shift(time_point, offset_str)
return self.date_time_oper.date_format(parse_format, time_point)
except OffsetValueError:
raise
except ValueError:
raise CycleTimeError(cycle)
class SuiteEngineProcessor(object):
"""An abstract suite engine processor."""
TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
SCHEME_HANDLER_MANAGER = None
SCHEME_DEFAULT = "cylc" # TODO: site configuration?
TIMEOUT = 5 # seconds
@classmethod
def get_processor(cls, key=None, event_handler=None, popen=None,
fs_util=None, host_selector=None):
"""Return a processor for the suite engine named by "key"."""
if cls.SCHEME_HANDLER_MANAGER is None:
p = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
[p], ns="rose.suite_engine_procs", attrs=["SCHEME"],
can_handle=None, event_handler=event_handler, popen=popen,
fs_util=fs_util, host_selector=host_selector)
if key is None:
key = cls.SCHEME_DEFAULT
return cls.SCHEME_HANDLER_MANAGER.get_handler(key)
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **kwargs):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def check_global_conf_compat(self):
"""Raise exception on suite engine specific incompatibity.
Should raise SuiteEngineGlobalConfCompatError.
"""
raise NotImplementedError()
def check_suite_not_running(self, suite_name, hosts=None):
"""Raise SuiteStillRunningError if suite is still running."""
reasons = self.is_suite_running(
None, suite_name, self.get_suite_hosts(suite_name, hosts))
if reasons:
raise SuiteStillRunningError(suite_name, reasons)
def clean_hook(self, suite_name=None):
"""Run suite engine dependent logic (at end of "rose suite-clean")."""
raise NotImplementedError()
def cmp_suite_conf(self, suite_name, strict_mode=False, debug_mode=False):
"""Compare current suite configuration with that in the previous run.
An implementation of this method should:
* Raise an exception on failure.
* Return True if suite configuration is unmodified c.f. previous run.
* Return False otherwise.
"""
raise NotImplementedError()
def get_suite_dir(self, suite_name, *paths):
"""Return the path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
return os.path.join(os.path.expanduser("~"),
self.get_suite_dir_rel(suite_name, *paths))
def get_suite_dir_rel(self, suite_name, *paths):
"""Return the relative path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
raise NotImplementedError()
def get_suite_hosts(self, suite_name=None, hosts=None):
"""Return names of potential suite hosts.
If "suite_name" is specified, return all possible hosts including what
is written in the "log/rose-suite-run.host" file.
If "suite_name" is not specified and if "[rose-suite-run]hosts" is
defined, split and return the setting.
Otherwise, return ["localhost"].
"""
conf = ResourceLocator.default().get_conf()
hostnames = []
if hosts:
hostnames += hosts
if suite_name:
hostnames.append("localhost")
# Get host name from "log/rose-suite-run.host" file
host_file_path = self.get_suite_dir(
suite_name, "log", "rose-suite-run.host")
try:
for line in open(host_file_path):
hostnames.append(line.strip())
except IOError:
pass
# Scan-able list
suite_hosts = conf.get_value(
["rose-suite-run", "scan-hosts"], "").split()
if suite_hosts:
hostnames += self.host_selector.expand(suite_hosts)[0]
# Normal list
suite_hosts = conf.get_value(["rose-suite-run", "hosts"], "").split()
if suite_hosts:
hostnames += self.host_selector.expand(suite_hosts)[0]
hostnames = list(set(hostnames))
return hostnames
def get_suite_job_events(self, user_name, suite_name, cycles, tasks,
no_statuses, order, limit, offset):
"""Return suite job events.
user -- A string containing a valid user ID
suite -- A string containing a valid suite ID
cycles -- Display only task jobs matching these cycles. A value in the
list can be a cycle, the string "before|after CYCLE", or a
glob to match cycles.
tasks -- Display only jobs for task names matching these names. Values
can be a valid task name or a glob like pattern for matching
valid task names.
no_statues -- Do not display jobs with these statuses. Valid values are
the keys of CylcProcessor.STATUSES.
order -- Order search in a predetermined way. A valid value is one of
the keys in CylcProcessor.ORDERS.
limit -- Limit number of returned entries
offset -- Offset entry number
Return (entries, of_n_entries) where:
entries -- A list of matching entries
of_n_entries -- Total number of entries matching query
Each entry is a dict:
{"cycle": cycle, "name": name, "submit_num": submit_num,
"events": [time_submit, time_init, time_exit],
"status": None|"submit|fail(submit)|init|success|fail|fail(%s)",
"logs": {"script": {"path": path, "path_in_tar", path_in_tar,
"size": size, "mtime": mtime},
"out": {...},
"err": {...},
...}}
"""
raise NotImplementedError()
def get_suite_log_url(self, user_name, suite_name):
"""Return the "rose bush" URL for a user's suite."""
prefix = "~"
if user_name:
prefix += user_name
suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
suite_d = os.path.expanduser(suite_d)
if not os.path.isdir(suite_d):
raise NoSuiteLogError(user_name, suite_name)
rose_bush_url = None
for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")):
status = {}
for line in open(f_name):
k, v = line.strip().split("=", 1)
status[k] = v
if status.get("host"):
rose_bush_url = "http://" + status["host"]
if status.get("port"):
rose_bush_url += ":" + status["port"]
rose_bush_url += "/"
break
if not rose_bush_url:
conf = ResourceLocator.default().get_conf()
rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
if not rose_bush_url:
return "file://" + suite_d
if not rose_bush_url.endswith("/"):
rose_bush_url += "/"
if not user_name:
user_name = pwd.getpwuid(os.getuid()).pw_name
return rose_bush_url + "/".join(["list", user_name, suite_name])
def get_suite_logs_info(self, user_name, suite_name):
"""Return the information of the suite logs.
Return a tuple that looks like:
("cylc-run",
{"err": {"path": "log/suite/err", "mtime": mtime, "size": size},
"log": {"path": "log/suite/log", "mtime": mtime, "size": size},
"out": {"path": "log/suite/out", "mtime": mtime, "size": size}})
"""
raise NotImplementedError()
def get_suite_state_summary(self, user_name, suite_name):
"""Return a the state summary of a user's suite.
Return {"last_activity_time": s, "is_running": b, "is_failed": b}
where:
* last_activity_time is a string in %Y-%m-%dT%H:%M:%S format,
the time of the latest activity in the suite
* is_running is a boolean to indicate if the suite is running
* is_failed: a boolean to indicate if any tasks (submit) failed
"""
raise NotImplementedError()
def get_task_auth(self, suite_name, task_name):
"""Return [user@]host for a remote task in a suite."""
raise NotImplementedError()
def get_tasks_auths(self, suite_name):
"""Return a list of [user@]host for remote tasks in a suite."""
raise NotImplementedError()
def get_task_props(self, *args, **kwargs):
"""Return a TaskProps object containing the attributes of a suite task.
"""
t = TaskProps()
# If suite_name and task_id are defined, we can assume that the rest
# are defined as well.
if t.suite_name is not None and t.task_id is not None:
return t
t = self.get_task_props_from_env()
if kwargs["cycle"] is not None:
try:
cycle_offset = get_cycle_offset(kwargs["cycle"])
except ISO8601SyntaxError:
t.task_cycle_time = kwargs["cycle"]
else:
if t.task_cycle_time:
t.task_cycle_time = self._get_offset_cycle_time(
t.task_cycle_time, cycle_offset)
else:
t.task_cycle_time = kwargs["cycle"]
# Etc directory
if os.path.exists(os.path.join(t.suite_dir, "etc")):
t.dir_etc = os.path.join(t.suite_dir, "etc")
# Data directory: generic, current cycle, and previous cycle
t.dir_data = os.path.join(t.suite_dir, "share", "data")
if t.task_cycle_time is not None:
task_cycle_time = t.task_cycle_time
t.dir_data_cycle = os.path.join(
t.suite_dir, "share", "cycle", str(task_cycle_time))
# Offset cycles
if kwargs.get("cycle_offsets"):
cycle_offset_strings = []
for v in kwargs.get("cycle_offsets"):
cycle_offset_strings.extend(v.split(","))
for v in cycle_offset_strings:
if t.cycling_mode == "integer":
cycle_offset = v
if cycle_offset.startswith("__"):
sign_factor = 1
else:
sign_factor = -1
offset_val = cycle_offset.replace("__", "")
cycle_time = str(
int(task_cycle_time) +
sign_factor * int(offset_val.replace("P", "")))
else:
cycle_offset = get_cycle_offset(v)
cycle_time = self._get_offset_cycle_time(
task_cycle_time, cycle_offset)
t.dir_data_cycle_offsets[str(cycle_offset)] = os.path.join(
t.suite_dir, "share", "cycle", cycle_time)
# Create data directories if necessary
# Note: should we create the offsets directories?
for d in ([t.dir_data, t.dir_data_cycle] +
t.dir_data_cycle_offsets.values()):
if d is None:
continue
if os.path.exists(d) and not os.path.isdir(d):
self.fs_util.delete(d)
self.fs_util.makedirs(d)
# Task prefix and suffix
for key, split, index in [("prefix", str.split, 0),
("suffix", str.rsplit, 1)]:
delim = self.TASK_NAME_DELIM[key]
if kwargs.get(key + "_delim"):
delim = kwargs.get(key + "_delim")
if delim in t.task_name:
res = split(t.task_name, delim, 1)
setattr(t, "task_" + key, res[index])
return t
def get_task_props_from_env(self):
"""Return a TaskProps object.
This method should not be used directly. Call get_task_props() instead.
"""
raise NotImplementedError()
def get_version(self):
"""Return the version string of the suite engine."""
raise NotImplementedError()
def get_version_env_name(self):
"""Return the name of the suite engine version environment variable."""
return self.SCHEME.upper() + "_VERSION"
def handle_event(self, *args, **kwargs):
"""Call self.event_handler if it is callable."""
if callable(self.event_handler):
return self.event_handler(*args, **kwargs)
def gcontrol(self, suite_name, host=None, engine_version=None, args=None):
"""Launch control GUI for a suite_name running at a host."""
raise NotImplementedError()
def is_conf(self, path):
"""Return the file type if path is a config of this suite engine."""
raise NotImplementedError()
def is_suite_registered(self, suite_name):
"""Return whether or not a suite is registered."""
raise NotImplementedError()
def is_suite_running(self, user_name, suite_name, hosts=None):
"""Return a list of reasons if it looks like suite is running.
Each reason should be a dict with the following keys:
* "host": the host name where the suite appears to be running on.
* "reason_key": a key, such as "process-id", "port-file", etc.
* "reason_value": the value of the reason, e.g. the process ID, the
path to a port file, etc.
"""
raise NotImplementedError()
def job_logs_archive(self, suite_name, items):
"""Archive cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def job_logs_db_create(self, suite_name, close=False):
"""Create the job logs database."""
raise NotImplementedError()
def job_logs_pull_remote(self, suite_name, items,
prune_remote_mode=False, force_mode=False):
"""Pull and housekeep the job logs on remote task hosts.
suite_name -- The name of a suite.
items -- A list of relevant items.
prune_remote_mode -- Remove remote job logs after pulling them.
force_mode -- Force retrieval, even if it may not be necessary.
"""
raise NotImplementedError()
def job_logs_remove_on_server(self, suite_name, items):
"""Remove cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def launch_suite_log_browser(self, user_name, suite_name):
"""Launch web browser to view suite log.
Return URL of suite log on success, None otherwise.
"""
url = self.get_suite_log_url(user_name, suite_name)
w = webbrowser.get()
w.open(url, new=True, autoraise=True)
self.handle_event(WebBrowserEvent(w.name, url))
return url
def parse_job_log_rel_path(self, f_name):
"""Return (cycle, task, submit_num, ext) for a job log rel path."""
raise NotImplementedError()
def ping(self, suite_name, hosts=None, timeout=10):
"""Return a list of host names where suite_name is running."""
raise NotImplementedError()
def run(self, suite_name, host=None, host_environ=None, restart_mode=False,
args=None):
"""Start a suite (in a specified host)."""
raise NotImplementedError()
def scan(self, host_names=None, timeout=TIMEOUT):
"""Scan for running suites (in hosts).
Return (suite_scan_results, exceptions) where
suite_scan_results is a list of SuiteScanResult instances and
exceptions is a list of exceptions resulting from any failed scans
Default timeout for SSH and "cylc scan" command is 5 seconds.
"""
raise NotImplementedError()
def shutdown(self, suite_name, host=None, engine_version=None, args=None,
stderr=None, stdout=None):
"""Shut down the suite."""
raise NotImplementedError()
def _get_offset_cycle_time(self, cycle, cycle_offset):
"""Return the actual date time of an BaseCycleOffset against cycle.
cycle: a YYYYmmddHH or ISO 8601 date/time string.
cycle_offset: an instance of BaseCycleOffset.
The returned date time would be an YYYYmmdd[HH[MM]] string.
Note: It would be desirable to switch to a ISO 8601 format,
but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
at the moment.
"""
offset_str = str(cycle_offset.to_duration())
try:
time_point, parse_format = self.date_time_oper.date_parse(cycle)
time_point = self.date_time_oper.date_shift(time_point, offset_str)
return self.date_time_oper.date_format(parse_format, time_point)
except OffsetValueError:
raise
except ValueError:
raise CycleTimeError(cycle)
class SuiteEngineProcessor(object):
"""An abstract suite engine processor."""
TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
SCHEME = None
SCHEME_HANDLER_MANAGER = None
SCHEME_DEFAULT = "cylc" # TODO: site configuration?
TIMEOUT = 5 # seconds
@classmethod
def get_processor(cls, key=None, event_handler=None, popen=None,
fs_util=None, host_selector=None):
"""Return a processor for the suite engine named by "key"."""
if cls.SCHEME_HANDLER_MANAGER is None:
path = os.path.dirname(
os.path.dirname(sys.modules["rose"].__file__))
cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
[path], ns="rose.suite_engine_procs", attrs=["SCHEME"],
can_handle=None, event_handler=event_handler, popen=popen,
fs_util=fs_util, host_selector=host_selector)
if key is None:
key = cls.SCHEME_DEFAULT
return cls.SCHEME_HANDLER_MANAGER.get_handler(key)
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **_):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def check_global_conf_compat(self):
"""Raise exception on suite engine specific incompatibity.
Should raise SuiteEngineGlobalConfCompatError.
"""
raise NotImplementedError()
def check_suite_not_running(self, suite_name, hosts=None):
"""Raise SuiteStillRunningError if suite is still running."""
host_names = self.get_suite_run_hosts(None, suite_name, hosts)
if host_names:
raise SuiteStillRunningError(suite_name, host_names)
def cmp_suite_conf(
self, suite_name, run_mode, strict_mode=False, debug_mode=False):
"""Compare current suite configuration with that in the previous run.
An implementation of this method should:
* Raise an exception on failure.
* Return True if suite configuration is unmodified c.f. previous run.
* Return False otherwise.
"""
raise NotImplementedError()
def get_suite_dir(self, suite_name, *paths):
"""Return the path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
return os.path.join(os.path.expanduser("~"),
self.get_suite_dir_rel(suite_name, *paths))
def get_suite_dir_rel(self, suite_name, *paths):
"""Return the relative path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
raise NotImplementedError()
def get_suite_run_hosts(self, user_name, suite_name, host_names=None):
"""Return host(s) where suite_name is running."""
raise NotImplementedError()
def get_suite_log_url(self, user_name, suite_name):
"""Return the "rose bush" URL for a user's suite."""
prefix = "~"
if user_name:
prefix += user_name
suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
suite_d = os.path.expanduser(suite_d)
if not os.path.isdir(suite_d):
raise NoSuiteLogError(user_name, suite_name)
rose_bush_url = None
for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")):
status = {}
for line in open(f_name):
key, value = line.strip().split("=", 1)
status[key] = value
if status.get("host"):
rose_bush_url = "http://" + status["host"]
if status.get("port"):
rose_bush_url += ":" + status["port"]
rose_bush_url += "/"
break
if not rose_bush_url:
conf = ResourceLocator.default().get_conf()
rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
if not rose_bush_url:
return "file://" + suite_d
if not rose_bush_url.endswith("/"):
rose_bush_url += "/"
if not user_name:
user_name = pwd.getpwuid(os.getuid()).pw_name
return rose_bush_url + "/".join(["taskjobs", user_name, suite_name])
def get_task_auth(self, suite_name, task_name):
"""Return [user@]host for a remote task in a suite."""
raise NotImplementedError()
def get_tasks_auths(self, suite_name):
"""Return a list of [user@]host for remote tasks in a suite."""
raise NotImplementedError()
def get_task_props(self, *args, **kwargs):
"""Return a TaskProps object containing suite task's attributes."""
calendar_mode = self.date_time_oper.get_calendar_mode()
try:
return self._get_task_props(*args, **kwargs)
finally:
# Restore calendar mode if changed
self.date_time_oper.set_calendar_mode(calendar_mode)
def _get_task_props(self, *_, **kwargs):
"""Helper for get_task_props."""
tprops = TaskProps()
# If suite_name and task_id are defined, we can assume that the rest
# are defined as well.
if tprops.suite_name is not None and tprops.task_id is not None:
return tprops
tprops = self.get_task_props_from_env()
# Modify calendar mode, if possible
self.date_time_oper.set_calendar_mode(tprops.cycling_mode)
if kwargs["cycle"] is not None:
try:
cycle_offset = get_cycle_offset(kwargs["cycle"])
except ISO8601SyntaxError:
tprops.task_cycle_time = kwargs["cycle"]
else:
if tprops.task_cycle_time:
tprops.task_cycle_time = self._get_offset_cycle_time(
tprops.task_cycle_time, cycle_offset)
else:
tprops.task_cycle_time = kwargs["cycle"]
# Etc directory
if os.path.exists(os.path.join(tprops.suite_dir, "etc")):
tprops.dir_etc = os.path.join(tprops.suite_dir, "etc")
# Data directory: generic, current cycle, and previous cycle
tprops.dir_data = os.path.join(tprops.suite_dir, "share", "data")
if tprops.task_cycle_time is not None:
task_cycle_time = tprops.task_cycle_time
tprops.dir_data_cycle = os.path.join(
tprops.suite_dir, "share", "cycle", str(task_cycle_time))
# Offset cycles
if kwargs.get("cycle_offsets"):
cycle_offset_strings = []
for value in kwargs.get("cycle_offsets"):
cycle_offset_strings.extend(value.split(","))
for value in cycle_offset_strings:
if tprops.cycling_mode == "integer":
cycle_offset = value
if cycle_offset.startswith("__"):
sign_factor = 1
else:
sign_factor = -1
offset_val = cycle_offset.replace("__", "")
cycle_time = str(
int(task_cycle_time) +
sign_factor * int(offset_val.replace("P", "")))
else:
cycle_offset = get_cycle_offset(value)
cycle_time = self._get_offset_cycle_time(
task_cycle_time, cycle_offset)
tprops.dir_data_cycle_offsets[str(cycle_offset)] = (
os.path.join(
tprops.suite_dir, "share", "cycle", cycle_time))
# Create data directories if necessary
# Note: should we create the offsets directories?
for dir_ in (
[tprops.dir_data, tprops.dir_data_cycle] +
tprops.dir_data_cycle_offsets.values()):
if dir_ is None:
continue
if os.path.exists(dir_) and not os.path.isdir(dir_):
self.fs_util.delete(dir_)
self.fs_util.makedirs(dir_)
# Task prefix and suffix
for key, split, index in [("prefix", str.split, 0),
("suffix", str.rsplit, 1)]:
delim = self.TASK_NAME_DELIM[key]
if kwargs.get(key + "_delim"):
delim = kwargs.get(key + "_delim")
if delim in tprops.task_name:
res = split(tprops.task_name, delim, 1)
setattr(tprops, "task_" + key, res[index])
return tprops
def get_task_props_from_env(self):
"""Return a TaskProps object.
This method should not be used directly. Call get_task_props() instead.
"""
raise NotImplementedError()
def get_version(self):
"""Return the version string of the suite engine."""
raise NotImplementedError()
def get_version_env_name(self):
"""Return the name of the suite engine version environment variable."""
return self.SCHEME.upper() + "_VERSION"
def handle_event(self, *args, **kwargs):
"""Call self.event_handler if it is callable."""
if callable(self.event_handler):
return self.event_handler(*args, **kwargs)
def gcontrol(self, suite_name, host=None, engine_version=None, args=None):
"""Launch control GUI for a suite_name running at a host."""
raise NotImplementedError()
def is_suite_registered(self, suite_name):
"""Return whether or not a suite is registered."""
raise NotImplementedError()
def job_logs_archive(self, suite_name, items):
"""Archive cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def job_logs_pull_remote(self, suite_name, items,
prune_remote_mode=False, force_mode=False):
"""Pull and housekeep the job logs on remote task hosts.
suite_name -- The name of a suite.
items -- A list of relevant items.
prune_remote_mode -- Remove remote job logs after pulling them.
force_mode -- Force retrieval, even if it may not be necessary.
"""
raise NotImplementedError()
def job_logs_remove_on_server(self, suite_name, items):
"""Remove cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def launch_suite_log_browser(self, user_name, suite_name):
"""Launch web browser to view suite log.
Return URL of suite log on success, None otherwise.
"""
url = self.get_suite_log_url(user_name, suite_name)
browser = webbrowser.get()
browser.open(url, new=True, autoraise=True)
self.handle_event(WebBrowserEvent(browser.name, url))
return url
def parse_job_log_rel_path(self, f_name):
"""Return (cycle, task, submit_num, ext) for a job log rel path."""
raise NotImplementedError()
def run(self, suite_name, host=None, host_environ=None, restart_mode=False,
args=None):
"""Start a suite (in a specified host)."""
raise NotImplementedError()
def scan(self, host_names=None, timeout=TIMEOUT):
"""Scan for running suites (in hosts).
Return (suite_scan_results, exceptions) where
suite_scan_results is a list of SuiteScanResult instances and
exceptions is a list of exceptions resulting from any failed scans
Default timeout for SSH and "cylc scan" command is 5 seconds.
"""
raise NotImplementedError()
def shutdown(self, suite_name, host=None, engine_version=None, args=None,
stderr=None, stdout=None):
"""Shut down the suite."""
raise NotImplementedError()
def _get_offset_cycle_time(self, cycle, cycle_offset):
"""Return the actual date time of an BaseCycleOffset against cycle.
cycle: a YYYYmmddHH or ISO 8601 date/time string.
cycle_offset: an instance of BaseCycleOffset.
Return date time in the same format as cycle.
Note: It would be desirable to switch to a ISO 8601 format,
but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
at the moment.
"""
offset_str = str(cycle_offset.to_duration())
try:
time_point, parse_format = self.date_time_oper.date_parse(cycle)
time_point = self.date_time_oper.date_shift(time_point, offset_str)
return self.date_time_oper.date_format(parse_format, time_point)
except OffsetValueError:
raise
except ValueError:
raise CycleTimeError(cycle)
def _get_conf(self, conf_tree, key, max_args=0):
"""Get a list of cycles from a configuration setting.
key -- An option key in self.SECTION to locate the setting.
max_args -- Maximum number of extra arguments for an item in the list.
The value of the setting is expected to be split by shlex.split into a
list of items. If max_args == 0, an item should be a string
representing a cycle or an cycle offset. If max_args > 0, the cycle
or cycle offset string can, optionally, have arguments. The arguments
are delimited by colons ":".
E.g.:
prune-remote-logs-at=-PT6H -PT12H
prune-server-logs-at=-P7D
prune-datac-at=-PT6H:foo/* -PT12H:'bar/* baz/*' -P1D
prune-work-at=-PT6H:t1*:*.tar -PT12H:t1*: -PT12H:*.gz -P1D
If max_args == 0, return a list of cycles.
If max_args > 0, return a list of (cycle, [arg, ...])
"""
items_str = conf_tree.node.get_value([self.SECTION, key])
if items_str is None:
return []
try:
items_str = env_var_process(items_str)
except UnboundEnvironmentVariableError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
items = []
ref_point_str = os.getenv(
RoseDateTimeOperator.TASK_CYCLE_TIME_MODE_ENV)
try:
date_time_oper = None
ref_time_point = None
parse_format = None
for item_str in shlex.split(items_str):
args = item_str.split(":", max_args)
when = args.pop(0)
cycle = when
if (ref_point_str and
os.getenv("ROSE_CYCLING_MODE") == "integer"):
# Integer cycling
if "P" in when: # "when" is an offset
cycle = str(int(ref_point_str) +
int(when.replace("P", "")))
else: # "when" is a cycle point
cycle = str(when)
elif ref_point_str:
# Date-time cycling
if date_time_oper is None:
date_time_oper = RoseDateTimeOperator(
ref_time_point=ref_point_str)
ref_time_point, ref_fmt = date_time_oper.date_parse()
try:
time_point = date_time_oper.date_parse(when)[0]
except ValueError:
time_point = date_time_oper.date_shift(
ref_time_point, when)
cycle = date_time_oper.date_format(ref_fmt, time_point)
if max_args:
items.append((cycle, args))
else:
items.append(cycle)
except ValueError as exc:
raise ConfigValueError([self.SECTION, key], items_str, exc)
return items