class SuiteEngineProcessor:
"""An abstract suite engine processor."""
TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
SCHEME = None
SCHEME_HANDLER_MANAGER = None
SCHEME_DEFAULT = "cylc" # TODO: site configuration?
TIMEOUT = 5 # seconds
@classmethod
def get_processor(cls,
key=None,
event_handler=None,
popen=None,
fs_util=None,
host_selector=None):
"""Return a processor for the suite engine named by "key"."""
if cls.SCHEME_HANDLER_MANAGER is None:
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rose"].__file__))
cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
[path],
ns="rose.suite_engine_procs",
attrs=["SCHEME"],
can_handle=None,
event_handler=event_handler,
popen=popen,
fs_util=fs_util,
host_selector=host_selector)
if key is None:
key = cls.SCHEME_DEFAULT
x = cls.SCHEME_HANDLER_MANAGER.get_handler(key)
return x
def __init__(self,
event_handler=None,
popen=None,
fs_util=None,
host_selector=None,
**_):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def get_suite_dir(self, suite_name, *paths):
"""Return the path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
return os.path.join(os.path.expanduser("~"),
self.get_suite_dir_rel(suite_name, *paths))
def get_suite_dir_rel(self, suite_name, *paths):
"""Return the relative path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
raise NotImplementedError()
def get_task_auth(self, suite_name, task_name):
"""Return [user@]host for a remote task in a suite."""
raise NotImplementedError()
def get_task_props(self, *args, **kwargs):
"""Return a TaskProps object containing suite task's attributes."""
calendar_mode = self.date_time_oper.get_calendar_mode()
try:
return self._get_task_props(*args, **kwargs)
finally:
# Restore calendar mode if changed
self.date_time_oper.set_calendar_mode(calendar_mode)
def _get_task_props(self, *_, **kwargs):
"""Helper for get_task_props."""
tprops = TaskProps()
# If suite_name and task_id are defined, we can assume that the rest
# are defined as well.
if tprops.suite_name is not None and tprops.task_id is not None:
return tprops
tprops = self.get_task_props_from_env()
# Modify calendar mode, if possible
self.date_time_oper.set_calendar_mode(tprops.cycling_mode)
if kwargs["cycle"] is not None:
try:
cycle_offset = get_cycle_offset(kwargs["cycle"])
except ISO8601SyntaxError:
tprops.task_cycle_time = kwargs["cycle"]
else:
if tprops.task_cycle_time:
tprops.task_cycle_time = self._get_offset_cycle_time(
tprops.task_cycle_time, cycle_offset)
else:
tprops.task_cycle_time = kwargs["cycle"]
# Etc directory
if os.path.exists(os.path.join(tprops.suite_dir, "etc")):
tprops.dir_etc = os.path.join(tprops.suite_dir, "etc")
# Data directory: generic, current cycle, and previous cycle
tprops.dir_data = os.path.join(tprops.suite_dir, "share", "data")
if tprops.task_cycle_time is not None:
task_cycle_time = tprops.task_cycle_time
tprops.dir_data_cycle = os.path.join(tprops.suite_dir, "share",
"cycle", str(task_cycle_time))
# Offset cycles
if kwargs.get("cycle_offsets"):
cycle_offset_strings = []
for value in kwargs.get("cycle_offsets"):
cycle_offset_strings.extend(value.split(","))
for value in cycle_offset_strings:
if tprops.cycling_mode == "integer":
cycle_offset = value
if cycle_offset.startswith("__"):
sign_factor = 1
else:
sign_factor = -1
offset_val = cycle_offset.replace("__", "")
cycle_time = str(
int(task_cycle_time) +
sign_factor * int(offset_val.replace("P", "")))
else:
cycle_offset = get_cycle_offset(value)
cycle_time = self._get_offset_cycle_time(
task_cycle_time, cycle_offset)
tprops.dir_data_cycle_offsets[str(cycle_offset)] = (
os.path.join(tprops.suite_dir, "share", "cycle",
cycle_time))
# Create data directories if necessary
# Note: should we create the offsets directories?
for dir_ in ([tprops.dir_data, tprops.dir_data_cycle] +
list(tprops.dir_data_cycle_offsets.values())):
if dir_ is None:
continue
if os.path.exists(dir_) and not os.path.isdir(dir_):
self.fs_util.delete(dir_)
self.fs_util.makedirs(dir_)
# Task prefix and suffix
for key, split, index in [("prefix", str.split, 0),
("suffix", str.rsplit, 1)]:
delim = self.TASK_NAME_DELIM[key]
if kwargs.get(key + "_delim"):
delim = kwargs.get(key + "_delim")
if delim in tprops.task_name:
res = split(tprops.task_name, delim, 1)
setattr(tprops, "task_" + key, res[index])
return tprops
def get_task_props_from_env(self):
"""Return a TaskProps object.
This method should not be used directly. Call get_task_props() instead.
"""
raise NotImplementedError()
def handle_event(self, *args, **kwargs):
"""Call self.event_handler if it is callable."""
if callable(self.event_handler):
return self.event_handler(*args, **kwargs)
def job_logs_archive(self, suite_name, items):
"""Archive cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def job_logs_pull_remote(self,
suite_name,
items,
prune_remote_mode=False,
force_mode=False):
"""Pull and housekeep the job logs on remote task hosts.
suite_name -- The name of a suite.
items -- A list of relevant items.
prune_remote_mode -- Remove remote job logs after pulling them.
force_mode -- Force retrieval, even if it may not be necessary.
"""
raise NotImplementedError()
def job_logs_remove_on_server(self, suite_name, items):
"""Remove cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def parse_job_log_rel_path(self, f_name):
"""Return (cycle, task, submit_num, ext) for a job log rel path."""
raise NotImplementedError()
def _get_offset_cycle_time(self, cycle, cycle_offset):
"""Return the actual date time of an BaseCycleOffset against cycle.
cycle: a YYYYmmddHH or ISO 8601 date/time string.
cycle_offset: an instance of BaseCycleOffset.
Return date time in the same format as cycle.
Note: It would be desirable to switch to a ISO 8601 format,
but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
at the moment.
"""
offset_str = str(cycle_offset.to_duration())
try:
time_point, parse_format = self.date_time_oper.date_parse(cycle)
time_point = self.date_time_oper.date_shift(time_point, offset_str)
return self.date_time_oper.date_format(parse_format, time_point)
except OffsetValueError:
raise
except ValueError:
raise CycleTimeError(cycle)
class SuiteEngineProcessor(object):
"""An abstract suite engine processor."""
TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
SCHEME = None
SCHEME_HANDLER_MANAGER = None
SCHEME_DEFAULT = "cylc" # TODO: site configuration?
TIMEOUT = 5 # seconds
@classmethod
def get_processor(cls, key=None, event_handler=None, popen=None,
fs_util=None, host_selector=None):
"""Return a processor for the suite engine named by "key"."""
if cls.SCHEME_HANDLER_MANAGER is None:
path = os.path.dirname(
os.path.dirname(sys.modules["metomi.rose"].__file__))
cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
[path], ns="rose.suite_engine_procs", attrs=["SCHEME"],
can_handle=None, event_handler=event_handler, popen=popen,
fs_util=fs_util, host_selector=host_selector)
if key is None:
key = cls.SCHEME_DEFAULT
x = cls.SCHEME_HANDLER_MANAGER.get_handler(key)
return x
def __init__(self, event_handler=None, popen=None, fs_util=None,
host_selector=None, **_):
self.event_handler = event_handler
if popen is None:
popen = RosePopener(event_handler)
self.popen = popen
if fs_util is None:
fs_util = FileSystemUtil(event_handler)
self.fs_util = fs_util
if host_selector is None:
host_selector = HostSelector(event_handler, popen)
self.host_selector = host_selector
self.date_time_oper = RoseDateTimeOperator()
def check_global_conf_compat(self):
"""Raise exception on suite engine specific incompatibity.
Should raise SuiteEngineGlobalConfCompatError.
"""
raise NotImplementedError()
def check_suite_not_running(self, suite_name):
"""Check that suite is not running.
This method is not implemented. Sub-class should override.
Arguments:
suite_name: name of suite to check.
Raise:
SuiteStillRunningError:
Should raise SuiteStillRunningError if suite is still running.
"""
raise NotImplementedError()
def cmp_suite_conf(
self, suite_name, run_mode, strict_mode=False, debug_mode=False):
"""Compare current suite configuration with that in the previous run.
An implementation of this method should:
* Raise an exception on failure.
* Return True if suite configuration is unmodified c.f. previous run.
* Return False otherwise.
"""
raise NotImplementedError()
def get_suite_contact(self, suite_name):
"""Return suite contact information for a user suite.
Return (dict): suite contact information.
"""
raise NotImplementedError()
def get_suite_dir(self, suite_name, *paths):
"""Return the path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
return os.path.join(os.path.expanduser("~"),
self.get_suite_dir_rel(suite_name, *paths))
def get_suite_dir_rel(self, suite_name, *paths):
"""Return the relative path to the suite running directory.
paths -- if specified, are added to the end of the path.
"""
raise NotImplementedError()
def get_suite_log_url(self, user_name, suite_name):
"""Return the "rose bush" URL for a user's suite."""
prefix = "~"
if user_name:
prefix += user_name
suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
suite_d = os.path.expanduser(suite_d)
if not os.path.isdir(suite_d):
raise NoSuiteLogError(user_name, suite_name)
rose_bush_url = None
for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")):
status = {}
for line in open(f_name):
key, value = line.strip().split("=", 1)
status[key] = value
if status.get("host"):
rose_bush_url = "http://" + status["host"]
if status.get("port"):
rose_bush_url += ":" + status["port"]
rose_bush_url += "/"
break
if not rose_bush_url:
conf = ResourceLocator.default().get_conf()
rose_bush_url = conf.get_value(
["rose-suite-log", "rose-bush"])
if not rose_bush_url:
return "file://" + suite_d
if not rose_bush_url.endswith("/"):
rose_bush_url += "/"
if not user_name:
user_name = pwd.getpwuid(os.getuid()).pw_name
return rose_bush_url + "/".join(
["taskjobs", user_name, suite_name])
def get_task_auth(self, suite_name, task_name):
"""Return [user@]host for a remote task in a suite."""
raise NotImplementedError()
def get_tasks_auths(self, suite_name):
"""Return a list of [user@]host for remote tasks in a suite."""
raise NotImplementedError()
def get_task_props(self, *args, **kwargs):
"""Return a TaskProps object containing suite task's attributes."""
calendar_mode = self.date_time_oper.get_calendar_mode()
try:
return self._get_task_props(*args, **kwargs)
finally:
# Restore calendar mode if changed
self.date_time_oper.set_calendar_mode(calendar_mode)
def _get_task_props(self, *_, **kwargs):
"""Helper for get_task_props."""
tprops = TaskProps()
# If suite_name and task_id are defined, we can assume that the rest
# are defined as well.
if tprops.suite_name is not None and tprops.task_id is not None:
return tprops
tprops = self.get_task_props_from_env()
# Modify calendar mode, if possible
self.date_time_oper.set_calendar_mode(tprops.cycling_mode)
if kwargs["cycle"] is not None:
try:
cycle_offset = get_cycle_offset(kwargs["cycle"])
except ISO8601SyntaxError:
tprops.task_cycle_time = kwargs["cycle"]
else:
if tprops.task_cycle_time:
tprops.task_cycle_time = self._get_offset_cycle_time(
tprops.task_cycle_time, cycle_offset)
else:
tprops.task_cycle_time = kwargs["cycle"]
# Etc directory
if os.path.exists(os.path.join(tprops.suite_dir, "etc")):
tprops.dir_etc = os.path.join(tprops.suite_dir, "etc")
# Data directory: generic, current cycle, and previous cycle
tprops.dir_data = os.path.join(tprops.suite_dir, "share", "data")
if tprops.task_cycle_time is not None:
task_cycle_time = tprops.task_cycle_time
tprops.dir_data_cycle = os.path.join(
tprops.suite_dir, "share", "cycle", str(task_cycle_time))
# Offset cycles
if kwargs.get("cycle_offsets"):
cycle_offset_strings = []
for value in kwargs.get("cycle_offsets"):
cycle_offset_strings.extend(value.split(","))
for value in cycle_offset_strings:
if tprops.cycling_mode == "integer":
cycle_offset = value
if cycle_offset.startswith("__"):
sign_factor = 1
else:
sign_factor = -1
offset_val = cycle_offset.replace("__", "")
cycle_time = str(
int(task_cycle_time) +
sign_factor * int(offset_val.replace("P", "")))
else:
cycle_offset = get_cycle_offset(value)
cycle_time = self._get_offset_cycle_time(
task_cycle_time, cycle_offset)
tprops.dir_data_cycle_offsets[str(cycle_offset)] = (
os.path.join(
tprops.suite_dir, "share", "cycle", cycle_time))
# Create data directories if necessary
# Note: should we create the offsets directories?
for dir_ in (
[tprops.dir_data, tprops.dir_data_cycle] +
list(tprops.dir_data_cycle_offsets.values())):
if dir_ is None:
continue
if os.path.exists(dir_) and not os.path.isdir(dir_):
self.fs_util.delete(dir_)
self.fs_util.makedirs(dir_)
# Task prefix and suffix
for key, split, index in [("prefix", str.split, 0),
("suffix", str.rsplit, 1)]:
delim = self.TASK_NAME_DELIM[key]
if kwargs.get(key + "_delim"):
delim = kwargs.get(key + "_delim")
if delim in tprops.task_name:
res = split(tprops.task_name, delim, 1)
setattr(tprops, "task_" + key, res[index])
return tprops
def get_task_props_from_env(self):
"""Return a TaskProps object.
This method should not be used directly. Call get_task_props() instead.
"""
raise NotImplementedError()
def get_version(self):
"""Return the version string of the suite engine."""
raise NotImplementedError()
def get_version_env_name(self):
"""Return the name of the suite engine version environment variable."""
return self.SCHEME.upper() + "_VERSION"
def handle_event(self, *args, **kwargs):
"""Call self.event_handler if it is callable."""
if callable(self.event_handler):
return self.event_handler(*args, **kwargs)
def is_suite_registered(self, suite_name):
"""Return whether or not a suite is registered."""
raise NotImplementedError()
def job_logs_archive(self, suite_name, items):
"""Archive cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def job_logs_pull_remote(self, suite_name, items,
prune_remote_mode=False, force_mode=False):
"""Pull and housekeep the job logs on remote task hosts.
suite_name -- The name of a suite.
items -- A list of relevant items.
prune_remote_mode -- Remove remote job logs after pulling them.
force_mode -- Force retrieval, even if it may not be necessary.
"""
raise NotImplementedError()
def job_logs_remove_on_server(self, suite_name, items):
"""Remove cycle job logs.
suite_name -- The name of a suite.
items -- A list of relevant items.
"""
raise NotImplementedError()
def launch_suite_log_browser(self, user_name, suite_name):
"""Launch web browser to view suite log.
Return URL of suite log on success, None otherwise.
"""
url = self.get_suite_log_url(user_name, suite_name)
browser = webbrowser.get()
browser.open(url, new=True, autoraise=True)
self.handle_event(WebBrowserEvent(browser.name, url))
return url
def parse_job_log_rel_path(self, f_name):
"""Return (cycle, task, submit_num, ext) for a job log rel path."""
raise NotImplementedError()
def run(self, suite_name, host=None, run_mode=None, args=None):
"""Start a suite (in a specified host)."""
raise NotImplementedError()
def shutdown(self, suite_name, args=None, stderr=None, stdout=None):
"""Shut down the suite."""
raise NotImplementedError()
def _get_offset_cycle_time(self, cycle, cycle_offset):
"""Return the actual date time of an BaseCycleOffset against cycle.
cycle: a YYYYmmddHH or ISO 8601 date/time string.
cycle_offset: an instance of BaseCycleOffset.
Return date time in the same format as cycle.
Note: It would be desirable to switch to a ISO 8601 format,
but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
at the moment.
"""
offset_str = str(cycle_offset.to_duration())
try:
time_point, parse_format = self.date_time_oper.date_parse(cycle)
time_point = self.date_time_oper.date_shift(time_point, offset_str)
return self.date_time_oper.date_format(parse_format, time_point)
except OffsetValueError:
raise
except ValueError:
raise CycleTimeError(cycle)