コード例 #1
0
    def _get_conf(self, conf_tree, key, max_args=0):
        """Get a list of cycles from a configuration setting.

        key -- An option key in self.SECTION to locate the setting.
        max_args -- Maximum number of extra arguments for an item in the list.

        The value of the setting is expected to be split by shlex.split into a
        list of items. If max_args == 0, an item should be a string
        representing a cycle or an cycle offset. If max_args > 0, the cycle
        or cycle offset string can, optionally, have arguments. The arguments
        are delimited by colons ":".
        E.g.:

        prune-remote-logs-at=-6h -12h
        prune-server-logs-at=-7d
        prune-datac-at=-6h:foo/* -12h:'bar/* baz/*' -1d
        prune-work-at=-6h:t1*:*.tar -12h:t1*: -12h:*.gz -1d

        If max_args == 0, return a list of cycles.
        If max_args > 0, return a list of (cycle, [arg, ...])

        """
        items_str = conf_tree.node.get_value([self.SECTION, key])
        if items_str is None:
            return []
        try:
            items_str = env_var_process(items_str)
        except UnboundEnvironmentVariableError as exc:
            raise ConfigValueError([self.SECTION, key], items_str, exc)
        items = []
        ref_time_point = os.getenv(
            RoseDateTimeOperator.TASK_CYCLE_TIME_MODE_ENV)
        date_time_oper = RoseDateTimeOperator(ref_time_point=ref_time_point)
        for item_str in shlex.split(items_str):
            args = item_str.split(":", max_args)
            offset = args.pop(0)
            cycle = offset
            if ref_time_point:
                if os.getenv("ROSE_CYCLING_MODE") == "integer":
                    try:
                        cycle = str(int(ref_time_point) + 
                                    int(offset.replace("P","")))
                    except ValueError:
                        pass
                else:
                    try:
                        time_point, parse_format = date_time_oper.date_parse()
                        time_point = date_time_oper.date_shift(time_point, offset)
                        cycle = date_time_oper.date_format(
                            parse_format,
                            time_point)
                    except ValueError:
                        pass
            if max_args:
                items.append((cycle, args))
            else:
                items.append(cycle)
        return items
コード例 #2
0
ファイル: suite_engine_proc.py プロジェクト: hjoliver/rose
class SuiteEngineProcessor(object):
    """An abstract suite engine processor."""

    TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
    SCHEME = None
    SCHEME_HANDLER_MANAGER = None
    SCHEME_DEFAULT = "cylc"  # TODO: site configuration?
    TIMEOUT = 5  # seconds

    @classmethod
    def get_processor(cls, key=None, event_handler=None, popen=None,
                      fs_util=None, host_selector=None):
        """Return a processor for the suite engine named by "key"."""

        if cls.SCHEME_HANDLER_MANAGER is None:
            path = os.path.dirname(
                os.path.dirname(sys.modules["rose"].__file__))
            cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
                [path], ns="rose.suite_engine_procs", attrs=["SCHEME"],
                can_handle=None, event_handler=event_handler, popen=popen,
                fs_util=fs_util, host_selector=host_selector)
        if key is None:
            key = cls.SCHEME_DEFAULT
        return cls.SCHEME_HANDLER_MANAGER.get_handler(key)

    def __init__(self, event_handler=None, popen=None, fs_util=None,
                 host_selector=None, **_):
        self.event_handler = event_handler
        if popen is None:
            popen = RosePopener(event_handler)
        self.popen = popen
        if fs_util is None:
            fs_util = FileSystemUtil(event_handler)
        self.fs_util = fs_util
        if host_selector is None:
            host_selector = HostSelector(event_handler, popen)
        self.host_selector = host_selector
        self.date_time_oper = RoseDateTimeOperator()

    def check_global_conf_compat(self):
        """Raise exception on suite engine specific incompatibity.

        Should raise SuiteEngineGlobalConfCompatError.

        """
        raise NotImplementedError()

    def check_suite_not_running(self, suite_name):
        """Check that suite is not running.

        This method is not implemented. Sub-class should override.

        Arguments:
            suite_name: name of suite to check.

        Raise:
            SuiteStillRunningError:
                Should raise SuiteStillRunningError if suite is still running.
        """
        raise NotImplementedError()

    def cmp_suite_conf(
            self, suite_name, run_mode, strict_mode=False, debug_mode=False):
        """Compare current suite configuration with that in the previous run.

        An implementation of this method should:
        * Raise an exception on failure.
        * Return True if suite configuration is unmodified c.f. previous run.
        * Return False otherwise.

        """
        raise NotImplementedError()

    def get_suite_contact(self, suite_name):
        """Return suite contact information for a user suite.

        Return (dict): suite contact information.
        """
        raise NotImplementedError()

    def get_suite_dir(self, suite_name, *paths):
        """Return the path to the suite running directory.

        paths -- if specified, are added to the end of the path.

        """
        return os.path.join(os.path.expanduser("~"),
                            self.get_suite_dir_rel(suite_name, *paths))

    def get_suite_dir_rel(self, suite_name, *paths):
        """Return the relative path to the suite running directory.

        paths -- if specified, are added to the end of the path.

        """
        raise NotImplementedError()

    def get_suite_log_url(self, user_name, suite_name):
        """Return the "rose bush" URL for a user's suite."""
        prefix = "~"
        if user_name:
            prefix += user_name
        suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
        suite_d = os.path.expanduser(suite_d)
        if not os.path.isdir(suite_d):
            raise NoSuiteLogError(user_name, suite_name)
        rose_bush_url = None
        for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")):
            status = {}
            for line in open(f_name):
                key, value = line.strip().split("=", 1)
                status[key] = value
            if status.get("host"):
                rose_bush_url = "http://" + status["host"]
                if status.get("port"):
                    rose_bush_url += ":" + status["port"]
            rose_bush_url += "/"
            break
        if not rose_bush_url:
            conf = ResourceLocator.default().get_conf()
            rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
        if not rose_bush_url:
            return "file://" + suite_d
        if not rose_bush_url.endswith("/"):
            rose_bush_url += "/"
        if not user_name:
            user_name = pwd.getpwuid(os.getuid()).pw_name
        return rose_bush_url + "/".join(["taskjobs", user_name, suite_name])

    def get_task_auth(self, suite_name, task_name):
        """Return [user@]host for a remote task in a suite."""
        raise NotImplementedError()

    def get_tasks_auths(self, suite_name):
        """Return a list of [user@]host for remote tasks in a suite."""
        raise NotImplementedError()

    def get_task_props(self, *args, **kwargs):
        """Return a TaskProps object containing suite task's attributes."""
        calendar_mode = self.date_time_oper.get_calendar_mode()
        try:
            return self._get_task_props(*args, **kwargs)
        finally:
            # Restore calendar mode if changed
            self.date_time_oper.set_calendar_mode(calendar_mode)

    def _get_task_props(self, *_, **kwargs):
        """Helper for get_task_props."""
        tprops = TaskProps()
        # If suite_name and task_id are defined, we can assume that the rest
        # are defined as well.
        if tprops.suite_name is not None and tprops.task_id is not None:
            return tprops

        tprops = self.get_task_props_from_env()
        # Modify calendar mode, if possible
        self.date_time_oper.set_calendar_mode(tprops.cycling_mode)

        if kwargs["cycle"] is not None:

            try:
                cycle_offset = get_cycle_offset(kwargs["cycle"])
            except ISO8601SyntaxError:
                tprops.task_cycle_time = kwargs["cycle"]
            else:
                if tprops.task_cycle_time:
                    tprops.task_cycle_time = self._get_offset_cycle_time(
                        tprops.task_cycle_time, cycle_offset)
                else:
                    tprops.task_cycle_time = kwargs["cycle"]

        # Etc directory
        if os.path.exists(os.path.join(tprops.suite_dir, "etc")):
            tprops.dir_etc = os.path.join(tprops.suite_dir, "etc")

        # Data directory: generic, current cycle, and previous cycle
        tprops.dir_data = os.path.join(tprops.suite_dir, "share", "data")
        if tprops.task_cycle_time is not None:
            task_cycle_time = tprops.task_cycle_time
            tprops.dir_data_cycle = os.path.join(
                tprops.suite_dir, "share", "cycle", str(task_cycle_time))

            # Offset cycles
            if kwargs.get("cycle_offsets"):
                cycle_offset_strings = []
                for value in kwargs.get("cycle_offsets"):
                    cycle_offset_strings.extend(value.split(","))
                for value in cycle_offset_strings:
                    if tprops.cycling_mode == "integer":
                        cycle_offset = value
                        if cycle_offset.startswith("__"):
                            sign_factor = 1
                        else:
                            sign_factor = -1
                        offset_val = cycle_offset.replace("__", "")
                        cycle_time = str(
                            int(task_cycle_time) +
                            sign_factor * int(offset_val.replace("P", "")))
                    else:
                        cycle_offset = get_cycle_offset(value)
                        cycle_time = self._get_offset_cycle_time(
                            task_cycle_time, cycle_offset)
                    tprops.dir_data_cycle_offsets[str(cycle_offset)] = (
                        os.path.join(
                            tprops.suite_dir, "share", "cycle", cycle_time))

        # Create data directories if necessary
        # Note: should we create the offsets directories?
        for dir_ in (
                [tprops.dir_data, tprops.dir_data_cycle] +
                tprops.dir_data_cycle_offsets.values()):
            if dir_ is None:
                continue
            if os.path.exists(dir_) and not os.path.isdir(dir_):
                self.fs_util.delete(dir_)
            self.fs_util.makedirs(dir_)

        # Task prefix and suffix
        for key, split, index in [("prefix", str.split, 0),
                                  ("suffix", str.rsplit, 1)]:
            delim = self.TASK_NAME_DELIM[key]
            if kwargs.get(key + "_delim"):
                delim = kwargs.get(key + "_delim")
            if delim in tprops.task_name:
                res = split(tprops.task_name, delim, 1)
                setattr(tprops, "task_" + key, res[index])

        return tprops

    def get_task_props_from_env(self):
        """Return a TaskProps object.

        This method should not be used directly. Call get_task_props() instead.

        """
        raise NotImplementedError()

    def get_version(self):
        """Return the version string of the suite engine."""
        raise NotImplementedError()

    def get_version_env_name(self):
        """Return the name of the suite engine version environment variable."""
        return self.SCHEME.upper() + "_VERSION"

    def handle_event(self, *args, **kwargs):
        """Call self.event_handler if it is callable."""
        if callable(self.event_handler):
            return self.event_handler(*args, **kwargs)

    def gcontrol(self, suite_name, args=None):
        """Launch control GUI for a suite_name."""
        raise NotImplementedError()

    def gscan(self, args=None):
        """Launch suites scan GUI."""
        raise NotImplementedError()

    def is_suite_registered(self, suite_name):
        """Return whether or not a suite is registered."""
        raise NotImplementedError()

    def job_logs_archive(self, suite_name, items):
        """Archive cycle job logs.

        suite_name -- The name of a suite.
        items -- A list of relevant items.

        """
        raise NotImplementedError()

    def job_logs_pull_remote(self, suite_name, items,
                             prune_remote_mode=False, force_mode=False):
        """Pull and housekeep the job logs on remote task hosts.

        suite_name -- The name of a suite.
        items -- A list of relevant items.
        prune_remote_mode -- Remove remote job logs after pulling them.
        force_mode -- Force retrieval, even if it may not be necessary.

        """
        raise NotImplementedError()

    def job_logs_remove_on_server(self, suite_name, items):
        """Remove cycle job logs.

        suite_name -- The name of a suite.
        items -- A list of relevant items.

        """
        raise NotImplementedError()

    def launch_suite_log_browser(self, user_name, suite_name):
        """Launch web browser to view suite log.

        Return URL of suite log on success, None otherwise.

        """
        url = self.get_suite_log_url(user_name, suite_name)
        browser = webbrowser.get()
        browser.open(url, new=True, autoraise=True)
        self.handle_event(WebBrowserEvent(browser.name, url))
        return url

    def parse_job_log_rel_path(self, f_name):
        """Return (cycle, task, submit_num, ext) for a job log rel path."""
        raise NotImplementedError()

    def run(self, suite_name, host=None, run_mode=None, args=None):
        """Start a suite (in a specified host)."""
        raise NotImplementedError()

    def shutdown(self, suite_name, args=None, stderr=None, stdout=None):
        """Shut down the suite."""
        raise NotImplementedError()

    def _get_offset_cycle_time(self, cycle, cycle_offset):
        """Return the actual date time of an BaseCycleOffset against cycle.

        cycle: a YYYYmmddHH or ISO 8601 date/time string.
        cycle_offset: an instance of BaseCycleOffset.

        Return date time in the same format as cycle.

        Note: It would be desirable to switch to a ISO 8601 format,
        but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
        at the moment.

        """
        offset_str = str(cycle_offset.to_duration())
        try:
            time_point, parse_format = self.date_time_oper.date_parse(cycle)
            time_point = self.date_time_oper.date_shift(time_point, offset_str)
            return self.date_time_oper.date_format(parse_format, time_point)
        except OffsetValueError:
            raise
        except ValueError:
            raise CycleTimeError(cycle)
コード例 #3
0
class SuiteEngineProcessor(object):
    """An abstract suite engine processor."""

    TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
    SCHEME_HANDLER_MANAGER = None
    SCHEME_DEFAULT = "cylc" # TODO: site configuration?
    TIMEOUT = 5 # seconds

    @classmethod
    def get_processor(cls, key=None, event_handler=None, popen=None,
                      fs_util=None, host_selector=None):
        """Return a processor for the suite engine named by "key"."""

        if cls.SCHEME_HANDLER_MANAGER is None:
            p = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
            cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
                    [p], ns="rose.suite_engine_procs", attrs=["SCHEME"],
                    can_handle=None, event_handler=event_handler, popen=popen,
                    fs_util=fs_util, host_selector=host_selector)
        if key is None:
            key = cls.SCHEME_DEFAULT
        return cls.SCHEME_HANDLER_MANAGER.get_handler(key)

    def __init__(self, event_handler=None, popen=None, fs_util=None,
                 host_selector=None, **kwargs):
        self.event_handler = event_handler
        if popen is None:
            popen = RosePopener(event_handler)
        self.popen = popen
        if fs_util is None:
            fs_util = FileSystemUtil(event_handler)
        self.fs_util = fs_util
        if host_selector is None:
            host_selector = HostSelector(event_handler, popen)
        self.host_selector = host_selector
        self.date_time_oper = RoseDateTimeOperator()

    def check_global_conf_compat(self):
        """Raise exception on suite engine specific incompatibity.

        Should raise SuiteEngineGlobalConfCompatError.

        """
        raise NotImplementedError()

    def check_suite_not_running(self, suite_name, hosts=None):
        """Raise SuiteStillRunningError if suite is still running."""
        reasons = self.is_suite_running(
            None, suite_name, self.get_suite_hosts(suite_name, hosts))
        if reasons:
            raise SuiteStillRunningError(suite_name, reasons)

    def clean_hook(self, suite_name=None):
        """Run suite engine dependent logic (at end of "rose suite-clean")."""
        raise NotImplementedError()

    def cmp_suite_conf(self, suite_name, strict_mode=False, debug_mode=False):
        """Compare current suite configuration with that in the previous run.

        An implementation of this method should:
        * Raise an exception on failure.
        * Return True if suite configuration is unmodified c.f. previous run.
        * Return False otherwise.

        """
        raise NotImplementedError()

    def get_cycle_items_globs(self, name, cycle):
        """Return a glob to match named items created for a given cycle.

        E.g.:
        suite_engine_proc.get_cycle_items_globs("datac", "20130101T0000Z")
        # return "share/data/20130101T0000Z"

        Return None if named item not supported.

        """
        raise NotImplementedError()

    def get_suite_dir(self, suite_name, *paths):
        """Return the path to the suite running directory.

        paths -- if specified, are added to the end of the path.

        """
        return os.path.join(os.path.expanduser("~"),
                            self.get_suite_dir_rel(suite_name, *paths))

    def get_suite_dir_rel(self, suite_name, *paths):
        """Return the relative path to the suite running directory.

        paths -- if specified, are added to the end of the path.

        """
        raise NotImplementedError()

    def get_suite_hosts(self, suite_name=None, hosts=None):
        """Return names of potential suite hosts.

        If "suite_name" is specified, return all possible hosts including what
        is written in the "log/rose-suite-run.host" file.

        If "suite_name" is not specified and if "[rose-suite-run]hosts" is
        defined, split and return the setting.

        Otherwise, return ["localhost"].

        """
        conf = ResourceLocator.default().get_conf()
        hostnames = []
        if hosts:
            hostnames += hosts
        if suite_name:
            hostnames.append("localhost")
            # Get host name from "log/rose-suite-run.host" file
            host_file_path = self.get_suite_dir(
                suite_name, "log", "rose-suite-run.host")
            try:
                for line in open(host_file_path):
                    hostnames.append(line.strip())
            except IOError:
                pass
            # Scan-able list
            suite_hosts = conf.get_value(
                ["rose-suite-run", "scan-hosts"], "").split()
            if suite_hosts:
                hostnames += self.host_selector.expand(suite_hosts)[0]
        # Normal list
        suite_hosts = conf.get_value(["rose-suite-run", "hosts"], "").split()
        if suite_hosts:
            hostnames += self.host_selector.expand(suite_hosts)[0]
        hostnames = list(set(hostnames))
        return hostnames

    def get_suite_job_events(self, user_name, suite_name, cycles, tasks,
                             no_statuses, order, limit, offset):
        """Return suite job events.

        user -- A string containing a valid user ID
        suite -- A string containing a valid suite ID
        cycles -- Display only task jobs matching these cycles. A value in the
                  list can be a cycle, the string "before|after CYCLE", or a
                  glob to match cycles.
        tasks -- Display only jobs for task names matching these names. Values
                 can be a valid task name or a glob like pattern for matching
                 valid task names.
        no_statues -- Do not display jobs with these statuses. Valid values are
                      the keys of CylcProcessor.STATUSES.
        order -- Order search in a predetermined way. A valid value is one of
                 the keys in CylcProcessor.ORDERS.
        limit -- Limit number of returned entries
        offset -- Offset entry number

        Return (entries, of_n_entries) where:
        entries -- A list of matching entries
        of_n_entries -- Total number of entries matching query

        Each entry is a dict:
            {"cycle": cycle, "name": name, "submit_num": submit_num,
             "events": [time_submit, time_init, time_exit],
             "status": None|"submit|fail(submit)|init|success|fail|fail(%s)",
             "logs": {"script": {"path": path, "path_in_tar", path_in_tar,
                                 "size": size, "mtime": mtime},
                      "out": {...},
                      "err": {...},
                      ...}}

        """
        raise NotImplementedError()

    def get_suite_log_url(self, user_name, suite_name):
        """Return the "rose bush" URL for a user's suite."""
        prefix = "~"
        if user_name:
            prefix += user_name
        suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
        suite_d = os.path.expanduser(suite_d)
        if not os.path.isdir(suite_d):
            raise NoSuiteLogError(user_name, suite_name)
        rose_bush_status_f_name = os.path.expanduser(
                    "~/.metomi/rose-bush.status")
        rose_bush_url = None
        if os.path.isfile(rose_bush_status_f_name):
            status = {}
            for line in open(rose_bush_status_f_name):
                k, v = line.strip().split("=", 1)
                status[k] = v
            if status.get("host"):
                rose_bush_url = "http://" + status["host"]
                if status.get("port"):
                    rose_bush_url += ":" + status["port"]
            rose_bush_url += "/"
        if not rose_bush_url:
            conf = ResourceLocator.default().get_conf()
            rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
        if not rose_bush_url:
            return "file://" + suite_d
        if not rose_bush_url.endswith("/"):
            rose_bush_url += "/"
        if not user_name:
            user_name = pwd.getpwuid(os.getuid()).pw_name
        return rose_bush_url + "/".join(["list", user_name, suite_name])

    def get_suite_logs_info(self, user_name, suite_name):
        """Return the information of the suite logs.

        Return a tuple that looks like:
            ("cylc-run",
             {"err": {"path": "log/suite/err", "mtime": mtime, "size": size},
              "log": {"path": "log/suite/log", "mtime": mtime, "size": size},
              "out": {"path": "log/suite/out", "mtime": mtime, "size": size}})

        """
        raise NotImplementedError()

    def get_suite_state_summary(self, user_name, suite_name):
        """Return a the state summary of a user's suite.

        Return {"last_activity_time": s, "is_running": b, "is_failed": b}
        where:
        * last_activity_time is a string in %Y-%m-%dT%H:%M:%S format,
          the time of the latest activity in the suite
        * is_running is a boolean to indicate if the suite is running
        * is_failed: a boolean to indicate if any tasks (submit) failed

        """
        raise NotImplementedError()

    def get_task_auth(self, suite_name, task_name):
        """Return [user@]host for a remote task in a suite."""
        raise NotImplementedError()

    def get_tasks_auths(self, suite_name):
        """Return a list of [user@]host for remote tasks in a suite."""
        raise NotImplementedError()

    def get_task_props(self, *args, **kwargs):
        """Return a TaskProps object containing the attributes of a suite task.
        """

        t = TaskProps()
        # If suite_name and task_id are defined, we can assume that the rest
        # are defined as well.
        if t.suite_name is not None and t.task_id is not None:
            return t

        t = self.get_task_props_from_env()

        if kwargs["cycle"] is not None:

            try:
                cycle_offset = get_cycle_offset(kwargs["cycle"])
            except Exception:
                t.task_cycle_time = kwargs["cycle"]
            else:
                if t.task_cycle_time:
                    t.task_cycle_time = self._get_offset_cycle_time(
                        t.task_cycle_time, cycle_offset)
                else:
                    t.task_cycle_time = kwargs["cycle"]

        # Etc directory
        if os.path.exists(os.path.join(t.suite_dir, "etc")):
            t.dir_etc = os.path.join(t.suite_dir, "etc")

        # Data directory: generic, current cycle, and previous cycle
        t.dir_data = os.path.join(t.suite_dir, "share", "data")
        if t.task_cycle_time is not None:
            task_cycle_time = t.task_cycle_time
            t.dir_data_cycle = os.path.join(t.dir_data, str(task_cycle_time))

            # Offset cycles
            if kwargs.get("cycle_offsets"):
                cycle_offset_strings = []
                for v in kwargs.get("cycle_offsets"):
                    cycle_offset_strings.extend(v.split(","))
                for v in cycle_offset_strings:
                    if t.cycling_mode == "integer":
                        cycle_offset = v
                        if cycle_offset.startswith("__"):
                            sign_factor = 1
                        else:
                            sign_factor = -1
                        offset_val = cycle_offset.replace("__", "")
                        cycle_time = str(int(task_cycle_time)
                             + sign_factor * int(offset_val.replace("P", "")))
                    else:
                        cycle_offset = get_cycle_offset(v)
                        cycle_time = self._get_offset_cycle_time(
                                task_cycle_time, cycle_offset)
                    t.dir_data_cycle_offsets[str(cycle_offset)] = os.path.join(
                            t.dir_data, cycle_time)

        # Create data directories if necessary
        # Note: should we create the offsets directories?
        for d in ([t.dir_data, t.dir_data_cycle] +
                  t.dir_data_cycle_offsets.values()):
            if d is None:
                continue
            if os.path.exists(d) and not os.path.isdir(d):
                self.fs_util.delete(d)
            self.fs_util.makedirs(d)

        # Task prefix and suffix
        for key, split, index in [("prefix", str.split, 0),
                                  ("suffix", str.rsplit, 1)]:
            delim = self.TASK_NAME_DELIM[key]
            if kwargs.get(key + "_delim"):
                delim = kwargs.get(key + "_delim")
            if delim in t.task_name:
                res = split(t.task_name, delim, 1)
                setattr(t, "task_" + key, res[index])

        return t

    def get_task_props_from_env(self):
        """Return a TaskProps object.

        This method should not be used directly. Call get_task_props() instead.

        """
        raise NotImplementedError()

    def get_version(self):
        """Return the version string of the suite engine."""
        raise NotImplementedError()

    def get_version_env_name(self):
        """Return the name of the suite engine version environment variable."""
        return self.SCHEME.upper() + "_VERSION"

    def handle_event(self, *args, **kwargs):
        """Call self.event_handler if it is callable."""
        if callable(self.event_handler):
            return self.event_handler(*args, **kwargs)

    def gcontrol(self, suite_name, host=None, engine_version=None, args=None):
        """Launch control GUI for a suite_name running at a host."""
        raise NotImplementedError()

    def is_conf(self, path):
        """Return the file type if path is a config of this suite engine."""
        raise NotImplementedError()

    def is_suite_registered(self, suite_name):
        """Return whether or not a suite is registered."""
        raise NotImplementedError()

    def is_suite_running(self, user_name, suite_name, hosts=None):
        """Return a list of reasons if it looks like suite is running.

        Each reason should be a dict with the following keys:
        * "host": the host name where the suite appears to be running on.
        * "reason_key": a key, such as "process-id", "port-file", etc.
        * "reason_value": the value of the reason, e.g. the process ID, the
                          path to a port file, etc.

        """
        raise NotImplementedError()

    def job_logs_archive(self, suite_name, items):
        """Archive cycle job logs.

        suite_name -- The name of a suite.
        items -- A list of relevant items.

        """
        raise NotImplementedError()

    def job_logs_db_create(self, suite_name, close=False):
        """Create the job logs database."""
        raise NotImplementedError()

    def job_logs_pull_remote(self, suite_name, items, prune_remote_mode=False):
        """Pull and housekeep the job logs on remote task hosts.

        suite_name -- The name of a suite.
        items -- A list of relevant items.
        prune_remote_mode -- Remove remote job logs after pulling them.

        """
        raise NotImplementedError()

    def job_logs_remove_on_server(self, suite_name, items):
        """Remove cycle job logs.

        suite_name -- The name of a suite.
        items -- A list of relevant items.

        """
        raise NotImplementedError()

    def launch_suite_log_browser(self, user_name, suite_name):
        """Launch web browser to view suite log.

        Return URL of suite log on success, None otherwise.

        """
        url = self.get_suite_log_url(user_name, suite_name)
        w = webbrowser.get()
        w.open(url, new=True, autoraise=True)
        self.handle_event(WebBrowserEvent(w.name, url))
        return url

    def ping(self, suite_name, hosts=None, timeout=10):
        """Return a list of host names where suite_name is running."""
        raise NotImplementedError()

    def run(self, suite_name, host=None, host_environ=None, restart_mode=False,
            args=None):
        """Start a suite (in a specified host)."""
        raise NotImplementedError()

    def scan(self, host_names=None, timeout=TIMEOUT):
        """Scan for running suites (in hosts).

        Return (suite_scan_results, exceptions) where
        suite_scan_results is a list of SuiteScanResult instances and
        exceptions is a list of exceptions resulting from any failed scans

        Default timeout for SSH and "cylc scan" command is 5 seconds.

        """
        raise NotImplementedError()

    def shutdown(self, suite_name, host=None, engine_version=None, args=None,
                 stderr=None, stdout=None):
        """Shut down the suite."""
        raise NotImplementedError()

    def _get_offset_cycle_time(self, cycle, cycle_offset):
        """Return the actual date time of an BaseCycleOffset against cycle.

        cycle: a YYYYmmddHH or ISO 8601 date/time string.
        cycle_offset: an instance of BaseCycleOffset.

        The returned date time would be an YYYYmmdd[HH[MM]] string.

        Note: It would be desirable to switch to a ISO 8601 format,
        but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
        at the moment.

        """
        offset_str = str(cycle_offset.to_duration())
        try:
            time_point, parse_format = self.date_time_oper.date_parse(cycle)
            time_point = self.date_time_oper.date_shift(time_point, offset_str)
            return self.date_time_oper.date_format(parse_format, time_point)
        except OffsetValueError:
            raise
        except ValueError:
            raise CycleTimeError(cycle)
コード例 #4
0
ファイル: suite_engine_proc.py プロジェクト: kaday/rose
class SuiteEngineProcessor(object):
    """An abstract suite engine processor."""

    TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
    SCHEME_HANDLER_MANAGER = None
    SCHEME_DEFAULT = "cylc"  # TODO: site configuration?
    TIMEOUT = 5  # seconds

    @classmethod
    def get_processor(cls, key=None, event_handler=None, popen=None,
                      fs_util=None, host_selector=None):
        """Return a processor for the suite engine named by "key"."""

        if cls.SCHEME_HANDLER_MANAGER is None:
            p = os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))
            cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
                [p], ns="rose.suite_engine_procs", attrs=["SCHEME"],
                can_handle=None, event_handler=event_handler, popen=popen,
                fs_util=fs_util, host_selector=host_selector)
        if key is None:
            key = cls.SCHEME_DEFAULT
        return cls.SCHEME_HANDLER_MANAGER.get_handler(key)

    def __init__(self, event_handler=None, popen=None, fs_util=None,
                 host_selector=None, **kwargs):
        self.event_handler = event_handler
        if popen is None:
            popen = RosePopener(event_handler)
        self.popen = popen
        if fs_util is None:
            fs_util = FileSystemUtil(event_handler)
        self.fs_util = fs_util
        if host_selector is None:
            host_selector = HostSelector(event_handler, popen)
        self.host_selector = host_selector
        self.date_time_oper = RoseDateTimeOperator()

    def check_global_conf_compat(self):
        """Raise exception on suite engine specific incompatibity.

        Should raise SuiteEngineGlobalConfCompatError.

        """
        raise NotImplementedError()

    def check_suite_not_running(self, suite_name, hosts=None):
        """Raise SuiteStillRunningError if suite is still running."""
        reasons = self.is_suite_running(
            None, suite_name, self.get_suite_hosts(suite_name, hosts))
        if reasons:
            raise SuiteStillRunningError(suite_name, reasons)

    def clean_hook(self, suite_name=None):
        """Run suite engine dependent logic (at end of "rose suite-clean")."""
        raise NotImplementedError()

    def cmp_suite_conf(self, suite_name, strict_mode=False, debug_mode=False):
        """Compare current suite configuration with that in the previous run.

        An implementation of this method should:
        * Raise an exception on failure.
        * Return True if suite configuration is unmodified c.f. previous run.
        * Return False otherwise.

        """
        raise NotImplementedError()

    def get_suite_dir(self, suite_name, *paths):
        """Return the path to the suite running directory.

        paths -- if specified, are added to the end of the path.

        """
        return os.path.join(os.path.expanduser("~"),
                            self.get_suite_dir_rel(suite_name, *paths))

    def get_suite_dir_rel(self, suite_name, *paths):
        """Return the relative path to the suite running directory.

        paths -- if specified, are added to the end of the path.

        """
        raise NotImplementedError()

    def get_suite_hosts(self, suite_name=None, hosts=None):
        """Return names of potential suite hosts.

        If "suite_name" is specified, return all possible hosts including what
        is written in the "log/rose-suite-run.host" file.

        If "suite_name" is not specified and if "[rose-suite-run]hosts" is
        defined, split and return the setting.

        Otherwise, return ["localhost"].

        """
        conf = ResourceLocator.default().get_conf()
        hostnames = []
        if hosts:
            hostnames += hosts
        if suite_name:
            hostnames.append("localhost")
            # Get host name from "log/rose-suite-run.host" file
            host_file_path = self.get_suite_dir(
                suite_name, "log", "rose-suite-run.host")
            try:
                for line in open(host_file_path):
                    hostnames.append(line.strip())
            except IOError:
                pass
            # Scan-able list
            suite_hosts = conf.get_value(
                ["rose-suite-run", "scan-hosts"], "").split()
            if suite_hosts:
                hostnames += self.host_selector.expand(suite_hosts)[0]
        # Normal list
        suite_hosts = conf.get_value(["rose-suite-run", "hosts"], "").split()
        if suite_hosts:
            hostnames += self.host_selector.expand(suite_hosts)[0]
        hostnames = list(set(hostnames))
        return hostnames

    def get_suite_job_events(self, user_name, suite_name, cycles, tasks,
                             no_statuses, order, limit, offset):
        """Return suite job events.

        user -- A string containing a valid user ID
        suite -- A string containing a valid suite ID
        cycles -- Display only task jobs matching these cycles. A value in the
                  list can be a cycle, the string "before|after CYCLE", or a
                  glob to match cycles.
        tasks -- Display only jobs for task names matching these names. Values
                 can be a valid task name or a glob like pattern for matching
                 valid task names.
        no_statues -- Do not display jobs with these statuses. Valid values are
                      the keys of CylcProcessor.STATUSES.
        order -- Order search in a predetermined way. A valid value is one of
                 the keys in CylcProcessor.ORDERS.
        limit -- Limit number of returned entries
        offset -- Offset entry number

        Return (entries, of_n_entries) where:
        entries -- A list of matching entries
        of_n_entries -- Total number of entries matching query

        Each entry is a dict:
            {"cycle": cycle, "name": name, "submit_num": submit_num,
             "events": [time_submit, time_init, time_exit],
             "status": None|"submit|fail(submit)|init|success|fail|fail(%s)",
             "logs": {"script": {"path": path, "path_in_tar", path_in_tar,
                                 "size": size, "mtime": mtime},
                      "out": {...},
                      "err": {...},
                      ...}}

        """
        raise NotImplementedError()

    def get_suite_log_url(self, user_name, suite_name):
        """Return the "rose bush" URL for a user's suite."""
        prefix = "~"
        if user_name:
            prefix += user_name
        suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
        suite_d = os.path.expanduser(suite_d)
        if not os.path.isdir(suite_d):
            raise NoSuiteLogError(user_name, suite_name)
        rose_bush_url = None
        for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")):
            status = {}
            for line in open(f_name):
                k, v = line.strip().split("=", 1)
                status[k] = v
            if status.get("host"):
                rose_bush_url = "http://" + status["host"]
                if status.get("port"):
                    rose_bush_url += ":" + status["port"]
            rose_bush_url += "/"
            break
        if not rose_bush_url:
            conf = ResourceLocator.default().get_conf()
            rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
        if not rose_bush_url:
            return "file://" + suite_d
        if not rose_bush_url.endswith("/"):
            rose_bush_url += "/"
        if not user_name:
            user_name = pwd.getpwuid(os.getuid()).pw_name
        return rose_bush_url + "/".join(["list", user_name, suite_name])

    def get_suite_logs_info(self, user_name, suite_name):
        """Return the information of the suite logs.

        Return a tuple that looks like:
            ("cylc-run",
             {"err": {"path": "log/suite/err", "mtime": mtime, "size": size},
              "log": {"path": "log/suite/log", "mtime": mtime, "size": size},
              "out": {"path": "log/suite/out", "mtime": mtime, "size": size}})

        """
        raise NotImplementedError()

    def get_suite_state_summary(self, user_name, suite_name):
        """Return a the state summary of a user's suite.

        Return {"last_activity_time": s, "is_running": b, "is_failed": b}
        where:
        * last_activity_time is a string in %Y-%m-%dT%H:%M:%S format,
          the time of the latest activity in the suite
        * is_running is a boolean to indicate if the suite is running
        * is_failed: a boolean to indicate if any tasks (submit) failed

        """
        raise NotImplementedError()

    def get_task_auth(self, suite_name, task_name):
        """Return [user@]host for a remote task in a suite."""
        raise NotImplementedError()

    def get_tasks_auths(self, suite_name):
        """Return a list of [user@]host for remote tasks in a suite."""
        raise NotImplementedError()

    def get_task_props(self, *args, **kwargs):
        """Return a TaskProps object containing the attributes of a suite task.
        """

        t = TaskProps()
        # If suite_name and task_id are defined, we can assume that the rest
        # are defined as well.
        if t.suite_name is not None and t.task_id is not None:
            return t

        t = self.get_task_props_from_env()

        if kwargs["cycle"] is not None:

            try:
                cycle_offset = get_cycle_offset(kwargs["cycle"])
            except ISO8601SyntaxError:
                t.task_cycle_time = kwargs["cycle"]
            else:
                if t.task_cycle_time:
                    t.task_cycle_time = self._get_offset_cycle_time(
                        t.task_cycle_time, cycle_offset)
                else:
                    t.task_cycle_time = kwargs["cycle"]

        # Etc directory
        if os.path.exists(os.path.join(t.suite_dir, "etc")):
            t.dir_etc = os.path.join(t.suite_dir, "etc")

        # Data directory: generic, current cycle, and previous cycle
        t.dir_data = os.path.join(t.suite_dir, "share", "data")
        if t.task_cycle_time is not None:
            task_cycle_time = t.task_cycle_time
            t.dir_data_cycle = os.path.join(
                t.suite_dir, "share", "cycle", str(task_cycle_time))

            # Offset cycles
            if kwargs.get("cycle_offsets"):
                cycle_offset_strings = []
                for v in kwargs.get("cycle_offsets"):
                    cycle_offset_strings.extend(v.split(","))
                for v in cycle_offset_strings:
                    if t.cycling_mode == "integer":
                        cycle_offset = v
                        if cycle_offset.startswith("__"):
                            sign_factor = 1
                        else:
                            sign_factor = -1
                        offset_val = cycle_offset.replace("__", "")
                        cycle_time = str(
                            int(task_cycle_time) +
                            sign_factor * int(offset_val.replace("P", "")))
                    else:
                        cycle_offset = get_cycle_offset(v)
                        cycle_time = self._get_offset_cycle_time(
                            task_cycle_time, cycle_offset)
                    t.dir_data_cycle_offsets[str(cycle_offset)] = os.path.join(
                        t.suite_dir, "share", "cycle", cycle_time)

        # Create data directories if necessary
        # Note: should we create the offsets directories?
        for d in ([t.dir_data, t.dir_data_cycle] +
                  t.dir_data_cycle_offsets.values()):
            if d is None:
                continue
            if os.path.exists(d) and not os.path.isdir(d):
                self.fs_util.delete(d)
            self.fs_util.makedirs(d)

        # Task prefix and suffix
        for key, split, index in [("prefix", str.split, 0),
                                  ("suffix", str.rsplit, 1)]:
            delim = self.TASK_NAME_DELIM[key]
            if kwargs.get(key + "_delim"):
                delim = kwargs.get(key + "_delim")
            if delim in t.task_name:
                res = split(t.task_name, delim, 1)
                setattr(t, "task_" + key, res[index])

        return t

    def get_task_props_from_env(self):
        """Return a TaskProps object.

        This method should not be used directly. Call get_task_props() instead.

        """
        raise NotImplementedError()

    def get_version(self):
        """Return the version string of the suite engine."""
        raise NotImplementedError()

    def get_version_env_name(self):
        """Return the name of the suite engine version environment variable."""
        return self.SCHEME.upper() + "_VERSION"

    def handle_event(self, *args, **kwargs):
        """Call self.event_handler if it is callable."""
        if callable(self.event_handler):
            return self.event_handler(*args, **kwargs)

    def gcontrol(self, suite_name, host=None, engine_version=None, args=None):
        """Launch control GUI for a suite_name running at a host."""
        raise NotImplementedError()

    def is_conf(self, path):
        """Return the file type if path is a config of this suite engine."""
        raise NotImplementedError()

    def is_suite_registered(self, suite_name):
        """Return whether or not a suite is registered."""
        raise NotImplementedError()

    def is_suite_running(self, user_name, suite_name, hosts=None):
        """Return a list of reasons if it looks like suite is running.

        Each reason should be a dict with the following keys:
        * "host": the host name where the suite appears to be running on.
        * "reason_key": a key, such as "process-id", "port-file", etc.
        * "reason_value": the value of the reason, e.g. the process ID, the
                          path to a port file, etc.

        """
        raise NotImplementedError()

    def job_logs_archive(self, suite_name, items):
        """Archive cycle job logs.

        suite_name -- The name of a suite.
        items -- A list of relevant items.

        """
        raise NotImplementedError()

    def job_logs_db_create(self, suite_name, close=False):
        """Create the job logs database."""
        raise NotImplementedError()

    def job_logs_pull_remote(self, suite_name, items,
                             prune_remote_mode=False, force_mode=False):
        """Pull and housekeep the job logs on remote task hosts.

        suite_name -- The name of a suite.
        items -- A list of relevant items.
        prune_remote_mode -- Remove remote job logs after pulling them.
        force_mode -- Force retrieval, even if it may not be necessary.

        """
        raise NotImplementedError()

    def job_logs_remove_on_server(self, suite_name, items):
        """Remove cycle job logs.

        suite_name -- The name of a suite.
        items -- A list of relevant items.

        """
        raise NotImplementedError()

    def launch_suite_log_browser(self, user_name, suite_name):
        """Launch web browser to view suite log.

        Return URL of suite log on success, None otherwise.

        """
        url = self.get_suite_log_url(user_name, suite_name)
        w = webbrowser.get()
        w.open(url, new=True, autoraise=True)
        self.handle_event(WebBrowserEvent(w.name, url))
        return url

    def parse_job_log_rel_path(self, f_name):
        """Return (cycle, task, submit_num, ext) for a job log rel path."""
        raise NotImplementedError()

    def ping(self, suite_name, hosts=None, timeout=10):
        """Return a list of host names where suite_name is running."""
        raise NotImplementedError()

    def run(self, suite_name, host=None, host_environ=None, restart_mode=False,
            args=None):
        """Start a suite (in a specified host)."""
        raise NotImplementedError()

    def scan(self, host_names=None, timeout=TIMEOUT):
        """Scan for running suites (in hosts).

        Return (suite_scan_results, exceptions) where
        suite_scan_results is a list of SuiteScanResult instances and
        exceptions is a list of exceptions resulting from any failed scans

        Default timeout for SSH and "cylc scan" command is 5 seconds.

        """
        raise NotImplementedError()

    def shutdown(self, suite_name, host=None, engine_version=None, args=None,
                 stderr=None, stdout=None):
        """Shut down the suite."""
        raise NotImplementedError()

    def _get_offset_cycle_time(self, cycle, cycle_offset):
        """Return the actual date time of an BaseCycleOffset against cycle.

        cycle: a YYYYmmddHH or ISO 8601 date/time string.
        cycle_offset: an instance of BaseCycleOffset.

        The returned date time would be an YYYYmmdd[HH[MM]] string.

        Note: It would be desirable to switch to a ISO 8601 format,
        but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
        at the moment.

        """
        offset_str = str(cycle_offset.to_duration())
        try:
            time_point, parse_format = self.date_time_oper.date_parse(cycle)
            time_point = self.date_time_oper.date_shift(time_point, offset_str)
            return self.date_time_oper.date_format(parse_format, time_point)
        except OffsetValueError:
            raise
        except ValueError:
            raise CycleTimeError(cycle)
コード例 #5
0
class SuiteEngineProcessor(object):
    """An abstract suite engine processor."""

    TASK_NAME_DELIM = {"prefix": "_", "suffix": "_"}
    SCHEME = None
    SCHEME_HANDLER_MANAGER = None
    SCHEME_DEFAULT = "cylc"  # TODO: site configuration?
    TIMEOUT = 5  # seconds

    @classmethod
    def get_processor(cls, key=None, event_handler=None, popen=None,
                      fs_util=None, host_selector=None):
        """Return a processor for the suite engine named by "key"."""

        if cls.SCHEME_HANDLER_MANAGER is None:
            path = os.path.dirname(
                os.path.dirname(sys.modules["rose"].__file__))
            cls.SCHEME_HANDLER_MANAGER = SchemeHandlersManager(
                [path], ns="rose.suite_engine_procs", attrs=["SCHEME"],
                can_handle=None, event_handler=event_handler, popen=popen,
                fs_util=fs_util, host_selector=host_selector)
        if key is None:
            key = cls.SCHEME_DEFAULT
        return cls.SCHEME_HANDLER_MANAGER.get_handler(key)

    def __init__(self, event_handler=None, popen=None, fs_util=None,
                 host_selector=None, **_):
        self.event_handler = event_handler
        if popen is None:
            popen = RosePopener(event_handler)
        self.popen = popen
        if fs_util is None:
            fs_util = FileSystemUtil(event_handler)
        self.fs_util = fs_util
        if host_selector is None:
            host_selector = HostSelector(event_handler, popen)
        self.host_selector = host_selector
        self.date_time_oper = RoseDateTimeOperator()

    def check_global_conf_compat(self):
        """Raise exception on suite engine specific incompatibity.

        Should raise SuiteEngineGlobalConfCompatError.

        """
        raise NotImplementedError()

    def check_suite_not_running(self, suite_name, hosts=None):
        """Raise SuiteStillRunningError if suite is still running."""
        host_names = self.get_suite_run_hosts(None, suite_name, hosts)
        if host_names:
            raise SuiteStillRunningError(suite_name, host_names)

    def cmp_suite_conf(
            self, suite_name, run_mode, strict_mode=False, debug_mode=False):
        """Compare current suite configuration with that in the previous run.

        An implementation of this method should:
        * Raise an exception on failure.
        * Return True if suite configuration is unmodified c.f. previous run.
        * Return False otherwise.

        """
        raise NotImplementedError()

    def get_suite_dir(self, suite_name, *paths):
        """Return the path to the suite running directory.

        paths -- if specified, are added to the end of the path.

        """
        return os.path.join(os.path.expanduser("~"),
                            self.get_suite_dir_rel(suite_name, *paths))

    def get_suite_dir_rel(self, suite_name, *paths):
        """Return the relative path to the suite running directory.

        paths -- if specified, are added to the end of the path.

        """
        raise NotImplementedError()

    def get_suite_run_hosts(self, user_name, suite_name, host_names=None):
        """Return host(s) where suite_name is running."""
        raise NotImplementedError()

    def get_suite_log_url(self, user_name, suite_name):
        """Return the "rose bush" URL for a user's suite."""
        prefix = "~"
        if user_name:
            prefix += user_name
        suite_d = os.path.join(prefix, self.get_suite_dir_rel(suite_name))
        suite_d = os.path.expanduser(suite_d)
        if not os.path.isdir(suite_d):
            raise NoSuiteLogError(user_name, suite_name)
        rose_bush_url = None
        for f_name in glob(os.path.expanduser("~/.metomi/rose-bush*.status")):
            status = {}
            for line in open(f_name):
                key, value = line.strip().split("=", 1)
                status[key] = value
            if status.get("host"):
                rose_bush_url = "http://" + status["host"]
                if status.get("port"):
                    rose_bush_url += ":" + status["port"]
            rose_bush_url += "/"
            break
        if not rose_bush_url:
            conf = ResourceLocator.default().get_conf()
            rose_bush_url = conf.get_value(["rose-suite-log", "rose-bush"])
        if not rose_bush_url:
            return "file://" + suite_d
        if not rose_bush_url.endswith("/"):
            rose_bush_url += "/"
        if not user_name:
            user_name = pwd.getpwuid(os.getuid()).pw_name
        return rose_bush_url + "/".join(["taskjobs", user_name, suite_name])

    def get_task_auth(self, suite_name, task_name):
        """Return [user@]host for a remote task in a suite."""
        raise NotImplementedError()

    def get_tasks_auths(self, suite_name):
        """Return a list of [user@]host for remote tasks in a suite."""
        raise NotImplementedError()

    def get_task_props(self, *args, **kwargs):
        """Return a TaskProps object containing suite task's attributes."""
        calendar_mode = self.date_time_oper.get_calendar_mode()
        try:
            return self._get_task_props(*args, **kwargs)
        finally:
            # Restore calendar mode if changed
            self.date_time_oper.set_calendar_mode(calendar_mode)

    def _get_task_props(self, *_, **kwargs):
        """Helper for get_task_props."""
        tprops = TaskProps()
        # If suite_name and task_id are defined, we can assume that the rest
        # are defined as well.
        if tprops.suite_name is not None and tprops.task_id is not None:
            return tprops

        tprops = self.get_task_props_from_env()
        # Modify calendar mode, if possible
        self.date_time_oper.set_calendar_mode(tprops.cycling_mode)

        if kwargs["cycle"] is not None:

            try:
                cycle_offset = get_cycle_offset(kwargs["cycle"])
            except ISO8601SyntaxError:
                tprops.task_cycle_time = kwargs["cycle"]
            else:
                if tprops.task_cycle_time:
                    tprops.task_cycle_time = self._get_offset_cycle_time(
                        tprops.task_cycle_time, cycle_offset)
                else:
                    tprops.task_cycle_time = kwargs["cycle"]

        # Etc directory
        if os.path.exists(os.path.join(tprops.suite_dir, "etc")):
            tprops.dir_etc = os.path.join(tprops.suite_dir, "etc")

        # Data directory: generic, current cycle, and previous cycle
        tprops.dir_data = os.path.join(tprops.suite_dir, "share", "data")
        if tprops.task_cycle_time is not None:
            task_cycle_time = tprops.task_cycle_time
            tprops.dir_data_cycle = os.path.join(
                tprops.suite_dir, "share", "cycle", str(task_cycle_time))

            # Offset cycles
            if kwargs.get("cycle_offsets"):
                cycle_offset_strings = []
                for value in kwargs.get("cycle_offsets"):
                    cycle_offset_strings.extend(value.split(","))
                for value in cycle_offset_strings:
                    if tprops.cycling_mode == "integer":
                        cycle_offset = value
                        if cycle_offset.startswith("__"):
                            sign_factor = 1
                        else:
                            sign_factor = -1
                        offset_val = cycle_offset.replace("__", "")
                        cycle_time = str(
                            int(task_cycle_time) +
                            sign_factor * int(offset_val.replace("P", "")))
                    else:
                        cycle_offset = get_cycle_offset(value)
                        cycle_time = self._get_offset_cycle_time(
                            task_cycle_time, cycle_offset)
                    tprops.dir_data_cycle_offsets[str(cycle_offset)] = (
                        os.path.join(
                            tprops.suite_dir, "share", "cycle", cycle_time))

        # Create data directories if necessary
        # Note: should we create the offsets directories?
        for dir_ in (
                [tprops.dir_data, tprops.dir_data_cycle] +
                tprops.dir_data_cycle_offsets.values()):
            if dir_ is None:
                continue
            if os.path.exists(dir_) and not os.path.isdir(dir_):
                self.fs_util.delete(dir_)
            self.fs_util.makedirs(dir_)

        # Task prefix and suffix
        for key, split, index in [("prefix", str.split, 0),
                                  ("suffix", str.rsplit, 1)]:
            delim = self.TASK_NAME_DELIM[key]
            if kwargs.get(key + "_delim"):
                delim = kwargs.get(key + "_delim")
            if delim in tprops.task_name:
                res = split(tprops.task_name, delim, 1)
                setattr(tprops, "task_" + key, res[index])

        return tprops

    def get_task_props_from_env(self):
        """Return a TaskProps object.

        This method should not be used directly. Call get_task_props() instead.

        """
        raise NotImplementedError()

    def get_version(self):
        """Return the version string of the suite engine."""
        raise NotImplementedError()

    def get_version_env_name(self):
        """Return the name of the suite engine version environment variable."""
        return self.SCHEME.upper() + "_VERSION"

    def handle_event(self, *args, **kwargs):
        """Call self.event_handler if it is callable."""
        if callable(self.event_handler):
            return self.event_handler(*args, **kwargs)

    def gcontrol(self, suite_name, host=None, engine_version=None, args=None):
        """Launch control GUI for a suite_name running at a host."""
        raise NotImplementedError()

    def is_suite_registered(self, suite_name):
        """Return whether or not a suite is registered."""
        raise NotImplementedError()

    def job_logs_archive(self, suite_name, items):
        """Archive cycle job logs.

        suite_name -- The name of a suite.
        items -- A list of relevant items.

        """
        raise NotImplementedError()

    def job_logs_pull_remote(self, suite_name, items,
                             prune_remote_mode=False, force_mode=False):
        """Pull and housekeep the job logs on remote task hosts.

        suite_name -- The name of a suite.
        items -- A list of relevant items.
        prune_remote_mode -- Remove remote job logs after pulling them.
        force_mode -- Force retrieval, even if it may not be necessary.

        """
        raise NotImplementedError()

    def job_logs_remove_on_server(self, suite_name, items):
        """Remove cycle job logs.

        suite_name -- The name of a suite.
        items -- A list of relevant items.

        """
        raise NotImplementedError()

    def launch_suite_log_browser(self, user_name, suite_name):
        """Launch web browser to view suite log.

        Return URL of suite log on success, None otherwise.

        """
        url = self.get_suite_log_url(user_name, suite_name)
        browser = webbrowser.get()
        browser.open(url, new=True, autoraise=True)
        self.handle_event(WebBrowserEvent(browser.name, url))
        return url

    def parse_job_log_rel_path(self, f_name):
        """Return (cycle, task, submit_num, ext) for a job log rel path."""
        raise NotImplementedError()

    def run(self, suite_name, host=None, host_environ=None, restart_mode=False,
            args=None):
        """Start a suite (in a specified host)."""
        raise NotImplementedError()

    def scan(self, host_names=None, timeout=TIMEOUT):
        """Scan for running suites (in hosts).

        Return (suite_scan_results, exceptions) where
        suite_scan_results is a list of SuiteScanResult instances and
        exceptions is a list of exceptions resulting from any failed scans

        Default timeout for SSH and "cylc scan" command is 5 seconds.

        """
        raise NotImplementedError()

    def shutdown(self, suite_name, host=None, engine_version=None, args=None,
                 stderr=None, stdout=None):
        """Shut down the suite."""
        raise NotImplementedError()

    def _get_offset_cycle_time(self, cycle, cycle_offset):
        """Return the actual date time of an BaseCycleOffset against cycle.

        cycle: a YYYYmmddHH or ISO 8601 date/time string.
        cycle_offset: an instance of BaseCycleOffset.

        Return date time in the same format as cycle.

        Note: It would be desirable to switch to a ISO 8601 format,
        but due to Cylc's YYYYmmddHH format, it would be too confusing to do so
        at the moment.

        """
        offset_str = str(cycle_offset.to_duration())
        try:
            time_point, parse_format = self.date_time_oper.date_parse(cycle)
            time_point = self.date_time_oper.date_shift(time_point, offset_str)
            return self.date_time_oper.date_format(parse_format, time_point)
        except OffsetValueError:
            raise
        except ValueError:
            raise CycleTimeError(cycle)
コード例 #6
0
ファイル: rose_prune.py プロジェクト: ScottWales/rose
    def _get_conf(self, conf_tree, key, max_args=0):
        """Get a list of cycles from a configuration setting.

        key -- An option key in self.SECTION to locate the setting.
        max_args -- Maximum number of extra arguments for an item in the list.

        The value of the setting is expected to be split by shlex.split into a
        list of items. If max_args == 0, an item should be a string
        representing a cycle or an cycle offset. If max_args > 0, the cycle
        or cycle offset string can, optionally, have arguments. The arguments
        are delimited by colons ":".
        E.g.:

        prune-remote-logs-at=-PT6H -PT12H
        prune-server-logs-at=-P7D
        prune-datac-at=-PT6H:foo/* -PT12H:'bar/* baz/*' -P1D
        prune-work-at=-PT6H:t1*:*.tar -PT12H:t1*: -PT12H:*.gz -P1D

        If max_args == 0, return a list of cycles.
        If max_args > 0, return a list of (cycle, [arg, ...])

        """
        items_str = conf_tree.node.get_value([self.SECTION, key])
        if items_str is None:
            return []
        try:
            items_str = env_var_process(items_str)
        except UnboundEnvironmentVariableError as exc:
            raise ConfigValueError([self.SECTION, key], items_str, exc)
        items = []
        ref_point_str = os.getenv(
            RoseDateTimeOperator.TASK_CYCLE_TIME_MODE_ENV)
        try:
            date_time_oper = None
            ref_time_point = None
            parse_format = None
            for item_str in shlex.split(items_str):
                args = item_str.split(":", max_args)
                when = args.pop(0)
                cycle = when
                if (ref_point_str and
                        os.getenv("ROSE_CYCLING_MODE") == "integer"):
                    # Integer cycling
                    if "P" in when:  # "when" is an offset
                        cycle = str(int(ref_point_str) +
                                    int(when.replace("P", "")))
                    else:  # "when" is a cycle point
                        cycle = str(when)
                elif ref_point_str:
                    # Date-time cycling
                    if date_time_oper is None:
                        date_time_oper = RoseDateTimeOperator(
                            ref_time_point=ref_point_str)
                        ref_time_point, ref_fmt = date_time_oper.date_parse()
                    try:
                        time_point = date_time_oper.date_parse(when)[0]
                    except ValueError:
                        time_point = date_time_oper.date_shift(
                            ref_time_point, when)
                    cycle = date_time_oper.date_format(ref_fmt, time_point)
                if max_args:
                    items.append((cycle, args))
                else:
                    items.append(cycle)
        except ValueError as exc:
            raise ConfigValueError([self.SECTION, key], items_str, exc)
        return items