예제 #1
0
    def _run(self, dao, app_runner, config):
        """Transform and archive suite files.

        This application is designed to work under "rose task-run" in a suite.

        """
        compress_manager = SchemeHandlersManager(
            [os.path.dirname(os.path.dirname(sys.modules["rose"].__file__))],
            "rose.apps.rose_arch_compressions",
            ["compress_sources"],
            None, app_runner)

        # Set up the targets
        s_key_tails = set()
        targets = []
        for t_key, t_node in sorted(config.value.items()):
            if t_node.is_ignored() or ":" not in t_key:
                continue
            s_key_head, s_key_tail = t_key.split(":", 1)
            if s_key_head != self.SECTION or not s_key_tail:
                continue

            # Determine target path.
            s_key_tail = t_key.split(":", 1)[1]
            try:
                s_key_tail = env_var_process(s_key_tail)
            except UnboundEnvironmentVariableError as exc:
                raise ConfigValueError([t_key, ""], "", exc)

            # If parenthesised target is optional.
            is_compulsory_target = True
            if s_key_tail.startswith("(") and s_key_tail.endswith(")"):
                s_key_tail = s_key_tail[1:-1]
                is_compulsory_target = False

            # Don't permit duplicate targets.
            if s_key_tail in s_key_tails:
                raise RoseArchDuplicateError([t_key], '', s_key_tail)
            else:
                s_key_tails.add(s_key_tail)

            target = self._run_target_setup(
                app_runner, compress_manager, config, t_key, s_key_tail,
                t_node, is_compulsory_target)
            old_target = dao.select(target.name)
            if old_target is None or old_target != target:
                dao.delete(target)
            else:
                target.status = target.ST_OLD
            targets.append(target)
        targets.sort(key=lambda target: target.name)
        # Delete from database items that are no longer relevant
        dao.delete_all(filter_targets=targets)
        # Update the targets
        for target in targets:
            self._run_target_update(dao, app_runner, compress_manager, target)
        return [target.status for target in targets].count(
            RoseArchTarget.ST_BAD)
예제 #2
0
파일: fcm_make.py 프로젝트: wxtim/rose
def _conf_value(conf_tree, keys, default=None):
    """Return conf setting value, with env var processed."""
    value = conf_tree.node.get_value(keys, default)
    if value is None:
        return
    try:
        return env_var_process(value)
    except UnboundEnvironmentVariableError as exc:
        raise ConfigValueError(keys, value, exc)
예제 #3
0
 def _get_conf(self, r_node, t_node, key, compulsory=False, default=None):
     """Return the value of a configuration."""
     value = t_node.get_value([key],
                              r_node.get_value([self.SECTION, key],
                                               default=default))
     if compulsory and not value:
         raise CompulsoryConfigValueError([key], None, KeyError(key))
     if value:
         try:
             value = env_var_process(value)
         except UnboundEnvironmentVariableError as exc:
             raise ConfigValueError([key], value, exc)
     return value
예제 #4
0
    def _get_prune_globs(self, app_runner, conf_tree):
        """Return (globs, cycles).

        where:
        * globs is for matching items to prune.
        * cycles is a set of relevant cycles.
        """
        globs = []
        nodes = conf_tree.node.get_value([self.SECTION])
        if nodes is None:
            return [], set()
        cycle_formats = {}
        for key, node in nodes.items():
            if node.is_ignored():
                continue
            if key.startswith("cycle-format{") and key.endswith("}"):
                fmt = key[len("cycle-format{"):-1]
                try:
                    cycle_formats[fmt] = env_var_process(node.value)
                    # Check formats are valid
                    if self._get_cycling_mode() == "integer":
                        cycle_formats[fmt] % 0
                    else:
                        app_runner.date_time_oper.date_format(
                            cycle_formats[fmt])
                except (UnboundEnvironmentVariableError, ValueError) as exc:
                    raise ConfigValueError([self.SECTION, key], node.value,
                                           exc)
        cycle_set = set()
        for key, node in sorted(nodes.items()):
            if node.is_ignored():
                continue
            if key == "prune-datac-at":  # backward compat
                head = "share/cycle"
            elif key == "prune-work-at":  # backward compat
                head = "work"
            elif key.startswith("prune{") and key.endswith("}"):
                head = key[len("prune{"):-1].strip()  # remove "prune{" and "}"
            else:
                continue
            for cycle, cycle_args in self._get_conf(app_runner,
                                                    conf_tree,
                                                    key,
                                                    max_args=1):
                cycle_set.add(cycle)
                if cycle_args:
                    cycle_strs = {"cycle": cycle}
                    for cycle_key, cycle_format in cycle_formats.items():
                        if self._get_cycling_mode() == "integer":
                            cycle_strs[cycle_key] = cycle_format % int(cycle)
                        else:  # date time cycling
                            cycle_point = app_runner.date_time_oper.date_parse(
                                cycle)[0]
                            cycle_strs[
                                cycle_key] = app_runner.date_time_oper.date_format(
                                    cycle_format, cycle_point)
                    for tail_glob in shlex.split(cycle_args.pop()):
                        glob_ = tail_glob % cycle_strs
                        if glob_ == tail_glob:  # no substitution
                            glob_ = os.path.join(cycle, tail_glob)
                        globs.append(os.path.join(head, glob_))
                else:
                    globs.append(os.path.join(head, cycle))
        return globs, cycle_set
예제 #5
0
    def _get_conf(self, app_runner, conf_tree, key, max_args=0):
        """Get a list of cycles from a configuration setting.

        key -- An option key in self.SECTION to locate the setting.
        max_args -- Maximum number of extra arguments for an item in the list.

        The value of the setting is expected to be split by shlex.split into a
        list of items. If max_args == 0, an item should be a string
        representing a cycle or an cycle offset. If max_args > 0, the cycle
        or cycle offset string can, optionally, have arguments. The arguments
        are delimited by colons ":".
        E.g.:

        prune-remote-logs-at=-PT6H -PT12H
        prune-server-logs-at=-P7D
        prune-datac-at=-PT6H:foo/* -PT12H:'bar/* baz/*' -P1D
        prune-work-at=-PT6H:t1*:*.tar -PT12H:t1*: -PT12H:*.gz -P1D

        If max_args == 0, return a list of cycles.
        If max_args > 0, return a list of (cycle, [arg, ...])

        """
        items_str = conf_tree.node.get_value([self.SECTION, key])
        if items_str is None:
            return []
        try:
            items_str = env_var_process(items_str)
        except UnboundEnvironmentVariableError as exc:
            raise ConfigValueError([self.SECTION, key], items_str, exc)
        items = []
        ref_point_str = os.getenv(RoseDateTimeOperator.TASK_CYCLE_TIME_ENV)
        try:
            ref_point = None
            ref_fmt = None
            for item_str in shlex.split(items_str):
                args = item_str.split(":", max_args)
                when = args.pop(0)
                cycle = when
                if ref_point_str is not None:
                    if self._get_cycling_mode() == "integer":
                        # Integer cycling
                        if "P" in when:  # "when" is an offset
                            cycle = str(
                                int(ref_point_str) +
                                int(when.replace("P", "")))
                        else:  # "when" is a cycle point
                            cycle = str(when)
                    else:
                        # Date-time cycling
                        if ref_fmt is None:
                            (
                                ref_point,
                                ref_fmt,
                            ) = app_runner.date_time_oper.date_parse(
                                ref_point_str)
                        try:
                            time_point = app_runner.date_time_oper.date_parse(
                                when)[0]
                        except ValueError:
                            time_point = app_runner.date_time_oper.date_shift(
                                ref_point, when)
                        cycle = app_runner.date_time_oper.date_format(
                            ref_fmt, time_point)
                if max_args:
                    items.append((cycle, args))
                else:
                    items.append(cycle)
        except ValueError as exc:
            raise ConfigValueError([self.SECTION, key], items_str, exc)
        return items
예제 #6
0
파일: rose_bunch.py 프로젝트: rolinzcy/rose
    def run(self, app_runner, conf_tree, opts, args, uuid, work_files):
        """ Run multiple instances of a command using sets of specified args"""

        # Counts for reporting purposes
        run_ok = 0
        run_fail = 0
        run_skip = 0
        notrun = 0

        # Allow naming of individual calls
        self.invocation_names = conf_tree.node.get_value([self.BUNCH_SECTION,
                                                         "names"])
        if self.invocation_names:
            self.invocation_names = shlex.split(
                metomi.rose.env.env_var_process(self.invocation_names))
            if len(set(self.invocation_names)) != len(self.invocation_names):
                raise ConfigValueError([self.BUNCH_SECTION, "names"],
                                       self.invocation_names,
                                       "names must be unique")

        self.fail_mode = metomi.rose.env.env_var_process(
            conf_tree.node.get_value(
                [self.BUNCH_SECTION, "fail-mode"], self.TYPE_CONTINUE_ON_FAIL))

        if self.fail_mode not in self.FAIL_MODE_TYPES:
            raise ConfigValueError([self.BUNCH_SECTION, "fail-mode"],
                                   self.fail_mode,
                                   "not a valid setting")

        self.incremental = conf_tree.node.get_value([self.BUNCH_SECTION,
                                                    "incremental"],
                                                    "true")
        if self.incremental:
            self.incremental = metomi.rose.env.env_var_process(
                self.incremental)

        self.isformatted = True
        self.command = metomi.rose.env.env_var_process(
            conf_tree.node.get_value([self.BUNCH_SECTION, "command-format"]))

        if not self.command:
            self.isformatted = False
            self.command = app_runner.get_command(conf_tree, opts, args)

        if not self.command:
            raise CommandNotDefinedError()

        # Set up command-instances if needed
        instances = conf_tree.node.get_value([self.BUNCH_SECTION,
                                              "command-instances"])

        if instances:
            try:
                instances = range(
                    int(metomi.rose.env.env_var_process(instances)))
            except ValueError:
                raise ConfigValueError([self.BUNCH_SECTION,
                                        "command-instances"],
                                       instances,
                                       "not an integer value")

        # Argument lists
        multi_args = conf_tree.node.get_value([self.ARGS_SECTION], {})
        bunch_args_names = []
        bunch_args_values = []
        for key, val in multi_args.items():
            bunch_args_names.append(key)
            bunch_args_values.append(
                shlex.split(metomi.rose.env.env_var_process(val.value)))

        # Update the argument values based on the argument-mode
        argument_mode = conf_tree.node.get_value([self.BUNCH_SECTION,
                                                  "argument-mode"],
                                                 self.DEFAULT_ARGUMENT_MODE)
        if argument_mode == self.DEFAULT_ARGUMENT_MODE:
            pass
        elif argument_mode in self.ACCEPTED_ARGUMENT_MODES:
            # The behaviour of of izip and izip_longest are special cases
            # because:
            # * izip was deprecated in Python3 use zip
            # * itertools.izip_longest was renamed and requires the fillvalue
            #     kwarg
            if argument_mode in ['zip', 'izip']:
                _permutations = zip(*bunch_args_values)
            elif argument_mode in ['zip_longest', 'izip_longest']:
                _permutations = itertools.zip_longest(*bunch_args_values,
                                                      fillvalue="")
            else:
                iteration_cmd = getattr(itertools, argument_mode)
                _permutations = iteration_cmd(*bunch_args_values)

            # Reconstruct the bunch_args_values
            _permutations = list(_permutations)
            for index, _ in enumerate(bunch_args_values):
                bunch_args_values[index] = [v[index] for v in _permutations]
        else:
            raise ConfigValueError([self.BUNCH_SECTION,
                                    "argument-mode"],
                                   argument_mode,
                                   "must be one of %s" %
                                   self.ACCEPTED_ARGUMENT_MODES)

        # Validate runlists
        if not self.invocation_names:
            if instances:
                arglength = len(instances)
            else:
                arglength = len(bunch_args_values[0])
            self.invocation_names = list(range(0, arglength))
        else:
            arglength = len(self.invocation_names)

        for item, vals in zip(bunch_args_names, bunch_args_values):
            if len(vals) != arglength:
                raise ConfigValueError([self.ARGS_SECTION, item],
                                       conf_tree.node.get_value(
                                       [self.ARGS_SECTION, item]),
                                       "inconsistent arg lengths")

        if conf_tree.node.get_value([self.ARGS_SECTION, "command-instances"]):
            raise ConfigValueError([self.ARGS_SECTION, "command-instances"],
                                   conf_tree.node.get_value(
                                   [self.ARGS_SECTION, "command-instances"]),
                                   "reserved keyword")

        if conf_tree.node.get_value([self.ARGS_SECTION, "COMMAND_INSTANCES"]):
            raise ConfigValueError([self.ARGS_SECTION, "COMMAND_INSTANCES"],
                                   conf_tree.node.get_value(
                                   [self.ARGS_SECTION, "COMMAND_INSTANCES"]),
                                   "reserved keyword")

        if instances and arglength != len(instances):
            raise ConfigValueError([self.BUNCH_SECTION, "command-instances"],
                                   instances, "inconsistent arg lengths")

        # Set max number of processes to run at once
        max_procs = conf_tree.node.get_value([self.BUNCH_SECTION, "pool-size"])

        if max_procs:
            max_procs = int(metomi.rose.env.env_var_process(max_procs))
        else:
            max_procs = arglength

        if self.incremental == "true":
            self.dao = RoseBunchDAO(conf_tree)
        else:
            self.dao = None

        commands = {}
        for vals in zip(range(arglength), self.invocation_names,
                        *bunch_args_values):
            index, name, bunch_args_vals = vals[0], vals[1], vals[2:]
            argsdict = dict(zip(bunch_args_names, bunch_args_vals))
            if instances:
                if self.isformatted:
                    argsdict["command-instances"] = instances[index]
                else:
                    argsdict["COMMAND_INSTANCES"] = str(instances[index])
            commands[name] = RoseBunchCmd(name, self.command, argsdict,
                                          self.isformatted)

        procs = {}
        if 'ROSE_TASK_LOG_DIR' in os.environ:
            log_format = os.path.join(os.environ['ROSE_TASK_LOG_DIR'], "%s")
        else:
            log_format = os.path.join(os.getcwd(), "%s")

        failed = {}
        abort = False

        while procs or (commands and not abort):
            for key, proc in list(procs.items()):
                if proc.poll() is not None:
                    procs.pop(key)
                    if proc.returncode:
                        failed[key] = proc.returncode
                        run_fail += 1
                        app_runner.handle_event(RosePopenError(str(key),
                                                proc.returncode,
                                                None, None))
                        if self.dao:
                            self.dao.update_command_state(key, self.dao.S_FAIL)
                        if self.fail_mode == self.TYPE_ABORT_ON_FAIL:
                            abort = True
                            app_runner.handle_event(AbortEvent())
                    else:
                        run_ok += 1
                        app_runner.handle_event(SucceededEvent(key),
                                                prefix=self.PREFIX_OK)
                        if self.dao:
                            self.dao.update_command_state(key, self.dao.S_PASS)

            while len(procs) < max_procs and commands and not abort:
                key = self.invocation_names[0]
                command = commands.pop(key)
                self.invocation_names.pop(0)
                cmd = command.get_command()
                cmd_stdout = log_format % command.get_out_file()
                cmd_stderr = log_format % command.get_err_file()
                prefix = command.get_log_prefix()
                bunch_environ = os.environ
                if not command.isformatted:
                    bunch_environ.update(command.argsdict)
                bunch_environ['ROSE_BUNCH_LOG_PREFIX'] = prefix

                if self.dao:
                    if self.dao.check_has_succeeded(key):
                        run_skip += 1
                        app_runner.handle_event(PreviousSuccessEvent(key),
                                                prefix=self.PREFIX_PASS)
                        continue
                    else:
                        self.dao.add_command(key)

                app_runner.handle_event(LaunchEvent(key, cmd))
                procs[key] = app_runner.popen.run_bg(
                    cmd,
                    shell=True,
                    stdout=open(cmd_stdout, 'w'),
                    stderr=open(cmd_stderr, 'w'),
                    env=bunch_environ)

            sleep(self.SLEEP_DURATION)

        if abort and commands:
            for key in self.invocation_names:
                notrun += 1
                cmd = commands.pop(key).get_command()
                app_runner.handle_event(NotRunEvent(key, cmd),
                                        prefix=self.PREFIX_NOTRUN)

        if self.dao:
            self.dao.close()

        # Report summary data in job.out file
        app_runner.handle_event(SummaryEvent(
                                run_ok, run_fail, run_skip, notrun))

        if failed:
            return 1
        else:
            return 0
예제 #7
0
 def _run_target_setup(self,
                       app_runner,
                       compress_manager,
                       config,
                       t_key,
                       s_key_tail,
                       t_node,
                       is_compulsory_target=True):
     """Helper for _run. Set up a target."""
     target_prefix = self._get_conf(config,
                                    t_node,
                                    "target-prefix",
                                    default="")
     target = RoseArchTarget(target_prefix + s_key_tail)
     target.command_format = self._get_conf(config,
                                            t_node,
                                            "command-format",
                                            compulsory=True)
     try:
         target.command_format % {"sources": "", "target": ""}
     except KeyError as exc:
         target.status = target.ST_BAD
         app_runner.handle_event(
             RoseArchValueError(target.name, "command-format",
                                target.command_format,
                                type(exc).__name__, exc))
     target.source_edit_format = self._get_conf(config,
                                                t_node,
                                                "source-edit-format",
                                                default="")
     try:
         target.source_edit_format % {"in": "", "out": ""}
     except KeyError as exc:
         target.status = target.ST_BAD
         app_runner.handle_event(
             RoseArchValueError(target.name, "source-edit-format",
                                target.source_edit_format,
                                type(exc).__name__, exc))
     update_check_str = self._get_conf(config, t_node, "update-check")
     try:
         checksum_func = get_checksum_func(update_check_str)
     except ValueError as exc:
         raise RoseArchValueError(target.name, "update-check",
                                  update_check_str,
                                  type(exc).__name__, exc)
     source_prefix = self._get_conf(config,
                                    t_node,
                                    "source-prefix",
                                    default="")
     for source_glob in shlex.split(
             self._get_conf(config, t_node, "source", compulsory=True)):
         is_compulsory_source = is_compulsory_target
         if source_glob.startswith("(") and source_glob.endswith(")"):
             source_glob = source_glob[1:-1]
             is_compulsory_source = False
         paths = glob(source_prefix + source_glob)
         if not paths:
             exc = OSError(errno.ENOENT, os.strerror(errno.ENOENT),
                           source_prefix + source_glob)
             app_runner.handle_event(
                 ConfigValueError([t_key, "source"], source_glob, exc))
             if is_compulsory_source:
                 target.status = target.ST_BAD
             continue
         for path in paths:
             # N.B. source_prefix may not be a directory
             name = path[len(source_prefix):]
             for path_, checksum, _ in get_checksum(path, checksum_func):
                 if checksum is None:  # is directory
                     continue
                 if path_:
                     target.sources[checksum] = RoseArchSource(
                         checksum, os.path.join(name, path_),
                         os.path.join(path, path_))
                 else:  # path is a file
                     target.sources[checksum] = RoseArchSource(
                         checksum, name, path)
     if not target.sources:
         if is_compulsory_target:
             target.status = target.ST_BAD
         else:
             target.status = target.ST_NULL
     target.compress_scheme = self._get_conf(config, t_node, "compress")
     if not target.compress_scheme:
         target_base = target.name
         if "/" in target.name:
             target_base = target.name.rsplit("/", 1)[1]
         if "." in target_base:
             tail = target_base.split(".", 1)[1]
             if compress_manager.get_handler(tail):
                 target.compress_scheme = tail
     elif compress_manager.get_handler(target.compress_scheme) is None:
         app_runner.handle_event(
             ConfigValueError([t_key, "compress"], target.compress_scheme,
                              KeyError(target.compress_scheme)))
         target.status = target.ST_BAD
     rename_format = self._get_conf(config, t_node, "rename-format")
     if rename_format:
         rename_parser_str = self._get_conf(config, t_node, "rename-parser")
         if rename_parser_str:
             try:
                 rename_parser = re.compile(rename_parser_str)
             except re.error as exc:
                 raise RoseArchValueError(target.name, "rename-parser",
                                          rename_parser_str,
                                          type(exc).__name__, exc)
         else:
             rename_parser = None
         for source in target.sources.values():
             dict_ = {
                 "cycle": os.getenv("ROSE_TASK_CYCLE_TIME"),
                 "name": source.name
             }
             if rename_parser:
                 match = rename_parser.match(source.name)
                 if match:
                     dict_.update(match.groupdict())
             try:
                 source.name = rename_format % dict_
             except (KeyError, ValueError) as exc:
                 raise RoseArchValueError(target.name, "rename-format",
                                          rename_format,
                                          type(exc).__name__, exc)
     return target