Exemplo n.º 1
0
 def process(self,
             conf_tree,
             item,
             orig_keys=None,
             orig_value=None,
             **kwargs):
     """Export environment variables in an [env] in "conf_tree.node"."""
     env_node = conf_tree.node.get([item], no_ignore=True)
     if env_node is None:
         return
     if "UNDEF" in os.environ:
         os.environ.pop("UNDEF")
     environ = {}
     if env_node and not env_node.state:
         for key, node in env_node.value.items():
             if node.state:
                 continue
             try:
                 environ[key] = env_var_process(node.value)
             except UnboundEnvironmentVariableError as exc:
                 raise ConfigProcessError([item, key], node.value, exc)
             environ[key] = os.path.expanduser(environ[key])  # ~ expansion
     for key, value in sorted(environ.items()):
         env_export(key, value, self.manager.event_handler)
     return environ
Exemplo n.º 2
0
    def process(
        self, conf_tree, item, orig_keys=None, orig_value=None, **kwargs
    ):
        """Install files according to [file:*] in conf_tree.

        kwargs["no_overwrite_mode"]: fail if a target file already exists.

        """

        # Find all the "file:*" nodes.
        nodes = {}
        if item == self.SCHEME:
            for key, node in list(conf_tree.node.value.items()):
                if node.is_ignored() or not key.startswith(self.PREFIX):
                    continue
                nodes[key] = node
        else:
            node = conf_tree.node.get([item], no_ignore=True)
            if node is None:
                raise ConfigProcessError(orig_keys, item)
            nodes[item] = node

        if not nodes:
            return

        # Create database to store information for incremental updates,
        # if it does not already exist.
        loc_dao = LocDAO()
        loc_dao.create()

        cwd = os.getcwd()
        file_install_root = conf_tree.node.get_value(
            ["file-install-root"], os.getenv("ROSE_FILE_INSTALL_ROOT", None)
        )
        if file_install_root:
            file_install_root = env_var_process(file_install_root)
            self.manager.fs_util.makedirs(file_install_root)
            self.manager.fs_util.chdir(file_install_root)
        try:
            self._process(conf_tree, nodes, loc_dao, **kwargs)
        finally:
            if cwd != os.getcwd():
                self.manager.fs_util.chdir(cwd)
Exemplo n.º 3
0
 async def pull(self, loc, conf_tree):
     """Write namelist to loc.cache."""
     sections = self.parse(loc, conf_tree)
     if loc.name.endswith("(:)"):
         sections.sort(key=cmp_to_key(metomi.rose.config.sort_settings))
     with open(loc.cache, "wb") as handle:
         for section in sections:
             section_value = conf_tree.node.get_value([section])
             group = RE_NAMELIST_GROUP.match(section).group(1)
             nlg = "&" + group + "\n"
             for key, node in sorted(section_value.items()):
                 if node.state:
                     continue
                 try:
                     value = env_var_process(node.value)
                 except UnboundEnvironmentVariableError as exc:
                     raise ConfigProcessError([section, key], node.value,
                                              exc)
                 nlg += "%s=%s,\n" % (key, value)
             nlg += "/" + "\n"
             handle.write(nlg.encode('UTF-8'))
             self.manager.handle_event(NamelistEvent(nlg))
Exemplo n.º 4
0
def get_rose_vars_from_config_node(config, config_node, environ):
    """Load template variables from a Rose config node.

    This uses only the provided config node and environment variables
    - there is no system interaction.

    Args:
        config (dict):
            Object which will be populated with the results.
        config_node (metomi.rose.config.ConfigNode):
            Configuration node representing the Rose suite configuration.
        environ (dict):
            Dictionary of environment variables

    """
    templating = None

    # Don't allow multiple templating sections.
    templating = identify_templating_section(config_node)

    if templating != 'template variables':
        config['templating_detected'] = templating.replace(':suite.rc', '')
    else:
        config['templating_detected'] = templating

    # Create env section if it doesn't already exist.
    if 'env' not in config_node.value:
        config_node.set(['env'])
    if templating not in config_node.value:
        config_node.set([templating])

    # Get Rose Orig host:
    rose_orig_host = get_host()

    # For each section process variables and add standard variables.
    for section in ['env', templating]:

        # This loop handles standard variables.
        # CYLC_VERSION - If it's in the config, remove it.
        # ROSE_VERSION - If it's in the config, replace it.
        # ROSE_ORIG_HOST - If it's the config, replace it, unless it has a
        # comment marking it as having been saved by ``cylc install``.
        # In all cases warn users if the value in their config is not used.
        for var_name, replace_with in [('ROSE_ORIG_HOST', rose_orig_host),
                                       ('ROSE_VERSION', ROSE_VERSION),
                                       ('CYLC_VERSION', SET_BY_CYLC)]:
            # Warn if we're we're going to override a variable:
            if override_this_variable(config_node, section, var_name):
                user_var = config_node[section].value[var_name].value
                LOG.warning(
                    f'[{section}]{var_name}={user_var} from rose-suite.conf '
                    f'will be ignored: {var_name} will be: {replace_with}')

            # Handle replacement of stored variable if appropriate:
            if replace_with == SET_BY_CYLC:
                config_node[section].unset([var_name])
            elif not rose_orig_host_set_by_cylc_install(
                    config_node, section, var_name):
                config_node[section].set([var_name], replace_with)

        # Use env_var_process to process variables which may need expanding.
        for key, node in config_node.value[section].value.items():
            try:
                config_node.value[section].value[key].value = env_var_process(
                    node.value, environ=environ)
                if section == 'env':
                    environ[key] = node.value
            except UnboundEnvironmentVariableError as exc:
                raise ConfigProcessError(['env', key], node.value, exc)

    # For each of the template language sections extract items to a simple
    # dict to be returned.
    if 'env' in config_node.value:
        config['env'] = {
            item[0][1]: item[1].value
            for item in config_node.value['env'].walk()
        }
    if templating in config_node.value:
        config['template_variables'] = {
            item[0][1]: item[1].value
            for item in config_node.value[templating].walk()
        }
    elif 'template variables' in config_node.value:
        config['template_variables'] = {
            item[0][1]: item[1].value
            for item in config_node.value['template variables'].walk()
        }

    # Add the entire config to ROSE_SUITE_VARIABLES to allow for programatic
    # access.
    if templating is not None:
        with patch_jinja2_leading_zeros():
            # BACK COMPAT: patch_jinja2_leading_zeros
            # back support zero-padded integers for a limited time to help
            # users migrate before upgrading cylc-flow to Jinja2>=3.1
            parser = Parser()
            for key, value in config['template_variables'].items():
                # The special variables are already Python variables.
                if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']:
                    try:
                        config['template_variables'][key] = (
                            parser.literal_eval(value))
                    except Exception:
                        raise ConfigProcessError(
                            [templating, key], value,
                            f'Invalid template variable: {value}'
                            '\nMust be a valid Python or Jinja2 literal'
                            ' (note strings "must be quoted").') from None

    # Add ROSE_SUITE_VARIABLES to config of templating engines in use.
    if templating is not None:
        config['template_variables']['ROSE_SUITE_VARIABLES'] = config[
            'template_variables']
Exemplo n.º 5
0
    def process(self,
                conf_tree,
                item,
                orig_keys=None,
                orig_value=None,
                **kwargs):
        """Process [jinja2:*] in "conf_tree.node".

        Arguments:
            conf_tree:
                The relevant metomi.rose.config_tree.ConfigTree object with the
                full configuration.
            item: The current configuration item to process.
            orig_keys:
                The keys for locating the originating setting in conf_tree in a
                recursive processing. None implies a top level call.
            orig_value: The value of orig_keys in conf_tree.
            **kwargs:
                environ (dict): suite level environment variables.
        """
        for s_key, s_node in sorted(conf_tree.node.value.items()):
            if (s_node.is_ignored() or not s_key.startswith(self.PREFIX)
                    or not s_node.value):
                continue
            target = s_key[len(self.PREFIX):]
            source = os.path.join(conf_tree.files[target], target)
            if not os.access(source, os.F_OK | os.R_OK):
                continue
            scheme_ln = self.SCHEME_TEMPL % self.SCHEME
            msg_init_ln = self.COMMENT_TEMPL % self.MSG_INIT
            msg_done_ln = self.COMMENT_TEMPL % self.MSG_DONE
            tmp_file = NamedTemporaryFile()
            tmp_file.write(scheme_ln.encode('UTF-8'))
            tmp_file.write(msg_init_ln.encode('UTF-8'))
            suite_variables = ['{']
            for key, node in sorted(s_node.value.items()):
                if node.is_ignored():
                    continue
                try:
                    value = env_var_process(node.value)
                except UnboundEnvironmentVariableError as exc:
                    raise ConfigProcessError([s_key, key], node.value, exc)
                tmp_file.write(
                    (self.ASSIGN_TEMPL % (key, value)).encode('UTF-8'))
                suite_variables.append("    '%s': %s," % (key, key))
            suite_variables.append('}')
            suite_variables = self.ASSIGN_TEMPL % ('ROSE_SUITE_VARIABLES',
                                                   '\n'.join(suite_variables))
            tmp_file.write(suite_variables.encode('UTF-8'))
            environ = kwargs.get("environ")
            if environ:
                tmp_file.write('[cylc]\n'.encode('UTF-8'))
                tmp_file.write('    [[environment]]\n'.encode('UTF-8'))
                for key, value in sorted(environ.items()):
                    tmp_file.write(
                        ('        %s=%s\n' % (key, value)).encode('UTF-8'))
            tmp_file.write(msg_done_ln.encode('UTF-8'))
            line_n = 0
            is_in_old_insert = False
            for line in open(source):
                line_n += 1
                if line_n == 1 and line.strip().lower() == scheme_ln.strip():
                    continue
                elif line_n == 2 and line == msg_init_ln:
                    is_in_old_insert = True
                    continue
                elif is_in_old_insert and line == msg_done_ln:
                    is_in_old_insert = False
                    continue
                elif is_in_old_insert:
                    continue
                tmp_file.write(line.encode('UTF-8'))
            tmp_file.seek(0)
            if os.access(target, os.F_OK | os.R_OK):
                if filecmp.cmp(target, tmp_file.name):  # identical
                    tmp_file.close()
                    continue
                else:
                    self.manager.fs_util.delete(target)
            # Write content to target
            target_file = open(target, "w")
            for line in tmp_file:
                try:
                    target_file.write(line)
                except TypeError:
                    target_file.write(line.decode())
            event = FileSystemEvent(FileSystemEvent.INSTALL, target)
            self.manager.handle_event(event)
            tmp_file.close()
Exemplo n.º 6
0
    def _process(self, conf_tree, nodes, loc_dao, **kwargs):
        """Helper for self.process."""
        # Ensure that everything is overwritable
        # Ensure that container directories exist
        for key, node in sorted(nodes.items()):
            try:
                name = env_var_process(key[len(self.PREFIX) :])
            except UnboundEnvironmentVariableError as exc:
                raise ConfigProcessError([key], key, exc)
            if os.path.exists(name) and kwargs.get("no_overwrite_mode"):
                raise ConfigProcessError([key], None, FileOverwriteError(name))
            self.manager.fs_util.makedirs(self.manager.fs_util.dirname(name))
        # Gets a list of sources and targets
        sources = {}
        targets = {}
        for key, node in sorted(nodes.items()):
            # N.B. no need to catch UnboundEnvironmentVariableError here
            #      because any exception should been caught earlier.
            name = env_var_process(key[len(self.PREFIX) :])
            targets[name] = Loc(name)
            targets[name].action_key = Loc.A_INSTALL
            targets[name].mode = node.get_value(["mode"])
            if targets[name].mode and targets[name].mode not in Loc.MODES:
                raise ConfigProcessError([key, "mode"], targets[name].mode)
            target_sources = []
            for k in ["content", "source"]:  # "content" for back compat
                source_str = node.get_value([k])
                if source_str is None:
                    continue
                try:
                    source_str = env_var_process(source_str)
                except UnboundEnvironmentVariableError as exc:
                    raise ConfigProcessError([key, k], source_str, exc)
                source_names = []
                for raw_source_glob in shlex.split(source_str):
                    source_glob = raw_source_glob
                    if raw_source_glob.startswith(
                        "("
                    ) and raw_source_glob.endswith(")"):
                        source_glob = raw_source_glob[1:-1]
                    names = glob(source_glob)
                    if names:
                        source_names += sorted(names)
                    else:
                        source_names.append(raw_source_glob)
                for raw_source_name in source_names:
                    source_name = raw_source_name
                    is_optional = raw_source_name.startswith(
                        "("
                    ) and raw_source_name.endswith(")")
                    if is_optional:
                        source_name = raw_source_name[1:-1]
                    if source_name.startswith("~"):
                        source_name = os.path.expanduser(source_name)
                    if targets[name].mode in [
                        targets[name].MODE_SYMLINK,
                        targets[name].MODE_SYMLINK_PLUS,
                    ]:
                        if targets[name].real_name:
                            # Symlink mode can only have 1 source
                            raise ConfigProcessError([key, k], source_str)
                        targets[name].real_name = source_name
                    else:
                        if source_name not in sources:
                            sources[source_name] = Loc(source_name)
                            sources[source_name].action_key = Loc.A_SOURCE
                            sources[source_name].is_optional = is_optional
                        sources[source_name].used_by_names.append(name)
                        target_sources.append(sources[source_name])
            targets[name].dep_locs = target_sources
            if not targets[name].real_name and targets[name].mode in [
                targets[name].MODE_SYMLINK,
                targets[name].MODE_SYMLINK_PLUS,
            ]:
                raise ConfigProcessError([key, "source"], None)

        # Determine the scheme of the location from configuration.
        config_schemes_str = conf_tree.node.get_value(["schemes"])
        config_schemes = []  # [(pattern, scheme), ...]
        if config_schemes_str:
            for line in config_schemes_str.splitlines():
                pattern, scheme = line.split("=", 1)
                pattern = pattern.strip()
                scheme = scheme.strip()
                config_schemes.append((pattern, scheme))

        # Where applicable, determine for each source:
        # * Its real name.
        # * The checksums of its paths.
        # * Whether it can be considered unchanged.
        for source in list(sources.values()):
            try:
                for pattern, scheme in config_schemes:
                    if fnmatch(source.name, pattern):
                        source.scheme = scheme
                        break
                self.loc_handlers_manager.parse(source, conf_tree)
            except ValueError:
                if source.is_optional:
                    sources.pop(source.name)
                    for name in source.used_by_names:
                        targets[name].dep_locs.remove(source)
                        event = SourceSkipEvent(name, source.name)
                        self.handle_event(event)
                    continue
                else:
                    raise ConfigProcessError(
                        ["file:" + source.used_by_names[0], "source"],
                        source.name,
                    )
            prev_source = loc_dao.select(source.name)
            source.is_out_of_date = (
                not prev_source
                or (not source.key and not source.paths)
                or prev_source.scheme != source.scheme
                or prev_source.loc_type != source.loc_type
                or prev_source.key != source.key
                or sorted(prev_source.paths) != sorted(source.paths)
            )

        # Inspect each target to see if it is out of date:
        # * Target does not already exist.
        # * Target exists, but does not have a database entry.
        # * Target exists, but does not match settings in database.
        # * Target exists, but a source cannot be considered unchanged.
        for target in list(targets.values()):
            if target.real_name:
                target.is_out_of_date = not os.path.islink(
                    target.name
                ) or target.real_name != os.readlink(target.name)
            elif target.mode == target.MODE_MKDIR:
                target.is_out_of_date = os.path.islink(
                    target.name
                ) or not os.path.isdir(target.name)
            else:
                if os.path.exists(target.name) and not os.path.islink(
                    target.name
                ):
                    for path, checksum, access_mode in get_checksum(
                        target.name
                    ):
                        target.add_path(path, checksum, access_mode)
                    target.paths.sort()
                prev_target = loc_dao.select(target.name)
                target.is_out_of_date = (
                    os.path.islink(target.name)
                    or not os.path.exists(target.name)
                    or prev_target is None
                    or prev_target.mode != target.mode
                    or len(prev_target.paths) != len(target.paths)
                )
                if not target.is_out_of_date:
                    prev_target.paths.sort()
                    for prev_path, path in zip(
                        prev_target.paths, target.paths
                    ):
                        if prev_path != path:
                            target.is_out_of_date = True
                            break
                # See if any sources have changed names.
                if not target.is_out_of_date:
                    conn = loc_dao.get_conn()
                    prev_dep_locs = conn.execute(
                        """
                            SELECT *
                            FROM dep_names
                            WHERE name=?
                            ORDER BY ROWID
                        """,
                        [target.name],
                    ).fetchall()
                    prev_dep_locs = [i[1] for i in prev_dep_locs]
                    prev_dep_locs = [loc_dao.select(i) for i in prev_dep_locs]
                    if [i.name for i in prev_dep_locs] != [
                        i.name for i in target.dep_locs
                    ]:
                        target.is_out_of_date = True
                # See if any sources out of date
                if not target.is_out_of_date:
                    for dep_loc in target.dep_locs:
                        if dep_loc.is_out_of_date:
                            target.is_out_of_date = True
                            break
            if target.is_out_of_date:
                target.paths = None
                loc_dao.delete_locs.append(target)

        # Set up jobs for rebuilding all out-of-date targets.
        jobs = {}
        for name, target in sorted(targets.items()):
            if not target.is_out_of_date:
                self.handle_event(FileUnchangedEvent(target, level=Event.V))
                continue
            if target.mode in [target.MODE_SYMLINK, target.MODE_SYMLINK_PLUS]:
                if target.mode == target.MODE_SYMLINK_PLUS:
                    try:
                        os.stat(target.real_name)
                    except OSError as exc:
                        raise ConfigProcessError(
                            [self.PREFIX + target.name, "source"],
                            target.real_name,
                            exc,
                        )
                self.manager.fs_util.symlink(target.real_name, target.name)
                loc_dao.update_locs.append(target)
            elif target.mode == target.MODE_MKDIR:
                if os.path.islink(target.name):
                    self.manager.fs_util.delete(target.name)
                self.manager.fs_util.makedirs(target.name)
                loc_dao.update_locs.append(target)
                target.loc_type = target.TYPE_TREE
                target.add_path(target.BLOB, None, None)
            elif target.dep_locs:
                if os.path.islink(target.name):
                    self.manager.fs_util.delete(target.name)
                jobs[target.name] = JobProxy(target)
                for source in target.dep_locs:
                    if source.name not in jobs:
                        jobs[source.name] = JobProxy(source)
                        jobs[source.name].event_level = Event.V
                    job = jobs[source.name]
                    jobs[target.name].pending_for[source.name] = job
                p_name = target.name
                while (
                    os.path.dirname(p_name)
                    and os.path.dirname(p_name) != p_name
                ):
                    p_name = os.path.dirname(p_name)
                    if p_name in jobs:
                        jobs[target.name].pending_for[p_name] = jobs[p_name]
            else:
                self.manager.fs_util.install(target.name)
                target.loc_type = target.TYPE_BLOB
                for path, checksum, access_mode in get_checksum(target.name):
                    target.add_path(path, checksum, access_mode)
                loc_dao.update_locs.append(target)
        loc_dao.execute_queued_items()

        # If relevant, use job runner to get sources and build targets
        if jobs:
            work_dir = mkdtemp()
            try:
                nproc_keys = ["rose.config_processors.fileinstall", "nproc"]
                nproc_str = conf_tree.node.get_value(nproc_keys)
                nproc = None
                if nproc_str is not None:
                    nproc = int(nproc_str)
                job_runner = JobRunner(self, nproc)
                job_runner(JobManager(jobs), conf_tree, loc_dao, work_dir)
            except ValueError as exc:
                if exc.args and exc.args[0] in jobs:
                    job = jobs[exc.args[0]]
                    if job.context.action_key == Loc.A_SOURCE:
                        source = job.context
                        keys = [
                            self.PREFIX + source.used_by_names[0],
                            "source",
                        ]
                        raise ConfigProcessError(keys, source.name)
                raise exc
            finally:
                loc_dao.execute_queued_items()
                rmtree(work_dir)

        # Target checksum compare and report
        for target in list(targets.values()):
            if (
                not target.is_out_of_date
                or target.loc_type == target.TYPE_TREE
            ):
                continue
            keys = [self.PREFIX + target.name, "checksum"]
            checksum_expected = conf_tree.node.get_value(keys)
            if checksum_expected is None:
                continue
            checksum = target.paths[0].checksum
            if checksum_expected:
                if len(checksum_expected) != len(checksum):
                    algorithm = guess_checksum_algorithm(checksum_expected)
                    if algorithm:
                        checksum = get_checksum_func(algorithm)(target.name)
                if checksum_expected != checksum:
                    raise ConfigProcessError(
                        keys,
                        checksum_expected,
                        ChecksumError(checksum_expected, checksum),
                    )
            event = ChecksumEvent(target.name, target.paths[0].checksum)
            self.handle_event(event)
Exemplo n.º 7
0
def get_rose_vars_from_config_node(config, config_node, environ):
    """Load template variables from a Rose config node.

    This uses only the provided config node and environment variables
    - there is no system interaction.

    Args:
        config (dict):
            Object which will be populated with the results.
        config_node (metomi.rose.config.ConfigNode):
            Configuration node representing the Rose suite configuration.
        environ (dict):
            Dictionary of environment variables

    """
    templating = None
    sections = {'jinja2:suite.rc', 'empy:suite.rc', 'template variables'}

    # Don't allow multiple templating sections.
    defined_sections = sections.intersection(set(config_node.value))
    if len(defined_sections) > 1:
        raise MultipleTemplatingEnginesError(
            "You should not define more than one templating section. "
            f"You defined:\n\t{'; '.join(defined_sections)}")
    elif len(defined_sections) == 1:
        templating, = defined_sections
        if templating != 'template variables':
            config['templating_detected'] = templating.replace(':suite.rc', '')
        else:
            config['templating_detected'] = templating

    # Create env section if it doesn't already exist.
    if 'env' not in config_node.value:
        config_node.set(['env'])

    # Get Values for standard ROSE variables (ROSE_ORIG_HOST and ROSE_SITE).
    rose_orig_host = get_host()
    rose_site = ResourceLocator().get_conf().get_value(['site'], '')

    # For each section process variables and add standard variables.
    for section in ['env', templating]:
        if section not in config_node.value:
            continue

        # Add standard ROSE_VARIABLES
        config_node[section].set(['ROSE_SITE'], rose_site)
        config_node[section].set(['ROSE_VERSION'], ROSE_VERSION)
        config_node[section].set(['ROSE_ORIG_HOST'], rose_orig_host)

        # Use env_var_process to process variables which may need expanding.
        for key, node in config_node.value[section].value.items():
            try:
                config_node.value[section].value[key].value = env_var_process(
                    node.value, environ=environ)
                if section == 'env':
                    environ[key] = node.value
            except UnboundEnvironmentVariableError as exc:
                raise ConfigProcessError(['env', key], node.value, exc)

    # For each of the template language sections extract items to a simple
    # dict to be returned.
    if 'env' in config_node.value:
        config['env'] = {
            item[0][1]: item[1].value
            for item in config_node.value['env'].walk()
        }
    if templating in config_node.value:
        config['template_variables'] = {
            item[0][1]: item[1].value
            for item in config_node.value[templating].walk()
        }
    elif 'template variables' in config_node.value:
        config['template_variables'] = {
            item[0][1]: item[1].value
            for item in config_node.value['template variables'].walk()
        }

    # Add the entire config to ROSE_SUITE_VARIABLES to allow for programatic
    # access.
    if templating is not None:
        parser = Parser()
        for key, value in config['template_variables'].items():
            # The special variables are already Python variables.
            if key not in ['ROSE_ORIG_HOST', 'ROSE_VERSION', 'ROSE_SITE']:
                try:
                    config['template_variables'][key] = (
                        parser.literal_eval(value))
                except Exception:
                    raise ConfigProcessError(
                        [templating, key], value,
                        f'Invalid template variable: {value}'
                        '\nMust be a valid Python or Jinja2 literal'
                        ' (note strings "must be quoted").') from None

    # Add ROSE_SUITE_VARIABLES to config of templating engines in use.
    if templating is not None:
        config['template_variables']['ROSE_SUITE_VARIABLES'] = config[
            'template_variables']