Example #1
0
    def test_sorted_scan(self):
        path = os.path.join(self.dir, "sorted_scan")
        os.mkdir(path)
        files = [
            os.path.normpath(os.path.join(path, x))
            for x in ["tmp", "blah", "dar"]
        ]
        # cheap version of a touch.
        for x in files:
            open(x, "w").close()
        dirs = [
            os.path.normpath(os.path.join(path, x)) for x in ["a", "b", "c"]
        ]
        for x in dirs:
            os.mkdir(x)

        # regular directory scanning
        sorted_files = livefs.sorted_scan(path)
        self.assertEqual(
            list([pjoin(path, x) for x in ['blah', 'dar', 'tmp']]),
            sorted_files)

        # nonexistent paths
        nonexistent_path = os.path.join(self.dir, 'foobar')
        sorted_files = livefs.sorted_scan(nonexistent_path)
        self.assertEqual(sorted_files, [])
        sorted_files = livefs.sorted_scan(nonexistent_path, nonexistent=True)
        self.assertEqual(sorted_files, [nonexistent_path])
Example #2
0
    def test_sorted_scan(self):
        path = os.path.join(self.dir, "sorted_scan")
        os.mkdir(path)
        files = [os.path.normpath(os.path.join(path, x)) for x in
                 ["tmp", "blah", "dar"]]
        # cheap version of a touch.
        for x in files:
            open(x, "w").close()
        dirs = [os.path.normpath(os.path.join(path, x)) for x in [
                "a", "b", "c"]]
        for x in dirs:
            os.mkdir(x)

        # regular directory scanning
        sorted_files = livefs.sorted_scan(path)
        self.assertEqual(
            list([pjoin(path, x) for x in ['blah', 'dar', 'tmp']]),
            sorted_files)

        # nonexistent paths
        nonexistent_path = os.path.join(self.dir, 'foobar')
        sorted_files = livefs.sorted_scan(nonexistent_path)
        self.assertEqual(sorted_files, [])
        sorted_files = livefs.sorted_scan(nonexistent_path, nonexistent=True)
        self.assertEqual(sorted_files, [nonexistent_path])
Example #3
0
 def test_sorted_scan_backup(self):
     path = os.path.join(self.dir, "sorted_scan")
     os.mkdir(path)
     files = [os.path.normpath(os.path.join(path, x)) for x in ["blah", "blah~"]]
     # cheap version of a touch.
     for x in files:
         open(x, "w").close()
     sorted_files = livefs.sorted_scan(path)
     assert list([pjoin(path, x) for x in ['blah', 'blah~']]) == sorted_files
     sorted_files = livefs.sorted_scan(path, backup=False)
     assert list([pjoin(path, x) for x in ['blah']]) == sorted_files
Example #4
0
 def test_sorted_scan_backup(self):
     path = os.path.join(self.dir, "sorted_scan")
     os.mkdir(path)
     files = [
         os.path.normpath(os.path.join(path, x)) for x in ["blah", "blah~"]
     ]
     # cheap version of a touch.
     for x in files:
         open(x, "w").close()
     sorted_files = livefs.sorted_scan(path)
     assert list([pjoin(path, x)
                  for x in ['blah', 'blah~']]) == sorted_files
     sorted_files = livefs.sorted_scan(path, backup=False)
     assert list([pjoin(path, x) for x in ['blah']]) == sorted_files
Example #5
0
def load_make_conf(vars_dict, path, allow_sourcing=False, required=True,
                   incrementals=False):
    """parse make.conf files

    Args:
        vars_dict (dict): dictionary to add parsed variables to
        path (str): path to the make.conf which can be a regular file or
            directory, if a directory is passed all the non-hidden files within
            that directory are parsed in alphabetical order.
    """
    sourcing_command = None
    if allow_sourcing:
        sourcing_command = 'source'

    for fp in sorted_scan(os.path.realpath(path), follow_symlinks=True, nonexistent=True):
        try:
            new_vars = read_bash_dict(
                fp, vars_dict=vars_dict, sourcing_command=sourcing_command)
        except EnvironmentError as e:
            if e.errno == errno.EACCES:
                raise_from(errors.PermissionDeniedError(fp, write=False))
            if e.errno != errno.ENOENT or required:
                raise_from(errors.ParsingError("parsing %r" % (fp,), exception=e))
            return

        if incrementals:
            for key in econst.incrementals:
                if key in vars_dict and key in new_vars:
                    new_vars[key] = "%s %s" % (vars_dict[key], new_vars[key])
        # quirk of read_bash_dict; it returns only what was mutated.
        vars_dict.update(new_vars)
Example #6
0
def load_make_conf(vars_dict, path, allow_sourcing=False, required=True,
                   incrementals=False):
    """parse make.conf files

    Args:
        vars_dict (dict): dictionary to add parsed variables to
        path (str): path to the make.conf which can be a regular file or
            directory, if a directory is passed all the non-hidden files within
            that directory are parsed in alphabetical order.
    """
    sourcing_command = None
    if allow_sourcing:
        sourcing_command = 'source'

    for fp in sorted_scan(os.path.realpath(path), follow_symlinks=True, nonexistent=True):
        try:
            new_vars = read_bash_dict(
                fp, vars_dict=vars_dict, sourcing_command=sourcing_command)
        except EnvironmentError as e:
            if e.errno == errno.EACCES:
                raise_from(errors.PermissionDeniedError(fp, write=False))
            if e.errno != errno.ENOENT or required:
                raise_from(errors.ParsingError("parsing %r" % (fp,), exception=e))
            return

        if incrementals:
            for key in econst.incrementals:
                if key in vars_dict and key in new_vars:
                    new_vars[key] = "%s %s" % (vars_dict[key], new_vars[key])
        # quirk of read_bash_dict; it returns only what was mutated.
        vars_dict.update(new_vars)
Example #7
0
def _load_and_invoke(func, filename, read_func, fallback, allow_recurse,
                     allow_line_cont, parse_func, eapi_optional, self):
    if eapi_optional is not None and not getattr(self.eapi.options, eapi_optional, None):
        return func(self, fallback)

    profile_path = self.path.rstrip('/')
    base = pjoin(profile_path, filename)

    files = []
    if self.pms_strict or not allow_recurse:
        if os.path.exists(base):
            files.append(base)
    else:
        # Skip hidden files and backup files, those beginning with '.' or
        # ending with '~', respectively.
        files.extend(sorted_scan(base, hidden=False, backup=False))

    try:
        if files:
            if read_func is None:
                data = parse_func(files)
            else:
                data = parse_func(read_func(
                    files, allow_line_cont=allow_line_cont))
        else:
            data = fallback
        return func(self, data)
    except (ValueError, IndexError, EnvironmentError) as e:
        raise ProfileError(profile_path, filename, e) from e
    except IsADirectoryError as e:
        raise ProfileError(
            self.path, filename,
            "path is a directory, but this profile is PMS format- "
            "directories aren't allowed. See layout.conf profile-formats "
            "to enable directory support") from e
Example #8
0
    def load_make_conf(vars_dict,
                       path,
                       allow_sourcing=False,
                       required=True,
                       allow_recurse=True,
                       incrementals=False):
        """parse make.conf files

        Args:
            vars_dict (dict): dictionary to add parsed variables to
            path (str): path to the make.conf which can be a regular file or
                directory, if a directory is passed all the non-hidden files within
                that directory are parsed in alphabetical order.
        """
        sourcing_command = 'source' if allow_sourcing else None

        if allow_recurse:
            files = sorted_scan(os.path.realpath(path),
                                follow_symlinks=True,
                                nonexistent=True,
                                hidden=False,
                                backup=False)
        else:
            files = (path, )

        for fp in files:
            try:
                new_vars = read_bash_dict(fp,
                                          vars_dict=vars_dict,
                                          sourcing_command=sourcing_command)
            except PermissionError as e:
                raise base_errors.PermissionDenied(fp, write=False) from e
            except EnvironmentError as e:
                if e.errno != errno.ENOENT or required:
                    raise config_errors.ParsingError(f"parsing {fp!r}",
                                                     exception=e) from e
                return

            if incrementals:
                for key in econst.incrementals:
                    if key in vars_dict and key in new_vars:
                        new_vars[key] = f"{vars_dict[key]} {new_vars[key]}"
            # quirk of read_bash_dict; it returns only what was mutated.
            vars_dict.update(new_vars)
Example #9
0
    def load_make_conf(vars_dict, path, allow_sourcing=False, required=True,
                       allow_recurse=True, incrementals=False):
        """parse make.conf files

        Args:
            vars_dict (dict): dictionary to add parsed variables to
            path (str): path to the make.conf which can be a regular file or
                directory, if a directory is passed all the non-hidden files within
                that directory are parsed in alphabetical order.
        """
        sourcing_command = 'source' if allow_sourcing else None

        if allow_recurse:
            files = sorted_scan(
                os.path.realpath(path), follow_symlinks=True, nonexistent=True,
                hidden=False, backup=False)
        else:
            files = (path,)

        for fp in files:
            try:
                new_vars = read_bash_dict(
                    fp, vars_dict=vars_dict, sourcing_command=sourcing_command)
            except PermissionError as e:
                raise base_errors.PermissionDenied(fp, write=False) from e
            except EnvironmentError as e:
                if e.errno != errno.ENOENT or required:
                    raise errors.ParsingError(f"parsing {fp!r}", exception=e) from e
                return

            if incrementals:
                for key in econst.incrementals:
                    if key in vars_dict and key in new_vars:
                        new_vars[key] = f"{vars_dict[key]} {new_vars[key]}"
            # quirk of read_bash_dict; it returns only what was mutated.
            vars_dict.update(new_vars)
Example #10
0
    def load_repos_conf(cls, path):
        """parse repos.conf files

        Args:
            path (str): path to the repos.conf which can be a regular file or
                directory, if a directory is passed all the non-hidden files within
                that directory are parsed in alphabetical order.

        Returns:
            dict: global repo settings
            dict: repo settings
        """
        main_defaults = {}
        repos = {}

        parser = ParseConfig()

        for fp in sorted_scan(os.path.realpath(path),
                              follow_symlinks=True,
                              nonexistent=True,
                              hidden=False,
                              backup=False):
            try:
                with open(fp) as f:
                    defaults, repo_confs = parser.parse_file(f)
            except PermissionError as e:
                raise base_errors.PermissionDenied(fp, write=False) from e
            except EnvironmentError as e:
                raise config_errors.ParsingError(f"parsing {fp!r}",
                                                 exception=e) from e
            except configparser.Error as e:
                raise config_errors.ParsingError(f"repos.conf: {fp!r}",
                                                 exception=e) from e

            if defaults and main_defaults:
                logger.warning(
                    f"repos.conf: parsing {fp!r}: overriding DEFAULT section")
            main_defaults.update(defaults)

            for name, repo_conf in repo_confs.items():
                if name in repos:
                    logger.warning(
                        f"repos.conf: parsing {fp!r}: overriding {name!r} repo"
                    )

                # ignore repo if location is unset
                location = repo_conf.get('location', None)
                if location is None:
                    logger.warning(
                        f"repos.conf: parsing {fp!r}: "
                        f"{name!r} repo missing location setting, ignoring repo"
                    )
                    continue
                repo_conf['location'] = os.path.abspath(location)

                # repo type defaults to ebuild for compat with portage
                repo_type = repo_conf.get('repo-type', 'ebuild-v1')
                try:
                    repo_conf['repo-type'] = cls._supported_repo_types[
                        repo_type]
                except KeyError:
                    logger.warning(
                        f"repos.conf: parsing {fp!r}: "
                        f"{name!r} repo has unsupported repo-type {repo_type!r}, "
                        "ignoring repo")
                    continue

                # Priority defaults to zero if unset or invalid for ebuild repos
                # while binpkg repos have the lowest priority by default.
                priority = repo_conf.get('priority', None)
                if priority is None:
                    if repo_type.startswith('binpkg'):
                        priority = -10000
                    else:
                        priority = 0

                try:
                    priority = int(priority)
                except ValueError:
                    logger.warning(
                        f"repos.conf: parsing {fp!r}: {name!r} repo has invalid priority "
                        f"setting: {priority!r} (defaulting to 0)")
                    priority = 0
                finally:
                    repo_conf['priority'] = priority

                # register repo
                repos[name] = repo_conf

        if repos:
            # the default repo is gentoo if unset and gentoo exists
            default_repo = main_defaults.get('main-repo', 'gentoo')
            if default_repo not in repos:
                raise config_errors.UserConfigError(
                    f"repos.conf: default repo {default_repo!r} is undefined or invalid"
                )

            if 'main-repo' not in main_defaults:
                main_defaults['main-repo'] = default_repo

            # the default repo has a low priority if unset or zero
            if repos[default_repo]['priority'] == 0:
                repos[default_repo]['priority'] = -1000

        # sort repos via priority, in this case high values map to high priorities
        repos = OrderedDict((k, v) for k, v in sorted(
            repos.items(), key=lambda d: d[1]['priority'], reverse=True))

        return main_defaults, repos
Example #11
0
def load_repos_conf(path):
    """parse repos.conf files

    Args:
        path (str): path to the repos.conf which can be a regular file or
            directory, if a directory is passed all the non-hidden files within
            that directory are parsed in alphabetical order.

    Returns:
        dict: global repo settings
        dict: repo settings
    """
    defaults = {}
    repos = {}

    for fp in sorted_scan(os.path.realpath(path),
                          follow_symlinks=True,
                          nonexistent=True):
        try:
            with open(fp) as f:
                config = ConfigParser()
                config.read_file(f)
        except EnvironmentError as e:
            if e.errno == errno.EACCES:
                raise_from(errors.PermissionDeniedError(fp, write=False))
            raise_from(errors.ParsingError("parsing %r" % (fp, ), exception=e))

        defaults.update(config.defaults())
        for name in config.sections():
            # note we don't check for duplicate entries so older matching
            # repos will be overridden
            repos[name] = dict(config.items(name))

            # repo priority defaults to zero if unset
            priority = repos[name].get('priority', 0)
            try:
                repos[name]['priority'] = int(priority)
            except ValueError:
                raise errors.ParsingError(
                    "%s: repo '%s' has invalid priority setting: %s" %
                    (fp, name, priority))

            # only the location setting is strictly required
            location = repos[name].get('location', None)
            if location is None:
                raise errors.ParsingError(
                    "%s: repo '%s' missing location setting" % (fp, name))
            repos[name]['location'] = os.path.abspath(location)

    if not repos:
        raise errors.ConfigurationError(
            "No repos are defined, please fix your repos.conf settings")

    # the default repo is gentoo if unset and gentoo exists
    default_repo = defaults.get('main-repo', 'gentoo')
    if default_repo not in repos:
        raise errors.ConfigurationError(
            "The main repo is undefined or invalid, "
            "please fix your repos.conf settings")

    if 'main-repo' not in defaults:
        defaults['main-repo'] = default_repo

    # the default repo has a low priority if unset or zero
    if repos[default_repo]['priority'] == 0:
        repos[default_repo]['priority'] = -1000

    # sort repos via priority, in this case high values map to high priorities
    repos = OrderedDict((k, v) for k, v in sorted(
        repos.iteritems(), key=lambda d: d[1]['priority'], reverse=True))

    del config
    return defaults, repos
Example #12
0
def load_repos_conf(path):
    """parse repos.conf files

    Args:
        path (str): path to the repos.conf which can be a regular file or
            directory, if a directory is passed all the non-hidden files within
            that directory are parsed in alphabetical order.

    Returns:
        dict: global repo settings
        dict: repo settings
    """
    defaults = {}
    repos = {}

    for fp in sorted_scan(os.path.realpath(path), follow_symlinks=True, nonexistent=True):
        try:
            with open(fp) as f:
                config = ConfigParser()
                config.read_file(f)
        except EnvironmentError as e:
            if e.errno == errno.EACCES:
                raise_from(errors.PermissionDeniedError(fp, write=False))
            raise_from(errors.ParsingError("parsing %r" % (fp,), exception=e))

        defaults.update(config.defaults())
        for name in config.sections():
            # note we don't check for duplicate entries so older matching
            # repos will be overridden
            repos[name] = dict(config.items(name))

            # repo priority defaults to zero if unset
            priority = repos[name].get('priority', 0)
            try:
                repos[name]['priority'] = int(priority)
            except ValueError:
                raise errors.ParsingError(
                    "%s: repo '%s' has invalid priority setting: %s" %
                    (fp, name, priority))

            # only the location setting is strictly required
            location = repos[name].get('location', None)
            if location is None:
                raise errors.ParsingError(
                    "%s: repo '%s' missing location setting" %
                    (fp, name))
            repos[name]['location'] = os.path.abspath(location)

    if not repos:
        raise errors.ConfigurationError(
            "No repos are defined, please fix your repos.conf settings")

    # the default repo is gentoo if unset and gentoo exists
    default_repo = defaults.get('main-repo', 'gentoo')
    if default_repo not in repos:
        raise errors.ConfigurationError(
            "The main repo is undefined or invalid, "
            "please fix your repos.conf settings")

    if 'main-repo' not in defaults:
        defaults['main-repo'] = default_repo

    # the default repo has a low priority if unset or zero
    if repos[default_repo]['priority'] == 0:
        repos[default_repo]['priority'] = -1000

    # sort repos via priority, in this case high values map to high priorities
    repos = OrderedDict(
        (k, v) for k, v in
        sorted(repos.iteritems(), key=lambda d: d[1]['priority'], reverse=True))

    del config
    return defaults, repos
Example #13
0
def load_repos_conf(path):
    """parse repos.conf files

    Args:
        path (str): path to the repos.conf which can be a regular file or
            directory, if a directory is passed all the non-hidden files within
            that directory are parsed in alphabetical order.

    Returns:
        dict: global repo settings
        dict: repo settings
    """
    defaults = {}
    repos = {}

    for fp in sorted_scan(os.path.realpath(path), follow_symlinks=True, nonexistent=True):
        config = ConfigParser()
        try:
            with open(fp) as f:
                config.read_file(f)
        except EnvironmentError as e:
            if e.errno == errno.EACCES:
                raise_from(errors.PermissionDeniedError(fp))
            raise_from(errors.ParsingError("repos.conf: '%s'" % (fp,), exception=e))
        except ConfigParser.exceptions as e:
            raise_from(errors.ParsingError("repos.conf: '%s'" % (fp,), exception=e))

        defaults_data = config.defaults()
        if defaults_data and defaults:
            logger.warning("repos.conf: parsing '%s': overriding DEFAULT section", fp)
        defaults.update(defaults_data)

        for name in config.sections():
            if name in repos:
                logger.warning("repos.conf: parsing '%s': overriding '%s' repo", fp, name)
            repo_data = dict(config.items(name))

            # ignore repo if location is unset
            location = repo_data.get('location', None)
            if location is None:
                logger.warning(
                    "repos.conf: parsing '%s': "
                    "'%s' repo missing location setting, ignoring repo", fp, name)
                continue
            repo_data['location'] = os.path.abspath(location)

            # repo priority defaults to zero if unset or invalid
            priority = repo_data.get('priority', 0)
            try:
                priority = int(priority)
            except ValueError:
                logger.warning(
                    "repos.conf: parsing '%s': '%s' repo has invalid priority "
                    "setting: '%s' (defaulting to 0)",
                    fp, name, priority)
                priority = 0
            finally:
                repo_data['priority'] = priority

            # register repo
            repos[name] = repo_data

    if not repos:
        raise errors.ConfigurationError("empty repos.conf, no repos defined")

    # the default repo is gentoo if unset and gentoo exists
    default_repo = defaults.get('main-repo', 'gentoo')
    if default_repo not in repos:
        raise errors.ConfigurationError(
            "default repo '%s' is undefined or invalid" % (default_repo,))

    if 'main-repo' not in defaults:
        defaults['main-repo'] = default_repo

    # the default repo has a low priority if unset or zero
    if repos[default_repo]['priority'] == 0:
        repos[default_repo]['priority'] = -1000

    # sort repos via priority, in this case high values map to high priorities
    repos = OrderedDict(
        (k, v) for k, v in
        sorted(repos.iteritems(), key=lambda d: d[1]['priority'], reverse=True))

    del config
    return defaults, repos
Example #14
0
 def bashrcs(self):
     files = sorted_scan(pjoin(self.config_dir, 'bashrc'), follow_symlinks=True)
     return tuple(local_source(x) for x in files)
Example #15
0
    def load_repos_conf(cls, path):
        """parse repos.conf files

        Args:
            path (str): path to the repos.conf which can be a regular file or
                directory, if a directory is passed all the non-hidden files within
                that directory are parsed in alphabetical order.

        Returns:
            dict: global repo settings
            dict: repo settings
        """
        main_defaults = {}
        repos = {}

        parser = ParseConfig()

        for fp in sorted_scan(
                os.path.realpath(path), follow_symlinks=True, nonexistent=True,
                hidden=False, backup=False):
            try:
                with open(fp) as f:
                    defaults, repo_confs = parser.parse_file(f)
            except PermissionError as e:
                raise base_errors.PermissionDenied(fp, write=False) from e
            except EnvironmentError as e:
                raise errors.ParsingError(f"parsing {fp!r}", exception=e) from e
            except configparser.Error as e:
                raise errors.ParsingError(f"repos.conf: {fp!r}", exception=e) from e

            if defaults and main_defaults:
                logger.warning(f"repos.conf: parsing {fp!r}: overriding DEFAULT section")
            main_defaults.update(defaults)

            for name, repo_conf in repo_confs.items():
                if name in repos:
                    logger.warning(f"repos.conf: parsing {fp!r}: overriding {name!r} repo")

                # ignore repo if location is unset
                location = repo_conf.get('location', None)
                if location is None:
                    logger.warning(
                        f"repos.conf: parsing {fp!r}: "
                        f"{name!r} repo missing location setting, ignoring repo")
                    continue
                repo_conf['location'] = os.path.abspath(location)

                # repo type defaults to ebuild for compat with portage
                repo_type = repo_conf.get('repo-type', 'ebuild-v1')
                try:
                    repo_conf['repo-type'] = cls._supported_repo_types[repo_type]
                except KeyError:
                    logger.warning(
                        f"repos.conf: parsing {fp!r}: "
                        f"{name!r} repo has unsupported repo-type {repo_type!r}, "
                        "ignoring repo")
                    continue

                # Priority defaults to zero if unset or invalid for ebuild repos
                # while binpkg repos have the lowest priority by default.
                priority = repo_conf.get('priority', None)
                if priority is None:
                    if repo_type.startswith('binpkg'):
                        priority = -10000
                    else:
                        priority = 0

                try:
                    priority = int(priority)
                except ValueError:
                    logger.warning(
                        f"repos.conf: parsing {fp!r}: {name!r} repo has invalid priority "
                        f"setting: {priority!r} (defaulting to 0)")
                    priority = 0
                finally:
                    repo_conf['priority'] = priority

                # register repo
                repos[name] = repo_conf

        if repos:
            # the default repo is gentoo if unset and gentoo exists
            default_repo = main_defaults.get('main-repo', 'gentoo')
            if default_repo not in repos:
                raise errors.UserConfigError(
                    f"repos.conf: default repo {default_repo!r} is undefined or invalid")

            if 'main-repo' not in main_defaults:
                main_defaults['main-repo'] = default_repo

            # the default repo has a low priority if unset or zero
            if repos[default_repo]['priority'] == 0:
                repos[default_repo]['priority'] = -1000

        # sort repos via priority, in this case high values map to high priorities
        repos = OrderedDict(
            (k, v) for k, v in
            sorted(repos.items(), key=lambda d: d[1]['priority'], reverse=True))

        return main_defaults, repos