def test_sorted_scan(self):
path = os.path.join(self.dir, "sorted_scan")
os.mkdir(path)
files = [
os.path.normpath(os.path.join(path, x))
for x in ["tmp", "blah", "dar"]
]
# cheap version of a touch.
for x in files:
open(x, "w").close()
dirs = [
os.path.normpath(os.path.join(path, x)) for x in ["a", "b", "c"]
]
for x in dirs:
os.mkdir(x)
# regular directory scanning
sorted_files = livefs.sorted_scan(path)
self.assertEqual(
list([pjoin(path, x) for x in ['blah', 'dar', 'tmp']]),
sorted_files)
# nonexistent paths
nonexistent_path = os.path.join(self.dir, 'foobar')
sorted_files = livefs.sorted_scan(nonexistent_path)
self.assertEqual(sorted_files, [])
sorted_files = livefs.sorted_scan(nonexistent_path, nonexistent=True)
self.assertEqual(sorted_files, [nonexistent_path])
def test_sorted_scan(self):
path = os.path.join(self.dir, "sorted_scan")
os.mkdir(path)
files = [os.path.normpath(os.path.join(path, x)) for x in
["tmp", "blah", "dar"]]
# cheap version of a touch.
for x in files:
open(x, "w").close()
dirs = [os.path.normpath(os.path.join(path, x)) for x in [
"a", "b", "c"]]
for x in dirs:
os.mkdir(x)
# regular directory scanning
sorted_files = livefs.sorted_scan(path)
self.assertEqual(
list([pjoin(path, x) for x in ['blah', 'dar', 'tmp']]),
sorted_files)
# nonexistent paths
nonexistent_path = os.path.join(self.dir, 'foobar')
sorted_files = livefs.sorted_scan(nonexistent_path)
self.assertEqual(sorted_files, [])
sorted_files = livefs.sorted_scan(nonexistent_path, nonexistent=True)
self.assertEqual(sorted_files, [nonexistent_path])
def test_sorted_scan_backup(self):
path = os.path.join(self.dir, "sorted_scan")
os.mkdir(path)
files = [os.path.normpath(os.path.join(path, x)) for x in ["blah", "blah~"]]
# cheap version of a touch.
for x in files:
open(x, "w").close()
sorted_files = livefs.sorted_scan(path)
assert list([pjoin(path, x) for x in ['blah', 'blah~']]) == sorted_files
sorted_files = livefs.sorted_scan(path, backup=False)
assert list([pjoin(path, x) for x in ['blah']]) == sorted_files
def test_sorted_scan_backup(self):
path = os.path.join(self.dir, "sorted_scan")
os.mkdir(path)
files = [
os.path.normpath(os.path.join(path, x)) for x in ["blah", "blah~"]
]
# cheap version of a touch.
for x in files:
open(x, "w").close()
sorted_files = livefs.sorted_scan(path)
assert list([pjoin(path, x)
for x in ['blah', 'blah~']]) == sorted_files
sorted_files = livefs.sorted_scan(path, backup=False)
assert list([pjoin(path, x) for x in ['blah']]) == sorted_files
def load_make_conf(vars_dict, path, allow_sourcing=False, required=True,
incrementals=False):
"""parse make.conf files
Args:
vars_dict (dict): dictionary to add parsed variables to
path (str): path to the make.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
"""
sourcing_command = None
if allow_sourcing:
sourcing_command = 'source'
for fp in sorted_scan(os.path.realpath(path), follow_symlinks=True, nonexistent=True):
try:
new_vars = read_bash_dict(
fp, vars_dict=vars_dict, sourcing_command=sourcing_command)
except EnvironmentError as e:
if e.errno == errno.EACCES:
raise_from(errors.PermissionDeniedError(fp, write=False))
if e.errno != errno.ENOENT or required:
raise_from(errors.ParsingError("parsing %r" % (fp,), exception=e))
return
if incrementals:
for key in econst.incrementals:
if key in vars_dict and key in new_vars:
new_vars[key] = "%s %s" % (vars_dict[key], new_vars[key])
# quirk of read_bash_dict; it returns only what was mutated.
vars_dict.update(new_vars)
def load_make_conf(vars_dict, path, allow_sourcing=False, required=True,
incrementals=False):
"""parse make.conf files
Args:
vars_dict (dict): dictionary to add parsed variables to
path (str): path to the make.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
"""
sourcing_command = None
if allow_sourcing:
sourcing_command = 'source'
for fp in sorted_scan(os.path.realpath(path), follow_symlinks=True, nonexistent=True):
try:
new_vars = read_bash_dict(
fp, vars_dict=vars_dict, sourcing_command=sourcing_command)
except EnvironmentError as e:
if e.errno == errno.EACCES:
raise_from(errors.PermissionDeniedError(fp, write=False))
if e.errno != errno.ENOENT or required:
raise_from(errors.ParsingError("parsing %r" % (fp,), exception=e))
return
if incrementals:
for key in econst.incrementals:
if key in vars_dict and key in new_vars:
new_vars[key] = "%s %s" % (vars_dict[key], new_vars[key])
# quirk of read_bash_dict; it returns only what was mutated.
vars_dict.update(new_vars)
def _load_and_invoke(func, filename, read_func, fallback, allow_recurse,
allow_line_cont, parse_func, eapi_optional, self):
if eapi_optional is not None and not getattr(self.eapi.options, eapi_optional, None):
return func(self, fallback)
profile_path = self.path.rstrip('/')
base = pjoin(profile_path, filename)
files = []
if self.pms_strict or not allow_recurse:
if os.path.exists(base):
files.append(base)
else:
# Skip hidden files and backup files, those beginning with '.' or
# ending with '~', respectively.
files.extend(sorted_scan(base, hidden=False, backup=False))
try:
if files:
if read_func is None:
data = parse_func(files)
else:
data = parse_func(read_func(
files, allow_line_cont=allow_line_cont))
else:
data = fallback
return func(self, data)
except (ValueError, IndexError, EnvironmentError) as e:
raise ProfileError(profile_path, filename, e) from e
except IsADirectoryError as e:
raise ProfileError(
self.path, filename,
"path is a directory, but this profile is PMS format- "
"directories aren't allowed. See layout.conf profile-formats "
"to enable directory support") from e
def load_make_conf(vars_dict,
path,
allow_sourcing=False,
required=True,
allow_recurse=True,
incrementals=False):
"""parse make.conf files
Args:
vars_dict (dict): dictionary to add parsed variables to
path (str): path to the make.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
"""
sourcing_command = 'source' if allow_sourcing else None
if allow_recurse:
files = sorted_scan(os.path.realpath(path),
follow_symlinks=True,
nonexistent=True,
hidden=False,
backup=False)
else:
files = (path, )
for fp in files:
try:
new_vars = read_bash_dict(fp,
vars_dict=vars_dict,
sourcing_command=sourcing_command)
except PermissionError as e:
raise base_errors.PermissionDenied(fp, write=False) from e
except EnvironmentError as e:
if e.errno != errno.ENOENT or required:
raise config_errors.ParsingError(f"parsing {fp!r}",
exception=e) from e
return
if incrementals:
for key in econst.incrementals:
if key in vars_dict and key in new_vars:
new_vars[key] = f"{vars_dict[key]} {new_vars[key]}"
# quirk of read_bash_dict; it returns only what was mutated.
vars_dict.update(new_vars)
def load_make_conf(vars_dict, path, allow_sourcing=False, required=True,
allow_recurse=True, incrementals=False):
"""parse make.conf files
Args:
vars_dict (dict): dictionary to add parsed variables to
path (str): path to the make.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
"""
sourcing_command = 'source' if allow_sourcing else None
if allow_recurse:
files = sorted_scan(
os.path.realpath(path), follow_symlinks=True, nonexistent=True,
hidden=False, backup=False)
else:
files = (path,)
for fp in files:
try:
new_vars = read_bash_dict(
fp, vars_dict=vars_dict, sourcing_command=sourcing_command)
except PermissionError as e:
raise base_errors.PermissionDenied(fp, write=False) from e
except EnvironmentError as e:
if e.errno != errno.ENOENT or required:
raise errors.ParsingError(f"parsing {fp!r}", exception=e) from e
return
if incrementals:
for key in econst.incrementals:
if key in vars_dict and key in new_vars:
new_vars[key] = f"{vars_dict[key]} {new_vars[key]}"
# quirk of read_bash_dict; it returns only what was mutated.
vars_dict.update(new_vars)
def load_repos_conf(cls, path):
"""parse repos.conf files
Args:
path (str): path to the repos.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
Returns:
dict: global repo settings
dict: repo settings
"""
main_defaults = {}
repos = {}
parser = ParseConfig()
for fp in sorted_scan(os.path.realpath(path),
follow_symlinks=True,
nonexistent=True,
hidden=False,
backup=False):
try:
with open(fp) as f:
defaults, repo_confs = parser.parse_file(f)
except PermissionError as e:
raise base_errors.PermissionDenied(fp, write=False) from e
except EnvironmentError as e:
raise config_errors.ParsingError(f"parsing {fp!r}",
exception=e) from e
except configparser.Error as e:
raise config_errors.ParsingError(f"repos.conf: {fp!r}",
exception=e) from e
if defaults and main_defaults:
logger.warning(
f"repos.conf: parsing {fp!r}: overriding DEFAULT section")
main_defaults.update(defaults)
for name, repo_conf in repo_confs.items():
if name in repos:
logger.warning(
f"repos.conf: parsing {fp!r}: overriding {name!r} repo"
)
# ignore repo if location is unset
location = repo_conf.get('location', None)
if location is None:
logger.warning(
f"repos.conf: parsing {fp!r}: "
f"{name!r} repo missing location setting, ignoring repo"
)
continue
repo_conf['location'] = os.path.abspath(location)
# repo type defaults to ebuild for compat with portage
repo_type = repo_conf.get('repo-type', 'ebuild-v1')
try:
repo_conf['repo-type'] = cls._supported_repo_types[
repo_type]
except KeyError:
logger.warning(
f"repos.conf: parsing {fp!r}: "
f"{name!r} repo has unsupported repo-type {repo_type!r}, "
"ignoring repo")
continue
# Priority defaults to zero if unset or invalid for ebuild repos
# while binpkg repos have the lowest priority by default.
priority = repo_conf.get('priority', None)
if priority is None:
if repo_type.startswith('binpkg'):
priority = -10000
else:
priority = 0
try:
priority = int(priority)
except ValueError:
logger.warning(
f"repos.conf: parsing {fp!r}: {name!r} repo has invalid priority "
f"setting: {priority!r} (defaulting to 0)")
priority = 0
finally:
repo_conf['priority'] = priority
# register repo
repos[name] = repo_conf
if repos:
# the default repo is gentoo if unset and gentoo exists
default_repo = main_defaults.get('main-repo', 'gentoo')
if default_repo not in repos:
raise config_errors.UserConfigError(
f"repos.conf: default repo {default_repo!r} is undefined or invalid"
)
if 'main-repo' not in main_defaults:
main_defaults['main-repo'] = default_repo
# the default repo has a low priority if unset or zero
if repos[default_repo]['priority'] == 0:
repos[default_repo]['priority'] = -1000
# sort repos via priority, in this case high values map to high priorities
repos = OrderedDict((k, v) for k, v in sorted(
repos.items(), key=lambda d: d[1]['priority'], reverse=True))
return main_defaults, repos
def load_repos_conf(path):
"""parse repos.conf files
Args:
path (str): path to the repos.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
Returns:
dict: global repo settings
dict: repo settings
"""
defaults = {}
repos = {}
for fp in sorted_scan(os.path.realpath(path),
follow_symlinks=True,
nonexistent=True):
try:
with open(fp) as f:
config = ConfigParser()
config.read_file(f)
except EnvironmentError as e:
if e.errno == errno.EACCES:
raise_from(errors.PermissionDeniedError(fp, write=False))
raise_from(errors.ParsingError("parsing %r" % (fp, ), exception=e))
defaults.update(config.defaults())
for name in config.sections():
# note we don't check for duplicate entries so older matching
# repos will be overridden
repos[name] = dict(config.items(name))
# repo priority defaults to zero if unset
priority = repos[name].get('priority', 0)
try:
repos[name]['priority'] = int(priority)
except ValueError:
raise errors.ParsingError(
"%s: repo '%s' has invalid priority setting: %s" %
(fp, name, priority))
# only the location setting is strictly required
location = repos[name].get('location', None)
if location is None:
raise errors.ParsingError(
"%s: repo '%s' missing location setting" % (fp, name))
repos[name]['location'] = os.path.abspath(location)
if not repos:
raise errors.ConfigurationError(
"No repos are defined, please fix your repos.conf settings")
# the default repo is gentoo if unset and gentoo exists
default_repo = defaults.get('main-repo', 'gentoo')
if default_repo not in repos:
raise errors.ConfigurationError(
"The main repo is undefined or invalid, "
"please fix your repos.conf settings")
if 'main-repo' not in defaults:
defaults['main-repo'] = default_repo
# the default repo has a low priority if unset or zero
if repos[default_repo]['priority'] == 0:
repos[default_repo]['priority'] = -1000
# sort repos via priority, in this case high values map to high priorities
repos = OrderedDict((k, v) for k, v in sorted(
repos.iteritems(), key=lambda d: d[1]['priority'], reverse=True))
del config
return defaults, repos
def load_repos_conf(path):
"""parse repos.conf files
Args:
path (str): path to the repos.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
Returns:
dict: global repo settings
dict: repo settings
"""
defaults = {}
repos = {}
for fp in sorted_scan(os.path.realpath(path), follow_symlinks=True, nonexistent=True):
try:
with open(fp) as f:
config = ConfigParser()
config.read_file(f)
except EnvironmentError as e:
if e.errno == errno.EACCES:
raise_from(errors.PermissionDeniedError(fp, write=False))
raise_from(errors.ParsingError("parsing %r" % (fp,), exception=e))
defaults.update(config.defaults())
for name in config.sections():
# note we don't check for duplicate entries so older matching
# repos will be overridden
repos[name] = dict(config.items(name))
# repo priority defaults to zero if unset
priority = repos[name].get('priority', 0)
try:
repos[name]['priority'] = int(priority)
except ValueError:
raise errors.ParsingError(
"%s: repo '%s' has invalid priority setting: %s" %
(fp, name, priority))
# only the location setting is strictly required
location = repos[name].get('location', None)
if location is None:
raise errors.ParsingError(
"%s: repo '%s' missing location setting" %
(fp, name))
repos[name]['location'] = os.path.abspath(location)
if not repos:
raise errors.ConfigurationError(
"No repos are defined, please fix your repos.conf settings")
# the default repo is gentoo if unset and gentoo exists
default_repo = defaults.get('main-repo', 'gentoo')
if default_repo not in repos:
raise errors.ConfigurationError(
"The main repo is undefined or invalid, "
"please fix your repos.conf settings")
if 'main-repo' not in defaults:
defaults['main-repo'] = default_repo
# the default repo has a low priority if unset or zero
if repos[default_repo]['priority'] == 0:
repos[default_repo]['priority'] = -1000
# sort repos via priority, in this case high values map to high priorities
repos = OrderedDict(
(k, v) for k, v in
sorted(repos.iteritems(), key=lambda d: d[1]['priority'], reverse=True))
del config
return defaults, repos
def load_repos_conf(path):
"""parse repos.conf files
Args:
path (str): path to the repos.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
Returns:
dict: global repo settings
dict: repo settings
"""
defaults = {}
repos = {}
for fp in sorted_scan(os.path.realpath(path), follow_symlinks=True, nonexistent=True):
config = ConfigParser()
try:
with open(fp) as f:
config.read_file(f)
except EnvironmentError as e:
if e.errno == errno.EACCES:
raise_from(errors.PermissionDeniedError(fp))
raise_from(errors.ParsingError("repos.conf: '%s'" % (fp,), exception=e))
except ConfigParser.exceptions as e:
raise_from(errors.ParsingError("repos.conf: '%s'" % (fp,), exception=e))
defaults_data = config.defaults()
if defaults_data and defaults:
logger.warning("repos.conf: parsing '%s': overriding DEFAULT section", fp)
defaults.update(defaults_data)
for name in config.sections():
if name in repos:
logger.warning("repos.conf: parsing '%s': overriding '%s' repo", fp, name)
repo_data = dict(config.items(name))
# ignore repo if location is unset
location = repo_data.get('location', None)
if location is None:
logger.warning(
"repos.conf: parsing '%s': "
"'%s' repo missing location setting, ignoring repo", fp, name)
continue
repo_data['location'] = os.path.abspath(location)
# repo priority defaults to zero if unset or invalid
priority = repo_data.get('priority', 0)
try:
priority = int(priority)
except ValueError:
logger.warning(
"repos.conf: parsing '%s': '%s' repo has invalid priority "
"setting: '%s' (defaulting to 0)",
fp, name, priority)
priority = 0
finally:
repo_data['priority'] = priority
# register repo
repos[name] = repo_data
if not repos:
raise errors.ConfigurationError("empty repos.conf, no repos defined")
# the default repo is gentoo if unset and gentoo exists
default_repo = defaults.get('main-repo', 'gentoo')
if default_repo not in repos:
raise errors.ConfigurationError(
"default repo '%s' is undefined or invalid" % (default_repo,))
if 'main-repo' not in defaults:
defaults['main-repo'] = default_repo
# the default repo has a low priority if unset or zero
if repos[default_repo]['priority'] == 0:
repos[default_repo]['priority'] = -1000
# sort repos via priority, in this case high values map to high priorities
repos = OrderedDict(
(k, v) for k, v in
sorted(repos.iteritems(), key=lambda d: d[1]['priority'], reverse=True))
del config
return defaults, repos
def bashrcs(self):
files = sorted_scan(pjoin(self.config_dir, 'bashrc'), follow_symlinks=True)
return tuple(local_source(x) for x in files)
def load_repos_conf(cls, path):
"""parse repos.conf files
Args:
path (str): path to the repos.conf which can be a regular file or
directory, if a directory is passed all the non-hidden files within
that directory are parsed in alphabetical order.
Returns:
dict: global repo settings
dict: repo settings
"""
main_defaults = {}
repos = {}
parser = ParseConfig()
for fp in sorted_scan(
os.path.realpath(path), follow_symlinks=True, nonexistent=True,
hidden=False, backup=False):
try:
with open(fp) as f:
defaults, repo_confs = parser.parse_file(f)
except PermissionError as e:
raise base_errors.PermissionDenied(fp, write=False) from e
except EnvironmentError as e:
raise errors.ParsingError(f"parsing {fp!r}", exception=e) from e
except configparser.Error as e:
raise errors.ParsingError(f"repos.conf: {fp!r}", exception=e) from e
if defaults and main_defaults:
logger.warning(f"repos.conf: parsing {fp!r}: overriding DEFAULT section")
main_defaults.update(defaults)
for name, repo_conf in repo_confs.items():
if name in repos:
logger.warning(f"repos.conf: parsing {fp!r}: overriding {name!r} repo")
# ignore repo if location is unset
location = repo_conf.get('location', None)
if location is None:
logger.warning(
f"repos.conf: parsing {fp!r}: "
f"{name!r} repo missing location setting, ignoring repo")
continue
repo_conf['location'] = os.path.abspath(location)
# repo type defaults to ebuild for compat with portage
repo_type = repo_conf.get('repo-type', 'ebuild-v1')
try:
repo_conf['repo-type'] = cls._supported_repo_types[repo_type]
except KeyError:
logger.warning(
f"repos.conf: parsing {fp!r}: "
f"{name!r} repo has unsupported repo-type {repo_type!r}, "
"ignoring repo")
continue
# Priority defaults to zero if unset or invalid for ebuild repos
# while binpkg repos have the lowest priority by default.
priority = repo_conf.get('priority', None)
if priority is None:
if repo_type.startswith('binpkg'):
priority = -10000
else:
priority = 0
try:
priority = int(priority)
except ValueError:
logger.warning(
f"repos.conf: parsing {fp!r}: {name!r} repo has invalid priority "
f"setting: {priority!r} (defaulting to 0)")
priority = 0
finally:
repo_conf['priority'] = priority
# register repo
repos[name] = repo_conf
if repos:
# the default repo is gentoo if unset and gentoo exists
default_repo = main_defaults.get('main-repo', 'gentoo')
if default_repo not in repos:
raise errors.UserConfigError(
f"repos.conf: default repo {default_repo!r} is undefined or invalid")
if 'main-repo' not in main_defaults:
main_defaults['main-repo'] = default_repo
# the default repo has a low priority if unset or zero
if repos[default_repo]['priority'] == 0:
repos[default_repo]['priority'] = -1000
# sort repos via priority, in this case high values map to high priorities
repos = OrderedDict(
(k, v) for k, v in
sorted(repos.items(), key=lambda d: d[1]['priority'], reverse=True))
return main_defaults, repos
