def parse_source_descriptions(data, file_path='<string>'):
"""Parse a YAML string as source descriptions.
If parsing failes an error message is printed to console and an
empty list is returned.
:param str data: string containing YAML representation of source
descriptions
:param str file_path: name of the file whose contents ``data``
contains
:returns: tuple of ``file_path`` and parsed source descriptions
:rtype: `tuple(str, list)`
"""
try:
descriptions = load_yaml(data)
verify_source_description_list(descriptions)
except yaml.YAMLError as exc:
if hasattr(exc, 'problem_mark'):
mark = exc.problem_mark.line
col = exc.problem_mark.column
error("Invalid YAML in source list file '{0}' at '{1}:{2}':\n"
.format(file_path, mark + 1, col + 1) + to_str(exc))
else:
error("Invalid YAML in source list file '{0}':\n"
.format(file_path) + to_str(exc))
descriptions = []
return (file_path, descriptions)
def read_stdout_err(cmd):
"""Execute a command synchronously and return stdout, stderr and exit code.
:param cmd: executable and arguments
:type cmd: `list` of `str`
:return: tuple of stdout, stderr and exit code
:rtype: ``(str, str, int)``
"""
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
return (to_str(std_out), to_str(std_err), p.returncode)
def read_stdout_err(cmd):
"""Execute a command synchronously and return stdout, stderr and exit code.
:param cmd: executable and arguments
:type cmd: `list` of `str`
:return: tuple of stdout, stderr and exit code
:rtype: ``(str, str, int)``
"""
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
return (to_str(std_out), to_str(std_err), p.returncode)
def verify_plugin_class(class_, base_class):
"""Verify class from plugin definition.
:raises ValueError: if class is invalid
"""
if not issubclass(base_class, PluginBase):
raise ValueError(
"Plugin base class '{0}' does not implement ABC PluginBase.".
format(to_str(base_class)))
if not issubclass(class_, base_class):
raise ValueError(
"Expected plugin class '{1}' to be subclass of '{0}'.".format(
to_str(class_), to_str(base_class)))
def verify_plugin_class(class_, base_class):
"""Verify class from plugin definition.
:raises ValueError: if class is invalid
"""
if not issubclass(base_class, PluginBase):
raise ValueError(
"Plugin base class '{0}' does not implement ABC PluginBase.".
format(to_str(base_class)))
if not issubclass(class_, base_class):
raise ValueError(
"Expected plugin class '{1}' to be subclass of '{0}'.".
format(to_str(class_), to_str(base_class)))
def options(self, value):
try:
self._options = config_from_parsed_yaml(value,
self.options_description,
use_defaults=True)
except ConfigValueError as e:
raise_from(
ConfigValueError, "invalid options `{}` for os plugin "
"'{}'".format(value, self.name), e)
unused_keys = set(value.keys()) - set(self._options.keys())
if unused_keys:
warning("ignoring the following unknown options for os plugin "
"'{}': {} -- known options are: {}".format(
self.name, to_str(list(unused_keys)),
to_str(self._options.keys())))
def load_url(url, retry=2, retry_period=1, timeout=10):
"""Load a given url with retries, retry_periods, and timeouts.
:param str url: URL to load and return contents of
:param int retry: number of times to retry the url on 503 or timeout
:param float retry_period: time to wait between retries in seconds
:param float timeout: timeout for opening the URL in seconds
:retunrs: loaded data as string
:rtype: str
:raises DownloadFailure: if loading fails even after retries
"""
retry = max(retry, 0) # negative retry count causes infinite loop
while True:
try:
req = urlopen(url, timeout=timeout)
except HTTPError as e:
if e.code == 503 and retry:
retry -= 1
time.sleep(retry_period)
else:
raise_from(DownloadFailure, "Failed to load url '{0}'.".
format(url), e)
except URLError as e:
if isinstance(e.reason, socket.timeout) and retry:
retry -= 1
time.sleep(retry_period)
else:
raise_from(DownloadFailure, "Failed to load url '{0}'.".
format(url), e)
else:
break
_, params = cgi.parse_header(req.headers.get('Content-Type', ''))
encoding = params.get('charset', 'utf-8')
data = req.read()
return to_str(data, encoding=encoding)
def load_url(url, retry=2, retry_period=1, timeout=10):
"""Load a given url with retries, retry_periods, and timeouts.
:param str url: URL to load and return contents of
:param int retry: number of times to retry the url on 503 or timeout
:param float retry_period: time to wait between retries in seconds
:param float timeout: timeout for opening the URL in seconds
:retunrs: loaded data as string
:rtype: str
:raises DownloadFailure: if loading fails even after retries
"""
retry = max(retry, 0) # negative retry count causes infinite loop
while True:
try:
req = urlopen(url, timeout=timeout)
except HTTPError as e:
if e.code == 503 and retry:
retry -= 1
time.sleep(retry_period)
else:
raise_from(DownloadFailure,
"Failed to load url '{0}'.".format(url), e)
except URLError as e:
if isinstance(e.reason, socket.timeout) and retry:
retry -= 1
time.sleep(retry_period)
else:
raise_from(DownloadFailure,
"Failed to load url '{0}'.".format(url), e)
else:
break
_, params = cgi.parse_header(req.headers.get('Content-Type', ''))
encoding = params.get('charset', 'utf-8')
data = req.read()
return to_str(data, encoding=encoding)
def main(args=None):
args = command_handle_args(args, definition)
try:
# prepare arguments
filepath = os.path.abspath(os.path.expanduser(args.rules_file))
# parse rules file
with open(filepath, 'rb') as f:
data = to_str(f.read())
rules = load_yaml(data)
rules = expand_rules(rules)
verify_rules_dict(rules)
# compact rules
compacted = compact_rules(
rules, OSSupport().get_default_installer_names())
# output result
dump = dump_yaml(compacted)
if args.write:
with open(filepath, 'wb') as f:
f.write(to_bytes(dump))
else:
info(dump)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
def load_config_file_yaml(path):
"""Utility for loading yaml config files.
Makes sure to always return a dict with strings as keys. Non-
existent or empty files result in an empty dictionary.
:raises ConfigValueError: if config file cannot be opened, decode or
parsed
"""
try:
if os.path.isfile(path):
with open(path, 'rb') as f:
binary_data = f.read()
config = load_yaml(to_str(binary_data))
else:
config = None
return process_config_file_yaml(config)
except EnvironmentError as e:
raise_from(ConfigValueError,
"failed to read config file `{}`".format(path), e)
except UnicodeError as e:
raise_from(ConfigValueError,
"failed to decode config file `{}`".format(path), e)
except YAMLError as e:
raise_from(ConfigValueError,
"failed to parse config file as YAML `{}`".format(path), e)
except ConfigValueError as e:
raise_from(ConfigValueError,
"config file has invalid structure `{}`".format(path), e)
def add_arguments(self, parser):
"""Create a group and add all command-line-enabled items to it.
See :func:`config_from_args` on how to process the passed
arguments to a config dict.
"""
typeset = {type(i.type) for i in self.itemlist
if i.type.command_line_parsing_help()}
if typeset:
typelist = list(typeset)
typelist.sort(key=to_str)
helplist = ["{}: {}".format(to_str(t),
t.command_line_parsing_help())
for t in typelist]
typehelp = "\n\n\nThere are some special cases and short hand " \
"notation for parsing config arguments:\n\n* " + "\n\n* ". \
join(helplist)
else:
typehelp = ""
subparser = parser.add_argument_group(
"config arguments", description="""The following typed
arguments correspond to entries in the config file. Command
line argument values are interpreted as YAML and override
the corresponding entries in user/system config files. """
+ typehelp)
for item in self.itemlist:
if item.command_line:
item.add_argument(subparser)
def main(args=None):
args = command_handle_args(args, definition)
try:
# prepare arguments
filepath = os.path.abspath(os.path.expanduser(args.rules_file))
# parse rules file
with open(filepath, 'rb') as f:
data = to_str(f.read())
rules = load_yaml(data)
rules = expand_rules(rules)
verify_rules_dict(rules)
# compact rules
compacted = compact_rules(rules,
OSSupport().get_default_installer_names())
# output result
dump = dump_yaml(compacted)
if args.write:
with open(filepath, 'wb') as f:
f.write(to_bytes(dump))
else:
info(dump)
except (KeyboardInterrupt, EOFError):
sys.exit(1)
def verify_plugin_name(name):
"""Verify name from plugin definition.
:raises ValueError: if name is invalid
"""
if not isinstance(name, text_type):
raise ValueError(
"Expected string as plugin name, got '{0}' of type '{1}'.".format(
name, to_str(type(name))))
def verify_plugin_description(decription):
"""Verify decription from plugin definition.
:raises ValueError: if decription is invalid
"""
if not isinstance(decription, text_type):
raise ValueError(
"Expected string as plugin decription, got '{0}' of type '{1}'.".
format(decription, to_str(type(decription))))
def verify_plugin_name(name):
"""Verify name from plugin definition.
:raises ValueError: if name is invalid
"""
if not isinstance(name, text_type):
raise ValueError(
"Expected string as plugin name, got '{0}' of type '{1}'.".
format(name, to_str(type(name))))
def verify_plugin_description(decription):
"""Verify decription from plugin definition.
:raises ValueError: if decription is invalid
"""
if not isinstance(decription, text_type):
raise ValueError(
"Expected string as plugin decription, got '{0}' of type '{1}'.".
format(decription, to_str(type(decription))))
def verify_spec_name(spec_name):
"""Verify that a ``spec_name`` is valid spec name.
:param str spec_name: spec name
:raises ValueError: if spec name is invalid
"""
if not isinstance(spec_name, text_type):
raise ValueError(
"expected spec name of string type, but got '{0}' of type '{1}'".
format(spec_name, to_str(type(spec_name))))
def read_stdout(cmd):
"""Execute a command synchronously and return stdout.
:param cmd: executable and arguments
:type cmd: `list` of `str`
:return str: captured stdout
"""
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
return to_str(std_out)
def read_stdout(cmd):
"""Execute a command synchronously and return stdout.
:param cmd: executable and arguments
:type cmd: `list` of `str`
:return str: captured stdout
"""
p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE)
std_out, std_err = p.communicate()
return to_str(std_out)
def _parse_installer_rule(self, installer_rule):
"""Helper to parse installer rule with the installer_description."""
try:
parsed_rule = config_from_parsed_yaml(
installer_rule,
self.installer_rule_description,
use_defaults=True)
except ConfigValueError as e:
raise_from(
InvalidRuleError, "invalid installer rule `{}` for "
"installer '{}'".format(installer_rule, self.name), e)
unused_keys = set(installer_rule.keys()) - set(parsed_rule.keys())
if unused_keys:
warning("ignoring the following unknown installer rule keys for "
"installer '{}' while parsing installer rule with "
"packages {}: {}".format(self.name,
to_str(parsed_rule.packages),
to_str(list(unused_keys))))
return parsed_rule
def parse_source_file(file_path):
"""Parse a given list file and returns a list of source urls.
:param file_path: path to file containing a list of source urls
:type file_path: str
:returns: lists of source urls keyed by spec type
:rtype: :py:obj:`dict`(:py:obj:`str`: :py:obj:`list`(:py:obj:`str`))
"""
with open(file_path, 'r') as f:
data = f.read()
return parse_source_descriptions(to_str(data), file_path)
def info(msg, file=None, *args, **kwargs):
"""Print info to console or file.
Works like :func:`print`, optionally uses terminal colors and
tries to handle unicode correctly by encoding to ``utf-8`` before
printing.
"""
file = file if file is not None else sys.stdout
msg = to_str(msg)
msg = msg + ansi('reset') # Assume that msg might contain colors
print(to_bytes(msg), file=file, *args, **kwargs)
return msg
def warning(msg, file=None, *args, **kwargs):
"""Print warning to console or file.
Works like :func:`print`, optionally uses terminal colors and
tries to handle unicode correctly by encoding to ``utf-8`` before
printing. Can be enabled or disabled with
:func:`enable_debug`.
"""
file = file if file is not None else sys.stderr
msg = to_str(msg)
msg = ansi('yellowf') + msg + ansi('reset')
print(to_bytes(msg), file=file, *args, **kwargs)
return msg
def error(msg, file=None, exit=False, *args, **kwargs):
"""Print error statement and optionally exit.
Works like :func:`print`, optionally uses terminal colors and
tries to handle unicode correctly by encoding to ``utf-8`` before
printing.
"""
file = file if file is not None else sys.stderr
msg = to_str(msg)
msg = ansi('redf') + ansi('boldon') + msg + ansi('reset')
if exit:
sys.exit(to_bytes(msg))
print(to_bytes(msg), file=file, *args, **kwargs)
return msg
def compact_os_dict(os_dict, default_installers):
for os_name, version_dict in os_dict.items():
try:
if os_name == 'any_os':
os_dict['any_os'] = compact_installer_dict(
version_dict['any_version'], None)
else:
os_dict[os_name] = compact_version_dict(
version_dict, default_installers.get(os_name, None))
except Exception as e:
error("Failed to expand version dict for os {0} with error: "
"{1}\n{2}".format(os_name, to_str(e), version_dict))
raise
return os_dict
def verify_source_description_list(descr_list):
"""Verify that a source description list has valid structure.
:param list descr_list: list of source descriptions
:raises ValueError: if structure of source descriptions is invalid
"""
try:
for descr in descr_list:
verify_source_description(descr)
except TypeError as e:
raise_from(ValueError, "failed to verify source description list of "
"type '{0}'".format(to_str(type(descr_list))), e)
except ValueError as e:
raise_from(ValueError, "source description list contains invalid "
"source descriptions", e)
def verify_source_description(descr):
"""Verify that a source description has valid structure.
:param dict descr_list: source description
:raises ValueError: if structure of source description is invalid
"""
if not isinstance(descr, dict):
raise ValueError("Expected source description to be a dictionary, but "
"got '{0}'.".format(to_str(type(descr))))
keys = descr.keys()
if not len(keys) == 1:
raise ValueError("Expected source description to have one entry, but "
"got keys '{0}'.".format(keys))
try:
verify_spec_name(keys[0])
except ValueError as e:
raise_from(ValueError, "source description does not have valid "
"spec name '{0}'".format(keys[0], e))
def update(self):
# TODO: save exceptions if they are not raised and then for the
# cli command recognize permission errors and suggest to use
# 'sudo'
# We don't just call `load_from_source` and `save_to_cache` here
# since we want errors with saving for each source to happen
# directly after loading, not at the end.
origins = set()
for source in self.sources:
if source.origin not in origins:
origins.add(source.origin)
if self.print_info:
info("Processing '{0}'...".format(source.origin))
if self.print_info:
info("Loading: {0} : {1}".format(source.spec.name,
source.arguments))
try:
source.load_from_source()
except Exception as e:
# TODO: be more specific about which exceptions to catch here
if self.raise_on_error:
raise
else:
error("Failed to load source '{0}':\n{1}".format(
source.unique_id(), e))
else:
try:
source.save_to_cache()
except Exception as e:
# TODO: be more specific about which exceptions to catch
if self.raise_on_error:
raise
else:
error("Failed to save source '{0}' to cache:\n{1}".
format(source.unique_id(), to_str(e)))
def update(self):
# TODO: save exceptions if they are not raised and then for the
# cli command recognize permission errors and suggest to use
# 'sudo'
# We don't just call `load_from_source` and `save_to_cache` here
# since we want errors with saving for each source to happen
# directly after loading, not at the end.
origins = set()
for source in self.sources:
if source.origin not in origins:
origins.add(source.origin)
if self.print_info:
info("Processing '{0}'...".format(source.origin))
if self.print_info:
info("Loading: {0} : {1}".
format(source.spec.name, source.arguments))
try:
source.load_from_source()
except Exception as e:
# TODO: be more specific about which exceptions to catch here
if self.raise_on_error:
raise
else:
error("Failed to load source '{0}':\n{1}".
format(source.unique_id(), e))
else:
try:
source.save_to_cache()
except Exception as e:
# TODO: be more specific about which exceptions to catch
if self.raise_on_error:
raise
else:
error("Failed to save source '{0}' to cache:\n{1}".
format(source.unique_id(), to_str(e)))
def from_command_line(self, value):
if value == UNSET_COMMAND_LINE:
return self.unset_value()
return self.from_yaml(to_str(value))
def __init__(self, msg, related_snippet=None):
if related_snippet:
msg += "\n\n" + to_str(related_snippet)
ValueError.__init__(self, msg)
def verify_arguments(self, arguments):
if not isinstance(arguments, text_type):
raise ValueError(
"Expected string (URL) as arguments for 'rules' spec, "
"but got '{0}' of type '{1}'.".
format(arguments, to_str(type(arguments))))
def resolve(xylem_keys,
all_keys=False,
config=None,
database=None,
sources_context=None,
installer_context=None):
"""TODO"""
# 1. Prepare config and contexts and load database
config = ensure_config(config)
ic = ensure_installer_context(installer_context, config)
del installer_context # don't use further down, use `ic` only
if not database:
sources_context = ensure_sources_context(sources_context, config)
database = RulesDatabase(sources_context)
database.load_from_cache()
del sources_context # don't use further down, use `database` only
# 2. Prepare set of keys to look up
if all_keys:
lookup_keys = remove_duplicates(xylem_keys + sorted(database.keys(ic)))
else:
lookup_keys = remove_duplicates(xylem_keys)
result = []
errors = []
# 3. Create an inverse install-from mapping
# TODO: maybe allow pattern matching here like
# `install-from "pip: python-*"`
install_from_map = dict()
for inst, keys in six.iteritems(config.install_from):
for k in keys:
if k in install_from_map:
error("ignoring 'install from {}' for key '{}'; "
"already configured to install from '{}'".
format(inst, k, install_from_map[k]))
else:
install_from_map[k] = inst
# 4. Resolve each key
for key in lookup_keys:
# 4.1. Lookup key in the database
try:
installer_dict = database.lookup(key, ic)
if not installer_dict:
errors.append((key, ResolutionError(
"could not find rule for xylem key '{}' on '{}'.".
format(key, ic.get_os_string()))))
continue
except LookupError as e:
errors.append((key, chain_exception(
ResolutionError, "lookup for key '{}' failed".format(key), e)))
continue
# 4.2. Decide which installer to use
if key in install_from_map:
inst_name = install_from_map[key]
if not ic.lookup_installer(inst_name):
errors.append((key, ResolutionError(
"explicitly requested to install '{}' from '{}', but that "
"installer is not loaded".format(key, inst_name))))
continue
if inst_name not in installer_dict:
errors.append((key, ResolutionError(
"explicitly requested to install '{}' from '{}', but no "
"rule for that installer was found; found rules for "
"installers: `{}`".
format(key, inst_name, to_str(installer_dict.keys())))))
continue
info_v("found rule for key '{}' for explicitly requested "
"installer '{}'".format(key, inst_name))
rule = installer_dict[inst_name]
installer = ic.lookup_installer(inst_name)
else:
installer = None
for inst in ic.core_installers:
if inst.name in installer_dict:
info_v("found rule for key '{}' for core installer '{}'".
format(key, inst.name))
rule = installer_dict[inst.name]
installer = inst
break
if installer is None:
for inst in ic.additional_installers:
if inst.name in installer_dict:
info_v("found rule for key '{}' for additional "
"installer '{}'".format(key, inst.name))
rule = installer_dict[inst.name]
installer = inst
if installer is None:
errors.append((key, ResolutionError(
"did not find rule for key '{}' for neither core "
"installers '{}' nor additional installers '{}'; rules "
"found for installers: '{}'".
format(key,
to_str(ic.core_installer_names),
to_str(ic.additional_installer_names),
to_str(installer_dict.keys())))))
continue
# 4.3. Resolve with determined installer
try:
resolutions = installer.resolve(rule)
except InstallerError as e:
errors.append((key, chain_exception(
ResolutionError,
"failed to resolve with installer '{}'".format(installer.name),
e)))
else:
result.append((key, (installer.name, resolutions)))
return result, errors
