def load_url(url, retry=2, retry_period=1, timeout=10):
"""Load a given url with retries, retry_periods, and timeouts.
:param str url: URL to load and return contents of
:param int retry: number of times to retry the url on 503 or timeout
:param float retry_period: time to wait between retries in seconds
:param float timeout: timeout for opening the URL in seconds
:retunrs: loaded data as string
:rtype: str
:raises DownloadFailure: if loading fails even after retries
"""
retry = max(retry, 0) # negative retry count causes infinite loop
while True:
try:
req = urlopen(url, timeout=timeout)
except HTTPError as e:
if e.code == 503 and retry:
retry -= 1
time.sleep(retry_period)
else:
raise_from(DownloadFailure, "Failed to load url '{0}'.".
format(url), e)
except URLError as e:
if isinstance(e.reason, socket.timeout) and retry:
retry -= 1
time.sleep(retry_period)
else:
raise_from(DownloadFailure, "Failed to load url '{0}'.".
format(url), e)
else:
break
_, params = cgi.parse_header(req.headers.get('Content-Type', ''))
encoding = params.get('charset', 'utf-8')
data = req.read()
return to_str(data, encoding=encoding)
def verify_installer_dict(installer_dict, allow_default_installer=True):
"""Verify that an expanded installer dict has valid structure.
:param dict installer_dict: dictionary mapping installer names to
installer rules
:param bool allow_default_installer: indicates if
'default_installer' installer name is allowed
:raises ValueError: if installer dict does not have valid structure
"""
if not isinstance(installer_dict, dict):
raise ValueError("Expected installer dict of type 'dict', but got "
"'{0}'.".format(type(installer_dict)))
for installer_name, installer_rule in installer_dict.items():
if not allow_default_installer:
if installer_name == 'default_installer':
raise ValueError("Default installer is not allowed here.")
try:
verify_installer_name(installer_name)
except ValueError as e:
raise_from(ValueError, "Expected installer dict to have installer "
"names as keys.", e)
try:
verify_installer_rule(installer_rule)
except ValueError as e:
raise_from(ValueError, "Expected installer dict to have installer "
"rules as values, but got error for installer '{0}.".
format(installer_name), e)
def verify_installer_rule(installer_rule):
"""Verify that an expanded installer rule has valid structure.
:param dict installer_rule: dictionary describing an installer command
:raises ValueError: if installer rule does not have valid structure
"""
if not isinstance(installer_rule, dict):
raise ValueError("Expected installer rule of type 'dict', but got "
"'{0}'.".format(type(installer_rule)))
for key, value in installer_rule.items():
if not isinstance(key, text_type):
raise ValueError("Expected installer rule to have keys of text"
" type, but got '{0}'.".format(type(key)))
# The contents of the installer rule is specific to the
# according installer plugin, but we check for a few common keys here
if key == "packages":
if not isinstance(value, list):
raise ValueError(
"Expected 'packages' entry of installer rule to be of "
"type 'list', but got '{0}'".format(type(value)))
if key == "depends":
if not isinstance(value, list):
raise ValueError(
"Expected 'depends' entry of installer rule to be of "
"type 'list', but got '{0}'".format(type(value)))
try:
for xylem_key in value:
verify_xylem_key(xylem_key)
except ValueError as e:
raise_from(ValueError, "Expected 'depends' entry of installer "
"rule to be list of xylem keys.", e)
def from_command_line(self, value):
"""Create config value from value of command line parser.
A value of `UNSET_COMMAND_LINE` should be interpreted as
'unset'.
The default implementation parses the input string as yaml and
then invokes :meth:`from_yaml`.
Before returning a value, :meth:`verify` should be used to
verify the validity of the parsed value.
:param value: value from the command line parser, which is a
string when :meth:`command_line_multiple()` is false and a
list of strings otherwise, or `UNSET_COMMAND_LINE` if the
argument was omitted on the command line.
:raises ConfigValueError: if ``value`` cannot be parsed properly
or has invalid type or structure
"""
if value == UNSET_COMMAND_LINE:
return self.unset_value()
try:
return self.from_yaml(load_yaml(value))
except YAMLError as e:
raise_from(
ConfigValueError, "parsing command line value failed", e)
def load_url(url, retry=2, retry_period=1, timeout=10):
"""Load a given url with retries, retry_periods, and timeouts.
:param str url: URL to load and return contents of
:param int retry: number of times to retry the url on 503 or timeout
:param float retry_period: time to wait between retries in seconds
:param float timeout: timeout for opening the URL in seconds
:retunrs: loaded data as string
:rtype: str
:raises DownloadFailure: if loading fails even after retries
"""
retry = max(retry, 0) # negative retry count causes infinite loop
while True:
try:
req = urlopen(url, timeout=timeout)
except HTTPError as e:
if e.code == 503 and retry:
retry -= 1
time.sleep(retry_period)
else:
raise_from(DownloadFailure,
"Failed to load url '{0}'.".format(url), e)
except URLError as e:
if isinstance(e.reason, socket.timeout) and retry:
retry -= 1
time.sleep(retry_period)
else:
raise_from(DownloadFailure,
"Failed to load url '{0}'.".format(url), e)
else:
break
_, params = cgi.parse_header(req.headers.get('Content-Type', ''))
encoding = params.get('charset', 'utf-8')
data = req.read()
return to_str(data, encoding=encoding)
def verify(self, value):
if not self.is_unset(value):
if not isinstance(value, list):
raise ConfigValueError(type_error_msg('list', value))
try:
for elt in value:
self.element_type.verify(elt)
except ConfigValueError as e:
raise_from(ConfigValueError, "invalid list element", e)
return value
def verify(self, value):
if not self.is_unset(value):
if not isinstance(value, dict):
raise ConfigValueError(type_error_msg('dict', value))
try:
for k, v in six.iteritems(value):
self.key_type.verify(k)
self.value_type.verify(v)
except ConfigValueError as e:
raise_from(ConfigValueError, "invalid key or value", e)
return value
def verify_plugin(cls, obj):
"""Verify validity of a plugin object.
:param PluginBase obj: instantiated plugin object
:raises InvalidPluginError: if ``obj`` is invalid
"""
# delay import to avoid circular dependency
from xylem.sources.rules_dict import verify_rules_dict_identifier
try:
verify_rules_dict_identifier(obj.name, "plugin name")
except ValueError as e:
raise_from(InvalidPluginError, "plugin name invalid", e)
def verify_plugin(cls, obj):
"""Verify validity of a plugin object.
:param PluginBase obj: instantiated plugin object
:raises InvalidPluginError: if ``obj`` is invalid
"""
# delay import to avoid circular dependency
from xylem.sources.rules_dict import verify_rules_dict_identifier
try:
verify_rules_dict_identifier(obj.name, "plugin name")
except ValueError as e:
raise_from(InvalidPluginError, "plugin name invalid", e)
def load_config_file_yaml(path):
"""Utility for loading yaml config files.
Makes sure to always return a dict with strings as keys. Non-
existent or empty files result in an empty dictionary.
:raises ConfigValueError: if config file cannot be opened, decode or
parsed
"""
try:
if os.path.isfile(path):
with open(path, 'rb') as f:
binary_data = f.read()
config = load_yaml(to_str(binary_data))
else:
config = None
return process_config_file_yaml(config)
except EnvironmentError as e:
raise_from(ConfigValueError,
"failed to read config file `{}`".format(path), e)
except UnicodeError as e:
raise_from(ConfigValueError,
"failed to decode config file `{}`".format(path), e)
except YAMLError as e:
raise_from(ConfigValueError,
"failed to parse config file as YAML `{}`".format(path), e)
except ConfigValueError as e:
raise_from(ConfigValueError,
"config file has invalid structure `{}`".format(path), e)
def verify_plugin_definition(definition, kind, base_class):
"""Verify plugin definition.
:param dict definition: definition of plugin as loaded from entry point
:param str kind: kind of plugin (e.g. "installer")
:param type base_class: (abstract) base class plugins must derive from
:raises InvalidPluginError: if plugin definition is invalid
"""
try:
verify_plugin_name(definition['plugin_name'])
verify_plugin_description(definition['description'])
verify_plugin_class(definition[kind], base_class)
except (TypeError, KeyError, ValueError) as e:
raise_from(InvalidPluginError, "The following definition of {0} "
"plugin is invalid:\n{1}".format(kind, definition), e)
def verify_source_description_list(descr_list):
"""Verify that a source description list has valid structure.
:param list descr_list: list of source descriptions
:raises ValueError: if structure of source descriptions is invalid
"""
try:
for descr in descr_list:
verify_source_description(descr)
except TypeError as e:
raise_from(ValueError, "failed to verify source description list of "
"type '{0}'".format(to_str(type(descr_list))), e)
except ValueError as e:
raise_from(ValueError, "source description list contains invalid "
"source descriptions", e)
def parse_single(input):
try:
result = load_yaml(input)
except YAMLError:
result = None
if not isinstance(result, dict):
# special parsing allowing lists without enclosing `{}`
try:
result = load_yaml("{" + input + "}")
except YAMLError as e:
raise_from(ConfigValueError,
"failed to parse `{}` as dict".format(input), e)
if not isinstance(result, dict):
raise ConfigValueError(type_error_msg("dict", result))
return result
def options(self, value):
try:
self._options = config_from_parsed_yaml(value,
self.options_description,
use_defaults=True)
except ConfigValueError as e:
raise_from(
ConfigValueError, "invalid options `{}` for os plugin "
"'{}'".format(value, self.name), e)
unused_keys = set(value.keys()) - set(self._options.keys())
if unused_keys:
warning("ignoring the following unknown options for os plugin "
"'{}': {} -- known options are: {}".format(
self.name, to_str(list(unused_keys)),
to_str(self._options.keys())))
def verify_plugin_definition(definition, kind, base_class):
"""Verify plugin definition.
:param dict definition: definition of plugin as loaded from entry point
:param str kind: kind of plugin (e.g. "installer")
:param type base_class: (abstract) base class plugins must derive from
:raises InvalidPluginError: if plugin definition is invalid
"""
try:
verify_plugin_name(definition['plugin_name'])
verify_plugin_description(definition['description'])
verify_plugin_class(definition[kind], base_class)
except (TypeError, KeyError, ValueError) as e:
raise_from(
InvalidPluginError, "The following definition of {0} "
"plugin is invalid:\n{1}".format(kind, definition), e)
def verify_source_description(descr):
"""Verify that a source description has valid structure.
:param dict descr_list: source description
:raises ValueError: if structure of source description is invalid
"""
if not isinstance(descr, dict):
raise ValueError("Expected source description to be a dictionary, but "
"got '{0}'.".format(to_str(type(descr))))
keys = descr.keys()
if not len(keys) == 1:
raise ValueError("Expected source description to have one entry, but "
"got keys '{0}'.".format(keys))
try:
verify_spec_name(keys[0])
except ValueError as e:
raise_from(ValueError, "source description does not have valid "
"spec name '{0}'".format(keys[0], e))
def _parse_installer_rule(self, installer_rule):
"""Helper to parse installer rule with the installer_description."""
try:
parsed_rule = config_from_parsed_yaml(
installer_rule,
self.installer_rule_description,
use_defaults=True)
except ConfigValueError as e:
raise_from(
InvalidRuleError, "invalid installer rule `{}` for "
"installer '{}'".format(installer_rule, self.name), e)
unused_keys = set(installer_rule.keys()) - set(parsed_rule.keys())
if unused_keys:
warning("ignoring the following unknown installer rule keys for "
"installer '{}' while parsing installer rule with "
"packages {}: {}".format(self.name,
to_str(parsed_rule.packages),
to_str(list(unused_keys))))
return parsed_rule
def filter_uninstalled(resolved, installer_context):
errors = []
uninstalled = []
for installer_name, resolutions in resolved:
installer = installer_context.lookup_installer(installer_name)
if installer is None:
raise XylemInternalError("did not find resolved installer '{}'".
format(installer_name))
try:
resolutions = installer.filter_uninstalled(resolutions)
except InstallerError as e: # TODO: does this here make sense?
errors.append(chain_exception(
InstallError, "installer '{}' failed to determine "
"uninstalled resolutions out of {}".
format(installer_name, resolutions), e))
except Exception as e:
raise_from(
XylemInternalError, "unexpected error in installer '{}' "
"while trying to determine uninstalled resolutions out of {}".
format(installer_name, resolutions), e)
# only create a tuple if there is something to do
if resolutions:
uninstalled.append((installer_name, resolutions))
return uninstalled, errors
def verify_rules_dict(rules_dict, allow_default_installer=True):
"""Verify that an expanded rules dict has valid structure.
:param dict rules_dict: dictionary mapping xylem keys to os dicts
:param bool allow_default_installer: indicates if
'default_installer' installer name is allowed
:raises ValueError: if rules dict does not have valid structure
"""
if not isinstance(rules_dict, dict):
raise ValueError("Expected rules dict of type 'dict', but got '{0}'.".
format(type(rules_dict)))
for xylem_key, os_dict in rules_dict.items():
try:
verify_xylem_key(xylem_key)
except ValueError as e:
raise_from(ValueError, "Expected rules dict to have xylem keys as "
"keys.", e)
try:
verify_os_dict(os_dict)
except ValueError as e:
raise_from(ValueError, "Expected rules dict to have valid os "
"dicts as values, but got error for key '{0}'.".
format(xylem_key), e)
raise
def verify_version_dict(version_dict, allow_default_installer=True):
"""Verify that an expanded version dict has valid structure.
:param dict version_dict: dictionary mapping os versions to
installer dicts
:param bool allow_default_installer: indicates if
'default_installer' installer name is allowed
:raises ValueError: if version dict does not have valid structure
"""
if not isinstance(version_dict, dict):
raise ValueError("Expected version dict of type 'dict', but got "
"'{0}'.".format(type(version_dict)))
for os_version, installer_dict in version_dict.items():
try:
verify_os_version(os_version)
except ValueError as e:
raise_from(ValueError, "Expected version dict to have os versions "
"as keys.", e)
try:
verify_installer_dict(installer_dict, allow_default_installer)
except ValueError as e:
raise_from(ValueError, "Expected version dict to have valid "
"installer dicts as values, but got error for version "
"'{0}'.".format(os_version), e)
def verify_os_dict(os_dict, allow_default_installer=True):
"""Verify that an expanded os dict has valid structure.
:param dict os_dict: dictionary mapping os names to version dicts
:param bool allow_default_installer: indicates if
'default_installer' installer name is allowed
:raises ValueError: if os dict does not have valid structure
"""
if not isinstance(os_dict, dict):
raise ValueError("Expected os dict of type 'dict', but got '{0}'.".
format(type(os_dict)))
for os_name, version_dict in os_dict.items():
try:
verify_os_name(os_name)
except ValueError as e:
raise_from(ValueError, "Expected os dict to have os names as "
"keys.", e)
try:
def_inst = allow_default_installer and os_name != 'any_os'
verify_version_dict(version_dict, allow_default_installer=def_inst)
except ValueError as e:
raise_from(ValueError, "Expected os dict to have valid version "
"dicts as values, but got error for os '{0}'.".
format(os_name), e)
def install_resolutions(installer_name,
resolutions,
installer_context,
interactive=True,
reinstall=False,
simulate=False,
continue_on_error=False):
installer = installer_context.lookup_installer(installer_name)
if installer is None:
raise XylemInternalError("did not find resolved installer '{}'".
format(installer_name))
errors = []
try:
commands = installer.get_install_commands(resolutions,
interactive=interactive,
reinstall=reinstall)
except InstallerError as e: # TODO: does InstallerError here make sense?
errors.append(chain_exception(
InstallError, "installer '{}' failed to compose install commands "
"for resolutions {} with options `interactive={}` and "
"`reinstall={}`".
format(installer_name, resolutions, interactive, reinstall), e))
except Exception as e:
raise_from(
XylemInternalError, "unexpected error in installer '{}' while "
"composing install commands for resolutions {} with options "
"`interactive={}` and `reinstall={}`".
format(installer_name, resolutions, interactive, reinstall), e)
if not commands:
info_v("# [%s] no packages to install" % installer_name)
return errors
# 1. when simulating, only print commands to screen
if simulate:
print("# [%s] installation commands:" % installer_name)
for cmd in commands:
info(' ' + ' '.join(cmd))
return errors
# 2. else, run each install command set and collect errors
for cmd in commands:
info(fmt("@!executing command: %s@|" % ' '.join(cmd)))
exitcode = subprocess.call(cmd)
info_v(fmt("@!command return code: %s@|" % exitcode))
if exitcode != 0:
errors.append(InstallError(
"command `{}` for installer '{}' failed with return code {}".
format(' '.join(cmd), installer, exitcode)))
if not continue_on_error:
return errors
# 3. test installation of each resolution item
for item in resolutions:
try:
if not installer.is_installed(item):
errors.append(InstallError(
"failed to detect successful installation of '{}' "
"resolution `{}`".format(installer_name, item)))
except InstallerError as e: # TODO: does this here make sense?
errors.append(chain_exception(
InstallError, "installer '{}' failed to determine if `{}` "
"was successfully installed or not".
format(installer_name, item), e))
except Exception as e:
raise_from(
XylemInternalError, "unexpected error in installer '{}' while "
"checking successful installation of `{}`".
format(installer_name, item), e)
# 4. return list of failures
if errors:
info_v("# [%s] errors during installation" % installer_name)
else:
info_v("# [%s] successfully installed" % installer_name)
return errors
