def parse_source_descriptions(data, file_path='<string>'): """Parse a YAML string as source descriptions. If parsing failes an error message is printed to console and an empty list is returned. :param str data: string containing YAML representation of source descriptions :param str file_path: name of the file whose contents ``data`` contains :returns: tuple of ``file_path`` and parsed source descriptions :rtype: `tuple(str, list)` """ try: descriptions = load_yaml(data) verify_source_description_list(descriptions) except yaml.YAMLError as exc: if hasattr(exc, 'problem_mark'): mark = exc.problem_mark.line col = exc.problem_mark.column error("Invalid YAML in source list file '{0}' at '{1}:{2}':\n" .format(file_path, mark + 1, col + 1) + to_str(exc)) else: error("Invalid YAML in source list file '{0}':\n" .format(file_path) + to_str(exc)) descriptions = [] return (file_path, descriptions)
def read_stdout_err(cmd): """Execute a command synchronously and return stdout, stderr and exit code. :param cmd: executable and arguments :type cmd: `list` of `str` :return: tuple of stdout, stderr and exit code :rtype: ``(str, str, int)`` """ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) std_out, std_err = p.communicate() return (to_str(std_out), to_str(std_err), p.returncode)
def verify_plugin_class(class_, base_class): """Verify class from plugin definition. :raises ValueError: if class is invalid """ if not issubclass(base_class, PluginBase): raise ValueError( "Plugin base class '{0}' does not implement ABC PluginBase.". format(to_str(base_class))) if not issubclass(class_, base_class): raise ValueError( "Expected plugin class '{1}' to be subclass of '{0}'.".format( to_str(class_), to_str(base_class)))
def verify_plugin_class(class_, base_class): """Verify class from plugin definition. :raises ValueError: if class is invalid """ if not issubclass(base_class, PluginBase): raise ValueError( "Plugin base class '{0}' does not implement ABC PluginBase.". format(to_str(base_class))) if not issubclass(class_, base_class): raise ValueError( "Expected plugin class '{1}' to be subclass of '{0}'.". format(to_str(class_), to_str(base_class)))
def options(self, value): try: self._options = config_from_parsed_yaml(value, self.options_description, use_defaults=True) except ConfigValueError as e: raise_from( ConfigValueError, "invalid options `{}` for os plugin " "'{}'".format(value, self.name), e) unused_keys = set(value.keys()) - set(self._options.keys()) if unused_keys: warning("ignoring the following unknown options for os plugin " "'{}': {} -- known options are: {}".format( self.name, to_str(list(unused_keys)), to_str(self._options.keys())))
def load_url(url, retry=2, retry_period=1, timeout=10): """Load a given url with retries, retry_periods, and timeouts. :param str url: URL to load and return contents of :param int retry: number of times to retry the url on 503 or timeout :param float retry_period: time to wait between retries in seconds :param float timeout: timeout for opening the URL in seconds :retunrs: loaded data as string :rtype: str :raises DownloadFailure: if loading fails even after retries """ retry = max(retry, 0) # negative retry count causes infinite loop while True: try: req = urlopen(url, timeout=timeout) except HTTPError as e: if e.code == 503 and retry: retry -= 1 time.sleep(retry_period) else: raise_from(DownloadFailure, "Failed to load url '{0}'.". format(url), e) except URLError as e: if isinstance(e.reason, socket.timeout) and retry: retry -= 1 time.sleep(retry_period) else: raise_from(DownloadFailure, "Failed to load url '{0}'.". format(url), e) else: break _, params = cgi.parse_header(req.headers.get('Content-Type', '')) encoding = params.get('charset', 'utf-8') data = req.read() return to_str(data, encoding=encoding)
def load_url(url, retry=2, retry_period=1, timeout=10): """Load a given url with retries, retry_periods, and timeouts. :param str url: URL to load and return contents of :param int retry: number of times to retry the url on 503 or timeout :param float retry_period: time to wait between retries in seconds :param float timeout: timeout for opening the URL in seconds :retunrs: loaded data as string :rtype: str :raises DownloadFailure: if loading fails even after retries """ retry = max(retry, 0) # negative retry count causes infinite loop while True: try: req = urlopen(url, timeout=timeout) except HTTPError as e: if e.code == 503 and retry: retry -= 1 time.sleep(retry_period) else: raise_from(DownloadFailure, "Failed to load url '{0}'.".format(url), e) except URLError as e: if isinstance(e.reason, socket.timeout) and retry: retry -= 1 time.sleep(retry_period) else: raise_from(DownloadFailure, "Failed to load url '{0}'.".format(url), e) else: break _, params = cgi.parse_header(req.headers.get('Content-Type', '')) encoding = params.get('charset', 'utf-8') data = req.read() return to_str(data, encoding=encoding)
def main(args=None): args = command_handle_args(args, definition) try: # prepare arguments filepath = os.path.abspath(os.path.expanduser(args.rules_file)) # parse rules file with open(filepath, 'rb') as f: data = to_str(f.read()) rules = load_yaml(data) rules = expand_rules(rules) verify_rules_dict(rules) # compact rules compacted = compact_rules( rules, OSSupport().get_default_installer_names()) # output result dump = dump_yaml(compacted) if args.write: with open(filepath, 'wb') as f: f.write(to_bytes(dump)) else: info(dump) except (KeyboardInterrupt, EOFError): sys.exit(1)
def load_config_file_yaml(path): """Utility for loading yaml config files. Makes sure to always return a dict with strings as keys. Non- existent or empty files result in an empty dictionary. :raises ConfigValueError: if config file cannot be opened, decode or parsed """ try: if os.path.isfile(path): with open(path, 'rb') as f: binary_data = f.read() config = load_yaml(to_str(binary_data)) else: config = None return process_config_file_yaml(config) except EnvironmentError as e: raise_from(ConfigValueError, "failed to read config file `{}`".format(path), e) except UnicodeError as e: raise_from(ConfigValueError, "failed to decode config file `{}`".format(path), e) except YAMLError as e: raise_from(ConfigValueError, "failed to parse config file as YAML `{}`".format(path), e) except ConfigValueError as e: raise_from(ConfigValueError, "config file has invalid structure `{}`".format(path), e)
def add_arguments(self, parser): """Create a group and add all command-line-enabled items to it. See :func:`config_from_args` on how to process the passed arguments to a config dict. """ typeset = {type(i.type) for i in self.itemlist if i.type.command_line_parsing_help()} if typeset: typelist = list(typeset) typelist.sort(key=to_str) helplist = ["{}: {}".format(to_str(t), t.command_line_parsing_help()) for t in typelist] typehelp = "\n\n\nThere are some special cases and short hand " \ "notation for parsing config arguments:\n\n* " + "\n\n* ". \ join(helplist) else: typehelp = "" subparser = parser.add_argument_group( "config arguments", description="""The following typed arguments correspond to entries in the config file. Command line argument values are interpreted as YAML and override the corresponding entries in user/system config files. """ + typehelp) for item in self.itemlist: if item.command_line: item.add_argument(subparser)
def main(args=None): args = command_handle_args(args, definition) try: # prepare arguments filepath = os.path.abspath(os.path.expanduser(args.rules_file)) # parse rules file with open(filepath, 'rb') as f: data = to_str(f.read()) rules = load_yaml(data) rules = expand_rules(rules) verify_rules_dict(rules) # compact rules compacted = compact_rules(rules, OSSupport().get_default_installer_names()) # output result dump = dump_yaml(compacted) if args.write: with open(filepath, 'wb') as f: f.write(to_bytes(dump)) else: info(dump) except (KeyboardInterrupt, EOFError): sys.exit(1)
def verify_plugin_name(name): """Verify name from plugin definition. :raises ValueError: if name is invalid """ if not isinstance(name, text_type): raise ValueError( "Expected string as plugin name, got '{0}' of type '{1}'.".format( name, to_str(type(name))))
def verify_plugin_description(decription): """Verify decription from plugin definition. :raises ValueError: if decription is invalid """ if not isinstance(decription, text_type): raise ValueError( "Expected string as plugin decription, got '{0}' of type '{1}'.". format(decription, to_str(type(decription))))
def verify_plugin_name(name): """Verify name from plugin definition. :raises ValueError: if name is invalid """ if not isinstance(name, text_type): raise ValueError( "Expected string as plugin name, got '{0}' of type '{1}'.". format(name, to_str(type(name))))
def verify_spec_name(spec_name): """Verify that a ``spec_name`` is valid spec name. :param str spec_name: spec name :raises ValueError: if spec name is invalid """ if not isinstance(spec_name, text_type): raise ValueError( "expected spec name of string type, but got '{0}' of type '{1}'". format(spec_name, to_str(type(spec_name))))
def read_stdout(cmd): """Execute a command synchronously and return stdout. :param cmd: executable and arguments :type cmd: `list` of `str` :return str: captured stdout """ p = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE) std_out, std_err = p.communicate() return to_str(std_out)
def _parse_installer_rule(self, installer_rule): """Helper to parse installer rule with the installer_description.""" try: parsed_rule = config_from_parsed_yaml( installer_rule, self.installer_rule_description, use_defaults=True) except ConfigValueError as e: raise_from( InvalidRuleError, "invalid installer rule `{}` for " "installer '{}'".format(installer_rule, self.name), e) unused_keys = set(installer_rule.keys()) - set(parsed_rule.keys()) if unused_keys: warning("ignoring the following unknown installer rule keys for " "installer '{}' while parsing installer rule with " "packages {}: {}".format(self.name, to_str(parsed_rule.packages), to_str(list(unused_keys)))) return parsed_rule
def parse_source_file(file_path): """Parse a given list file and returns a list of source urls. :param file_path: path to file containing a list of source urls :type file_path: str :returns: lists of source urls keyed by spec type :rtype: :py:obj:`dict`(:py:obj:`str`: :py:obj:`list`(:py:obj:`str`)) """ with open(file_path, 'r') as f: data = f.read() return parse_source_descriptions(to_str(data), file_path)
def info(msg, file=None, *args, **kwargs): """Print info to console or file. Works like :func:`print`, optionally uses terminal colors and tries to handle unicode correctly by encoding to ``utf-8`` before printing. """ file = file if file is not None else sys.stdout msg = to_str(msg) msg = msg + ansi('reset') # Assume that msg might contain colors print(to_bytes(msg), file=file, *args, **kwargs) return msg
def warning(msg, file=None, *args, **kwargs): """Print warning to console or file. Works like :func:`print`, optionally uses terminal colors and tries to handle unicode correctly by encoding to ``utf-8`` before printing. Can be enabled or disabled with :func:`enable_debug`. """ file = file if file is not None else sys.stderr msg = to_str(msg) msg = ansi('yellowf') + msg + ansi('reset') print(to_bytes(msg), file=file, *args, **kwargs) return msg
def error(msg, file=None, exit=False, *args, **kwargs): """Print error statement and optionally exit. Works like :func:`print`, optionally uses terminal colors and tries to handle unicode correctly by encoding to ``utf-8`` before printing. """ file = file if file is not None else sys.stderr msg = to_str(msg) msg = ansi('redf') + ansi('boldon') + msg + ansi('reset') if exit: sys.exit(to_bytes(msg)) print(to_bytes(msg), file=file, *args, **kwargs) return msg
def compact_os_dict(os_dict, default_installers): for os_name, version_dict in os_dict.items(): try: if os_name == 'any_os': os_dict['any_os'] = compact_installer_dict( version_dict['any_version'], None) else: os_dict[os_name] = compact_version_dict( version_dict, default_installers.get(os_name, None)) except Exception as e: error("Failed to expand version dict for os {0} with error: " "{1}\n{2}".format(os_name, to_str(e), version_dict)) raise return os_dict
def verify_source_description_list(descr_list): """Verify that a source description list has valid structure. :param list descr_list: list of source descriptions :raises ValueError: if structure of source descriptions is invalid """ try: for descr in descr_list: verify_source_description(descr) except TypeError as e: raise_from(ValueError, "failed to verify source description list of " "type '{0}'".format(to_str(type(descr_list))), e) except ValueError as e: raise_from(ValueError, "source description list contains invalid " "source descriptions", e)
def verify_source_description(descr): """Verify that a source description has valid structure. :param dict descr_list: source description :raises ValueError: if structure of source description is invalid """ if not isinstance(descr, dict): raise ValueError("Expected source description to be a dictionary, but " "got '{0}'.".format(to_str(type(descr)))) keys = descr.keys() if not len(keys) == 1: raise ValueError("Expected source description to have one entry, but " "got keys '{0}'.".format(keys)) try: verify_spec_name(keys[0]) except ValueError as e: raise_from(ValueError, "source description does not have valid " "spec name '{0}'".format(keys[0], e))
def update(self): # TODO: save exceptions if they are not raised and then for the # cli command recognize permission errors and suggest to use # 'sudo' # We don't just call `load_from_source` and `save_to_cache` here # since we want errors with saving for each source to happen # directly after loading, not at the end. origins = set() for source in self.sources: if source.origin not in origins: origins.add(source.origin) if self.print_info: info("Processing '{0}'...".format(source.origin)) if self.print_info: info("Loading: {0} : {1}".format(source.spec.name, source.arguments)) try: source.load_from_source() except Exception as e: # TODO: be more specific about which exceptions to catch here if self.raise_on_error: raise else: error("Failed to load source '{0}':\n{1}".format( source.unique_id(), e)) else: try: source.save_to_cache() except Exception as e: # TODO: be more specific about which exceptions to catch if self.raise_on_error: raise else: error("Failed to save source '{0}' to cache:\n{1}". format(source.unique_id(), to_str(e)))
def update(self): # TODO: save exceptions if they are not raised and then for the # cli command recognize permission errors and suggest to use # 'sudo' # We don't just call `load_from_source` and `save_to_cache` here # since we want errors with saving for each source to happen # directly after loading, not at the end. origins = set() for source in self.sources: if source.origin not in origins: origins.add(source.origin) if self.print_info: info("Processing '{0}'...".format(source.origin)) if self.print_info: info("Loading: {0} : {1}". format(source.spec.name, source.arguments)) try: source.load_from_source() except Exception as e: # TODO: be more specific about which exceptions to catch here if self.raise_on_error: raise else: error("Failed to load source '{0}':\n{1}". format(source.unique_id(), e)) else: try: source.save_to_cache() except Exception as e: # TODO: be more specific about which exceptions to catch if self.raise_on_error: raise else: error("Failed to save source '{0}' to cache:\n{1}". format(source.unique_id(), to_str(e)))
def from_command_line(self, value): if value == UNSET_COMMAND_LINE: return self.unset_value() return self.from_yaml(to_str(value))
def __init__(self, msg, related_snippet=None): if related_snippet: msg += "\n\n" + to_str(related_snippet) ValueError.__init__(self, msg)
def verify_arguments(self, arguments): if not isinstance(arguments, text_type): raise ValueError( "Expected string (URL) as arguments for 'rules' spec, " "but got '{0}' of type '{1}'.". format(arguments, to_str(type(arguments))))
def resolve(xylem_keys, all_keys=False, config=None, database=None, sources_context=None, installer_context=None): """TODO""" # 1. Prepare config and contexts and load database config = ensure_config(config) ic = ensure_installer_context(installer_context, config) del installer_context # don't use further down, use `ic` only if not database: sources_context = ensure_sources_context(sources_context, config) database = RulesDatabase(sources_context) database.load_from_cache() del sources_context # don't use further down, use `database` only # 2. Prepare set of keys to look up if all_keys: lookup_keys = remove_duplicates(xylem_keys + sorted(database.keys(ic))) else: lookup_keys = remove_duplicates(xylem_keys) result = [] errors = [] # 3. Create an inverse install-from mapping # TODO: maybe allow pattern matching here like # `install-from "pip: python-*"` install_from_map = dict() for inst, keys in six.iteritems(config.install_from): for k in keys: if k in install_from_map: error("ignoring 'install from {}' for key '{}'; " "already configured to install from '{}'". format(inst, k, install_from_map[k])) else: install_from_map[k] = inst # 4. Resolve each key for key in lookup_keys: # 4.1. Lookup key in the database try: installer_dict = database.lookup(key, ic) if not installer_dict: errors.append((key, ResolutionError( "could not find rule for xylem key '{}' on '{}'.". format(key, ic.get_os_string())))) continue except LookupError as e: errors.append((key, chain_exception( ResolutionError, "lookup for key '{}' failed".format(key), e))) continue # 4.2. Decide which installer to use if key in install_from_map: inst_name = install_from_map[key] if not ic.lookup_installer(inst_name): errors.append((key, ResolutionError( "explicitly requested to install '{}' from '{}', but that " "installer is not loaded".format(key, inst_name)))) continue if inst_name not in installer_dict: errors.append((key, ResolutionError( "explicitly requested to install '{}' from '{}', but no " "rule for that installer was found; found rules for " "installers: `{}`". format(key, inst_name, to_str(installer_dict.keys()))))) continue info_v("found rule for key '{}' for explicitly requested " "installer '{}'".format(key, inst_name)) rule = installer_dict[inst_name] installer = ic.lookup_installer(inst_name) else: installer = None for inst in ic.core_installers: if inst.name in installer_dict: info_v("found rule for key '{}' for core installer '{}'". format(key, inst.name)) rule = installer_dict[inst.name] installer = inst break if installer is None: for inst in ic.additional_installers: if inst.name in installer_dict: info_v("found rule for key '{}' for additional " "installer '{}'".format(key, inst.name)) rule = installer_dict[inst.name] installer = inst if installer is None: errors.append((key, ResolutionError( "did not find rule for key '{}' for neither core " "installers '{}' nor additional installers '{}'; rules " "found for installers: '{}'". format(key, to_str(ic.core_installer_names), to_str(ic.additional_installer_names), to_str(installer_dict.keys()))))) continue # 4.3. Resolve with determined installer try: resolutions = installer.resolve(rule) except InstallerError as e: errors.append((key, chain_exception( ResolutionError, "failed to resolve with installer '{}'".format(installer.name), e))) else: result.append((key, (installer.name, resolutions))) return result, errors