Пример #1
0
    def __init__(self, filepath, logger=None):
        self.logger = logger or Logging.get_logger()
        logger.connect(self)

        pattern_map = load_yaml(filepath, logger=self.logger)

        if pattern_map is None:
            raise GlueError(
                "pattern map '{}' does not contain any patterns".format(
                    filepath))

        self._compiled_map = []

        for pattern_dict in pattern_map:
            if not isinstance(pattern_dict, dict):
                raise GlueError(
                    "Invalid format: '- <pattern>: <result>' expected, '{}' found"
                    .format(pattern_dict))

            pattern = pattern_dict.keys()[0]
            result = pattern_dict[pattern].strip()

            try:
                pattern = re.compile(pattern)

            except re.error as exc:
                raise GlueError("Pattern '{}' is not valid: {}".format(
                    pattern, str(exc)))

            self._compiled_map.append((pattern, result))
Пример #2
0
def load_yaml(filepath, logger=None):
    """
    Load data stored in YAML file, and return their Python representation.

    :param str filepath: Path to a file. ``~`` or ``~<username>`` are expanded before using.
    :param gluetool.log.ContextLogger logger: Logger used for logging.
    :rtype: object
    :returns: structures representing data in the file.
    :raises gluetool.glue.GlueError: if it was not possible to successfully load content of the file.
    """

    if not filepath:
        raise GlueError('File path is not valid: {}'.format(filepath))

    logger = logger or Logging.get_logger()

    real_filepath = normalize_path(filepath)

    logger.debug("attempt to load YAML from '{}' (maps to '{}')".format(
        filepath, real_filepath))

    if not os.path.exists(real_filepath):
        raise GlueError("File '{}' does not exist".format(filepath))

    try:
        with open(real_filepath, 'r') as f:
            data = YAML.load(f)
            logger.debug("loaded YAML data from '{}':\n{}".format(
                filepath, format_dict(data)))

            return data

    except ruamel.yaml.YAMLError as e:
        raise GlueError("Unable to load YAML file '{}': {}".format(
            filepath, str(e)))
Пример #3
0
def dump_yaml(data, filepath, logger=None):
    """
    Save data stored in variable to YAML file.

    :param object data: Data to store in YAML file
    :param str filepath: Path to an output file.
    :raises gluetool.glue.GlueError: if it was not possible to successfully save data to file.
    """
    if not filepath:
        raise GlueError("File path is not valid: '{}'".format(filepath))

    logger = logger or Logging.get_logger()

    real_filepath = normalize_path(filepath)
    dirpath = os.path.dirname(real_filepath)

    if not os.path.exists(dirpath):
        raise GlueError(
            "Cannot save file in nonexistent directory '{}'".format(dirpath))

    try:

        with open(real_filepath, 'w') as f:
            YAML.dump(data, f)
            f.flush()

    except ruamel.yaml.YAMLError as e:
        raise GlueError("Unable to save YAML file '{}': {}".format(
            filepath, str(e)))
Пример #4
0
def treat_url(url, logger=None):
    """
    Remove "weird" artifacts from the given URL. Collapse adjacent '.'s, apply '..', etc.

    :param str url: URL to clear.
    :param gluetool.log.ContextAdapter logger: logger to use for logging.
    :rtype: str
    :returns: Treated URL.
    """

    logger = logger or Logging.get_logger()

    logger.debug("treating a URL '{}'".format(url))

    try:
        url = str(urlnorm.norm(url))

    except urlnorm.InvalidUrl as exc:
        # urlnorm cannot handle localhost: https://github.com/jehiah/urlnorm/issues/3
        if exc.message == "host u'localhost' is not valid":
            pass

        else:
            raise exc

    return url.strip()
Пример #5
0
def fetch_url(url, logger=None, success_codes=(200, )):
    """
    "Get me content of this URL" helper.

    Very thin wrapper around urllib. Added value is logging, and converting
    possible errors to :py:class:`gluetool.glue.GlueError` exception.

    :param str url: URL to get.
    :param gluetool.log.ContextLogger logger: Logger used for logging.
    :param tuple success_codes: tuple of HTTP response codes representing successfull request.
    :returns: tuple ``(response, content)`` where ``response`` is what :py:func:`urllib2.urlopen`
      returns, and ``content`` is the payload of the response.
    """

    logger = logger or Logging.get_logger()

    logger.debug("opening URL '{}'".format(url))

    try:
        response = urllib2.urlopen(url)
        code, content = response.getcode(), response.read()

    except urllib2.HTTPError as exc:
        logger.exception(exc)
        raise GlueError("Failed to fetch URL '{}': {}".format(
            url, exc.message))

    log_blob(logger.debug, '{}: {}'.format(url, code), content)

    if code not in success_codes:
        raise GlueError("Unsuccessfull response from '{}'".format(url))

    return response, content
Пример #6
0
def wait(label, check, timeout=None, tick=30, logger=None):
    """
    Wait for a condition to be true.

    :param str label: printable label used for logging.
    :param callable check: called to test the condition. If its return value evaluates as ``True``,
        the condition is assumed to pass the test and waiting ends.
    :param int timeout: fail after this many seconds. ``None`` means test forever.
    :param int tick: test condition every ``tick`` seconds.
    :param gluetool.log.ContextAdapter logger: parent logger whose methods will be used for logging.
    :raises gluetool.glue.GlueError: when ``timeout`` elapses while condition did not pass the check.
    """

    if not isinstance(tick, int):
        raise GlueError('Tick must be an integer')

    if tick < 0:
        raise GlueError('Tick must be a positive integer')

    logger = logger or Logging.get_logger()

    if timeout is not None:
        end_time = time.time() + timeout

    def _timeout():
        return '{} seconds'.format(
            int(end_time - time.time())) if timeout is not None else 'infinite'

    logger.debug(
        "waiting for condition '{}', timeout {}, check every {} seconds".
        format(label, _timeout(), tick))

    while timeout is None or time.time() < end_time:
        logger.debug("calling callback function")
        ret = check()
        if ret:
            logger.debug('check passed, assuming success')
            return ret

        logger.debug("check failed with {}, assuming failure".format(ret))

        logger.debug('{} left, sleeping for {} seconds'.format(
            _timeout(), tick))
        time.sleep(tick)

    raise GlueError(
        "Condition '{}' failed to pass within given time".format(label))
Пример #7
0
    def __init__(self, executable, options=None, logger=None):
        self.logger = logger or ContextAdapter(Logging.get_logger())
        self.logger.connect(self)

        self.executable = executable
        self.options = options or []

        self.use_shell = False
        self.quote_args = False

        self._command = None
        self._popen_kwargs = None
        self._process = None
        self._exit_code = None

        self._stdout = None
        self._stderr = None
Пример #8
0
def render_template(template, logger=None, **kwargs):
    """
    Render Jinja2 template. Logs errors, and raises an exception when it's not possible
    to correctly render the template.

    :param template: Template to render. It can be either :py:class:`jinja2.environment.Template` instance,
        or a string.
    :param dict kwargs: Keyword arguments passed to render process.
    :rtype: str
    :returns: Rendered template.
    :raises gluetool.glue.GlueError: when the rednering failed.
    """

    logger = logger or Logging.get_logger()

    try:

        def _render(template):
            log_blob(logger.debug, 'rendering template', template.source)
            log_dict(logger.debug, 'context', kwargs)

            return str(template.render(**kwargs).strip())

        if isinstance(template, (str, unicode)):
            source, template = template, jinja2.Template(template)
            template.source = source

            return _render(template)

        if isinstance(template, jinja2.environment.Template):
            if template.filename != '<template>':
                with open(template.filename, 'r') as f:
                    template.source = f.read()

            else:
                template.source = '<unknown template source>'

            return _render(template)

        raise GlueError('Unhandled template type {}'.format(type(template)))

    except Exception as exc:
        raise GlueError('Cannot render template: {}'.format(exc))
Пример #9
0
    def __init__(self, filepath, spices=None, logger=None):
        self.logger = logger or Logging.get_logger()
        logger.connect(self)

        spices = spices or {}

        pattern_map = load_yaml(filepath, logger=self.logger)

        if pattern_map is None:
            raise GlueError(
                "pattern map '{}' does not contain any patterns".format(
                    filepath))

        def _create_simple_repl(repl):
            def _replace(pattern, target):
                """
                Use `repl` to construct image from `target`, honoring all backreferences made by `pattern`.
                """

                self.debug("pattern '{}', repl '{}', target '{}'".format(
                    pattern.pattern, repl, target))

                try:
                    return pattern.sub(repl, target)

                except re.error as e:
                    raise GlueError(
                        "Cannot transform pattern '{}' with target '{}', repl '{}': {}"
                        .format(pattern.pattern, target, repl, str(e)))

            return _replace

        self._compiled_map = []

        for pattern_dict in pattern_map:
            log_dict(logger.debug, 'pattern dict', pattern_dict)

            if not isinstance(pattern_dict, dict):
                raise GlueError(
                    "Invalid format: '- <pattern>: <transform>' expected, '{}' found"
                    .format(pattern_dict))

            pattern = pattern_dict.keys()[0]
            converter_chains = pattern_dict[pattern]

            if isinstance(converter_chains, str):
                converter_chains = [converter_chains]

            try:
                pattern = re.compile(pattern)

            except re.error as e:
                raise GlueError("Pattern '{}' is not valid: {}".format(
                    pattern, str(e)))

            compiled_chains = []

            for chain in converter_chains:
                converters = [s.strip() for s in chain.split(',')]

                # first item in `converters` is always a simple string used by `pattern.sub()` call
                converter = _create_simple_repl(converters.pop(0))

                # if there any any items left, they name "spices" to apply, one by one,
                # on the result of the first operation
                for spice in converters:
                    if spice not in spices:
                        raise GlueError(
                            "Unknown 'spice' function '{}'".format(spice))

                    converter = spices[spice](converter)

                compiled_chains.append(converter)

            self._compiled_map.append((pattern, compiled_chains))
Пример #10
0
def run_command(cmd, logger=None, inspect=False, inspect_callback=None, **kwargs):
    """
    Run external command, and return it's exit code and output.

    This is a very thin and simple wrapper above :py:class:`subprocess.Popen`,
    and its main purpose is to log everything that happens before and after
    execution. All additional arguments are passed directly to `Popen` constructor.

    If ``stdout`` or ``stderr`` keyword arguments are not specified, function
    will set them to :py:const:`subprocess.PIPE`, to capture both output streams
    in separate strings.

    By default, output of the process is captured for both ``stdout`` and ``stderr``,
    and returned back to the caller. Under some conditions, caller might want to see
    the output in "real-time". For that purpose, it can pass callable via ``inspect_callback``
    parameter - such callable will be called for every received bit of input on both
    ``stdout`` and ``stderr``. E.g.

    .. code-block:: python

       def foo(stream, s, flush=False):
         if s is not None and 'a' in s:
           print s

       run_command(['/bin/foo'], inspect=foo)

    This example will print all substrings containing letter `a`. Strings passed to ``foo``
    may be of arbitrary lengths, and may change between subsequent calls of ``run_command``.

    :param list cmd: command to execute.
    :param gluetool.log.ContextAdapter logger: parent logger whose methods will be used for logging.
    :param bool inspect: if set, ``inspect_callback`` will receive the output of command in "real-time".
    :param callable inspect_callback: callable that will receive command output. If not set,
        default "write to ``sys.stdout``" is used.
    :rtype: gluetool.utils.ProcessOutput instance
    :returns: :py:class:`gluetool.utils.ProcessOutput` instance whose attributes contain
        data returned by the process.
    :raises gluetool.glue.GlueError: when command was not found.
    :raises gluetool.glue.GlueCommandError: when command exited with non-zero exit code.
    :raises Exception: when anything else breaks.
    """

    assert isinstance(cmd, list), 'Only list of strings accepted as a command'

    if not all((isinstance(s, str) for s in cmd)):
        raise GlueError('Only list of strings accepted as a command, {} found'.format([type(s) for s in cmd]))

    logger = logger or Logging.get_logger()

    stdout, stderr = None, None

    # Set default stdout/stderr, unless told otherwise
    if 'stdout' not in kwargs:
        kwargs['stdout'] = subprocess.PIPE

    if 'stderr' not in kwargs:
        kwargs['stderr'] = subprocess.PIPE

    def _format_stream(stream):
        if stream == subprocess.PIPE:
            return 'PIPE'
        if stream == DEVNULL:
            return 'DEVNULL'
        if stream == subprocess.STDOUT:
            return 'STDOUT'
        return stream

    printable_kwargs = kwargs.copy()
    for stream in ('stdout', 'stderr'):
        if stream in printable_kwargs:
            printable_kwargs[stream] = _format_stream(printable_kwargs[stream])

    # Make tests happy by sorting kwargs - it's a dictionary, therefore
    # unpredictable from the observer's point of view. Can print its entries
    # in different order with different Pythons, making tests a mess.
    # sorted_kwargs = ', '.join(["'%s': '%s'" % (k, printable_kwargs[k]) for k in sorted(printable_kwargs.iterkeys())])

    log_dict(logger.debug, 'command', cmd)
    log_dict(logger.debug, 'kwargs', printable_kwargs)
    log_blob(logger.debug, 'runnable (copy & paste)', format_command_line([cmd]))

    try:
        p = subprocess.Popen(cmd, **kwargs)

        if inspect is True:
            # let's capture *both* streams - capturing just a single one leads to so many ifs
            # and elses and messy code
            p_stdout = StreamReader(p.stdout, name='<stdout>')
            p_stderr = StreamReader(p.stderr, name='<stderr>')

            if inspect_callback is None:
                def stdout_write(stream, data, flush=False):
                    # pylint: disable=unused-argument

                    if data is None:
                        return

                    # Not suitable for multiple simultaneous commands. Shuffled output will
                    # ruin your day. And night. And few following weeks, full of debugging, as well.
                    sys.stdout.write(data)
                    sys.stdout.flush()

                inspect_callback = stdout_write

            inputs = (p_stdout, p_stderr)

            with BlobLogger('Output of command: {}'.format(format_command_line([cmd])), outro='End of command output',
                            writer=logger.info):
                logger.debug("output of command is inspected by the caller")
                logger.debug('following blob-like header and footer are expected to be empty')
                logger.debug('the captured output will follow them')

                # As long as process runs, keep calling callbacks with incoming data
                while True:
                    for stream in inputs:
                        inspect_callback(stream, stream.read())

                    if p.poll() is not None:
                        break

                    # give up OS' attention and let others run
                    # time.sleep(0) is a Python synonym for "thread yields the rest of its quantum"
                    time.sleep(0.1)

                # OK, process finished but we have to wait for our readers to finish as well
                p_stdout.wait()
                p_stderr.wait()

                for stream in inputs:
                    while True:
                        data = stream.read()

                        if data in ('', None):
                            break

                        inspect_callback(stream, data)

                    inspect_callback(stream, None, flush=True)

            stdout, stderr = p_stdout.content, p_stderr.content

        else:
            stdout, stderr = p.communicate()

    except OSError as e:
        if e.errno == errno.ENOENT:
            raise GlueError("Command '{}' not found".format(cmd[0]))

        raise e

    exit_code = p.poll()

    output = ProcessOutput(cmd, exit_code, stdout, stderr, kwargs)

    output.log(logger.debug)

    if exit_code != 0:
        raise GlueCommandError(cmd, output)

    return output