Exemple #1
0
def test_console_to_str_warning(monkeypatch):
    some_bytes = b"a\xE9b"

    def check_warning(msg):
        assert msg.startswith(
            "Subprocess output does not appear to be encoded as")

    monkeypatch.setattr(locale, 'getpreferredencoding', lambda: 'utf-8')
    monkeypatch.setattr(pip.compat.logger, 'warning', check_warning)
    console_to_str(some_bytes)
Exemple #2
0
def test_console_to_str_warning(monkeypatch):
    some_bytes = b"a\xE9b"

    def check_warning(msg):
        assert msg.startswith(
            "Subprocess output does not appear to be encoded as")

    monkeypatch.setattr(locale, 'getpreferredencoding', lambda: 'utf-8')
    monkeypatch.setattr(pip.compat.logger, 'warning', check_warning)
    console_to_str(some_bytes)
Exemple #3
0
def test_console_to_str(monkeypatch):
    some_bytes = b"a\xE9\xC3\xE9b"
    encodings = ('ascii', 'utf-8', 'iso-8859-1', 'iso-8859-5',
                 'koi8_r', 'cp850')
    for e in encodings:
        monkeypatch.setattr(locale, 'getpreferredencoding', lambda: e)
        result = console_to_str(some_bytes)
        assert result.startswith("a")
        assert result.endswith("b")
Exemple #4
0
def test_console_to_str(monkeypatch):
    some_bytes = b"a\xE9\xC3\xE9b"
    encodings = ('ascii', 'utf-8', 'iso-8859-1', 'iso-8859-5', 'koi8_r',
                 'cp850')
    for e in encodings:
        monkeypatch.setattr(locale, 'getpreferredencoding', lambda: e)
        result = console_to_str(some_bytes)
        assert result.startswith("a")
        assert result.endswith("b")
def call_subprocess(cmd,
                    show_stdout=True,
                    cwd=None,
                    on_returncode='raise',
                    command_desc=None,
                    extra_environ=None,
                    spinner=None):
    # This function's handling of subprocess output is confusing and I
    # previously broke it terribly, so as penance I will write a long comment
    # explaining things.
    #
    # The obvious thing that affects output is the show_stdout=
    # kwarg. show_stdout=True means, let the subprocess write directly to our
    # stdout. Even though it is nominally the default, it is almost never used
    # inside pip (and should not be used in new code without a very good
    # reason); as of 2016-02-22 it is only used in a few places inside the VCS
    # wrapper code. Ideally we should get rid of it entirely, because it
    # creates a lot of complexity here for a rarely used feature.
    #
    # Most places in pip set show_stdout=False. What this means is:
    # - We connect the child stdout to a pipe, which we read.
    # - By default, we hide the output but show a spinner -- unless the
    #   subprocess exits with an error, in which case we show the output.
    # - If the --verbose option was passed (= loglevel is DEBUG), then we show
    #   the output unconditionally. (But in this case we don't want to show
    #   the output a second time if it turns out that there was an error.)
    #
    # stderr is always merged with stdout (even if show_stdout=True).
    if show_stdout:
        stdout = None
    else:
        stdout = subprocess.PIPE
    if command_desc is None:
        cmd_parts = []
        for part in cmd:
            if ' ' in part or '\n' in part or '"' in part or "'" in part:
                part = '"%s"' % part.replace('"', '\\"')
            cmd_parts.append(part)
        command_desc = ' '.join(cmd_parts)
    logger.debug("Running command %s", command_desc)
    env = os.environ.copy()
    if extra_environ:
        env.update(extra_environ)
    try:
        proc = subprocess.Popen(cmd,
                                stderr=subprocess.STDOUT,
                                stdin=None,
                                stdout=stdout,
                                cwd=cwd,
                                env=env)
    except Exception as exc:
        logger.critical(
            "Error %s while executing command %s",
            exc,
            command_desc,
        )
        raise
    if stdout is not None:
        all_output = []
        while True:
            line = console_to_str(proc.stdout.readline())
            if not line:
                break
            line = line.rstrip()
            all_output.append(line + '\n')
            if logger.getEffectiveLevel() <= std_logging.DEBUG:
                # Show the line immediately
                logger.debug(line)
            else:
                # Update the spinner
                if spinner is not None:
                    spinner.spin()
    proc.wait()
    if spinner is not None:
        if proc.returncode:
            spinner.finish("error")
        else:
            spinner.finish("done")
    if proc.returncode:
        if on_returncode == 'raise':
            if (logger.getEffectiveLevel() > std_logging.DEBUG
                    and not show_stdout):
                logger.info(
                    'Complete output from command %s:',
                    command_desc,
                )
                logger.info(''.join(all_output) +
                            '\n----------------------------------------')
            raise InstallationError(
                'Command "%s" failed with error code %s in %s' %
                (command_desc, proc.returncode, cwd))
        elif on_returncode == 'warn':
            logger.warning(
                'Command "%s" had error code %s in %s',
                command_desc,
                proc.returncode,
                cwd,
            )
        elif on_returncode == 'ignore':
            pass
        else:
            raise ValueError('Invalid value: on_returncode=%s' %
                             repr(on_returncode))
    if not show_stdout:
        return ''.join(all_output)
def call_subprocess(cmd, show_stdout=True, cwd=None,
                    on_returncode='raise',
                    command_level=std_logging.DEBUG, command_desc=None,
                    extra_environ=None, spinner=None):
    # This function's handling of subprocess output is confusing and I
    # previously broke it terribly, so as penance I will write a long comment
    # explaining things.
    #
    # The obvious thing that affects output is the show_stdout=
    # kwarg. show_stdout=True means, let the subprocess write directly to our
    # stdout. Even though it is nominally the default, it is almost never used
    # inside pip (and should not be used in new code without a very good
    # reason); as of 2016-02-22 it is only used in a few places inside the VCS
    # wrapper code. Ideally we should get rid of it entirely, because it
    # creates a lot of complexity here for a rarely used feature.
    #
    # Most places in pip set show_stdout=False. What this means is:
    # - We connect the child stdout to a pipe, which we read.
    # - By default, we hide the output but show a spinner -- unless the
    #   subprocess exits with an error, in which case we show the output.
    # - If the --verbose option was passed (= loglevel is DEBUG), then we show
    #   the output unconditionally. (But in this case we don't want to show
    #   the output a second time if it turns out that there was an error.)
    #
    # stderr is always merged with stdout (even if show_stdout=True).
    if show_stdout:
        stdout = None
    else:
        stdout = subprocess.PIPE
    if command_desc is None:
        cmd_parts = []
        for part in cmd:
            if ' ' in part or '\n' in part or '"' in part or "'" in part:
                part = '"%s"' % part.replace('"', '\\"')
            cmd_parts.append(part)
        command_desc = ' '.join(cmd_parts)
    logger.log(command_level, "Running command %s", command_desc)
    env = os.environ.copy()
    if extra_environ:
        env.update(extra_environ)
    try:
        proc = subprocess.Popen(
            cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout,
            cwd=cwd, env=env)
    except Exception as exc:
        logger.critical(
            "Error %s while executing command %s", exc, command_desc,
        )
        raise
    if stdout is not None:
        all_output = []
        while True:
            line = console_to_str(proc.stdout.readline())
            if not line:
                break
            line = line.rstrip()
            all_output.append(line + '\n')
            if logger.getEffectiveLevel() <= std_logging.DEBUG:
                # Show the line immediately
                logger.debug(line)
            else:
                # Update the spinner
                if spinner is not None:
                    spinner.spin()
    proc.wait()
    if spinner is not None:
        if proc.returncode:
            spinner.finish("error")
        else:
            spinner.finish("done")
    if proc.returncode:
        if on_returncode == 'raise':
            if (logger.getEffectiveLevel() > std_logging.DEBUG and
                    not show_stdout):
                logger.info(
                    'Complete output from command %s:', command_desc,
                )
                logger.info(
                    ''.join(all_output) +
                    '\n----------------------------------------'
                )
            raise InstallationError(
                'Command "%s" failed with error code %s in %s'
                % (command_desc, proc.returncode, cwd))
        elif on_returncode == 'warn':
            logger.warning(
                'Command "%s" had error code %s in %s',
                command_desc, proc.returncode, cwd,
            )
        elif on_returncode == 'ignore':
            pass
        else:
            raise ValueError('Invalid value: on_returncode=%s' %
                             repr(on_returncode))
    if not show_stdout:
        return remove_tracebacks(''.join(all_output))
Exemple #7
0
def call_subprocess(cmd,
                    show_stdout=True,
                    cwd=None,
                    raise_on_returncode=True,
                    command_level=logging.DEBUG,
                    command_desc=None,
                    extra_environ=None):
    if command_desc is None:
        cmd_parts = []
        for part in cmd:
            if ' ' in part or '\n' in part or '"' in part or "'" in part:
                part = '"%s"' % part.replace('"', '\\"')
            cmd_parts.append(part)
        command_desc = ' '.join(cmd_parts)
    if show_stdout:
        stdout = None
    else:
        stdout = subprocess.PIPE
    logger.log(command_level, "Running command %s", command_desc)
    env = os.environ.copy()
    if extra_environ:
        env.update(extra_environ)
    try:
        proc = subprocess.Popen(cmd,
                                stderr=subprocess.STDOUT,
                                stdin=None,
                                stdout=stdout,
                                cwd=cwd,
                                env=env)
    except Exception as exc:
        logger.critical(
            "Error %s while executing command %s",
            exc,
            command_desc,
        )
        raise
    all_output = []
    if stdout is not None:
        while True:
            line = console_to_str(proc.stdout.readline())
            if not line:
                break
            line = line.rstrip()
            all_output.append(line + '\n')
            logger.debug(line)
    if not all_output:
        returned_stdout, returned_stderr = proc.communicate()
        all_output = [returned_stdout or '']
    proc.wait()
    if proc.returncode:
        if raise_on_returncode:
            if all_output:
                logger.info(
                    'Complete output from command %s:',
                    command_desc,
                )
                logger.info(''.join(all_output) +
                            '\n----------------------------------------')
            raise InstallationError(
                'Command "%s" failed with error code %s in %s' %
                (command_desc, proc.returncode, cwd))
        else:
            logger.warning(
                'Command "%s" had error code %s in %s',
                command_desc,
                proc.returncode,
                cwd,
            )
    if stdout is not None:
        return remove_tracebacks(''.join(all_output))
def call_subprocess(cmd, show_stdout=True, cwd=None,
                    raise_on_returncode=True,
                    command_level=logging.DEBUG, command_desc=None,
                    extra_environ=None):
    if command_desc is None:
        cmd_parts = []
        for part in cmd:
            if ' ' in part or '\n' in part or '"' in part or "'" in part:
                part = '"%s"' % part.replace('"', '\\"')
            cmd_parts.append(part)
        command_desc = ' '.join(cmd_parts)
    if show_stdout:
        stdout = None
    else:
        stdout = subprocess.PIPE
    logger.log(command_level, "Running command %s", command_desc)
    env = os.environ.copy()
    if extra_environ:
        env.update(extra_environ)
    try:
        proc = subprocess.Popen(
            cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout,
            cwd=cwd, env=env)
    except Exception as exc:
        logger.critical(
            "Error %s while executing command %s", exc, command_desc,
        )
        raise
    all_output = []
    if stdout is not None:
        while True:
            line = console_to_str(proc.stdout.readline())
            if not line:
                break
            line = line.rstrip()
            all_output.append(line + '\n')
            logger.debug(line)
    if not all_output:
        returned_stdout, returned_stderr = proc.communicate()
        all_output = [returned_stdout or '']
    proc.wait()
    if proc.returncode:
        if raise_on_returncode:
            if all_output:
                logger.info(
                    'Complete output from command %s:', command_desc,
                )
                logger.info(
                    ''.join(all_output) +
                    '\n----------------------------------------'
                )
            raise InstallationError(
                'Command "%s" failed with error code %s in %s'
                % (command_desc, proc.returncode, cwd))
        else:
            logger.warning(
                'Command "%s" had error code %s in %s',
                command_desc, proc.returncode, cwd,
            )
    if stdout is not None:
        return remove_tracebacks(''.join(all_output))
Exemple #9
0
def call_subprocess(cmd,
                    show_stdout=True,
                    cwd=None,
                    on_returncode='raise',
                    command_level=std_logging.DEBUG,
                    command_desc=None,
                    extra_environ=None,
                    spinner=None):
    if command_desc is None:
        cmd_parts = []
        for part in cmd:
            if ' ' in part or '\n' in part or '"' in part or "'" in part:
                part = '"%s"' % part.replace('"', '\\"')
            cmd_parts.append(part)
        command_desc = ' '.join(cmd_parts)
    logger.log(command_level, "Running command %s", command_desc)
    env = os.environ.copy()
    if extra_environ:
        env.update(extra_environ)
    try:
        proc = subprocess.Popen(cmd,
                                stderr=subprocess.STDOUT,
                                stdin=None,
                                stdout=subprocess.PIPE,
                                cwd=cwd,
                                env=env)
    except Exception as exc:
        logger.critical(
            "Error %s while executing command %s",
            exc,
            command_desc,
        )
        raise
    all_output = []
    while True:
        line = console_to_str(proc.stdout.readline())
        if not line:
            break
        line = line.rstrip()
        all_output.append(line + '\n')
        if show_stdout:
            logger.debug(line)
        if spinner is not None:
            spinner.spin()
    proc.wait()
    if spinner is not None:
        if proc.returncode:
            spinner.finish("error")
        else:
            spinner.finish("done")
    if proc.returncode:
        if on_returncode == 'raise':
            if all_output:
                logger.info(
                    'Complete output from command %s:',
                    command_desc,
                )
                logger.info(''.join(all_output) +
                            '\n----------------------------------------')
            raise InstallationError(
                'Command "%s" failed with error code %s in %s' %
                (command_desc, proc.returncode, cwd))
        elif on_returncode == 'warn':
            logger.warning(
                'Command "%s" had error code %s in %s',
                command_desc,
                proc.returncode,
                cwd,
            )
        elif on_returncode == 'ignore':
            pass
        else:
            raise ValueError('Invalid value: on_returncode=%s' %
                             repr(on_returncode))
    if not show_stdout:
        return remove_tracebacks(''.join(all_output))
def call_subprocess(cmd, show_stdout=True,
                    filter_stdout=None, cwd=None,
                    raise_on_returncode=True,
                    command_level=logger.DEBUG, command_desc=None,
                    extra_environ=None):
    if command_desc is None:
        cmd_parts = []
        for part in cmd:
            if ' ' in part or '\n' in part or '"' in part or "'" in part:
                part = '"%s"' % part.replace('"', '\\"')
            cmd_parts.append(part)
        command_desc = ' '.join(cmd_parts)
    if show_stdout:
        stdout = None
    else:
        stdout = subprocess.PIPE
    logger.log(command_level, "Running command %s" % command_desc)
    env = os.environ.copy()
    if extra_environ:
        env.update(extra_environ)
    try:
        proc = subprocess.Popen(
            cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout,
            cwd=cwd, env=env)
    except Exception as exc:
        logger.fatal(
            "Error %s while executing command %s" % (exc, command_desc))
        raise
    all_output = []
    if stdout is not None:
        stdout = proc.stdout
        while 1:
            line = console_to_str(stdout.readline())
            if not line:
                break
            line = line.rstrip()
            all_output.append(line + '\n')
            if filter_stdout:
                level = filter_stdout(line)
                if isinstance(level, tuple):
                    level, line = level
                logger.log(level, line)
                if not logger.stdout_level_matches(level):
                    logger.show_progress()
            else:
                logger.info(line)
    else:
        returned_stdout, returned_stderr = proc.communicate()
        all_output = [returned_stdout or '']
    proc.wait()
    if proc.returncode:
        if raise_on_returncode:
            if all_output:
                logger.notify(
                    'Complete output from command %s:' % command_desc
                )
                logger.notify(
                    '\n'.join(all_output) +
                    '\n----------------------------------------'
                )
            raise InstallationError(
                'Command "%s" failed with error code %s in %s'
                % (command_desc, proc.returncode, cwd))
        else:
            logger.warn(
                'Command "%s" had error code %s in %s'
                % (command_desc, proc.returncode, cwd))
    if stdout is not None:
        return ''.join(all_output)
Exemple #11
0
def call_subprocess(cmd, show_stdout=True, cwd=None,
                    on_returncode='raise',
                    command_level=std_logging.DEBUG, command_desc=None,
                    extra_environ=None, spinner=None):
    if command_desc is None:
        cmd_parts = []
        for part in cmd:
            if ' ' in part or '\n' in part or '"' in part or "'" in part:
                part = '"%s"' % part.replace('"', '\\"')
            cmd_parts.append(part)
        command_desc = ' '.join(cmd_parts)
    logger.log(command_level, "Running command %s", command_desc)
    env = os.environ.copy()
    if extra_environ:
        env.update(extra_environ)
    try:
        proc = subprocess.Popen(
            cmd, stderr=subprocess.STDOUT, stdin=None, stdout=subprocess.PIPE,
            cwd=cwd, env=env)
    except Exception as exc:
        logger.critical(
            "Error %s while executing command %s", exc, command_desc,
        )
        raise
    all_output = []
    while True:
        line = console_to_str(proc.stdout.readline())
        if not line:
            break
        line = line.rstrip()
        all_output.append(line + '\n')
        if show_stdout:
            logger.debug(line)
        if spinner is not None:
            spinner.spin()
    proc.wait()
    if spinner is not None:
        if proc.returncode:
            spinner.finish("error")
        else:
            spinner.finish("done")
    if proc.returncode:
        if on_returncode == 'raise':
            if all_output:
                logger.info(
                    'Complete output from command %s:', command_desc,
                )
                logger.info(
                    ''.join(all_output) +
                    '\n----------------------------------------'
                )
            raise InstallationError(
                'Command "%s" failed with error code %s in %s'
                % (command_desc, proc.returncode, cwd))
        elif on_returncode == 'warn':
            logger.warning(
                'Command "%s" had error code %s in %s',
                command_desc, proc.returncode, cwd,
            )
        elif on_returncode == 'ignore':
            pass
        else:
            raise ValueError('Invalid value: on_returncode=%s' %
                             repr(on_returncode))
    if not show_stdout:
        return remove_tracebacks(''.join(all_output))
Exemple #12
0
    logger.log(command_level, "Running command %s", command_desc)
    env = os.environ.copy()
    if extra_environ:
        env.update(extra_environ)
    try:
        proc = subprocess.Popen(
            cmd, stderr=subprocess.STDOUT, stdin=None, stdout=stdout,
            cwd=cwd, env=env)
    except Exception as exc:
        logger.critical(
            "Error %s while executing command %s", exc, command_desc,
        )
        raise
    all_output = []
    if stdout is not None:
        stdout = remove_tracebacks(console_to_str(proc.stdout.read()))
        stdout = cStringIO(stdout)
<<<<<<< HEAD
        all_output = stdout.readlines()
        if show_stdout:
            while 1:
                line = stdout.readline()
                if not line:
                    break
                line = line.rstrip()
                all_output.append(line + '\n')
                if filter_stdout:
                    level = filter_stdout(line)
                    if isinstance(level, tuple):
                        level, line = level
                    logger.log(level, line)