示例#1
0
    def wait(self):
        """Wait for instance to respond to ansible ping."""
        extra_vars = [
            'ansible_connection=winrm',
            'ansible_host=%s' % self.core_ci.connection.hostname,
            'ansible_user=%s' % self.core_ci.connection.username,
            'ansible_password=%s' % self.core_ci.connection.password,
            'ansible_port=%s' % self.core_ci.connection.port,
            'ansible_winrm_server_cert_validation=ignore',
        ]

        name = 'windows_%s' % self.core_ci.version

        env = ansible_environment(self.core_ci.args)
        cmd = ['ansible', '-m', 'win_ping', '-i', '%s,' % name, name, '-e', ' '.join(extra_vars)]

        for _ in range(1, 120):
            try:
                run_command(self.core_ci.args, cmd, env=env)
                return
            except SubprocessError:
                sleep(10)
                continue

        raise ApplicationError('Timeout waiting for %s/%s instance %s.' %
                               (self.core_ci.platform, self.core_ci.version, self.core_ci.instance_id))
示例#2
0
def delegate_tox(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    if args.python:
        versions = args.python,

        if args.python not in SUPPORTED_PYTHON_VERSIONS:
            raise ApplicationError('tox does not support Python version %s' % args.python)
    else:
        versions = SUPPORTED_PYTHON_VERSIONS

    options = {
        '--tox': args.tox_args,
    }

    for version in versions:
        tox = ['tox', '-c', 'test/runner/tox.ini', '-e', 'py' + version.replace('.', ''), '--']
        cmd = generate_command(args, os.path.abspath('test/runner/test.py'), options, exclude, require)

        if not args.python:
            cmd += ['--python', version]

        run_command(args, tox + cmd)
示例#3
0
    def __init__(self, args):
        """
        :type args: EnvironmentConfig
        """
        cache_dir = 'test/cache'

        self.key = os.path.join(cache_dir, self.KEY_NAME)
        self.pub = os.path.join(cache_dir, self.PUB_NAME)

        if not os.path.isfile(self.key) or not os.path.isfile(self.pub):
            base_dir = os.path.expanduser('~/.ansible/test/')

            key = os.path.join(base_dir, self.KEY_NAME)
            pub = os.path.join(base_dir, self.PUB_NAME)

            if not args.explain:
                make_dirs(base_dir)

            if not os.path.isfile(key) or not os.path.isfile(pub):
                run_command(args, ['ssh-keygen', '-q', '-t', 'rsa', '-N', '', '-f', key])

            if not args.explain:
                shutil.copy2(key, self.key)
                shutil.copy2(pub, self.pub)

        if args.explain:
            self.pub_contents = None
        else:
            with open(self.pub, 'r') as pub_fd:
                self.pub_contents = pub_fd.read().strip()
示例#4
0
    def wait(self):
        """Wait for instance to respond to ansible ping."""
        extra_vars = [
            'ansible_host=%s' % self.core_ci.connection.hostname,
            'ansible_port=%s' % self.core_ci.connection.port,
            'ansible_connection=local',
            'ansible_ssh_private_key_file=%s' % self.core_ci.ssh_key.key,
        ]

        name = '%s-%s' % (self.core_ci.platform, self.core_ci.version.replace('.', '-'))

        env = ansible_environment(self.core_ci.args)
        cmd = [
            'ansible',
            '-m', '%s_command' % self.core_ci.platform,
            '-a', 'commands=?',
            '-u', self.core_ci.connection.username,
            '-i', '%s,' % name,
            '-e', ' '.join(extra_vars),
            name,
        ]

        for _ in range(1, 90):
            try:
                run_command(self.core_ci.args, cmd, env=env)
                return
            except SubprocessError:
                sleep(10)
                continue

        raise ApplicationError('Timeout waiting for %s/%s instance %s.' %
                               (self.core_ci.platform, self.core_ci.version, self.core_ci.instance_id))
示例#5
0
def command_sanity_validate_modules(args, targets):
    """
    :type args: SanityConfig
    :type targets: SanityTargets
    """
    env = ansible_environment(args)

    paths = [deepest_path(i.path, 'lib/ansible/modules/') for i in targets.include_external]
    paths = sorted(set(p for p in paths if p))

    if not paths:
        display.info('No tests applicable.', verbosity=1)
        return

    cmd = ['test/sanity/validate-modules/validate-modules'] + paths

    with open('test/sanity/validate-modules/skip.txt', 'r') as skip_fd:
        skip_paths = skip_fd.read().splitlines()

    skip_paths += [e.path for e in targets.exclude_external]

    if skip_paths:
        cmd += ['--exclude', '^(%s)' % '|'.join(skip_paths)]

    run_command(args, cmd, env=env)
示例#6
0
 def scp(self, src, dst):
     """
     :type src: str
     :type dst: str
     """
     run_command(self.core_ci.args,
                 ['scp'] + self.ssh_args +
                 ['-P', str(self.core_ci.connection.port), '-q', '-r', src, dst])
示例#7
0
def generate_egg_info(args):
    """
    :type args: EnvironmentConfig
    """
    if os.path.isdir('lib/ansible.egg-info'):
        return

    run_command(args, ['python', 'setup.py', 'egg_info'], capture=args.verbosity < 3)
示例#8
0
    def setup_cli(self):
        """Install the correct Tower CLI for the version of Tower being tested."""
        tower_cli_version = self._get_cloud_config('tower_cli_version')

        display.info('Installing Tower CLI version: %s' % tower_cli_version)

        cmd = self.args.pip_command + ['install', '--disable-pip-version-check', 'ansible-tower-cli==%s' % tower_cli_version]

        run_command(self.args, cmd)
示例#9
0
def command_coverage_xml(args):
    """
    :type args: CoverageConfig
    """
    output_files = command_coverage_combine(args)

    for output_file in output_files:
        xml_name = 'test/results/reports/%s.xml' % os.path.basename(output_file)
        env = common_environment()
        env.update(dict(COVERAGE_FILE=output_file))
        run_command(args, env=env, cmd=['coverage', 'xml', '-i', '-o', xml_name])
示例#10
0
def command_shell(args):
    """
    :type args: ShellConfig
    """
    if args.delegate:
        raise Delegate()

    install_command_requirements(args)

    cmd = create_shell_command(['bash', '-i'])
    run_command(args, cmd)
示例#11
0
def command_shell(args):
    """
    :type args: ShellConfig
    """
    if args.delegate:
        raise Delegate()

    install_command_requirements(args)

    cmd = create_shell_command(['bash', '-i'])
    run_command(args, cmd)
示例#12
0
文件: tower.py 项目: zy1911/ansible
    def setup_cli(self):
        """Install the correct Tower CLI for the version of Tower being tested."""
        tower_cli_version = self._get_cloud_config('tower_cli_version')

        display.info('Installing Tower CLI version: %s' % tower_cli_version)

        cmd = self.args.pip_command + [
            'install', '--disable-pip-version-check',
            'ansible-tower-cli==%s' % tower_cli_version
        ]

        run_command(self.args, cmd)
示例#13
0
    def ssh(self, command):
        """
        :type command: str | list[str]
        """
        if isinstance(command, list):
            command = ' '.join(pipes.quote(c) for c in command)

        run_command(self.core_ci.args,
                    ['ssh', '-tt', '-q'] + self.ssh_args +
                    ['-p', str(self.core_ci.connection.port),
                     '%s@%s' % (self.core_ci.connection.username, self.core_ci.connection.hostname)] +
                    self.become + [pipes.quote(command)])
示例#14
0
    def ssh(self, command):
        """
        :type command: str | list[str]
        """
        if isinstance(command, list):
            command = ' '.join(pipes.quote(c) for c in command)

        run_command(self.core_ci.args,
                    ['ssh', '-tt', '-q'] + self.ssh_args +
                    ['-p', str(self.core_ci.connection.port),
                     '%s@%s' % (self.core_ci.connection.username, self.core_ci.connection.hostname)] +
                    self.become + [pipes.quote(command)])
示例#15
0
def docker_get(args, container_id, src, dst):
    """
    :type args: EnvironmentConfig
    :type container_id: str
    :type src: str
    :type dst: str
    """
    # avoid 'docker cp' due to a bug which causes 'docker rm' to fail
    cmd = ['docker', 'exec', '-i', container_id, 'dd', 'if=%s' % src, 'bs=%s' % BUFFER_SIZE]

    with open(dst, 'wb') as dst_fd:
        run_command(args, cmd, stdout=dst_fd, capture=True)
示例#16
0
文件: cover.py 项目: zdoop/ansible
def command_coverage_xml(args):
    """
    :type args: CoverageConfig
    """
    output_files = command_coverage_combine(args)

    for output_file in output_files:
        xml_name = 'test/results/reports/%s.xml' % os.path.basename(
            output_file)
        env = common_environment()
        env.update(dict(COVERAGE_FILE=output_file))
        run_command(args, env=env, cmd=['coverage', 'xml', '-o', xml_name])
示例#17
0
def command_sanity_yamllint(args, targets):
    """
    :type args: SanityConfig
    :type targets: SanityTargets
    """
    paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.yml', '.yaml'))

    if not paths:
        display.info('No tests applicable.', verbosity=1)
        return

    run_command(args, ['yamllint'] + paths)
示例#18
0
def command_compile(args):
    """
    :type args: CompileConfig
    """
    changes = get_changes_filter(args)
    require = (args.require or []) + changes
    include, exclude = walk_external_targets(walk_compile_targets(),
                                             args.include, args.exclude,
                                             require)

    if not include:
        raise AllTargetsSkipped()

    if args.delegate:
        raise Delegate(require=changes)

    install_command_requirements(args)

    version_commands = []

    for version in COMPILE_PYTHON_VERSIONS:
        # run all versions unless version given, in which case run only that version
        if args.python and version != args.python:
            continue

        # optional list of regex patterns to exclude from tests
        skip_file = 'test/compile/python%s-skip.txt' % version

        if os.path.exists(skip_file):
            with open(skip_file, 'r') as skip_fd:
                skip_paths = skip_fd.read().splitlines()
        else:
            skip_paths = []

        # augment file exclusions
        skip_paths += [e.path for e in exclude]
        skip_paths.append('/.tox/')

        skip_paths = sorted(skip_paths)

        python = 'python%s' % version
        cmd = [python, '-m', 'compileall', '-fq']

        if skip_paths:
            cmd += ['-x', '|'.join(skip_paths)]

        cmd += [target.path for target in include]

        version_commands.append((version, cmd))

    for version, command in version_commands:
        display.info('Compile with Python %s' % version)
        run_command(args, command)
示例#19
0
def delegate_tox(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    if args.python:
        versions = args.python,

        if args.python not in SUPPORTED_PYTHON_VERSIONS:
            raise ApplicationError('tox does not support Python version %s' %
                                   args.python)
    else:
        versions = SUPPORTED_PYTHON_VERSIONS

    options = {
        '--tox': args.tox_args,
        '--tox-sitepackages': 0,
    }

    for version in versions:
        tox = [
            'tox', '-c', 'test/runner/tox.ini', '-e',
            'py' + version.replace('.', '')
        ]

        if args.tox_sitepackages:
            tox.append('--sitepackages')

        tox.append('--')

        cmd = generate_command(args, os.path.abspath('test/runner/test.py'),
                               options, exclude, require)

        if not args.python:
            cmd += ['--python', version]

        if isinstance(args, TestConfig):
            if args.coverage and not args.coverage_label:
                cmd += ['--coverage-label', 'tox-%s' % version]

        env = common_environment()

        # temporary solution to permit ansible-test delegated to tox to provision remote resources
        optional = (
            'SHIPPABLE',
            'SHIPPABLE_BUILD_ID',
            'SHIPPABLE_JOB_NUMBER',
        )

        env.update(pass_vars(required=[], optional=optional))

        run_command(args, tox + cmd, env=env)
示例#20
0
def command_sanity_yamllint(args, targets):
    """
    :type args: SanityConfig
    :type targets: SanityTargets
    """
    paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.yml', '.yaml'))

    if not paths:
        display.info('No tests applicable.', verbosity=1)
        return

    run_command(args, ['yamllint'] + paths)
示例#21
0
def docker_get(args, container_id, src, dst):
    """
    :type args: EnvironmentConfig
    :type container_id: str
    :type src: str
    :type dst: str
    """
    # avoid 'docker cp' due to a bug which causes 'docker rm' to fail
    cmd = ['docker', 'exec', '-i', container_id, 'dd', 'if=%s' % src, 'bs=%s' % BUFFER_SIZE]

    with open(dst, 'wb') as dst_fd:
        run_command(args, cmd, stdout=dst_fd, capture=True)
示例#22
0
    def disable_pendo(self):
        """Disable Pendo tracking."""
        display.info('Disable Pendo tracking')

        config = TowerConfig.parse(self.config_path)

        # tower-cli does not recognize TOWER_ environment variables
        cmd = [
            'tower-cli', 'setting', 'modify', 'PENDO_TRACKING_STATE', 'off',
            '-h', config.host, '-u', config.username, '-p', config.password
        ]

        run_command(self.args, cmd)
示例#23
0
def command_coverage_report(args):
    """
    :type args: CoverageConfig
    """
    output_files = command_coverage_combine(args)

    for output_file in output_files:
        if args.group_by or args.stub:
            display.info('>>> Coverage Group: %s' % ' '.join(os.path.basename(output_file).split('=')[1:]))

        env = common_environment()
        env.update(dict(COVERAGE_FILE=output_file))
        run_command(args, env=env, cmd=['coverage', 'report'])
示例#24
0
def command_compile(args):
    """
    :type args: CompileConfig
    """
    changes = get_changes_filter(args)
    require = (args.require or []) + changes
    include, exclude = walk_external_targets(walk_compile_targets(), args.include, args.exclude, require)

    if not include:
        raise AllTargetsSkipped()

    if args.delegate:
        raise Delegate(require=changes)

    install_command_requirements(args)

    version_commands = []

    for version in COMPILE_PYTHON_VERSIONS:
        # run all versions unless version given, in which case run only that version
        if args.python and version != args.python:
            continue

        # optional list of regex patterns to exclude from tests
        skip_file = 'test/compile/python%s-skip.txt' % version

        if os.path.exists(skip_file):
            with open(skip_file, 'r') as skip_fd:
                skip_paths = skip_fd.read().splitlines()
        else:
            skip_paths = []

        # augment file exclusions
        skip_paths += [e.path for e in exclude]
        skip_paths.append('/.tox/')

        skip_paths = sorted(skip_paths)

        python = 'python%s' % version
        cmd = [python, '-m', 'compileall', '-fq']

        if skip_paths:
            cmd += ['-x', '|'.join(skip_paths)]

        cmd += [target.path if target.path == '.' else './%s' % target.path for target in include]

        version_commands.append((version, cmd))

    for version, command in version_commands:
        display.info('Compile with Python %s' % version)
        run_command(args, command)
示例#25
0
def command_sanity_code_smell(args, _):
    """
    :type args: SanityConfig
    :type _: SanityTargets
    """
    with open('test/sanity/code-smell/skip.txt', 'r') as skip_fd:
        skip_tests = skip_fd.read().splitlines()

    tests = glob.glob('test/sanity/code-smell/*')
    tests = sorted(p for p in tests if os.access(p, os.X_OK) and os.path.basename(p) not in skip_tests)

    for test in tests:
        display.info('Code smell check using %s' % os.path.basename(test))
        run_command(args, [test])
示例#26
0
def command_sanity_code_smell(args, _):
    """
    :type args: SanityConfig
    :type _: SanityTargets
    """
    with open('test/sanity/code-smell/skip.txt', 'r') as skip_fd:
        skip_tests = skip_fd.read().splitlines()

    tests = glob.glob('test/sanity/code-smell/*')
    tests = sorted(p for p in tests if os.access(p, os.X_OK) and os.path.basename(p) not in skip_tests)

    for test in tests:
        display.info('Code smell check using %s' % os.path.basename(test))
        run_command(args, [test])
示例#27
0
    def scp(self, src, dst):
        """
        :type src: str
        :type dst: str
        """
        for dummy in range(1, 10):
            try:
                run_command(self.core_ci.args, ['scp'] + self.ssh_args +
                            ['-P', '22', '-q', '-r', src, dst])
                return
            except SubprocessError:
                time.sleep(10)

        raise ApplicationError('Failed transfer: %s -> %s' % (src, dst))
示例#28
0
def command_sanity_shellcheck(args, targets):
    """
    :type args: SanityConfig
    :type targets: SanityTargets
    """
    with open('test/sanity/shellcheck/skip.txt', 'r') as skip_fd:
        skip_paths = set(skip_fd.read().splitlines())

    paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths)

    if not paths:
        display.info('No tests applicable.', verbosity=1)
        return

    run_command(args, ['shellcheck'] + paths)
示例#29
0
    def scp(self, src, dst):
        """
        :type src: str
        :type dst: str
        """
        for dummy in range(1, 10):
            try:
                run_command(self.core_ci.args,
                            ['scp'] + self.ssh_args +
                            ['-P', str(self.core_ci.connection.port), '-q', '-r', src, dst])
                return
            except SubprocessError:
                time.sleep(10)

        raise ApplicationError('Failed transfer: %s -> %s' % (src, dst))
示例#30
0
def command_sanity_shellcheck(args, targets):
    """
    :type args: SanityConfig
    :type targets: SanityTargets
    """
    with open('test/sanity/shellcheck/skip.txt', 'r') as skip_fd:
        skip_paths = set(skip_fd.read().splitlines())

    paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths)

    if not paths:
        display.info('No tests applicable.', verbosity=1)
        return

    run_command(args, ['shellcheck'] + paths)
示例#31
0
def delegate_tox(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    if args.python:
        versions = args.python,

        if args.python not in SUPPORTED_PYTHON_VERSIONS:
            raise ApplicationError('tox does not support Python version %s' % args.python)
    else:
        versions = SUPPORTED_PYTHON_VERSIONS

    options = {
        '--tox': args.tox_args,
        '--tox-sitepackages': 0,
    }

    for version in versions:
        tox = ['tox', '-c', 'test/runner/tox.ini', '-e', 'py' + version.replace('.', '')]

        if args.tox_sitepackages:
            tox.append('--sitepackages')

        tox.append('--')

        cmd = generate_command(args, os.path.abspath('test/runner/test.py'), options, exclude, require)

        if not args.python:
            cmd += ['--python', version]

        if isinstance(args, TestConfig):
            if args.coverage and not args.coverage_label:
                cmd += ['--coverage-label', 'tox-%s' % version]

        env = common_environment()

        # temporary solution to permit ansible-test delegated to tox to provision remote resources
        optional = (
            'SHIPPABLE',
            'SHIPPABLE_BUILD_ID',
            'SHIPPABLE_JOB_NUMBER',
        )

        env.update(pass_vars(required=[], optional=optional))

        run_command(args, tox + cmd, env=env)
示例#32
0
    def test(self, args):
        """
        :type args: SanityConfig
        :rtype: SanityResult
        """
        cmd = [self.path]
        env = ansible_environment(args, color=False)

        try:
            stdout, stderr = run_command(args, cmd, env=env, capture=True)
            status = 0
        except SubprocessError as ex:
            stdout = ex.stdout
            stderr = ex.stderr
            status = ex.status

        if stderr or status:
            summary = str(
                SubprocessError(cmd=cmd,
                                status=status,
                                stderr=stderr,
                                stdout=stdout))
            return SanityFailure(self.name, summary=summary)

        return SanitySuccess(self.name)
示例#33
0
def intercept_command(args,
                      cmd,
                      capture=False,
                      env=None,
                      data=None,
                      cwd=None,
                      python_version=None):
    """
    :type args: TestConfig
    :type cmd: collections.Iterable[str]
    :type capture: bool
    :type env: dict[str, str] | None
    :type data: str | None
    :type cwd: str | None
    :type python_version: str | None
    :rtype: str | None, str | None
    """
    if not env:
        env = common_environment()

    cmd = list(cmd)
    escaped_cmd = ' '.join(pipes.quote(c) for c in cmd)
    inject_path = get_coverage_path(args)

    env['PATH'] = inject_path + os.pathsep + env['PATH']
    env['ANSIBLE_TEST_COVERAGE'] = 'coverage' if args.coverage else 'version'
    env['ANSIBLE_TEST_PYTHON_VERSION'] = python_version or args.python_version
    env['ANSIBLE_TEST_CMD'] = escaped_cmd

    return run_command(args, cmd, capture=capture, env=env, data=data, cwd=cwd)
示例#34
0
def check_pyyaml(args, version):
    """
    :type args: EnvironmentConfig
    :type version: str
    """
    if version in CHECK_YAML_VERSIONS:
        return

    python = find_python(version)
    stdout, _dummy = run_command(args, [python, 'test/runner/yamlcheck.py'],
                                 capture=True)

    if args.explain:
        return

    CHECK_YAML_VERSIONS[version] = result = json.loads(stdout)

    yaml = result['yaml']
    cloader = result['cloader']

    if not yaml:
        display.warning('PyYAML is not installed for interpreter: %s' % python)
    elif not cloader:
        display.warning(
            'PyYAML will be slow due to installation without libyaml support for interpreter: %s'
            % python)
示例#35
0
def pip_list(args):
    """
    :type args: EnvironmentConfig
    :rtype: str
    """
    stdout, _ = run_command(args, ['pip', 'list'], capture=True, always=True)
    return stdout
示例#36
0
 def run_git(self, cmd, str_errors='strict'):
     """
     :type cmd: list[str]
     :type str_errors: str
     :rtype: str
     """
     return run_command(self.args, [self.git] + cmd, capture=True, always=True, str_errors=str_errors)[0]
示例#37
0
def command_sanity_code_smell(args, _, script):
    """
    :type args: SanityConfig
    :type _: SanityTargets
    :type script: str
    :rtype: SanityResult
    """
    test = os.path.splitext(os.path.basename(script))[0]

    cmd = [script]
    env = ansible_environment(args)

    # Since the output from scripts end up in other places besides the console, we don't want color here.
    env.pop('ANSIBLE_FORCE_COLOR')

    try:
        stdout, stderr = run_command(args, cmd, env=env, capture=True)
        status = 0
    except SubprocessError as ex:
        stdout = ex.stdout
        stderr = ex.stderr
        status = ex.status

    if stderr or status:
        summary = str(
            SubprocessError(cmd=cmd,
                            status=status,
                            stderr=stderr,
                            stdout=stdout))
        return SanityFailure(test, summary=summary)

    return SanitySuccess(test)
示例#38
0
 def run_git(self, cmd, str_errors='strict'):
     """
     :type cmd: list[str]
     :type str_errors: str
     :rtype: str
     """
     return run_command(self.args, [self.git] + cmd, capture=True, always=True, str_errors=str_errors)[0]
示例#39
0
def command_sanity_code_smell(args, _, script):
    """
    :type args: SanityConfig
    :type _: SanityTargets
    :type script: str
    :rtype: SanityResult
    """
    test = os.path.splitext(os.path.basename(script))[0]

    cmd = [script]
    env = ansible_environment(args, color=False)

    try:
        stdout, stderr = run_command(args, cmd, env=env, capture=True)
        status = 0
    except SubprocessError as ex:
        stdout = ex.stdout
        stderr = ex.stderr
        status = ex.status

    if stderr or status:
        summary = str(SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout))
        return SanityFailure(test, summary=summary)

    return SanitySuccess(test)
示例#40
0
def command_sanity_shellcheck(args, targets):
    """
    :type args: SanityConfig
    :type targets: SanityTargets
    :rtype: SanityResult
    """
    test = 'shellcheck'

    with open('test/sanity/shellcheck/skip.txt', 'r') as skip_fd:
        skip_paths = set(skip_fd.read().splitlines())

    with open('test/sanity/shellcheck/exclude.txt', 'r') as exclude_fd:
        exclude = set(exclude_fd.read().splitlines())

    paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths)

    if not paths:
        return SanitySkipped(test)

    cmd = [
        'shellcheck',
        '-e', ','.join(sorted(exclude)),
        '--format', 'checkstyle',
    ] + paths

    try:
        stdout, stderr = run_command(args, cmd, capture=True)
        status = 0
    except SubprocessError as ex:
        stdout = ex.stdout
        stderr = ex.stderr
        status = ex.status

    if stderr or status > 1:
        raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)

    if args.explain:
        return SanitySkipped(test)

    # json output is missing file paths in older versions of shellcheck, so we'll use xml instead
    root = fromstring(stdout)  # type: Element

    results = []

    for item in root:  # type: Element
        for entry in item:  # type: Element
            results.append(SanityMessage(
                message=entry.attrib['message'],
                path=item.attrib['name'],
                line=int(entry.attrib['line']),
                column=int(entry.attrib['column']),
                level=entry.attrib['severity'],
                code=entry.attrib['source'].replace('ShellCheck.', ''),
            ))

    if results:
        return SanityFailure(test, messages=results)

    return SanitySuccess(test)
示例#41
0
def pip_list(args, pip):
    """
    :type args: EnvironmentConfig
    :type pip: str
    :rtype: str
    """
    stdout, _ = run_command(args, [pip, 'list'], capture=True)
    return stdout
示例#42
0
 def run_git(self, cmd):
     """
     :type cmd: list[str]
     :rtype: str
     """
     return run_command(self.args, [self.git] + cmd,
                        capture=True,
                        always=True)[0]
示例#43
0
def inject_httptester(args):
    """
    :type args: CommonConfig
    """
    comment = ' # ansible-test httptester\n'
    append_lines = ['127.0.0.1 %s%s' % (host, comment) for host in HTTPTESTER_HOSTS]

    with open('/etc/hosts', 'r+') as hosts_fd:
        original_lines = hosts_fd.readlines()

        if not any(line.endswith(comment) for line in original_lines):
            hosts_fd.writelines(append_lines)

    # determine which forwarding mechanism to use
    pfctl = find_executable('pfctl', required=False)
    iptables = find_executable('iptables', required=False)

    if pfctl:
        kldload = find_executable('kldload', required=False)

        if kldload:
            try:
                run_command(args, ['kldload', 'pf'], capture=True)
            except SubprocessError:
                pass  # already loaded

        rules = '''
rdr pass inet proto tcp from any to any port 80 -> 127.0.0.1 port 8080
rdr pass inet proto tcp from any to any port 443 -> 127.0.0.1 port 8443
'''
        cmd = ['pfctl', '-ef', '-']

        try:
            run_command(args, cmd, capture=True, data=rules)
        except SubprocessError:
            pass  # non-zero exit status on success

    elif iptables:
        ports = [
            (80, 8080),
            (443, 8443),
        ]

        for src, dst in ports:
            rule = ['-o', 'lo', '-p', 'tcp', '--dport', str(src), '-j', 'REDIRECT', '--to-port', str(dst)]

            try:
                # check for existing rule
                cmd = ['iptables', '-t', 'nat', '-C', 'OUTPUT'] + rule
                run_command(args, cmd, capture=True)
            except SubprocessError:
                # append rule when it does not exist
                cmd = ['iptables', '-t', 'nat', '-A', 'OUTPUT'] + rule
                run_command(args, cmd, capture=True)
    else:
        raise ApplicationError('No supported port forwarding mechanism detected.')
示例#44
0
文件: sanity.py 项目: wdegeus/ansible
def command_sanity_yamllint(args, targets):
    """
    :type args: SanityConfig
    :type targets: SanityTargets
    :rtype: SanityResult
    """
    test = 'yamllint'

    paths = sorted(i.path for i in targets.include
                   if os.path.splitext(i.path)[1] in ('.yml', '.yaml'))

    if not paths:
        return SanitySkipped(test)

    cmd = [
        'yamllint',
        '--format',
        'parsable',
    ] + paths

    try:
        stdout, stderr = run_command(args, cmd, capture=True)
        status = 0
    except SubprocessError as ex:
        stdout = ex.stdout
        stderr = ex.stderr
        status = ex.status

    if stderr:
        raise SubprocessError(cmd=cmd,
                              status=status,
                              stderr=stderr,
                              stdout=stdout)

    if args.explain:
        return SanitySkipped(test)

    pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): \[(?P<level>warning|error)\] (?P<message>.*)$'

    results = [
        re.search(pattern, line).groupdict() for line in stdout.splitlines()
    ]

    results = [
        SanityMessage(
            message=r['message'],
            path=r['path'],
            line=int(r['line']),
            column=int(r['column']),
            level=r['level'],
        ) for r in results
    ]

    if results:
        return SanityFailure(test, messages=results)

    return SanitySuccess(test)
示例#45
0
def docker_pull(args, image):
    """
    :type args: EnvironmentConfig
    :type image: str
    """
    if not args.docker_pull:
        display.warning('Skipping docker pull for "%s". Image may be out-of-date.' % image)
        return

    for _ in range(1, 10):
        try:
            run_command(args, ['docker', 'pull', image])
            return
        except SubprocessError:
            display.warning('Failed to pull docker image "%s". Waiting a few seconds before trying again.' % image)
            time.sleep(3)

    raise ApplicationError('Failed to pull docker image "%s".' % image)
示例#46
0
    def pylint(self, args, context, paths):
        """
        :type args: SanityConfig
        :type context: str
        :type paths: list[str]
        :rtype: list[dict[str, str]]
        """
        rcfile = 'test/sanity/pylint/config/%s' % context

        if not os.path.exists(rcfile):
            rcfile = 'test/sanity/pylint/config/default'

        parser = configparser.SafeConfigParser()
        parser.read(rcfile)

        if parser.has_section('ansible-test'):
            config = dict(parser.items('ansible-test'))
        else:
            config = dict()

        disable_plugins = set(i.strip() for i in config.get('disable-plugins', '').split(',') if i)
        load_plugins = set(self.plugin_names) - disable_plugins

        cmd = [
            args.python_executable,
            '-m', 'pylint',
            '--jobs', '0',
            '--reports', 'n',
            '--max-line-length', '160',
            '--rcfile', rcfile,
            '--output-format', 'json',
            '--load-plugins', ','.join(load_plugins),
        ] + paths

        env = ansible_environment(args)
        env['PYTHONPATH'] += '%s%s' % (os.pathsep, self.plugin_dir)

        if paths:
            try:
                stdout, stderr = run_command(args, cmd, env=env, capture=True)
                status = 0
            except SubprocessError as ex:
                stdout = ex.stdout
                stderr = ex.stderr
                status = ex.status

            if stderr or status >= 32:
                raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)
        else:
            stdout = None

        if not args.explain and stdout:
            messages = json.loads(stdout)
        else:
            messages = []

        return messages
示例#47
0
    def request(self, method, url, data=None, headers=None):
        """
        :type method: str
        :type url: str
        :type data: str | None
        :type headers: dict[str, str] | None
        :rtype: HttpResponse
        """
        cmd = ['curl', '-s', '-S', '-i', '-X', method]

        if headers is None:
            headers = {}

        headers['Expect'] = ''  # don't send expect continue header

        for header in headers.keys():
            cmd += ['-H', '%s: %s' % (header, headers[header])]

        if data is not None:
            cmd += ['-d', data]

        cmd += [url]

        attempts = 0
        max_attempts = 3
        sleep_seconds = 3

        # curl error codes which are safe to retry (request never sent to server)
        retry_on_status = (
            6,  # CURLE_COULDNT_RESOLVE_HOST
        )

        while True:
            attempts += 1

            try:
                stdout, _ = run_command(self.args, cmd, capture=True, always=self.always, cmd_verbosity=2)
                break
            except SubprocessError as ex:
                if ex.status in retry_on_status and attempts < max_attempts:
                    display.warning(u'%s' % ex)
                    time.sleep(sleep_seconds)
                    continue

                raise

        if self.args.explain and not self.always:
            return HttpResponse(method, url, 200, '')

        header, body = stdout.split('\r\n\r\n', 1)

        response_headers = header.split('\r\n')
        first_line = response_headers[0]
        http_response = first_line.split(' ')
        status_code = int(http_response[1])

        return HttpResponse(method, url, status_code, body)
示例#48
0
    def test(self, args, targets):
        """
        :type args: SanityConfig
        :type targets: SanityTargets
        :rtype: TestResult
        """
        with open('test/sanity/shellcheck/skip.txt', 'r') as skip_fd:
            skip_paths = set(skip_fd.read().splitlines())

        with open('test/sanity/shellcheck/exclude.txt', 'r') as exclude_fd:
            exclude = set(exclude_fd.read().splitlines())

        paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] == '.sh' and i.path not in skip_paths)

        if not paths:
            return SanitySkipped(self.name)

        cmd = [
            'shellcheck',
            '-e', ','.join(sorted(exclude)),
            '--format', 'checkstyle',
        ] + paths

        try:
            stdout, stderr = run_command(args, cmd, capture=True)
            status = 0
        except SubprocessError as ex:
            stdout = ex.stdout
            stderr = ex.stderr
            status = ex.status

        if stderr or status > 1:
            raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)

        if args.explain:
            return SanitySuccess(self.name)

        # json output is missing file paths in older versions of shellcheck, so we'll use xml instead
        root = fromstring(stdout)  # type: Element

        results = []

        for item in root:  # type: Element
            for entry in item:  # type: Element
                results.append(SanityMessage(
                    message=entry.attrib['message'],
                    path=item.attrib['name'],
                    line=int(entry.attrib['line']),
                    column=int(entry.attrib['column']),
                    level=entry.attrib['severity'],
                    code=entry.attrib['source'].replace('ShellCheck.', ''),
                ))

        if results:
            return SanityFailure(self.name, messages=results)

        return SanitySuccess(self.name)
示例#49
0
    def test(self, args, targets):
        """
        :type args: SanityConfig
        :type targets: SanityTargets
        :rtype: TestResult
        """
        if args.python_version in UNSUPPORTED_PYTHON_VERSIONS:
            display.warning('Skipping rstcheck on unsupported Python version %s.' % args.python_version)
            return SanitySkipped(self.name)

        with open('test/sanity/rstcheck/ignore-substitutions.txt', 'r') as ignore_fd:
            ignore_substitutions = sorted(set(ignore_fd.read().splitlines()))

        paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.rst',))

        if not paths:
            return SanitySkipped(self.name)

        cmd = [
            args.python_executable,
            '-m', 'rstcheck',
            '--report', 'warning',
            '--ignore-substitutions', ','.join(ignore_substitutions),
        ] + paths

        try:
            stdout, stderr = run_command(args, cmd, capture=True)
            status = 0
        except SubprocessError as ex:
            stdout = ex.stdout
            stderr = ex.stderr
            status = ex.status

        if stdout:
            raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)

        if args.explain:
            return SanitySuccess(self.name)

        pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+): \((?P<level>INFO|WARNING|ERROR|SEVERE)/[0-4]\) (?P<message>.*)$'

        results = [parse_to_dict(pattern, line) for line in stderr.splitlines()]

        results = [SanityMessage(
            message=r['message'],
            path=r['path'],
            line=int(r['line']),
            column=0,
            level=r['level'],
        ) for r in results]

        if results:
            return SanityFailure(self.name, messages=results)

        return SanitySuccess(self.name)
示例#50
0
    def test(self, args, targets):
        """
        :type args: SanityConfig
        :type targets: SanityTargets
        :rtype: TestResult
        """
        if args.python_version in UNSUPPORTED_PYTHON_VERSIONS:
            display.warning('Skipping rstcheck on unsupported Python version %s.' % args.python_version)
            return SanitySkipped(self.name)

        ignore_file = 'test/sanity/rstcheck/ignore-substitutions.txt'
        ignore_substitutions = sorted(set(read_lines_without_comments(ignore_file, remove_blank_lines=True)))

        paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.rst',))

        if not paths:
            return SanitySkipped(self.name)

        cmd = [
            args.python_executable,
            '-m', 'rstcheck',
            '--report', 'warning',
            '--ignore-substitutions', ','.join(ignore_substitutions),
        ] + paths

        try:
            stdout, stderr = run_command(args, cmd, capture=True)
            status = 0
        except SubprocessError as ex:
            stdout = ex.stdout
            stderr = ex.stderr
            status = ex.status

        if stdout:
            raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)

        if args.explain:
            return SanitySuccess(self.name)

        pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+): \((?P<level>INFO|WARNING|ERROR|SEVERE)/[0-4]\) (?P<message>.*)$'

        results = [parse_to_dict(pattern, line) for line in stderr.splitlines()]

        results = [SanityMessage(
            message=r['message'],
            path=r['path'],
            line=int(r['line']),
            column=0,
            level=r['level'],
        ) for r in results]

        if results:
            return SanityFailure(self.name, messages=results)

        return SanitySuccess(self.name)
示例#51
0
def delegate_tox(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    if args.python:
        versions = args.python,

        if args.python not in SUPPORTED_PYTHON_VERSIONS:
            raise ApplicationError('tox does not support Python version %s' %
                                   args.python)
    else:
        versions = SUPPORTED_PYTHON_VERSIONS

    options = {
        '--tox': args.tox_args,
        '--tox-sitepackages': 0,
    }

    for version in versions:
        tox = [
            'tox', '-c', 'test/runner/tox.ini', '-e',
            'py' + version.replace('.', '')
        ]

        if args.tox_sitepackages:
            tox.append('--sitepackages')

        tox.append('--')

        cmd = generate_command(args, os.path.abspath('test/runner/test.py'),
                               options, exclude, require)

        if not args.python:
            cmd += ['--python', version]

        if isinstance(args, TestConfig):
            if args.coverage and not args.coverage_label:
                cmd += ['--coverage-label', 'tox-%s' % version]

        run_command(args, tox + cmd)
示例#52
0
    def __init__(self, args):
        """
        :type args: CommonConfig
        """
        tmp = os.path.expanduser('~/.ansible/test/')

        self.key = os.path.join(tmp, 'id_rsa')
        self.pub = os.path.join(tmp, 'id_rsa.pub')

        if not os.path.isfile(self.pub):
            if not args.explain:
                make_dirs(tmp)

            run_command(args, ['ssh-keygen', '-q', '-t', 'rsa', '-N', '', '-f', self.key])

        if args.explain:
            self.pub_contents = None
        else:
            with open(self.pub, 'r') as pub_fd:
                self.pub_contents = pub_fd.read().strip()
示例#53
0
def docker_command(args, cmd, capture=False, stdin=None, stdout=None):
    """
    :type args: EnvironmentConfig
    :type cmd: list[str]
    :type capture: bool
    :type stdin: file | None
    :type stdout: file | None
    :rtype: str | None, str | None
    """
    env = docker_environment()
    return run_command(args, ['docker'] + cmd, env=env, capture=capture, stdin=stdin, stdout=stdout)
示例#54
0
def install_command_requirements(args):
    """
    :type args: EnvironmentConfig
    """
    generate_egg_info(args)

    if not args.requirements:
        return

    cmd = generate_pip_install(args.command)

    if not cmd:
        return

    if isinstance(args, TestConfig):
        if args.coverage:
            cmd += ['coverage']

    try:
        run_command(args, cmd)
    except SubprocessError as ex:
        if ex.status != 2:
            raise

        # If pip is too old it won't understand the arguments we passed in, so we'll need to upgrade it.

        # Installing "coverage" on ubuntu 16.04 fails with the error:
        # AttributeError: 'Requirement' object has no attribute 'project_name'
        # See: https://bugs.launchpad.net/ubuntu/xenial/+source/python-pip/+bug/1626258
        # Upgrading pip works around the issue.
        run_command(args, ['pip', 'install', '--upgrade', 'pip'])
        run_command(args, cmd)
示例#55
0
    def disable_pendo(self):
        """Disable Pendo tracking."""
        display.info('Disable Pendo tracking')

        config = TowerConfig.parse(self.config_path)

        # tower-cli does not recognize TOWER_ environment variables
        cmd = ['tower-cli', 'setting', 'modify', 'PENDO_TRACKING_STATE', 'off',
               '-h', config.host, '-u', config.username, '-p', config.password]

        attempts = 60

        while True:
            attempts -= 1

            try:
                run_command(self.args, cmd, capture=True)
                return
            except SubprocessError as ex:
                if not attempts:
                    raise ApplicationError('Timed out trying to disable Pendo tracking:\n%s' % ex)

            time.sleep(5)
示例#56
0
def command_coverage_report(args):
    """
    :type args: CoverageReportConfig
    """
    output_files = command_coverage_combine(args)

    for output_file in output_files:
        if args.group_by or args.stub:
            display.info('>>> Coverage Group: %s' % ' '.join(os.path.basename(output_file).split('=')[1:]))

        options = []

        if args.show_missing:
            options.append('--show-missing')

        if args.include:
            options.extend(['--include', args.include])

        if args.omit:
            options.extend(['--omit', args.omit])

        env = common_environment()
        env.update(dict(COVERAGE_FILE=output_file))
        run_command(args, env=env, cmd=['coverage', 'report'] + options)
示例#57
0
 def get_containers(status=Status.All, filter=None):
     cmd = CMD
     if filter:
         cmd = "{} --filter '{}'".format(cmd, filter)
     try:
         ps_strings = run_command(cmd).split(os.linesep)
     except:
         log.error("Unable to retrieve containers")
     containers = []
     for str in ps_strings:
         if len(str.split(DELIMITER)) != 10:
             continue
         container = Container(str)
         if status == Status.All or container.status == status:
             containers.append(container)
     return containers
示例#58
0
    def test(self, args, targets):
        """
        :type args: SanityConfig
        :type targets: SanityTargets
        :rtype: SanityResult
        """
        paths = sorted(i.path for i in targets.include if os.path.splitext(i.path)[1] in ('.yml', '.yaml'))

        if not paths:
            return SanitySkipped(self.name)

        cmd = [
            'yamllint',
            '--format', 'parsable',
        ] + paths

        try:
            stdout, stderr = run_command(args, cmd, capture=True)
            status = 0
        except SubprocessError as ex:
            stdout = ex.stdout
            stderr = ex.stderr
            status = ex.status

        if stderr:
            raise SubprocessError(cmd=cmd, status=status, stderr=stderr, stdout=stdout)

        if args.explain:
            return SanitySuccess(self.name)

        pattern = r'^(?P<path>[^:]*):(?P<line>[0-9]+):(?P<column>[0-9]+): \[(?P<level>warning|error)\] (?P<message>.*)$'

        results = [re.search(pattern, line).groupdict() for line in stdout.splitlines()]

        results = [SanityMessage(
            message=r['message'],
            path=r['path'],
            line=int(r['line']),
            column=int(r['column']),
            level=r['level'],
        ) for r in results]

        if results:
            return SanityFailure(self.name, messages=results)

        return SanitySuccess(self.name)