Ejemplo n.º 1
0
def delegate_remote(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    parts = args.remote.split('/', 1)

    platform = parts[0]
    version = parts[1]

    core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage)
    success = False

    try:
        core_ci.start()
        core_ci.wait()

        options = {
            '--remote': 1,
        }

        cmd = generate_command(args, 'ansible/test/runner/test.py', options, exclude, require)

        if isinstance(args, TestConfig):
            if args.coverage and not args.coverage_label:
                cmd += ['--coverage-label', 'remote-%s-%s' % (platform, version)]

        if isinstance(args, IntegrationConfig):
            if not args.allow_destructive:
                cmd.append('--allow-destructive')

        # remote instances are only expected to have a single python version available
        if isinstance(args, UnitsConfig) and not args.python:
            cmd += ['--python', 'default']

        manage = ManagePosixCI(core_ci)
        manage.setup()

        ssh_options = []

        if isinstance(args, IntegrationConfig):
            cloud_platforms = get_cloud_providers(args)

            for cloud_platform in cloud_platforms:
                ssh_options += cloud_platform.get_remote_ssh_options()

        try:
            manage.ssh(cmd, ssh_options)
            success = True
        finally:
            manage.ssh('rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results')
            manage.download('/tmp/results', 'test')
    finally:
        if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success):
            core_ci.stop()
Ejemplo n.º 2
0
def delegate_docker(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    util_image = args.docker_util
    test_image = args.docker
    privileged = args.docker_privileged

    if util_image:
        docker_pull(args, util_image)

    docker_pull(args, test_image)

    util_id = None
    test_id = None

    options = {
        '--docker': 1,
        '--docker-privileged': 0,
        '--docker-util': 1,
    }

    cmd = generate_command(args, '/root/ansible/test/runner/test.py', options, exclude, require)

    if isinstance(args, TestConfig):
        if args.coverage and not args.coverage_label:
            image_label = re.sub('^ansible/ansible:', '', args.docker)
            image_label = re.sub('[^a-zA-Z0-9]+', '-', image_label)
            cmd += ['--coverage-label', 'docker-%s' % image_label]

    if isinstance(args, IntegrationConfig):
        if not args.allow_destructive:
            cmd.append('--allow-destructive')

    cmd_options = []

    if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig) and args.debug_strategy):
        cmd_options.append('-it')

    with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd:
        try:
            if not args.explain:
                lib.pytar.create_tarfile(local_source_fd.name, '.', lib.pytar.ignore)

            if util_image:
                util_options = [
                    '--detach',
                ]

                util_id, _ = docker_run(args, util_image, options=util_options)

                if args.explain:
                    util_id = 'util_id'
                else:
                    util_id = util_id.strip()
            else:
                util_id = None

            test_options = [
                '--detach',
                '--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro',
                '--privileged=%s' % str(privileged).lower(),
            ]

            docker_socket = '/var/run/docker.sock'

            if os.path.exists(docker_socket):
                test_options += ['--volume', '%s:%s' % (docker_socket, docker_socket)]

            if util_id:
                test_options += [
                    '--link', '%s:ansible.http.tests' % util_id,
                    '--link', '%s:sni1.ansible.http.tests' % util_id,
                    '--link', '%s:sni2.ansible.http.tests' % util_id,
                    '--link', '%s:fail.ansible.http.tests' % util_id,
                    '--env', 'HTTPTESTER=1',
                ]

            if isinstance(args, IntegrationConfig):
                cloud_platforms = get_cloud_providers(args)

                for cloud_platform in cloud_platforms:
                    test_options += cloud_platform.get_docker_run_options()

            test_id, _ = docker_run(args, test_image, options=test_options)

            if args.explain:
                test_id = 'test_id'
            else:
                test_id = test_id.strip()

            # write temporary files to /root since /tmp isn't ready immediately on container start
            docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh')
            docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh'])
            docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz')
            docker_exec(args, test_id, ['mkdir', '/root/ansible'])
            docker_exec(args, test_id, ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible'])

            # docker images are only expected to have a single python version available
            if isinstance(args, UnitsConfig) and not args.python:
                cmd += ['--python', 'default']

            try:
                docker_exec(args, test_id, cmd, options=cmd_options)
            finally:
                with tempfile.NamedTemporaryFile(prefix='ansible-result-', suffix='.tgz') as local_result_fd:
                    docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', '/root/ansible/test', 'results'])
                    docker_get(args, test_id, '/root/results.tgz', local_result_fd.name)
                    run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test'])
        finally:
            if util_id:
                docker_rm(args, util_id)

            if test_id:
                docker_rm(args, test_id)
Ejemplo n.º 3
0
def delegate_docker(args, exclude, require, integration_targets):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    :type integration_targets: tuple[IntegrationTarget]
    """
    test_image = args.docker
    privileged = args.docker_privileged

    if isinstance(args, ShellConfig):
        use_httptester = args.httptester
    else:
        use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets)

    if use_httptester:
        docker_pull(args, args.httptester)

    docker_pull(args, test_image)

    httptester_id = None
    test_id = None

    options = {
        '--docker': 1,
        '--docker-privileged': 0,
        '--docker-util': 1,
    }

    python_interpreter = get_python_interpreter(args, get_docker_completion(), args.docker_raw)

    install_root = '/root/ansible'

    if data_context().content.collection:
        content_root = os.path.join(install_root, data_context().content.collection.directory)
    else:
        content_root = install_root

    cmd = generate_command(args, python_interpreter, install_root, content_root, options, exclude, require)

    if isinstance(args, TestConfig):
        if args.coverage and not args.coverage_label:
            image_label = args.docker_raw
            image_label = re.sub('[^a-zA-Z0-9]+', '-', image_label)
            cmd += ['--coverage-label', 'docker-%s' % image_label]

    if isinstance(args, IntegrationConfig):
        if not args.allow_destructive:
            cmd.append('--allow-destructive')

    cmd_options = []

    if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig) and args.debug_strategy):
        cmd_options.append('-it')

    with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd:
        try:
            create_payload(args, local_source_fd.name)

            if use_httptester:
                httptester_id = run_httptester(args)
            else:
                httptester_id = None

            test_options = [
                '--detach',
                '--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro',
                '--privileged=%s' % str(privileged).lower(),
            ]

            if args.docker_memory:
                test_options.extend([
                    '--memory=%d' % args.docker_memory,
                    '--memory-swap=%d' % args.docker_memory,
                ])

            docker_socket = '/var/run/docker.sock'

            if args.docker_seccomp != 'default':
                test_options += ['--security-opt', 'seccomp=%s' % args.docker_seccomp]

            if os.path.exists(docker_socket):
                test_options += ['--volume', '%s:%s' % (docker_socket, docker_socket)]

            if httptester_id:
                test_options += ['--env', 'HTTPTESTER=1']

                for host in HTTPTESTER_HOSTS:
                    test_options += ['--link', '%s:%s' % (httptester_id, host)]

            if isinstance(args, IntegrationConfig):
                cloud_platforms = get_cloud_providers(args)

                for cloud_platform in cloud_platforms:
                    test_options += cloud_platform.get_docker_run_options()

            test_id = docker_run(args, test_image, options=test_options)[0]

            if args.explain:
                test_id = 'test_id'
            else:
                test_id = test_id.strip()

            # write temporary files to /root since /tmp isn't ready immediately on container start
            docker_put(args, test_id, os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'docker.sh'), '/root/docker.sh')
            docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh'])
            docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz')
            docker_exec(args, test_id, ['mkdir', '/root/ansible'])
            docker_exec(args, test_id, ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible'])

            # docker images are only expected to have a single python version available
            if isinstance(args, UnitsConfig) and not args.python:
                cmd += ['--python', 'default']

            # run unit tests unprivileged to prevent stray writes to the source tree
            # also disconnect from the network once requirements have been installed
            if isinstance(args, UnitsConfig):
                writable_dirs = [
                    os.path.join(install_root, '.pytest_cache'),
                ]

                if content_root != install_root:
                    writable_dirs.append(os.path.join(content_root, 'test/results/junit'))
                    writable_dirs.append(os.path.join(content_root, 'test/results/coverage'))

                docker_exec(args, test_id, ['mkdir', '-p'] + writable_dirs)
                docker_exec(args, test_id, ['chmod', '777'] + writable_dirs)

                if content_root == install_root:
                    docker_exec(args, test_id, ['find', os.path.join(content_root, 'test/results/'), '-type', 'd', '-exec', 'chmod', '777', '{}', '+'])

                docker_exec(args, test_id, ['chmod', '755', '/root'])
                docker_exec(args, test_id, ['chmod', '644', os.path.join(content_root, args.metadata_path)])

                docker_exec(args, test_id, ['useradd', 'pytest', '--create-home'])

                docker_exec(args, test_id, cmd + ['--requirements-mode', 'only'], options=cmd_options)

                networks = get_docker_networks(args, test_id)

                for network in networks:
                    docker_network_disconnect(args, test_id, network)

                cmd += ['--requirements-mode', 'skip']

                cmd_options += ['--user', 'pytest']

            try:
                docker_exec(args, test_id, cmd, options=cmd_options)
            finally:
                with tempfile.NamedTemporaryFile(prefix='ansible-result-', suffix='.tgz') as local_result_fd:
                    docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', os.path.join(content_root, 'test'), 'results'])
                    docker_get(args, test_id, '/root/results.tgz', local_result_fd.name)
                    run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test'])
        finally:
            if httptester_id:
                docker_rm(args, httptester_id)

            if test_id:
                docker_rm(args, test_id)
Ejemplo n.º 4
0
def delegate_remote(args, exclude, require, integration_targets):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    :type integration_targets: tuple[IntegrationTarget]
    """
    parts = args.remote.split('/', 1)

    platform = parts[0]
    version = parts[1]

    core_ci = AnsibleCoreCI(args,
                            platform,
                            version,
                            stage=args.remote_stage,
                            provider=args.remote_provider)
    success = False

    if isinstance(args, ShellConfig):
        use_httptester = args.httptester
    else:
        use_httptester = args.httptester and any(
            'needs/httptester/' in target.aliases
            for target in integration_targets)

    if use_httptester and not docker_available():
        display.warning(
            'Assuming --disable-httptester since `docker` is not available.')
        use_httptester = False

    httptester_id = None
    ssh_options = []

    try:
        core_ci.start()

        if use_httptester:
            httptester_id, ssh_options = start_httptester(args)

        core_ci.wait()

        if platform == 'windows':
            # Windows doesn't need the ansible-test fluff, just run the SSH command
            manage = ManageWindowsCI(core_ci)
            cmd = ['powershell.exe']
        else:
            options = {
                '--remote': 1,
            }

            cmd = generate_command(args, 'ansible/test/runner/test.py',
                                   options, exclude, require)

            if httptester_id:
                cmd += ['--inject-httptester']

            if isinstance(args, TestConfig):
                if args.coverage and not args.coverage_label:
                    cmd += [
                        '--coverage-label',
                        'remote-%s-%s' % (platform, version)
                    ]

            if isinstance(args, IntegrationConfig):
                if not args.allow_destructive:
                    cmd.append('--allow-destructive')

            # remote instances are only expected to have a single python version available
            if isinstance(args, UnitsConfig) and not args.python:
                cmd += ['--python', 'default']

            manage = ManagePosixCI(core_ci)

        manage.setup()
        if isinstance(args, IntegrationConfig):
            cloud_platforms = get_cloud_providers(args)

            for cloud_platform in cloud_platforms:
                ssh_options += cloud_platform.get_remote_ssh_options()

        try:
            manage.ssh(cmd, ssh_options)
            success = True
        finally:
            if platform != 'windows':
                manage.ssh(
                    'rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results'
                )
                manage.download('/tmp/results', 'test')
    finally:
        if args.remote_terminate == 'always' or (args.remote_terminate
                                                 == 'success' and success):
            core_ci.stop()

        if httptester_id:
            docker_rm(args, httptester_id)
Ejemplo n.º 5
0
def delegate_docker(args, exclude, require, integration_targets):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    :type integration_targets: tuple[IntegrationTarget]
    """
    test_image = args.docker
    privileged = args.docker_privileged

    if isinstance(args, ShellConfig):
        use_httptester = args.httptester
    else:
        use_httptester = args.httptester and any(
            'needs/httptester/' in target.aliases
            for target in integration_targets)

    if use_httptester:
        docker_pull(args, args.httptester)

    docker_pull(args, test_image)

    httptester_id = None
    test_id = None

    options = {
        '--docker': 1,
        '--docker-privileged': 0,
        '--docker-util': 1,
    }

    cmd = generate_command(args, '/root/ansible/test/runner/test.py', options,
                           exclude, require)

    if isinstance(args, TestConfig):
        if args.coverage and not args.coverage_label:
            image_label = re.sub('^ansible/ansible:', '', args.docker)
            image_label = re.sub('[^a-zA-Z0-9]+', '-', image_label)
            cmd += ['--coverage-label', 'docker-%s' % image_label]

    if isinstance(args, IntegrationConfig):
        if not args.allow_destructive:
            cmd.append('--allow-destructive')

    cmd_options = []

    if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig)
                                         and args.debug_strategy):
        cmd_options.append('-it')

    with tempfile.NamedTemporaryFile(prefix='ansible-source-',
                                     suffix='.tgz') as local_source_fd:
        try:
            if not args.explain:
                if args.docker_keep_git:
                    tar_filter = lib.pytar.AllowGitTarFilter()
                else:
                    tar_filter = lib.pytar.DefaultTarFilter()

                lib.pytar.create_tarfile(local_source_fd.name, '.', tar_filter)

            if use_httptester:
                httptester_id = run_httptester(args)
            else:
                httptester_id = None

            test_options = [
                '--detach',
                '--volume',
                '/sys/fs/cgroup:/sys/fs/cgroup:ro',
                '--privileged=%s' % str(privileged).lower(),
            ]

            if args.docker_memory:
                test_options.extend([
                    '--memory=%d' % args.docker_memory,
                    '--memory-swap=%d' % args.docker_memory,
                ])

            docker_socket = '/var/run/docker.sock'

            if os.path.exists(docker_socket):
                test_options += [
                    '--volume',
                    '%s:%s' % (docker_socket, docker_socket)
                ]

            if httptester_id:
                test_options += ['--env', 'HTTPTESTER=1']

                for host in HTTPTESTER_HOSTS:
                    test_options += ['--link', '%s:%s' % (httptester_id, host)]

            if isinstance(args, IntegrationConfig):
                cloud_platforms = get_cloud_providers(args)

                for cloud_platform in cloud_platforms:
                    test_options += cloud_platform.get_docker_run_options()

            test_id, _ = docker_run(args, test_image, options=test_options)

            if args.explain:
                test_id = 'test_id'
            else:
                test_id = test_id.strip()

            # write temporary files to /root since /tmp isn't ready immediately on container start
            docker_put(args, test_id, 'test/runner/setup/docker.sh',
                       '/root/docker.sh')
            docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh'])
            docker_put(args, test_id, local_source_fd.name,
                       '/root/ansible.tgz')
            docker_exec(args, test_id, ['mkdir', '/root/ansible'])
            docker_exec(
                args, test_id,
                ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible'])

            # docker images are only expected to have a single python version available
            if isinstance(args, UnitsConfig) and not args.python:
                cmd += ['--python', 'default']

            try:
                docker_exec(args, test_id, cmd, options=cmd_options)
            finally:
                with tempfile.NamedTemporaryFile(
                        prefix='ansible-result-',
                        suffix='.tgz') as local_result_fd:
                    docker_exec(args, test_id, [
                        'tar', 'czf', '/root/results.tgz', '-C',
                        '/root/ansible/test', 'results'
                    ])
                    docker_get(args, test_id, '/root/results.tgz',
                               local_result_fd.name)
                    run_command(
                        args,
                        ['tar', 'oxzf', local_result_fd.name, '-C', 'test'])
        finally:
            if httptester_id:
                docker_rm(args, httptester_id)

            if test_id:
                docker_rm(args, test_id)
Ejemplo n.º 6
0
def delegate_docker(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    util_image = args.docker_util
    test_image = args.docker
    privileged = args.docker_privileged

    if util_image:
        docker_pull(args, util_image)

    docker_pull(args, test_image)

    util_id = None
    test_id = None

    options = {
        '--docker': 1,
        '--docker-privileged': 0,
        '--docker-util': 1,
    }

    cmd = generate_command(args, '/root/ansible/test/runner/test.py', options,
                           exclude, require)

    if isinstance(args, IntegrationConfig):
        if not args.allow_destructive:
            cmd.append('--allow-destructive')

    cmd_options = []

    if isinstance(args, ShellConfig):
        cmd_options.append('-it')

    if not args.explain:
        lib.pytar.create_tarfile('/tmp/ansible.tgz', '.', lib.pytar.ignore)

    try:
        if util_image:
            util_options = [
                '--detach',
            ]

            util_id, _ = docker_run(args, util_image, options=util_options)

            if args.explain:
                util_id = 'util_id'
            else:
                util_id = util_id.strip()
        else:
            util_id = None

        test_options = [
            '--detach',
            '--volume',
            '/sys/fs/cgroup:/sys/fs/cgroup:ro',
            '--privileged=%s' % str(privileged).lower(),
        ]

        if util_id:
            test_options += [
                '--link',
                '%s:ansible.http.tests' % util_id,
                '--link',
                '%s:sni1.ansible.http.tests' % util_id,
                '--link',
                '%s:sni2.ansible.http.tests' % util_id,
                '--link',
                '%s:fail.ansible.http.tests' % util_id,
                '--env',
                'HTTPTESTER=1',
            ]

        if isinstance(args, TestConfig):
            cloud_platforms = get_cloud_providers(args)

            for cloud_platform in cloud_platforms:
                test_options += cloud_platform.get_docker_run_options()

        test_id, _ = docker_run(args, test_image, options=test_options)

        if args.explain:
            test_id = 'test_id'
        else:
            test_id = test_id.strip()

        # write temporary files to /root since /tmp isn't ready immediately on container start
        docker_put(args, test_id, 'test/runner/setup/docker.sh',
                   '/root/docker.sh')
        docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh'])
        docker_put(args, test_id, '/tmp/ansible.tgz', '/root/ansible.tgz')
        docker_exec(args, test_id, ['mkdir', '/root/ansible'])
        docker_exec(
            args, test_id,
            ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible'])

        # docker images are only expected to have a single python version available
        if isinstance(args, UnitsConfig) and not args.python:
            cmd += ['--python', 'default']

        try:
            docker_exec(args, test_id, cmd, options=cmd_options)
        finally:
            docker_exec(args, test_id, [
                'tar', 'czf', '/root/results.tgz', '-C', '/root/ansible/test',
                'results'
            ])
            docker_get(args, test_id, '/root/results.tgz', '/tmp/results.tgz')
            run_command(args,
                        ['tar', 'oxzf', '/tmp/results.tgz', '-C', 'test'])
    finally:
        if util_id:
            docker_rm(args, util_id)

        if test_id:
            docker_rm(args, test_id)
Ejemplo n.º 7
0
def delegate_remote(args, exclude, require):
    """
    :type args: EnvironmentConfig
    :type exclude: list[str]
    :type require: list[str]
    """
    parts = args.remote.split('/', 1)

    platform = parts[0]
    version = parts[1]

    core_ci = AnsibleCoreCI(args,
                            platform,
                            version,
                            stage=args.remote_stage,
                            provider=args.remote_provider)
    success = False

    try:
        core_ci.start()
        core_ci.wait()

        options = {
            '--remote': 1,
        }

        cmd = generate_command(args, 'ansible/test/runner/test.py', options,
                               exclude, require)

        if isinstance(args, TestConfig):
            if args.coverage and not args.coverage_label:
                cmd += [
                    '--coverage-label',
                    'remote-%s-%s' % (platform, version)
                ]

        if isinstance(args, IntegrationConfig):
            if not args.allow_destructive:
                cmd.append('--allow-destructive')

        # remote instances are only expected to have a single python version available
        if isinstance(args, UnitsConfig) and not args.python:
            cmd += ['--python', 'default']

        manage = ManagePosixCI(core_ci)
        manage.setup()

        ssh_options = []

        if isinstance(args, IntegrationConfig):
            cloud_platforms = get_cloud_providers(args)

            for cloud_platform in cloud_platforms:
                ssh_options += cloud_platform.get_remote_ssh_options()

        try:
            manage.ssh(cmd, ssh_options)
            success = True
        finally:
            manage.ssh(
                'rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results'
            )
            manage.download('/tmp/results', 'test')
    finally:
        if args.remote_terminate == 'always' or (args.remote_terminate
                                                 == 'success' and success):
            core_ci.stop()