def delegate_remote(args, exclude, require): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] """ parts = args.remote.split('/', 1) platform = parts[0] version = parts[1] core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage) success = False try: core_ci.start() core_ci.wait() options = { '--remote': 1, } cmd = generate_command(args, 'ansible/test/runner/test.py', options, exclude, require) if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: cmd += ['--coverage-label', 'remote-%s-%s' % (platform, version)] if isinstance(args, IntegrationConfig): if not args.allow_destructive: cmd.append('--allow-destructive') # remote instances are only expected to have a single python version available if isinstance(args, UnitsConfig) and not args.python: cmd += ['--python', 'default'] manage = ManagePosixCI(core_ci) manage.setup() ssh_options = [] if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) for cloud_platform in cloud_platforms: ssh_options += cloud_platform.get_remote_ssh_options() try: manage.ssh(cmd, ssh_options) success = True finally: manage.ssh('rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results') manage.download('/tmp/results', 'test') finally: if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success): core_ci.stop()
def delegate_docker(args, exclude, require): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] """ util_image = args.docker_util test_image = args.docker privileged = args.docker_privileged if util_image: docker_pull(args, util_image) docker_pull(args, test_image) util_id = None test_id = None options = { '--docker': 1, '--docker-privileged': 0, '--docker-util': 1, } cmd = generate_command(args, '/root/ansible/test/runner/test.py', options, exclude, require) if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: image_label = re.sub('^ansible/ansible:', '', args.docker) image_label = re.sub('[^a-zA-Z0-9]+', '-', image_label) cmd += ['--coverage-label', 'docker-%s' % image_label] if isinstance(args, IntegrationConfig): if not args.allow_destructive: cmd.append('--allow-destructive') cmd_options = [] if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig) and args.debug_strategy): cmd_options.append('-it') with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd: try: if not args.explain: lib.pytar.create_tarfile(local_source_fd.name, '.', lib.pytar.ignore) if util_image: util_options = [ '--detach', ] util_id, _ = docker_run(args, util_image, options=util_options) if args.explain: util_id = 'util_id' else: util_id = util_id.strip() else: util_id = None test_options = [ '--detach', '--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro', '--privileged=%s' % str(privileged).lower(), ] docker_socket = '/var/run/docker.sock' if os.path.exists(docker_socket): test_options += ['--volume', '%s:%s' % (docker_socket, docker_socket)] if util_id: test_options += [ '--link', '%s:ansible.http.tests' % util_id, '--link', '%s:sni1.ansible.http.tests' % util_id, '--link', '%s:sni2.ansible.http.tests' % util_id, '--link', '%s:fail.ansible.http.tests' % util_id, '--env', 'HTTPTESTER=1', ] if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) for cloud_platform in cloud_platforms: test_options += cloud_platform.get_docker_run_options() test_id, _ = docker_run(args, test_image, options=test_options) if args.explain: test_id = 'test_id' else: test_id = test_id.strip() # write temporary files to /root since /tmp isn't ready immediately on container start docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh') docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh']) docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz') docker_exec(args, test_id, ['mkdir', '/root/ansible']) docker_exec(args, test_id, ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible']) # docker images are only expected to have a single python version available if isinstance(args, UnitsConfig) and not args.python: cmd += ['--python', 'default'] try: docker_exec(args, test_id, cmd, options=cmd_options) finally: with tempfile.NamedTemporaryFile(prefix='ansible-result-', suffix='.tgz') as local_result_fd: docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', '/root/ansible/test', 'results']) docker_get(args, test_id, '/root/results.tgz', local_result_fd.name) run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test']) finally: if util_id: docker_rm(args, util_id) if test_id: docker_rm(args, test_id)
def delegate_docker(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] :type integration_targets: tuple[IntegrationTarget] """ test_image = args.docker privileged = args.docker_privileged if isinstance(args, ShellConfig): use_httptester = args.httptester else: use_httptester = args.httptester and any('needs/httptester/' in target.aliases for target in integration_targets) if use_httptester: docker_pull(args, args.httptester) docker_pull(args, test_image) httptester_id = None test_id = None options = { '--docker': 1, '--docker-privileged': 0, '--docker-util': 1, } python_interpreter = get_python_interpreter(args, get_docker_completion(), args.docker_raw) install_root = '/root/ansible' if data_context().content.collection: content_root = os.path.join(install_root, data_context().content.collection.directory) else: content_root = install_root cmd = generate_command(args, python_interpreter, install_root, content_root, options, exclude, require) if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: image_label = args.docker_raw image_label = re.sub('[^a-zA-Z0-9]+', '-', image_label) cmd += ['--coverage-label', 'docker-%s' % image_label] if isinstance(args, IntegrationConfig): if not args.allow_destructive: cmd.append('--allow-destructive') cmd_options = [] if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig) and args.debug_strategy): cmd_options.append('-it') with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd: try: create_payload(args, local_source_fd.name) if use_httptester: httptester_id = run_httptester(args) else: httptester_id = None test_options = [ '--detach', '--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro', '--privileged=%s' % str(privileged).lower(), ] if args.docker_memory: test_options.extend([ '--memory=%d' % args.docker_memory, '--memory-swap=%d' % args.docker_memory, ]) docker_socket = '/var/run/docker.sock' if args.docker_seccomp != 'default': test_options += ['--security-opt', 'seccomp=%s' % args.docker_seccomp] if os.path.exists(docker_socket): test_options += ['--volume', '%s:%s' % (docker_socket, docker_socket)] if httptester_id: test_options += ['--env', 'HTTPTESTER=1'] for host in HTTPTESTER_HOSTS: test_options += ['--link', '%s:%s' % (httptester_id, host)] if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) for cloud_platform in cloud_platforms: test_options += cloud_platform.get_docker_run_options() test_id = docker_run(args, test_image, options=test_options)[0] if args.explain: test_id = 'test_id' else: test_id = test_id.strip() # write temporary files to /root since /tmp isn't ready immediately on container start docker_put(args, test_id, os.path.join(ANSIBLE_TEST_DATA_ROOT, 'setup', 'docker.sh'), '/root/docker.sh') docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh']) docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz') docker_exec(args, test_id, ['mkdir', '/root/ansible']) docker_exec(args, test_id, ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible']) # docker images are only expected to have a single python version available if isinstance(args, UnitsConfig) and not args.python: cmd += ['--python', 'default'] # run unit tests unprivileged to prevent stray writes to the source tree # also disconnect from the network once requirements have been installed if isinstance(args, UnitsConfig): writable_dirs = [ os.path.join(install_root, '.pytest_cache'), ] if content_root != install_root: writable_dirs.append(os.path.join(content_root, 'test/results/junit')) writable_dirs.append(os.path.join(content_root, 'test/results/coverage')) docker_exec(args, test_id, ['mkdir', '-p'] + writable_dirs) docker_exec(args, test_id, ['chmod', '777'] + writable_dirs) if content_root == install_root: docker_exec(args, test_id, ['find', os.path.join(content_root, 'test/results/'), '-type', 'd', '-exec', 'chmod', '777', '{}', '+']) docker_exec(args, test_id, ['chmod', '755', '/root']) docker_exec(args, test_id, ['chmod', '644', os.path.join(content_root, args.metadata_path)]) docker_exec(args, test_id, ['useradd', 'pytest', '--create-home']) docker_exec(args, test_id, cmd + ['--requirements-mode', 'only'], options=cmd_options) networks = get_docker_networks(args, test_id) for network in networks: docker_network_disconnect(args, test_id, network) cmd += ['--requirements-mode', 'skip'] cmd_options += ['--user', 'pytest'] try: docker_exec(args, test_id, cmd, options=cmd_options) finally: with tempfile.NamedTemporaryFile(prefix='ansible-result-', suffix='.tgz') as local_result_fd: docker_exec(args, test_id, ['tar', 'czf', '/root/results.tgz', '-C', os.path.join(content_root, 'test'), 'results']) docker_get(args, test_id, '/root/results.tgz', local_result_fd.name) run_command(args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test']) finally: if httptester_id: docker_rm(args, httptester_id) if test_id: docker_rm(args, test_id)
def delegate_remote(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] :type integration_targets: tuple[IntegrationTarget] """ parts = args.remote.split('/', 1) platform = parts[0] version = parts[1] core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider) success = False if isinstance(args, ShellConfig): use_httptester = args.httptester else: use_httptester = args.httptester and any( 'needs/httptester/' in target.aliases for target in integration_targets) if use_httptester and not docker_available(): display.warning( 'Assuming --disable-httptester since `docker` is not available.') use_httptester = False httptester_id = None ssh_options = [] try: core_ci.start() if use_httptester: httptester_id, ssh_options = start_httptester(args) core_ci.wait() if platform == 'windows': # Windows doesn't need the ansible-test fluff, just run the SSH command manage = ManageWindowsCI(core_ci) cmd = ['powershell.exe'] else: options = { '--remote': 1, } cmd = generate_command(args, 'ansible/test/runner/test.py', options, exclude, require) if httptester_id: cmd += ['--inject-httptester'] if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: cmd += [ '--coverage-label', 'remote-%s-%s' % (platform, version) ] if isinstance(args, IntegrationConfig): if not args.allow_destructive: cmd.append('--allow-destructive') # remote instances are only expected to have a single python version available if isinstance(args, UnitsConfig) and not args.python: cmd += ['--python', 'default'] manage = ManagePosixCI(core_ci) manage.setup() if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) for cloud_platform in cloud_platforms: ssh_options += cloud_platform.get_remote_ssh_options() try: manage.ssh(cmd, ssh_options) success = True finally: if platform != 'windows': manage.ssh( 'rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results' ) manage.download('/tmp/results', 'test') finally: if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success): core_ci.stop() if httptester_id: docker_rm(args, httptester_id)
def delegate_docker(args, exclude, require, integration_targets): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] :type integration_targets: tuple[IntegrationTarget] """ test_image = args.docker privileged = args.docker_privileged if isinstance(args, ShellConfig): use_httptester = args.httptester else: use_httptester = args.httptester and any( 'needs/httptester/' in target.aliases for target in integration_targets) if use_httptester: docker_pull(args, args.httptester) docker_pull(args, test_image) httptester_id = None test_id = None options = { '--docker': 1, '--docker-privileged': 0, '--docker-util': 1, } cmd = generate_command(args, '/root/ansible/test/runner/test.py', options, exclude, require) if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: image_label = re.sub('^ansible/ansible:', '', args.docker) image_label = re.sub('[^a-zA-Z0-9]+', '-', image_label) cmd += ['--coverage-label', 'docker-%s' % image_label] if isinstance(args, IntegrationConfig): if not args.allow_destructive: cmd.append('--allow-destructive') cmd_options = [] if isinstance(args, ShellConfig) or (isinstance(args, IntegrationConfig) and args.debug_strategy): cmd_options.append('-it') with tempfile.NamedTemporaryFile(prefix='ansible-source-', suffix='.tgz') as local_source_fd: try: if not args.explain: if args.docker_keep_git: tar_filter = lib.pytar.AllowGitTarFilter() else: tar_filter = lib.pytar.DefaultTarFilter() lib.pytar.create_tarfile(local_source_fd.name, '.', tar_filter) if use_httptester: httptester_id = run_httptester(args) else: httptester_id = None test_options = [ '--detach', '--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro', '--privileged=%s' % str(privileged).lower(), ] if args.docker_memory: test_options.extend([ '--memory=%d' % args.docker_memory, '--memory-swap=%d' % args.docker_memory, ]) docker_socket = '/var/run/docker.sock' if os.path.exists(docker_socket): test_options += [ '--volume', '%s:%s' % (docker_socket, docker_socket) ] if httptester_id: test_options += ['--env', 'HTTPTESTER=1'] for host in HTTPTESTER_HOSTS: test_options += ['--link', '%s:%s' % (httptester_id, host)] if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) for cloud_platform in cloud_platforms: test_options += cloud_platform.get_docker_run_options() test_id, _ = docker_run(args, test_image, options=test_options) if args.explain: test_id = 'test_id' else: test_id = test_id.strip() # write temporary files to /root since /tmp isn't ready immediately on container start docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh') docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh']) docker_put(args, test_id, local_source_fd.name, '/root/ansible.tgz') docker_exec(args, test_id, ['mkdir', '/root/ansible']) docker_exec( args, test_id, ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible']) # docker images are only expected to have a single python version available if isinstance(args, UnitsConfig) and not args.python: cmd += ['--python', 'default'] try: docker_exec(args, test_id, cmd, options=cmd_options) finally: with tempfile.NamedTemporaryFile( prefix='ansible-result-', suffix='.tgz') as local_result_fd: docker_exec(args, test_id, [ 'tar', 'czf', '/root/results.tgz', '-C', '/root/ansible/test', 'results' ]) docker_get(args, test_id, '/root/results.tgz', local_result_fd.name) run_command( args, ['tar', 'oxzf', local_result_fd.name, '-C', 'test']) finally: if httptester_id: docker_rm(args, httptester_id) if test_id: docker_rm(args, test_id)
def delegate_docker(args, exclude, require): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] """ util_image = args.docker_util test_image = args.docker privileged = args.docker_privileged if util_image: docker_pull(args, util_image) docker_pull(args, test_image) util_id = None test_id = None options = { '--docker': 1, '--docker-privileged': 0, '--docker-util': 1, } cmd = generate_command(args, '/root/ansible/test/runner/test.py', options, exclude, require) if isinstance(args, IntegrationConfig): if not args.allow_destructive: cmd.append('--allow-destructive') cmd_options = [] if isinstance(args, ShellConfig): cmd_options.append('-it') if not args.explain: lib.pytar.create_tarfile('/tmp/ansible.tgz', '.', lib.pytar.ignore) try: if util_image: util_options = [ '--detach', ] util_id, _ = docker_run(args, util_image, options=util_options) if args.explain: util_id = 'util_id' else: util_id = util_id.strip() else: util_id = None test_options = [ '--detach', '--volume', '/sys/fs/cgroup:/sys/fs/cgroup:ro', '--privileged=%s' % str(privileged).lower(), ] if util_id: test_options += [ '--link', '%s:ansible.http.tests' % util_id, '--link', '%s:sni1.ansible.http.tests' % util_id, '--link', '%s:sni2.ansible.http.tests' % util_id, '--link', '%s:fail.ansible.http.tests' % util_id, '--env', 'HTTPTESTER=1', ] if isinstance(args, TestConfig): cloud_platforms = get_cloud_providers(args) for cloud_platform in cloud_platforms: test_options += cloud_platform.get_docker_run_options() test_id, _ = docker_run(args, test_image, options=test_options) if args.explain: test_id = 'test_id' else: test_id = test_id.strip() # write temporary files to /root since /tmp isn't ready immediately on container start docker_put(args, test_id, 'test/runner/setup/docker.sh', '/root/docker.sh') docker_exec(args, test_id, ['/bin/bash', '/root/docker.sh']) docker_put(args, test_id, '/tmp/ansible.tgz', '/root/ansible.tgz') docker_exec(args, test_id, ['mkdir', '/root/ansible']) docker_exec( args, test_id, ['tar', 'oxzf', '/root/ansible.tgz', '-C', '/root/ansible']) # docker images are only expected to have a single python version available if isinstance(args, UnitsConfig) and not args.python: cmd += ['--python', 'default'] try: docker_exec(args, test_id, cmd, options=cmd_options) finally: docker_exec(args, test_id, [ 'tar', 'czf', '/root/results.tgz', '-C', '/root/ansible/test', 'results' ]) docker_get(args, test_id, '/root/results.tgz', '/tmp/results.tgz') run_command(args, ['tar', 'oxzf', '/tmp/results.tgz', '-C', 'test']) finally: if util_id: docker_rm(args, util_id) if test_id: docker_rm(args, test_id)
def delegate_remote(args, exclude, require): """ :type args: EnvironmentConfig :type exclude: list[str] :type require: list[str] """ parts = args.remote.split('/', 1) platform = parts[0] version = parts[1] core_ci = AnsibleCoreCI(args, platform, version, stage=args.remote_stage, provider=args.remote_provider) success = False try: core_ci.start() core_ci.wait() options = { '--remote': 1, } cmd = generate_command(args, 'ansible/test/runner/test.py', options, exclude, require) if isinstance(args, TestConfig): if args.coverage and not args.coverage_label: cmd += [ '--coverage-label', 'remote-%s-%s' % (platform, version) ] if isinstance(args, IntegrationConfig): if not args.allow_destructive: cmd.append('--allow-destructive') # remote instances are only expected to have a single python version available if isinstance(args, UnitsConfig) and not args.python: cmd += ['--python', 'default'] manage = ManagePosixCI(core_ci) manage.setup() ssh_options = [] if isinstance(args, IntegrationConfig): cloud_platforms = get_cloud_providers(args) for cloud_platform in cloud_platforms: ssh_options += cloud_platform.get_remote_ssh_options() try: manage.ssh(cmd, ssh_options) success = True finally: manage.ssh( 'rm -rf /tmp/results && cp -a ansible/test/results /tmp/results && chmod -R a+r /tmp/results' ) manage.download('/tmp/results', 'test') finally: if args.remote_terminate == 'always' or (args.remote_terminate == 'success' and success): core_ci.stop()