def test_compose_buildkit_inline_cache(arrow_compose_path): compose = DockerCompose(arrow_compose_path) expected_calls = [ "build --build-arg BUILDKIT_INLINE_CACHE=1 conda-cpp", ] with assert_compose_calls(compose, expected_calls): compose.build('conda-cpp')
def test_compose_pull_params(arrow_compose_path): expected_calls = [ "pull --ignore-pull-failures conda-cpp", "pull --ignore-pull-failures conda-python", ] compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04')) expected_env = PartialEnv(PYTHON='3.6', PANDAS='latest') with assert_compose_calls(compose, expected_calls, env=expected_env): compose.pull('conda-python-pandas', pull_leaf=False)
def test_image_with_gpu(arrow_compose_path): compose = DockerCompose(arrow_compose_path) expected_calls = [[ "run", "--rm", "--gpus", "all", "-e", "CUDA_ENV=1", "-e", "OTHER_ENV=2", "-v", "/host:/container:rw", "org/ubuntu-cuda", '/bin/bash -c "echo 1 > /tmp/dummy && cat /tmp/dummy"' ]] with assert_docker_calls(compose, expected_calls): compose.run('ubuntu-cuda')
def test_image_with_gpu(arrow_compose_path): compose = DockerCompose(arrow_compose_path) expected_calls = [[ "run", "--rm", "-it", "--gpus", "all", "-e", "CUDA_ENV=1", "-e", "OTHER_ENV=2", "-v", "/host:/container:rw", "dummy-cuda", "/bin/bash", "-c", "echo 1 > /tmp/dummy && cat /tmp/dummy" ]] with assert_docker_calls(compose, expected_calls): compose.run('ubuntu-cuda', force_pull=False, force_build=False)
def test_compose_run_with_resource_limits(arrow_compose_path): expected_calls = [ format_run([ "--cpuset-cpus=0,1", "--memory=7g", "--memory-swap=7g", "org/conda-cpp" ]), ] compose = DockerCompose(arrow_compose_path) with assert_docker_calls(compose, expected_calls): compose.run('conda-cpp', resource_limit="github")
def test_compose_push(arrow_compose_path): compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8')) expected_env = PartialEnv(PYTHON="3.8") expected_calls = [ mock.call(["docker", "login", "-u", "user", "-p", "pass"], check=True), mock.call(["docker-compose", "--file", str(compose.config_path), "push", "conda-python"], check=True, env=expected_env) ] with assert_subprocess_calls(expected_calls): compose.push('conda-python', user='******', password='******')
def test_forwarding_env_variables(arrow_compose_path): expected_calls = [ "pull --ignore-pull-failures conda-cpp", "build conda-cpp", ] expected_env = PartialEnv(MY_CUSTOM_VAR_A='a', MY_CUSTOM_VAR_B='b') with override_env({'MY_CUSTOM_VAR_A': 'a', 'MY_CUSTOM_VAR_B': 'b'}): compose = DockerCompose(arrow_compose_path) with assert_compose_calls(compose, expected_calls, env=expected_env): assert os.environ['MY_CUSTOM_VAR_A'] == 'a' assert os.environ['MY_CUSTOM_VAR_B'] == 'b' compose.build('conda-cpp')
def test_compose_error(arrow_compose_path): compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8', PANDAS='master')) error = subprocess.CalledProcessError(99, []) with mock.patch('subprocess.run', side_effect=error): with pytest.raises(RuntimeError) as exc: compose.run('conda-cpp') exception_message = str(exc.value) assert "exited with a non-zero exit code 99" in exception_message assert "PANDAS: latest" in exception_message assert "export PANDAS=master" in exception_message
def test_listing_images(arrow_compose_path): compose = DockerCompose(arrow_compose_path) assert sorted(compose.images()) == [ 'conda-cpp', 'conda-python', 'conda-python-dask', 'conda-python-pandas', 'ubuntu-c-glib', 'ubuntu-cpp', 'ubuntu-cpp-cmake32', 'ubuntu-cuda', 'ubuntu-ruby', ]
def test_config_validation(tmpdir): config_path = create_config(tmpdir, missing_service_compose_yml) msg = "`sub-foo` is defined in `x-hierarchy` bot not in `services`" with pytest.raises(ValueError, match=msg): DockerCompose(config_path) config_path = create_config(tmpdir, missing_node_compose_yml) msg = "`sub-bar` is defined in `services` but not in `x-hierarchy`" with pytest.raises(ValueError, match=msg): DockerCompose(config_path) config_path = create_config(tmpdir, ok_compose_yml) DockerCompose(config_path) # no issue
def test_compose_push(arrow_compose_path): compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.9')) expected_env = PartialEnv(PYTHON="3.9") expected_calls = [ mock.call(["docker", "login", "-u", "user", "-p", "pass"], check=True), ] for image in ["conda-cpp", "conda-python", "conda-python-pandas"]: expected_calls.append( mock.call([ "docker-compose", "--file", str(compose.config.path), "push", image ], check=True, env=expected_env)) with assert_subprocess_calls(expected_calls): compose.push('conda-python-pandas', user='******', password='******')
def test_compose_default_params_and_env(arrow_compose_path): compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04', DASK='master')) assert compose.dotenv == arrow_compose_env assert compose.params == { 'UBUNTU': '18.04', 'DASK': 'master', }
def test_compose_pull(arrow_compose_path): compose = DockerCompose(arrow_compose_path) expected_calls = [ "pull --ignore-pull-failures conda-cpp", ] with assert_compose_calls(compose, expected_calls): compose.pull('conda-cpp') expected_calls = [ "pull --ignore-pull-failures conda-cpp", "pull --ignore-pull-failures conda-python", "pull --ignore-pull-failures conda-python-pandas" ] with assert_compose_calls(compose, expected_calls): compose.pull('conda-python-pandas') expected_calls = [ "pull --ignore-pull-failures conda-cpp", "pull --ignore-pull-failures conda-python", ] with assert_compose_calls(compose, expected_calls): compose.pull('conda-python-pandas', pull_leaf=False)
def test_compose_run(arrow_compose_path): expected_calls = [ "run --rm conda-cpp", ] compose = DockerCompose(arrow_compose_path) with assert_compose_calls(compose, expected_calls): compose.run('conda-cpp') expected_calls = ["run --rm conda-python"] expected_env = PartialEnv(PYTHON='3.6') with assert_compose_calls(compose, expected_calls, env=expected_env): compose.run('conda-python') compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8')) expected_env = PartialEnv(PYTHON='3.8') with assert_compose_calls(compose, expected_calls, env=expected_env): compose.run('conda-python') compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8')) for command in ["bash", "echo 1"]: expected_calls = [["run", "--rm", "conda-python", command]] expected_env = PartialEnv(PYTHON='3.8') with assert_compose_calls(compose, expected_calls, env=expected_env): compose.run('conda-python', command) expected_calls = [ ("run --rm -e CONTAINER_ENV_VAR_A=a -e CONTAINER_ENV_VAR_B=b " "conda-python") ] compose = DockerCompose(arrow_compose_path) expected_env = PartialEnv(PYTHON='3.6') with assert_compose_calls(compose, expected_calls, env=expected_env): env = collections.OrderedDict([("CONTAINER_ENV_VAR_A", "a"), ("CONTAINER_ENV_VAR_B", "b")]) compose.run('conda-python', env=env)
def test_compose_run_force_pull_and_build(arrow_compose_path): compose = DockerCompose(arrow_compose_path) expected_calls = [ "pull --ignore-pull-failures conda-cpp", format_run("conda-cpp") ] with assert_compose_calls(compose, expected_calls): compose.run('conda-cpp', force_pull=True) expected_calls = [ "build conda-cpp", format_run("conda-cpp") ] with assert_compose_calls(compose, expected_calls): compose.run('conda-cpp', force_build=True) expected_calls = [ "pull --ignore-pull-failures conda-cpp", "build conda-cpp", format_run("conda-cpp") ] with assert_compose_calls(compose, expected_calls): compose.run('conda-cpp', force_pull=True, force_build=True) expected_calls = [ "pull --ignore-pull-failures conda-cpp", "pull --ignore-pull-failures conda-python", "pull --ignore-pull-failures conda-python-pandas", "build conda-cpp", "build conda-python", "build conda-python-pandas", format_run("conda-python-pandas bash") ] with assert_compose_calls(compose, expected_calls): compose.run('conda-python-pandas', command='bash', force_build=True, force_pull=True) expected_calls = [ "pull --ignore-pull-failures conda-cpp", "pull --ignore-pull-failures conda-python", "build conda-cpp", "build conda-python", "build --no-cache conda-python-pandas", format_run("conda-python-pandas bash") ] with assert_compose_calls(compose, expected_calls): compose.run('conda-python-pandas', command='bash', force_build=True, force_pull=True, use_leaf_cache=False)
def test_compose_run(arrow_compose_path): expected_calls = [ format_run("conda-cpp"), ] compose = DockerCompose(arrow_compose_path) with assert_compose_calls(compose, expected_calls): compose.run('conda-cpp') expected_calls = [ format_run("conda-python") ] expected_env = PartialEnv(PYTHON='3.6') with assert_compose_calls(compose, expected_calls, env=expected_env): compose.run('conda-python') compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8')) expected_env = PartialEnv(PYTHON='3.8') with assert_compose_calls(compose, expected_calls, env=expected_env): compose.run('conda-python') compose = DockerCompose(arrow_compose_path, params=dict(PYTHON='3.8')) for command in ["bash", "echo 1"]: expected_calls = [ format_run(["conda-python", command]), ] expected_env = PartialEnv(PYTHON='3.8') with assert_compose_calls(compose, expected_calls, env=expected_env): compose.run('conda-python', command) expected_calls = [ ( format_run("-e CONTAINER_ENV_VAR_A=a -e CONTAINER_ENV_VAR_B=b " "conda-python") ) ] compose = DockerCompose(arrow_compose_path) expected_env = PartialEnv(PYTHON='3.6') with assert_compose_calls(compose, expected_calls, env=expected_env): env = collections.OrderedDict([ ("CONTAINER_ENV_VAR_A", "a"), ("CONTAINER_ENV_VAR_B", "b") ]) compose.run('conda-python', env=env) expected_calls = [ ( format_run("--volume /host/build:/build --volume " "/host/ccache:/ccache:delegated conda-python") ) ] compose = DockerCompose(arrow_compose_path) with assert_compose_calls(compose, expected_calls): volumes = ("/host/build:/build", "/host/ccache:/ccache:delegated") compose.run('conda-python', volumes=volumes)
def test_compose_build_params(arrow_compose_path): expected_calls = [ "build ubuntu-cpp", ] compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04')) expected_env = PartialEnv(UBUNTU="18.04") with assert_compose_calls(compose, expected_calls, env=expected_env): compose.build('ubuntu-cpp') compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='16.04')) expected_env = PartialEnv(UBUNTU="16.04") with assert_compose_calls(compose, expected_calls, env=expected_env): compose.build('ubuntu-cpp') expected_calls = [ "build --no-cache conda-cpp", "build --no-cache conda-python", "build --no-cache conda-python-pandas", ] compose = DockerCompose(arrow_compose_path, params=dict(UBUNTU='18.04')) expected_env = PartialEnv(PYTHON='3.6', PANDAS='latest') with assert_compose_calls(compose, expected_calls, env=expected_env): compose.build('conda-python-pandas', use_cache=False)
def test_compose_build(arrow_compose_path): compose = DockerCompose(arrow_compose_path) expected_calls = [ "build conda-cpp", ] with assert_compose_calls(compose, expected_calls): compose.build('conda-cpp') expected_calls = [ "build --no-cache conda-cpp" ] with assert_compose_calls(compose, expected_calls): compose.build('conda-cpp', use_cache=False) expected_calls = [ "build conda-cpp", "build conda-python", "build conda-python-pandas" ] with assert_compose_calls(compose, expected_calls): compose.build('conda-python-pandas') expected_calls = [ "build --no-cache conda-cpp", "build --no-cache conda-python", "build --no-cache conda-python-pandas", ] with assert_compose_calls(compose, expected_calls): compose.build('conda-python-pandas', use_cache=False) expected_calls = [ "build conda-cpp", "build conda-python", "build --no-cache conda-python-pandas", ] with assert_compose_calls(compose, expected_calls): compose.build('conda-python-pandas', use_cache=True, use_leaf_cache=False)
def test_arrow_example_validation_passes(arrow_compose_path): DockerCompose(arrow_compose_path)