def test_run_detach(self):
"""
Tests detach=True is passed to the underlying call by default
"""
mock_container = create_mock_container()
# Test detach=True is passed in even if not specified
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command") == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["detach"] is True
# Test passing in detach=True does not cause any issues
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command", detach=True) == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["detach"] is True
# Test detach=False fails
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
with self.assertRaises(ValueError):
safe_docker.run("image", "command", detach=False)
assert mock_client.containers.run.call_args_list == []
def test_run_successful(self, mock_pthread_sigmask):
"""
Tests successful run
"""
mock_container = create_mock_container()
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
# Check return code is 0
assert safe_docker.run("image", "command") == 0
# Check call to container is correct
assert mock_client.containers.run.call_args_list == [
call("image",
"command",
detach=True,
environment={
"BUILD_NUMBER": "BUILD_NUMBER_5",
"BUILD_ID": "BUILD_ID_1",
"BUILD_TAG": "BUILD_TAG_7"
})
]
# Check correct signals are blocked then unblocked
assert mock_pthread_sigmask.call_args_list == [
call(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM}),
call(signal.SIG_UNBLOCK, {signal.SIGINT, signal.SIGTERM})
]
# Assert container is stopped and removed
assert mock_container.stop.call_count == 1
assert mock_container.remove.call_count == 1
assert len(safe_docker._containers) == 0
def test_jenkins_vars(self):
"""
Tests jenkins environment variables are appropriately passed to the underlying docker run call
"""
# NOTE: It's important that these variables are passed to the underlying docker container
# These variables are passed to the container so the process tree killer can find runaway
# process inside the container
# https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
# https://github.com/jenkinsci/jenkins/blob/578d6bacb33a5e99f149de504c80275796f0b231/core/src/main/java/hudson/model/Run.java#L2393
jenkins_vars = {
"BUILD_NUMBER": "BUILD_NUMBER_5",
"BUILD_ID": "BUILD_ID_1",
"BUILD_TAG": "BUILD_TAG_7"
}
mock_container = create_mock_container()
# Test environment is empty if the jenkins vars are not present
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command") == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["environment"] == {}
# Test environment contains jenkins env vars if they are present
with MockDockerClient(mock_container) as mock_client:
with patch.dict(os.environ, jenkins_vars):
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command") == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["environment"] == jenkins_vars
# Test jenkins env vars are added to callers env vars
user_env = {"key1": "value1", "key2": "value2"}
with MockDockerClient(mock_container) as mock_client:
with patch.dict(os.environ, jenkins_vars):
safe_docker = SafeDockerClient()
assert safe_docker.run("image",
"command",
environment=user_env) == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["environment"] == {**jenkins_vars, **user_env}
def test_container_remove_raises_returns_152(self):
"""
Tests 152 is returned if an error is raised when calling container.remove
"""
mock_container = create_mock_container()
mock_container.remove.side_effect = RuntimeError(
"Something bad happened")
with MockDockerClient(mock_container):
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command") == 152
def test_container_returns_non_zero_status_code(self):
"""
Tests non-zero code from container is returned and the container
is cleaned up
"""
mock_container = create_mock_container(status_code=10)
with MockDockerClient(mock_container):
safe_docker = SafeDockerClient()
# check return code and that container gets cleaned up
assert safe_docker.run("image", "command") == 10
assert mock_container.stop.call_count == 1
assert mock_container.remove.call_count == 1
assert len(safe_docker._containers) == 0
def test_run_args_kwargs_passed(self):
"""
Tests args and kwargs are passed to the container run call
"""
mock_container = create_mock_container()
# Test detach=True is passed in even if not specified
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
assert safe_docker.run("image",
"command",
"another_arg",
str_param="value",
bool_param=True,
none_param=None,
int_param=5,
float_param=5.2,
list_param=["this", "is", "a", "list"],
map_param={
"a": "5",
"b": True,
"c": 2
}) == 0
assert mock_client.containers.run.call_args_list == [
call("image",
"command",
"another_arg",
detach=True,
environment={},
str_param="value",
bool_param=True,
none_param=None,
int_param=5,
float_param=5.2,
list_param=["this", "is", "a", "list"],
map_param={
"a": "5",
"b": True,
"c": 2
})
]
def container_run(docker_client: SafeDockerClient,
platform: str,
nvidia_runtime: bool,
docker_registry: str,
shared_memory_size: str,
local_ccache_dir: str,
command: List[str],
environment: Dict[str, str],
dry_run: bool = False) -> int:
"""Run command in a container"""
container_wait_s = 600
#
# Environment setup
#
environment.update({
'CCACHE_MAXSIZE': '500G',
'CCACHE_TEMPDIR':
'/tmp/ccache', # temp dir should be local and not shared
'CCACHE_DIR':
'/work/ccache', # this path is inside the container as /work/ccache is
# mounted
'CCACHE_LOGFILE':
'/tmp/ccache.log', # a container-scoped log, useful for ccache
# verification.
})
environment.update(
{k: os.environ[k]
for k in ['CCACHE_MAXSIZE'] if k in os.environ})
tag = get_docker_tag(platform=platform, registry=docker_registry)
mx_root = get_mxnet_root()
local_build_folder = buildir()
# We need to create it first, otherwise it will be created by the docker daemon with root only permissions
os.makedirs(local_build_folder, exist_ok=True)
os.makedirs(local_ccache_dir, exist_ok=True)
logging.info("Using ccache directory: %s", local_ccache_dir)
# Equivalent command
docker_cmd_list = [
get_docker_binary(nvidia_runtime),
'run',
"--cap-add",
"SYS_PTRACE", # Required by ASAN
'--rm',
'--shm-size={}'.format(shared_memory_size),
# mount mxnet root
'-v',
"{}:/work/mxnet".format(mx_root),
# mount mxnet/build for storing build
'-v',
"{}:/work/build".format(local_build_folder),
'-v',
"{}:/work/ccache".format(local_ccache_dir),
'-u',
'{}:{}'.format(os.getuid(), os.getgid()),
'-e',
'CCACHE_MAXSIZE={}'.format(environment['CCACHE_MAXSIZE']),
# temp dir should be local and not shared
'-e',
'CCACHE_TEMPDIR={}'.format(environment['CCACHE_TEMPDIR']),
# this path is inside the container as /work/ccache is mounted
'-e',
"CCACHE_DIR={}".format(environment['CCACHE_DIR']),
# a container-scoped log, useful for ccache verification.
'-e',
"CCACHE_LOGFILE={}".format(environment['CCACHE_LOGFILE']),
'-ti',
tag
]
docker_cmd_list.extend(command)
docker_cmd = ' \\\n\t'.join(docker_cmd_list)
logging.info("Running %s in container %s", command, tag)
logging.info("Executing the equivalent of:\n%s\n", docker_cmd)
if not dry_run:
#############################
#
signal.pthread_sigmask(signal.SIG_BLOCK,
{signal.SIGINT, signal.SIGTERM})
# noinspection PyShadowingNames
runtime = None
if nvidia_runtime:
# noinspection PyShadowingNames
# runc is default (docker info | grep -i runtime)
runtime = 'nvidia'
return docker_client.run(tag,
runtime=runtime,
command=command,
shm_size=shared_memory_size,
user='******'.format(os.getuid(), os.getgid()),
cap_add='SYS_PTRACE',
volumes={
mx_root: {
'bind': '/work/mxnet',
'mode': 'rw'
},
local_build_folder: {
'bind': '/work/build',
'mode': 'rw'
},
local_ccache_dir: {
'bind': '/work/ccache',
'mode': 'rw'
},
},
environment=environment)
return 0