def test_run_successful(self, mock_pthread_sigmask):
"""
Tests successful run
"""
mock_container = create_mock_container()
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
# Check return code is 0
assert safe_docker.run("image", "command") == 0
# Check call to container is correct
assert mock_client.containers.run.call_args_list == [
call("image",
"command",
detach=True,
environment={
"BUILD_NUMBER": "BUILD_NUMBER_5",
"BUILD_ID": "BUILD_ID_1",
"BUILD_TAG": "BUILD_TAG_7"
})
]
# Check correct signals are blocked then unblocked
assert mock_pthread_sigmask.call_args_list == [
call(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM}),
call(signal.SIG_UNBLOCK, {signal.SIGINT, signal.SIGTERM})
]
# Assert container is stopped and removed
assert mock_container.stop.call_count == 1
assert mock_container.remove.call_count == 1
assert len(safe_docker._containers) == 0
def test_run_detach(self):
"""
Tests detach=True is passed to the underlying call by default
"""
mock_container = create_mock_container()
# Test detach=True is passed in even if not specified
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command") == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["detach"] is True
# Test passing in detach=True does not cause any issues
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command", detach=True) == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["detach"] is True
# Test detach=False fails
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
with self.assertRaises(ValueError):
safe_docker.run("image", "command", detach=False)
assert mock_client.containers.run.call_args_list == []
def test_container_remove_raises_returns_152(self):
"""
Tests 152 is returned if an error is raised when calling container.remove
"""
mock_container = create_mock_container()
mock_container.remove.side_effect = RuntimeError(
"Something bad happened")
with MockDockerClient(mock_container):
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command") == 152
def test_container_returns_non_zero_status_code(self):
"""
Tests non-zero code from container is returned and the container
is cleaned up
"""
mock_container = create_mock_container(status_code=10)
with MockDockerClient(mock_container):
safe_docker = SafeDockerClient()
# check return code and that container gets cleaned up
assert safe_docker.run("image", "command") == 10
assert mock_container.stop.call_count == 1
assert mock_container.remove.call_count == 1
assert len(safe_docker._containers) == 0
def test_run_args_kwargs_passed(self):
"""
Tests args and kwargs are passed to the container run call
"""
mock_container = create_mock_container()
# Test detach=True is passed in even if not specified
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
assert safe_docker.run("image",
"command",
"another_arg",
str_param="value",
bool_param=True,
none_param=None,
int_param=5,
float_param=5.2,
list_param=["this", "is", "a", "list"],
map_param={
"a": "5",
"b": True,
"c": 2
}) == 0
assert mock_client.containers.run.call_args_list == [
call("image",
"command",
"another_arg",
detach=True,
environment={},
str_param="value",
bool_param=True,
none_param=None,
int_param=5,
float_param=5.2,
list_param=["this", "is", "a", "list"],
map_param={
"a": "5",
"b": True,
"c": 2
})
]
def test_jenkins_vars(self):
"""
Tests jenkins environment variables are appropriately passed to the underlying docker run call
"""
# NOTE: It's important that these variables are passed to the underlying docker container
# These variables are passed to the container so the process tree killer can find runaway
# process inside the container
# https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller
# https://github.com/jenkinsci/jenkins/blob/578d6bacb33a5e99f149de504c80275796f0b231/core/src/main/java/hudson/model/Run.java#L2393
jenkins_vars = {
"BUILD_NUMBER": "BUILD_NUMBER_5",
"BUILD_ID": "BUILD_ID_1",
"BUILD_TAG": "BUILD_TAG_7"
}
mock_container = create_mock_container()
# Test environment is empty if the jenkins vars are not present
with MockDockerClient(mock_container) as mock_client:
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command") == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["environment"] == {}
# Test environment contains jenkins env vars if they are present
with MockDockerClient(mock_container) as mock_client:
with patch.dict(os.environ, jenkins_vars):
safe_docker = SafeDockerClient()
assert safe_docker.run("image", "command") == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["environment"] == jenkins_vars
# Test jenkins env vars are added to callers env vars
user_env = {"key1": "value1", "key2": "value2"}
with MockDockerClient(mock_container) as mock_client:
with patch.dict(os.environ, jenkins_vars):
safe_docker = SafeDockerClient()
assert safe_docker.run("image",
"command",
environment=user_env) == 0
assert mock_client.containers.run.call_count == 1
_, kwargs = mock_client.containers.run.call_args
assert kwargs["environment"] == {**jenkins_vars, **user_env}
def main() -> int:
config_logging()
logging.info("MXNet container based build tool.")
log_environment()
chdir_to_script_directory()
parser = argparse.ArgumentParser(
description="""Utility for building and testing MXNet on docker
containers""",
epilog="")
parser.add_argument("-p", "--platform", help="platform", type=str)
parser.add_argument(
"-b",
"--build-only",
help="Only build the container, don't build the project",
action='store_true')
parser.add_argument(
"-R",
"--run-only",
help="Only run the container, don't rebuild the container",
action='store_true')
parser.add_argument("-a",
"--all",
help="build for all platforms",
action='store_true')
parser.add_argument("-n",
"--nvidiadocker",
help="Use nvidia docker",
action='store_true')
parser.add_argument(
"--shm-size",
help=
"Size of the shared memory /dev/shm allocated in the container (e.g '1g')",
default='500m',
dest="shared_memory_size")
parser.add_argument("-l",
"--list",
help="List platforms",
action='store_true')
parser.add_argument("--print-docker-run",
help="print docker run command for manual inspection",
action='store_true')
parser.add_argument("-d",
"--docker-registry",
help="Dockerhub registry name to retrieve cache from.",
default='mxnetci',
type=str)
parser.add_argument(
"-r",
"--docker-build-retries",
help="Number of times to retry building the docker image. Default is 1",
default=1,
type=int)
parser.add_argument("--no-cache",
action="store_true",
help="passes --no-cache to docker build")
parser.add_argument("--cache-intermediate",
action="store_true",
help="passes --rm=false to docker build")
parser.add_argument(
"-e",
"--environment",
nargs="*",
default=[],
help="Environment variables for the docker container. "
"Specify with a list containing either names or name=value")
parser.add_argument("command",
help="command to run in the container",
nargs='*',
action='append',
type=str)
parser.add_argument("--ccache-dir",
default=default_ccache_dir(),
help="ccache directory",
type=str)
args = parser.parse_args()
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
docker_client = SafeDockerClient()
environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e]))
for e in args.environment])
if args.list:
print(list_platforms())
elif args.platform:
platform = args.platform
tag = get_docker_tag(platform=platform, registry=args.docker_registry)
if args.docker_registry:
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
if not args.run_only:
build_docker(platform=platform,
docker_binary=docker_binary,
registry=args.docker_registry,
num_retries=args.docker_build_retries,
no_cache=args.no_cache,
cache_intermediate=args.cache_intermediate)
else:
logging.info("Skipping docker build step.")
if args.build_only:
logging.warning(
"Container was just built. Exiting due to build-only.")
return 0
# noinspection PyUnusedLocal
ret = 0
if command:
ret = container_run(docker_client=docker_client,
platform=platform,
nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size,
command=command,
docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir,
environment=environment)
elif args.print_docker_run:
command = []
ret = container_run(docker_client=docker_client,
platform=platform,
nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size,
command=command,
docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir,
dry_run=True,
environment=environment)
else:
# With no commands, execute a build function for the target platform
command = [
"/work/mxnet/ci/docker/runtime_functions.sh",
"build_{}".format(platform)
]
logging.info("No command specified, trying default build: %s",
' '.join(command))
ret = container_run(docker_client=docker_client,
platform=platform,
nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size,
command=command,
docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir,
environment=environment)
if ret != 0:
logging.critical("Execution of %s failed with status: %d", command,
ret)
return ret
elif args.all:
platforms = get_platforms()
platforms = [
platform for platform in platforms if 'build.' in platform
]
logging.info("Building for all architectures: %s", platforms)
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
tag = get_docker_tag(platform=platform,
registry=args.docker_registry)
load_docker_cache(tag=tag, docker_registry=args.docker_registry)
build_docker(platform,
docker_binary=docker_binary,
registry=args.docker_registry,
num_retries=args.docker_build_retries,
no_cache=args.no_cache)
if args.build_only:
continue
shutil.rmtree(buildir(), ignore_errors=True)
build_platform = "build_{}".format(platform)
plat_buildir = os.path.abspath(
os.path.join(get_mxnet_root(), '..',
"mxnet_{}".format(build_platform)))
if os.path.exists(plat_buildir):
logging.warning("%s already exists, skipping", plat_buildir)
continue
command = [
"/work/mxnet/ci/docker/runtime_functions.sh", build_platform
]
container_run(docker_client=docker_client,
platform=platform,
nvidia_runtime=args.nvidiadocker,
shared_memory_size=args.shared_memory_size,
command=command,
docker_registry=args.docker_registry,
local_ccache_dir=args.ccache_dir,
environment=environment)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)
else:
parser.print_help()
list_platforms()
print("""
Examples:
./build.py -p armv7
Will build a docker container with cross compilation tools and build MXNet for armv7 by
running: ci/docker/runtime_functions.sh build_armv7 inside the container.
./build.py -p armv7 ls
Will execute the given command inside the armv7 container
./build.py -p armv7 --print-docker-run
Will print a docker run command to get inside the container in a shell
./build.py -a
Builds for all platforms and leaves artifacts in build_<platform>
""")
return 0
def container_run(docker_client: SafeDockerClient,
platform: str,
nvidia_runtime: bool,
docker_registry: str,
shared_memory_size: str,
local_ccache_dir: str,
command: List[str],
environment: Dict[str, str],
dry_run: bool = False) -> int:
"""Run command in a container"""
container_wait_s = 600
#
# Environment setup
#
environment.update({
'CCACHE_MAXSIZE': '500G',
'CCACHE_TEMPDIR':
'/tmp/ccache', # temp dir should be local and not shared
'CCACHE_DIR':
'/work/ccache', # this path is inside the container as /work/ccache is
# mounted
'CCACHE_LOGFILE':
'/tmp/ccache.log', # a container-scoped log, useful for ccache
# verification.
})
environment.update(
{k: os.environ[k]
for k in ['CCACHE_MAXSIZE'] if k in os.environ})
tag = get_docker_tag(platform=platform, registry=docker_registry)
mx_root = get_mxnet_root()
local_build_folder = buildir()
# We need to create it first, otherwise it will be created by the docker daemon with root only permissions
os.makedirs(local_build_folder, exist_ok=True)
os.makedirs(local_ccache_dir, exist_ok=True)
logging.info("Using ccache directory: %s", local_ccache_dir)
# Equivalent command
docker_cmd_list = [
get_docker_binary(nvidia_runtime),
'run',
"--cap-add",
"SYS_PTRACE", # Required by ASAN
'--rm',
'--shm-size={}'.format(shared_memory_size),
# mount mxnet root
'-v',
"{}:/work/mxnet".format(mx_root),
# mount mxnet/build for storing build
'-v',
"{}:/work/build".format(local_build_folder),
'-v',
"{}:/work/ccache".format(local_ccache_dir),
'-u',
'{}:{}'.format(os.getuid(), os.getgid()),
'-e',
'CCACHE_MAXSIZE={}'.format(environment['CCACHE_MAXSIZE']),
# temp dir should be local and not shared
'-e',
'CCACHE_TEMPDIR={}'.format(environment['CCACHE_TEMPDIR']),
# this path is inside the container as /work/ccache is mounted
'-e',
"CCACHE_DIR={}".format(environment['CCACHE_DIR']),
# a container-scoped log, useful for ccache verification.
'-e',
"CCACHE_LOGFILE={}".format(environment['CCACHE_LOGFILE']),
'-ti',
tag
]
docker_cmd_list.extend(command)
docker_cmd = ' \\\n\t'.join(docker_cmd_list)
logging.info("Running %s in container %s", command, tag)
logging.info("Executing the equivalent of:\n%s\n", docker_cmd)
if not dry_run:
#############################
#
signal.pthread_sigmask(signal.SIG_BLOCK,
{signal.SIGINT, signal.SIGTERM})
# noinspection PyShadowingNames
runtime = None
if nvidia_runtime:
# noinspection PyShadowingNames
# runc is default (docker info | grep -i runtime)
runtime = 'nvidia'
return docker_client.run(tag,
runtime=runtime,
command=command,
shm_size=shared_memory_size,
user='******'.format(os.getuid(), os.getgid()),
cap_add='SYS_PTRACE',
volumes={
mx_root: {
'bind': '/work/mxnet',
'mode': 'rw'
},
local_build_folder: {
'bind': '/work/build',
'mode': 'rw'
},
local_ccache_dir: {
'bind': '/work/ccache',
'mode': 'rw'
},
},
environment=environment)
return 0
