def test_run_successful(self, mock_pthread_sigmask): """ Tests successful run """ mock_container = create_mock_container() with MockDockerClient(mock_container) as mock_client: safe_docker = SafeDockerClient() # Check return code is 0 assert safe_docker.run("image", "command") == 0 # Check call to container is correct assert mock_client.containers.run.call_args_list == [ call("image", "command", detach=True, environment={ "BUILD_NUMBER": "BUILD_NUMBER_5", "BUILD_ID": "BUILD_ID_1", "BUILD_TAG": "BUILD_TAG_7" }) ] # Check correct signals are blocked then unblocked assert mock_pthread_sigmask.call_args_list == [ call(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM}), call(signal.SIG_UNBLOCK, {signal.SIGINT, signal.SIGTERM}) ] # Assert container is stopped and removed assert mock_container.stop.call_count == 1 assert mock_container.remove.call_count == 1 assert len(safe_docker._containers) == 0
def test_run_detach(self): """ Tests detach=True is passed to the underlying call by default """ mock_container = create_mock_container() # Test detach=True is passed in even if not specified with MockDockerClient(mock_container) as mock_client: safe_docker = SafeDockerClient() assert safe_docker.run("image", "command") == 0 assert mock_client.containers.run.call_count == 1 _, kwargs = mock_client.containers.run.call_args assert kwargs["detach"] is True # Test passing in detach=True does not cause any issues with MockDockerClient(mock_container) as mock_client: safe_docker = SafeDockerClient() assert safe_docker.run("image", "command", detach=True) == 0 assert mock_client.containers.run.call_count == 1 _, kwargs = mock_client.containers.run.call_args assert kwargs["detach"] is True # Test detach=False fails with MockDockerClient(mock_container) as mock_client: safe_docker = SafeDockerClient() with self.assertRaises(ValueError): safe_docker.run("image", "command", detach=False) assert mock_client.containers.run.call_args_list == []
def test_container_remove_raises_returns_152(self): """ Tests 152 is returned if an error is raised when calling container.remove """ mock_container = create_mock_container() mock_container.remove.side_effect = RuntimeError( "Something bad happened") with MockDockerClient(mock_container): safe_docker = SafeDockerClient() assert safe_docker.run("image", "command") == 152
def test_container_returns_non_zero_status_code(self): """ Tests non-zero code from container is returned and the container is cleaned up """ mock_container = create_mock_container(status_code=10) with MockDockerClient(mock_container): safe_docker = SafeDockerClient() # check return code and that container gets cleaned up assert safe_docker.run("image", "command") == 10 assert mock_container.stop.call_count == 1 assert mock_container.remove.call_count == 1 assert len(safe_docker._containers) == 0
def test_run_args_kwargs_passed(self): """ Tests args and kwargs are passed to the container run call """ mock_container = create_mock_container() # Test detach=True is passed in even if not specified with MockDockerClient(mock_container) as mock_client: safe_docker = SafeDockerClient() assert safe_docker.run("image", "command", "another_arg", str_param="value", bool_param=True, none_param=None, int_param=5, float_param=5.2, list_param=["this", "is", "a", "list"], map_param={ "a": "5", "b": True, "c": 2 }) == 0 assert mock_client.containers.run.call_args_list == [ call("image", "command", "another_arg", detach=True, environment={}, str_param="value", bool_param=True, none_param=None, int_param=5, float_param=5.2, list_param=["this", "is", "a", "list"], map_param={ "a": "5", "b": True, "c": 2 }) ]
def test_jenkins_vars(self): """ Tests jenkins environment variables are appropriately passed to the underlying docker run call """ # NOTE: It's important that these variables are passed to the underlying docker container # These variables are passed to the container so the process tree killer can find runaway # process inside the container # https://wiki.jenkins.io/display/JENKINS/ProcessTreeKiller # https://github.com/jenkinsci/jenkins/blob/578d6bacb33a5e99f149de504c80275796f0b231/core/src/main/java/hudson/model/Run.java#L2393 jenkins_vars = { "BUILD_NUMBER": "BUILD_NUMBER_5", "BUILD_ID": "BUILD_ID_1", "BUILD_TAG": "BUILD_TAG_7" } mock_container = create_mock_container() # Test environment is empty if the jenkins vars are not present with MockDockerClient(mock_container) as mock_client: safe_docker = SafeDockerClient() assert safe_docker.run("image", "command") == 0 assert mock_client.containers.run.call_count == 1 _, kwargs = mock_client.containers.run.call_args assert kwargs["environment"] == {} # Test environment contains jenkins env vars if they are present with MockDockerClient(mock_container) as mock_client: with patch.dict(os.environ, jenkins_vars): safe_docker = SafeDockerClient() assert safe_docker.run("image", "command") == 0 assert mock_client.containers.run.call_count == 1 _, kwargs = mock_client.containers.run.call_args assert kwargs["environment"] == jenkins_vars # Test jenkins env vars are added to callers env vars user_env = {"key1": "value1", "key2": "value2"} with MockDockerClient(mock_container) as mock_client: with patch.dict(os.environ, jenkins_vars): safe_docker = SafeDockerClient() assert safe_docker.run("image", "command", environment=user_env) == 0 assert mock_client.containers.run.call_count == 1 _, kwargs = mock_client.containers.run.call_args assert kwargs["environment"] == {**jenkins_vars, **user_env}
def main() -> int: config_logging() logging.info("MXNet container based build tool.") log_environment() chdir_to_script_directory() parser = argparse.ArgumentParser( description="""Utility for building and testing MXNet on docker containers""", epilog="") parser.add_argument("-p", "--platform", help="platform", type=str) parser.add_argument( "-b", "--build-only", help="Only build the container, don't build the project", action='store_true') parser.add_argument( "-R", "--run-only", help="Only run the container, don't rebuild the container", action='store_true') parser.add_argument("-a", "--all", help="build for all platforms", action='store_true') parser.add_argument("-n", "--nvidiadocker", help="Use nvidia docker", action='store_true') parser.add_argument( "--shm-size", help= "Size of the shared memory /dev/shm allocated in the container (e.g '1g')", default='500m', dest="shared_memory_size") parser.add_argument("-l", "--list", help="List platforms", action='store_true') parser.add_argument("--print-docker-run", help="print docker run command for manual inspection", action='store_true') parser.add_argument("-d", "--docker-registry", help="Dockerhub registry name to retrieve cache from.", default='mxnetci', type=str) parser.add_argument( "-r", "--docker-build-retries", help="Number of times to retry building the docker image. Default is 1", default=1, type=int) parser.add_argument("--no-cache", action="store_true", help="passes --no-cache to docker build") parser.add_argument("--cache-intermediate", action="store_true", help="passes --rm=false to docker build") parser.add_argument( "-e", "--environment", nargs="*", default=[], help="Environment variables for the docker container. " "Specify with a list containing either names or name=value") parser.add_argument("command", help="command to run in the container", nargs='*', action='append', type=str) parser.add_argument("--ccache-dir", default=default_ccache_dir(), help="ccache directory", type=str) args = parser.parse_args() command = list(chain(*args.command)) docker_binary = get_docker_binary(args.nvidiadocker) docker_client = SafeDockerClient() environment = dict([(e.split('=')[:2] if '=' in e else (e, os.environ[e])) for e in args.environment]) if args.list: print(list_platforms()) elif args.platform: platform = args.platform tag = get_docker_tag(platform=platform, registry=args.docker_registry) if args.docker_registry: load_docker_cache(tag=tag, docker_registry=args.docker_registry) if not args.run_only: build_docker(platform=platform, docker_binary=docker_binary, registry=args.docker_registry, num_retries=args.docker_build_retries, no_cache=args.no_cache, cache_intermediate=args.cache_intermediate) else: logging.info("Skipping docker build step.") if args.build_only: logging.warning( "Container was just built. Exiting due to build-only.") return 0 # noinspection PyUnusedLocal ret = 0 if command: ret = container_run(docker_client=docker_client, platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir, environment=environment) elif args.print_docker_run: command = [] ret = container_run(docker_client=docker_client, platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir, dry_run=True, environment=environment) else: # With no commands, execute a build function for the target platform command = [ "/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform) ] logging.info("No command specified, trying default build: %s", ' '.join(command)) ret = container_run(docker_client=docker_client, platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir, environment=environment) if ret != 0: logging.critical("Execution of %s failed with status: %d", command, ret) return ret elif args.all: platforms = get_platforms() platforms = [ platform for platform in platforms if 'build.' in platform ] logging.info("Building for all architectures: %s", platforms) logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: tag = get_docker_tag(platform=platform, registry=args.docker_registry) load_docker_cache(tag=tag, docker_registry=args.docker_registry) build_docker(platform, docker_binary=docker_binary, registry=args.docker_registry, num_retries=args.docker_build_retries, no_cache=args.no_cache) if args.build_only: continue shutil.rmtree(buildir(), ignore_errors=True) build_platform = "build_{}".format(platform) plat_buildir = os.path.abspath( os.path.join(get_mxnet_root(), '..', "mxnet_{}".format(build_platform))) if os.path.exists(plat_buildir): logging.warning("%s already exists, skipping", plat_buildir) continue command = [ "/work/mxnet/ci/docker/runtime_functions.sh", build_platform ] container_run(docker_client=docker_client, platform=platform, nvidia_runtime=args.nvidiadocker, shared_memory_size=args.shared_memory_size, command=command, docker_registry=args.docker_registry, local_ccache_dir=args.ccache_dir, environment=environment) shutil.move(buildir(), plat_buildir) logging.info("Built files left in: %s", plat_buildir) else: parser.print_help() list_platforms() print(""" Examples: ./build.py -p armv7 Will build a docker container with cross compilation tools and build MXNet for armv7 by running: ci/docker/runtime_functions.sh build_armv7 inside the container. ./build.py -p armv7 ls Will execute the given command inside the armv7 container ./build.py -p armv7 --print-docker-run Will print a docker run command to get inside the container in a shell ./build.py -a Builds for all platforms and leaves artifacts in build_<platform> """) return 0
def container_run(docker_client: SafeDockerClient, platform: str, nvidia_runtime: bool, docker_registry: str, shared_memory_size: str, local_ccache_dir: str, command: List[str], environment: Dict[str, str], dry_run: bool = False) -> int: """Run command in a container""" container_wait_s = 600 # # Environment setup # environment.update({ 'CCACHE_MAXSIZE': '500G', 'CCACHE_TEMPDIR': '/tmp/ccache', # temp dir should be local and not shared 'CCACHE_DIR': '/work/ccache', # this path is inside the container as /work/ccache is # mounted 'CCACHE_LOGFILE': '/tmp/ccache.log', # a container-scoped log, useful for ccache # verification. }) environment.update( {k: os.environ[k] for k in ['CCACHE_MAXSIZE'] if k in os.environ}) tag = get_docker_tag(platform=platform, registry=docker_registry) mx_root = get_mxnet_root() local_build_folder = buildir() # We need to create it first, otherwise it will be created by the docker daemon with root only permissions os.makedirs(local_build_folder, exist_ok=True) os.makedirs(local_ccache_dir, exist_ok=True) logging.info("Using ccache directory: %s", local_ccache_dir) # Equivalent command docker_cmd_list = [ get_docker_binary(nvidia_runtime), 'run', "--cap-add", "SYS_PTRACE", # Required by ASAN '--rm', '--shm-size={}'.format(shared_memory_size), # mount mxnet root '-v', "{}:/work/mxnet".format(mx_root), # mount mxnet/build for storing build '-v', "{}:/work/build".format(local_build_folder), '-v', "{}:/work/ccache".format(local_ccache_dir), '-u', '{}:{}'.format(os.getuid(), os.getgid()), '-e', 'CCACHE_MAXSIZE={}'.format(environment['CCACHE_MAXSIZE']), # temp dir should be local and not shared '-e', 'CCACHE_TEMPDIR={}'.format(environment['CCACHE_TEMPDIR']), # this path is inside the container as /work/ccache is mounted '-e', "CCACHE_DIR={}".format(environment['CCACHE_DIR']), # a container-scoped log, useful for ccache verification. '-e', "CCACHE_LOGFILE={}".format(environment['CCACHE_LOGFILE']), '-ti', tag ] docker_cmd_list.extend(command) docker_cmd = ' \\\n\t'.join(docker_cmd_list) logging.info("Running %s in container %s", command, tag) logging.info("Executing the equivalent of:\n%s\n", docker_cmd) if not dry_run: ############################# # signal.pthread_sigmask(signal.SIG_BLOCK, {signal.SIGINT, signal.SIGTERM}) # noinspection PyShadowingNames runtime = None if nvidia_runtime: # noinspection PyShadowingNames # runc is default (docker info | grep -i runtime) runtime = 'nvidia' return docker_client.run(tag, runtime=runtime, command=command, shm_size=shared_memory_size, user='******'.format(os.getuid(), os.getgid()), cap_add='SYS_PTRACE', volumes={ mx_root: { 'bind': '/work/mxnet', 'mode': 'rw' }, local_build_folder: { 'bind': '/work/build', 'mode': 'rw' }, local_ccache_dir: { 'bind': '/work/ccache', 'mode': 'rw' }, }, environment=environment) return 0