def load_docker_cache(tag, docker_registry) -> None: if docker_registry: try: import docker_cache logging.info('Docker cache download is enabled from registry %s', docker_registry) docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag) except Exception: logging.exception('Unable to retrieve Docker cache. Continue without...') else: logging.info('Distributed docker cache disabled')
def load_docker_cache(tag, docker_registry) -> None: """Imports tagged container from the given docker registry""" if docker_registry: # noinspection PyBroadException try: import docker_cache logging.info('Docker cache download is enabled from registry %s', docker_registry) docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag) except Exception: logging.exception('Unable to retrieve Docker cache. Continue without...') else: logging.info('Distributed docker cache disabled')
def load_docker_cache(platform, tag, docker_registry) -> None: """Imports tagged container from the given docker registry""" if docker_registry: if platform in DOCKER_COMPOSE_WHITELIST: env = os.environ.copy() env["DOCKER_CACHE_REGISTRY"] = docker_registry cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'pull', platform] logging.info("Running command: 'DOCKER_CACHE_REGISTRY=%s %s'", docker_registry, ' '.join(cmd)) check_call(cmd, env=env) return # noinspection PyBroadException try: import docker_cache logging.info('Docker cache download is enabled from registry %s', docker_registry) docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag) except Exception: logging.exception('Unable to retrieve Docker cache. Continue without...') else: logging.info('Distributed docker cache disabled')
def main() -> int: # We need to be in the same directory than the script so the commands in the dockerfiles work as # expected. But the script can be invoked from a different path base = os.path.split(os.path.realpath(__file__))[0] os.chdir(base) logging.getLogger().setLevel(logging.INFO) def script_name() -> str: return os.path.split(sys.argv[0])[1] logging.basicConfig( format='{}: %(asctime)-15s %(message)s'.format(script_name())) parser = argparse.ArgumentParser( description="""Utility for building and testing MXNet on docker containers""", epilog="") parser.add_argument("-p", "--platform", help="platform", type=str) parser.add_argument( "--build-only", help="Only build the container, don't build the project", action='store_true') parser.add_argument("-a", "--all", help="build for all platforms", action='store_true') parser.add_argument("-n", "--nvidiadocker", help="Use nvidia docker", action='store_true') parser.add_argument( "--shm-size", help= "Size of the shared memory /dev/shm allocated in the container (e.g '1g')", default='500m', dest="shared_memory_size") parser.add_argument("-l", "--list", help="List platforms", action='store_true') parser.add_argument("--print-docker-run", help="print docker run command for manual inspection", action='store_true') parser.add_argument("-i", "--into-container", help="go in a shell inside the container", action='store_true') parser.add_argument( "--download-docker-cache", help= "Download the docker cache from our central repository instead of rebuilding locally", action='store_true') parser.add_argument( "--docker-cache-bucket", help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache", type=str) parser.add_argument("command", help="command to run in the container", nargs='*', action='append', type=str) args = parser.parse_args() command = list(chain(*args.command)) docker_binary = get_docker_binary(args.nvidiadocker) shared_memory_size = args.shared_memory_size print("into container: {}".format(args.into_container)) if args.list: list_platforms() elif args.platform: platform = args.platform tag = get_docker_tag(platform) if args.download_docker_cache: import docker_cache logging.info('Docker cache download is enabled') docker_cache.load_docker_cache( bucket_name=args.docker_cache_bucket, docker_tag=tag) build_docker(platform, docker_binary) if args.build_only: logging.warning( "Container was just built. Exiting due to build-only.") return 0 if command: container_run(platform, docker_binary, shared_memory_size, command) elif args.print_docker_run: print( container_run(platform, docker_binary, shared_memory_size, [], True)) elif args.into_container: container_run(platform, docker_binary, shared_memory_size, [], False, True) else: cmd = [ "/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform) ] logging.info("No command specified, trying default build: %s", ' '.join(cmd)) container_run(platform, docker_binary, shared_memory_size, cmd) elif args.all: platforms = get_platforms() logging.info("Building for all architectures: {}".format(platforms)) logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: if args.download_docker_cache: import docker_cache tag = get_docker_tag(platform) logging.info('Docker cache download is enabled') docker_cache.load_docker_cache( bucket_name=args.docker_cache_bucket, docker_tag=tag) build_docker(platform, docker_binary) if args.build_only: continue build_platform = "build_{}".format(platform) cmd = [ "/work/mxnet/ci/docker/runtime_functions.sh", build_platform ] shutil.rmtree(buildir(), ignore_errors=True) container_run(platform, docker_binary, shared_memory_size, cmd) plat_buildir = os.path.join(get_mxnet_root(), build_platform) shutil.move(buildir(), plat_buildir) logging.info("Built files left in: %s", plat_buildir) else: parser.print_help() list_platforms() print(""" Examples: ./build.py -p armv7 Will build a docker container with cross compilation tools and build MXNet for armv7 by running: ci/docker/runtime_functions.sh build_armv7 inside the container. ./build.py -p armv7 ls Will execute the given command inside the armv7 container ./build.py -p armv7 --print-docker-run Will print a docker run command to get inside the container in an interactive shell ./build.py -p armv7 --into-container Will execute a shell into the container ./build.py -a Builds for all platforms and leaves artifacts in build_<platform> """) return 0
def main() -> int: # We need to be in the same directory than the script so the commands in the dockerfiles work as # expected. But the script can be invoked from a different path base = os.path.split(os.path.realpath(__file__))[0] os.chdir(base) logging.getLogger().setLevel(logging.INFO) def script_name() -> str: return os.path.split(sys.argv[0])[1] logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name())) parser = argparse.ArgumentParser(description="""Utility for building and testing MXNet on docker containers""",epilog="") parser.add_argument("-p", "--platform", help="platform", type=str) parser.add_argument("--build-only", help="Only build the container, don't build the project", action='store_true') parser.add_argument("-a", "--all", help="build for all platforms", action='store_true') parser.add_argument("-n", "--nvidiadocker", help="Use nvidia docker", action='store_true') parser.add_argument("--shm-size", help="Size of the shared memory /dev/shm allocated in the container (e.g '1g')", default='500m', dest="shared_memory_size") parser.add_argument("-l", "--list", help="List platforms", action='store_true') parser.add_argument("--print-docker-run", help="print docker run command for manual inspection", action='store_true') parser.add_argument("-i", "--into-container", help="go in a shell inside the container", action='store_true') parser.add_argument("--download-docker-cache", help="Download the docker cache from our central repository instead of rebuilding locally", action='store_true') parser.add_argument("--docker-cache-bucket", help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache", type=str) parser.add_argument("command", help="command to run in the container", nargs='*', action='append', type=str) args = parser.parse_args() command = list(chain(*args.command)) docker_binary = get_docker_binary(args.nvidiadocker) shared_memory_size = args.shared_memory_size print("into container: {}".format(args.into_container)) if args.list: list_platforms() elif args.platform: platform = args.platform tag = get_docker_tag(platform) if args.download_docker_cache: logging.info('Docker cache download is enabled') docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag) build_docker(platform, docker_binary) if args.build_only: logging.warning("Container was just built. Exiting due to build-only.") return 0 if command: container_run(platform, docker_binary, shared_memory_size, command) elif args.print_docker_run: print(container_run(platform, docker_binary, shared_memory_size, [], True)) elif args.into_container: container_run(platform, docker_binary, shared_memory_size, [], False, True) else: cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)] logging.info("No command specified, trying default build: %s", ' '.join(cmd)) container_run(platform, docker_binary, shared_memory_size, cmd) elif args.all: platforms = get_platforms() logging.info("Building for all architectures: {}".format(platforms)) logging.info("Artifacts will be produced in the build/ directory.") for platform in platforms: if args.download_docker_cache: tag = get_docker_tag(platform) logging.info('Docker cache download is enabled') docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag) build_docker(platform, docker_binary) if args.build_only: continue build_platform = "build_{}".format(platform) cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform] shutil.rmtree(buildir(), ignore_errors=True) container_run(platform, docker_binary, shared_memory_size, cmd) plat_buildir = os.path.join(get_mxnet_root(), build_platform) shutil.move(buildir(), plat_buildir) logging.info("Built files left in: %s", plat_buildir) else: parser.print_help() list_platforms() print(""" Examples: ./build.py -p armv7 Will build a docker container with cross compilation tools and build MXNet for armv7 by running: ci/docker/runtime_functions.sh build_armv7 inside the container. ./build.py -p armv7 ls Will execute the given command inside the armv7 container ./build.py -p armv7 --print-docker-run Will print a docker run command to get inside the container in an interactive shell ./build.py -p armv7 --into-container Will execute a shell into the container ./build.py -a Builds for all platforms and leaves artifacts in build_<platform> """) return 0