def load_docker_cache(tag, docker_registry) -> None:
if docker_registry:
try:
import docker_cache
logging.info('Docker cache download is enabled from registry %s', docker_registry)
docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag)
except Exception:
logging.exception('Unable to retrieve Docker cache. Continue without...')
else:
logging.info('Distributed docker cache disabled')
def load_docker_cache(tag, docker_registry) -> None:
if docker_registry:
try:
import docker_cache
logging.info('Docker cache download is enabled from registry %s', docker_registry)
docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag)
except Exception:
logging.exception('Unable to retrieve Docker cache. Continue without...')
else:
logging.info('Distributed docker cache disabled')
def load_docker_cache(tag, docker_registry) -> None:
"""Imports tagged container from the given docker registry"""
if docker_registry:
# noinspection PyBroadException
try:
import docker_cache
logging.info('Docker cache download is enabled from registry %s', docker_registry)
docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag)
except Exception:
logging.exception('Unable to retrieve Docker cache. Continue without...')
else:
logging.info('Distributed docker cache disabled')
def load_docker_cache(tag, docker_registry) -> None:
"""Imports tagged container from the given docker registry"""
if docker_registry:
# noinspection PyBroadException
try:
import docker_cache
logging.info('Docker cache download is enabled from registry %s', docker_registry)
docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag)
except Exception:
logging.exception('Unable to retrieve Docker cache. Continue without...')
else:
logging.info('Distributed docker cache disabled')
def load_docker_cache(platform, tag, docker_registry) -> None:
"""Imports tagged container from the given docker registry"""
if docker_registry:
if platform in DOCKER_COMPOSE_WHITELIST:
env = os.environ.copy()
env["DOCKER_CACHE_REGISTRY"] = docker_registry
cmd = ['docker-compose', '-f', 'docker/docker-compose.yml', 'pull', platform]
logging.info("Running command: 'DOCKER_CACHE_REGISTRY=%s %s'", docker_registry, ' '.join(cmd))
check_call(cmd, env=env)
return
# noinspection PyBroadException
try:
import docker_cache
logging.info('Docker cache download is enabled from registry %s', docker_registry)
docker_cache.load_docker_cache(registry=docker_registry, docker_tag=tag)
except Exception:
logging.exception('Unable to retrieve Docker cache. Continue without...')
else:
logging.info('Distributed docker cache disabled')
def main() -> int:
# We need to be in the same directory than the script so the commands in the dockerfiles work as
# expected. But the script can be invoked from a different path
base = os.path.split(os.path.realpath(__file__))[0]
os.chdir(base)
logging.getLogger().setLevel(logging.INFO)
def script_name() -> str:
return os.path.split(sys.argv[0])[1]
logging.basicConfig(
format='{}: %(asctime)-15s %(message)s'.format(script_name()))
parser = argparse.ArgumentParser(
description="""Utility for building and testing MXNet on docker
containers""",
epilog="")
parser.add_argument("-p", "--platform", help="platform", type=str)
parser.add_argument(
"--build-only",
help="Only build the container, don't build the project",
action='store_true')
parser.add_argument("-a",
"--all",
help="build for all platforms",
action='store_true')
parser.add_argument("-n",
"--nvidiadocker",
help="Use nvidia docker",
action='store_true')
parser.add_argument(
"--shm-size",
help=
"Size of the shared memory /dev/shm allocated in the container (e.g '1g')",
default='500m',
dest="shared_memory_size")
parser.add_argument("-l",
"--list",
help="List platforms",
action='store_true')
parser.add_argument("--print-docker-run",
help="print docker run command for manual inspection",
action='store_true')
parser.add_argument("-i",
"--into-container",
help="go in a shell inside the container",
action='store_true')
parser.add_argument(
"--download-docker-cache",
help=
"Download the docker cache from our central repository instead of rebuilding locally",
action='store_true')
parser.add_argument(
"--docker-cache-bucket",
help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache",
type=str)
parser.add_argument("command",
help="command to run in the container",
nargs='*',
action='append',
type=str)
args = parser.parse_args()
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
shared_memory_size = args.shared_memory_size
print("into container: {}".format(args.into_container))
if args.list:
list_platforms()
elif args.platform:
platform = args.platform
tag = get_docker_tag(platform)
if args.download_docker_cache:
import docker_cache
logging.info('Docker cache download is enabled')
docker_cache.load_docker_cache(
bucket_name=args.docker_cache_bucket, docker_tag=tag)
build_docker(platform, docker_binary)
if args.build_only:
logging.warning(
"Container was just built. Exiting due to build-only.")
return 0
if command:
container_run(platform, docker_binary, shared_memory_size, command)
elif args.print_docker_run:
print(
container_run(platform, docker_binary, shared_memory_size, [],
True))
elif args.into_container:
container_run(platform, docker_binary, shared_memory_size, [],
False, True)
else:
cmd = [
"/work/mxnet/ci/docker/runtime_functions.sh",
"build_{}".format(platform)
]
logging.info("No command specified, trying default build: %s",
' '.join(cmd))
container_run(platform, docker_binary, shared_memory_size, cmd)
elif args.all:
platforms = get_platforms()
logging.info("Building for all architectures: {}".format(platforms))
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
if args.download_docker_cache:
import docker_cache
tag = get_docker_tag(platform)
logging.info('Docker cache download is enabled')
docker_cache.load_docker_cache(
bucket_name=args.docker_cache_bucket, docker_tag=tag)
build_docker(platform, docker_binary)
if args.build_only:
continue
build_platform = "build_{}".format(platform)
cmd = [
"/work/mxnet/ci/docker/runtime_functions.sh", build_platform
]
shutil.rmtree(buildir(), ignore_errors=True)
container_run(platform, docker_binary, shared_memory_size, cmd)
plat_buildir = os.path.join(get_mxnet_root(), build_platform)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)
else:
parser.print_help()
list_platforms()
print("""
Examples:
./build.py -p armv7
Will build a docker container with cross compilation tools and build MXNet for armv7 by
running: ci/docker/runtime_functions.sh build_armv7 inside the container.
./build.py -p armv7 ls
Will execute the given command inside the armv7 container
./build.py -p armv7 --print-docker-run
Will print a docker run command to get inside the container in an interactive shell
./build.py -p armv7 --into-container
Will execute a shell into the container
./build.py -a
Builds for all platforms and leaves artifacts in build_<platform>
""")
return 0
def main() -> int:
# We need to be in the same directory than the script so the commands in the dockerfiles work as
# expected. But the script can be invoked from a different path
base = os.path.split(os.path.realpath(__file__))[0]
os.chdir(base)
logging.getLogger().setLevel(logging.INFO)
def script_name() -> str:
return os.path.split(sys.argv[0])[1]
logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
parser = argparse.ArgumentParser(description="""Utility for building and testing MXNet on docker
containers""",epilog="")
parser.add_argument("-p", "--platform",
help="platform",
type=str)
parser.add_argument("--build-only",
help="Only build the container, don't build the project",
action='store_true')
parser.add_argument("-a", "--all",
help="build for all platforms",
action='store_true')
parser.add_argument("-n", "--nvidiadocker",
help="Use nvidia docker",
action='store_true')
parser.add_argument("--shm-size",
help="Size of the shared memory /dev/shm allocated in the container (e.g '1g')",
default='500m',
dest="shared_memory_size")
parser.add_argument("-l", "--list",
help="List platforms",
action='store_true')
parser.add_argument("--print-docker-run",
help="print docker run command for manual inspection",
action='store_true')
parser.add_argument("-i", "--into-container",
help="go in a shell inside the container",
action='store_true')
parser.add_argument("--download-docker-cache",
help="Download the docker cache from our central repository instead of rebuilding locally",
action='store_true')
parser.add_argument("--docker-cache-bucket",
help="S3 docker cache bucket, e.g. mxnet-ci-docker-cache",
type=str)
parser.add_argument("command",
help="command to run in the container",
nargs='*', action='append', type=str)
args = parser.parse_args()
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
shared_memory_size = args.shared_memory_size
print("into container: {}".format(args.into_container))
if args.list:
list_platforms()
elif args.platform:
platform = args.platform
tag = get_docker_tag(platform)
if args.download_docker_cache:
logging.info('Docker cache download is enabled')
docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag)
build_docker(platform, docker_binary)
if args.build_only:
logging.warning("Container was just built. Exiting due to build-only.")
return 0
if command:
container_run(platform, docker_binary, shared_memory_size, command)
elif args.print_docker_run:
print(container_run(platform, docker_binary, shared_memory_size, [], True))
elif args.into_container:
container_run(platform, docker_binary, shared_memory_size, [], False, True)
else:
cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
logging.info("No command specified, trying default build: %s", ' '.join(cmd))
container_run(platform, docker_binary, shared_memory_size, cmd)
elif args.all:
platforms = get_platforms()
logging.info("Building for all architectures: {}".format(platforms))
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
if args.download_docker_cache:
tag = get_docker_tag(platform)
logging.info('Docker cache download is enabled')
docker_cache.load_docker_cache(bucket_name=args.docker_cache_bucket, docker_tag=tag)
build_docker(platform, docker_binary)
if args.build_only:
continue
build_platform = "build_{}".format(platform)
cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform]
shutil.rmtree(buildir(), ignore_errors=True)
container_run(platform, docker_binary, shared_memory_size, cmd)
plat_buildir = os.path.join(get_mxnet_root(), build_platform)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)
else:
parser.print_help()
list_platforms()
print("""
Examples:
./build.py -p armv7
Will build a docker container with cross compilation tools and build MXNet for armv7 by
running: ci/docker/runtime_functions.sh build_armv7 inside the container.
./build.py -p armv7 ls
Will execute the given command inside the armv7 container
./build.py -p armv7 --print-docker-run
Will print a docker run command to get inside the container in an interactive shell
./build.py -p armv7 --into-container
Will execute a shell into the container
./build.py -a
Builds for all platforms and leaves artifacts in build_<platform>
""")
return 0
