def _run_container(image, container_args, **kwargs):
# TODO we could allow configuration of non default socket
client = docker.from_env(version='auto')
runtime = kwargs.pop('runtime', None)
origRuntime = runtime
if runtime is None and nvidia.is_nvidia_image(client.api, image):
runtime = 'nvidia'
logger.info(
'Running container: image: %s args: %s runtime: %s kwargs: %s' %
(image, container_args, runtime, kwargs))
try:
try:
return client.containers.run(image,
container_args,
runtime=runtime,
**kwargs)
except APIError:
if origRuntime is None and runtime is not None:
return client.containers.run(image, container_args, **kwargs)
else:
raise
except DockerException:
logger.exception('Exception when running docker container')
raise
def _run_container(image, container_args, **kwargs):
# TODO we could allow configuration of non default socket
if 'DOCKER_CLIENT_TIMEOUT' in os.environ:
timeout = int(os.environ['DOCKER_CLIENT_TIMEOUT'])
client = docker.from_env(version='auto', timeout=timeout)
else:
client = docker.from_env(version='auto')
runtime = kwargs.pop('runtime', None)
origRuntime = runtime
if runtime is None and nvidia.is_nvidia_image(client.api, image):
runtime = 'nvidia'
container_args = [str(arg) for arg in container_args]
logger.info(
'Running container: image: %s args: %s runtime: %s kwargs: %s' %
(image, container_args, runtime, kwargs))
try:
try:
return client.containers.run(image,
container_args,
runtime=runtime,
**kwargs)
except APIError:
if origRuntime is None and runtime is not None:
return client.containers.run(image, container_args, **kwargs)
else:
raise
except DockerException:
logger.exception('Exception when running docker container')
raise
def _run_container(image, container_args, **kwargs):
# TODO we could allow configuration of non default socket
client = docker.from_env(version='auto')
if nvidia.is_nvidia_image(client.api, image):
client = nvidia.NvidiaDockerClient.from_env(version='auto')
logger.info('Running container: image: %s args: %s kwargs: %s' %
(image, container_args, kwargs))
try:
return client.containers.run(image, container_args, **kwargs)
except nvidia.NvidiaConnectionError:
try:
logger.info(
'Running nvidia container without nvidia support: image: %s' %
image)
client = docker.from_env(version='auto')
return client.containers.run(image, container_args, **kwargs)
except DockerException:
logger.exception(
'Exception when running docker container without nvidia support.'
)
raise
except DockerException:
logger.exception('Exception when running docker container')
raise
def test_is_nvidia_image_returns_true():
api = mock.MagicMock(spec=NvidiaAPIClient)
api.inspect_image.return_value = {
'Config': {
'Labels': {
'com.nvidia.volumes.needed': 'nvidia_driver'
}
}
}
assert is_nvidia_image(api, 'bogus/image:latest') is True
def _run_container(image, container_args, **kwargs):
# TODO we could allow configuration of non default socket
if 'DOCKER_CLIENT_TIMEOUT' in os.environ:
timeout = int(os.environ['DOCKER_CLIENT_TIMEOUT'])
client = docker.from_env(version='auto', timeout=timeout)
else:
client = docker.from_env(version='auto')
runtime = kwargs.pop('runtime', None)
origRuntime = runtime
if runtime is None and nvidia.is_nvidia_image(client.api, image):
runtime = 'nvidia'
container_args = [str(arg) for arg in container_args]
docker_network = _get_docker_network()
if docker_network and 'network' not in kwargs:
kwargs = kwargs.copy()
kwargs['network'] = docker_network
logger.info(
'Running container: image: %s args: %s runtime: %s kwargs: %s' %
(image, container_args, runtime, kwargs))
try:
try:
if runtime == 'nvidia' and kwargs.get('device_requests') is None:
# Docker < 19.03 required the runtime='nvidia' argument.
# Newer versions require a device request for some number of
# GPUs. This should handle either version of the docker
# daemon.
try:
device_requests_kwargs = kwargs.copy()
device_requests_kwargs['device_requests'] = [
docker.types.DeviceRequest(count=-1,
capabilities=[['gpu']])
]
return client.containers.run(image, container_args,
**device_requests_kwargs)
except (APIError, InvalidVersion):
pass
return client.containers.run(image,
container_args,
runtime=runtime,
**kwargs)
except APIError:
if origRuntime is None and runtime is not None:
return client.containers.run(image, container_args, **kwargs)
else:
raise
except DockerException:
logger.exception('Exception when running docker container')
raise
def test_is_nvidia_image_no_nvidia_labels_returns_false():
api = mock.MagicMock(spec=NvidiaAPIClient)
api.inspect_image.return_value = {'Config': {'Labels': {'some': 'label'}}}
assert is_nvidia_image(api, 'bogus/image:latest') is False
def test_is_nvidia_image_no_labels_returns_false():
api = mock.MagicMock(spec=NvidiaAPIClient)
api.inspect_image.return_value = {}
assert is_nvidia_image(api, 'bogus/image:latest') is False