def task_cleanup(e):
"""
Since files written by docker containers are owned by root, we can't
clean them up in the worker process since that typically doesn't run
as root. So, we run a lightweight container to make the temp dir cleanable.
"""
from .executor import DATA_VOLUME
if e.info['task']['mode'] == 'docker' and '_tempdir' in e.info['kwargs']:
tmpdir = e.info['kwargs']['_tempdir']
client = docker.from_env(version='auto')
config = {
'tty': True,
'volumes': {
tmpdir: {
'bind': DATA_VOLUME,
'mode': 'rw'
}
},
'detach': False,
'remove': True
}
args = ['chmod', '-R', 'a+rw', DATA_VOLUME]
try:
client.containers.run('busybox:latest', args, **config)
except DockerException as dex:
logger.error('Error setting perms on docker tempdir %s.' % tmpdir)
logger.exception(dex)
raise
def _run_container(image, args, **kwargs):
# TODO we could allow configuration of non default socket
client = docker.from_env(version='auto')
logger.info('Running container: image: %s args: %s kwargs: %s' % (image, args, kwargs))
try:
return client.containers.run(image, args, **kwargs)
except DockerException as dex:
logger.error(dex)
raise
def _pull_image(image):
"""
Pulls the specified Docker image onto this worker.
"""
client = docker.from_env(version='auto')
try:
client.images.pull(image)
except DockerException as dex:
logger.error('Error pulling Docker image %s:' % image)
logger.exception(dex)
raise
def _run_select_loop(task, container, read_stream_connectors,
write_stream_connectors):
stdout = None
stderr = None
try:
# attach to standard streams
stdout = container.attach_socket(params={
'stdout': True,
'logs': True,
'stream': True
})
stderr = container.attach_socket(params={
'stderr': True,
'logs': True,
'stream': True
})
def exit_condition():
container.reload()
return container.status in {'exited', 'dead'} or task.canceled
# Look for ContainerStdOut and ContainerStdErr instances that need
# to be replace with the real container streams.
stdout_connected = False
for read_stream_connector in read_stream_connectors:
if isinstance(read_stream_connector.input, ContainerStdOut):
stdout_reader = _SocketReader(stdout)
read_stream_connector.output = DockerStreamPushAdapter(
read_stream_connector.output)
read_stream_connector.input = stdout_reader
stdout_connected = True
break
stderr_connected = False
for read_stream_connector in read_stream_connectors:
if isinstance(read_stream_connector.input, ContainerStdErr):
stderr_reader = _SocketReader(stderr)
read_stream_connector.output = DockerStreamPushAdapter(
read_stream_connector.output)
read_stream_connector.input = stderr_reader
stderr_connected = True
break
# If not stdout and stderr connection has been provided just use
# sys.stdXXX
if not stdout_connected:
stdout_reader = _SocketReader(stdout)
connector = FDReadStreamConnector(
stdout_reader,
DockerStreamPushAdapter(StdStreamWriter(sys.stdout)))
read_stream_connectors.append(connector)
if not stderr_connected:
stderr_reader = _SocketReader(stderr)
connector = FDReadStreamConnector(
stderr_reader,
DockerStreamPushAdapter(StdStreamWriter(sys.stderr)))
read_stream_connectors.append(connector)
# Run select loop
utils.select_loop(exit_condition=exit_condition,
readers=read_stream_connectors,
writers=write_stream_connectors)
if task.canceled:
try:
container.stop()
# Catch the ReadTimeout from requests and wait for container to
# exit. See https://github.com/docker/docker-py/issues/1374 for
# more details.
except ReadTimeout:
tries = 10
while tries > 0:
container.reload()
if container.status == 'exited':
break
if container.status != 'exited':
msg = 'Unable to stop container: %s' % container.id
logger.error(msg)
except DockerException as dex:
logger.error(dex)
raise
container.reload()
exit_code = container.attrs['State']['ExitCode']
if not task.canceled and exit_code != 0:
raise DockerException(
'Non-zero exit code from docker container (%d).' % exit_code)
finally:
# Close our stdout and stderr sockets
if stdout:
stdout.close()
if stderr:
stderr.close()