def _dockerKill(containerName, action):
"""
Kills the specified container.
:param str containerName: The name of the container created by docker_call
:param int action: What action should be taken on the container? See `defer=` in
:func:`docker_call`
"""
running = containerIsRunning(containerName)
if running is None:
# This means that the container doesn't exist. We will see this if the container was run
# with --rm and has already exited before this call.
logger.debug(
'The container with name "%s" appears to have already been removed. Nothing to '
'do.', containerName)
else:
if action in (None, FORGO):
logger.debug(
'The container with name %s continues to exist as we were asked to forgo a '
'post-job action on it.', containerName)
else:
logger.debug(
'The container with name %s exists. Running user-specified defer functions.',
containerName)
if running and action >= STOP:
logger.debug('Stopping container "%s".', containerName)
for attempt in retry(predicate=dockerPredicate):
with attempt:
subprocess.check_call(
['docker', 'stop', containerName])
else:
logger.debug('The container "%s" was not found to be running.',
containerName)
if action >= RM:
# If the container was run with --rm, then stop will most likely remove the
# container. We first check if it is running then remove it.
running = containerIsRunning(containerName)
if running is not None:
logger.debug('Removing container "%s".', containerName)
for attempt in retry(predicate=dockerPredicate):
with attempt:
subprocess.check_call(
['docker', 'rm', '-f', containerName])
else:
logger.debug(
'The container "%s" was not found on the system. Nothing to remove.',
containerName)
def testDockerClean(self,
disableCaching=True,
detached=True,
rm=True,
deferParam=None):
"""
Run the test container that creates a file in the work dir, and sleeps
for 5 minutes.
Ensure that the calling job gets SIGKILLed after a minute, leaving
behind the spooky/ghost/zombie container. Ensure that the container is
killed on batch system shutdown (through the deferParam mechanism).
"""
# We need to test the behaviour of `deferParam` with `rm` and
# `detached`. We do not look at the case where `rm` and `detached` are
# both True. This is the truth table for the different combinations at
# the end of the test. R = Running, X = Does not exist, E = Exists but
# not running.
# None FORGO STOP RM
# rm X R X X
# detached R R E X
# Neither R R E X
data_dir = os.path.join(self.tempDir, 'data')
working_dir = os.path.join(self.tempDir, 'working')
test_file = os.path.join(working_dir, 'test.txt')
mkdir_p(data_dir)
mkdir_p(working_dir)
options = Job.Runner.getDefaultOptions(
os.path.join(self.tempDir, 'jobstore'))
options.logLevel = self.dockerTestLogLevel
options.workDir = working_dir
options.clean = 'always'
options.disableCaching = disableCaching
# No base64 logic since it might create a name starting with a `-`.
container_name = uuid.uuid4().hex
A = Job.wrapJobFn(_testDockerCleanFn, working_dir, detached, rm,
deferParam, container_name)
try:
Job.Runner.startToil(A, options)
except FailedJobsException:
# The file created by spooky_container would remain in the directory
# and since it was created inside the container, it would have had
# uid and gid == 0 (root) which may cause problems when docker
# attempts to clean up the jobstore.
file_stats = os.stat(test_file)
assert file_stats.st_gid != 0
assert file_stats.st_uid != 0
if (rm and (deferParam != FORGO)) or deferParam == RM:
# These containers should not exist
assert containerIsRunning(container_name) is None, \
'Container was not removed.'
elif deferParam == STOP:
# These containers should exist but be non-running
assert containerIsRunning(container_name) == False, \
'Container was not stopped.'
else:
# These containers will be running
assert containerIsRunning(container_name) == True, \
'Container was not running.'
client = docker.from_env(version='auto')
dockerKill(container_name, client)
try:
os.remove(test_file)
except:
pass
def testSubprocessDockerClean(self, caching=True):
"""
Run the test container that creates a file in the work dir, and sleeps for 5 minutes. Ensure
that the calling job gets SIGKILLed after a minute, leaving behind the spooky/ghost/zombie
container. Ensure that the container is killed on batch system shutdown (through the defer
mechanism).
This inherently also tests _docker
:returns: None
"""
# We need to test the behaviour of `defer` with `rm` and `detached`. We do not look at the case
# where `rm` and `detached` are both True. This is the truth table for the different
# combinations at the end of the test. R = Running, X = Does not exist, E = Exists but not
# running.
# None FORGO STOP RM
# rm X R X X
# detached R R E X
# Neither R R E X
assert os.getuid() != 0, "Cannot test this if the user is root."
data_dir = os.path.join(self.tempDir, 'data')
work_dir = os.path.join(self.tempDir, 'working')
test_file = os.path.join(data_dir, 'test.txt')
mkdir_p(data_dir)
mkdir_p(work_dir)
options = Job.Runner.getDefaultOptions(
os.path.join(self.tempDir, 'jobstore'))
options.logLevel = 'INFO'
options.workDir = work_dir
options.clean = 'always'
if not caching:
options.disableCaching = True
for rm in (True, False):
for detached in (True, False):
if detached and rm:
continue
for defer in (FORGO, STOP, RM, None):
# Not using base64 logic here since it might create a name starting with a `-`.
container_name = uuid.uuid4().hex
A = Job.wrapJobFn(_testSubprocessDockerCleanFn, data_dir,
detached, rm, defer, container_name)
try:
Job.Runner.startToil(A, options)
except FailedJobsException:
# The file created by spooky_container would remain in the directory, and since
# it was created inside the container, it would have had uid and gid == 0 (root)
# upon creation. If the defer mechanism worked, it should now be non-zero and we
# check for that.
file_stats = os.stat(test_file)
assert file_stats.st_gid != 0
assert file_stats.st_uid != 0
if (rm and defer != FORGO) or defer == RM:
# These containers should not exist
assert containerIsRunning(container_name) is None, \
'Container was not removed.'
elif defer == STOP:
# These containers should exist but be non-running
assert containerIsRunning(container_name) == False, \
'Container was not stopped.'
else:
# These containers will be running
assert containerIsRunning(container_name) == True, \
'Container was not running.'
finally:
# Prepare for the next test.
_dockerKill(container_name, RM)
os.remove(test_file)