def _dockerKill(containerName, action): """ Kills the specified container. :param str containerName: The name of the container created by docker_call :param int action: What action should be taken on the container? See `defer=` in :func:`docker_call` """ running = containerIsRunning(containerName) if running is None: # This means that the container doesn't exist. We will see this if the container was run # with --rm and has already exited before this call. logger.debug( 'The container with name "%s" appears to have already been removed. Nothing to ' 'do.', containerName) else: if action in (None, FORGO): logger.debug( 'The container with name %s continues to exist as we were asked to forgo a ' 'post-job action on it.', containerName) else: logger.debug( 'The container with name %s exists. Running user-specified defer functions.', containerName) if running and action >= STOP: logger.debug('Stopping container "%s".', containerName) for attempt in retry(predicate=dockerPredicate): with attempt: subprocess.check_call( ['docker', 'stop', containerName]) else: logger.debug('The container "%s" was not found to be running.', containerName) if action >= RM: # If the container was run with --rm, then stop will most likely remove the # container. We first check if it is running then remove it. running = containerIsRunning(containerName) if running is not None: logger.debug('Removing container "%s".', containerName) for attempt in retry(predicate=dockerPredicate): with attempt: subprocess.check_call( ['docker', 'rm', '-f', containerName]) else: logger.debug( 'The container "%s" was not found on the system. Nothing to remove.', containerName)
def testDockerClean(self, disableCaching=True, detached=True, rm=True, deferParam=None): """ Run the test container that creates a file in the work dir, and sleeps for 5 minutes. Ensure that the calling job gets SIGKILLed after a minute, leaving behind the spooky/ghost/zombie container. Ensure that the container is killed on batch system shutdown (through the deferParam mechanism). """ # We need to test the behaviour of `deferParam` with `rm` and # `detached`. We do not look at the case where `rm` and `detached` are # both True. This is the truth table for the different combinations at # the end of the test. R = Running, X = Does not exist, E = Exists but # not running. # None FORGO STOP RM # rm X R X X # detached R R E X # Neither R R E X data_dir = os.path.join(self.tempDir, 'data') working_dir = os.path.join(self.tempDir, 'working') test_file = os.path.join(working_dir, 'test.txt') mkdir_p(data_dir) mkdir_p(working_dir) options = Job.Runner.getDefaultOptions( os.path.join(self.tempDir, 'jobstore')) options.logLevel = self.dockerTestLogLevel options.workDir = working_dir options.clean = 'always' options.disableCaching = disableCaching # No base64 logic since it might create a name starting with a `-`. container_name = uuid.uuid4().hex A = Job.wrapJobFn(_testDockerCleanFn, working_dir, detached, rm, deferParam, container_name) try: Job.Runner.startToil(A, options) except FailedJobsException: # The file created by spooky_container would remain in the directory # and since it was created inside the container, it would have had # uid and gid == 0 (root) which may cause problems when docker # attempts to clean up the jobstore. file_stats = os.stat(test_file) assert file_stats.st_gid != 0 assert file_stats.st_uid != 0 if (rm and (deferParam != FORGO)) or deferParam == RM: # These containers should not exist assert containerIsRunning(container_name) is None, \ 'Container was not removed.' elif deferParam == STOP: # These containers should exist but be non-running assert containerIsRunning(container_name) == False, \ 'Container was not stopped.' else: # These containers will be running assert containerIsRunning(container_name) == True, \ 'Container was not running.' client = docker.from_env(version='auto') dockerKill(container_name, client) try: os.remove(test_file) except: pass
def testSubprocessDockerClean(self, caching=True): """ Run the test container that creates a file in the work dir, and sleeps for 5 minutes. Ensure that the calling job gets SIGKILLed after a minute, leaving behind the spooky/ghost/zombie container. Ensure that the container is killed on batch system shutdown (through the defer mechanism). This inherently also tests _docker :returns: None """ # We need to test the behaviour of `defer` with `rm` and `detached`. We do not look at the case # where `rm` and `detached` are both True. This is the truth table for the different # combinations at the end of the test. R = Running, X = Does not exist, E = Exists but not # running. # None FORGO STOP RM # rm X R X X # detached R R E X # Neither R R E X assert os.getuid() != 0, "Cannot test this if the user is root." data_dir = os.path.join(self.tempDir, 'data') work_dir = os.path.join(self.tempDir, 'working') test_file = os.path.join(data_dir, 'test.txt') mkdir_p(data_dir) mkdir_p(work_dir) options = Job.Runner.getDefaultOptions( os.path.join(self.tempDir, 'jobstore')) options.logLevel = 'INFO' options.workDir = work_dir options.clean = 'always' if not caching: options.disableCaching = True for rm in (True, False): for detached in (True, False): if detached and rm: continue for defer in (FORGO, STOP, RM, None): # Not using base64 logic here since it might create a name starting with a `-`. container_name = uuid.uuid4().hex A = Job.wrapJobFn(_testSubprocessDockerCleanFn, data_dir, detached, rm, defer, container_name) try: Job.Runner.startToil(A, options) except FailedJobsException: # The file created by spooky_container would remain in the directory, and since # it was created inside the container, it would have had uid and gid == 0 (root) # upon creation. If the defer mechanism worked, it should now be non-zero and we # check for that. file_stats = os.stat(test_file) assert file_stats.st_gid != 0 assert file_stats.st_uid != 0 if (rm and defer != FORGO) or defer == RM: # These containers should not exist assert containerIsRunning(container_name) is None, \ 'Container was not removed.' elif defer == STOP: # These containers should exist but be non-running assert containerIsRunning(container_name) == False, \ 'Container was not stopped.' else: # These containers will be running assert containerIsRunning(container_name) == True, \ 'Container was not running.' finally: # Prepare for the next test. _dockerKill(container_name, RM) os.remove(test_file)