def testConcurrencyWithDisk(self): """ Tests that the batch system is allocating disk resources properly """ tempDir = self._createTempDir('testFiles') options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.workDir = tempDir from toil import physicalDisk availableDisk = physicalDisk('', toilWorkflowDir=options.workDir) options.batchSystem = self.batchSystemName counterPath = os.path.join(tempDir, 'counter') resetCounters(counterPath) value, maxValue = getCounters(counterPath) assert (value, maxValue) == (0, 0) root = Job() # Physically, we're asking for 50% of disk and 50% of disk + 500bytes in the two jobs. The # batchsystem should not allow the 2 child jobs to run concurrently. root.addChild(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1, memory='1M', disk=availableDisk/2)) root.addChild(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1, memory='1M', disk=(availableDisk / 2) + 500)) Job.Runner.startToil(root, options) _, maxValue = getCounters(counterPath) self.assertEqual(maxValue, 1)
def testEncapsulation(self): """ Tests the Job.encapsulation method, which uses the EncapsulationJob class. """ #Temporary file outFile = getTempFile(rootDir=os.getcwd()) #Make a job graph a = T.wrapFn(f, "A", outFile) b = a.addChildFn(f, a.rv(), outFile) c = a.addFollowOnFn(f, b.rv(), outFile) #Encapsulate it a = a.encapsulate() #Now add children/follow to the encapsulated graph d = T.wrapFn(f, c.rv(), outFile) e = T.wrapFn(f, d.rv(), outFile) a.addChild(d) a.addFollowOn(e) #Create the runner for the workflow. options = T.Runner.getDefaultOptions() options.logLevel = "INFO" #Run the workflow, the return value being the number of failed jobs self.assertEquals(T.Runner.startToil(a, options), 0) T.Runner.cleanup(options) #This removes the jobStore #Check output self.assertEquals(open(outFile, 'r').readline(), "ABCDE") #Cleanup os.remove(outFile)
def testEncapsulation(self): """ Tests the Job.encapsulation method, which uses the EncapsulationJob class. """ # Temporary file outFile = getTempFile(rootDir=self._createTempDir()) try: # Encapsulate a job graph a = T.wrapJobFn(encapsulatedJobFn, "A", outFile) a = a.encapsulate() # Now add children/follow to the encapsulated graph d = T.wrapFn(f, a.rv(), outFile) e = T.wrapFn(f, d.rv(), outFile) a.addChild(d) a.addFollowOn(e) # Create the runner for the workflow. options = T.Runner.getDefaultOptions(self._getTestJobStorePath()) options.logLevel = "INFO" # Run the workflow, the return value being the number of failed jobs T.Runner.startToil(a, options) # Check output self.assertEquals(open(outFile, 'r').readline(), "ABCDE") finally: os.remove(outFile)
def testAddChildEncapsulate(self): """ Make sure that the encapsulate child does not have two pareents with unique roots. """ # Temporary file a = T.wrapFn(noOp) b = T.wrapFn(noOp) a.addChild(b).encapsulate() self.assertEquals(len(a.getRootJobs()), 1)
def testAddChildEncapsulate(self): """ Make sure that the encapsulate child does not have two parents with unique roots. """ # Temporary file a = T.wrapFn(noOp) b = T.wrapFn(noOp) a.addChild(b).encapsulate() self.assertEqual(len(a.getRootJobs()), 1)
def testConcurrencyWithDisk(self): """ Tests that the batch system is allocating disk resources properly """ tempDir = self._createTempDir('testFiles') options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.workDir = tempDir from toil import physicalDisk availableDisk = physicalDisk(options.workDir) logger.info('Testing disk concurrency limits with %s disk space', availableDisk) # More disk might become available by the time Toil starts, so we limit it here options.maxDisk = availableDisk options.batchSystem = self.batchSystemName counterPath = os.path.join(tempDir, 'counter') resetCounters(counterPath) value, maxValue = getCounters(counterPath) assert (value, maxValue) == (0, 0) half_disk = availableDisk // 2 more_than_half_disk = half_disk + 500 logger.info('Dividing into parts of %s and %s', half_disk, more_than_half_disk) root = Job() # Physically, we're asking for 50% of disk and 50% of disk + 500bytes in the two jobs. The # batchsystem should not allow the 2 child jobs to run concurrently. root.addChild( Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1, memory='1M', disk=half_disk)) root.addChild( Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1, memory='1M', disk=more_than_half_disk)) Job.Runner.startToil(root, options) _, maxValue = getCounters(counterPath) logger.info('After run: %s disk space', physicalDisk(options.workDir)) self.assertEqual(maxValue, 1)
def makeJob(string): promises = [] job = Job.wrapFn( fn2Test, promises, string, None if outPath is None else os.path.join(outPath, string)) jobsToPromisesMap[job] = promises return job
def testJobConcurrency(self): """ Tests that the batch system is allocating core resources properly for concurrent tasks. """ for cores_per_job in self.allocated_cores: temp_dir = self._createTempDir('testFiles') options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.workDir = temp_dir options.maxCores = self.cpu_count options.batchSystem = self.batchSystemName counter_path = os.path.join(temp_dir, 'counter') resetCounters(counter_path) value, max_value = getCounters(counter_path) assert (value, max_value) == (0, 0) root = Job() for _ in range(self.cpu_count): root.addFollowOn(Job.wrapFn(measureConcurrency, counter_path, self.sleep_time, cores=cores_per_job, memory='1M', disk='1Mi')) Job.Runner.startToil(root, options) _, max_value = getCounters(counter_path) self.assertEqual(max_value, self.cpu_count / cores_per_job)
def testJobConcurrency(self): """ Tests that the batch system is allocating core resources properly for concurrent tasks. """ for coresPerJob in self.allocatedCores: tempDir = self._createTempDir('testFiles') options = self.getOptions(tempDir) counterPath = os.path.join(tempDir, 'counter') resetCounters(counterPath) value, maxValue = getCounters(counterPath) assert (value, maxValue) == (0, 0) root = Job() for _ in range(self.cpuCount): root.addFollowOn( Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=coresPerJob, memory='1M', disk='1Mi')) Job.Runner.startToil(root, options) _, maxValue = getCounters(counterPath) self.assertEqual(maxValue, old_div(self.cpuCount, coresPerJob))
def makeJob(string): promises = [] job = Job.wrapFn(fn2Test, promises, string, None if outPath is None else os.path.join(outPath, string), cores=0.1, memory="0.5G", disk="0.1G") jobsToPromisesMap[job] = promises return job
def makeJobGraph(nodeNumber, childEdges, followOnEdges, outFile): """ Converts a DAG into a job graph. childEdges and followOnEdges are the lists of child and followOn edges. """ jobs = map(lambda i: Job.wrapFn(f, str(i) + " ", outFile), xrange(nodeNumber)) for fNode, tNode in childEdges: jobs[fNode].addChild(jobs[tNode]) for fNode, tNode in followOnEdges: jobs[fNode].addFollowOn(jobs[tNode]) return jobs[0]
def testPromisedRequirementStatic(self): """ Asserts that promised core resources are allocated properly using a static DAG """ for coresPerJob in self.allocatedCores: tempDir = self._createTempDir('testFiles') counterPath = self.getCounterPath(tempDir) root = Job() one = Job.wrapFn(getOne, cores=0.1, memory='32M', disk='1M') thirtyTwoMb = Job.wrapFn(getThirtyTwoMb, cores=0.1, memory='32M', disk='1M') root.addChild(one) root.addChild(thirtyTwoMb) for _ in range(self.cpuCount): root.addFollowOn(Job.wrapFn(batchSystemTest.measureConcurrency, counterPath, cores=PromisedRequirement(lambda x: x * coresPerJob, one.rv()), memory=PromisedRequirement(thirtyTwoMb.rv()), disk='1M')) Job.Runner.startToil(root, self.getOptions(tempDir)) _, maxValue = batchSystemTest.getCounters(counterPath) self.assertEqual(maxValue, self.cpuCount / coresPerJob)
def testStatic2(self): """ Create a DAG of jobs non-dynamically and run it. DAG is: A -> F \------- B -> D \ \ \ ------- C -> E Follow on is marked by -> """ outFile = getTempFile(rootDir=self._createTempDir()) try: # Create the jobs A = Job.wrapFn(fn1Test, "A", outFile) B = Job.wrapFn(fn1Test, A.rv(), outFile) C = Job.wrapFn(fn1Test, B.rv(), outFile) D = Job.wrapFn(fn1Test, C.rv(), outFile) # Connect them into a workflow A.addChild(B) A.addFollowOn(C) C.addChild(D) # Create the runner for the workflow. options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.logLevel = "INFO" options.retryCount = 100 options.badWorker = 0.5 options.badWorkerFailInterval = 0.01 # Run the workflow, the return value being the number of failed jobs Job.Runner.startToil(A, options) # Check output self.assertEquals(open(outFile, 'r').readline(), "ABCDE") finally: os.remove(outFile)
def makeJobGraph(nodeNumber, childEdges, followOnEdges, outFile): """ Converts a DAG into a job graph. childEdges and followOnEdges are the lists of child and followOn edges. """ jobs = map(lambda i: Job.wrapFn(f, str(i) + " ", outFile), xrange(nodeNumber)) for fNode, tNode in childEdges: jobs[fNode].addChild(jobs[tNode]) for fNode, tNode in followOnEdges: jobs[fNode].addFollowOn(jobs[tNode]) return jobs[0]
def testStatic2(self): """ Create a DAG of jobs non-dynamically and run it. DAG is: A -> F \------- B -> D \ \ \ ------- C -> E Follow on is marked by -> """ outFile = getTempFile(rootDir=self._createTempDir()) try: # Create the jobs A = Job.wrapFn(fn1Test, "A", outFile) B = Job.wrapFn(fn1Test, A.rv(), outFile) C = Job.wrapFn(fn1Test, B.rv(), outFile) D = Job.wrapFn(fn1Test, C.rv(), outFile) # Connect them into a workflow A.addChild(B) A.addFollowOn(C) C.addChild(D) # Create the runner for the workflow. options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.logLevel = "INFO" options.retryCount = 100 options.badWorker = 0.5 options.badWorkerFailInterval = 0.01 # Run the workflow, the return value being the number of failed jobs Job.Runner.startToil(A, options) # Check output self.assertEquals(open(outFile, 'r').readline(), "ABCDE") finally: os.remove(outFile)
def hello_world( ): # noinspection PyUnresolvedReferences from toil.job import Job from subprocess import check_output def hello( name ): return check_output( [ 'docker', 'run', '-e', 'FOO=' + name, 'ubuntu', 'bash', '-c', 'echo -n Hello, $FOO!' ] ) if __name__ == '__main__': options = Job.Runner.getDefaultArgumentParser( ).parse_args( ) job = Job.wrapFn( hello, "world", cores=1, memory=1e6, disk=1e6, cache=1e6 ) result = Job.Runner.startToil( job, options ) assert result == 'Hello, world!'
def testConcurrencyWithDisk(self): """ Tests that the batch system is allocating disk resources properly """ tempDir = self._createTempDir('testFiles') options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.workDir = tempDir from toil import physicalDisk availableDisk = physicalDisk('', toilWorkflowDir=options.workDir) options.batchSystem = self.batchSystemName counterPath = os.path.join(tempDir, 'counter') resetCounters(counterPath) value, maxValue = getCounters(counterPath) assert (value, maxValue) == (0, 0) root = Job() # Physically, we're asking for 50% of disk and 50% of disk + 500bytes in the two jobs. The # batchsystem should not allow the 2 child jobs to run concurrently. root.addChild( Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1, memory='1M', disk=old_div(availableDisk, 2))) root.addChild( Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1, memory='1M', disk=(old_div(availableDisk, 2)) + 500)) Job.Runner.startToil(root, options) _, maxValue = getCounters(counterPath) self.assertEqual(maxValue, 1)
def script(): import argparse from toil.job import Job from toil.common import Toil def fn(): pass if __name__ == '__main__': parser = argparse.ArgumentParser() Job.Runner.addToilOptions(parser) options = parser.parse_args() job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M') with Toil(options) as toil: toil.start(job)
def script(): import argparse from toil.job import Job from toil.common import Toil def fn(): pass if __name__ == '__main__': parser = argparse.ArgumentParser() Job.Runner.addToilOptions(parser) options = parser.parse_args() job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M') with Toil(options) as toil: toil.start(job)
def hello_world( ): # noinspection PyUnresolvedReferences from toil.job import Job from subprocess import check_output import os def hello( name ): assert os.environ[ 'TOIL_WORKDIR' ] == '/var/lib/toil' return check_output( [ 'docker', 'run', '-e', 'FOO=' + name, 'ubuntu', 'bash', '-c', 'echo -n Hello, $FOO!' ] ) if __name__ == '__main__': options = Job.Runner.getDefaultArgumentParser( ).parse_args( ) job = Job.wrapFn( hello, "world", cores=1, memory=1e6, disk=1e6 ) result = Job.Runner.startToil( job, options ) assert result == 'Hello, world!'
def testService(self): """ Tests the creation of a Job.Service. """ # Temporary file outFile = getTempFile(rootDir=self._createTempDir()) try: # Wire up the services/jobs t = Job.wrapFn(f, "1", outFile) t.addChildFn(f, t.addService(TestService("2", "3", outFile)), outFile) # Create the runner for the workflow. options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.logLevel = "INFO" # Run the workflow, the return value being the number of failed jobs Job.Runner.startToil(t, options) # Check output self.assertEquals(open(outFile, 'r').readline(), "123") finally: os.remove(outFile)
def testService(self): """ Tests the creation of a Job.Service. """ # Temporary file outFile = getTempFile(rootDir=self._createTempDir()) try: # Wire up the services/jobs t = Job.wrapFn(fn1Test, "1", outFile) t.addChildFn(fn1Test, t.addService(TestService("2", "3", outFile)), outFile) # Create the runner for the workflow. options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.logLevel = "INFO" # Run the workflow Job.Runner.startToil(t, options) # Check output self.assertEquals(open(outFile, 'r').readline(), "123") finally: os.remove(outFile)
def testJobConcurrency(self): """ Tests that the batch system is allocating core resources properly for concurrent tasks. """ for coresPerJob in self.allocatedCores: tempDir = self._createTempDir('testFiles') options = self.getOptions(tempDir) counterPath = os.path.join(tempDir, 'counter') resetCounters(counterPath) value, maxValue = getCounters(counterPath) assert (value, maxValue) == (0, 0) root = Job() for _ in range(self.cpuCount): root.addFollowOn(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=coresPerJob, memory='1M', disk='1Mi')) Job.Runner.startToil(root, options) _, maxValue = getCounters(counterPath) self.assertEqual(maxValue, old_div(self.cpuCount, coresPerJob))
def test_omp_threads(self): """ Test if the OMP_NUM_THREADS env var is set correctly based on jobs.cores. """ test_cases = { # mapping of the number of cores to the OMP_NUM_THREADS value 0.1: "1", 1: "1", 2: "2" } temp_dir = self._createTempDir() options = self.getOptions(temp_dir) for cores, expected_omp_threads in test_cases.items(): if os.environ.get('OMP_NUM_THREADS'): expected_omp_threads = os.environ.get('OMP_NUM_THREADS') logger.info(f"OMP_NUM_THREADS is set. Using OMP_NUM_THREADS={expected_omp_threads} instead.") with Toil(options) as toil: output = toil.start(Job.wrapFn(get_omp_threads, memory='1Mi', cores=cores, disk='1Mi')) self.assertEqual(output, expected_omp_threads)
def testStatic(self): """ Create a DAG of jobs non-dynamically and run it. DAG is: A -> F \------- B -> D \ \ \ ------- C -> E Follow on is marked by -> """ #Temporary file outFile = getTempFile(rootDir=os.getcwd()) #Create the jobs A = Job.wrapFn(f, "A", outFile) B = Job.wrapFn(f, A.rv(0), outFile) C = Job.wrapFn(f, B.rv(0), outFile) D = Job.wrapFn(f, C.rv(0), outFile) E = Job.wrapFn(f, D.rv(0), outFile) F = Job.wrapFn(f, E.rv(0), outFile) #Connect them into a workflow A.addChild(B) A.addChild(C) B.addChild(C) B.addFollowOn(E) C.addFollowOn(D) A.addFollowOn(F) #Create the runner for the workflow. options = Job.Runner.getDefaultOptions() options.logLevel = "INFO" #Run the workflow, the return value being the number of failed jobs self.assertEquals(Job.Runner.startToil(A, options), 0) Job.Runner.cleanup(options) #This removes the jobStore #Check output self.assertEquals(open(outFile, 'r').readline(), "ABCDEF") #Cleanup os.remove(outFile)
def testStatic(self): """ Create a DAG of jobs non-dynamically and run it. DAG is: A -> F \------- B -> D \ \ \ ------- C -> E Follow on is marked by -> """ #Temporary file outFile = getTempFile(rootDir=os.getcwd()) #Create the jobs A = Job.wrapFn(f, "A", outFile) B = Job.wrapFn(f, A.rv(0), outFile) C = Job.wrapFn(f, B.rv(0), outFile) D = Job.wrapFn(f, C.rv(0), outFile) E = Job.wrapFn(f, D.rv(0), outFile) F = Job.wrapFn(f, E.rv(0), outFile) #Connect them into a workflow A.addChild(B) A.addChild(C) B.addChild(C) B.addFollowOn(E) C.addFollowOn(D) A.addFollowOn(F) #Create the runner for the workflow. options = Job.Runner.getDefaultOptions() options.logLevel = "INFO" #Run the workflow, the return value being the number of failed jobs self.assertEquals(Job.Runner.startToil(A, options), 0) Job.Runner.cleanup(options) #This removes the jobStore #Check output self.assertEquals(open(outFile, 'r').readline(), "ABCDEF") #Cleanup os.remove(outFile)
def testUnicodeSupport(self): options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.clean = 'always' options.logLevel = 'debug' Job.Runner.startToil(Job.wrapFn(printUnicodeCharacter), options)
def testUnicodeSupport(self): options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.clean = "always" options.logLevel = "debug" Job.Runner.startToil(Job.wrapFn(printUnicodeCharacter), options)
from toil.common import Toil from toil.job import Job def helloWorld(message, memory="1G", cores=1, disk="1G"): return "Hello, world!, here's a message: %s" % message if __name__ == "__main__": parser = Job.Runner.getDefaultArgumentParser() options = parser.parse_args() options.clean = "always" with Toil(options) as toil: output = toil.start(Job.wrapFn(helloWorld, "You did it!")) print(output)
def testNestedResourcesDoNotBlock(self): """ Resources are requested in the order Memory > Cpu > Disk. Test that inavailability of cpus for one job that is scheduled does not block another job that can run. """ tempDir = self._createTempDir('testFiles') options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.workDir = tempDir options.maxCores = 4 from toil import physicalMemory availableMemory = physicalMemory() options.batchSystem = self.batchSystemName outFile = os.path.join(tempDir, 'counter') open(outFile, 'w').close() root = Job() blocker = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=30, writeVal='b', cores=2, memory='1M', disk='1M') firstJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='fJ', cores=1, memory='1M', disk='1M') secondJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=10, writeVal='sJ', cores=1, memory='1M', disk='1M') # Should block off 50% of memory while waiting for it's 3 cores firstJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=0, writeVal='fJC', cores=3, memory=int(old_div(availableMemory,2)), disk='1M') # These two shouldn't be able to run before B because there should be only # (50% of memory - 1M) available (firstJobChild should be blocking 50%) secondJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='sJC', cores=2, memory=int(old_div(availableMemory,1.5)), disk='1M') secondJobGrandChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='sJGC', cores=2, memory=int(old_div(availableMemory,1.5)), disk='1M') root.addChild(blocker) root.addChild(firstJob) root.addChild(secondJob) firstJob.addChild(firstJobChild) secondJob.addChild(secondJobChild) secondJobChild.addChild(secondJobGrandChild) """ The tree is: root / | \ b fJ sJ | | fJC sJC | sJGC But the order of execution should be root > b , fJ, sJ > sJC > sJGC > fJC since fJC cannot run till bl finishes but sJC and sJGC can(fJC blocked by disk). If the resource acquisition is written properly, then fJC which is scheduled before sJC and sJGC should not block them, and should only run after they finish. """ Job.Runner.startToil(root, options) with open(outFile) as oFH: outString = oFH.read() # The ordering of b, fJ and sJ is non-deterministic since they are scheduled at the same # time. We look for all possible permutations. possibleStarts = tuple([''.join(x) for x in itertools.permutations(['b', 'fJ', 'sJ'])]) assert outString.startswith(possibleStarts) assert outString.endswith('sJCsJGCfJC')
def makeJob(string): promises = [] job = Job.wrapFn(f, string, outFile, promises) jobsToPromisesMap[job] = promises return job
def makeJob(string): promises = [] job = Job.wrapFn(fn2Test, promises, string, None if outPath is None else os.path.join(outPath, string)) jobsToPromisesMap[job] = promises return job
from toil.common import Toil from toil.job import Job def helloWorld(message, memory="2G", cores=2, disk="3G"): return "Hello, world!, here's a message: %s" % message if __name__ == "__main__": options = Job.Runner.getDefaultOptions("./toilWorkflowRun") options.logLevel = "OFF" options.clean = "always" hello_job = Job.wrapFn(helloWorld, "Woot") with Toil(options) as toil: print(toil.start(hello_job)) #Prints Hello, world!, ...
def testNestedResourcesDoNotBlock(self): """ Resources are requested in the order Memory > Cpu > Disk. Test that inavailability of cpus for one job that is scheduled does not block another job that can run. """ tempDir = self._createTempDir('testFiles') options = Job.Runner.getDefaultOptions(self._getTestJobStorePath()) options.workDir = tempDir options.maxCores = 4 from toil import physicalMemory availableMemory = physicalMemory() options.batchSystem = self.batchSystemName outFile = os.path.join(tempDir, 'counter') open(outFile, 'w').close() root = Job() blocker = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=30, writeVal='b', cores=2, memory='1M', disk='1M') firstJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='fJ', cores=1, memory='1M', disk='1M') secondJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=10, writeVal='sJ', cores=1, memory='1M', disk='1M') # Should block off 50% of memory while waiting for it's 3 cores firstJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=0, writeVal='fJC', cores=3, memory=int(availableMemory/2), disk='1M') # These two shouldn't be able to run before B because there should be only # (50% of memory - 1M) available (firstJobChild should be blocking 50%) secondJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='sJC', cores=2, memory=int(availableMemory/1.5), disk='1M') secondJobGrandChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='sJGC', cores=2, memory=int(availableMemory/1.5), disk='1M') root.addChild(blocker) root.addChild(firstJob) root.addChild(secondJob) firstJob.addChild(firstJobChild) secondJob.addChild(secondJobChild) secondJobChild.addChild(secondJobGrandChild) """ The tree is: root / | \ b fJ sJ | | fJC sJC | sJGC But the order of execution should be root > b , fJ, sJ > sJC > sJGC > fJC since fJC cannot run till bl finishes but sJC and sJGC can(fJC blocked by disk). If the resource acquisition is written properly, then fJC which is scheduled before sJC and sJGC should not block them, and should only run after they finish. """ Job.Runner.startToil(root, options) with open(outFile) as oFH: outString = oFH.read() # The ordering of b, fJ and sJ is non-deterministic since they are scheduled at the same # time. We look for all possible permutations. possibleStarts = tuple([''.join(x) for x in itertools.permutations(['b', 'fJ', 'sJ'])]) assert outString.startswith(possibleStarts) assert outString.endswith('sJCsJGCfJC')