def testConcurrencyWithDisk(self):
"""
Tests that the batch system is allocating disk resources properly
"""
tempDir = self._createTempDir('testFiles')
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.workDir = tempDir
from toil import physicalDisk
availableDisk = physicalDisk('', toilWorkflowDir=options.workDir)
options.batchSystem = self.batchSystemName
counterPath = os.path.join(tempDir, 'counter')
resetCounters(counterPath)
value, maxValue = getCounters(counterPath)
assert (value, maxValue) == (0, 0)
root = Job()
# Physically, we're asking for 50% of disk and 50% of disk + 500bytes in the two jobs. The
# batchsystem should not allow the 2 child jobs to run concurrently.
root.addChild(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1,
memory='1M', disk=availableDisk/2))
root.addChild(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime, cores=1,
memory='1M', disk=(availableDisk / 2) + 500))
Job.Runner.startToil(root, options)
_, maxValue = getCounters(counterPath)
self.assertEqual(maxValue, 1)
def testEncapsulation(self):
"""
Tests the Job.encapsulation method, which uses the EncapsulationJob
class.
"""
#Temporary file
outFile = getTempFile(rootDir=os.getcwd())
#Make a job graph
a = T.wrapFn(f, "A", outFile)
b = a.addChildFn(f, a.rv(), outFile)
c = a.addFollowOnFn(f, b.rv(), outFile)
#Encapsulate it
a = a.encapsulate()
#Now add children/follow to the encapsulated graph
d = T.wrapFn(f, c.rv(), outFile)
e = T.wrapFn(f, d.rv(), outFile)
a.addChild(d)
a.addFollowOn(e)
#Create the runner for the workflow.
options = T.Runner.getDefaultOptions()
options.logLevel = "INFO"
#Run the workflow, the return value being the number of failed jobs
self.assertEquals(T.Runner.startToil(a, options), 0)
T.Runner.cleanup(options) #This removes the jobStore
#Check output
self.assertEquals(open(outFile, 'r').readline(), "ABCDE")
#Cleanup
os.remove(outFile)
def testEncapsulation(self):
"""
Tests the Job.encapsulation method, which uses the EncapsulationJob
class.
"""
# Temporary file
outFile = getTempFile(rootDir=self._createTempDir())
try:
# Encapsulate a job graph
a = T.wrapJobFn(encapsulatedJobFn, "A", outFile)
a = a.encapsulate()
# Now add children/follow to the encapsulated graph
d = T.wrapFn(f, a.rv(), outFile)
e = T.wrapFn(f, d.rv(), outFile)
a.addChild(d)
a.addFollowOn(e)
# Create the runner for the workflow.
options = T.Runner.getDefaultOptions(self._getTestJobStorePath())
options.logLevel = "INFO"
# Run the workflow, the return value being the number of failed jobs
T.Runner.startToil(a, options)
# Check output
self.assertEquals(open(outFile, 'r').readline(), "ABCDE")
finally:
os.remove(outFile)
def testAddChildEncapsulate(self):
"""
Make sure that the encapsulate child does not have two pareents
with unique roots.
"""
# Temporary file
a = T.wrapFn(noOp)
b = T.wrapFn(noOp)
a.addChild(b).encapsulate()
self.assertEquals(len(a.getRootJobs()), 1)
def testAddChildEncapsulate(self):
"""
Make sure that the encapsulate child does not have two parents
with unique roots.
"""
# Temporary file
a = T.wrapFn(noOp)
b = T.wrapFn(noOp)
a.addChild(b).encapsulate()
self.assertEqual(len(a.getRootJobs()), 1)
def testConcurrencyWithDisk(self):
"""
Tests that the batch system is allocating disk resources properly
"""
tempDir = self._createTempDir('testFiles')
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.workDir = tempDir
from toil import physicalDisk
availableDisk = physicalDisk(options.workDir)
logger.info('Testing disk concurrency limits with %s disk space',
availableDisk)
# More disk might become available by the time Toil starts, so we limit it here
options.maxDisk = availableDisk
options.batchSystem = self.batchSystemName
counterPath = os.path.join(tempDir, 'counter')
resetCounters(counterPath)
value, maxValue = getCounters(counterPath)
assert (value, maxValue) == (0, 0)
half_disk = availableDisk // 2
more_than_half_disk = half_disk + 500
logger.info('Dividing into parts of %s and %s', half_disk,
more_than_half_disk)
root = Job()
# Physically, we're asking for 50% of disk and 50% of disk + 500bytes in the two jobs. The
# batchsystem should not allow the 2 child jobs to run concurrently.
root.addChild(
Job.wrapFn(measureConcurrency,
counterPath,
self.sleepTime,
cores=1,
memory='1M',
disk=half_disk))
root.addChild(
Job.wrapFn(measureConcurrency,
counterPath,
self.sleepTime,
cores=1,
memory='1M',
disk=more_than_half_disk))
Job.Runner.startToil(root, options)
_, maxValue = getCounters(counterPath)
logger.info('After run: %s disk space', physicalDisk(options.workDir))
self.assertEqual(maxValue, 1)
def makeJob(string):
promises = []
job = Job.wrapFn(
fn2Test, promises, string,
None if outPath is None else os.path.join(outPath, string))
jobsToPromisesMap[job] = promises
return job
def testJobConcurrency(self):
"""
Tests that the batch system is allocating core resources properly for concurrent tasks.
"""
for cores_per_job in self.allocated_cores:
temp_dir = self._createTempDir('testFiles')
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.workDir = temp_dir
options.maxCores = self.cpu_count
options.batchSystem = self.batchSystemName
counter_path = os.path.join(temp_dir, 'counter')
resetCounters(counter_path)
value, max_value = getCounters(counter_path)
assert (value, max_value) == (0, 0)
root = Job()
for _ in range(self.cpu_count):
root.addFollowOn(Job.wrapFn(measureConcurrency, counter_path, self.sleep_time,
cores=cores_per_job, memory='1M', disk='1Mi'))
Job.Runner.startToil(root, options)
_, max_value = getCounters(counter_path)
self.assertEqual(max_value, self.cpu_count / cores_per_job)
def testJobConcurrency(self):
"""
Tests that the batch system is allocating core resources properly for concurrent tasks.
"""
for coresPerJob in self.allocatedCores:
tempDir = self._createTempDir('testFiles')
options = self.getOptions(tempDir)
counterPath = os.path.join(tempDir, 'counter')
resetCounters(counterPath)
value, maxValue = getCounters(counterPath)
assert (value, maxValue) == (0, 0)
root = Job()
for _ in range(self.cpuCount):
root.addFollowOn(
Job.wrapFn(measureConcurrency,
counterPath,
self.sleepTime,
cores=coresPerJob,
memory='1M',
disk='1Mi'))
Job.Runner.startToil(root, options)
_, maxValue = getCounters(counterPath)
self.assertEqual(maxValue, old_div(self.cpuCount, coresPerJob))
def makeJob(string):
promises = []
job = Job.wrapFn(fn2Test, promises, string,
None if outPath is None else os.path.join(outPath, string),
cores=0.1, memory="0.5G", disk="0.1G")
jobsToPromisesMap[job] = promises
return job
def makeJobGraph(nodeNumber, childEdges, followOnEdges, outFile):
"""
Converts a DAG into a job graph. childEdges and followOnEdges are
the lists of child and followOn edges.
"""
jobs = map(lambda i: Job.wrapFn(f, str(i) + " ", outFile), xrange(nodeNumber))
for fNode, tNode in childEdges:
jobs[fNode].addChild(jobs[tNode])
for fNode, tNode in followOnEdges:
jobs[fNode].addFollowOn(jobs[tNode])
return jobs[0]
def testPromisedRequirementStatic(self):
"""
Asserts that promised core resources are allocated properly using a static DAG
"""
for coresPerJob in self.allocatedCores:
tempDir = self._createTempDir('testFiles')
counterPath = self.getCounterPath(tempDir)
root = Job()
one = Job.wrapFn(getOne, cores=0.1, memory='32M', disk='1M')
thirtyTwoMb = Job.wrapFn(getThirtyTwoMb, cores=0.1, memory='32M', disk='1M')
root.addChild(one)
root.addChild(thirtyTwoMb)
for _ in range(self.cpuCount):
root.addFollowOn(Job.wrapFn(batchSystemTest.measureConcurrency, counterPath,
cores=PromisedRequirement(lambda x: x * coresPerJob, one.rv()),
memory=PromisedRequirement(thirtyTwoMb.rv()),
disk='1M'))
Job.Runner.startToil(root, self.getOptions(tempDir))
_, maxValue = batchSystemTest.getCounters(counterPath)
self.assertEqual(maxValue, self.cpuCount / coresPerJob)
def testStatic2(self):
"""
Create a DAG of jobs non-dynamically and run it. DAG is:
A -> F
\-------
B -> D \
\ \
------- C -> E
Follow on is marked by ->
"""
outFile = getTempFile(rootDir=self._createTempDir())
try:
# Create the jobs
A = Job.wrapFn(fn1Test, "A", outFile)
B = Job.wrapFn(fn1Test, A.rv(), outFile)
C = Job.wrapFn(fn1Test, B.rv(), outFile)
D = Job.wrapFn(fn1Test, C.rv(), outFile)
# Connect them into a workflow
A.addChild(B)
A.addFollowOn(C)
C.addChild(D)
# Create the runner for the workflow.
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.logLevel = "INFO"
options.retryCount = 100
options.badWorker = 0.5
options.badWorkerFailInterval = 0.01
# Run the workflow, the return value being the number of failed jobs
Job.Runner.startToil(A, options)
# Check output
self.assertEquals(open(outFile, 'r').readline(), "ABCDE")
finally:
os.remove(outFile)
def makeJobGraph(nodeNumber, childEdges, followOnEdges, outFile):
"""
Converts a DAG into a job graph. childEdges and followOnEdges are
the lists of child and followOn edges.
"""
jobs = map(lambda i: Job.wrapFn(f,
str(i) + " ", outFile),
xrange(nodeNumber))
for fNode, tNode in childEdges:
jobs[fNode].addChild(jobs[tNode])
for fNode, tNode in followOnEdges:
jobs[fNode].addFollowOn(jobs[tNode])
return jobs[0]
def testStatic2(self):
"""
Create a DAG of jobs non-dynamically and run it. DAG is:
A -> F
\-------
B -> D \
\ \
------- C -> E
Follow on is marked by ->
"""
outFile = getTempFile(rootDir=self._createTempDir())
try:
# Create the jobs
A = Job.wrapFn(fn1Test, "A", outFile)
B = Job.wrapFn(fn1Test, A.rv(), outFile)
C = Job.wrapFn(fn1Test, B.rv(), outFile)
D = Job.wrapFn(fn1Test, C.rv(), outFile)
# Connect them into a workflow
A.addChild(B)
A.addFollowOn(C)
C.addChild(D)
# Create the runner for the workflow.
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.logLevel = "INFO"
options.retryCount = 100
options.badWorker = 0.5
options.badWorkerFailInterval = 0.01
# Run the workflow, the return value being the number of failed jobs
Job.Runner.startToil(A, options)
# Check output
self.assertEquals(open(outFile, 'r').readline(), "ABCDE")
finally:
os.remove(outFile)
def hello_world( ):
# noinspection PyUnresolvedReferences
from toil.job import Job
from subprocess import check_output
def hello( name ):
return check_output( [ 'docker', 'run', '-e', 'FOO=' + name, 'ubuntu',
'bash', '-c', 'echo -n Hello, $FOO!' ] )
if __name__ == '__main__':
options = Job.Runner.getDefaultArgumentParser( ).parse_args( )
job = Job.wrapFn( hello, "world", cores=1, memory=1e6, disk=1e6, cache=1e6 )
result = Job.Runner.startToil( job, options )
assert result == 'Hello, world!'
def testConcurrencyWithDisk(self):
"""
Tests that the batch system is allocating disk resources properly
"""
tempDir = self._createTempDir('testFiles')
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.workDir = tempDir
from toil import physicalDisk
availableDisk = physicalDisk('', toilWorkflowDir=options.workDir)
options.batchSystem = self.batchSystemName
counterPath = os.path.join(tempDir, 'counter')
resetCounters(counterPath)
value, maxValue = getCounters(counterPath)
assert (value, maxValue) == (0, 0)
root = Job()
# Physically, we're asking for 50% of disk and 50% of disk + 500bytes in the two jobs. The
# batchsystem should not allow the 2 child jobs to run concurrently.
root.addChild(
Job.wrapFn(measureConcurrency,
counterPath,
self.sleepTime,
cores=1,
memory='1M',
disk=old_div(availableDisk, 2)))
root.addChild(
Job.wrapFn(measureConcurrency,
counterPath,
self.sleepTime,
cores=1,
memory='1M',
disk=(old_div(availableDisk, 2)) + 500))
Job.Runner.startToil(root, options)
_, maxValue = getCounters(counterPath)
self.assertEqual(maxValue, 1)
def script():
import argparse
from toil.job import Job
from toil.common import Toil
def fn():
pass
if __name__ == '__main__':
parser = argparse.ArgumentParser()
Job.Runner.addToilOptions(parser)
options = parser.parse_args()
job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M')
with Toil(options) as toil:
toil.start(job)
def script():
import argparse
from toil.job import Job
from toil.common import Toil
def fn():
pass
if __name__ == '__main__':
parser = argparse.ArgumentParser()
Job.Runner.addToilOptions(parser)
options = parser.parse_args()
job = Job.wrapFn(fn, memory='10M', cores=0.1, disk='10M')
with Toil(options) as toil:
toil.start(job)
def hello_world( ):
# noinspection PyUnresolvedReferences
from toil.job import Job
from subprocess import check_output
import os
def hello( name ):
assert os.environ[ 'TOIL_WORKDIR' ] == '/var/lib/toil'
return check_output( [ 'docker', 'run', '-e', 'FOO=' + name, 'ubuntu',
'bash', '-c', 'echo -n Hello, $FOO!' ] )
if __name__ == '__main__':
options = Job.Runner.getDefaultArgumentParser( ).parse_args( )
job = Job.wrapFn( hello, "world", cores=1, memory=1e6, disk=1e6 )
result = Job.Runner.startToil( job, options )
assert result == 'Hello, world!'
def testService(self):
"""
Tests the creation of a Job.Service.
"""
# Temporary file
outFile = getTempFile(rootDir=self._createTempDir())
try:
# Wire up the services/jobs
t = Job.wrapFn(f, "1", outFile)
t.addChildFn(f, t.addService(TestService("2", "3", outFile)), outFile)
# Create the runner for the workflow.
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.logLevel = "INFO"
# Run the workflow, the return value being the number of failed jobs
Job.Runner.startToil(t, options)
# Check output
self.assertEquals(open(outFile, 'r').readline(), "123")
finally:
os.remove(outFile)
def testService(self):
"""
Tests the creation of a Job.Service.
"""
# Temporary file
outFile = getTempFile(rootDir=self._createTempDir())
try:
# Wire up the services/jobs
t = Job.wrapFn(fn1Test, "1", outFile)
t.addChildFn(fn1Test, t.addService(TestService("2", "3", outFile)),
outFile)
# Create the runner for the workflow.
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.logLevel = "INFO"
# Run the workflow
Job.Runner.startToil(t, options)
# Check output
self.assertEquals(open(outFile, 'r').readline(), "123")
finally:
os.remove(outFile)
def testJobConcurrency(self):
"""
Tests that the batch system is allocating core resources properly for concurrent tasks.
"""
for coresPerJob in self.allocatedCores:
tempDir = self._createTempDir('testFiles')
options = self.getOptions(tempDir)
counterPath = os.path.join(tempDir, 'counter')
resetCounters(counterPath)
value, maxValue = getCounters(counterPath)
assert (value, maxValue) == (0, 0)
root = Job()
for _ in range(self.cpuCount):
root.addFollowOn(Job.wrapFn(measureConcurrency, counterPath, self.sleepTime,
cores=coresPerJob, memory='1M', disk='1Mi'))
Job.Runner.startToil(root, options)
_, maxValue = getCounters(counterPath)
self.assertEqual(maxValue, old_div(self.cpuCount, coresPerJob))
def test_omp_threads(self):
"""
Test if the OMP_NUM_THREADS env var is set correctly based on jobs.cores.
"""
test_cases = {
# mapping of the number of cores to the OMP_NUM_THREADS value
0.1: "1",
1: "1",
2: "2"
}
temp_dir = self._createTempDir()
options = self.getOptions(temp_dir)
for cores, expected_omp_threads in test_cases.items():
if os.environ.get('OMP_NUM_THREADS'):
expected_omp_threads = os.environ.get('OMP_NUM_THREADS')
logger.info(f"OMP_NUM_THREADS is set. Using OMP_NUM_THREADS={expected_omp_threads} instead.")
with Toil(options) as toil:
output = toil.start(Job.wrapFn(get_omp_threads, memory='1Mi', cores=cores, disk='1Mi'))
self.assertEqual(output, expected_omp_threads)
def testStatic(self):
"""
Create a DAG of jobs non-dynamically and run it. DAG is:
A -> F
\-------
B -> D \
\ \
------- C -> E
Follow on is marked by ->
"""
#Temporary file
outFile = getTempFile(rootDir=os.getcwd())
#Create the jobs
A = Job.wrapFn(f, "A", outFile)
B = Job.wrapFn(f, A.rv(0), outFile)
C = Job.wrapFn(f, B.rv(0), outFile)
D = Job.wrapFn(f, C.rv(0), outFile)
E = Job.wrapFn(f, D.rv(0), outFile)
F = Job.wrapFn(f, E.rv(0), outFile)
#Connect them into a workflow
A.addChild(B)
A.addChild(C)
B.addChild(C)
B.addFollowOn(E)
C.addFollowOn(D)
A.addFollowOn(F)
#Create the runner for the workflow.
options = Job.Runner.getDefaultOptions()
options.logLevel = "INFO"
#Run the workflow, the return value being the number of failed jobs
self.assertEquals(Job.Runner.startToil(A, options), 0)
Job.Runner.cleanup(options) #This removes the jobStore
#Check output
self.assertEquals(open(outFile, 'r').readline(), "ABCDEF")
#Cleanup
os.remove(outFile)
def testStatic(self):
"""
Create a DAG of jobs non-dynamically and run it. DAG is:
A -> F
\-------
B -> D \
\ \
------- C -> E
Follow on is marked by ->
"""
#Temporary file
outFile = getTempFile(rootDir=os.getcwd())
#Create the jobs
A = Job.wrapFn(f, "A", outFile)
B = Job.wrapFn(f, A.rv(0), outFile)
C = Job.wrapFn(f, B.rv(0), outFile)
D = Job.wrapFn(f, C.rv(0), outFile)
E = Job.wrapFn(f, D.rv(0), outFile)
F = Job.wrapFn(f, E.rv(0), outFile)
#Connect them into a workflow
A.addChild(B)
A.addChild(C)
B.addChild(C)
B.addFollowOn(E)
C.addFollowOn(D)
A.addFollowOn(F)
#Create the runner for the workflow.
options = Job.Runner.getDefaultOptions()
options.logLevel = "INFO"
#Run the workflow, the return value being the number of failed jobs
self.assertEquals(Job.Runner.startToil(A, options), 0)
Job.Runner.cleanup(options) #This removes the jobStore
#Check output
self.assertEquals(open(outFile, 'r').readline(), "ABCDEF")
#Cleanup
os.remove(outFile)
def testUnicodeSupport(self):
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.clean = 'always'
options.logLevel = 'debug'
Job.Runner.startToil(Job.wrapFn(printUnicodeCharacter), options)
def testUnicodeSupport(self):
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.clean = "always"
options.logLevel = "debug"
Job.Runner.startToil(Job.wrapFn(printUnicodeCharacter), options)
from toil.common import Toil
from toil.job import Job
def helloWorld(message, memory="1G", cores=1, disk="1G"):
return "Hello, world!, here's a message: %s" % message
if __name__ == "__main__":
parser = Job.Runner.getDefaultArgumentParser()
options = parser.parse_args()
options.clean = "always"
with Toil(options) as toil:
output = toil.start(Job.wrapFn(helloWorld, "You did it!"))
print(output)
def testNestedResourcesDoNotBlock(self):
"""
Resources are requested in the order Memory > Cpu > Disk.
Test that inavailability of cpus for one job that is scheduled does not block another job
that can run.
"""
tempDir = self._createTempDir('testFiles')
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.workDir = tempDir
options.maxCores = 4
from toil import physicalMemory
availableMemory = physicalMemory()
options.batchSystem = self.batchSystemName
outFile = os.path.join(tempDir, 'counter')
open(outFile, 'w').close()
root = Job()
blocker = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=30, writeVal='b',
cores=2, memory='1M', disk='1M')
firstJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='fJ',
cores=1, memory='1M', disk='1M')
secondJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=10,
writeVal='sJ', cores=1, memory='1M', disk='1M')
# Should block off 50% of memory while waiting for it's 3 cores
firstJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=0,
writeVal='fJC', cores=3, memory=int(old_div(availableMemory,2)), disk='1M')
# These two shouldn't be able to run before B because there should be only
# (50% of memory - 1M) available (firstJobChild should be blocking 50%)
secondJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
writeVal='sJC', cores=2, memory=int(old_div(availableMemory,1.5)),
disk='1M')
secondJobGrandChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
writeVal='sJGC', cores=2, memory=int(old_div(availableMemory,1.5)),
disk='1M')
root.addChild(blocker)
root.addChild(firstJob)
root.addChild(secondJob)
firstJob.addChild(firstJobChild)
secondJob.addChild(secondJobChild)
secondJobChild.addChild(secondJobGrandChild)
"""
The tree is:
root
/ | \
b fJ sJ
| |
fJC sJC
|
sJGC
But the order of execution should be
root > b , fJ, sJ > sJC > sJGC > fJC
since fJC cannot run till bl finishes but sJC and sJGC can(fJC blocked by disk). If the
resource acquisition is written properly, then fJC which is scheduled before sJC and sJGC
should not block them, and should only run after they finish.
"""
Job.Runner.startToil(root, options)
with open(outFile) as oFH:
outString = oFH.read()
# The ordering of b, fJ and sJ is non-deterministic since they are scheduled at the same
# time. We look for all possible permutations.
possibleStarts = tuple([''.join(x) for x in itertools.permutations(['b', 'fJ', 'sJ'])])
assert outString.startswith(possibleStarts)
assert outString.endswith('sJCsJGCfJC')
def makeJob(string):
promises = []
job = Job.wrapFn(f, string, outFile, promises)
jobsToPromisesMap[job] = promises
return job
def makeJob(string):
promises = []
job = Job.wrapFn(fn2Test, promises, string,
None if outPath is None else os.path.join(outPath, string))
jobsToPromisesMap[job] = promises
return job
from toil.common import Toil
from toil.job import Job
def helloWorld(message, memory="2G", cores=2, disk="3G"):
return "Hello, world!, here's a message: %s" % message
if __name__ == "__main__":
options = Job.Runner.getDefaultOptions("./toilWorkflowRun")
options.logLevel = "OFF"
options.clean = "always"
hello_job = Job.wrapFn(helloWorld, "Woot")
with Toil(options) as toil:
print(toil.start(hello_job)) #Prints Hello, world!, ...
def testNestedResourcesDoNotBlock(self):
"""
Resources are requested in the order Memory > Cpu > Disk.
Test that inavailability of cpus for one job that is scheduled does not block another job
that can run.
"""
tempDir = self._createTempDir('testFiles')
options = Job.Runner.getDefaultOptions(self._getTestJobStorePath())
options.workDir = tempDir
options.maxCores = 4
from toil import physicalMemory
availableMemory = physicalMemory()
options.batchSystem = self.batchSystemName
outFile = os.path.join(tempDir, 'counter')
open(outFile, 'w').close()
root = Job()
blocker = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=30, writeVal='b',
cores=2, memory='1M', disk='1M')
firstJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5, writeVal='fJ',
cores=1, memory='1M', disk='1M')
secondJob = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=10,
writeVal='sJ', cores=1, memory='1M', disk='1M')
# Should block off 50% of memory while waiting for it's 3 cores
firstJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=0,
writeVal='fJC', cores=3, memory=int(availableMemory/2), disk='1M')
# These two shouldn't be able to run before B because there should be only
# (50% of memory - 1M) available (firstJobChild should be blocking 50%)
secondJobChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
writeVal='sJC', cores=2, memory=int(availableMemory/1.5),
disk='1M')
secondJobGrandChild = Job.wrapFn(_resourceBlockTestAuxFn, outFile=outFile, sleepTime=5,
writeVal='sJGC', cores=2, memory=int(availableMemory/1.5),
disk='1M')
root.addChild(blocker)
root.addChild(firstJob)
root.addChild(secondJob)
firstJob.addChild(firstJobChild)
secondJob.addChild(secondJobChild)
secondJobChild.addChild(secondJobGrandChild)
"""
The tree is:
root
/ | \
b fJ sJ
| |
fJC sJC
|
sJGC
But the order of execution should be
root > b , fJ, sJ > sJC > sJGC > fJC
since fJC cannot run till bl finishes but sJC and sJGC can(fJC blocked by disk). If the
resource acquisition is written properly, then fJC which is scheduled before sJC and sJGC
should not block them, and should only run after they finish.
"""
Job.Runner.startToil(root, options)
with open(outFile) as oFH:
outString = oFH.read()
# The ordering of b, fJ and sJ is non-deterministic since they are scheduled at the same
# time. We look for all possible permutations.
possibleStarts = tuple([''.join(x) for x in itertools.permutations(['b', 'fJ', 'sJ'])])
assert outString.startswith(possibleStarts)
assert outString.endswith('sJCsJGCfJC')
