def runMain(options):
"""
make sure the output file is deleted every time main is run
"""
main(options)
yield
if os.path.exists(options.outputFile):
os.remove(options.outputFile)
def runMain(options):
"""
make sure the output file is deleted every time main is run
"""
main(options)
yield
try:
os.remove('sortedFile.txt')
except OSError as e:
if e.errno == errno.ENOENT:
pass
else:
raise
def _toilSort(self, jobStoreLocator, batchSystem,
lines=defaultLines, N=defaultN, testNo=1, lineLen=defaultLineLen,
retryCount=2, badWorker=0.5, downCheckpoints=False, disableCaching=False):
"""
Generate a file consisting of the given number of random lines, each line of the given
length. Sort the file with Toil by splitting the file recursively until each part is less
than the given number of bytes, sorting each part and merging them back together. Then
verify the result.
:param jobStoreLocator: The location of the job store.
:param batchSystem: the name of the batch system
:param lines: the number of random lines to generate
:param N: the size in bytes of each split
:param testNo: the number of repeats of this test
:param lineLen: the length of each random line in the file
"""
for test in xrange(testNo):
try:
# Specify options
options = Job.Runner.getDefaultOptions(jobStoreLocator)
options.logLevel = getLogLevelString()
options.retryCount = retryCount
options.batchSystem = batchSystem
options.clean = "never"
options.badWorker = badWorker
options.badWorkerFailInterval = 0.05
options.disableCaching = disableCaching # FIXME maybe this line should be deleted
options.downCheckpoints = downCheckpoints
options.N = N
# Make the file to sort
tempSortFile = os.path.join(self.tempDir, "fileToSort.txt")
makeFileToSort(tempSortFile, lines=lines, lineLen=lineLen)
options.fileToSort = tempSortFile
# First make our own sorted version
with open(tempSortFile, 'r') as fileHandle:
l = fileHandle.readlines()
l.sort()
# Check we get an exception if we try to restart a workflow that doesn't exist
options.restart = True
with self.assertRaises(NoSuchJobStoreException):
main(options)
options.restart = False
# Now actually run the workflow
try:
main(options)
i = 0
except FailedJobsException as e:
i = e.numberOfFailedJobs
# Check we get an exception if we try to run without restart on an existing store
with self.assertRaises(JobStoreExistsException):
main(options)
options.restart = True
# This loop tests the restart behavior
totalTrys = 1
while i != 0:
options.useExistingOptions = random.random() > 0.5
try:
main(options)
i = 0
except FailedJobsException as e:
i = e.numberOfFailedJobs
if totalTrys > 32: # p(fail after this many restarts) = 0.5**32
self.fail('Exceeded a reasonable number of restarts')
totalTrys += 1
# Now check that if you try to restart from here it will raise an exception
# indicating that there are no jobs remaining in the workflow.
with self.assertRaises(JobException):
main(options)
# Now check the file is properly sorted..
with open(tempSortFile, 'r') as fileHandle:
l2 = fileHandle.readlines()
self.assertEquals(l, l2)
finally:
subprocess.check_call([resolveEntryPoint('toil'), 'clean', jobStoreLocator])
def _toilSort(self,
jobStoreLocator,
batchSystem,
lines=defaultLines,
N=defaultN,
testNo=1,
lineLen=defaultLineLen,
retryCount=2,
badWorker=0.5,
downCheckpoints=False,
disableCaching=False):
"""
Generate a file consisting of the given number of random lines, each line of the given
length. Sort the file with Toil by splitting the file recursively until each part is less
than the given number of bytes, sorting each part and merging them back together. Then
verify the result.
:param jobStoreLocator: The location of the job store.
:param batchSystem: the name of the batch system
:param lines: the number of random lines to generate
:param N: the size in bytes of each split
:param testNo: the number of repeats of this test
:param lineLen: the length of each random line in the file
"""
for test in xrange(testNo):
try:
# Specify options
options = Job.Runner.getDefaultOptions(jobStoreLocator)
options.logLevel = getLogLevelString()
options.retryCount = retryCount
options.batchSystem = batchSystem
options.clean = "never"
options.badWorker = badWorker
options.badWorkerFailInterval = 0.05
options.disableCaching = disableCaching # FIXME maybe this line should be deleted
options.downCheckpoints = downCheckpoints
options.N = N
# Make the file to sort
tempSortFile = os.path.join(self.tempDir, "fileToSort.txt")
makeFileToSort(tempSortFile, lines=lines, lineLen=lineLen)
options.fileToSort = tempSortFile
# First make our own sorted version
with open(tempSortFile, 'r') as fileHandle:
l = fileHandle.readlines()
l.sort()
# Check we get an exception if we try to restart a workflow that doesn't exist
options.restart = True
with self.assertRaises(NoSuchJobStoreException):
main(options)
options.restart = False
# Now actually run the workflow
try:
main(options)
i = 0
except FailedJobsException as e:
i = e.numberOfFailedJobs
# Check we get an exception if we try to run without restart on an existing store
with self.assertRaises(JobStoreExistsException):
main(options)
options.restart = True
# This loop tests the restart behavior
totalTrys = 1
while i != 0:
options.useExistingOptions = random.random() > 0.5
try:
main(options)
i = 0
except FailedJobsException as e:
i = e.numberOfFailedJobs
if totalTrys > 32: # p(fail after this many restarts) = 0.5**32
self.fail(
'Exceeded a reasonable number of restarts')
totalTrys += 1
# Now check that if you try to restart from here it will raise an exception
# indicating that there are no jobs remaining in the workflow.
with self.assertRaises(JobException):
main(options)
# Now check the file is properly sorted..
with open(tempSortFile, 'r') as fileHandle:
l2 = fileHandle.readlines()
self.assertEquals(l, l2)
finally:
subprocess.check_call(
[resolveEntryPoint('toil'), 'clean', jobStoreLocator])