def testScriptTree_Example2(self):
"""Tests that the global and local temp dirs of a job behave as expected.
"""
for test in xrange(self.testNo):
command = "scriptTreeTest_Wrapper2.py --jobTree %s --logLevel=INFO --retryCount=0" % self.jobTreeDir
system(command)
runJobTreeStatusAndFailIfNotComplete(self.jobTreeDir)
def testScriptTree_Example(self):
"""Uses the jobTreeTest code to test the scriptTree Target wrapper.
"""
for test in xrange(self.testNo):
command = "scriptTreeTest_Wrapper.py --jobTree %s --logLevel=INFO --retryCount=10" % self.jobTreeDir
system(command)
runJobTreeStatusAndFailIfNotComplete(self.jobTreeDir)
def execute(self):
task_id = int(os.environ['SGE_TASK_ID'])
if task_id is None:
RuntimeError("Multi-target launched without task id")
if task_id < 1 or task_id > len(self.commands):
RuntimeError("Task ID not within the array range 1 <= %i <= %i", task_id, len(self.commands))
(job, outfile) = self.commands[task_id - 1]
if outfile is None:
system(job)
else:
system("%s > %s" % (job, outfile))
def testSort(self):
for test in xrange(self.testNo):
tempDir = getTempDirectory(os.getcwd())
tempFile1 = getTempFile(rootDir=tempDir)
makeFileToSort(tempFile1)
lines1 = loadFile(tempFile1)
lines1.sort()
sort(tempFile1)
lines2 = loadFile(tempFile1)
checkEqual(lines1, lines2)
system("rm -rf %s" % tempDir)
def testGetMidPoint(self):
for test in xrange(self.testNo):
tempDir = getTempDirectory(os.getcwd())
tempFile = getTempFile(rootDir=tempDir)
makeFileToSort(tempFile)
l = open(tempFile, 'r').read()
fileSize = os.path.getsize(tempFile)
midPoint = getMidPoint(tempFile, 0, fileSize)
print "the mid point is %i of a file of %i bytes woth byte" % (midPoint, fileSize)
assert midPoint < fileSize
assert l[midPoint] == '\n'
assert midPoint >= 0
system("rm -rf %s" % tempDir)
def runJobTree(command, jobTreeDir, logLevel="DEBUG", retryCount=0, batchSystem="single_machine",
rescueJobFrequency=None):
"""A convenience function for running job tree from within a python script.
"""
if rescueJobFrequency != None:
rescueJobFrequencyString = "--rescueJobsFrequency %s" % float(rescueJobFrequency)
else:
rescueJobFrequencyString = ""
command = "jobTree --command \"%s\" --jobTree %s --logLevel %s \
--retryCount %i --batchSystem %s %s" % \
(command, jobTreeDir, logLevel, retryCount, batchSystem, rescueJobFrequencyString)
logger.info("Running command : %s" % command)
system(command)
logger.info("Ran the jobtree apparently okay")
def run(self):
while True:
command, logFile, jobID = self.inputQueue.get()
#fnull = open(os.devnull, 'w') #Pipe the output to dev/null (it is caught by the slave and will be reported if there is an error)
tempLogFile = getTempFile()
fileHandle = open(tempLogFile, 'w')
process = subprocess.Popen(command, shell=True, stdout = fileHandle, stderr = fileHandle)
sts = os.waitpid(process.pid, 0)
fileHandle.close()
#fnull.close()
if os.path.exists(tempLogFile):
system("mv %s %s" % (tempLogFile, logFile))
self.outputQueue.put((command, sts[1], jobID))
self.inputQueue.task_done()
def testCopySubRangeOfFile(self):
for test in xrange(self.testNo):
tempDir = getTempDirectory(os.getcwd())
tempFile = getTempFile(rootDir=tempDir)
outputFile = getTempFile(rootDir=tempDir)
makeFileToSort(tempFile)
fileSize = os.path.getsize(tempFile)
assert fileSize > 0
fileStart = random.choice(xrange(0, fileSize))
fileEnd = random.choice(xrange(fileStart, fileSize))
copySubRangeOfFile(tempFile, fileStart, fileEnd, outputFile)
l = open(outputFile, 'r').read()
l2 = open(tempFile, 'r').read()[fileStart:fileEnd]
checkEqual(l, l2)
system("rm -rf %s" % tempDir)
def scriptTree_SortTest(testNo, batchSystem, lines=100000, maxLineLength=10, N=1000):
"""Tests scriptTree/jobTree by sorting a file in parallel.
"""
for test in xrange(testNo):
tempDir = getTempDirectory(os.getcwd())
tempFile = getTempFile(rootDir=tempDir)
jobTreeDir = os.path.join(tempDir, "jobTree")
makeFileToSort(tempFile, lines=lines, maxLineLength=maxLineLength)
#First make our own sorted version
fileHandle = open(tempFile, 'r')
l = fileHandle.readlines()
l.sort()
fileHandle.close()
#Sort the file
command = "scriptTreeTest_Sort.py --jobTree %s --logLevel=DEBUG --fileToSort=%s --N %i --batchSystem %s --jobTime 1.0" % (jobTreeDir, tempFile, N, batchSystem)
system(command)
#Now check the file is properly sorted..
#Now get the sorted file
fileHandle = open(tempFile, 'r')
l2 = fileHandle.readlines()
fileHandle.close()
checkEqual(l, l2)
system("rm -rf %s" % tempDir)
def testJobTreeStats_SortSimple(self):
"""Tests the jobTreeStats utility using the scriptTree_sort example.
"""
for test in xrange(self.testNo):
tempDir = getTempDirectory(os.getcwd())
tempFile = getTempFile(rootDir=tempDir)
outputFile = getTempFile(rootDir=tempDir)
jobTreeDir = os.path.join(tempDir, "jobTree")
lines=100000
maxLineLength=10
N=1000
makeFileToSort(tempFile, lines, maxLineLength)
#Sort the file
command = "scriptTreeTest_Sort.py --jobTree %s --logLevel=DEBUG --fileToSort=%s --N %s --stats --jobTime 0.5" % (jobTreeDir, tempFile, N)
system(command)
#Now get the stats
system("jobTreeStats --jobTree %s --outputFile %s" % (jobTreeDir, outputFile))
#Cleanup
system("rm -rf %s" % tempDir)
def processJob(job, jobToRun, memoryAvailable, cpuAvailable, stats):
from workflow.jobTree.lib.bioio import getTempFile
from workflow.jobTree.lib.bioio import getTempDirectory
from workflow.jobTree.lib.bioio import logger
from workflow.jobTree.lib.bioio import system
from workflow.jobTree.lib.bioio import getTotalCpuTime
assert len(job.find("children").findall("child")) == 0
assert int(job.attrib["child_count"]) == int(job.attrib["black_child_count"])
command = jobToRun.attrib["command"]
#Copy the job file to be edited
tempJob = ET.Element("job")
ET.SubElement(tempJob, "children")
#Log for job
tempJob.attrib["log_level"] = job.attrib["log_level"]
#Time length of 'ideal' job before further parallelism is required
tempJob.attrib["job_time"] = job.attrib["job_time"]
#Dir to put all the temp files in.
localSlaveTempDir = getTempDirectory()
#Temp file dirs for job.
localTempDir = getTempDirectory(rootDir=localSlaveTempDir)
tempJob.attrib["local_temp_dir"] = localTempDir
depth = len(job.find("followOns").findall("followOn"))
tempJob.attrib["global_temp_dir"] = os.path.join(job.attrib["global_temp_dir"], str(depth))
if not os.path.isdir(tempJob.attrib["global_temp_dir"]): #Ensures that the global temp dirs of each level are kept separate.
os.mkdir(tempJob.attrib["global_temp_dir"])
os.chmod(tempJob.attrib["global_temp_dir"], 0777)
if os.path.isdir(os.path.join(job.attrib["global_temp_dir"], str(depth+1))):
system("rm -rf %s" % os.path.join(job.attrib["global_temp_dir"], str(depth+1)))
assert not os.path.isdir(os.path.join(job.attrib["global_temp_dir"], str(depth+2)))
#Deal with memory and cpu requirements (this pass tells the running job how much cpu and memory they have,
#according to the batch system
tempJob.attrib["available_memory"] = str(memoryAvailable)
tempJob.attrib["available_cpu"] = str(cpuAvailable)
if stats != None:
tempJob.attrib["stats"] = getTempFile(rootDir=localSlaveTempDir)
os.remove(tempJob.attrib["stats"])
#Now write the temp job file
tempFile = getTempFile(rootDir=localSlaveTempDir)
fileHandle = open(tempFile, 'w')
tree = ET.ElementTree(tempJob)
tree.write(fileHandle)
fileHandle.close()
logger.info("Copied the jobs files ready for the job")
if "JOB_FILE" not in command:
logger.critical("There is no 'JOB_FILE' string in the command to be run to take the job-file argument: %s" % command)
job.attrib["colour"] = "red" #Update the colour
else:
#First load the environment for the job.
fileHandle = open(job.attrib["environment_file"], 'r')
environment = cPickle.load(fileHandle)
fileHandle.close()
logger.info("Loaded the environment for the process")
#Run the actual command
tempLogFile = getTempFile(suffix=".log", rootDir=localSlaveTempDir)
fileHandle = open(tempLogFile, 'w')
finalCommand = command.replace("JOB_FILE", tempFile)
if stats != None:
startTime = time.time()
startClock = getTotalCpuTime()
process = subprocess.Popen(finalCommand, shell=True, stdout=fileHandle, stderr=subprocess.STDOUT, env=environment)
sts = os.waitpid(process.pid, 0)
fileHandle.close()
truncateFile(tempLogFile, int(job.attrib["max_log_file_size"]))
#Copy across the log file
system("mv %s %s" % (tempLogFile, job.attrib["log_file"]))
i = sts[1]
logger.info("Ran the job command=%s with exit status %i" % (finalCommand, i))
if i == 0:
logger.info("Passed the job, okay")
if stats != None:
jobTag = ET.SubElement(stats, "job", { "time":str(time.time() - startTime), "clock":str(getTotalCpuTime() - startClock) })
if os.path.exists(tempJob.attrib["stats"]):
jobTag.append(ET.parse(tempJob.attrib["stats"]).getroot())
tempJob = ET.parse(tempFile).getroot()
job.attrib["colour"] = "black" #Update the colour
#Update the runtime of the stack..
totalRuntime = float(job.attrib["total_time"]) #This is the estimate runtime of the jobs on the followon stack
runtime = float(jobToRun.attrib["time"])
totalRuntime -= runtime
if totalRuntime < 0.0:
totalRuntime = 0.0
#The children
children = job.find("children")
assert len(children.findall("child")) == 0 #The children
assert tempJob.find("children") != None
for child in tempJob.find("children").findall("child"):
memory, cpu, compTime = getMemoryCpuAndTimeRequirements(job, child)
ET.SubElement(children, "child", { "command":child.attrib["command"],
"time":str(compTime), "memory":str(memory), "cpu":str(cpu) })
logger.info("Making a child with command: %s" % (child.attrib["command"]))
#The follow on command
followOns = job.find("followOns")
followOns.remove(followOns.findall("followOn")[-1]) #Remove the old job
if tempJob.attrib.has_key("command"):
memory, cpu, compTime = getMemoryCpuAndTimeRequirements(job, tempJob)
ET.SubElement(followOns, "followOn", { "command":tempJob.attrib["command"],
"time":str(compTime), "memory":str(memory), "cpu":str(cpu) })
##Add the runtime to the total runtime..
totalRuntime += compTime
logger.info("Making a follow on job with command: %s" % tempJob.attrib["command"])
elif len(tempJob.find("children").findall("child")) != 0: #This is to keep the stack of follow on jobs consistent.
ET.SubElement(followOns, "followOn", { "command":"echo JOB_FILE", "time":"0", "memory":"1000000", "cpu":"1" })
logger.info("Making a stub follow on job")
#Write back the runtime, after addin the follow on time and subtracting the time of the run job.
job.attrib["total_time"] = str(totalRuntime)
else:
logger.info("Failed the job")
job.attrib["colour"] = "red" #Update the colour
#Clean up
system("rm -rf %s" % (localSlaveTempDir))
logger.info("Cleaned up by removing temp jobfile (the copy), and the temporary file directory for the job")
def mainLoop(config, batchSystem):
"""This is the main loop from which jobs are issued and processed.
"""
waitDuration = float(config.attrib["wait_duration"])
assert waitDuration >= 0
rescueJobsFrequency = float(config.attrib["rescue_jobs_frequency"])
maxJobDuration = float(config.attrib["max_job_duration"])
assert maxJobDuration >= 0
logger.info("Got parameters, wait duration %s, rescue jobs frequency: %s max job duration: %s" % \
(waitDuration, rescueJobsFrequency, maxJobDuration))
#Kill any jobs on the batch system queue from the last time.
assert len(batchSystem.getIssuedJobIDs()) == 0 #Batch system must start with no active jobs!
logger.info("Checked batch system has no running jobs and no updated jobs")
jobFiles = config.attrib["job_file_dir"].listFiles()
logger.info("Got a list of job files")
#Repair the job tree using any .old files
fixJobsList(config, jobFiles)
logger.info("Fixed the job files using any .old files")
#Get jobs that were running, or that had failed reset to 'white' status
restartFailedJobs(config, jobFiles)
logger.info("Reworked failed jobs")
updatedJobFiles = set() #Jobs whose status needs updating, either because they have finished, or because they need to be started.
for jobFile in jobFiles:
job = ET.parse(jobFile).getroot()
if job.attrib["colour"] not in ("grey", "blue"):
updatedJobFiles.add(jobFile)
logger.info("Got the active (non grey/blue) job files")
totalJobFiles = len(jobFiles) #Total number of job files we have.
jobIDsToJobsHash = {} #A hash of the currently running jobs ids, made by the batch system.
idealJobTime = float(config.attrib["job_time"])
assert idealJobTime > 0.0
maxIssuedJobs = int(config.attrib["max_jobs"]) #The maximum number of jobs to issue to the batch system
assert maxIssuedJobs >= 1
stats = config.attrib.has_key("stats")
if stats:
startTime = time.time()
startClock = getTotalCpuTime()
logger.info("Starting the main loop")
timeSinceJobsLastRescued = time.time() - rescueJobsFrequency + 100 #We hack it so that we rescue jobs after the first 100 seconds to get around an apparent parasol bug
while True:
if len(updatedJobFiles) > 0:
logger.debug("Built the jobs list, currently have %i job files, %i jobs to update and %i jobs currently issued" % (totalJobFiles, len(updatedJobFiles), len(jobIDsToJobsHash)))
jobsToIssue = []
for jobFile in list(updatedJobFiles):
job = ET.parse(jobFile).getroot()
assert job.attrib["colour"] not in ("grey", "blue")
if job.attrib["colour"] == "white": #Get ready to start the job
if len(jobIDsToJobsHash) < maxIssuedJobs:
logger.debug("Job: %s is being started" % job.attrib["file"])
updatedJobFiles.remove(job.attrib["file"])
#Reset the log files for the job.
open(job.attrib["slave_log_file"], 'w').close()
open(job.attrib["log_file"], 'w').close()
job.attrib["colour"] = "grey"
#writeJobs([ job ]) #Check point, do this before issuing job, so state is not read until issued
#issueJobs([ job ], jobIDsToJobsHash, batchSystem)
jobsToIssue.append(job)
else:
logger.debug("Job: %s is not being issued yet because we have %i jobs issued" % (job.attrib["file"], len(jobIDsToJobsHash)))
elif job.attrib["colour"] == "black": #Job has finished okay
logger.debug("Job: %s has finished okay" % job.attrib["file"])
#Deal with stats
if stats:
system("cat %s >> %s" % (job.attrib["stats"], config.attrib["stats"]))
open(job.attrib["stats"], 'w').close() #Reset the stats file
childCount = int(job.attrib["child_count"])
blackChildCount = int(job.attrib["black_child_count"])
assert childCount == blackChildCount #Has no currently running child jobs
#Launch any unborn children
unbornChildren = job.find("children")
unbornChild = unbornChildren.find("child")
if unbornChild != None: #We must give birth to the unborn children
logger.debug("Job: %s has children to schedule" % job.attrib["file"])
newChildren = []
while unbornChild != None:
cummulativeChildTime = float(unbornChild.attrib["time"])
newJob = createJob(unbornChild.attrib.copy(), job.attrib["file"], config)
totalJobFiles += 1
updatedJobFiles.add(newJob.attrib["file"])
newChildren.append(newJob)
unbornChildren.remove(unbornChild)
unbornChild = unbornChildren.find("child")
#This was code to aggregate groups of children into one job, but we don't do this now
#while cummulativeChildTime < idealJobTime and unbornChild != None: #Cummulate a series of children into each job as a stack of jobs (to balance cost of parellelism with cost of running stuff in serially).
# cummulativeChildTime += float(unbornChild.attrib["time"])
# ET.SubElement(newJob.find("followOns"), "followOn", unbornChild.attrib.copy())
# unbornChildren.remove(unbornChild)
# unbornChild = unbornChildren.find("child")
newJob.attrib["total_time"] = str(cummulativeChildTime)
updatedJobFiles.remove(job.attrib["file"])
job.attrib["child_count"] = str(childCount + len(newChildren))
job.attrib["colour"] = "blue" #Blue - has children running.
writeJobs([ job ] + newChildren ) #Check point
elif len(job.find("followOns").findall("followOn")) != 0: #Has another job
logger.debug("Job: %s has a new command that we can now issue" % job.attrib["file"])
##Reset the job run info
job.attrib["remaining_retry_count"] = config.attrib["retry_count"]
job.attrib["colour"] = "white"
##End resetting the job
writeJobs([ job ])
else: #Job has finished, so we can defer to any parent
logger.debug("Job: %s is now dead" % job.attrib["file"])
job.attrib["colour"] = "dead"
if job.attrib.has_key("parent"):
parent = ET.parse(job.attrib["parent"]).getroot()
assert parent.attrib["colour"] == "blue"
assert int(parent.attrib["black_child_count"]) < int(parent.attrib["child_count"])
parent.attrib["black_child_count"] = str(int(parent.attrib["black_child_count"]) + 1)
if int(parent.attrib["child_count"]) == int(parent.attrib["black_child_count"]):
parent.attrib["colour"] = "black"
assert parent.attrib["file"] not in updatedJobFiles
updatedJobFiles.add(parent.attrib["file"])
writeJobs([ job, parent ]) #Check point
else:
writeJobs([ job ])
elif job.attrib["colour"] == "red": #Job failed
logger.critical("Job: %s failed" % job.attrib["file"])
logger.critical("The log file of the failed job")
logFile(job.attrib["log_file"], logger.critical)
logger.critical("The log file of the slave for the failed job")
logFile(job.attrib["slave_log_file"], logger.critical) #We log the job log file in the main loop
#Checks
assert len(job.find("children").findall("child")) == 0
assert int(job.attrib["child_count"]) == int(job.attrib["black_child_count"])
remainingRetyCount = int(job.attrib["remaining_retry_count"])
if remainingRetyCount > 0: #Give it another try, maybe there is a bad node somewhere
job.attrib["remaining_retry_count"] = str(remainingRetyCount-1)
job.attrib["colour"] = "white"
logger.critical("Job: %s will be restarted, it has %s goes left" % (job.attrib["file"], job.attrib["remaining_retry_count"]))
writeJobs([ job ]) #Check point
else:
assert remainingRetyCount == 0
updatedJobFiles.remove(job.attrib["file"])
logger.critical("Job: %s is completely failed" % job.attrib["file"])
else:
logger.debug("Job: %s is already dead, we'll get rid of it" % job.attrib["file"])
assert job.attrib["colour"] == "dead"
updatedJobFiles.remove(job.attrib["file"])
totalJobFiles -= 1
deleteJob(job, config) #This could be done earlier, but I like it this way.
###End of for loop
writeJobs(jobsToIssue) #Check point, do this before issuing job, so state is not read until issued
issueJobs(jobsToIssue, jobIDsToJobsHash, batchSystem)
if len(jobIDsToJobsHash) == 0 and len(updatedJobFiles) == 0:
logger.info("Only failed jobs and their dependents (%i total) are remaining, so exiting." % totalJobFiles)
break
if len(updatedJobFiles) > 0:
updatedJobs = batchSystem.getUpdatedJobs() #Asks the batch system what jobs have been completed.
else:
updatedJobs = pauseForUpdatedJobs(batchSystem.getUpdatedJobs) #Asks the batch system what jobs have been completed.
for jobID in updatedJobs.keys(): #Runs through a map of updated jobs and there status,
result = updatedJobs[jobID]
if jobIDsToJobsHash.has_key(jobID):
if result == 0:
logger.debug("Batch system is reporting that the job %s ended sucessfully" % jobIDsToJobsHash[jobID])
else:
logger.critical("Batch system is reporting that the job %s failed with exit value %i" % (jobIDsToJobsHash[jobID], result))
processFinishedJob(jobID, result, updatedJobFiles, jobIDsToJobsHash)
else:
logger.info("A result seems to already have been processed: %i" % jobID) #T
if time.time() - timeSinceJobsLastRescued >= rescueJobsFrequency: #We only rescue jobs every N seconds
reissueOverLongJobs(updatedJobFiles, jobIDsToJobsHash, config, batchSystem)
logger.info("Reissued any over long jobs")
reissueMissingJobs(updatedJobFiles, jobIDsToJobsHash, batchSystem)
logger.info("Rescued any (long) missing jobs")
timeSinceJobsLastRescued = time.time()
#Going to sleep to let the job system catch up.
time.sleep(waitDuration)
if stats:
fileHandle = open(config.attrib["stats"], 'a')
fileHandle.write("<total_time time='%s' clock='%s'/></stats>" % (str(time.time() - startTime), str(getTotalCpuTime() - startClock)))
fileHandle.close()
logger.info("Finished the main loop")
return totalJobFiles #Returns number of failed jobs
def runJobTreeStats(jobTree, outputFile):
system("jobTreeStats --jobTree %s --outputFile %s" % (jobTree, outputFile))
logger.info("Ran the job-tree stats command apparently okay")
def run(self):
logger.info("At the end, this is the contents of the global temp dir...")
system("ls -l %s" % self.getGlobalTempDir())
logger.info("And done....")
self.tempFileTree.destroyTempFiles()
def runJobTreeStatusAndFailIfNotComplete(jobTreeDir):
command = "jobTreeStatus --jobTree %s --failIfNotComplete --verbose" % jobTreeDir
system(command)
def tearDown(self):
unittest.TestCase.tearDown(self)
system("rm -rf %s" % self.tempDir) #Cleanup the job tree in case it hasn't already been cleaned up.
def run(self):
system("rm -rf %s" % self.getGlobalTempDir())
def execute(self, jobFile):
job = ET.parse(jobFile).getroot()
setLogLevel(job.attrib["log_level"])
logger.info("Setup logging with level: %s" % job.attrib["log_level"])
self.tempDirAccessed = False
self.localTempDir = job.attrib["local_temp_dir"]
self.globalTempDir = job.attrib["global_temp_dir"]
maxTime = float(job.attrib["job_time"])
memory = int(job.attrib["available_memory"])
cpu = int(job.attrib["available_cpu"])
if job.attrib.has_key("stats"):
stats = ET.Element("stack")
startTime = time.time()
startClock = time.clock()
else:
stats = None
newChildren = [] #List to add all the children to before we package them
#off into stacks
newChildCommands = [] #Ditto for the child commands
newFollowOns = [] #Ditto for the follow-ons
while self.hasRemaining():
if stats != None: #Getting the runtime of the stats module
targetStartTime = time.time()
targetStartClock = getTotalCpuTime()
target = self.popTarget()
target.setStack(self)
#Debug check that we have the right amount of CPU and memory for the job in hand
targetMemory = target.getMemory()
if targetMemory != sys.maxint:
assert targetMemory <= memory
targetCpu = target.getCpu()
if targetCpu != sys.maxint:
assert targetCpu <= cpu
#Run the target, first cleanup then run.
target.run()
#Cleanup after the target
if self.tempDirAccessed:
system("rm -rf %s/*" % self.localTempDir)
self.tempDirAccessed = False
#Handle the follow on
followOn = target.getFollowOn()
#if target.__class__ != CleanupGlobalTempDirTarget and followOn == None:
# followOn = CleanupGlobalTempDirTarget()
if followOn != None: #Target to get rid of follow on when done.
if target.isGlobalTempDirSet():
followOn.setGlobalTempDir(target.getGlobalTempDir())
newFollowOns.append(followOn)
#Now add the children to the newChildren stack
newChildren += target.getChildren()
#Now add the child commands to the newChildCommands stack
newChildCommands += target.getChildCommands()
if stats != None:
ET.SubElement(stats, "target", { "time":str(time.time() - targetStartTime),
"clock":str(getTotalCpuTime() - targetStartClock),
"class":".".join((target.__class__.__name__,)),
"e_time":str(target.getRunTime())})
#######
#Now build the new stacks and corresponding jobs
#######
#First add all the follow ons to the existing stack and make it a follow on job for job-tree
assert not self.hasRemaining()
#First sort out the follow on job
if len(newFollowOns) > 0: #If we have follow ons
followOnRuntime = sum([ followOn.getRunTime() for followOn in newFollowOns ])
if followOnRuntime > maxTime: #We create a parallel list of follow ons
followOnStack = Stack(ParallelFollowOnTarget(newFollowOns))
else:
followOnStack = Stack(newFollowOns.pop())
while len(newFollowOns) > 0:
followOnStack.addTarget(newFollowOns.pop())
job.attrib["command"] = followOnStack.makeRunnable(self.globalTempDir)
job.attrib["time"] = str(followOnStack.getRunTime())
followOnMemory = followOnStack.getMemory()
assert not job.attrib.has_key("memory")
if followOnMemory != sys.maxint:
job.attrib["memory"] = str(followOnMemory)
assert not job.attrib.has_key("cpu")
followOnCpu = followOnStack.getCpu()
if followOnCpu != sys.maxint:
job.attrib["cpu"] = str(followOnCpu)
#Now build stacks of children..
childrenTag = job.find("children")
while len(newChildren) > 0:
childStack = Stack(newChildren.pop())
while len(newChildren) > 0 and childStack.getRunTime() <= maxTime:
childStack.addTarget(newChildren.pop())
childJob = ET.SubElement(childrenTag, "child", { "command":childStack.makeRunnable(self.globalTempDir),
"time":str(childStack.getRunTime()) })
childMemory = childStack.getMemory()
assert not childJob.attrib.has_key("memory")
if childMemory != sys.maxint:
childJob.attrib["memory"] = str(childMemory)
assert not childJob.attrib.has_key("cpu")
childCpu = childStack.getCpu()
if childCpu != sys.maxint:
childJob.attrib["cpu"] = str(childCpu)
#Now build jobs for each child command
for childCommand, runTime in newChildCommands:
ET.SubElement(childrenTag, "child", { "command":str(childCommand),
"time":str(runTime) })
#Now write the updated job file
fileHandle = open(jobFile, 'w')
ET.ElementTree(job).write(fileHandle)
fileHandle.close()
#Finish up the stats
if stats != None:
stats.attrib["time"] = str(time.time() - startTime)
stats.attrib["clock"] = str(getTotalCpuTime() - startClock)
fileHandle = open(job.attrib["stats"], 'w')
ET.ElementTree(stats).write(fileHandle)
fileHandle.close()
def tearDown(self):
unittest.TestCase.tearDown(self)
self.tempFileTree.destroyTempFiles()
system("rm -rf %s %s" % (self.jobTreeDir, self.tempFileTreeDir)) #Cleanup the job tree in case it hasn't already been cleaned up.
