def __init__(self, config, maxCpus, maxMemory):
AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor
if maxMemory != sys.maxint:
logger.critical("A max memory has been specified for the parasol batch system class of %i, but currently this batchsystem interface does not support such limiting" % maxMemory)
#Keep the name of the results file for the pstat2 command..
self.parasolCommand = config.attrib["parasol_command"]
self.parasolResultsFile = getParasolResultsFileName(config.attrib["job_tree"])
#Reset the job queue and results (initially, we do this again once we've killed the jobs)
self.queuePattern = re.compile("q\s+([0-9]+)")
self.runningPattern = re.compile("r\s+([0-9]+)\s+[\S]+\s+[\S]+\s+([0-9]+)\s+[\S]+")
self.killJobs(self.getIssuedJobIDs()) #Kill any jobs on the current stack
logger.info("Going to sleep for a few seconds to kill any existing jobs")
time.sleep(5) #Give batch system a second to sort itself out.
logger.info("Removed any old jobs from the queue")
#Reset the job queue and results
exitValue = popenParasolCommand("%s -results=%s clear sick" % (self.parasolCommand, self.parasolResultsFile), False)[0]
if exitValue != None:
logger.critical("Could not clear sick status of the parasol batch %s" % self.parasolResultsFile)
exitValue = popenParasolCommand("%s -results=%s flushResults" % (self.parasolCommand, self.parasolResultsFile), False)[0]
if exitValue != None:
logger.critical("Could not flush the parasol batch %s" % self.parasolResultsFile)
open(self.parasolResultsFile, 'w').close()
logger.info("Reset the results queue")
#Stuff to allow max cpus to be work
self.outputQueue1 = Queue()
self.outputQueue2 = Queue()
#worker = Thread(target=getUpdatedJob, args=(self.parasolResultsFileHandle, self.outputQueue1, self.outputQueue2))
#worker.setDaemon(True)
worker = Process(target=getUpdatedJob, args=(self.parasolResultsFile, self.outputQueue1, self.outputQueue2))
worker.daemon = True
worker.start()
self.usedCpus = 0
self.jobIDsToCpu = {}
def __init__(self, config, maxCpus, maxMemory, workerFn=worker):
AbstractBatchSystem.__init__(self, config, maxCpus,
maxMemory) #Call the parent constructor
self.jobIndex = 0
self.jobs = {}
self.maxThreads = int(config.attrib["max_threads"])
logger.info(
"Setting up the thread pool with %i threads given the max threads %i and the max cpus %i"
% (min(self.maxThreads,
self.maxCpus), self.maxThreads, self.maxCpus))
self.maxThreads = min(self.maxThreads, self.maxCpus)
self.cpusPerThread = float(self.maxCpus) / float(self.maxThreads)
self.memoryPerThread = self.maxThreads + float(self.maxMemory) / float(
self.maxThreads
) #Add the maxThreads to avoid losing memory by rounding.
assert self.cpusPerThread >= 1
assert self.maxThreads >= 1
assert self.maxMemory >= 1
assert self.memoryPerThread >= 1
self.inputQueue = Queue()
self.outputQueue = Queue()
self.workerFn = workerFn
for i in xrange(self.maxThreads): #Setup the threads
worker = Process(target=workerFn,
args=(self.inputQueue, self.outputQueue))
worker.daemon = True
worker.start()
def __init__(self, config, batchSystem1, batchSystem2,
batchSystemChoiceFn):
AbstractBatchSystem.__init__(self, config, 0,
0) #Call the parent constructor
self.batchSystem1 = batchSystem1
self.batchSystem2 = batchSystem2
self.batchSystemChoiceFn = batchSystemChoiceFn
def __init__(self, config, workerClass=Worker):
AbstractBatchSystem.__init__(self, config) #Call the parent constructor
self.jobIndex = 0
self.jobs = {}
self.maxThreads = int(config.attrib["max_threads"])
assert self.maxThreads >= 1
self.inputQueue = Queue()
self.outputQueue = Queue()
for i in xrange(int(config.attrib["max_threads"])): #Setup the threads
worker = workerClass(self.inputQueue, self.outputQueue)
worker.setDaemon(True)
worker.start()
def __init__(self, config, maxCpus, maxMemory):
AbstractBatchSystem.__init__(self, config, maxCpus,
maxMemory) #Call the parent constructor
if maxMemory != sys.maxint:
logger.critical(
"A max memory has been specified for the parasol batch system class of %i, but currently this batchsystem interface does not support such limiting"
% maxMemory)
#Keep the name of the results file for the pstat2 command..
self.parasolCommand = config.attrib["parasol_command"]
self.parasolResultsFile = getParasolResultsFileName(
config.attrib["job_tree"])
#Reset the job queue and results (initially, we do this again once we've killed the jobs)
self.queuePattern = re.compile("q\s+([0-9]+)")
self.runningPattern = re.compile(
"r\s+([0-9]+)\s+[\S]+\s+[\S]+\s+([0-9]+)\s+[\S]+")
self.killJobs(
self.getIssuedJobIDs()) #Kill any jobs on the current stack
logger.info(
"Going to sleep for a few seconds to kill any existing jobs")
time.sleep(5) #Give batch system a second to sort itself out.
logger.info("Removed any old jobs from the queue")
#Reset the job queue and results
exitValue = popenParasolCommand(
"%s -results=%s clear sick" %
(self.parasolCommand, self.parasolResultsFile), False)[0]
if exitValue != None:
logger.critical(
"Could not clear sick status of the parasol batch %s" %
self.parasolResultsFile)
exitValue = popenParasolCommand(
"%s -results=%s flushResults" %
(self.parasolCommand, self.parasolResultsFile), False)[0]
if exitValue != None:
logger.critical("Could not flush the parasol batch %s" %
self.parasolResultsFile)
open(self.parasolResultsFile, 'w').close()
logger.info("Reset the results queue")
#Stuff to allow max cpus to be work
self.outputQueue1 = Queue()
self.outputQueue2 = Queue()
#worker = Thread(target=getUpdatedJob, args=(self.parasolResultsFileHandle, self.outputQueue1, self.outputQueue2))
#worker.setDaemon(True)
worker = Process(target=getUpdatedJob,
args=(self.parasolResultsFile, self.outputQueue1,
self.outputQueue2))
worker.daemon = True
worker.start()
self.usedCpus = 0
self.jobIDsToCpu = {}
def __init__(self, config, maxCpus, maxMemory):
AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor
self.lsfResultsFile = getParasolResultsFileName(config.attrib["job_tree"])
#Reset the job queue and results (initially, we do this again once we've killed the jobs)
self.lsfResultsFileHandle = open(self.lsfResultsFile, 'w')
self.lsfResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence
self.currentjobs = set()
self.obtainSystemConstants()
self.jobIDs = dict()
self.lsfJobIDs = dict()
self.nextJobID = 0
self.newJobsQueue = Queue()
self.updatedJobsQueue = Queue()
self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self)
self.worker.setDaemon(True)
self.worker.start()
def __init__(self, config, maxCpus, maxMemory):
AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor
self.gridengineResultsFile = getParasolResultsFileName(config.attrib["job_tree"])
#Reset the job queue and results (initially, we do this again once we've killed the jobs)
self.gridengineResultsFileHandle = open(self.gridengineResultsFile, 'w')
self.gridengineResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence
self.currentjobs = set()
self.obtainSystemConstants()
self.nextJobID = 0
self.newJobsQueue = Queue()
self.updatedJobsQueue = Queue()
self.killQueue = Queue()
self.killedJobsQueue = Queue()
self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self.killQueue, self.killedJobsQueue, self)
self.worker.setDaemon(True)
self.worker.start()
def __init__(self, config):
AbstractBatchSystem.__init__(self, config) #Call the parent constructor
self.gridengineResultsFile = config.attrib["results_file"]
#Reset the job queue and results (initially, we do this again once we've killed the jobs)
self.gridengineResultsFileHandle = open(self.gridengineResultsFile, 'w')
self.gridengineResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence
self.scratchFile = self.config.attrib["scratch_file"]
self.currentjobs = set()
self.obtainSystemConstants()
self.jobIDs = dict()
self.sgeJobIDs = dict()
self.nextJobID = 0
self.newJobsQueue = Queue()
self.updatedJobsQueue = Queue()
self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self)
self.worker.setDaemon(True)
self.worker.start()
def __init__(self, config):
AbstractBatchSystem.__init__(self, config) #Call the parent constructor
#Keep the name of the results file for the pstat2 command..
self.parasolResultsFile = config.attrib["results_file"]
#Reset the job queue and results (initially, we do this again once we've killed the jobs)
self.parasolResultsFileHandle = open(self.parasolResultsFile, 'w')
self.parasolResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence
self.queuePattern = re.compile("q\s+([0-9]+)")
self.runningPattern = re.compile("r\s+([0-9]+)\s+[\S]+\s+[\S]+\s+([0-9]+)\s+[\S]+")
#The scratch file
self.scratchFile = self.config.attrib["scratch_file"]
self.killJobs(self.getIssuedJobIDs()) #Kill any jobs on the current stack
logger.info("Going to sleep for a few seconds to kill any existing jobs")
time.sleep(5) #Give batch system a second to sort itself out.
logger.info("Removed any old jobs from the queue")
#Reset the job queue and results
self.parasolResultsFileHandle = open(self.parasolResultsFile, 'w')
self.parasolResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence
self.parasolResultsFileHandle = open(self.parasolResultsFile, 'r')
logger.info("Reset the results queue")
def __init__(self, config, maxCpus, maxMemory, workerFn=worker):
AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor
self.jobIndex = 0
self.jobs = {}
self.maxThreads = int(config.attrib["max_threads"])
logger.info("Setting up the thread pool with %i threads given the max threads %i and the max cpus %i" % (min(self.maxThreads, self.maxCpus), self.maxThreads, self.maxCpus))
self.maxThreads = min(self.maxThreads, self.maxCpus)
self.cpusPerThread = float(self.maxCpus) / float(self.maxThreads)
self.memoryPerThread = self.maxThreads + float(self.maxMemory) / float(self.maxThreads) #Add the maxThreads to avoid losing memory by rounding.
assert self.cpusPerThread >= 1
assert self.maxThreads >= 1
assert self.maxMemory >= 1
assert self.memoryPerThread >= 1
self.inputQueue = Queue()
self.outputQueue = Queue()
self.workerFn = workerFn
for i in xrange(self.maxThreads): #Setup the threads
worker = Process(target=workerFn, args=(self.inputQueue, self.outputQueue))
worker.daemon = True
worker.start()
def __init__(self, config, maxCpus, maxMemory):
"""
Stuff
"""
AbstractBatchSystem.__init__(self, config, maxCpus,
maxMemory) #Call the parent constructor
self.resultsFile = getParasolResultsFileName(config.attrib["job_tree"])
#Reset the job queue and results (initially, we do this again once we've killed the jobs)
self.resultsFileHandle = open(self.resultsFile, 'w')
self.resultsFileHandle.close(
) #We lose any previous state in this file, and ensure the files existence
self.currentjobs = set()
self.obtainSystemConstants()
self.jobIDs = dict()
self.slurmJobTasks = dict()
self.nextJobID = 0
self.newJobsQueue = Queue()
self.updatedJobsQueue = Queue()
# store any of the slurm options
slurmopts = dict()
optiondata = SlurmBatchSystem.getOptionData()
for switch, data in optiondata.iteritems():
key = data['dest']
if key in config.attrib:
sbatchopt = re.sub(r'^slurm_', '', key)
slurmopts[sbatchopt] = config.attrib[key]
self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self,
slurmopts)
self.worker.setDaemon(True)
self.worker.start()
def __init__(self, config, batchSystem1, batchSystem2, batchSystemChoiceFn):
AbstractBatchSystem.__init__(self, config, 0, 0) #Call the parent constructor
self.batchSystem1 = batchSystem1
self.batchSystem2 = batchSystem2
self.batchSystemChoiceFn = batchSystemChoiceFn
def __init__(self, config, maxCpus, maxMemory):
AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor
self.job_type_queue = {}
self.currentjobs = 0
