def __init__(self, config, maxCpus, maxMemory): AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor if maxMemory != sys.maxint: logger.critical("A max memory has been specified for the parasol batch system class of %i, but currently this batchsystem interface does not support such limiting" % maxMemory) #Keep the name of the results file for the pstat2 command.. self.parasolCommand = config.attrib["parasol_command"] self.parasolResultsFile = getParasolResultsFileName(config.attrib["job_tree"]) #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.queuePattern = re.compile("q\s+([0-9]+)") self.runningPattern = re.compile("r\s+([0-9]+)\s+[\S]+\s+[\S]+\s+([0-9]+)\s+[\S]+") self.killJobs(self.getIssuedJobIDs()) #Kill any jobs on the current stack logger.info("Going to sleep for a few seconds to kill any existing jobs") time.sleep(5) #Give batch system a second to sort itself out. logger.info("Removed any old jobs from the queue") #Reset the job queue and results exitValue = popenParasolCommand("%s -results=%s clear sick" % (self.parasolCommand, self.parasolResultsFile), False)[0] if exitValue != None: logger.critical("Could not clear sick status of the parasol batch %s" % self.parasolResultsFile) exitValue = popenParasolCommand("%s -results=%s flushResults" % (self.parasolCommand, self.parasolResultsFile), False)[0] if exitValue != None: logger.critical("Could not flush the parasol batch %s" % self.parasolResultsFile) open(self.parasolResultsFile, 'w').close() logger.info("Reset the results queue") #Stuff to allow max cpus to be work self.outputQueue1 = Queue() self.outputQueue2 = Queue() #worker = Thread(target=getUpdatedJob, args=(self.parasolResultsFileHandle, self.outputQueue1, self.outputQueue2)) #worker.setDaemon(True) worker = Process(target=getUpdatedJob, args=(self.parasolResultsFile, self.outputQueue1, self.outputQueue2)) worker.daemon = True worker.start() self.usedCpus = 0 self.jobIDsToCpu = {}
def __init__(self, config, maxCpus, maxMemory, workerFn=worker): AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor self.jobIndex = 0 self.jobs = {} self.maxThreads = int(config.attrib["max_threads"]) logger.info( "Setting up the thread pool with %i threads given the max threads %i and the max cpus %i" % (min(self.maxThreads, self.maxCpus), self.maxThreads, self.maxCpus)) self.maxThreads = min(self.maxThreads, self.maxCpus) self.cpusPerThread = float(self.maxCpus) / float(self.maxThreads) self.memoryPerThread = self.maxThreads + float(self.maxMemory) / float( self.maxThreads ) #Add the maxThreads to avoid losing memory by rounding. assert self.cpusPerThread >= 1 assert self.maxThreads >= 1 assert self.maxMemory >= 1 assert self.memoryPerThread >= 1 self.inputQueue = Queue() self.outputQueue = Queue() self.workerFn = workerFn for i in xrange(self.maxThreads): #Setup the threads worker = Process(target=workerFn, args=(self.inputQueue, self.outputQueue)) worker.daemon = True worker.start()
def __init__(self, config, batchSystem1, batchSystem2, batchSystemChoiceFn): AbstractBatchSystem.__init__(self, config, 0, 0) #Call the parent constructor self.batchSystem1 = batchSystem1 self.batchSystem2 = batchSystem2 self.batchSystemChoiceFn = batchSystemChoiceFn
def __init__(self, config, workerClass=Worker): AbstractBatchSystem.__init__(self, config) #Call the parent constructor self.jobIndex = 0 self.jobs = {} self.maxThreads = int(config.attrib["max_threads"]) assert self.maxThreads >= 1 self.inputQueue = Queue() self.outputQueue = Queue() for i in xrange(int(config.attrib["max_threads"])): #Setup the threads worker = workerClass(self.inputQueue, self.outputQueue) worker.setDaemon(True) worker.start()
def __init__(self, config, maxCpus, maxMemory): AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor if maxMemory != sys.maxint: logger.critical( "A max memory has been specified for the parasol batch system class of %i, but currently this batchsystem interface does not support such limiting" % maxMemory) #Keep the name of the results file for the pstat2 command.. self.parasolCommand = config.attrib["parasol_command"] self.parasolResultsFile = getParasolResultsFileName( config.attrib["job_tree"]) #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.queuePattern = re.compile("q\s+([0-9]+)") self.runningPattern = re.compile( "r\s+([0-9]+)\s+[\S]+\s+[\S]+\s+([0-9]+)\s+[\S]+") self.killJobs( self.getIssuedJobIDs()) #Kill any jobs on the current stack logger.info( "Going to sleep for a few seconds to kill any existing jobs") time.sleep(5) #Give batch system a second to sort itself out. logger.info("Removed any old jobs from the queue") #Reset the job queue and results exitValue = popenParasolCommand( "%s -results=%s clear sick" % (self.parasolCommand, self.parasolResultsFile), False)[0] if exitValue != None: logger.critical( "Could not clear sick status of the parasol batch %s" % self.parasolResultsFile) exitValue = popenParasolCommand( "%s -results=%s flushResults" % (self.parasolCommand, self.parasolResultsFile), False)[0] if exitValue != None: logger.critical("Could not flush the parasol batch %s" % self.parasolResultsFile) open(self.parasolResultsFile, 'w').close() logger.info("Reset the results queue") #Stuff to allow max cpus to be work self.outputQueue1 = Queue() self.outputQueue2 = Queue() #worker = Thread(target=getUpdatedJob, args=(self.parasolResultsFileHandle, self.outputQueue1, self.outputQueue2)) #worker.setDaemon(True) worker = Process(target=getUpdatedJob, args=(self.parasolResultsFile, self.outputQueue1, self.outputQueue2)) worker.daemon = True worker.start() self.usedCpus = 0 self.jobIDsToCpu = {}
def __init__(self, config, maxCpus, maxMemory): AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor self.lsfResultsFile = getParasolResultsFileName(config.attrib["job_tree"]) #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.lsfResultsFileHandle = open(self.lsfResultsFile, 'w') self.lsfResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence self.currentjobs = set() self.obtainSystemConstants() self.jobIDs = dict() self.lsfJobIDs = dict() self.nextJobID = 0 self.newJobsQueue = Queue() self.updatedJobsQueue = Queue() self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self) self.worker.setDaemon(True) self.worker.start()
def __init__(self, config, maxCpus, maxMemory): AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor self.gridengineResultsFile = getParasolResultsFileName(config.attrib["job_tree"]) #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.gridengineResultsFileHandle = open(self.gridengineResultsFile, 'w') self.gridengineResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence self.currentjobs = set() self.obtainSystemConstants() self.nextJobID = 0 self.newJobsQueue = Queue() self.updatedJobsQueue = Queue() self.killQueue = Queue() self.killedJobsQueue = Queue() self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self.killQueue, self.killedJobsQueue, self) self.worker.setDaemon(True) self.worker.start()
def __init__(self, config): AbstractBatchSystem.__init__(self, config) #Call the parent constructor self.gridengineResultsFile = config.attrib["results_file"] #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.gridengineResultsFileHandle = open(self.gridengineResultsFile, 'w') self.gridengineResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence self.scratchFile = self.config.attrib["scratch_file"] self.currentjobs = set() self.obtainSystemConstants() self.jobIDs = dict() self.sgeJobIDs = dict() self.nextJobID = 0 self.newJobsQueue = Queue() self.updatedJobsQueue = Queue() self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self) self.worker.setDaemon(True) self.worker.start()
def __init__(self, config): AbstractBatchSystem.__init__(self, config) #Call the parent constructor #Keep the name of the results file for the pstat2 command.. self.parasolResultsFile = config.attrib["results_file"] #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.parasolResultsFileHandle = open(self.parasolResultsFile, 'w') self.parasolResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence self.queuePattern = re.compile("q\s+([0-9]+)") self.runningPattern = re.compile("r\s+([0-9]+)\s+[\S]+\s+[\S]+\s+([0-9]+)\s+[\S]+") #The scratch file self.scratchFile = self.config.attrib["scratch_file"] self.killJobs(self.getIssuedJobIDs()) #Kill any jobs on the current stack logger.info("Going to sleep for a few seconds to kill any existing jobs") time.sleep(5) #Give batch system a second to sort itself out. logger.info("Removed any old jobs from the queue") #Reset the job queue and results self.parasolResultsFileHandle = open(self.parasolResultsFile, 'w') self.parasolResultsFileHandle.close() #We lose any previous state in this file, and ensure the files existence self.parasolResultsFileHandle = open(self.parasolResultsFile, 'r') logger.info("Reset the results queue")
def __init__(self, config, maxCpus, maxMemory, workerFn=worker): AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor self.jobIndex = 0 self.jobs = {} self.maxThreads = int(config.attrib["max_threads"]) logger.info("Setting up the thread pool with %i threads given the max threads %i and the max cpus %i" % (min(self.maxThreads, self.maxCpus), self.maxThreads, self.maxCpus)) self.maxThreads = min(self.maxThreads, self.maxCpus) self.cpusPerThread = float(self.maxCpus) / float(self.maxThreads) self.memoryPerThread = self.maxThreads + float(self.maxMemory) / float(self.maxThreads) #Add the maxThreads to avoid losing memory by rounding. assert self.cpusPerThread >= 1 assert self.maxThreads >= 1 assert self.maxMemory >= 1 assert self.memoryPerThread >= 1 self.inputQueue = Queue() self.outputQueue = Queue() self.workerFn = workerFn for i in xrange(self.maxThreads): #Setup the threads worker = Process(target=workerFn, args=(self.inputQueue, self.outputQueue)) worker.daemon = True worker.start()
def __init__(self, config, maxCpus, maxMemory): """ Stuff """ AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor self.resultsFile = getParasolResultsFileName(config.attrib["job_tree"]) #Reset the job queue and results (initially, we do this again once we've killed the jobs) self.resultsFileHandle = open(self.resultsFile, 'w') self.resultsFileHandle.close( ) #We lose any previous state in this file, and ensure the files existence self.currentjobs = set() self.obtainSystemConstants() self.jobIDs = dict() self.slurmJobTasks = dict() self.nextJobID = 0 self.newJobsQueue = Queue() self.updatedJobsQueue = Queue() # store any of the slurm options slurmopts = dict() optiondata = SlurmBatchSystem.getOptionData() for switch, data in optiondata.iteritems(): key = data['dest'] if key in config.attrib: sbatchopt = re.sub(r'^slurm_', '', key) slurmopts[sbatchopt] = config.attrib[key] self.worker = Worker(self.newJobsQueue, self.updatedJobsQueue, self, slurmopts) self.worker.setDaemon(True) self.worker.start()
def __init__(self, config, maxCpus, maxMemory): AbstractBatchSystem.__init__(self, config, maxCpus, maxMemory) #Call the parent constructor self.job_type_queue = {} self.currentjobs = 0