def mainLoop(self):
#Start monitor function for each batch in its own thread because can be very slow and block the server
tMon = threading.Thread(target=getLSFMonitorInstance().refreshInfo)
tMon.daemon = True
tMon.start()
for name,batch in self.listBatch.items():
#Update info for each monitor
batch["monitor"].monitor()
#Send summary to all clients
for clients in batch["clients"]:
clients.displaySummary(batch["monitor"].config.getStatusStats())
#We some jobs ready and we are not submitting
if batch["monitor"].submitReady and batch["monitor"].submitting==False:
#Reset either the jobs that are ready if the monitor does not have its own jobs or the monitor's jobs
if len(batch["monitor"].submitList)==0:
for clients in batch["clients"]:
clients.resetSubmit(batch["monitor"].config.getJobsNumberReady())
else:
for clients in batch["clients"]:
clients.resetSubmit(len(batch["monitor"].submitList))
#Start the submit loop in its own thread because can take a very long time again.
t = threading.Thread(target=self.submitLoopArrayed, args=(batch,))
t.setName(name)
t.daemon = True
t.start()
def monitorNormal(self):
self.activeJobs = 0
for key in self.config.jobCorrespondance.iterLayer1():
jobInfo = getLSFMonitorInstance().getInfoByJobID(key)
if not jobInfo is None:
for jobKey, job in jobInfo.iteritems():
if job.lsfStatus=="RUN" or job.lsfStatus=="PEND":
self.activeJobs += 1
redo,index = self.config.updateJob((job.lsfID,jobKey), {"status":job.lsfStatus}, self.keepOutput)
if redo:
self.reSubmit.append(index)
if self.submitting == False and len(self.reSubmit)>0:
self.submitReady = True
self.submitList.extend(self.reSubmit[:])
self.reSubmit = []
def submit(self, jobs):
printDebug(3, "Monitor submitting jobs")
return getLSFMonitorInstance().submitJob(jobs, self.config)
def deleteJobs(self):
printDebug(3, "Delete jobs " + self.config.name)
getLSFMonitorInstance().deleteJobs(self.config.name)
