def invertKeepOutput(self, name):
printDebug(3, "Invert keep output for batch %s" % name)
if name in self.listBatch:
self.listBatch[name]["monitor"].invertKeepOutput()
header = self.listBatch[name]["monitor"].config.getHeaders()
header['keep'] = self.listBatch[name]["monitor"].keepOutput
return header
def getBatchList(self):
printDebug(3, "Sending batch list")
l = []
orderedNames = sorted(self.listBatch.keys())
for name in orderedNames:
l.append({'name':name, 'stats':self.listBatch[name]["monitor"].config.getStatusStats()})
return l
def loadBatch(self, jsonFile, name, keep, limit):
printDebug(3, "Loading new batch %s from %s" % (name, jsonFile))
if name in self.listBatch:
printDebug(2, "Batch %s already exists" % name)
return
batch = Monitor2(keep, limit)
batch.loadBatch(str(jsonFile))
self.listBatch[name] = {"monitor":batch, "clients":[]}
def addBatch(self, cardFile, name, queue, test, keep, limit):
printDebug(3, "Adding new batch %s" % name)
if name in self.listBatch:
printDebug(2, "Batch %s already exists" % name)
return
batch = Monitor2(keep, limit)
batch.newBatch(cardFile, name, queue, test)
self.listBatch[name] = {"monitor":batch, "clients":[]}
def disconnectAllClients(self):
printDebug(3, "Disconnecting all clients")
for _,batch in self.listBatch.iteritems():
#list of clients can be manipulated by one thread at a time
self.mutex.acquire()
for client in batch["clients"]:
client._pyroRelease()
batch["clients"].remove(client)
self.mutex.release()
def disconnectClient(self, name, clientUri):
printDebug(3, "Disconnecting client %s for %s" % (clientUri, name))
if name in self.listBatch:
#list of clients can be manipulated by one thread at a time
self.mutex.acquire()
for client in self.listBatch[name]["clients"]:
if client._pyroUri == clientUri:
client._pyroRelease()
self.listBatch[name]["clients"].remove(client)
self.mutex.release()
def generateJobs(self):
printDebug(3, "Monitor generating jobs")
self.submitting = True
if len(self.submitList)==0:
subList = [job for job in self.config.jobsList if job.attempts==-1]
else:
subList = [self.config.jobsList[i] for i in self.submitList]
for job in subList:
yield job
self.activeJobs+=1
if self.jobsLimit>0 and self.activeJobs>=self.jobsLimit:
break
self.submitList = []
self.submitReady = False
self.submitting = False
def registerClient(self, name, clientUri):
printDebug(3, "Registering client %s for %s" % (clientUri, name))
if name in self.listBatch:
#List of clients can be manipulated only by one thread at a time
self.mutex.acquire()
client = Pyro4.Proxy(clientUri)
#Set the timeout for the calls. Client might not be running anymore and cannot block server indefinitely.
client._pyroTimeout = 1
self.listBatch[name]["clients"].append(client)
#Get the basic info about the batch and return them to the client
header = self.listBatch[name]["monitor"].config.getHeaders()
header['keep'] = self.listBatch[name]["monitor"].keepOutput
self.mutex.release()
return self.listBatch[name]["monitor"].config.startTime, header, len(self.listBatch[name]["monitor"].config.jobsList), self.listBatch[name]["monitor"].config.getStatusStats()
def submitInit(self):
printDebug(3, "Monitor initial submit")
self.config.enableNew()
self.submitReady = True
def reSubmitFailed(self):
printDebug(3, "Monitor resubmitting failed jobs")
self.config.resetFailed()
self.submitReady = True
def submit(self, jobs):
printDebug(3, "Monitor submitting jobs")
return getLSFMonitorInstance().submitJob(jobs, self.config)
def saveBatch(self, jsonFile):
printDebug(3, "Saving batch")
self.config.save(jsonFile)
def loadBatch(self, jsonFile):
printDebug(3, "Monitor loading new batch")
self.config.load(jsonFile)
self.submitReady = False
def newBatch(self, cfgFile, batchName, queue, test):
printDebug(3, "Monitor creating new batch")
self.config.initCardFile(cfgFile, batchName, queue, test)
self.submitReady = False
def deleteJobs(self):
printDebug(3, "Delete jobs " + self.config.name)
getLSFMonitorInstance().deleteJobs(self.config.name)
def submitInit(self, name):
printDebug(3, "Initial submit for batch %s" % name)
if name in self.listBatch:
self.listBatch[name]["monitor"].submitInit()
def stop(self):
global stopAll
printDebug(3, "Stopping server")
stopAll = True
def _shutilFunc(func, src, dst):
u.printDebug( 'shutil.'+ func.func_name +'('+ src +', '+ dst +')' )
func(src, dst)
def removeBatch(self, name):
printDebug(3, "Removing batch %s" % name)
t = threading.Thread(target=self.doRemove, args=(name,))
t.setName(name)
t.daemon = True
t.start()
def submitLoopArrayed(self, batch):
cThread = threading.currentThread()
printDebug(3, "["+cThread.name+"] Enter submit loop")
try:
#Reset submit flag
batch["monitor"].submitting = False
printDebug(3, "["+cThread.name+"] Number of jobs in Ready state: " + str(batch["monitor"].config.getJobsNumberReady()))
#Go through all the jobs that are ready and submit them
jList = [];
for i, job in enumerate(batch["monitor"].generateJobs()):
printDebug(3, "["+cThread.name+"] Generate job " + str(i))
jList.append(job)
lsfID,index = batch["monitor"].submit(jList)
printDebug(3, "["+cThread.name+"] acquire mutex")
#Notify the clients that the job was submitted
if self.mutex.acquire():
try:
for clients in batch["clients"]:
clients.displayJobSent(lsfID, index, len(jList)-1)
except Exception:
printDebug(1, "["+cThread.name+"] Exception:")
printDebug(1, "".join(Pyro4.util.getPyroTraceback()))
finally:
printDebug(3, "["+cThread.name+"] release mutex")
self.mutex.release()
except Exception:
printDebug(1, "["+cThread.name+"] Exception")
printDebug(1, "".join(Pyro4.util.getPyroTraceback()))
def kill(self):
printDebug(3, "Killing server properly")
self.disconnectAllClients()
self.saveAllBatches()
def removeFile(src):
u.printDebug( 'os.remove('+ src +')' )
os.remove(src)
def resubmitFailed(self, name):
printDebug(3, "Resubmiting failed jobs for batch %s" % name)
if name in self.listBatch:
self.listBatch[name]["monitor"].reSubmitFailed()
