class ThreadedMightyOptimizer(AgentModule): """ The specific agents must provide the following methods: - initialize() for initial settings - beginExecution() - execute() - the main method called in the agent cycle - endExecution() - finalize() - the graceful exit of the method, this one is usually used for the agent restart """ __jobStates = ["Received", "Checking"] __defaultValidOptimizers = [ "WorkloadManagement/JobPath", "WorkloadManagement/JobSanity", "WorkloadManagement/JobScheduling", "WorkloadManagement/TaskQueue", ] def initialize(self): """ Standard constructor """ self.jobDB = JobDB() self.jobLoggingDB = JobLoggingDB() self._optimizingJobs = JobsInTheWorks() self._optimizers = {} self._threadedOptimizers = {} self.am_setOption("PollingTime", 30) return S_OK() def execute(self): """ Standard Agent module execute method """ # Get jobs from DB result = self.jobDB.selectJobs({"Status": self.__jobStates}) if not result["OK"]: gLogger.error("Cannot retrieve jobs in states %s" % self.__jobStates) return result jobsList = result["Value"] for i in range(len(jobsList)): jobsList[i] = int(jobsList[i]) jobsList.sort() self.log.info("Got %s jobs for this iteration" % len(jobsList)) if not jobsList: return S_OK() # Check jobs that are already being optimized newJobsList = self._optimizingJobs.addJobs(jobsList) if not newJobsList: return S_OK() # Get attrs of jobs to be optimized result = self.jobDB.getAttributesForJobList(newJobsList) if not result["OK"]: gLogger.error("Cannot retrieve attributes for %s jobs %s" % len(newJobsList)) return result jobsToProcess = result["Value"] for jobId in jobsToProcess: self.log.info("== Processing job %s == " % jobId) jobAttrs = jobsToProcess[jobId] result = self.__dispatchJob(jobId, jobAttrs, False) if not result["OK"]: gLogger.error("There was a problem optimizing job", "JID %s: %s" % (jobId, result["Message"])) return S_OK() def __dispatchJob(self, jobId, jobAttrs, jobDef, keepOptimizing=True): """ Decide what to do with the Job """ returnValue = S_OK() if keepOptimizing: result = self.__sendJobToOptimizer(jobId, jobAttrs, jobDef) if result["OK"] and result["Value"]: return S_OK() if not result["OK"]: returnValue = result gLogger.error( "Could not send job to optimizer\n", "\tJob: %s\n\Message: %s" % (jobId, result["Message"]) ) self._optimizingJobs.deleteJob(jobId) return returnValue def __sendJobToOptimizer(self, jobId, jobAttrs, jobDef): """ Send Job to Optimizer queue """ optimizerName = self.__getNextOptimizerName(jobAttrs) if not optimizerName: return S_OK(False) if optimizerName not in self.am_getOption("ValidOptimizers", self.__defaultValidOptimizers): return S_OK(False) if optimizerName not in self._threadedOptimizers: to = ThreadedOptimizer(optimizerName, self.am_getModuleParam("fullName"), self.__dispatchJob) result = to.initialize(self.jobDB, self.jobLoggingDB) if not result["OK"]: return S_OK(False) self._threadedOptimizers[optimizerName] = to self._threadedOptimizers[optimizerName].optimizeJob(jobId, jobAttrs, jobDef) return S_OK(True) def __getNextOptimizerName(self, jobAttrs): """ Determine next Optimizer """ if jobAttrs["Status"] == "Received": optList = "JobPath" elif jobAttrs["Status"] == "Checking": optList = jobAttrs["MinorStatus"] else: return False optList = List.fromChar(optList, "/") if len(optList) == 1: optList.insert(0, "WorkloadManagement") if len(optList) > 2: optList[1] = "/".join(optList[1:]) return "/".join(optList)
class ThreadedMightyOptimizer( AgentModule ): """ The specific agents must provide the following methods: - initialize() for initial settings - beginExecution() - execute() - the main method called in the agent cycle - endExecution() - finalize() - the graceful exit of the method, this one is usually used for the agent restart """ __jobStates = [ 'Received', 'Checking' ] __defaultValidOptimizers = [ 'WorkloadManagement/JobPath', 'WorkloadManagement/JobSanity', 'WorkloadManagement/JobScheduling', 'WorkloadManagement/TaskQueue', ] def initialize( self ): """ Standard constructor """ self.jobDB = JobDB() self.jobLoggingDB = JobLoggingDB() self._optimizingJobs = JobsInTheWorks() self._optimizers = {} self._threadedOptimizers = {} self.am_setOption( "PollingTime", 30 ) return S_OK() def execute( self ): """ Standard Agent module execute method """ #Get jobs from DB result = self.jobDB.selectJobs( { 'Status': self.__jobStates } ) if not result[ 'OK' ]: gLogger.error( "Cannot retrieve jobs in states %s" % self.__jobStates ) return result jobsList = result[ 'Value' ] for i in range( len( jobsList ) ): jobsList[i] = int( jobsList[i] ) jobsList.sort() self.log.info( "Got %s jobs for this iteration" % len( jobsList ) ) if not jobsList: return S_OK() #Check jobs that are already being optimized newJobsList = self._optimizingJobs.addJobs( jobsList ) if not newJobsList: return S_OK() #Get attrs of jobs to be optimized result = self.jobDB.getAttributesForJobList( newJobsList ) if not result[ 'OK' ]: gLogger.error( "Cannot retrieve attributes for %s jobs %s" % len( newJobsList ) ) return result jobsToProcess = result[ 'Value' ] for jobId in jobsToProcess: self.log.info( "== Processing job %s == " % jobId ) jobAttrs = jobsToProcess[ jobId ] result = self.__dispatchJob( jobId, jobAttrs, False ) if not result[ 'OK' ]: gLogger.error( "There was a problem optimizing job", "JID %s: %s" % ( jobId, result[ 'Message' ] ) ) return S_OK() def __dispatchJob( self, jobId, jobAttrs, jobDef, keepOptimizing = True ): """ Decide what to do with the Job """ returnValue = S_OK() if keepOptimizing: result = self.__sendJobToOptimizer( jobId, jobAttrs, jobDef ) if result[ 'OK' ] and result[ 'Value' ]: return S_OK() if not result[ 'OK' ]: returnValue = result gLogger.error( "Could not send job to optimizer\n", "\tJob: %s\n\Message: %s" % ( jobId, result[ 'Message' ] ) ) self._optimizingJobs.deleteJob( jobId ) return returnValue def __sendJobToOptimizer( self, jobId, jobAttrs, jobDef ): """ Send Job to Optimizer queue """ optimizerName = self.__getNextOptimizerName( jobAttrs ) if not optimizerName: return S_OK( False ) if optimizerName not in self.am_getOption( "ValidOptimizers", self.__defaultValidOptimizers ): return S_OK( False ) if optimizerName not in self._threadedOptimizers: to = ThreadedOptimizer( optimizerName, self.am_getModuleParam( 'fullName' ), self.__dispatchJob ) result = to.initialize( self.jobDB, self.jobLoggingDB ) if not result[ 'OK' ]: return S_OK( False ) self._threadedOptimizers[ optimizerName ] = to self._threadedOptimizers[ optimizerName ].optimizeJob( jobId, jobAttrs, jobDef ) return S_OK( True ) def __getNextOptimizerName( self, jobAttrs ): """ Determine next Optimizer """ if jobAttrs[ 'Status' ] == 'Received': optList = [ "JobPath" ] elif jobAttrs[ 'Status' ] == 'Checking': optList = List.fromChar( jobAttrs[ 'MinorStatus' ], "/" ) else: return False if len( optList ) == 1: optList.insert( 0, "WorkloadManagement" ) if len( optList ) > 2: optList[1] = "/".join( optList[1:] ) return "/".join( optList )
class MightyOptimizer( AgentModule ): """ The specific agents must provide the following methods: - initialize() for initial settings - beginExecution() - execute() - the main method called in the agent cycle - endExecution() - finalize() - the graceful exit of the method, this one is usually used for the agent restart """ __jobStates = [ 'Received', 'Checking' ] def initialize( self ): """ Standard constructor """ self.jobDB = JobDB() self.jobLoggingDB = JobLoggingDB() self._optimizers = {} self.am_setOption( "PollingTime", 30 ) return S_OK() def execute( self ): """ The method call by AgentModule on each iteration """ jobTypeCondition = self.am_getOption( "JobTypeRestriction", [] ) jobCond = { 'Status': self.__jobStates } if jobTypeCondition: jobCond[ 'JobType' ] = jobTypeCondition result = self.jobDB.selectJobs( jobCond ) if not result[ 'OK' ]: return result jobsList = result[ 'Value' ] self.log.info( "Got %s jobs for this iteration" % len( jobsList ) ) if not jobsList: return S_OK() result = self.jobDB.getAttributesForJobList( jobsList ) if not result[ 'OK' ]: return result jobsToProcess = result[ 'Value' ] for jobId in jobsToProcess: self.log.info( "== Processing job %s == " % jobId ) jobAttrs = jobsToProcess[ jobId ] jobDef = False jobOptimized = False jobOK = True while not jobOptimized: result = self.optimizeJob( jobId, jobAttrs, jobDef ) if not result[ 'OK' ]: self.log.error( "Optimizer %s error" % jobAttrs[ 'MinorStatus' ], "Job %s: %s" % ( str(jobID), result[ 'Message' ] ) ) jobOK = False break optResult = result[ 'Value' ] jobOptimized = optResult[ 'done' ] if 'jobDef' in optResult: jobDef = optResult[ 'jobDef' ] if jobOK: self.log.info( "Finished optimizing job %s" % jobId ) return S_OK() def optimizeJob( self, jobId, jobAttrs, jobDef ): """ The method call for each Job to be optimized """ #Get the next optimizer result = self._getNextOptimizer( jobAttrs ) if not result[ 'OK' ]: return result optimizer = result[ 'Value' ] if not optimizer: return S_OK( { 'done' : True } ) #If there's no job def then get it if not jobDef: result = optimizer.getJobDefinition( jobId, jobDef ) if not result['OK']: optimizer.setFailedJob( jobId, result[ 'Message' ] ) return result jobDef = result[ 'Value' ] #Does the optimizer require a proxy? shifterEnv = False if optimizer.am_getModuleParam( 'shifterProxy' ): shifterEnv = True result = setupShifterProxyInEnv( optimizer.am_getModuleParam( 'shifterProxy' ), optimizer.am_getShifterProxyLocation() ) if not result[ 'OK' ]: return result #Call the initCycle function result = self.am_secureCall( optimizer.beginExecution, name = "beginExecution" ) if not result[ 'OK' ]: return result #Do the work result = optimizer.optimizeJob( jobId, jobDef[ 'classad' ] ) if not result[ 'OK' ]: return result nextOptimizer = result[ 'Value' ] #If there was a shifter proxy, unset it if shifterEnv: del( os.environ[ 'X509_USER_PROXY' ] ) #Check if the JDL has changed newJDL = jobDef[ 'classad' ].asJDL() if newJDL != jobDef[ 'jdl' ]: jobDef[ 'jdl' ] = newJDL #If there's a new optimizer set it! if nextOptimizer: jobAttrs[ 'Status' ] = 'Checking' jobAttrs[ 'MinorStatus' ] = nextOptimizer return S_OK( { 'done' : False, 'jobDef' : jobDef } ) return S_OK( { 'done' : True, 'jobDef' : jobDef } ) def _getNextOptimizer( self, jobAttrs ): """ Determine next Optimizer in the Path """ if jobAttrs[ 'Status' ] == 'Received': nextOptimizer = "JobPath" else: nextOptimizer = jobAttrs[ 'MinorStatus' ] if nextOptimizer in self.am_getOption( "FilteredOptimizers", "InputData, BKInputData" ): return S_OK( False ) gLogger.info( "Next optimizer for job %s is %s" % ( jobAttrs['JobID'], nextOptimizer ) ) if nextOptimizer not in self._optimizers: result = self.__loadOptimizer( nextOptimizer ) if not result[ 'OK' ]: return result self._optimizers[ nextOptimizer ] = result[ 'Value' ] return S_OK( self._optimizers[ nextOptimizer ] ) @gOptimizerLoadSync def __loadOptimizer( self, optimizerName ): """Need to load an optimizer """ gLogger.info( "Loading optimizer %s" % optimizerName ) try: agentName = "%sAgent" % optimizerName optimizerModule = __import__( 'DIRAC.WorkloadManagementSystem.Agent.%s' % agentName, globals(), locals(), agentName ) optimizerClass = getattr( optimizerModule, agentName ) optimizer = optimizerClass( "WorkloadManagement/%s" % agentName, self.am_getModuleParam( 'fullName' ) ) result = optimizer.am_initialize( self.jobDB, self.jobLoggingDB ) if not result[ 'OK' ]: return S_ERROR( "Can't initialize optimizer %s: %s" % ( optimizerName, result[ 'Message' ] ) ) except Exception, e: gLogger.exception( "LOADERROR" ) return S_ERROR( "Can't load optimizer %s: %s" % ( optimizerName, str( e ) ) ) return S_OK( optimizer )
class MightyOptimizer(AgentModule): """ The specific agents must provide the following methods: - initialize() for initial settings - beginExecution() - execute() - the main method called in the agent cycle - endExecution() - finalize() - the graceful exit of the method, this one is usually used for the agent restart """ __jobStates = ['Received', 'Checking'] def initialize(self): """ Standard constructor """ self.jobDB = JobDB() self.jobLoggingDB = JobLoggingDB() self._optimizers = {} self.am_setOption("PollingTime", 30) return S_OK() def execute(self): """ The method call by AgentModule on each iteration """ jobTypeCondition = self.am_getOption("JobTypeRestriction", []) jobCond = {'Status': self.__jobStates} if jobTypeCondition: jobCond['JobType'] = jobTypeCondition result = self.jobDB.selectJobs(jobCond) if not result['OK']: return result jobsList = result['Value'] self.log.info("Got %s jobs for this iteration" % len(jobsList)) if not jobsList: return S_OK() result = self.jobDB.getAttributesForJobList(jobsList) if not result['OK']: return result jobsToProcess = result['Value'] for jobId in jobsToProcess: self.log.info("== Processing job %s == " % jobId) jobAttrs = jobsToProcess[jobId] jobDef = False jobOptimized = False jobOK = True while not jobOptimized: result = self.optimizeJob(jobId, jobAttrs, jobDef) if not result['OK']: self.log.error( "Optimizer %s error" % jobAttrs['MinorStatus'], "Job %s: %s" % (str(jobId), result['Message'])) jobOK = False break optResult = result['Value'] jobOptimized = optResult['done'] if 'jobDef' in optResult: jobDef = optResult['jobDef'] if jobOK: self.log.info("Finished optimizing job %s" % jobId) return S_OK() def optimizeJob(self, jobId, jobAttrs, jobDef): """ The method call for each Job to be optimized """ #Get the next optimizer result = self._getNextOptimizer(jobAttrs) if not result['OK']: return result optimizer = result['Value'] if not optimizer: return S_OK({'done': True}) #If there's no job def then get it if not jobDef: result = optimizer.getJobDefinition(jobId, jobDef) if not result['OK']: optimizer.setFailedJob(jobId, result['Message']) return result jobDef = result['Value'] #Does the optimizer require a proxy? shifterEnv = False if optimizer.am_getModuleParam('shifterProxy'): shifterEnv = True result = setupShifterProxyInEnv( optimizer.am_getModuleParam('shifterProxy'), optimizer.am_getShifterProxyLocation()) if not result['OK']: return result #Call the initCycle function result = self.am_secureCall(optimizer.beginExecution, name="beginExecution") if not result['OK']: return result #Do the work result = optimizer.optimizeJob(jobId, jobDef['classad']) if not result['OK']: return result nextOptimizer = result['Value'] #If there was a shifter proxy, unset it if shifterEnv: del (os.environ['X509_USER_PROXY']) #Check if the JDL has changed newJDL = jobDef['classad'].asJDL() if newJDL != jobDef['jdl']: jobDef['jdl'] = newJDL #If there's a new optimizer set it! if nextOptimizer: jobAttrs['Status'] = 'Checking' jobAttrs['MinorStatus'] = nextOptimizer return S_OK({'done': False, 'jobDef': jobDef}) return S_OK({'done': True, 'jobDef': jobDef}) def _getNextOptimizer(self, jobAttrs): """ Determine next Optimizer in the Path """ if jobAttrs['Status'] == 'Received': nextOptimizer = "JobPath" else: nextOptimizer = jobAttrs['MinorStatus'] if nextOptimizer in self.am_getOption("FilteredOptimizers", "InputData, BKInputData"): return S_OK(False) gLogger.info("Next optimizer for job %s is %s" % (jobAttrs['JobID'], nextOptimizer)) if nextOptimizer not in self._optimizers: result = self.__loadOptimizer(nextOptimizer) if not result['OK']: return result self._optimizers[nextOptimizer] = result['Value'] return S_OK(self._optimizers[nextOptimizer]) @gOptimizerLoadSync def __loadOptimizer(self, optimizerName): """Need to load an optimizer """ gLogger.info("Loading optimizer %s" % optimizerName) try: agentName = "%sAgent" % optimizerName optimizerModule = __import__( 'DIRAC.WorkloadManagementSystem.Agent.%s' % agentName, globals(), locals(), agentName) optimizerClass = getattr(optimizerModule, agentName) optimizer = optimizerClass("WorkloadManagement/%s" % agentName, self.am_getModuleParam('fullName')) result = optimizer.am_initialize(self.jobDB, self.jobLoggingDB) if not result['OK']: return S_ERROR("Can't initialize optimizer %s: %s" % (optimizerName, result['Message'])) except Exception, e: gLogger.exception("LOADERROR") return S_ERROR("Can't load optimizer %s: %s" % (optimizerName, str(e))) return S_OK(optimizer)
class TaskAgent( AgentModule ): """ The specific agents must provide the following methods: - initialize() for initial settings - beginExecution() - execute() - the main method called in the agent cycle - endExecution() - finalize() - the graceful exit of the method, this one is usually used for the agent restart """ def initialize( self ): self.__taskDB = TaskDB() self.__jobDB = JobDB() return S_OK() def execute( self ): """ Main execution method """ condDict = { 'Status': ['Ready', 'Processing', 'Finished'] } result = self.__taskDB.getTasks( [ 'TaskID', 'Status' ], condDict ) if not result['OK']: return result tasks = result['Value'] self.log.info( '%d tasks will be refreshed' % len(tasks) ) for task in tasks: taskID = task[0] status = task[1] if status in ['Ready', 'Processing', 'Finished']: self.__refreshTask( taskID ) return S_OK() def __refreshTask( self, taskID ): result = self.__refreshTaskStringAttribute( taskID, 'Site' ) if result['OK']: self.log.debug( 'Task %d site is refreshed' % taskID ) else: self.log.error( 'Task %d site refresh failed: %s' % ( taskID, result['Message'] ) ) result = self.__refreshTaskStringAttribute( taskID, 'JobGroup' ) if result['OK']: self.log.debug( 'Task %d job group is refreshed' % taskID ) else: self.log.error( 'Task %d job group refresh failed: %s' % ( taskID, result['Message'] ) ) result = self.__refreshTaskStatus( taskID ) if result['OK']: self.log.debug( 'Task %d status is refreshed' % taskID ) else: self.log.error( 'Task %d status refresh failed: %s' % ( taskID, result['Message'] ) ) ################################################################################ def __getTaskProgress( self, taskID ): result = self.__taskDB.getTaskJobs( taskID ) if not result['OK']: return result jobIDs = result['Value'] result = self.__jobDB.getAttributesForJobList( jobIDs, ['Status'] ) if not result['OK']: return result statuses = result['Value'] progress = { 'Total': 0, 'Done': 0, 'Failed': 0, 'Running': 0, 'Waiting': 0, 'Deleted': 0 } progress['Total'] = len(jobIDs) for jobID in jobIDs: if jobID in statuses: status = statuses[jobID]['Status'] if status in ['Done']: progress['Done'] += 1 elif status in ['Failed', 'Stalled', 'Killed']: progress['Failed'] += 1 elif status in ['Running', 'Completed']: progress['Running'] += 1 else: progress['Waiting'] += 1 else: progress['Deleted'] += 1 return S_OK( progress ) def __analyseTaskStatus( self, progress ): totalJob = progress.get( 'Total', 0 ) runningJob = progress.get( 'Running', 0 ) waitingJob = progress.get( 'Waiting', 0 ) deletedJob = progress.get( 'Deleted', 0 ) status = 'Unknown' if deletedJob == totalJob: status = 'Expired' elif runningJob == 0 and waitingJob == 0: status = 'Finished' else: status = 'Processing' return status def __refreshTaskStatus( self, taskID ): """ Refresh the task status """ # get task progress from the job list result = self.__getTaskProgress( taskID ) if not result['OK']: return result progress = result['Value'] self.log.debug( 'Task %d Progress: %s' % ( taskID, progress ) ) result = self.__taskDB.updateTaskProgress( taskID, progress ) if not result['OK']: return result # get previous task status result = self.__taskDB.getTaskStatus( taskID ) if not result['OK']: return result status = result['Value'] # get current task status from the progress newStatus = self.__analyseTaskStatus( progress ) self.log.debug( 'Task %d new status: %s' % ( taskID, newStatus ) ) if newStatus != status: self.__taskDB.updateTaskStatus( taskID, newStatus, 'Status refreshed' ) if not result['OK']: return result return S_OK( newStatus ) ################################################################################ def __getTaskAttribute( self, taskID, attributeType ): """ Get all attributes of the jobs in the task """ result = self.__taskDB.getTaskJobs( taskID ) if not result['OK']: return result jobIDs = result['Value'] condDict = { 'JobID': jobIDs } result = self.__jobDB.getDistinctJobAttributes( attributeType, condDict ) if not result['OK']: return result attributes = result['Value'] return S_OK( attributes ) def __refreshTaskStringAttribute( self, taskID, attributeType ): """ Refresh the task attribute. The attribute type must be string and seperated by comma """ # get task attibutes from the job list result = self.__getTaskAttribute( taskID, attributeType ) if not result['OK']: return result newAttributes = result['Value'] # get previous task attributes result = self.__taskDB.getTask( taskID, [attributeType] ) if not result['OK']: return result oldAttributes = result['Value'][0].split( ',' ) # check whether there are differences if set( newAttributes ) == set( oldAttributes ): self.log.debug( 'Task %s attribute is the same: %s' % (attributeType, oldAttributes) ) return S_OK( oldAttributes ) # make a combination of old and new attributes attributes = list( set( oldAttributes ) | set( newAttributes ) ) for emptyAttr in [ '', 'ANY', 'Multiple' ]: if emptyAttr in attributes: attributes.remove( emptyAttr ) # generate a new attribute allAttributes = ','.join( attributes ) result = self.__taskDB.updateTask( taskID, [attributeType], [allAttributes] ) if not result['OK']: return result return S_OK( allAttributes )
class TaskAgent(AgentModule): """ The specific agents must provide the following methods: - initialize() for initial settings - beginExecution() - execute() - the main method called in the agent cycle - endExecution() - finalize() - the graceful exit of the method, this one is usually used for the agent restart """ def initialize(self): self.__taskDB = TaskDB() self.__jobDB = JobDB() return S_OK() def execute(self): """ Main execution method """ condDict = {'Status': ['Ready', 'Processing', 'Finished']} result = self.__taskDB.getTasks(['TaskID', 'Status'], condDict) if not result['OK']: return result tasks = result['Value'] self.log.info('%d tasks will be refreshed' % len(tasks)) for task in tasks: taskID = task[0] status = task[1] if status in ['Ready', 'Processing', 'Finished']: self.__refreshTask(taskID) return S_OK() def __refreshTask(self, taskID): result = self.__refreshTaskStringAttribute(taskID, 'Site') if result['OK']: self.log.debug('Task %d site is refreshed' % taskID) else: self.log.error('Task %d site refresh failed: %s' % (taskID, result['Message'])) result = self.__refreshTaskStringAttribute(taskID, 'JobGroup') if result['OK']: self.log.debug('Task %d job group is refreshed' % taskID) else: self.log.error('Task %d job group refresh failed: %s' % (taskID, result['Message'])) result = self.__refreshTaskStatus(taskID) if result['OK']: self.log.debug('Task %d status is refreshed' % taskID) else: self.log.error('Task %d status refresh failed: %s' % (taskID, result['Message'])) ################################################################################ def __getTaskProgress(self, taskID): result = self.__taskDB.getTaskJobs(taskID) if not result['OK']: return result jobIDs = result['Value'] result = self.__jobDB.getAttributesForJobList(jobIDs, ['Status']) if not result['OK']: return result statuses = result['Value'] progress = { 'Total': 0, 'Done': 0, 'Failed': 0, 'Running': 0, 'Waiting': 0, 'Deleted': 0 } progress['Total'] = len(jobIDs) for jobID in jobIDs: if jobID in statuses: status = statuses[jobID]['Status'] if status in ['Done']: progress['Done'] += 1 elif status in ['Failed', 'Stalled', 'Killed']: progress['Failed'] += 1 elif status in ['Running', 'Completed']: progress['Running'] += 1 else: progress['Waiting'] += 1 else: progress['Deleted'] += 1 return S_OK(progress) def __analyseTaskStatus(self, progress): totalJob = progress.get('Total', 0) runningJob = progress.get('Running', 0) waitingJob = progress.get('Waiting', 0) deletedJob = progress.get('Deleted', 0) status = 'Unknown' if deletedJob == totalJob: status = 'Expired' elif runningJob == 0 and waitingJob == 0: status = 'Finished' else: status = 'Processing' return status def __refreshTaskStatus(self, taskID): """ Refresh the task status """ # get task progress from the job list result = self.__getTaskProgress(taskID) if not result['OK']: return result progress = result['Value'] self.log.debug('Task %d Progress: %s' % (taskID, progress)) result = self.__taskDB.updateTaskProgress(taskID, progress) if not result['OK']: return result # get previous task status result = self.__taskDB.getTaskStatus(taskID) if not result['OK']: return result status = result['Value'] # get current task status from the progress newStatus = self.__analyseTaskStatus(progress) self.log.debug('Task %d new status: %s' % (taskID, newStatus)) if newStatus != status: self.__taskDB.updateTaskStatus(taskID, newStatus, 'Status refreshed') if not result['OK']: return result return S_OK(newStatus) ################################################################################ def __getTaskAttribute(self, taskID, attributeType): """ Get all attributes of the jobs in the task """ result = self.__taskDB.getTaskJobs(taskID) if not result['OK']: return result jobIDs = result['Value'] condDict = {'JobID': jobIDs} result = self.__jobDB.getDistinctJobAttributes(attributeType, condDict) if not result['OK']: return result attributes = result['Value'] return S_OK(attributes) def __refreshTaskStringAttribute(self, taskID, attributeType): """ Refresh the task attribute. The attribute type must be string and seperated by comma """ # get task attibutes from the job list result = self.__getTaskAttribute(taskID, attributeType) if not result['OK']: return result newAttributes = result['Value'] # get previous task attributes result = self.__taskDB.getTask(taskID, [attributeType]) if not result['OK']: return result oldAttributes = result['Value'][0].split(',') # check whether there are differences if set(newAttributes) == set(oldAttributes): self.log.debug('Task %s attribute is the same: %s' % (attributeType, oldAttributes)) return S_OK(oldAttributes) # make a combination of old and new attributes attributes = list(set(oldAttributes) | set(newAttributes)) for emptyAttr in ['', 'ANY', 'Multiple']: if emptyAttr in attributes: attributes.remove(emptyAttr) # generate a new attribute allAttributes = ','.join(attributes) result = self.__taskDB.updateTask(taskID, [attributeType], [allAttributes]) if not result['OK']: return result return S_OK(allAttributes)