class ThreadedMightyOptimizer(AgentModule):
"""
The specific agents must provide the following methods:
- initialize() for initial settings
- beginExecution()
- execute() - the main method called in the agent cycle
- endExecution()
- finalize() - the graceful exit of the method, this one is usually used
for the agent restart
"""
__jobStates = ["Received", "Checking"]
__defaultValidOptimizers = [
"WorkloadManagement/JobPath",
"WorkloadManagement/JobSanity",
"WorkloadManagement/JobScheduling",
"WorkloadManagement/TaskQueue",
]
def initialize(self):
""" Standard constructor
"""
self.jobDB = JobDB()
self.jobLoggingDB = JobLoggingDB()
self._optimizingJobs = JobsInTheWorks()
self._optimizers = {}
self._threadedOptimizers = {}
self.am_setOption("PollingTime", 30)
return S_OK()
def execute(self):
""" Standard Agent module execute method
"""
# Get jobs from DB
result = self.jobDB.selectJobs({"Status": self.__jobStates})
if not result["OK"]:
gLogger.error("Cannot retrieve jobs in states %s" % self.__jobStates)
return result
jobsList = result["Value"]
for i in range(len(jobsList)):
jobsList[i] = int(jobsList[i])
jobsList.sort()
self.log.info("Got %s jobs for this iteration" % len(jobsList))
if not jobsList:
return S_OK()
# Check jobs that are already being optimized
newJobsList = self._optimizingJobs.addJobs(jobsList)
if not newJobsList:
return S_OK()
# Get attrs of jobs to be optimized
result = self.jobDB.getAttributesForJobList(newJobsList)
if not result["OK"]:
gLogger.error("Cannot retrieve attributes for %s jobs %s" % len(newJobsList))
return result
jobsToProcess = result["Value"]
for jobId in jobsToProcess:
self.log.info("== Processing job %s == " % jobId)
jobAttrs = jobsToProcess[jobId]
result = self.__dispatchJob(jobId, jobAttrs, False)
if not result["OK"]:
gLogger.error("There was a problem optimizing job", "JID %s: %s" % (jobId, result["Message"]))
return S_OK()
def __dispatchJob(self, jobId, jobAttrs, jobDef, keepOptimizing=True):
""" Decide what to do with the Job
"""
returnValue = S_OK()
if keepOptimizing:
result = self.__sendJobToOptimizer(jobId, jobAttrs, jobDef)
if result["OK"] and result["Value"]:
return S_OK()
if not result["OK"]:
returnValue = result
gLogger.error(
"Could not send job to optimizer\n", "\tJob: %s\n\Message: %s" % (jobId, result["Message"])
)
self._optimizingJobs.deleteJob(jobId)
return returnValue
def __sendJobToOptimizer(self, jobId, jobAttrs, jobDef):
""" Send Job to Optimizer queue
"""
optimizerName = self.__getNextOptimizerName(jobAttrs)
if not optimizerName:
return S_OK(False)
if optimizerName not in self.am_getOption("ValidOptimizers", self.__defaultValidOptimizers):
return S_OK(False)
if optimizerName not in self._threadedOptimizers:
to = ThreadedOptimizer(optimizerName, self.am_getModuleParam("fullName"), self.__dispatchJob)
result = to.initialize(self.jobDB, self.jobLoggingDB)
if not result["OK"]:
return S_OK(False)
self._threadedOptimizers[optimizerName] = to
self._threadedOptimizers[optimizerName].optimizeJob(jobId, jobAttrs, jobDef)
return S_OK(True)
def __getNextOptimizerName(self, jobAttrs):
""" Determine next Optimizer
"""
if jobAttrs["Status"] == "Received":
optList = "JobPath"
elif jobAttrs["Status"] == "Checking":
optList = jobAttrs["MinorStatus"]
else:
return False
optList = List.fromChar(optList, "/")
if len(optList) == 1:
optList.insert(0, "WorkloadManagement")
if len(optList) > 2:
optList[1] = "/".join(optList[1:])
return "/".join(optList)
class ThreadedMightyOptimizer( AgentModule ):
"""
The specific agents must provide the following methods:
- initialize() for initial settings
- beginExecution()
- execute() - the main method called in the agent cycle
- endExecution()
- finalize() - the graceful exit of the method, this one is usually used
for the agent restart
"""
__jobStates = [ 'Received', 'Checking' ]
__defaultValidOptimizers = [ 'WorkloadManagement/JobPath',
'WorkloadManagement/JobSanity',
'WorkloadManagement/JobScheduling',
'WorkloadManagement/TaskQueue',
]
def initialize( self ):
""" Standard constructor
"""
self.jobDB = JobDB()
self.jobLoggingDB = JobLoggingDB()
self._optimizingJobs = JobsInTheWorks()
self._optimizers = {}
self._threadedOptimizers = {}
self.am_setOption( "PollingTime", 30 )
return S_OK()
def execute( self ):
""" Standard Agent module execute method
"""
#Get jobs from DB
result = self.jobDB.selectJobs( { 'Status': self.__jobStates } )
if not result[ 'OK' ]:
gLogger.error( "Cannot retrieve jobs in states %s" % self.__jobStates )
return result
jobsList = result[ 'Value' ]
for i in range( len( jobsList ) ):
jobsList[i] = int( jobsList[i] )
jobsList.sort()
self.log.info( "Got %s jobs for this iteration" % len( jobsList ) )
if not jobsList: return S_OK()
#Check jobs that are already being optimized
newJobsList = self._optimizingJobs.addJobs( jobsList )
if not newJobsList:
return S_OK()
#Get attrs of jobs to be optimized
result = self.jobDB.getAttributesForJobList( newJobsList )
if not result[ 'OK' ]:
gLogger.error( "Cannot retrieve attributes for %s jobs %s" % len( newJobsList ) )
return result
jobsToProcess = result[ 'Value' ]
for jobId in jobsToProcess:
self.log.info( "== Processing job %s == " % jobId )
jobAttrs = jobsToProcess[ jobId ]
result = self.__dispatchJob( jobId, jobAttrs, False )
if not result[ 'OK' ]:
gLogger.error( "There was a problem optimizing job", "JID %s: %s" % ( jobId, result[ 'Message' ] ) )
return S_OK()
def __dispatchJob( self, jobId, jobAttrs, jobDef, keepOptimizing = True ):
""" Decide what to do with the Job
"""
returnValue = S_OK()
if keepOptimizing:
result = self.__sendJobToOptimizer( jobId, jobAttrs, jobDef )
if result[ 'OK' ] and result[ 'Value' ]:
return S_OK()
if not result[ 'OK' ]:
returnValue = result
gLogger.error( "Could not send job to optimizer\n",
"\tJob: %s\n\Message: %s" % ( jobId,
result[ 'Message' ] ) )
self._optimizingJobs.deleteJob( jobId )
return returnValue
def __sendJobToOptimizer( self, jobId, jobAttrs, jobDef ):
""" Send Job to Optimizer queue
"""
optimizerName = self.__getNextOptimizerName( jobAttrs )
if not optimizerName:
return S_OK( False )
if optimizerName not in self.am_getOption( "ValidOptimizers", self.__defaultValidOptimizers ):
return S_OK( False )
if optimizerName not in self._threadedOptimizers:
to = ThreadedOptimizer( optimizerName, self.am_getModuleParam( 'fullName' ),
self.__dispatchJob )
result = to.initialize( self.jobDB, self.jobLoggingDB )
if not result[ 'OK' ]:
return S_OK( False )
self._threadedOptimizers[ optimizerName ] = to
self._threadedOptimizers[ optimizerName ].optimizeJob( jobId, jobAttrs, jobDef )
return S_OK( True )
def __getNextOptimizerName( self, jobAttrs ):
""" Determine next Optimizer
"""
if jobAttrs[ 'Status' ] == 'Received':
optList = [ "JobPath" ]
elif jobAttrs[ 'Status' ] == 'Checking':
optList = List.fromChar( jobAttrs[ 'MinorStatus' ], "/" )
else:
return False
if len( optList ) == 1:
optList.insert( 0, "WorkloadManagement" )
if len( optList ) > 2:
optList[1] = "/".join( optList[1:] )
return "/".join( optList )
class MightyOptimizer( AgentModule ):
"""
The specific agents must provide the following methods:
- initialize() for initial settings
- beginExecution()
- execute() - the main method called in the agent cycle
- endExecution()
- finalize() - the graceful exit of the method, this one is usually used
for the agent restart
"""
__jobStates = [ 'Received', 'Checking' ]
def initialize( self ):
""" Standard constructor
"""
self.jobDB = JobDB()
self.jobLoggingDB = JobLoggingDB()
self._optimizers = {}
self.am_setOption( "PollingTime", 30 )
return S_OK()
def execute( self ):
""" The method call by AgentModule on each iteration
"""
jobTypeCondition = self.am_getOption( "JobTypeRestriction", [] )
jobCond = { 'Status': self.__jobStates }
if jobTypeCondition:
jobCond[ 'JobType' ] = jobTypeCondition
result = self.jobDB.selectJobs( jobCond )
if not result[ 'OK' ]:
return result
jobsList = result[ 'Value' ]
self.log.info( "Got %s jobs for this iteration" % len( jobsList ) )
if not jobsList:
return S_OK()
result = self.jobDB.getAttributesForJobList( jobsList )
if not result[ 'OK' ]:
return result
jobsToProcess = result[ 'Value' ]
for jobId in jobsToProcess:
self.log.info( "== Processing job %s == " % jobId )
jobAttrs = jobsToProcess[ jobId ]
jobDef = False
jobOptimized = False
jobOK = True
while not jobOptimized:
result = self.optimizeJob( jobId, jobAttrs, jobDef )
if not result[ 'OK' ]:
self.log.error( "Optimizer %s error" % jobAttrs[ 'MinorStatus' ], "Job %s: %s" % ( str(jobID), result[ 'Message' ] ) )
jobOK = False
break
optResult = result[ 'Value' ]
jobOptimized = optResult[ 'done' ]
if 'jobDef' in optResult:
jobDef = optResult[ 'jobDef' ]
if jobOK:
self.log.info( "Finished optimizing job %s" % jobId )
return S_OK()
def optimizeJob( self, jobId, jobAttrs, jobDef ):
""" The method call for each Job to be optimized
"""
#Get the next optimizer
result = self._getNextOptimizer( jobAttrs )
if not result[ 'OK' ]:
return result
optimizer = result[ 'Value' ]
if not optimizer:
return S_OK( { 'done' : True } )
#If there's no job def then get it
if not jobDef:
result = optimizer.getJobDefinition( jobId, jobDef )
if not result['OK']:
optimizer.setFailedJob( jobId, result[ 'Message' ] )
return result
jobDef = result[ 'Value' ]
#Does the optimizer require a proxy?
shifterEnv = False
if optimizer.am_getModuleParam( 'shifterProxy' ):
shifterEnv = True
result = setupShifterProxyInEnv( optimizer.am_getModuleParam( 'shifterProxy' ),
optimizer.am_getShifterProxyLocation() )
if not result[ 'OK' ]:
return result
#Call the initCycle function
result = self.am_secureCall( optimizer.beginExecution, name = "beginExecution" )
if not result[ 'OK' ]:
return result
#Do the work
result = optimizer.optimizeJob( jobId, jobDef[ 'classad' ] )
if not result[ 'OK' ]:
return result
nextOptimizer = result[ 'Value' ]
#If there was a shifter proxy, unset it
if shifterEnv:
del( os.environ[ 'X509_USER_PROXY' ] )
#Check if the JDL has changed
newJDL = jobDef[ 'classad' ].asJDL()
if newJDL != jobDef[ 'jdl' ]:
jobDef[ 'jdl' ] = newJDL
#If there's a new optimizer set it!
if nextOptimizer:
jobAttrs[ 'Status' ] = 'Checking'
jobAttrs[ 'MinorStatus' ] = nextOptimizer
return S_OK( { 'done' : False, 'jobDef' : jobDef } )
return S_OK( { 'done' : True, 'jobDef' : jobDef } )
def _getNextOptimizer( self, jobAttrs ):
""" Determine next Optimizer in the Path
"""
if jobAttrs[ 'Status' ] == 'Received':
nextOptimizer = "JobPath"
else:
nextOptimizer = jobAttrs[ 'MinorStatus' ]
if nextOptimizer in self.am_getOption( "FilteredOptimizers", "InputData, BKInputData" ):
return S_OK( False )
gLogger.info( "Next optimizer for job %s is %s" % ( jobAttrs['JobID'], nextOptimizer ) )
if nextOptimizer not in self._optimizers:
result = self.__loadOptimizer( nextOptimizer )
if not result[ 'OK' ]:
return result
self._optimizers[ nextOptimizer ] = result[ 'Value' ]
return S_OK( self._optimizers[ nextOptimizer ] )
@gOptimizerLoadSync
def __loadOptimizer( self, optimizerName ):
"""Need to load an optimizer
"""
gLogger.info( "Loading optimizer %s" % optimizerName )
try:
agentName = "%sAgent" % optimizerName
optimizerModule = __import__( 'DIRAC.WorkloadManagementSystem.Agent.%s' % agentName,
globals(),
locals(), agentName )
optimizerClass = getattr( optimizerModule, agentName )
optimizer = optimizerClass( "WorkloadManagement/%s" % agentName, self.am_getModuleParam( 'fullName' ) )
result = optimizer.am_initialize( self.jobDB, self.jobLoggingDB )
if not result[ 'OK' ]:
return S_ERROR( "Can't initialize optimizer %s: %s" % ( optimizerName, result[ 'Message' ] ) )
except Exception, e:
gLogger.exception( "LOADERROR" )
return S_ERROR( "Can't load optimizer %s: %s" % ( optimizerName, str( e ) ) )
return S_OK( optimizer )
class MightyOptimizer(AgentModule):
"""
The specific agents must provide the following methods:
- initialize() for initial settings
- beginExecution()
- execute() - the main method called in the agent cycle
- endExecution()
- finalize() - the graceful exit of the method, this one is usually used
for the agent restart
"""
__jobStates = ['Received', 'Checking']
def initialize(self):
""" Standard constructor
"""
self.jobDB = JobDB()
self.jobLoggingDB = JobLoggingDB()
self._optimizers = {}
self.am_setOption("PollingTime", 30)
return S_OK()
def execute(self):
""" The method call by AgentModule on each iteration
"""
jobTypeCondition = self.am_getOption("JobTypeRestriction", [])
jobCond = {'Status': self.__jobStates}
if jobTypeCondition:
jobCond['JobType'] = jobTypeCondition
result = self.jobDB.selectJobs(jobCond)
if not result['OK']:
return result
jobsList = result['Value']
self.log.info("Got %s jobs for this iteration" % len(jobsList))
if not jobsList:
return S_OK()
result = self.jobDB.getAttributesForJobList(jobsList)
if not result['OK']:
return result
jobsToProcess = result['Value']
for jobId in jobsToProcess:
self.log.info("== Processing job %s == " % jobId)
jobAttrs = jobsToProcess[jobId]
jobDef = False
jobOptimized = False
jobOK = True
while not jobOptimized:
result = self.optimizeJob(jobId, jobAttrs, jobDef)
if not result['OK']:
self.log.error(
"Optimizer %s error" % jobAttrs['MinorStatus'],
"Job %s: %s" % (str(jobId), result['Message']))
jobOK = False
break
optResult = result['Value']
jobOptimized = optResult['done']
if 'jobDef' in optResult:
jobDef = optResult['jobDef']
if jobOK:
self.log.info("Finished optimizing job %s" % jobId)
return S_OK()
def optimizeJob(self, jobId, jobAttrs, jobDef):
""" The method call for each Job to be optimized
"""
#Get the next optimizer
result = self._getNextOptimizer(jobAttrs)
if not result['OK']:
return result
optimizer = result['Value']
if not optimizer:
return S_OK({'done': True})
#If there's no job def then get it
if not jobDef:
result = optimizer.getJobDefinition(jobId, jobDef)
if not result['OK']:
optimizer.setFailedJob(jobId, result['Message'])
return result
jobDef = result['Value']
#Does the optimizer require a proxy?
shifterEnv = False
if optimizer.am_getModuleParam('shifterProxy'):
shifterEnv = True
result = setupShifterProxyInEnv(
optimizer.am_getModuleParam('shifterProxy'),
optimizer.am_getShifterProxyLocation())
if not result['OK']:
return result
#Call the initCycle function
result = self.am_secureCall(optimizer.beginExecution,
name="beginExecution")
if not result['OK']:
return result
#Do the work
result = optimizer.optimizeJob(jobId, jobDef['classad'])
if not result['OK']:
return result
nextOptimizer = result['Value']
#If there was a shifter proxy, unset it
if shifterEnv:
del (os.environ['X509_USER_PROXY'])
#Check if the JDL has changed
newJDL = jobDef['classad'].asJDL()
if newJDL != jobDef['jdl']:
jobDef['jdl'] = newJDL
#If there's a new optimizer set it!
if nextOptimizer:
jobAttrs['Status'] = 'Checking'
jobAttrs['MinorStatus'] = nextOptimizer
return S_OK({'done': False, 'jobDef': jobDef})
return S_OK({'done': True, 'jobDef': jobDef})
def _getNextOptimizer(self, jobAttrs):
""" Determine next Optimizer in the Path
"""
if jobAttrs['Status'] == 'Received':
nextOptimizer = "JobPath"
else:
nextOptimizer = jobAttrs['MinorStatus']
if nextOptimizer in self.am_getOption("FilteredOptimizers",
"InputData, BKInputData"):
return S_OK(False)
gLogger.info("Next optimizer for job %s is %s" %
(jobAttrs['JobID'], nextOptimizer))
if nextOptimizer not in self._optimizers:
result = self.__loadOptimizer(nextOptimizer)
if not result['OK']:
return result
self._optimizers[nextOptimizer] = result['Value']
return S_OK(self._optimizers[nextOptimizer])
@gOptimizerLoadSync
def __loadOptimizer(self, optimizerName):
"""Need to load an optimizer
"""
gLogger.info("Loading optimizer %s" % optimizerName)
try:
agentName = "%sAgent" % optimizerName
optimizerModule = __import__(
'DIRAC.WorkloadManagementSystem.Agent.%s' % agentName,
globals(), locals(), agentName)
optimizerClass = getattr(optimizerModule, agentName)
optimizer = optimizerClass("WorkloadManagement/%s" % agentName,
self.am_getModuleParam('fullName'))
result = optimizer.am_initialize(self.jobDB, self.jobLoggingDB)
if not result['OK']:
return S_ERROR("Can't initialize optimizer %s: %s" %
(optimizerName, result['Message']))
except Exception, e:
gLogger.exception("LOADERROR")
return S_ERROR("Can't load optimizer %s: %s" %
(optimizerName, str(e)))
return S_OK(optimizer)
class TaskAgent( AgentModule ):
"""
The specific agents must provide the following methods:
- initialize() for initial settings
- beginExecution()
- execute() - the main method called in the agent cycle
- endExecution()
- finalize() - the graceful exit of the method, this one is usually used
for the agent restart
"""
def initialize( self ):
self.__taskDB = TaskDB()
self.__jobDB = JobDB()
return S_OK()
def execute( self ):
""" Main execution method
"""
condDict = { 'Status': ['Ready', 'Processing', 'Finished'] }
result = self.__taskDB.getTasks( [ 'TaskID', 'Status' ], condDict )
if not result['OK']:
return result
tasks = result['Value']
self.log.info( '%d tasks will be refreshed' % len(tasks) )
for task in tasks:
taskID = task[0]
status = task[1]
if status in ['Ready', 'Processing', 'Finished']:
self.__refreshTask( taskID )
return S_OK()
def __refreshTask( self, taskID ):
result = self.__refreshTaskStringAttribute( taskID, 'Site' )
if result['OK']:
self.log.debug( 'Task %d site is refreshed' % taskID )
else:
self.log.error( 'Task %d site refresh failed: %s' % ( taskID, result['Message'] ) )
result = self.__refreshTaskStringAttribute( taskID, 'JobGroup' )
if result['OK']:
self.log.debug( 'Task %d job group is refreshed' % taskID )
else:
self.log.error( 'Task %d job group refresh failed: %s' % ( taskID, result['Message'] ) )
result = self.__refreshTaskStatus( taskID )
if result['OK']:
self.log.debug( 'Task %d status is refreshed' % taskID )
else:
self.log.error( 'Task %d status refresh failed: %s' % ( taskID, result['Message'] ) )
################################################################################
def __getTaskProgress( self, taskID ):
result = self.__taskDB.getTaskJobs( taskID )
if not result['OK']:
return result
jobIDs = result['Value']
result = self.__jobDB.getAttributesForJobList( jobIDs, ['Status'] )
if not result['OK']:
return result
statuses = result['Value']
progress = { 'Total': 0, 'Done': 0, 'Failed': 0, 'Running': 0, 'Waiting': 0, 'Deleted': 0 }
progress['Total'] = len(jobIDs)
for jobID in jobIDs:
if jobID in statuses:
status = statuses[jobID]['Status']
if status in ['Done']:
progress['Done'] += 1
elif status in ['Failed', 'Stalled', 'Killed']:
progress['Failed'] += 1
elif status in ['Running', 'Completed']:
progress['Running'] += 1
else:
progress['Waiting'] += 1
else:
progress['Deleted'] += 1
return S_OK( progress )
def __analyseTaskStatus( self, progress ):
totalJob = progress.get( 'Total', 0 )
runningJob = progress.get( 'Running', 0 )
waitingJob = progress.get( 'Waiting', 0 )
deletedJob = progress.get( 'Deleted', 0 )
status = 'Unknown'
if deletedJob == totalJob:
status = 'Expired'
elif runningJob == 0 and waitingJob == 0:
status = 'Finished'
else:
status = 'Processing'
return status
def __refreshTaskStatus( self, taskID ):
""" Refresh the task status
"""
# get task progress from the job list
result = self.__getTaskProgress( taskID )
if not result['OK']:
return result
progress = result['Value']
self.log.debug( 'Task %d Progress: %s' % ( taskID, progress ) )
result = self.__taskDB.updateTaskProgress( taskID, progress )
if not result['OK']:
return result
# get previous task status
result = self.__taskDB.getTaskStatus( taskID )
if not result['OK']:
return result
status = result['Value']
# get current task status from the progress
newStatus = self.__analyseTaskStatus( progress )
self.log.debug( 'Task %d new status: %s' % ( taskID, newStatus ) )
if newStatus != status:
self.__taskDB.updateTaskStatus( taskID, newStatus, 'Status refreshed' )
if not result['OK']:
return result
return S_OK( newStatus )
################################################################################
def __getTaskAttribute( self, taskID, attributeType ):
""" Get all attributes of the jobs in the task
"""
result = self.__taskDB.getTaskJobs( taskID )
if not result['OK']:
return result
jobIDs = result['Value']
condDict = { 'JobID': jobIDs }
result = self.__jobDB.getDistinctJobAttributes( attributeType, condDict )
if not result['OK']:
return result
attributes = result['Value']
return S_OK( attributes )
def __refreshTaskStringAttribute( self, taskID, attributeType ):
""" Refresh the task attribute. The attribute type must be string and seperated by comma
"""
# get task attibutes from the job list
result = self.__getTaskAttribute( taskID, attributeType )
if not result['OK']:
return result
newAttributes = result['Value']
# get previous task attributes
result = self.__taskDB.getTask( taskID, [attributeType] )
if not result['OK']:
return result
oldAttributes = result['Value'][0].split( ',' )
# check whether there are differences
if set( newAttributes ) == set( oldAttributes ):
self.log.debug( 'Task %s attribute is the same: %s' % (attributeType, oldAttributes) )
return S_OK( oldAttributes )
# make a combination of old and new attributes
attributes = list( set( oldAttributes ) | set( newAttributes ) )
for emptyAttr in [ '', 'ANY', 'Multiple' ]:
if emptyAttr in attributes:
attributes.remove( emptyAttr )
# generate a new attribute
allAttributes = ','.join( attributes )
result = self.__taskDB.updateTask( taskID, [attributeType], [allAttributes] )
if not result['OK']:
return result
return S_OK( allAttributes )
class TaskAgent(AgentModule):
"""
The specific agents must provide the following methods:
- initialize() for initial settings
- beginExecution()
- execute() - the main method called in the agent cycle
- endExecution()
- finalize() - the graceful exit of the method, this one is usually used
for the agent restart
"""
def initialize(self):
self.__taskDB = TaskDB()
self.__jobDB = JobDB()
return S_OK()
def execute(self):
""" Main execution method
"""
condDict = {'Status': ['Ready', 'Processing', 'Finished']}
result = self.__taskDB.getTasks(['TaskID', 'Status'], condDict)
if not result['OK']:
return result
tasks = result['Value']
self.log.info('%d tasks will be refreshed' % len(tasks))
for task in tasks:
taskID = task[0]
status = task[1]
if status in ['Ready', 'Processing', 'Finished']:
self.__refreshTask(taskID)
return S_OK()
def __refreshTask(self, taskID):
result = self.__refreshTaskStringAttribute(taskID, 'Site')
if result['OK']:
self.log.debug('Task %d site is refreshed' % taskID)
else:
self.log.error('Task %d site refresh failed: %s' %
(taskID, result['Message']))
result = self.__refreshTaskStringAttribute(taskID, 'JobGroup')
if result['OK']:
self.log.debug('Task %d job group is refreshed' % taskID)
else:
self.log.error('Task %d job group refresh failed: %s' %
(taskID, result['Message']))
result = self.__refreshTaskStatus(taskID)
if result['OK']:
self.log.debug('Task %d status is refreshed' % taskID)
else:
self.log.error('Task %d status refresh failed: %s' %
(taskID, result['Message']))
################################################################################
def __getTaskProgress(self, taskID):
result = self.__taskDB.getTaskJobs(taskID)
if not result['OK']:
return result
jobIDs = result['Value']
result = self.__jobDB.getAttributesForJobList(jobIDs, ['Status'])
if not result['OK']:
return result
statuses = result['Value']
progress = {
'Total': 0,
'Done': 0,
'Failed': 0,
'Running': 0,
'Waiting': 0,
'Deleted': 0
}
progress['Total'] = len(jobIDs)
for jobID in jobIDs:
if jobID in statuses:
status = statuses[jobID]['Status']
if status in ['Done']:
progress['Done'] += 1
elif status in ['Failed', 'Stalled', 'Killed']:
progress['Failed'] += 1
elif status in ['Running', 'Completed']:
progress['Running'] += 1
else:
progress['Waiting'] += 1
else:
progress['Deleted'] += 1
return S_OK(progress)
def __analyseTaskStatus(self, progress):
totalJob = progress.get('Total', 0)
runningJob = progress.get('Running', 0)
waitingJob = progress.get('Waiting', 0)
deletedJob = progress.get('Deleted', 0)
status = 'Unknown'
if deletedJob == totalJob:
status = 'Expired'
elif runningJob == 0 and waitingJob == 0:
status = 'Finished'
else:
status = 'Processing'
return status
def __refreshTaskStatus(self, taskID):
""" Refresh the task status
"""
# get task progress from the job list
result = self.__getTaskProgress(taskID)
if not result['OK']:
return result
progress = result['Value']
self.log.debug('Task %d Progress: %s' % (taskID, progress))
result = self.__taskDB.updateTaskProgress(taskID, progress)
if not result['OK']:
return result
# get previous task status
result = self.__taskDB.getTaskStatus(taskID)
if not result['OK']:
return result
status = result['Value']
# get current task status from the progress
newStatus = self.__analyseTaskStatus(progress)
self.log.debug('Task %d new status: %s' % (taskID, newStatus))
if newStatus != status:
self.__taskDB.updateTaskStatus(taskID, newStatus,
'Status refreshed')
if not result['OK']:
return result
return S_OK(newStatus)
################################################################################
def __getTaskAttribute(self, taskID, attributeType):
""" Get all attributes of the jobs in the task
"""
result = self.__taskDB.getTaskJobs(taskID)
if not result['OK']:
return result
jobIDs = result['Value']
condDict = {'JobID': jobIDs}
result = self.__jobDB.getDistinctJobAttributes(attributeType, condDict)
if not result['OK']:
return result
attributes = result['Value']
return S_OK(attributes)
def __refreshTaskStringAttribute(self, taskID, attributeType):
""" Refresh the task attribute. The attribute type must be string and seperated by comma
"""
# get task attibutes from the job list
result = self.__getTaskAttribute(taskID, attributeType)
if not result['OK']:
return result
newAttributes = result['Value']
# get previous task attributes
result = self.__taskDB.getTask(taskID, [attributeType])
if not result['OK']:
return result
oldAttributes = result['Value'][0].split(',')
# check whether there are differences
if set(newAttributes) == set(oldAttributes):
self.log.debug('Task %s attribute is the same: %s' %
(attributeType, oldAttributes))
return S_OK(oldAttributes)
# make a combination of old and new attributes
attributes = list(set(oldAttributes) | set(newAttributes))
for emptyAttr in ['', 'ANY', 'Multiple']:
if emptyAttr in attributes:
attributes.remove(emptyAttr)
# generate a new attribute
allAttributes = ','.join(attributes)
result = self.__taskDB.updateTask(taskID, [attributeType],
[allAttributes])
if not result['OK']:
return result
return S_OK(allAttributes)