class MCExtensionAgent( AgentModule ):
#############################################################################
def initialize( self ):
"""Sets defaults """
self.transClient = TransformationClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'Simulation'] ) )
gLogger.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) )
self.maxIterationTasks = self.am_getOption( 'TasksPerIteration', 50 )
gLogger.info( "Will create a maximum of %s tasks per iteration" % self.maxIterationTasks )
self.maxFailRate = self.am_getOption( 'MaxFailureRate', 30 )
gLogger.info( "Will not submit tasks for transformations with failure rate greater than %s%s" % ( self.maxFailRate, '%' ) )
self.maxWaitingJobs = self.am_getOption( 'MaxWaitingJobs', 1000 )
gLogger.info( "Will not submit tasks for transformations with more than %d waiting jobs" % self.maxWaitingJobs )
return S_OK()
#############################################################################
def execute( self ):
""" The MCExtensionAgent execution method."""
self.enableFlag = self.am_getOption( 'EnableFlag', 'True' )
if not self.enableFlag == 'True':
self.log.info( 'MCExtensionAgent is disabled by configuration option EnableFlag' )
return S_OK( 'Disabled via CS flag' )
# Obtain the transformations in Cleaning status and remove any mention of the jobs/files
res = self.transClient.getTransformations( {'Status':'Active', 'Type':self.transformationTypes} )
if res['OK']:
for transDict in res['Value']:
transID = transDict['TransformationID']
maxTasks = transDict['MaxNumberOfTasks']
self.extendTransformation( transID, maxTasks )
return S_OK()
def extendTransformation( self, transID, maxTasks ):
gLogger.info( "Considering extension of transformation %d" % transID )
# Get the current count of tasks submitted for this transformation
res = self.transClient.getTransformationTaskStats( transID )
if not res['OK']:
if res['Message'] != 'No records found':
gLogger.error( "Failed to get task statistics", "%s %s" % ( transID, res['Message'] ) )
return res
else:
statusDict = {}
else:
statusDict = res['Value']
gLogger.verbose( "Current task count for transformation %d" % transID )
for status in sortList( statusDict.keys() ):
statusCount = statusDict[status]
gLogger.verbose( "%s : %s" % ( status.ljust( 20 ), str( statusCount ).rjust( 8 ) ) )
# Determine the number of tasks to be created
numberOfTasks = self.calculateTaskNumber( maxTasks, statusDict )
if not numberOfTasks:
gLogger.info( "No tasks required for transformation %d" % transID )
return S_OK()
# Extend the transformation by the determined number of tasks
res = self.transClient.extendTransformation( transID, numberOfTasks )
if not res['OK']:
gLogger.error( "Failed to extend transformation", "%s %s" % ( transID, res['Message'] ) )
return res
gLogger.info( "Successfully extended transformation %d by %d tasks" % ( transID, numberOfTasks ) )
return S_OK()
def calculateTaskNumber( self, maxTasks, statusDict ):
done = statusDict.get( 'Done', 0 )
failed = statusDict.get( 'Failed', 0 )
running = statusDict.get( 'Running', 0 )
waiting = statusDict.get( 'Waiting', 0 )
total = statusDict.get( 'Created', 0 )
# If the failure rate is higher than acceptable
if ( total != 0 ) and ( ( 100.0 * float( failed ) / float( total ) ) > self.maxFailRate ):
return 0
# If we already have enough completed jobs
if done >= maxTasks:
return 0
if waiting > self.maxWaitingJobs:
return 0
numberOfTasks = maxTasks - ( total - failed )
if numberOfTasks > self.maxIterationTasks:
numberOfTasks = self.maxIterationTasks
return numberOfTasks
class MCExtensionAgent(AgentModule):
#############################################################################
def initialize(self):
"""Sets defaults """
self.transClient = TransformationClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'DataManager')
self.transformationTypes = sortList(
self.am_getOption('TransformationTypes',
['MCSimulation', 'Simulation']))
gLogger.info("Will consider the following transformation types: %s" %
str(self.transformationTypes))
self.maxIterationTasks = self.am_getOption('TasksPerIteration', 50)
gLogger.info("Will create a maximum of %s tasks per iteration" %
self.maxIterationTasks)
self.maxFailRate = self.am_getOption('MaxFailureRate', 30)
gLogger.info(
"Will not submit tasks for transformations with failure rate greater than %s%s"
% (self.maxFailRate, '%'))
self.maxWaitingJobs = self.am_getOption('MaxWaitingJobs', 1000)
gLogger.info(
"Will not submit tasks for transformations with more than %d waiting jobs"
% self.maxWaitingJobs)
return S_OK()
#############################################################################
def execute(self):
""" The MCExtensionAgent execution method."""
self.enableFlag = self.am_getOption('EnableFlag', 'True')
if not self.enableFlag == 'True':
self.log.info(
'TransformationCleaningAgent is disabled by configuration option %s/EnableFlag'
% (self.section))
return S_OK('Disabled via CS flag')
# Obtain the transformations in Cleaning status and remove any mention of the jobs/files
res = self.transClient.getTransformations({
'Status':
'Active',
'Type':
self.transformationTypes
})
if res['OK']:
for transDict in res['Value']:
transID = transDict['TransformationID']
maxTasks = transDict['MaxNumberOfTasks']
self.extendTransformation(transID, maxTasks)
return S_OK()
def extendTransformation(self, transID, maxTasks):
gLogger.info("Considering extension of transformation %d" % transID)
# Get the current count of tasks submitted for this transformation
res = self.transClient.getTransformationTaskStats(transID)
if not res['OK']:
if res['Message'] != 'No records found':
gLogger.error("Failed to get task statistics",
"%s %s" % (transID, res['Message']))
return res
else:
statusDict = {}
else:
statusDict = res['Value']
gLogger.verbose("Current task count for transformation %d" % transID)
for status in sortList(statusDict.keys()):
statusCount = statusDict[status]
gLogger.verbose("%s : %s" %
(status.ljust(20), str(statusCount).rjust(8)))
# Determine the number of tasks to be created
numberOfTasks = self.calculateTaskNumber(maxTasks, statusDict)
if not numberOfTasks:
gLogger.info("No tasks required for transformation %d" % transID)
return S_OK()
# Extend the transformation by the determined number of tasks
res = self.transClient.extendTransformation(transID, numberOfTasks)
if not res['OK']:
gLogger.error("Failed to extend transformation",
"%s %s" % (transID, res['Message']))
return res
gLogger.info("Successfully extended transformation %d by %d tasks" %
(transID, numberOfTasks))
return S_OK()
def calculateTaskNumber(self, maxTasks, statusDict):
done = statusDict.get('Done', 0)
failed = statusDict.get('Failed', 0)
running = statusDict.get('Running', 0)
waiting = statusDict.get('Waiting', 0)
total = statusDict.get('Created', 0)
# If the failure rate is higher than acceptable
if (total != 0) and (
(100.0 * float(failed) / float(total)) > self.maxFailRate):
return 0
# If we already have enough completed jobs
if done >= maxTasks:
return 0
if waiting > self.maxWaitingJobs:
return 0
numberOfTasks = maxTasks - (total - failed)
if numberOfTasks > self.maxIterationTasks:
numberOfTasks = self.maxIterationTasks
return numberOfTasks
class MCExtensionAgent(AgentModule):
def __init__(self, *args, **kwargs):
''' c'tor
'''
AgentModule.__init__(self, *args, **kwargs)
self.transClient = TransformationClient()
agentTSTypes = self.am_getOption('TransformationTypes', [])
if agentTSTypes:
self.transformationTypes = sorted(agentTSTypes)
else:
self.transformationTypes = sorted(Operations().getValue(
'Transformations/ExtendableTransfTypes',
['MCSimulation', 'Simulation']))
self.maxIterationTasks = self.am_getOption('TasksPerIteration', 50)
self.maxFailRate = self.am_getOption('MaxFailureRate', 30)
self.maxWaitingJobs = self.am_getOption('MaxWaitingJobs', 1000)
#############################################################################
def initialize(self):
'''Sets defaults
'''
gLogger.info("Will consider the following transformation types: %s" %
str(self.transformationTypes))
gLogger.info("Will create a maximum of %s tasks per iteration" %
self.maxIterationTasks)
gLogger.info(
"Will not submit tasks for transformations with failure rate greater than %s%%"
% (self.maxFailRate))
gLogger.info(
"Will not submit tasks for transformations with more than %d waiting jobs"
% self.maxWaitingJobs)
return S_OK()
#############################################################################
def execute(self):
''' The MCExtensionAgent execution method.
'''
self.enableFlag = self.am_getOption('EnableFlag', 'True')
if not self.enableFlag == 'True':
self.log.info(
'MCExtensionAgent is disabled by configuration option EnableFlag'
)
return S_OK('Disabled via CS flag')
# Obtain the transformations in Cleaning status and remove any mention of the jobs/files
res = self.transClient.getTransformations({
'Status':
'Active',
'Type':
self.transformationTypes
})
if res['OK']:
for transDict in res['Value']:
transID = transDict['TransformationID']
maxTasks = transDict['MaxNumberOfTasks']
self.extendTransformation(transID, maxTasks)
return S_OK()
def extendTransformation(self, transID, maxTasks):
gLogger.info("Considering extension of transformation %d" % transID)
# Get the current count of tasks submitted for this transformation
res = self.transClient.getTransformationTaskStats(transID)
if not res['OK']:
if res['Message'] != 'No records found':
gLogger.error("Failed to get task statistics",
"%s %s" % (transID, res['Message']))
return res
else:
statusDict = {}
else:
statusDict = res['Value']
gLogger.verbose("Current task count for transformation %d" % transID)
for status in sorted(statusDict):
statusCount = statusDict[status]
gLogger.verbose("%s : %s" %
(status.ljust(20), str(statusCount).rjust(8)))
# Determine the number of tasks to be created
numberOfTasks = self._calculateTaskNumber(maxTasks, statusDict)
if not numberOfTasks:
gLogger.info("No tasks required for transformation %d" % transID)
return S_OK()
# Extend the transformation by the determined number of tasks
res = self.transClient.extendTransformation(transID, numberOfTasks)
if not res['OK']:
gLogger.error("Failed to extend transformation",
"%s %s" % (transID, res['Message']))
return res
gLogger.info("Successfully extended transformation %d by %d tasks" %
(transID, numberOfTasks))
return S_OK()
def _calculateTaskNumber(self, maxTasks, statusDict):
''' Utility function
'''
done = statusDict.get('Done', 0)
failed = statusDict.get('Failed', 0)
waiting = statusDict.get('Waiting', 0)
total = statusDict.get('TotalCreated', 0)
# If the failure rate is higher than acceptable
if (total != 0) and (
(100.0 * float(failed) / float(total)) > self.maxFailRate):
return 0
# If we already have enough completed jobs
if done >= maxTasks:
return 0
if waiting > self.maxWaitingJobs:
return 0
numberOfTasks = maxTasks - (total - failed)
if numberOfTasks < 0:
# this happens when people extend the transformation manually instead of increasing MaxNumberOfTasks
return 0
if numberOfTasks > self.maxIterationTasks:
numberOfTasks = self.maxIterationTasks
return numberOfTasks
class UpdateTransformationCounters( AgentModule ):
""" This agent is doing what getTransformationSummaryWeb does, but can take the time it needs
"""
def __init__( self, *args, **kwargs ):
""" c'tor
:param self: self reference
:param str agentName: name of agent
:param bool baseAgentName: whatever
:param dict properties: whatever else
"""
AgentModule.__init__( self, *args, **kwargs )
self.transClient = TransformationClient()
self.transfStatuses = self.am_getOption( 'TransformationStatuses', ['Active', 'Stopped'] )
def initialize( self ):
''' Make the necessary initializations
'''
gMonitor.registerActivity( "Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM )
return S_OK()
def execute( self ):
''' Main execution method
'''
gMonitor.addMark( 'Iteration', 1 )
# Get all the transformations
result = self.transClient.getTransformations( condDict = {'Status': self.transfStatuses }, timeout = 320 )
if not result['OK']:
gLogger.error( "UpdateTransformationCounters.execute: Failed to get transformations.", result['Message'] )
return S_OK()
# Process each transformation
jobsStates = self.transClient.getTransformationCountersStatuses( 'Tasks' )['Value']
filesStates = self.transClient.getTransformationCountersStatuses( 'Files' )['Value']
for transDict in result['Value']:
transID = long( transDict['TransformationID'] )
gLogger.debug( "Looking at transformationID %d" % transID )
counterDict = {}
counterDict['TransformationID'] = transID
#Take care of the Tasks' states
gLogger.verbose( "Getting the tasks stats for Transformation %s" % transID )
res = self.transClient.getTransformationTaskStats( transID )
if not res['OK']:
gLogger.warn( "Could not get Transformation Task Stats for transformation %s : %s" % ( transID,
res['Message'] ) )
break
else:
taskDict = {}
if res['Value']:
taskDict = res['Value']
gLogger.verbose( "Got %s tasks dict for transformation %s" % ( str( taskDict ), transID ) )
for state in jobsStates:
counterDict[state] = taskDict.get( state, 0 )
else:
gLogger.warn( "No Task Statuses found" )
break
#Now look for the files' states
gLogger.verbose( "Getting the files stats for Transformation %s" % transID )
res = self.transClient.getTransformationStats( transID )
if not res['OK']:
gLogger.warn( "Could not get Transformation Stats for transformation %s : %s" % ( transID,
res['Message'] ) )
break
else:
fileDict = {}
if res['Value']:
fileDict = res['Value']
gLogger.debug( "Got %s file dict for transformation %s" % ( str( fileDict ), transID ) )
for state in filesStates:
counterDict[state] = fileDict.get( state, 0 )
else:
gLogger.warn( "No File Statuses found" )
break
gLogger.verbose( "Updating the counters for transformation %s" % transID )
res = self.transClient.updateTransformationCounters( counterDict )
if not res['OK']:
gLogger.error( "Failed updating counters for transformation %s: %s" % ( transID, res['Message'] ) )
else:
gLogger.verbose( "Updated the counters of transformation %s" % transID )
return S_OK()