def __extendTransformation(self, transid): try: tasks = int(self.request.arguments["tasks"][-1]) except KeyError as excp: raise WErr(400, "Missing %s" % excp) gLogger.info("extend %s" % transid) tsClient = TransformationClient() gLogger.info("extendTransformation(%s,%s)" % (transid, tasks)) res = tsClient.extendTransformation(transid, tasks) if res["OK"]: resString = "%s extended by %s successfully" % (transid, tasks) else: resString = "%s failed to extend: %s" % (transid, res["Message"]) callback = {"success": "true", "showResult": [resString], "result": resString} gLogger.info("#######", res) return callback
def __extendTransformation( self, transid ): try: tasks = int( self.request.arguments["tasks"][-1] ) except KeyError as excp: raise WErr( 400, "Missing %s" % excp ) gLogger.info( "extend %s" % transid ) tsClient = TransformationClient() gLogger.info( "extendTransformation(%s,%s)" % ( transid, tasks ) ) res = tsClient.extendTransformation( transid, tasks ) if res["OK"]: resString = "%s extended by %s successfully" % ( transid, tasks ) else: resString = "%s failed to extend: %s" % ( transid, res["Message"] ) callback = {"success":"true", "showResult":[resString], "result":resString} gLogger.info( "#######", res ) return callback
class MCExtensionAgent( AgentModule ): ############################################################################# def initialize( self ): """Sets defaults """ self.transClient = TransformationClient() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'DataManager' ) self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'Simulation'] ) ) gLogger.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) ) self.maxIterationTasks = self.am_getOption( 'TasksPerIteration', 50 ) gLogger.info( "Will create a maximum of %s tasks per iteration" % self.maxIterationTasks ) self.maxFailRate = self.am_getOption( 'MaxFailureRate', 30 ) gLogger.info( "Will not submit tasks for transformations with failure rate greater than %s%s" % ( self.maxFailRate, '%' ) ) self.maxWaitingJobs = self.am_getOption( 'MaxWaitingJobs', 1000 ) gLogger.info( "Will not submit tasks for transformations with more than %d waiting jobs" % self.maxWaitingJobs ) return S_OK() ############################################################################# def execute( self ): """ The MCExtensionAgent execution method.""" self.enableFlag = self.am_getOption( 'EnableFlag', 'True' ) if not self.enableFlag == 'True': self.log.info( 'MCExtensionAgent is disabled by configuration option EnableFlag' ) return S_OK( 'Disabled via CS flag' ) # Obtain the transformations in Cleaning status and remove any mention of the jobs/files res = self.transClient.getTransformations( {'Status':'Active', 'Type':self.transformationTypes} ) if res['OK']: for transDict in res['Value']: transID = transDict['TransformationID'] maxTasks = transDict['MaxNumberOfTasks'] self.extendTransformation( transID, maxTasks ) return S_OK() def extendTransformation( self, transID, maxTasks ): gLogger.info( "Considering extension of transformation %d" % transID ) # Get the current count of tasks submitted for this transformation res = self.transClient.getTransformationTaskStats( transID ) if not res['OK']: if res['Message'] != 'No records found': gLogger.error( "Failed to get task statistics", "%s %s" % ( transID, res['Message'] ) ) return res else: statusDict = {} else: statusDict = res['Value'] gLogger.verbose( "Current task count for transformation %d" % transID ) for status in sortList( statusDict.keys() ): statusCount = statusDict[status] gLogger.verbose( "%s : %s" % ( status.ljust( 20 ), str( statusCount ).rjust( 8 ) ) ) # Determine the number of tasks to be created numberOfTasks = self.calculateTaskNumber( maxTasks, statusDict ) if not numberOfTasks: gLogger.info( "No tasks required for transformation %d" % transID ) return S_OK() # Extend the transformation by the determined number of tasks res = self.transClient.extendTransformation( transID, numberOfTasks ) if not res['OK']: gLogger.error( "Failed to extend transformation", "%s %s" % ( transID, res['Message'] ) ) return res gLogger.info( "Successfully extended transformation %d by %d tasks" % ( transID, numberOfTasks ) ) return S_OK() def calculateTaskNumber( self, maxTasks, statusDict ): done = statusDict.get( 'Done', 0 ) failed = statusDict.get( 'Failed', 0 ) running = statusDict.get( 'Running', 0 ) waiting = statusDict.get( 'Waiting', 0 ) total = statusDict.get( 'Created', 0 ) # If the failure rate is higher than acceptable if ( total != 0 ) and ( ( 100.0 * float( failed ) / float( total ) ) > self.maxFailRate ): return 0 # If we already have enough completed jobs if done >= maxTasks: return 0 if waiting > self.maxWaitingJobs: return 0 numberOfTasks = maxTasks - ( total - failed ) if numberOfTasks > self.maxIterationTasks: numberOfTasks = self.maxIterationTasks return numberOfTasks
class MCExtensionAgent(AgentModule): def __init__(self, *args, **kwargs): ''' c'tor ''' AgentModule.__init__(self, *args, **kwargs) self.transClient = TransformationClient() agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = sorted(agentTSTypes) else: self.transformationTypes = sorted(Operations().getValue( 'Transformations/ExtendableTransfTypes', ['MCSimulation', 'Simulation'])) self.maxIterationTasks = self.am_getOption('TasksPerIteration', 50) self.maxFailRate = self.am_getOption('MaxFailureRate', 30) self.maxWaitingJobs = self.am_getOption('MaxWaitingJobs', 1000) ############################################################################# def initialize(self): '''Sets defaults ''' gLogger.info("Will consider the following transformation types: %s" % str(self.transformationTypes)) gLogger.info("Will create a maximum of %s tasks per iteration" % self.maxIterationTasks) gLogger.info( "Will not submit tasks for transformations with failure rate greater than %s%%" % (self.maxFailRate)) gLogger.info( "Will not submit tasks for transformations with more than %d waiting jobs" % self.maxWaitingJobs) return S_OK() ############################################################################# def execute(self): ''' The MCExtensionAgent execution method. ''' self.enableFlag = self.am_getOption('EnableFlag', 'True') if not self.enableFlag == 'True': self.log.info( 'MCExtensionAgent is disabled by configuration option EnableFlag' ) return S_OK('Disabled via CS flag') # Obtain the transformations in Cleaning status and remove any mention of the jobs/files res = self.transClient.getTransformations({ 'Status': 'Active', 'Type': self.transformationTypes }) if res['OK']: for transDict in res['Value']: transID = transDict['TransformationID'] maxTasks = transDict['MaxNumberOfTasks'] self.extendTransformation(transID, maxTasks) return S_OK() def extendTransformation(self, transID, maxTasks): gLogger.info("Considering extension of transformation %d" % transID) # Get the current count of tasks submitted for this transformation res = self.transClient.getTransformationTaskStats(transID) if not res['OK']: if res['Message'] != 'No records found': gLogger.error("Failed to get task statistics", "%s %s" % (transID, res['Message'])) return res else: statusDict = {} else: statusDict = res['Value'] gLogger.verbose("Current task count for transformation %d" % transID) for status in sorted(statusDict): statusCount = statusDict[status] gLogger.verbose("%s : %s" % (status.ljust(20), str(statusCount).rjust(8))) # Determine the number of tasks to be created numberOfTasks = self._calculateTaskNumber(maxTasks, statusDict) if not numberOfTasks: gLogger.info("No tasks required for transformation %d" % transID) return S_OK() # Extend the transformation by the determined number of tasks res = self.transClient.extendTransformation(transID, numberOfTasks) if not res['OK']: gLogger.error("Failed to extend transformation", "%s %s" % (transID, res['Message'])) return res gLogger.info("Successfully extended transformation %d by %d tasks" % (transID, numberOfTasks)) return S_OK() def _calculateTaskNumber(self, maxTasks, statusDict): ''' Utility function ''' done = statusDict.get('Done', 0) failed = statusDict.get('Failed', 0) waiting = statusDict.get('Waiting', 0) total = statusDict.get('TotalCreated', 0) # If the failure rate is higher than acceptable if (total != 0) and ( (100.0 * float(failed) / float(total)) > self.maxFailRate): return 0 # If we already have enough completed jobs if done >= maxTasks: return 0 if waiting > self.maxWaitingJobs: return 0 numberOfTasks = maxTasks - (total - failed) if numberOfTasks < 0: # this happens when people extend the transformation manually instead of increasing MaxNumberOfTasks return 0 if numberOfTasks > self.maxIterationTasks: numberOfTasks = self.maxIterationTasks return numberOfTasks
class MCExtensionAgent(AgentModule): ############################################################################# def initialize(self): """Sets defaults """ self.transClient = TransformationClient() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption('shifterProxy', 'DataManager') self.transformationTypes = sortList( self.am_getOption('TransformationTypes', ['MCSimulation', 'Simulation'])) gLogger.info("Will consider the following transformation types: %s" % str(self.transformationTypes)) self.maxIterationTasks = self.am_getOption('TasksPerIteration', 50) gLogger.info("Will create a maximum of %s tasks per iteration" % self.maxIterationTasks) self.maxFailRate = self.am_getOption('MaxFailureRate', 30) gLogger.info( "Will not submit tasks for transformations with failure rate greater than %s%s" % (self.maxFailRate, '%')) self.maxWaitingJobs = self.am_getOption('MaxWaitingJobs', 1000) gLogger.info( "Will not submit tasks for transformations with more than %d waiting jobs" % self.maxWaitingJobs) return S_OK() ############################################################################# def execute(self): """ The MCExtensionAgent execution method.""" self.enableFlag = self.am_getOption('EnableFlag', 'True') if not self.enableFlag == 'True': self.log.info( 'TransformationCleaningAgent is disabled by configuration option %s/EnableFlag' % (self.section)) return S_OK('Disabled via CS flag') # Obtain the transformations in Cleaning status and remove any mention of the jobs/files res = self.transClient.getTransformations({ 'Status': 'Active', 'Type': self.transformationTypes }) if res['OK']: for transDict in res['Value']: transID = transDict['TransformationID'] maxTasks = transDict['MaxNumberOfTasks'] self.extendTransformation(transID, maxTasks) return S_OK() def extendTransformation(self, transID, maxTasks): gLogger.info("Considering extension of transformation %d" % transID) # Get the current count of tasks submitted for this transformation res = self.transClient.getTransformationTaskStats(transID) if not res['OK']: if res['Message'] != 'No records found': gLogger.error("Failed to get task statistics", "%s %s" % (transID, res['Message'])) return res else: statusDict = {} else: statusDict = res['Value'] gLogger.verbose("Current task count for transformation %d" % transID) for status in sortList(statusDict.keys()): statusCount = statusDict[status] gLogger.verbose("%s : %s" % (status.ljust(20), str(statusCount).rjust(8))) # Determine the number of tasks to be created numberOfTasks = self.calculateTaskNumber(maxTasks, statusDict) if not numberOfTasks: gLogger.info("No tasks required for transformation %d" % transID) return S_OK() # Extend the transformation by the determined number of tasks res = self.transClient.extendTransformation(transID, numberOfTasks) if not res['OK']: gLogger.error("Failed to extend transformation", "%s %s" % (transID, res['Message'])) return res gLogger.info("Successfully extended transformation %d by %d tasks" % (transID, numberOfTasks)) return S_OK() def calculateTaskNumber(self, maxTasks, statusDict): done = statusDict.get('Done', 0) failed = statusDict.get('Failed', 0) running = statusDict.get('Running', 0) waiting = statusDict.get('Waiting', 0) total = statusDict.get('Created', 0) # If the failure rate is higher than acceptable if (total != 0) and ( (100.0 * float(failed) / float(total)) > self.maxFailRate): return 0 # If we already have enough completed jobs if done >= maxTasks: return 0 if waiting > self.maxWaitingJobs: return 0 numberOfTasks = maxTasks - (total - failed) if numberOfTasks > self.maxIterationTasks: numberOfTasks = self.maxIterationTasks return numberOfTasks