class TransformationAgent( AgentModule ): def initialize( self ): """ standard init """ self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin' ) self.checkCatalog = self.am_getOption( 'CheckCatalog', 'yes' ) self.transformationStatus = self.am_getOption( 'transformationStatus', ['Active', 'Completing', 'Flush'] ) self.maxFiles = self.am_getOption( 'MaxFiles', 5000 ) self.am_setOption( 'shifterProxy', 'ProductionManager' ) self.transDB = TransformationClient( 'TransformationDB' ) self.rm = ReplicaManager() self.unusedFiles = {} return S_OK() def execute( self ): """ get and process the transformations to be processed """ res = self.getTransformations() if not res['OK']: gLogger.info( "execute: Failed to obtain transformations: %s" % res['Message'] ) return S_OK() # Process the transformations for transDict in res['Value']: transID = long( transDict['TransformationID'] ) gLogger.info( "execute: Processing transformation %s." % transID ) startTime = time.time() res = self.processTransformation( transDict ) if not res['OK']: gLogger.info( "execute: Failed to process transformation: %s" % res['Message'] ) else: gLogger.info( "execute: Processed transformation in %.1f seconds" % ( time.time() - startTime ) ) return S_OK() def getTransformations( self ): """ Obtain the transformations to be executed """ transName = self.am_getOption( 'Transformation', 'All' ) if transName == 'All': gLogger.info( "getTransformations: Initializing general purpose agent." ) res = self.transDB.getTransformations( {'Status':self.transformationStatus}, extraParams = True ) if not res['OK']: gLogger.error( "getTransformations: Failed to get transformations: %s" % res['Message'] ) return res transformations = res['Value'] gLogger.info( "getTransformations: Obtained %d transformations to process" % len( transformations ) ) else: gLogger.info( "getTransformations: Initializing for transformation %s." % transName ) res = self.transDB.getTransformation( transName, extraParams = True ) if not res['OK']: gLogger.error( "getTransformations: Failed to get transformation: %s." % res['Message'] ) return res transformations = [res['Value']] return S_OK( transformations ) def processTransformation( self, transDict ): transID = transDict['TransformationID'] # First get the LFNs associated to the transformation res = self.transDB.getTransformationFiles( condDict = {'TransformationID':transID, 'Status':'Unused'} ) if not res['OK']: gLogger.error( "processTransformation: Failed to obtain input data: %s." % res['Message'] ) return res transFiles = res['Value'] lfns = res['LFNs'] if not lfns: gLogger.info( "processTransformation: No 'Unused' files found for transformation." ) if transDict['Status'] == 'Flush': res = self.transDB.setTransformationParameter( transID, 'Status', 'Active' ) if not res['OK']: gLogger.error( "processTransformation: Failed to update transformation status to 'Active': %s." % res['Message'] ) else: gLogger.info( "processTransformation: Updated transformation status to 'Active'." ) return S_OK() #Check if something new happened if len( lfns ) == self.unusedFiles.get( transID, 0 ) and transDict['Status'] != 'Flush': gLogger.info( "processTransformation: No new 'Unused' files found for transformation." ) return S_OK() replicateOrRemove = transDict['Type'].lower() in ["replication", "removal"] # Limit the number of LFNs to be considered for replication or removal as they are treated individually if replicateOrRemove: lfns = lfns[0:self.maxFiles - 1] unusedFiles = len( lfns ) # Check the data is available with replicas res = self.__getDataReplicas( transID, lfns, active = not replicateOrRemove ) if not res['OK']: gLogger.error( "processTransformation: Failed to get data replicas: %s" % res['Message'] ) return res dataReplicas = res['Value'] # Get the plug-in type and create the plug-in object plugin = 'Standard' if transDict.has_key( 'Plugin' ) and transDict['Plugin']: plugin = transDict['Plugin'] gLogger.info( "processTransformation: Processing transformation with '%s' plug-in." % plugin ) res = self.__generatePluginObject( plugin ) if not res['OK']: return res oPlugin = res['Value'] # Get the plug-in and set the required params oPlugin.setParameters( transDict ) oPlugin.setInputData( dataReplicas ) oPlugin.setTransformationFiles( transFiles ) res = oPlugin.generateTasks() if not res['OK']: gLogger.error( "processTransformation: Failed to generate tasks for transformation: %s" % res['Message'] ) return res tasks = res['Value'] # Create the tasks allCreated = True created = 0 for se, lfns in tasks: res = self.transDB.addTaskForTransformation( transID, lfns, se ) if not res['OK']: gLogger.error( "processTransformation: Failed to add task generated by plug-in: %s." % res['Message'] ) allCreated = False else: created += 1 unusedFiles -= len( lfns ) if created: gLogger.info( "processTransformation: Successfully created %d tasks for transformation." % created ) self.unusedFiles[transID] = unusedFiles # If this production is to Flush if transDict['Status'] == 'Flush' and allCreated: res = self.transDB.setTransformationParameter( transID, 'Status', 'Active' ) if not res['OK']: gLogger.error( "processTransformation: Failed to update transformation status to 'Active': %s." % res['Message'] ) else: gLogger.info( "processTransformation: Updated transformation status to 'Active'." ) return S_OK() ###################################################################### # # Internal methods used by the agent # def __generatePluginObject( self, plugin ): """ This simply instantiates the TransformationPlugin class with the relevant plugin name """ try: plugModule = __import__( self.pluginLocation, globals(), locals(), ['TransformationPlugin'] ) except ImportError, e: gLogger.exception( "__generatePluginObject: Failed to import 'TransformationPlugin' %s: %s" % ( plugin, e ) ) return S_ERROR() try: plugin_o = getattr( plugModule, 'TransformationPlugin' )( '%s' % plugin, transClient = self.transDB, replicaManager = self.rm ) return S_OK( plugin_o ) except AttributeError, e: gLogger.exception( "__generatePluginObject: Failed to create %s(): %s." % ( plugin, e ) ) return S_ERROR()
class TransformationAgent(AgentModule): def initialize(self): """ standard init """ self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin') self.checkCatalog = self.am_getOption('CheckCatalog', 'yes') self.transformationStatus = self.am_getOption( 'transformationStatus', ['Active', 'Completing', 'Flush']) self.maxFiles = self.am_getOption('MaxFiles', 5000) self.am_setOption('shifterProxy', 'ProductionManager') self.transDB = TransformationClient('TransformationDB') self.rm = ReplicaManager() self.unusedFiles = {} return S_OK() def execute(self): """ get and process the transformations to be processed """ res = self.getTransformations() if not res['OK']: gLogger.info("execute: Failed to obtain transformations: %s" % res['Message']) return S_OK() # Process the transformations for transDict in res['Value']: transID = long(transDict['TransformationID']) gLogger.info("execute: Processing transformation %s." % transID) startTime = time.time() res = self.processTransformation(transDict) if not res['OK']: gLogger.info("execute: Failed to process transformation: %s" % res['Message']) else: gLogger.info( "execute: Processed transformation in %.1f seconds" % (time.time() - startTime)) return S_OK() def getTransformations(self): """ Obtain the transformations to be executed """ transName = self.am_getOption('Transformation', 'All') if transName == 'All': gLogger.info( "getTransformations: Initializing general purpose agent.") res = self.transDB.getTransformations( {'Status': self.transformationStatus}, extraParams=True) if not res['OK']: gLogger.error( "getTransformations: Failed to get transformations: %s" % res['Message']) return res transformations = res['Value'] gLogger.info( "getTransformations: Obtained %d transformations to process" % len(transformations)) else: gLogger.info( "getTransformations: Initializing for transformation %s." % transName) res = self.transDB.getTransformation(transName, extraParams=True) if not res['OK']: gLogger.error( "getTransformations: Failed to get transformation: %s." % res['Message']) return res transformations = [res['Value']] return S_OK(transformations) def processTransformation(self, transDict): transID = transDict['TransformationID'] # First get the LFNs associated to the transformation res = self.transDB.getTransformationFiles(condDict={ 'TransformationID': transID, 'Status': 'Unused' }) if not res['OK']: gLogger.error( "processTransformation: Failed to obtain input data: %s." % res['Message']) return res transFiles = res['Value'] lfns = res['LFNs'] if not lfns: gLogger.info( "processTransformation: No 'Unused' files found for transformation." ) if transDict['Status'] == 'Flush': res = self.transDB.setTransformationParameter( transID, 'Status', 'Active') if not res['OK']: gLogger.error( "processTransformation: Failed to update transformation status to 'Active': %s." % res['Message']) else: gLogger.info( "processTransformation: Updated transformation status to 'Active'." ) return S_OK() #Check if something new happened if len(lfns) == self.unusedFiles.get( transID, 0) and transDict['Status'] != 'Flush': gLogger.info( "processTransformation: No new 'Unused' files found for transformation." ) return S_OK() replicateOrRemove = transDict['Type'].lower() in [ "replication", "removal" ] # Limit the number of LFNs to be considered for replication or removal as they are treated individually if replicateOrRemove: lfns = lfns[0:self.maxFiles - 1] unusedFiles = len(lfns) # Check the data is available with replicas res = self.__getDataReplicas(transID, lfns, active=not replicateOrRemove) if not res['OK']: gLogger.error( "processTransformation: Failed to get data replicas: %s" % res['Message']) return res dataReplicas = res['Value'] # Get the plug-in type and create the plug-in object plugin = 'Standard' if transDict.has_key('Plugin') and transDict['Plugin']: plugin = transDict['Plugin'] gLogger.info( "processTransformation: Processing transformation with '%s' plug-in." % plugin) res = self.__generatePluginObject(plugin) if not res['OK']: return res oPlugin = res['Value'] # Get the plug-in and set the required params oPlugin.setParameters(transDict) oPlugin.setInputData(dataReplicas) oPlugin.setTransformationFiles(transFiles) res = oPlugin.generateTasks() if not res['OK']: gLogger.error( "processTransformation: Failed to generate tasks for transformation: %s" % res['Message']) return res tasks = res['Value'] # Create the tasks allCreated = True created = 0 for se, lfns in tasks: res = self.transDB.addTaskForTransformation(transID, lfns, se) if not res['OK']: gLogger.error( "processTransformation: Failed to add task generated by plug-in: %s." % res['Message']) allCreated = False else: created += 1 unusedFiles -= len(lfns) if created: gLogger.info( "processTransformation: Successfully created %d tasks for transformation." % created) self.unusedFiles[transID] = unusedFiles # If this production is to Flush if transDict['Status'] == 'Flush' and allCreated: res = self.transDB.setTransformationParameter( transID, 'Status', 'Active') if not res['OK']: gLogger.error( "processTransformation: Failed to update transformation status to 'Active': %s." % res['Message']) else: gLogger.info( "processTransformation: Updated transformation status to 'Active'." ) return S_OK() ###################################################################### # # Internal methods used by the agent # def __generatePluginObject(self, plugin): """ This simply instantiates the TransformationPlugin class with the relevant plugin name """ try: plugModule = __import__(self.pluginLocation, globals(), locals(), ['TransformationPlugin']) except ImportError, e: gLogger.exception( "__generatePluginObject: Failed to import 'TransformationPlugin' %s: %s" % (plugin, e)) return S_ERROR() try: plugin_o = getattr(plugModule, 'TransformationPlugin')( '%s' % plugin, transClient=self.transDB, replicaManager=self.rm) return S_OK(plugin_o) except AttributeError, e: gLogger.exception( "__generatePluginObject: Failed to create %s(): %s." % (plugin, e)) return S_ERROR()
class TransformationAgent(AgentModule): def initialize(self): self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin') self.checkCatalog = self.am_getOption('CheckCatalog', 'yes') # This sets the Default Proxy to used as that defined under # /Operations/Shifter/ProductionManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption('shifterProxy', 'ProductionManager') self.transDB = TransformationClient('TransformationDB') self.rm = ReplicaManager() return S_OK() def execute(self): # Get the transformations to process res = self.getTransformations() if not res['OK']: gLogger.info("%s.execute: Failed to obtain transformations: %s" % (AGENT_NAME, res['Message'])) return S_OK() # Process the transformations for transDict in res['Value']: transID = long(transDict['TransformationID']) gLogger.info("%s.execute: Processing transformation %s." % (AGENT_NAME, transID)) startTime = time.time() res = self.processTransformation(transDict) if not res['OK']: gLogger.info( "%s.execute: Failed to process transformation: %s" % (AGENT_NAME, res['Message'])) else: gLogger.info( "%s.execute: Processed transformation in %.1f seconds" % (AGENT_NAME, time.time() - startTime)) return S_OK() def getTransformations(self): # Obtain the transformations to be executed transName = self.am_getOption('Transformation', 'All') if transName == 'All': gLogger.info( "%s.getTransformations: Initializing general purpose agent." % AGENT_NAME) res = self.transDB.getTransformations( {'Status': ['Active', 'Completing', 'Flush']}, extraParams=True) if not res['OK']: gLogger.error( "%s.getTransformations: Failed to get transformations." % AGENT_NAME, res['Message']) return res transformations = res['Value'] gLogger.info( "%s.getTransformations: Obtained %d transformations to process" % (AGENT_NAME, len(transformations))) else: gLogger.info( "%s.getTransformations: Initializing for transformation %s." % (AGENT_NAME, transName)) res = self.transDB.getTransformation(transName, extraParams=True) if not res['OK']: gLogger.error( "%s.getTransformations: Failed to get transformation." % AGENT_NAME, res['Message']) return res transformations = [res['Value']] return S_OK(transformations) def processTransformation(self, transDict): transID = transDict['TransformationID'] # First get the LFNs associated to the transformation res = self.transDB.getTransformationFiles(condDict={ 'TransformationID': transID, 'Status': 'Unused' }) if not res['OK']: gLogger.error( "%s.processTransformation: Failed to obtain input data." % AGENT_NAME, res['Message']) return res transFiles = res['Value'] lfns = res['LFNs'] if not lfns: gLogger.info( "%s.processTransformation: No 'Unused' files found for transformation." % AGENT_NAME) if transDict['Status'] == 'Flush': res = self.transDB.setTransformationParameter( transID, 'Status', 'Active') if not res['OK']: gLogger.error( "%s.execute: Failed to update transformation status to 'Active'." % AGENT_NAME, res['Message']) else: gLogger.info( "%s.execute: Updated transformation status to 'Active'." % AGENT_NAME) return S_OK() # Check the data is available with replicas res = self.__getDataReplicas(transID, lfns, active=(transDict['Type'].lower() not in ["replication", "removal"])) if not res['OK']: gLogger.error( "%s.processTransformation: Failed to get data replicas" % AGENT_NAME, res['Message']) return res dataReplicas = res['Value'] # Get the plug-in type and create the plug-in object plugin = 'Standard' if transDict.has_key('Plugin') and transDict['Plugin']: plugin = transDict['Plugin'] gLogger.info( "%s.processTransformation: Processing transformation with '%s' plug-in." % (AGENT_NAME, plugin)) res = self.__generatePluginObject(plugin) if not res['OK']: return res oPlugin = res['Value'] # Get the plug-in and set the required params oPlugin.setParameters(transDict) oPlugin.setInputData(dataReplicas) oPlugin.setTransformationFiles(transFiles) res = oPlugin.generateTasks() if not res['OK']: gLogger.error( "%s.processTransformation: Failed to generate tasks for transformation." % AGENT_NAME, res['Message']) return res tasks = res['Value'] # Create the tasks allCreated = True created = 0 for se, lfns in tasks: res = self.transDB.addTaskForTransformation(transID, lfns, se) if not res['OK']: gLogger.error( "%s.processTransformation: Failed to add task generated by plug-in." % AGENT_NAME, res['Message']) allCreated = False else: created += 1 if created: gLogger.info( "%s.processTransformation: Successfully created %d tasks for transformation." % (AGENT_NAME, created)) # If this production is to Flush if transDict['Status'] == 'Flush' and allCreated: res = self.transDB.setTransformationParameter( transID, 'Status', 'Active') if not res['OK']: gLogger.error( "%s.execute: Failed to update transformation status to 'Active'." % AGENT_NAME, res['Message']) else: gLogger.info( "%s.execute: Updated transformation status to 'Active'." % AGENT_NAME) return S_OK() ###################################################################### # # Internal methods used by the agent # def __generatePluginObject(self, plugin): """ This simply instantiates the TransformationPlugin class with the relevant plugin name """ try: plugModule = __import__(self.pluginLocation, globals(), locals(), ['TransformationPlugin']) except Exception, x: gLogger.exception( "%s.__generatePluginObject: Failed to import 'TransformationPlugin'" % AGENT_NAME, '', x) return S_ERROR() try: evalString = "plugModule.TransformationPlugin('%s')" % plugin return S_OK(eval(evalString)) except Exception, x: gLogger.exception( "%s.__generatePluginObject: Failed to create %s()." % (AGENT_NAME, plugin), '', x) return S_ERROR()
class TransformationAgent( AgentModule ): def initialize( self ): self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin' ) self.checkCatalog = self.am_getOption( 'CheckCatalog', 'yes' ) # This sets the Default Proxy to used as that defined under # /Operations/Shifter/ProductionManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'ProductionManager' ) self.transDB = TransformationClient( 'TransformationDB' ) self.rm = ReplicaManager() return S_OK() def execute( self ): # Get the transformations to process res = self.getTransformations() if not res['OK']: gLogger.info( "%s.execute: Failed to obtain transformations: %s" % ( AGENT_NAME, res['Message'] ) ) return S_OK() # Process the transformations for transDict in res['Value']: transID = long( transDict['TransformationID'] ) gLogger.info( "%s.execute: Processing transformation %s." % ( AGENT_NAME, transID ) ) startTime = time.time() res = self.processTransformation( transDict ) if not res['OK']: gLogger.info( "%s.execute: Failed to process transformation: %s" % ( AGENT_NAME, res['Message'] ) ) else: gLogger.info( "%s.execute: Processed transformation in %.1f seconds" % ( AGENT_NAME, time.time() - startTime ) ) return S_OK() def getTransformations( self ): # Obtain the transformations to be executed transName = self.am_getOption( 'Transformation', 'All' ) if transName == 'All': gLogger.info( "%s.getTransformations: Initializing general purpose agent." % AGENT_NAME ) res = self.transDB.getTransformations( {'Status':['Active', 'Completing', 'Flush']}, extraParams = True ) if not res['OK']: gLogger.error( "%s.getTransformations: Failed to get transformations." % AGENT_NAME, res['Message'] ) return res transformations = res['Value'] gLogger.info( "%s.getTransformations: Obtained %d transformations to process" % ( AGENT_NAME, len( transformations ) ) ) else: gLogger.info( "%s.getTransformations: Initializing for transformation %s." % ( AGENT_NAME, transName ) ) res = self.transDB.getTransformation( transName, extraParams = True ) if not res['OK']: gLogger.error( "%s.getTransformations: Failed to get transformation." % AGENT_NAME, res['Message'] ) return res transformations = [res['Value']] return S_OK( transformations ) def processTransformation( self, transDict ): transID = transDict['TransformationID'] # First get the LFNs associated to the transformation res = self.transDB.getTransformationFiles( condDict = {'TransformationID':transID, 'Status':'Unused'} ) if not res['OK']: gLogger.error( "%s.processTransformation: Failed to obtain input data." % AGENT_NAME, res['Message'] ) return res transFiles = res['Value'] lfns = res['LFNs'] if not lfns: gLogger.info( "%s.processTransformation: No 'Unused' files found for transformation." % AGENT_NAME ) if transDict['Status'] == 'Flush': res = self.transDB.setTransformationParameter( transID, 'Status', 'Active' ) if not res['OK']: gLogger.error( "%s.execute: Failed to update transformation status to 'Active'." % AGENT_NAME, res['Message'] ) else: gLogger.info( "%s.execute: Updated transformation status to 'Active'." % AGENT_NAME ) return S_OK() # Check the data is available with replicas res = self.__getDataReplicas( transID, lfns, active = ( transDict['Type'].lower() not in ["replication", "removal"] ) ) if not res['OK']: gLogger.error( "%s.processTransformation: Failed to get data replicas" % AGENT_NAME, res['Message'] ) return res dataReplicas = res['Value'] # Get the plug-in type and create the plug-in object plugin = 'Standard' if transDict.has_key( 'Plugin' ) and transDict['Plugin']: plugin = transDict['Plugin'] gLogger.info( "%s.processTransformation: Processing transformation with '%s' plug-in." % ( AGENT_NAME, plugin ) ) res = self.__generatePluginObject( plugin ) if not res['OK']: return res oPlugin = res['Value'] # Get the plug-in and set the required params oPlugin.setParameters( transDict ) oPlugin.setInputData( dataReplicas ) oPlugin.setTransformationFiles( transFiles ) res = oPlugin.generateTasks() if not res['OK']: gLogger.error( "%s.processTransformation: Failed to generate tasks for transformation." % AGENT_NAME, res['Message'] ) return res tasks = res['Value'] # Create the tasks allCreated = True created = 0 for se, lfns in tasks: res = self.transDB.addTaskForTransformation( transID, lfns, se ) if not res['OK']: gLogger.error( "%s.processTransformation: Failed to add task generated by plug-in." % AGENT_NAME, res['Message'] ) allCreated = False else: created += 1 if created: gLogger.info( "%s.processTransformation: Successfully created %d tasks for transformation." % ( AGENT_NAME, created ) ) # If this production is to Flush if transDict['Status'] == 'Flush' and allCreated: res = self.transDB.setTransformationParameter( transID, 'Status', 'Active' ) if not res['OK']: gLogger.error( "%s.execute: Failed to update transformation status to 'Active'." % AGENT_NAME, res['Message'] ) else: gLogger.info( "%s.execute: Updated transformation status to 'Active'." % AGENT_NAME ) return S_OK() ###################################################################### # # Internal methods used by the agent # def __generatePluginObject( self, plugin ): """ This simply instantiates the TransformationPlugin class with the relevant plugin name """ try: plugModule = __import__( self.pluginLocation, globals(), locals(), ['TransformationPlugin'] ) except Exception, x: gLogger.exception( "%s.__generatePluginObject: Failed to import 'TransformationPlugin'" % AGENT_NAME, '', x ) return S_ERROR() try: evalString = "plugModule.TransformationPlugin('%s')" % plugin return S_OK( eval( evalString ) ) except Exception, x: gLogger.exception( "%s.__generatePluginObject: Failed to create %s()." % ( AGENT_NAME, plugin ), '', x ) return S_ERROR()