class TransformationAgent( AgentModule ):
def initialize( self ):
""" standard init
"""
self.pluginLocation = self.am_getOption( 'PluginLocation',
'DIRAC.TransformationSystem.Agent.TransformationPlugin' )
self.checkCatalog = self.am_getOption( 'CheckCatalog', 'yes' )
self.transformationStatus = self.am_getOption( 'transformationStatus', ['Active', 'Completing', 'Flush'] )
self.maxFiles = self.am_getOption( 'MaxFiles', 5000 )
self.am_setOption( 'shifterProxy', 'ProductionManager' )
self.transDB = TransformationClient( 'TransformationDB' )
self.rm = ReplicaManager()
self.unusedFiles = {}
return S_OK()
def execute( self ):
""" get and process the transformations to be processed
"""
res = self.getTransformations()
if not res['OK']:
gLogger.info( "execute: Failed to obtain transformations: %s" % res['Message'] )
return S_OK()
# Process the transformations
for transDict in res['Value']:
transID = long( transDict['TransformationID'] )
gLogger.info( "execute: Processing transformation %s." % transID )
startTime = time.time()
res = self.processTransformation( transDict )
if not res['OK']:
gLogger.info( "execute: Failed to process transformation: %s" % res['Message'] )
else:
gLogger.info( "execute: Processed transformation in %.1f seconds" % ( time.time() - startTime ) )
return S_OK()
def getTransformations( self ):
""" Obtain the transformations to be executed
"""
transName = self.am_getOption( 'Transformation', 'All' )
if transName == 'All':
gLogger.info( "getTransformations: Initializing general purpose agent." )
res = self.transDB.getTransformations( {'Status':self.transformationStatus}, extraParams = True )
if not res['OK']:
gLogger.error( "getTransformations: Failed to get transformations: %s" % res['Message'] )
return res
transformations = res['Value']
gLogger.info( "getTransformations: Obtained %d transformations to process" % len( transformations ) )
else:
gLogger.info( "getTransformations: Initializing for transformation %s." % transName )
res = self.transDB.getTransformation( transName, extraParams = True )
if not res['OK']:
gLogger.error( "getTransformations: Failed to get transformation: %s." % res['Message'] )
return res
transformations = [res['Value']]
return S_OK( transformations )
def processTransformation( self, transDict ):
transID = transDict['TransformationID']
# First get the LFNs associated to the transformation
res = self.transDB.getTransformationFiles( condDict = {'TransformationID':transID, 'Status':'Unused'} )
if not res['OK']:
gLogger.error( "processTransformation: Failed to obtain input data: %s." % res['Message'] )
return res
transFiles = res['Value']
lfns = res['LFNs']
if not lfns:
gLogger.info( "processTransformation: No 'Unused' files found for transformation." )
if transDict['Status'] == 'Flush':
res = self.transDB.setTransformationParameter( transID, 'Status', 'Active' )
if not res['OK']:
gLogger.error( "processTransformation: Failed to update transformation status to 'Active': %s." % res['Message'] )
else:
gLogger.info( "processTransformation: Updated transformation status to 'Active'." )
return S_OK()
#Check if something new happened
if len( lfns ) == self.unusedFiles.get( transID, 0 ) and transDict['Status'] != 'Flush':
gLogger.info( "processTransformation: No new 'Unused' files found for transformation." )
return S_OK()
replicateOrRemove = transDict['Type'].lower() in ["replication", "removal"]
# Limit the number of LFNs to be considered for replication or removal as they are treated individually
if replicateOrRemove:
lfns = lfns[0:self.maxFiles - 1]
unusedFiles = len( lfns )
# Check the data is available with replicas
res = self.__getDataReplicas( transID, lfns, active = not replicateOrRemove )
if not res['OK']:
gLogger.error( "processTransformation: Failed to get data replicas: %s" % res['Message'] )
return res
dataReplicas = res['Value']
# Get the plug-in type and create the plug-in object
plugin = 'Standard'
if transDict.has_key( 'Plugin' ) and transDict['Plugin']:
plugin = transDict['Plugin']
gLogger.info( "processTransformation: Processing transformation with '%s' plug-in." % plugin )
res = self.__generatePluginObject( plugin )
if not res['OK']:
return res
oPlugin = res['Value']
# Get the plug-in and set the required params
oPlugin.setParameters( transDict )
oPlugin.setInputData( dataReplicas )
oPlugin.setTransformationFiles( transFiles )
res = oPlugin.generateTasks()
if not res['OK']:
gLogger.error( "processTransformation: Failed to generate tasks for transformation: %s" % res['Message'] )
return res
tasks = res['Value']
# Create the tasks
allCreated = True
created = 0
for se, lfns in tasks:
res = self.transDB.addTaskForTransformation( transID, lfns, se )
if not res['OK']:
gLogger.error( "processTransformation: Failed to add task generated by plug-in: %s." % res['Message'] )
allCreated = False
else:
created += 1
unusedFiles -= len( lfns )
if created:
gLogger.info( "processTransformation: Successfully created %d tasks for transformation." % created )
self.unusedFiles[transID] = unusedFiles
# If this production is to Flush
if transDict['Status'] == 'Flush' and allCreated:
res = self.transDB.setTransformationParameter( transID, 'Status', 'Active' )
if not res['OK']:
gLogger.error( "processTransformation: Failed to update transformation status to 'Active': %s." % res['Message'] )
else:
gLogger.info( "processTransformation: Updated transformation status to 'Active'." )
return S_OK()
######################################################################
#
# Internal methods used by the agent
#
def __generatePluginObject( self, plugin ):
""" This simply instantiates the TransformationPlugin class with the relevant plugin name
"""
try:
plugModule = __import__( self.pluginLocation, globals(), locals(), ['TransformationPlugin'] )
except ImportError, e:
gLogger.exception( "__generatePluginObject: Failed to import 'TransformationPlugin' %s: %s" % ( plugin, e ) )
return S_ERROR()
try:
plugin_o = getattr( plugModule, 'TransformationPlugin' )( '%s' % plugin,
transClient = self.transDB,
replicaManager = self.rm )
return S_OK( plugin_o )
except AttributeError, e:
gLogger.exception( "__generatePluginObject: Failed to create %s(): %s." % ( plugin, e ) )
return S_ERROR()
class TransformationAgent(AgentModule):
def initialize(self):
""" standard init
"""
self.pluginLocation = self.am_getOption(
'PluginLocation',
'DIRAC.TransformationSystem.Agent.TransformationPlugin')
self.checkCatalog = self.am_getOption('CheckCatalog', 'yes')
self.transformationStatus = self.am_getOption(
'transformationStatus', ['Active', 'Completing', 'Flush'])
self.maxFiles = self.am_getOption('MaxFiles', 5000)
self.am_setOption('shifterProxy', 'ProductionManager')
self.transDB = TransformationClient('TransformationDB')
self.rm = ReplicaManager()
self.unusedFiles = {}
return S_OK()
def execute(self):
""" get and process the transformations to be processed
"""
res = self.getTransformations()
if not res['OK']:
gLogger.info("execute: Failed to obtain transformations: %s" %
res['Message'])
return S_OK()
# Process the transformations
for transDict in res['Value']:
transID = long(transDict['TransformationID'])
gLogger.info("execute: Processing transformation %s." % transID)
startTime = time.time()
res = self.processTransformation(transDict)
if not res['OK']:
gLogger.info("execute: Failed to process transformation: %s" %
res['Message'])
else:
gLogger.info(
"execute: Processed transformation in %.1f seconds" %
(time.time() - startTime))
return S_OK()
def getTransformations(self):
""" Obtain the transformations to be executed
"""
transName = self.am_getOption('Transformation', 'All')
if transName == 'All':
gLogger.info(
"getTransformations: Initializing general purpose agent.")
res = self.transDB.getTransformations(
{'Status': self.transformationStatus}, extraParams=True)
if not res['OK']:
gLogger.error(
"getTransformations: Failed to get transformations: %s" %
res['Message'])
return res
transformations = res['Value']
gLogger.info(
"getTransformations: Obtained %d transformations to process" %
len(transformations))
else:
gLogger.info(
"getTransformations: Initializing for transformation %s." %
transName)
res = self.transDB.getTransformation(transName, extraParams=True)
if not res['OK']:
gLogger.error(
"getTransformations: Failed to get transformation: %s." %
res['Message'])
return res
transformations = [res['Value']]
return S_OK(transformations)
def processTransformation(self, transDict):
transID = transDict['TransformationID']
# First get the LFNs associated to the transformation
res = self.transDB.getTransformationFiles(condDict={
'TransformationID': transID,
'Status': 'Unused'
})
if not res['OK']:
gLogger.error(
"processTransformation: Failed to obtain input data: %s." %
res['Message'])
return res
transFiles = res['Value']
lfns = res['LFNs']
if not lfns:
gLogger.info(
"processTransformation: No 'Unused' files found for transformation."
)
if transDict['Status'] == 'Flush':
res = self.transDB.setTransformationParameter(
transID, 'Status', 'Active')
if not res['OK']:
gLogger.error(
"processTransformation: Failed to update transformation status to 'Active': %s."
% res['Message'])
else:
gLogger.info(
"processTransformation: Updated transformation status to 'Active'."
)
return S_OK()
#Check if something new happened
if len(lfns) == self.unusedFiles.get(
transID, 0) and transDict['Status'] != 'Flush':
gLogger.info(
"processTransformation: No new 'Unused' files found for transformation."
)
return S_OK()
replicateOrRemove = transDict['Type'].lower() in [
"replication", "removal"
]
# Limit the number of LFNs to be considered for replication or removal as they are treated individually
if replicateOrRemove:
lfns = lfns[0:self.maxFiles - 1]
unusedFiles = len(lfns)
# Check the data is available with replicas
res = self.__getDataReplicas(transID,
lfns,
active=not replicateOrRemove)
if not res['OK']:
gLogger.error(
"processTransformation: Failed to get data replicas: %s" %
res['Message'])
return res
dataReplicas = res['Value']
# Get the plug-in type and create the plug-in object
plugin = 'Standard'
if transDict.has_key('Plugin') and transDict['Plugin']:
plugin = transDict['Plugin']
gLogger.info(
"processTransformation: Processing transformation with '%s' plug-in."
% plugin)
res = self.__generatePluginObject(plugin)
if not res['OK']:
return res
oPlugin = res['Value']
# Get the plug-in and set the required params
oPlugin.setParameters(transDict)
oPlugin.setInputData(dataReplicas)
oPlugin.setTransformationFiles(transFiles)
res = oPlugin.generateTasks()
if not res['OK']:
gLogger.error(
"processTransformation: Failed to generate tasks for transformation: %s"
% res['Message'])
return res
tasks = res['Value']
# Create the tasks
allCreated = True
created = 0
for se, lfns in tasks:
res = self.transDB.addTaskForTransformation(transID, lfns, se)
if not res['OK']:
gLogger.error(
"processTransformation: Failed to add task generated by plug-in: %s."
% res['Message'])
allCreated = False
else:
created += 1
unusedFiles -= len(lfns)
if created:
gLogger.info(
"processTransformation: Successfully created %d tasks for transformation."
% created)
self.unusedFiles[transID] = unusedFiles
# If this production is to Flush
if transDict['Status'] == 'Flush' and allCreated:
res = self.transDB.setTransformationParameter(
transID, 'Status', 'Active')
if not res['OK']:
gLogger.error(
"processTransformation: Failed to update transformation status to 'Active': %s."
% res['Message'])
else:
gLogger.info(
"processTransformation: Updated transformation status to 'Active'."
)
return S_OK()
######################################################################
#
# Internal methods used by the agent
#
def __generatePluginObject(self, plugin):
""" This simply instantiates the TransformationPlugin class with the relevant plugin name
"""
try:
plugModule = __import__(self.pluginLocation, globals(), locals(),
['TransformationPlugin'])
except ImportError, e:
gLogger.exception(
"__generatePluginObject: Failed to import 'TransformationPlugin' %s: %s"
% (plugin, e))
return S_ERROR()
try:
plugin_o = getattr(plugModule, 'TransformationPlugin')(
'%s' % plugin,
transClient=self.transDB,
replicaManager=self.rm)
return S_OK(plugin_o)
except AttributeError, e:
gLogger.exception(
"__generatePluginObject: Failed to create %s(): %s." %
(plugin, e))
return S_ERROR()
class TransformationAgent(AgentModule):
def initialize(self):
self.pluginLocation = self.am_getOption(
'PluginLocation',
'DIRAC.TransformationSystem.Agent.TransformationPlugin')
self.checkCatalog = self.am_getOption('CheckCatalog', 'yes')
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption('shifterProxy', 'ProductionManager')
self.transDB = TransformationClient('TransformationDB')
self.rm = ReplicaManager()
return S_OK()
def execute(self):
# Get the transformations to process
res = self.getTransformations()
if not res['OK']:
gLogger.info("%s.execute: Failed to obtain transformations: %s" %
(AGENT_NAME, res['Message']))
return S_OK()
# Process the transformations
for transDict in res['Value']:
transID = long(transDict['TransformationID'])
gLogger.info("%s.execute: Processing transformation %s." %
(AGENT_NAME, transID))
startTime = time.time()
res = self.processTransformation(transDict)
if not res['OK']:
gLogger.info(
"%s.execute: Failed to process transformation: %s" %
(AGENT_NAME, res['Message']))
else:
gLogger.info(
"%s.execute: Processed transformation in %.1f seconds" %
(AGENT_NAME, time.time() - startTime))
return S_OK()
def getTransformations(self):
# Obtain the transformations to be executed
transName = self.am_getOption('Transformation', 'All')
if transName == 'All':
gLogger.info(
"%s.getTransformations: Initializing general purpose agent." %
AGENT_NAME)
res = self.transDB.getTransformations(
{'Status': ['Active', 'Completing', 'Flush']},
extraParams=True)
if not res['OK']:
gLogger.error(
"%s.getTransformations: Failed to get transformations." %
AGENT_NAME, res['Message'])
return res
transformations = res['Value']
gLogger.info(
"%s.getTransformations: Obtained %d transformations to process"
% (AGENT_NAME, len(transformations)))
else:
gLogger.info(
"%s.getTransformations: Initializing for transformation %s." %
(AGENT_NAME, transName))
res = self.transDB.getTransformation(transName, extraParams=True)
if not res['OK']:
gLogger.error(
"%s.getTransformations: Failed to get transformation." %
AGENT_NAME, res['Message'])
return res
transformations = [res['Value']]
return S_OK(transformations)
def processTransformation(self, transDict):
transID = transDict['TransformationID']
# First get the LFNs associated to the transformation
res = self.transDB.getTransformationFiles(condDict={
'TransformationID': transID,
'Status': 'Unused'
})
if not res['OK']:
gLogger.error(
"%s.processTransformation: Failed to obtain input data." %
AGENT_NAME, res['Message'])
return res
transFiles = res['Value']
lfns = res['LFNs']
if not lfns:
gLogger.info(
"%s.processTransformation: No 'Unused' files found for transformation."
% AGENT_NAME)
if transDict['Status'] == 'Flush':
res = self.transDB.setTransformationParameter(
transID, 'Status', 'Active')
if not res['OK']:
gLogger.error(
"%s.execute: Failed to update transformation status to 'Active'."
% AGENT_NAME, res['Message'])
else:
gLogger.info(
"%s.execute: Updated transformation status to 'Active'."
% AGENT_NAME)
return S_OK()
# Check the data is available with replicas
res = self.__getDataReplicas(transID,
lfns,
active=(transDict['Type'].lower()
not in ["replication",
"removal"]))
if not res['OK']:
gLogger.error(
"%s.processTransformation: Failed to get data replicas" %
AGENT_NAME, res['Message'])
return res
dataReplicas = res['Value']
# Get the plug-in type and create the plug-in object
plugin = 'Standard'
if transDict.has_key('Plugin') and transDict['Plugin']:
plugin = transDict['Plugin']
gLogger.info(
"%s.processTransformation: Processing transformation with '%s' plug-in."
% (AGENT_NAME, plugin))
res = self.__generatePluginObject(plugin)
if not res['OK']:
return res
oPlugin = res['Value']
# Get the plug-in and set the required params
oPlugin.setParameters(transDict)
oPlugin.setInputData(dataReplicas)
oPlugin.setTransformationFiles(transFiles)
res = oPlugin.generateTasks()
if not res['OK']:
gLogger.error(
"%s.processTransformation: Failed to generate tasks for transformation."
% AGENT_NAME, res['Message'])
return res
tasks = res['Value']
# Create the tasks
allCreated = True
created = 0
for se, lfns in tasks:
res = self.transDB.addTaskForTransformation(transID, lfns, se)
if not res['OK']:
gLogger.error(
"%s.processTransformation: Failed to add task generated by plug-in."
% AGENT_NAME, res['Message'])
allCreated = False
else:
created += 1
if created:
gLogger.info(
"%s.processTransformation: Successfully created %d tasks for transformation."
% (AGENT_NAME, created))
# If this production is to Flush
if transDict['Status'] == 'Flush' and allCreated:
res = self.transDB.setTransformationParameter(
transID, 'Status', 'Active')
if not res['OK']:
gLogger.error(
"%s.execute: Failed to update transformation status to 'Active'."
% AGENT_NAME, res['Message'])
else:
gLogger.info(
"%s.execute: Updated transformation status to 'Active'." %
AGENT_NAME)
return S_OK()
######################################################################
#
# Internal methods used by the agent
#
def __generatePluginObject(self, plugin):
""" This simply instantiates the TransformationPlugin class with the relevant plugin name
"""
try:
plugModule = __import__(self.pluginLocation, globals(), locals(),
['TransformationPlugin'])
except Exception, x:
gLogger.exception(
"%s.__generatePluginObject: Failed to import 'TransformationPlugin'"
% AGENT_NAME, '', x)
return S_ERROR()
try:
evalString = "plugModule.TransformationPlugin('%s')" % plugin
return S_OK(eval(evalString))
except Exception, x:
gLogger.exception(
"%s.__generatePluginObject: Failed to create %s()." %
(AGENT_NAME, plugin), '', x)
return S_ERROR()
class TransformationAgent( AgentModule ):
def initialize( self ):
self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin' )
self.checkCatalog = self.am_getOption( 'CheckCatalog', 'yes' )
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'ProductionManager' )
self.transDB = TransformationClient( 'TransformationDB' )
self.rm = ReplicaManager()
return S_OK()
def execute( self ):
# Get the transformations to process
res = self.getTransformations()
if not res['OK']:
gLogger.info( "%s.execute: Failed to obtain transformations: %s" % ( AGENT_NAME, res['Message'] ) )
return S_OK()
# Process the transformations
for transDict in res['Value']:
transID = long( transDict['TransformationID'] )
gLogger.info( "%s.execute: Processing transformation %s." % ( AGENT_NAME, transID ) )
startTime = time.time()
res = self.processTransformation( transDict )
if not res['OK']:
gLogger.info( "%s.execute: Failed to process transformation: %s" % ( AGENT_NAME, res['Message'] ) )
else:
gLogger.info( "%s.execute: Processed transformation in %.1f seconds" % ( AGENT_NAME, time.time() - startTime ) )
return S_OK()
def getTransformations( self ):
# Obtain the transformations to be executed
transName = self.am_getOption( 'Transformation', 'All' )
if transName == 'All':
gLogger.info( "%s.getTransformations: Initializing general purpose agent." % AGENT_NAME )
res = self.transDB.getTransformations( {'Status':['Active', 'Completing', 'Flush']}, extraParams = True )
if not res['OK']:
gLogger.error( "%s.getTransformations: Failed to get transformations." % AGENT_NAME, res['Message'] )
return res
transformations = res['Value']
gLogger.info( "%s.getTransformations: Obtained %d transformations to process" % ( AGENT_NAME, len( transformations ) ) )
else:
gLogger.info( "%s.getTransformations: Initializing for transformation %s." % ( AGENT_NAME, transName ) )
res = self.transDB.getTransformation( transName, extraParams = True )
if not res['OK']:
gLogger.error( "%s.getTransformations: Failed to get transformation." % AGENT_NAME, res['Message'] )
return res
transformations = [res['Value']]
return S_OK( transformations )
def processTransformation( self, transDict ):
transID = transDict['TransformationID']
# First get the LFNs associated to the transformation
res = self.transDB.getTransformationFiles( condDict = {'TransformationID':transID, 'Status':'Unused'} )
if not res['OK']:
gLogger.error( "%s.processTransformation: Failed to obtain input data." % AGENT_NAME, res['Message'] )
return res
transFiles = res['Value']
lfns = res['LFNs']
if not lfns:
gLogger.info( "%s.processTransformation: No 'Unused' files found for transformation." % AGENT_NAME )
if transDict['Status'] == 'Flush':
res = self.transDB.setTransformationParameter( transID, 'Status', 'Active' )
if not res['OK']:
gLogger.error( "%s.execute: Failed to update transformation status to 'Active'." % AGENT_NAME, res['Message'] )
else:
gLogger.info( "%s.execute: Updated transformation status to 'Active'." % AGENT_NAME )
return S_OK()
# Check the data is available with replicas
res = self.__getDataReplicas( transID, lfns, active = ( transDict['Type'].lower() not in ["replication", "removal"] ) )
if not res['OK']:
gLogger.error( "%s.processTransformation: Failed to get data replicas" % AGENT_NAME, res['Message'] )
return res
dataReplicas = res['Value']
# Get the plug-in type and create the plug-in object
plugin = 'Standard'
if transDict.has_key( 'Plugin' ) and transDict['Plugin']:
plugin = transDict['Plugin']
gLogger.info( "%s.processTransformation: Processing transformation with '%s' plug-in." % ( AGENT_NAME, plugin ) )
res = self.__generatePluginObject( plugin )
if not res['OK']:
return res
oPlugin = res['Value']
# Get the plug-in and set the required params
oPlugin.setParameters( transDict )
oPlugin.setInputData( dataReplicas )
oPlugin.setTransformationFiles( transFiles )
res = oPlugin.generateTasks()
if not res['OK']:
gLogger.error( "%s.processTransformation: Failed to generate tasks for transformation." % AGENT_NAME, res['Message'] )
return res
tasks = res['Value']
# Create the tasks
allCreated = True
created = 0
for se, lfns in tasks:
res = self.transDB.addTaskForTransformation( transID, lfns, se )
if not res['OK']:
gLogger.error( "%s.processTransformation: Failed to add task generated by plug-in." % AGENT_NAME, res['Message'] )
allCreated = False
else:
created += 1
if created:
gLogger.info( "%s.processTransformation: Successfully created %d tasks for transformation." % ( AGENT_NAME, created ) )
# If this production is to Flush
if transDict['Status'] == 'Flush' and allCreated:
res = self.transDB.setTransformationParameter( transID, 'Status', 'Active' )
if not res['OK']:
gLogger.error( "%s.execute: Failed to update transformation status to 'Active'." % AGENT_NAME, res['Message'] )
else:
gLogger.info( "%s.execute: Updated transformation status to 'Active'." % AGENT_NAME )
return S_OK()
######################################################################
#
# Internal methods used by the agent
#
def __generatePluginObject( self, plugin ):
""" This simply instantiates the TransformationPlugin class with the relevant plugin name
"""
try:
plugModule = __import__( self.pluginLocation, globals(), locals(), ['TransformationPlugin'] )
except Exception, x:
gLogger.exception( "%s.__generatePluginObject: Failed to import 'TransformationPlugin'" % AGENT_NAME, '', x )
return S_ERROR()
try:
evalString = "plugModule.TransformationPlugin('%s')" % plugin
return S_OK( eval( evalString ) )
except Exception, x:
gLogger.exception( "%s.__generatePluginObject: Failed to create %s()." % ( AGENT_NAME, plugin ), '', x )
return S_ERROR()
