def __getLoggingInfo(self, transid):
callback = {}
tsClient = TransformationClient()
result = tsClient.getTransformationLogging(transid)
if result["OK"]:
result = result["Value"]
if len(result) > 0:
callback = []
resultUser = gConfig.getSections("/Security/Users")
if resultUser["OK"]:
users = resultUser["Value"]
dndb = {}
for j in users:
dndb[gConfig.getValue("/Security/Users/%s/DN" % j)] = j
else:
dndb = {}
for i in result:
DN = i["AuthorDN"]
if DN in dndb:
i["AuthorDN"] = dndb[DN]
else:
i["AuthorDN"] = DN # "Owner Unknown"
date = Time.toString(i["MessageDate"])
callback.append([i["Message"], date, i["AuthorDN"]])
callback = {"success": "true", "result": callback}
else:
callback = {"success": "false", "error": "Nothing to display"}
else:
callback = {"success": "false", "error": result["Message"]}
gLogger.info("PRODUCTION LOG:", id)
return callback
class FileReport( object ):
""" A stateful object for reporting to TransformationDB
"""
def __init__( self, server = 'Transformation/TransformationManager' ):
""" c'tor
self.transClient is a TransformationClient object
"""
self.transClient = TransformationClient()
self.transClient.setServer( server )
self.statusDict = {}
self.transformation = None
self.force = False
def setFileStatus( self, transformation, lfn, status, sendFlag = False ):
""" Set file status in the context of the given transformation
"""
if not self.transformation:
self.transformation = transformation
self.statusDict[lfn] = status
if sendFlag:
return self.commit()
return S_OK()
def setCommonStatus( self, status ):
""" Set common status for all files in the internal cache
"""
for lfn in self.statusDict.keys():
self.statusDict[lfn] = status
return S_OK()
def getFiles( self ):
""" Get the statuses of the files already accumulated in the FileReport object
"""
return copy.deepcopy( self.statusDict )
def commit( self ):
""" Commit pending file status update records
"""
if not self.statusDict:
return S_OK( {} )
result = self.transClient.setFileStatusForTransformation( self.transformation, self.statusDict, force = self.force )
if result['OK']:
self.statusDict = {}
return result
def generateForwardDISET( self ):
""" Commit the accumulated records and generate request eventually
"""
result = self.commit()
commitOp = None
if not result['OK']:
# Generate Request
commitOp = Operation()
commitOp.Type = 'SetFileStatus'
commitOp.Arguments = DEncode.encode( {'transformation':self.transformation, 'statusDict':self.statusDict, 'force':self.force} )
return S_OK( commitOp )
def _getProdInfoFromIDs(self):
"""get the processName, energy and eventsPerJob from the MetaData catalog
:raises: AttributeError if some of the information cannot be found
:returns: None
"""
if not self.prodIDs:
raise AttributeError("No prodIDs defined")
self.eventsPerJobs = []
self.processes = []
self.energies = []
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
trc = TransformationClient()
fc = FileCatalogClient()
for prodID in self.prodIDs:
gLogger.notice("Getting information for %s" % prodID)
tRes = trc.getTransformation(str(prodID))
if not tRes['OK']:
raise AttributeError("No prodInfo found for %s" % prodID)
self.eventsPerJobs.append(int(tRes['Value']['EventsPerTask']))
lfnRes = fc.findFilesByMetadata({'ProdID': prodID})
if not lfnRes['OK'] or not lfnRes['Value']:
raise AttributeError("Could not find files for %s: %s " % (prodID, lfnRes.get('Message', lfnRes.get('Value'))))
path = os.path.dirname(lfnRes['Value'][0])
fileRes = fc.getDirectoryUserMetadata(path)
self.processes.append(fileRes['Value']['EvtType'])
self.energies.append(fileRes['Value']['Energy'])
gLogger.notice("Found (Evts,Type,Energy): %s %s %s " %
(self.eventsPerJobs[-1], self.processes[-1], self.energies[-1]))
def __fileRetry(self, prodid, mode):
callback = {}
tsClient = TransformationClient()
if mode == "proc":
res = tsClient.getTransformationFilesCount(prodid, "ErrorCount", {'Status': 'Processed'})
elif mode == "not":
res = tsClient.getTransformationFilesCount(prodid, "ErrorCount", {'Status': ['Unused', 'Assigned', 'Failed']})
elif mode == "all":
res = tsClient.getTransformationFilesCount(prodid, "ErrorCount")
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
resList = []
total = res['Value'].pop('Total')
if total == 0:
callback = {"success": "false", "error": "No files found"}
else:
for status in sorted(res['Value'].keys()):
count = res['Value'][status]
percent = "%.1f" % ((count * 100.0) / total)
resList.append((status, str(count), percent))
resList.append(('Total', total, '-'))
callback = {"success": "true", "result": resList}
gLogger.info("#######", res)
return callback
def checkDatatype(prodID, datatype):
"""Check if the datatype makes sense for given production."""
# skip data type check when creating replications in development for prod productions this check doesn't work
if os.environ.get('SKIP_CHECK', False):
LOG.warn("Skipping Datatype check!")
return S_OK()
tClient = TransformationClient()
cond = dict(TransformationID=prodID)
trafo = tClient.getTransformations(cond)
if not trafo['OK']:
return trafo
if len(trafo['Value']) != 1:
return S_ERROR("Did not get unique production for this prodID")
trafoType = trafo['Value'][0]['Type'].split("_")[0]
dataTypes = Operations().getOptionsDict('Production/TransformationDatatypes')
if not dataTypes['OK']:
return dataTypes
dataTypes = dataTypes['Value']
if trafoType not in dataTypes[datatype]:
return S_ERROR("Datatype %r doesn't fit production type %r for prodID %s" % (datatype, trafoType, prodID))
return S_OK()
def __workflowxml(self, transid):
tsClient = TransformationClient()
retVal = tsClient.getTransformations({'TransformationID': transid})
if not retVal['OK']:
raise WErr.fromSERROR(retVal)
print retVal['Value']
return {"success": "true", "result": retVal['Value'][0]['Body']}
def getTransformationGroup(prodID, groupName):
"""Return TransformationGroup of prodID."""
if groupName:
return groupName
tClient = TransformationClient()
res = tClient.getTransformationParameters(prodID, 'TransformationGroup')
if not res['OK']:
return None
return res['Value']
def __additionalParams(self, prodid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getAdditionalParameters(prodid)
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
result = res["Value"]
back = []
for i in sorted(result.keys()):
back.append([i, result[i]])
callback = {"success": "true", "result": back}
return callback
def __dataQuery(self, prodid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getTransformationInputDataQuery(prodid)
gLogger.info("-= #######", res)
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
result = res["Value"]
back = []
for i in sorted(result.keys()):
back.append([i, result[i]])
callback = {"success": "true", "result": back}
return callback
class TaskBase:
def __init__(self):
self.transClient = TransformationClient()
def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''):
return S_ERROR("Not implemented")
def submitTransformationTasks(self,taskDict):
return S_ERROR("Not implemented")
def submitTasksToExternal(self,task):
return S_ERROR("Not implemented")
def updateDBAfterTaskSubmission(self,taskDict):
updated = 0
startTime = time.time()
for taskID in sortList(taskDict.keys()):
transID = taskDict[taskID]['TransformationID']
if taskDict[taskID]['Success']:
res = self.transClient.setTaskStatusAndWmsID(transID,taskID,'Submitted',str(taskDict[taskID]['ExternalID']))
if not res['OK']:
gLogger.warn("updateDBAfterSubmission: Failed to update task status after submission" , "%s %s" % (taskDict[taskID]['ExternalID'],res['Message']))
updated +=1
gLogger.info("updateDBAfterSubmission: Updated %d tasks in %.1f seconds" % (updated,time.time()-startTime))
return S_OK()
def updateTransformationReservedTasks(self,taskDicts):
return S_ERROR("Not implemented")
def getSubmittedTaskStatus(self,taskDicts):
return S_ERROR("Not implemented")
def getSubmittedFileStatus(self,fileDicts):
return S_ERROR("Not implemented")
def initialize(self):
"""Sets defaults
"""
self.am_setModuleParam("shifterProxy", "ProductionManager")
self.basepath = self.am_getOption("BasePath", "")
if not self.basepath:
return S_ERROR("Missing mandatory option BasePath")
self.baselogpath = self.am_getOption("BaseLogPath", "")
if not self.baselogpath:
return S_ERROR("Missing mandatory option BaseLogPath")
self.ops = Operations()
dest_se = self.ops.getValue("Transformations/ArchiveSE", "")
if not dest_se:
return S_ERROR("Missing mandatory option ArchiveSE")
self.storageElement = StorageElement( dest_se )
baselfn = self.ops.getValue("Transformations/BaseLogLFN", "")
if not baselfn:
return S_ERROR("Missing mandatory option Transformations/BaseLogLFN")
self.baselfn = baselfn
self.transclient = TransformationClient()
self.log.info("Running ")
return S_OK()
def initialize( self ):
"""Sets defaults """
self.replicaManager = ReplicaManager()
self.transClient = TransformationClient()
self.wmsClient = WMSClient()
self.requestClient = RequestClient()
self.metadataClient = FileCatalogClient()
self.storageUsageClient = StorageUsageClient()
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/DataManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'DataManager' )
self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge', 'Replication'] ) )
gLogger.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) )
self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'StorageUsage', 'MetadataCatalog'] ) )
gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) )
self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta )
self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days
gLogger.info( "Will archive Completed transformations after %d days" % self.archiveAfter )
self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) )
gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) )
self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' )
gLogger.info( "Will remove logs found on storage element: %s" % self.logSE )
return S_OK()
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
# # replica manager
self.replicaManager = ReplicaManager()
# # transformation client
self.transClient = TransformationClient()
# # wms client
self.wmsClient = WMSClient()
# # request client
self.requestClient = RequestClient()
# # file catalog clinet
self.metadataClient = FileCatalogClient()
# # placeholders for CS options
# # transformations types
self.transformationTypes = None
# # directory locations
self.directoryLocations = None
# # transformation metadata
self.transfidmeta = None
# # archive periof in days
self.archiveAfter = None
# # active SEs
self.activeStorages = None
# # transformation log SEs
self.logSE = None
# # enable/disable execution
self.enableFlag = None
def initialize( self ):
""" Agent initialization.
The extensions MUST provide in the initialize method the following data members:
- TransformationClient objects (self.transClient),
- set the shifterProxy if different from the default one set here ('ProductionManager')
- list of transformation types to be looked (self.transType)
"""
gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM )
self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Client.TaskManagerPlugin' )
# Default clients
self.transClient = TransformationClient()
# Bulk submission flag
self.bulkSubmissionFlag = self.am_getOption( 'BulkSubmission', False )
# setting up the threading
maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', 15 )
threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
self.log.verbose( "Multithreaded with %d threads" % maxNumberOfThreads )
for i in xrange( maxNumberOfThreads ):
threadPool.generateJobAndQueueIt( self._execute, [i] )
return S_OK()
def __transformationDetail(self, prodid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getTransformationParameters(prodid, ['DetailedInfo'])
if not res["OK"]:
callback = {"success": "false", "error": res["Message"]}
else:
callback = res['Value']
if callback:
callback = {"success": "true", "result": res['Value']}
else:
callback = {"success": "false", "error": "Production does not have parameter 'DetailedInfo'"}
gLogger.info("#######", res)
return callback
def __init__(self, script = None):
super(ProductionJob, self).__init__( script )
self.prodVersion = __RCSID__
self.dryrun = False
self.created = False
self.checked = False
self.call_finalization = False
self.finalsdict = {}
self.transfid = 0
self.type = 'Production'
self.csSection = '/Production/Defaults'
self.ops = Operations()
self.fc = FileCatalogClient()
self.trc = TransformationClient()
self.defaultProdID = '12345'
self.defaultProdJobID = '12345'
self.jobFileGroupSize = 1
self.nbtasks = 1
self.slicesize =0
self.basename = ''
self.basepath = self.ops.getValue('/Production/CLIC/BasePath','/ilc/prod/clic/')
self.evttype = ''
self.datatype = ''
self.energycat = ''
self.detector = ''
self.currtrans = None
self.description = ''
self.finalpaths = []
self.finalMetaDict = defaultdict( dict )
self.prodMetaDict = {}
self.finalMetaDictNonSearch = {}
self.metadict_external = {}
self.outputStorage = ''
self.proxyinfo = getProxyInfo()
self.inputdataquery = False
self.inputBKSelection = {}
self.plugin = 'Standard'
self.prodGroup = ''
self.prodTypes = ['MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction',
'MCReconstruction_Overlay', 'Merge', 'Split',
'MCGeneration_ILD',
'MCSimulation_ILD',
'MCReconstruction_ILD',
'MCReconstruction_Overlay_ILD',
'Split_ILD'
]
self.prodparameters = {}
self.prodparameters['NbInputFiles'] = 1
self.prodparameters['nbevts'] = 0
#self.prodparameters["SWPackages"] = ''
self._addParameter(self.workflow, "IS_PROD", 'JDL', True, "This job is a production job")
if not script:
self.__setDefaults()
self._recBasePaths = {}
self.maxFCFoldersToCheck = 100000
class TaskBase( TransformationAgentsUtilities ):
''' The other classes inside here inherits from this one.
'''
def __init__( self, transClient = None, logger = None ):
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if not logger:
self.log = gLogger.getSubLogger( 'TaskBase' )
else:
self.log = logger
self.pluginLocation = 'DIRAC.TransformationSystem.Client.TaskManagerPlugin'
self.transInThread = {}
self.debug = False
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '',
bulkSubmissionFlag = False ):
return S_ERROR( "Not implemented" )
def submitTransformationTasks( self, taskDict ):
return S_ERROR( "Not implemented" )
def submitTasksToExternal( self, task ):
return S_ERROR( "Not implemented" )
def updateDBAfterTaskSubmission( self, taskDict ):
""" Sets tasks status after the submission to "Submitted", in case of success
"""
updated = 0
startTime = time.time()
for taskID, task in taskDict.iteritems():
transID = task['TransformationID']
if task['Success']:
res = self.transClient.setTaskStatusAndWmsID( transID, taskID, 'Submitted',
str( task['ExternalID'] ) )
if not res['OK']:
self._logWarn( "Failed to update task status after submission" ,
"%s %s" % ( task['ExternalID'], res['Message'] ),
transID = transID, method = 'updateDBAfterSubmission' )
updated += 1
if updated:
self._logInfo( "Updated %d tasks in %.1f seconds" % ( updated, time.time() - startTime ),
transID = transID, method = 'updateDBAfterSubmission' )
return S_OK()
def updateTransformationReservedTasks( self, taskDicts ):
return S_ERROR( "Not implemented" )
def getSubmittedTaskStatus( self, taskDicts ):
return S_ERROR( "Not implemented" )
def getSubmittedFileStatus( self, fileDicts ):
return S_ERROR( "Not implemented" )
def __init__( self, *args, **kwargs ):
''' c'tor
'''
AgentModule.__init__( self, *args, **kwargs )
self.taskManager = None
self.shifterProxy = ''
self.transClient = TransformationClient()
self.transType = []
def __init__(self, transID=0, transClient=""):
API.__init__(self)
self.paramTypes = {
"TransformationID": [types.IntType, types.LongType],
"TransformationName": types.StringTypes,
"Status": types.StringTypes,
"Description": types.StringTypes,
"LongDescription": types.StringTypes,
"Type": types.StringTypes,
"Plugin": types.StringTypes,
"AgentType": types.StringTypes,
"FileMask": types.StringTypes,
"TransformationGroup": types.StringTypes,
"GroupSize": [types.IntType, types.LongType, types.FloatType],
"InheritedFrom": [types.IntType, types.LongType],
"Body": types.StringTypes,
"MaxNumberOfTasks": [types.IntType, types.LongType],
"EventsPerTask": [types.IntType, types.LongType],
}
self.paramValues = {
"TransformationID": 0,
"TransformationName": "",
"Status": "New",
"Description": "",
"LongDescription": "",
"Type": "",
"Plugin": "Standard",
"AgentType": "Manual",
"FileMask": "",
"TransformationGroup": "General",
"GroupSize": 1,
"InheritedFrom": 0,
"Body": "",
"MaxNumberOfTasks": 0,
"EventsPerTask": 0,
}
self.supportedPlugins = ["Broadcast", "Standard", "BySize", "ByShare"]
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues["TransformationID"] = transID
res = self.getTransformation()
if res["OK"]:
self.exists = True
elif res["Message"] == "Transformation does not exist":
raise AttributeError, "TransformationID %d does not exist" % transID
else:
self.paramValues["TransformationID"] = 0
gLogger.fatal(
"Failed to get transformation from database", "%s @ %s" % (transID, self.transClient.serverURL)
)
def __init__( self, server = 'Transformation/TransformationManager' ):
""" c'tor
self.transClient is a TransformationClient object
"""
self.transClient = TransformationClient()
self.transClient.setServer( server )
self.statusDict = {}
self.transformation = None
self.force = False
def __init__( self, transID = 0, transClient = None ):
""" c'tor
"""
super( Transformation, self ).__init__()
self.paramTypes = { 'TransformationID' : [types.IntType, types.LongType],
'TransformationName' : types.StringTypes,
'Status' : types.StringTypes,
'Description' : types.StringTypes,
'LongDescription' : types.StringTypes,
'Type' : types.StringTypes,
'Plugin' : types.StringTypes,
'AgentType' : types.StringTypes,
'FileMask' : types.StringTypes,
'TransformationGroup' : types.StringTypes,
'GroupSize' : [types.IntType, types.LongType, types.FloatType],
'InheritedFrom' : [types.IntType, types.LongType],
'Body' : types.StringTypes,
'MaxNumberOfTasks' : [types.IntType, types.LongType],
'EventsPerTask' : [types.IntType, types.LongType]}
self.paramValues = { 'TransformationID' : 0,
'TransformationName' : '',
'Status' : 'New',
'Description' : '',
'LongDescription' : '',
'Type' : '',
'Plugin' : 'Standard',
'AgentType' : 'Manual',
'FileMask' : '',
'TransformationGroup' : 'General',
'GroupSize' : 1,
'InheritedFrom' : 0,
'Body' : '',
'MaxNumberOfTasks' : 0,
'EventsPerTask' : 0}
self.ops = Operations()
self.supportedPlugins = self.ops.getValue( 'Transformations/AllowedPlugins',
['Broadcast', 'Standard', 'BySize', 'ByShare'] )
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
self.serverURL = self.transClient.getServer()
self.exists = False
if transID:
self.paramValues['TransformationID'] = transID
res = self.getTransformation()
if res['OK']:
self.exists = True
elif res['Message'] == 'Transformation does not exist':
raise AttributeError( 'TransformationID %d does not exist' % transID )
else:
self.paramValues['TransformationID'] = 0
gLogger.fatal( "Failed to get transformation from database", "%s @ %s" % ( transID,
self.transClient.serverURL ) )
def initialize(self):
""" Make the necessary initializations """
self.fileLog = {}
self.timeLog = {}
self.fullTimeLog = {}
self.pollingTime = self.am_getOption('PollingTime',120)
self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod',86400)
gMonitor.registerActivity("Iteration","Agent Loops",AGENT_NAME,"Loops/min",gMonitor.OP_SUM)
self.transClient = TransformationClient('TransformationDB')
self.metadataClient = FileCatalogClient()
return S_OK()
def __init__( self, transClient = None, logger = None ):
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if not logger:
self.log = gLogger.getSubLogger( 'TaskBase' )
else:
self.log = logger
def __transformationFileStatus(self, transid):
callback = {}
tsClient = TransformationClient()
res = tsClient.getTransformationFilesCount(transid, "Status")
if not res['OK']:
callback = {"success": "false", "error": res["Message"]}
else:
resList = []
total = res['Value'].pop('Total')
if total == 0:
callback = {"success": "false", "error": "No files found"}
else:
for status in sorted(res['Value'].keys()):
count = res['Value'][status]
percent = "%.1f" % ((count * 100.0) / total)
resList.append((status, str(count), percent))
resList.append(('Total', total, '-'))
callback = {"success": "true", "result": resList}
gLogger.info("#######", res)
return callback
def __extendTransformation(self, transid):
try:
tasks = int(self.request.arguments["tasks"][-1])
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
gLogger.info("extend %s" % transid)
tsClient = TransformationClient()
gLogger.info("extendTransformation(%s,%s)" % (transid, tasks))
res = tsClient.extendTransformation(transid, tasks)
if res["OK"]:
resString = "%s extended by %s successfully" % (transid, tasks)
else:
resString = "%s failed to extend: %s" % (transid, res["Message"])
callback = {"success": "true", "showResult": [resString], "result": resString}
gLogger.info("#######", res)
return callback
def _createReplication( targetSE, sourceSE, prodID, datatype, extraname=''):
"""Creates the replication transformation based on the given parameters"""
from DIRAC.TransformationSystem.Client.Transformation import Transformation
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
metadata = {"Datatype":datatype, "ProdID":prodID}
trans = Transformation()
transName = 'replicate_%s_%s' % ( str(prodID), ",".join(targetSE) )
if extraname:
transName += "_%s" % extraname
trans.setTransformationName( transName )
description = 'Replicate files for prodID %s to %s' % ( str(prodID), ",".join(targetSE) )
trans.setDescription( description )
trans.setLongDescription( description )
trans.setType( 'Replication' )
trans.setPlugin( 'Broadcast' )
res = trans.setSourceSE( sourceSE )
if not res['OK']:
exit(1)
res = trans.setTargetSE( targetSE )
if not res['OK']:
exit(1)
res = trans.addTransformation()
if not res['OK']:
gLogger.error(res['Message'])
exit(1)
gLogger.verbose(res)
trans.setStatus( 'Active' )
trans.setAgentType( 'Automatic' )
currtrans = trans.getTransformationID()['Value']
client = TransformationClient()
res = client.createTransformationInputDataQuery( currtrans, metadata )
if res['OK']:
gLogger.always("Successfully created replication transformation")
return S_OK()
else:
gLogger.error("Failure during replication creation", res['Message'])
return S_ERROR("Failed to create transformation")
def _getLogFolderFromID( clip ):
"""Obtain the folder of the logfiles from the prodID
Fills the clip.logD variable
"""
from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
## Check if transformation exists and get its type
server = TransformationClient()
result = server.getTransformation( clip.prodid )
if not result['OK']:
return result
transType = result['Value']['Type']
query = { 'ProdID' : clip.prodid }
if 'Reconstruction' in transType:
query['Datatype'] = 'REC'
result = FileCatalogClient().findFilesByMetadata( query, '/' )
if not result['OK']:
return result
elif result['Value']:
lfns = result['Value']
baseLFN = "/".join( lfns[0].split( '/' )[:-2] )
if not clip.getAllSubdirs:
lfns = lfns[:1]
clip.logD = []
lastdir = ""
for lfn in lfns:
subFolderNumber = lfn.split( '/' )[-2]
logdir = os.path.join( baseLFN, 'LOG', subFolderNumber )
if lastdir != logdir:
gLogger.notice( 'Setting logdir to %s' % logdir )
clip.logD.append(logdir)
lastdir=logdir
else:
return S_ERROR( "Cannot discover the LogFilePath: No output files yet" )
return S_OK()
def __init__( self, *args, **kwargs ):
""" c'tor
:param self: self reference
:param str agentName: name of agent
:param bool baseAgentName: whatever
:param dict properties: whatever else
"""
AgentModule.__init__( self, *args, **kwargs )
self.transClient = TransformationClient()
self.transfStatuses = self.am_getOption( 'TransformationStatuses', ['Active', 'Stopped'] )
def __init__( self, transClient = None, logger = None ):
if not transClient:
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
self.transClient = TransformationClient()
else:
self.transClient = transClient
if not logger:
self.log = gLogger.getSubLogger( 'TaskBase' )
else:
self.log = logger
def initialize( self ):
self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin' )
self.checkCatalog = self.am_getOption( 'CheckCatalog', 'yes' )
# This sets the Default Proxy to used as that defined under
# /Operations/Shifter/ProductionManager
# the shifterProxy option in the Configuration can be used to change this default.
self.am_setOption( 'shifterProxy', 'ProductionManager' )
self.transDB = TransformationClient( 'TransformationDB' )
self.rm = ReplicaManager()
return S_OK()
def __init__(self, *args, **kwargs):
"""c'tor"""
AgentModule.__init__(self, *args, **kwargs)
self.consistencyInspector = ConsistencyInspector()
self.integrityClient = DataIntegrityClient()
self.fc = FileCatalog()
self.transClient = TransformationClient()
self.fileCatalogClient = FileCatalogClient()
agentTSTypes = self.am_getOption("TransformationTypes", [])
if agentTSTypes:
self.transformationTypes = agentTSTypes
else:
self.transformationTypes = Operations().getValue(
"Transformations/DataProcessing", ["MCSimulation", "Merge"]
)
self.directoryLocations = sorted(
self.am_getOption("DirectoryLocations", ["TransformationDB", "MetadataCatalog"])
)
self.transfidmeta = self.am_getOption("TransfIDMeta", "TransformationID")
self.enableFlag = True
def submitTS(job, infileList):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType("SimtelMerging")
t.setDescription("Runs merge_simtel for array 3HB8")
t.setLongDescription("Merging array 3HB8 analysis") # mandatory
t.setGroupSize(10)
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files added here
return res
def _getClients(self):
""" returns the clients used in the threads - this is another function that should be extended.
The clients provided here are defaults, and should be adapted
"""
threadTransformationClient = TransformationClient()
threadTaskManager = WorkflowTasks(
) # this is for wms tasks, replace it with something else if needed
threadTaskManager.pluginLocation = self.pluginLocation
return {
'TransformationClient': threadTransformationClient,
'TaskManager': threadTaskManager
}
def checkDatatype( self, prodID, datatype ):
""" check if the datatype makes sense for given production """
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
tClient = TransformationClient()
cond = dict( TransformationID=prodID )
trafo = tClient.getTransformations( cond )
if not trafo['OK']:
return trafo
if len(trafo['Value']) != 1:
return S_ERROR( "Did not get unique production for this prodID" )
trafoType = trafo['Value'][0]['Type'].split("_")[0]
dataTypes = { 'MCGeneration': ['GEN'],
'Split': ['GEN'],
'MCSimulation': ['SIM'],
'MCReconstruction': ['REC', 'DST'],
}.get( trafoType, [] )
if datatype not in dataTypes:
return S_ERROR( "Datatype %s doesn't fit production type %s" %( datatype, trafoType ) )
return S_OK()
def submitTS(job, infileList):
""" Create a transformation executing the job workflow """
t = Transformation()
tc = TransformationClient()
t.setType("DataReprocessing")
t.setDescription("Mars example")
t.setLongDescription("Mars analysis") # mandatory
t.setGroupSize(10) # 1 for test
t.setBody(job.workflow.toXML())
res = t.addTransformation() # Transformation is created here
if not res['OK']:
print res['Message']
DIRAC.exit(-1)
t.setStatus("Active")
t.setAgentType("Automatic")
transID = t.getTransformationID()
tc.addFilesToTransformation(transID['Value'],
infileList) # Files added here
return res
def __init__(self, plugin='Standard', transClient=None, dataManager=None, fc=None,
debug=False, transInThread=None, transID=None):
"""
c'tor
Setting defaults
"""
# clients
if transClient is None:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if dataManager is None:
self.dm = DataManager()
else:
self.dm = dataManager
if fc is None:
self.fc = FileCatalog()
else:
self.fc = fc
self.dmsHelper = DMSHelpers()
self.plugin = plugin
self.transID = transID
self.params = {}
self.groupSize = 0
self.maxFiles = 0
self.cachedLFNSize = {}
self.transString = ''
self.debug = debug
if transInThread is None:
self.transInThread = {}
else:
self.transInThread = transInThread
self.log = gLogger.getSubLogger(plugin)
def initialize(self):
"""Agent initialization.
The extensions MUST provide in the initialize method the following data members:
- TransformationClient objects (self.transClient),
- set the shifterProxy if different from the default one set here ('ProductionManager')
- list of transformation types to be looked (self.transType)
"""
self.pluginLocation = self.am_getOption(
"PluginLocation",
"DIRAC.TransformationSystem.Client.TaskManagerPlugin")
# Default clients
self.transClient = TransformationClient()
self.jobManagerClient = JobManagerClient()
# Bulk submission flag
self.bulkSubmissionFlag = self.am_getOption("BulkSubmission",
self.bulkSubmissionFlag)
# Shifter credentials to use, could replace the use of shifterProxy eventually
self.shifterProxy = self.am_getOption("shifterProxy",
self.shifterProxy)
self.credentials = self.am_getOption("ShifterCredentials",
self.credentials)
resCred = self.__getCredentials()
if not resCred["OK"]:
return resCred
# setting up the threading
maxNumberOfThreads = self.am_getOption("maxNumberOfThreads", 15)
self.log.verbose("Multithreaded with %d threads" % maxNumberOfThreads)
self.threadPoolExecutor = concurrent.futures.ThreadPoolExecutor(
max_workers=maxNumberOfThreads)
return S_OK()
def __extendTransformation(self, transid):
try:
tasks = int(self.request.arguments["tasks"][-1])
except KeyError as excp:
raise WErr(400, "Missing %s" % excp)
gLogger.info("extend %s" % transid)
tsClient = TransformationClient()
gLogger.info("extendTransformation(%s,%s)" % (transid, tasks))
res = tsClient.extendTransformation(transid, tasks)
if res["OK"]:
resString = "%s extended by %s successfully" % (transid, tasks)
else:
resString = "%s failed to extend: %s" % (transid, res["Message"])
callback = {
"success": "true",
"showResult": [resString],
"result": resString
}
gLogger.info("#######", res)
return callback
def __init__( self, plugin, transClient = None, replicaManager = None ):
self.params = False
self.data = False
self.plugin = plugin
self.files = False
if transClient == None:
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
self.transClient = TransformationClient()
else:
self.transClient = transClient
if replicaManager == None:
from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager
self.rm = ReplicaManager()
else:
self.rm = replicaManager
def initialize(self):
""" Agent initialization.
The extensions MUST provide in the initialize method the following data members:
- TransformationClient objects (self.transClient),
- set the shifterProxy if different from the default one set here ('ProductionManager')
- list of transformation types to be looked (self.transType)
"""
gMonitor.registerActivity("SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM)
self.pluginLocation = self.am_getOption('PluginLocation', 'DIRAC.TransformationSystem.Client.TaskManagerPlugin')
# Default clients
self.transClient = TransformationClient()
self.jobManagerClient = JobManagerClient()
# Bulk submission flag
self.bulkSubmissionFlag = self.am_getOption('BulkSubmission', self.bulkSubmissionFlag)
# Shifter credentials to use, could replace the use of shifterProxy eventually
self.shifterProxy = self.am_getOption('shifterProxy', self.shifterProxy)
self.credentials = self.am_getOption('ShifterCredentials', self.credentials)
resCred = self.__getCredentials()
if not resCred['OK']:
return resCred
# setting up the threading
maxNumberOfThreads = self.am_getOption('maxNumberOfThreads', 15)
threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads)
self.log.verbose("Multithreaded with %d threads" % maxNumberOfThreads)
for i in xrange(maxNumberOfThreads):
threadPool.generateJobAndQueueIt(self._execute, [i])
return S_OK()
def applyInputDataQuery(self, metadata=None, prodid=None):
""" Tell the production to update itself using the metadata query specified, i.e. submit new jobs if new files
are added corresponding to same query.
"""
if not self.transfid and self.currtrans:
self.transfid = self.currtrans.getTransformationID()['Value'] #pylint: disable=E1101
elif prodid:
self.transfid = prodid
if not self.transfid:
print "Not transformation defined earlier"
return S_ERROR("No transformation defined")
if metadata:
self.inputBKSelection = metadata
client = TransformationClient()
if not self.dryrun:
res = client.createTransformationInputDataQuery(
self.transfid, self.inputBKSelection)
if not res['OK']:
return res
else:
self.log.notice("Would use %s as metadata query for production" %
str(self.inputBKSelection))
return S_OK()
def getRequestID(self, prod=None):
""" Get the request ID for a single production """
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
if not prod:
prod = self.options.get('Productions', [])
requestID = None
if isinstance(prod, basestring):
prods = [prod]
else:
prods = prod
if len(prods) == 1 and str(prods[0]).upper() != 'ALL':
res = TransformationClient().getTransformation(prods[0])
if res['OK']:
requestID = int(res['Value']['TransformationFamily'])
return requestID
def initialize( self ):
""" Agent initialization.
The extensions MUST provide in the initialize method the following data members:
- TransformationClient objects (self.transClient),
- set the shifterProxy if different from the default one set here ('ProductionManager')
- list of transformation types to be looked (self.transType)
"""
gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks",
gMonitor.OP_ACUM )
# Default clients
self.transClient = TransformationClient()
#setting up the threading
maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', 15 )
threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads )
self.log.verbose( "Multithreaded with %d threads" % maxNumberOfThreads )
for i in xrange( maxNumberOfThreads ):
threadPool.generateJobAndQueueIt( self._execute, [i] )
return S_OK()
def __init__(self, *args, **kwargs):
""" c'tor
"""
AgentModule.__init__(self, *args, **kwargs)
self.integrityClient = DataIntegrityClient()
self.replicaManager = ReplicaManager()
self.transClient = TransformationClient()
self.fileCatalogClient = FileCatalogClient()
agentTSTypes = self.am_getOption('TransformationTypes', [])
if agentTSTypes:
self.transformationTypes = agentTSTypes
else:
self.transformationTypes = Operations().getValue(
'Transformations/DataProcessing', ['MCSimulation', 'Merge'])
self.directoryLocations = sortList(
self.am_getOption('DirectoryLocations',
['TransformationDB', 'MetadataCatalog']))
self.activeStorages = sortList(self.am_getOption('ActiveSEs', []))
self.transfidmeta = self.am_getOption('TransfIDMeta',
"TransformationID")
self.enableFlag = True
class TaskBase( object ):
def __init__( self, transClient = None, logger = None ):
if not transClient:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if not logger:
self.log = gLogger.getSubLogger( 'TaskBase' )
else:
self.log = logger
def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ):
return S_ERROR( "Not implemented" )
def submitTransformationTasks( self, taskDict ):
return S_ERROR( "Not implemented" )
def submitTasksToExternal( self, task ):
return S_ERROR( "Not implemented" )
def updateDBAfterTaskSubmission( self, taskDict ):
""" Sets tasks status after the submission to "Submitted", in case of success
"""
updated = 0
startTime = time.time()
for taskID in sorted( taskDict ):
transID = taskDict[taskID]['TransformationID']
if taskDict[taskID]['Success']:
res = self.transClient.setTaskStatusAndWmsID( transID, taskID, 'Submitted',
str( taskDict[taskID]['ExternalID'] ) )
if not res['OK']:
self.log.warn( "updateDBAfterSubmission: Failed to update task status after submission" ,
"%s %s" % ( taskDict[taskID]['ExternalID'], res['Message'] ) )
updated += 1
self.log.info( "updateDBAfterSubmission: Updated %d tasks in %.1f seconds" % ( updated, time.time() - startTime ) )
return S_OK()
def updateTransformationReservedTasks( self, taskDicts ):
return S_ERROR( "Not implemented" )
def getSubmittedTaskStatus( self, taskDicts ):
return S_ERROR( "Not implemented" )
def getSubmittedFileStatus( self, fileDicts ):
return S_ERROR( "Not implemented" )
def web_executeOperation( self ):
try:
cmd = self.request.arguments[ 'action' ][-1]
ids = self.request.arguments["ids"][0].split( "," )
ids = [int( i ) for i in ids ]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
tsClient = TransformationClient()
agentType = 'Manual'
if cmd == 'clean':
status = 'Cleaning'
elif cmd == 'start':
status = 'Active'
agentType = 'Automatic'
elif cmd == 'flush':
status = 'Flush'
agentType = 'Automatic'
elif cmd == 'stop':
status = 'Stopped'
elif cmd == 'complete':
status = 'Completed'
else:
self.finish( {"success":"false", "error": "Unknown action"} )
callback = []
for i in ids:
try:
id = int( i )
result = yield self.threadTask( tsClient.setTransformationParameter, id, 'Status', status )
if result["OK"]:
resString = "ProdID: %s set to %s successfully" % ( i, cmd )
result = yield self.threadTask( tsClient.setTransformationParameter, id, 'AgentType', agentType )
if not result["OK"]:
resString = "ProdID: %s failed to set to %s: %s" % ( i, cmd, result["Message"] )
else:
resString = "ProdID: %s failed due the reason: %s" % ( i, result["Message"] )
except:
resString = "Unable to convert given ID %s to transformation ID" % i
callback.append( resString )
callback = {"success":"true", "showResult":callback}
gLogger.info( cmd, ids )
self.finish( callback )
def web_showFileStatus( self ):
callback = {}
start = int( self.request.arguments["start"][-1] )
limit = int( self.request.arguments["limit"][-1] )
try:
id = self.request.arguments[ 'transformationId' ][-1]
status = self.request.arguments[ 'status' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
tsClient = TransformationClient()
result = yield self.threadTask( tsClient.getTransformationFilesSummaryWeb, {'TransformationID':id, 'Status':status}, [["FileID", "ASC"]], start, limit )
if not result['OK']:
callback = {"success":"false", "error":result["Message"]}
else:
result = result["Value"]
if result.has_key( "TotalRecords" ) and result["TotalRecords"] > 0:
if result.has_key( "ParameterNames" ) and result.has_key( "Records" ):
if len( result["ParameterNames"] ) > 0:
if len( result["Records"] ) > 0:
callback = []
jobs = result["Records"]
head = result["ParameterNames"]
headLength = len( head )
for i in jobs:
tmp = {}
for j in range( 0, headLength ):
tmp[head[j]] = i[j]
callback.append( tmp )
total = result["TotalRecords"]
timestamp = Time.dateTime().strftime( "%Y-%m-%d %H:%M [UTC]" )
if result.has_key( "Extras" ):
extra = result["Extras"]
callback = {"success":"true", "result":callback, "total":total, "extra":extra, "date":timestamp}
else:
callback = {"success":"true", "result":callback, "total":total, "date":timestamp}
else:
callback = {"success":"false", "result":"", "error":"There are no data to display"}
else:
callback = {"success":"false", "result":"", "error":"ParameterNames field is undefined"}
else:
callback = {"success":"false", "result":"", "error":"Data structure is corrupted"}
else:
callback = {"success":"false", "result":"", "error":"There were no data matching your selection"}
self.finish( callback )
def __init__(self, plugin, transClient=None, dataManager=None):
""" plugin name has to be passed in: it will then be executed as one of the functions below, e.g.
plugin = 'BySize' will execute TransformationPlugin('BySize')._BySize()
"""
super(TransformationPlugin, self).__init__(plugin)
self.data = {}
self.files = False
self.startTime = time.time()
if transClient is None:
transClient = TransformationClient()
if dataManager is None:
dataManager = DataManager()
self.util = PluginUtilities(plugin, transClient, dataManager)
def __init__(self, plugin, transClient=None, replicaManager=None):
""" plugin name has to be passed in: it will then be executed as one of the functions below, e.g.
plugin = 'BySize' will execute TransformationPlugin('BySize')._BySize()
"""
self.params = {}
self.data = {}
self.plugin = plugin
self.files = False
if transClient is None:
self.transClient = TransformationClient()
else:
self.transClient = transClient
if replicaManager is None:
self.rm = ReplicaManager()
else:
self.rm = replicaManager
class TaskBase:
def __init__(self):
self.transClient = TransformationClient()
def prepareTransformationTasks(self,
transBody,
taskDict,
owner='',
ownerGroup=''):
return S_ERROR("Not implemented")
def submitTransformationTasks(self, taskDict):
return S_ERROR("Not implemented")
def submitTasksToExternal(self, task):
return S_ERROR("Not implemented")
def updateDBAfterTaskSubmission(self, taskDict):
updated = 0
startTime = time.time()
for taskID in sortList(taskDict.keys()):
transID = taskDict[taskID]['TransformationID']
if taskDict[taskID]['Success']:
res = self.transClient.setTaskStatusAndWmsID(
transID, taskID, 'Submitted',
str(taskDict[taskID]['ExternalID']))
if not res['OK']:
gLogger.warn(
"updateDBAfterSubmission: Failed to update task status after submission",
"%s %s" %
(taskDict[taskID]['ExternalID'], res['Message']))
updated += 1
gLogger.info(
"updateDBAfterSubmission: Updated %d tasks in %.1f seconds" %
(updated, time.time() - startTime))
return S_OK()
def updateTransformationReservedTasks(self, taskDicts):
return S_ERROR("Not implemented")
def getSubmittedTaskStatus(self, taskDicts):
return S_ERROR("Not implemented")
def getSubmittedFileStatus(self, fileDicts):
return S_ERROR("Not implemented")
def _getClients(self, ownerDN=None, ownerGroup=None):
"""Returns the clients used in the threads
This is another function that should be extended.
The clients provided here are defaults, and should be adapted
If ownerDN and ownerGroup are not None the clients will delegate to these credentials
:param str ownerDN: DN of the owner of the submitted jobs
:param str ownerGroup: group of the owner of the submitted jobs
:returns: dict of Clients
"""
threadTransformationClient = TransformationClient()
threadTaskManager = WorkflowTasks(ownerDN=ownerDN, ownerGroup=ownerGroup)
threadTaskManager.pluginLocation = self.pluginLocation
return {"TransformationClient": threadTransformationClient, "TaskManager": threadTaskManager}
def setUp(self):
self.prodClient = ProductionClient()
self.transClient = TransformationClient()
self.fc = FileCatalog()
# ## Add metadata fields to the DFC
self.MDFieldDict = {
"particle": "VARCHAR(128)",
"analysis_prog": "VARCHAR(128)",
"tel_sim_prog": "VARCHAR(128)",
"outputType": "VARCHAR(128)",
"zenith": "int",
"data_level": "int",
}
for MDField in self.MDFieldDict:
MDFieldType = self.MDFieldDict[MDField]
res = self.fc.addMetadataField(MDField, MDFieldType)
self.assert_(res["OK"])
def setUp(self):
self.prodClient = ProductionClient()
self.transClient = TransformationClient()
self.fc = FileCatalog()
# ## Add metadata fields to the DFC
self.MDFieldDict = {
'particle': 'VARCHAR(128)',
'analysis_prog': 'VARCHAR(128)',
'tel_sim_prog': 'VARCHAR(128)',
'outputType': 'VARCHAR(128)',
'zenith': 'int',
'data_level': 'int'
}
for MDField in self.MDFieldDict:
MDFieldType = self.MDFieldDict[MDField]
res = self.fc.addMetadataField(MDField, MDFieldType)
self.assert_(res['OK'])
def web_setSite( self ):
callback = {}
try:
transID = int( self.request.arguments[ 'TransformationId' ][-1] )
runID = int( self.request.arguments[ 'RunNumber' ][-1] )
site = self.request.arguments[ 'Site' ][-1]
except KeyError as excp:
raise WErr( 400, "Missing %s" % excp )
gLogger.info( "\033[0;31m setTransformationRunsSite(%s, %s, %s) \033[0m" % ( transID, runID, site ) )
tsClient = TransformationClient()
result = yield self.threadTask( tsClient.setTransformationRunsSite, transID, runID, site )
if result["OK"]:
callback = {"success":"true", "result":"true"}
else:
callback = {"success":"false", "error":result["Message"]}
self.finish( callback )
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["transID: transformation ID"])
_, args = Script.parseCommandLine()
transIDs = [int(arg) for arg in args]
from DIRAC.TransformationSystem.Agent.ValidateOutputDataAgent import ValidateOutputDataAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
agent = ValidateOutputDataAgent(
"Transformation/ValidateOutputDataAgent",
"Transformation/ValidateOutputDataAgent",
"dirac-transformation-verify-outputdata",
)
agent.initialize()
client = TransformationClient()
for transID in transIDs:
agent.checkTransformationIntegrity(transID)
def main():
Script.parseCommandLine()
args = Script.getPositionalArgs()
if not args:
Script.showHelp()
transIDs = [int(arg) for arg in args]
from DIRAC.TransformationSystem.Agent.ValidateOutputDataAgent import ValidateOutputDataAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
agent = ValidateOutputDataAgent('Transformation/ValidateOutputDataAgent',
'Transformation/ValidateOutputDataAgent',
'dirac-transformation-verify-outputdata')
agent.initialize()
client = TransformationClient()
for transID in transIDs:
agent.checkTransformationIntegrity(transID)
def main():
# Registering arguments will automatically add their description to the help menu
Script.registerArgument(["transID: transformation ID"])
_, args = Script.parseCommandLine()
transIDs = [int(arg) for arg in args]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
agent = TransformationCleaningAgent(
"Transformation/TransformationCleaningAgent",
"Transformation/TransformationCleaningAgent",
"dirac-transformation-archive",
)
agent.initialize()
client = TransformationClient()
for transID in transIDs:
agent.archiveTransformation(transID)
def setUp(self):
self.mockTransClient = MagicMock()
self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK': True}
self.WMSClientMock = MagicMock()
self.jobMonitoringClient = MagicMock()
self.mockReqClient = MagicMock()
self.jobMock = MagicMock()
self.jobMock2 = MagicMock()
mockWF = MagicMock()
mockPar = MagicMock()
mockWF.findParameter.return_value = mockPar
mockPar.getValue.return_value = 'MySite'
self.jobMock2.workflow = mockWF
self.jobMock2.setDestination.return_value = {'OK': True}
self.jobMock.workflow.return_value = ''
self.jobMock.return_value = self.jobMock2
self.reqValidatorMock = MagicMock()
self.reqValidatorMock.validate.return_value = {'OK': True}
self.taskBase = TaskBase(transClient=self.mockTransClient)
self.pu = PluginUtilities(transClient=self.mockTransClient)
self.wfTasks = WorkflowTasks(
transClient=self.mockTransClient,
submissionClient=self.WMSClientMock,
jobMonitoringClient=self.jobMonitoringClient,
outputDataModule="mock")
self.requestTasks = RequestTasks(transClient=self.mockTransClient,
requestClient=self.mockReqClient,
requestValidator=reqValFake)
self.tc = TransformationClient()
self.transformation = Transformation()
self.maxDiff = None
gLogger.setLevel('DEBUG')
class SetFileStatus(OperationHandlerBase):
"""
.. class:: SetFileStatus
SetFileStatus operation handler
"""
def __init__(self, operation=None, csPath=None):
"""c'tor
:param self: self reference
:param Operation operation: Operation instance
:param str csPath: CS path for this handler
"""
OperationHandlerBase.__init__(self, operation, csPath)
def __call__(self):
""" It expects to find the arguments for tc.setFileStatusForTransformation in operation.Arguments
"""
try:
setFileStatusDict = DEncode.decode(self.operation.Arguments)[0]
self.log.debug("decoded filStatusDict=%s" % str(setFileStatusDict))
except ValueError, error:
self.log.exception(error)
self.operation.Error = str(error)
self.operation.Status = "Failed"
return S_ERROR(str(error))
tc = TransformationClient()
setStatus = tc.setFileStatusForTransformation(
setFileStatusDict['transformation'],
setFileStatusDict['statusDict'], setFileStatusDict['force'])
if not setStatus['OK']:
errorStr = "failed to change status: %s" % setStatus['Message']
self.operation.Error = errorStr
self.log.warn(errorStr)
return S_ERROR(self.operation.Error)
else:
self.operation.Status = "Done"
return S_OK()
def main():
Script.parseCommandLine()
args = Script.getPositionalArgs()
if not args:
Script.showHelp()
transIDs = [int(arg) for arg in args]
from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent
from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient
agent = TransformationCleaningAgent(
'Transformation/TransformationCleaningAgent',
'Transformation/TransformationCleaningAgent',
'dirac-transformation-remove-output')
agent.initialize()
client = TransformationClient()
for transID in transIDs:
agent.removeTransformationOutput(transID)
