def __getLoggingInfo(self, transid): callback = {} tsClient = TransformationClient() result = tsClient.getTransformationLogging(transid) if result["OK"]: result = result["Value"] if len(result) > 0: callback = [] resultUser = gConfig.getSections("/Security/Users") if resultUser["OK"]: users = resultUser["Value"] dndb = {} for j in users: dndb[gConfig.getValue("/Security/Users/%s/DN" % j)] = j else: dndb = {} for i in result: DN = i["AuthorDN"] if DN in dndb: i["AuthorDN"] = dndb[DN] else: i["AuthorDN"] = DN # "Owner Unknown" date = Time.toString(i["MessageDate"]) callback.append([i["Message"], date, i["AuthorDN"]]) callback = {"success": "true", "result": callback} else: callback = {"success": "false", "error": "Nothing to display"} else: callback = {"success": "false", "error": result["Message"]} gLogger.info("PRODUCTION LOG:", id) return callback
class FileReport( object ): """ A stateful object for reporting to TransformationDB """ def __init__( self, server = 'Transformation/TransformationManager' ): """ c'tor self.transClient is a TransformationClient object """ self.transClient = TransformationClient() self.transClient.setServer( server ) self.statusDict = {} self.transformation = None self.force = False def setFileStatus( self, transformation, lfn, status, sendFlag = False ): """ Set file status in the context of the given transformation """ if not self.transformation: self.transformation = transformation self.statusDict[lfn] = status if sendFlag: return self.commit() return S_OK() def setCommonStatus( self, status ): """ Set common status for all files in the internal cache """ for lfn in self.statusDict.keys(): self.statusDict[lfn] = status return S_OK() def getFiles( self ): """ Get the statuses of the files already accumulated in the FileReport object """ return copy.deepcopy( self.statusDict ) def commit( self ): """ Commit pending file status update records """ if not self.statusDict: return S_OK( {} ) result = self.transClient.setFileStatusForTransformation( self.transformation, self.statusDict, force = self.force ) if result['OK']: self.statusDict = {} return result def generateForwardDISET( self ): """ Commit the accumulated records and generate request eventually """ result = self.commit() commitOp = None if not result['OK']: # Generate Request commitOp = Operation() commitOp.Type = 'SetFileStatus' commitOp.Arguments = DEncode.encode( {'transformation':self.transformation, 'statusDict':self.statusDict, 'force':self.force} ) return S_OK( commitOp )
def _getProdInfoFromIDs(self): """get the processName, energy and eventsPerJob from the MetaData catalog :raises: AttributeError if some of the information cannot be found :returns: None """ if not self.prodIDs: raise AttributeError("No prodIDs defined") self.eventsPerJobs = [] self.processes = [] self.energies = [] from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient trc = TransformationClient() fc = FileCatalogClient() for prodID in self.prodIDs: gLogger.notice("Getting information for %s" % prodID) tRes = trc.getTransformation(str(prodID)) if not tRes['OK']: raise AttributeError("No prodInfo found for %s" % prodID) self.eventsPerJobs.append(int(tRes['Value']['EventsPerTask'])) lfnRes = fc.findFilesByMetadata({'ProdID': prodID}) if not lfnRes['OK'] or not lfnRes['Value']: raise AttributeError("Could not find files for %s: %s " % (prodID, lfnRes.get('Message', lfnRes.get('Value')))) path = os.path.dirname(lfnRes['Value'][0]) fileRes = fc.getDirectoryUserMetadata(path) self.processes.append(fileRes['Value']['EvtType']) self.energies.append(fileRes['Value']['Energy']) gLogger.notice("Found (Evts,Type,Energy): %s %s %s " % (self.eventsPerJobs[-1], self.processes[-1], self.energies[-1]))
def __fileRetry(self, prodid, mode): callback = {} tsClient = TransformationClient() if mode == "proc": res = tsClient.getTransformationFilesCount(prodid, "ErrorCount", {'Status': 'Processed'}) elif mode == "not": res = tsClient.getTransformationFilesCount(prodid, "ErrorCount", {'Status': ['Unused', 'Assigned', 'Failed']}) elif mode == "all": res = tsClient.getTransformationFilesCount(prodid, "ErrorCount") if not res['OK']: callback = {"success": "false", "error": res["Message"]} else: resList = [] total = res['Value'].pop('Total') if total == 0: callback = {"success": "false", "error": "No files found"} else: for status in sorted(res['Value'].keys()): count = res['Value'][status] percent = "%.1f" % ((count * 100.0) / total) resList.append((status, str(count), percent)) resList.append(('Total', total, '-')) callback = {"success": "true", "result": resList} gLogger.info("#######", res) return callback
def checkDatatype(prodID, datatype): """Check if the datatype makes sense for given production.""" # skip data type check when creating replications in development for prod productions this check doesn't work if os.environ.get('SKIP_CHECK', False): LOG.warn("Skipping Datatype check!") return S_OK() tClient = TransformationClient() cond = dict(TransformationID=prodID) trafo = tClient.getTransformations(cond) if not trafo['OK']: return trafo if len(trafo['Value']) != 1: return S_ERROR("Did not get unique production for this prodID") trafoType = trafo['Value'][0]['Type'].split("_")[0] dataTypes = Operations().getOptionsDict('Production/TransformationDatatypes') if not dataTypes['OK']: return dataTypes dataTypes = dataTypes['Value'] if trafoType not in dataTypes[datatype]: return S_ERROR("Datatype %r doesn't fit production type %r for prodID %s" % (datatype, trafoType, prodID)) return S_OK()
def __workflowxml(self, transid): tsClient = TransformationClient() retVal = tsClient.getTransformations({'TransformationID': transid}) if not retVal['OK']: raise WErr.fromSERROR(retVal) print retVal['Value'] return {"success": "true", "result": retVal['Value'][0]['Body']}
def getTransformationGroup(prodID, groupName): """Return TransformationGroup of prodID.""" if groupName: return groupName tClient = TransformationClient() res = tClient.getTransformationParameters(prodID, 'TransformationGroup') if not res['OK']: return None return res['Value']
def __additionalParams(self, prodid): callback = {} tsClient = TransformationClient() res = tsClient.getAdditionalParameters(prodid) if not res['OK']: callback = {"success": "false", "error": res["Message"]} else: result = res["Value"] back = [] for i in sorted(result.keys()): back.append([i, result[i]]) callback = {"success": "true", "result": back} return callback
def __dataQuery(self, prodid): callback = {} tsClient = TransformationClient() res = tsClient.getTransformationInputDataQuery(prodid) gLogger.info("-= #######", res) if not res['OK']: callback = {"success": "false", "error": res["Message"]} else: result = res["Value"] back = [] for i in sorted(result.keys()): back.append([i, result[i]]) callback = {"success": "true", "result": back} return callback
class TaskBase: def __init__(self): self.transClient = TransformationClient() def prepareTransformationTasks(self,transBody,taskDict,owner='',ownerGroup=''): return S_ERROR("Not implemented") def submitTransformationTasks(self,taskDict): return S_ERROR("Not implemented") def submitTasksToExternal(self,task): return S_ERROR("Not implemented") def updateDBAfterTaskSubmission(self,taskDict): updated = 0 startTime = time.time() for taskID in sortList(taskDict.keys()): transID = taskDict[taskID]['TransformationID'] if taskDict[taskID]['Success']: res = self.transClient.setTaskStatusAndWmsID(transID,taskID,'Submitted',str(taskDict[taskID]['ExternalID'])) if not res['OK']: gLogger.warn("updateDBAfterSubmission: Failed to update task status after submission" , "%s %s" % (taskDict[taskID]['ExternalID'],res['Message'])) updated +=1 gLogger.info("updateDBAfterSubmission: Updated %d tasks in %.1f seconds" % (updated,time.time()-startTime)) return S_OK() def updateTransformationReservedTasks(self,taskDicts): return S_ERROR("Not implemented") def getSubmittedTaskStatus(self,taskDicts): return S_ERROR("Not implemented") def getSubmittedFileStatus(self,fileDicts): return S_ERROR("Not implemented")
def initialize(self): """Sets defaults """ self.am_setModuleParam("shifterProxy", "ProductionManager") self.basepath = self.am_getOption("BasePath", "") if not self.basepath: return S_ERROR("Missing mandatory option BasePath") self.baselogpath = self.am_getOption("BaseLogPath", "") if not self.baselogpath: return S_ERROR("Missing mandatory option BaseLogPath") self.ops = Operations() dest_se = self.ops.getValue("Transformations/ArchiveSE", "") if not dest_se: return S_ERROR("Missing mandatory option ArchiveSE") self.storageElement = StorageElement( dest_se ) baselfn = self.ops.getValue("Transformations/BaseLogLFN", "") if not baselfn: return S_ERROR("Missing mandatory option Transformations/BaseLogLFN") self.baselfn = baselfn self.transclient = TransformationClient() self.log.info("Running ") return S_OK()
def initialize( self ): """Sets defaults """ self.replicaManager = ReplicaManager() self.transClient = TransformationClient() self.wmsClient = WMSClient() self.requestClient = RequestClient() self.metadataClient = FileCatalogClient() self.storageUsageClient = StorageUsageClient() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'DataManager' ) self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge', 'Replication'] ) ) gLogger.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) ) self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'StorageUsage', 'MetadataCatalog'] ) ) gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) ) self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" ) gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta ) self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days gLogger.info( "Will archive Completed transformations after %d days" % self.archiveAfter ) self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) ) gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) ) self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' ) gLogger.info( "Will remove logs found on storage element: %s" % self.logSE ) return S_OK()
def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) # # replica manager self.replicaManager = ReplicaManager() # # transformation client self.transClient = TransformationClient() # # wms client self.wmsClient = WMSClient() # # request client self.requestClient = RequestClient() # # file catalog clinet self.metadataClient = FileCatalogClient() # # placeholders for CS options # # transformations types self.transformationTypes = None # # directory locations self.directoryLocations = None # # transformation metadata self.transfidmeta = None # # archive periof in days self.archiveAfter = None # # active SEs self.activeStorages = None # # transformation log SEs self.logSE = None # # enable/disable execution self.enableFlag = None
def initialize( self ): """ Agent initialization. The extensions MUST provide in the initialize method the following data members: - TransformationClient objects (self.transClient), - set the shifterProxy if different from the default one set here ('ProductionManager') - list of transformation types to be looked (self.transType) """ gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks", gMonitor.OP_ACUM ) self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Client.TaskManagerPlugin' ) # Default clients self.transClient = TransformationClient() # Bulk submission flag self.bulkSubmissionFlag = self.am_getOption( 'BulkSubmission', False ) # setting up the threading maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', 15 ) threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads ) self.log.verbose( "Multithreaded with %d threads" % maxNumberOfThreads ) for i in xrange( maxNumberOfThreads ): threadPool.generateJobAndQueueIt( self._execute, [i] ) return S_OK()
def __transformationDetail(self, prodid): callback = {} tsClient = TransformationClient() res = tsClient.getTransformationParameters(prodid, ['DetailedInfo']) if not res["OK"]: callback = {"success": "false", "error": res["Message"]} else: callback = res['Value'] if callback: callback = {"success": "true", "result": res['Value']} else: callback = {"success": "false", "error": "Production does not have parameter 'DetailedInfo'"} gLogger.info("#######", res) return callback
def __init__(self, script = None): super(ProductionJob, self).__init__( script ) self.prodVersion = __RCSID__ self.dryrun = False self.created = False self.checked = False self.call_finalization = False self.finalsdict = {} self.transfid = 0 self.type = 'Production' self.csSection = '/Production/Defaults' self.ops = Operations() self.fc = FileCatalogClient() self.trc = TransformationClient() self.defaultProdID = '12345' self.defaultProdJobID = '12345' self.jobFileGroupSize = 1 self.nbtasks = 1 self.slicesize =0 self.basename = '' self.basepath = self.ops.getValue('/Production/CLIC/BasePath','/ilc/prod/clic/') self.evttype = '' self.datatype = '' self.energycat = '' self.detector = '' self.currtrans = None self.description = '' self.finalpaths = [] self.finalMetaDict = defaultdict( dict ) self.prodMetaDict = {} self.finalMetaDictNonSearch = {} self.metadict_external = {} self.outputStorage = '' self.proxyinfo = getProxyInfo() self.inputdataquery = False self.inputBKSelection = {} self.plugin = 'Standard' self.prodGroup = '' self.prodTypes = ['MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction', 'MCReconstruction_Overlay', 'Merge', 'Split', 'MCGeneration_ILD', 'MCSimulation_ILD', 'MCReconstruction_ILD', 'MCReconstruction_Overlay_ILD', 'Split_ILD' ] self.prodparameters = {} self.prodparameters['NbInputFiles'] = 1 self.prodparameters['nbevts'] = 0 #self.prodparameters["SWPackages"] = '' self._addParameter(self.workflow, "IS_PROD", 'JDL', True, "This job is a production job") if not script: self.__setDefaults() self._recBasePaths = {} self.maxFCFoldersToCheck = 100000
class TaskBase( TransformationAgentsUtilities ): ''' The other classes inside here inherits from this one. ''' def __init__( self, transClient = None, logger = None ): if not transClient: self.transClient = TransformationClient() else: self.transClient = transClient if not logger: self.log = gLogger.getSubLogger( 'TaskBase' ) else: self.log = logger self.pluginLocation = 'DIRAC.TransformationSystem.Client.TaskManagerPlugin' self.transInThread = {} self.debug = False def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '', bulkSubmissionFlag = False ): return S_ERROR( "Not implemented" ) def submitTransformationTasks( self, taskDict ): return S_ERROR( "Not implemented" ) def submitTasksToExternal( self, task ): return S_ERROR( "Not implemented" ) def updateDBAfterTaskSubmission( self, taskDict ): """ Sets tasks status after the submission to "Submitted", in case of success """ updated = 0 startTime = time.time() for taskID, task in taskDict.iteritems(): transID = task['TransformationID'] if task['Success']: res = self.transClient.setTaskStatusAndWmsID( transID, taskID, 'Submitted', str( task['ExternalID'] ) ) if not res['OK']: self._logWarn( "Failed to update task status after submission" , "%s %s" % ( task['ExternalID'], res['Message'] ), transID = transID, method = 'updateDBAfterSubmission' ) updated += 1 if updated: self._logInfo( "Updated %d tasks in %.1f seconds" % ( updated, time.time() - startTime ), transID = transID, method = 'updateDBAfterSubmission' ) return S_OK() def updateTransformationReservedTasks( self, taskDicts ): return S_ERROR( "Not implemented" ) def getSubmittedTaskStatus( self, taskDicts ): return S_ERROR( "Not implemented" ) def getSubmittedFileStatus( self, fileDicts ): return S_ERROR( "Not implemented" )
def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) self.taskManager = None self.shifterProxy = '' self.transClient = TransformationClient() self.transType = []
def __init__(self, transID=0, transClient=""): API.__init__(self) self.paramTypes = { "TransformationID": [types.IntType, types.LongType], "TransformationName": types.StringTypes, "Status": types.StringTypes, "Description": types.StringTypes, "LongDescription": types.StringTypes, "Type": types.StringTypes, "Plugin": types.StringTypes, "AgentType": types.StringTypes, "FileMask": types.StringTypes, "TransformationGroup": types.StringTypes, "GroupSize": [types.IntType, types.LongType, types.FloatType], "InheritedFrom": [types.IntType, types.LongType], "Body": types.StringTypes, "MaxNumberOfTasks": [types.IntType, types.LongType], "EventsPerTask": [types.IntType, types.LongType], } self.paramValues = { "TransformationID": 0, "TransformationName": "", "Status": "New", "Description": "", "LongDescription": "", "Type": "", "Plugin": "Standard", "AgentType": "Manual", "FileMask": "", "TransformationGroup": "General", "GroupSize": 1, "InheritedFrom": 0, "Body": "", "MaxNumberOfTasks": 0, "EventsPerTask": 0, } self.supportedPlugins = ["Broadcast", "Standard", "BySize", "ByShare"] if not transClient: self.transClient = TransformationClient() else: self.transClient = transClient self.serverURL = self.transClient.getServer() self.exists = False if transID: self.paramValues["TransformationID"] = transID res = self.getTransformation() if res["OK"]: self.exists = True elif res["Message"] == "Transformation does not exist": raise AttributeError, "TransformationID %d does not exist" % transID else: self.paramValues["TransformationID"] = 0 gLogger.fatal( "Failed to get transformation from database", "%s @ %s" % (transID, self.transClient.serverURL) )
def __init__( self, server = 'Transformation/TransformationManager' ): """ c'tor self.transClient is a TransformationClient object """ self.transClient = TransformationClient() self.transClient.setServer( server ) self.statusDict = {} self.transformation = None self.force = False
def __init__( self, transID = 0, transClient = None ): """ c'tor """ super( Transformation, self ).__init__() self.paramTypes = { 'TransformationID' : [types.IntType, types.LongType], 'TransformationName' : types.StringTypes, 'Status' : types.StringTypes, 'Description' : types.StringTypes, 'LongDescription' : types.StringTypes, 'Type' : types.StringTypes, 'Plugin' : types.StringTypes, 'AgentType' : types.StringTypes, 'FileMask' : types.StringTypes, 'TransformationGroup' : types.StringTypes, 'GroupSize' : [types.IntType, types.LongType, types.FloatType], 'InheritedFrom' : [types.IntType, types.LongType], 'Body' : types.StringTypes, 'MaxNumberOfTasks' : [types.IntType, types.LongType], 'EventsPerTask' : [types.IntType, types.LongType]} self.paramValues = { 'TransformationID' : 0, 'TransformationName' : '', 'Status' : 'New', 'Description' : '', 'LongDescription' : '', 'Type' : '', 'Plugin' : 'Standard', 'AgentType' : 'Manual', 'FileMask' : '', 'TransformationGroup' : 'General', 'GroupSize' : 1, 'InheritedFrom' : 0, 'Body' : '', 'MaxNumberOfTasks' : 0, 'EventsPerTask' : 0} self.ops = Operations() self.supportedPlugins = self.ops.getValue( 'Transformations/AllowedPlugins', ['Broadcast', 'Standard', 'BySize', 'ByShare'] ) if not transClient: self.transClient = TransformationClient() else: self.transClient = transClient self.serverURL = self.transClient.getServer() self.exists = False if transID: self.paramValues['TransformationID'] = transID res = self.getTransformation() if res['OK']: self.exists = True elif res['Message'] == 'Transformation does not exist': raise AttributeError( 'TransformationID %d does not exist' % transID ) else: self.paramValues['TransformationID'] = 0 gLogger.fatal( "Failed to get transformation from database", "%s @ %s" % ( transID, self.transClient.serverURL ) )
def initialize(self): """ Make the necessary initializations """ self.fileLog = {} self.timeLog = {} self.fullTimeLog = {} self.pollingTime = self.am_getOption('PollingTime',120) self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod',86400) gMonitor.registerActivity("Iteration","Agent Loops",AGENT_NAME,"Loops/min",gMonitor.OP_SUM) self.transClient = TransformationClient('TransformationDB') self.metadataClient = FileCatalogClient() return S_OK()
def __init__( self, transClient = None, logger = None ): if not transClient: self.transClient = TransformationClient() else: self.transClient = transClient if not logger: self.log = gLogger.getSubLogger( 'TaskBase' ) else: self.log = logger
def __transformationFileStatus(self, transid): callback = {} tsClient = TransformationClient() res = tsClient.getTransformationFilesCount(transid, "Status") if not res['OK']: callback = {"success": "false", "error": res["Message"]} else: resList = [] total = res['Value'].pop('Total') if total == 0: callback = {"success": "false", "error": "No files found"} else: for status in sorted(res['Value'].keys()): count = res['Value'][status] percent = "%.1f" % ((count * 100.0) / total) resList.append((status, str(count), percent)) resList.append(('Total', total, '-')) callback = {"success": "true", "result": resList} gLogger.info("#######", res) return callback
def __extendTransformation(self, transid): try: tasks = int(self.request.arguments["tasks"][-1]) except KeyError as excp: raise WErr(400, "Missing %s" % excp) gLogger.info("extend %s" % transid) tsClient = TransformationClient() gLogger.info("extendTransformation(%s,%s)" % (transid, tasks)) res = tsClient.extendTransformation(transid, tasks) if res["OK"]: resString = "%s extended by %s successfully" % (transid, tasks) else: resString = "%s failed to extend: %s" % (transid, res["Message"]) callback = {"success": "true", "showResult": [resString], "result": resString} gLogger.info("#######", res) return callback
def _createReplication( targetSE, sourceSE, prodID, datatype, extraname=''): """Creates the replication transformation based on the given parameters""" from DIRAC.TransformationSystem.Client.Transformation import Transformation from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient metadata = {"Datatype":datatype, "ProdID":prodID} trans = Transformation() transName = 'replicate_%s_%s' % ( str(prodID), ",".join(targetSE) ) if extraname: transName += "_%s" % extraname trans.setTransformationName( transName ) description = 'Replicate files for prodID %s to %s' % ( str(prodID), ",".join(targetSE) ) trans.setDescription( description ) trans.setLongDescription( description ) trans.setType( 'Replication' ) trans.setPlugin( 'Broadcast' ) res = trans.setSourceSE( sourceSE ) if not res['OK']: exit(1) res = trans.setTargetSE( targetSE ) if not res['OK']: exit(1) res = trans.addTransformation() if not res['OK']: gLogger.error(res['Message']) exit(1) gLogger.verbose(res) trans.setStatus( 'Active' ) trans.setAgentType( 'Automatic' ) currtrans = trans.getTransformationID()['Value'] client = TransformationClient() res = client.createTransformationInputDataQuery( currtrans, metadata ) if res['OK']: gLogger.always("Successfully created replication transformation") return S_OK() else: gLogger.error("Failure during replication creation", res['Message']) return S_ERROR("Failed to create transformation")
def _getLogFolderFromID( clip ): """Obtain the folder of the logfiles from the prodID Fills the clip.logD variable """ from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient ## Check if transformation exists and get its type server = TransformationClient() result = server.getTransformation( clip.prodid ) if not result['OK']: return result transType = result['Value']['Type'] query = { 'ProdID' : clip.prodid } if 'Reconstruction' in transType: query['Datatype'] = 'REC' result = FileCatalogClient().findFilesByMetadata( query, '/' ) if not result['OK']: return result elif result['Value']: lfns = result['Value'] baseLFN = "/".join( lfns[0].split( '/' )[:-2] ) if not clip.getAllSubdirs: lfns = lfns[:1] clip.logD = [] lastdir = "" for lfn in lfns: subFolderNumber = lfn.split( '/' )[-2] logdir = os.path.join( baseLFN, 'LOG', subFolderNumber ) if lastdir != logdir: gLogger.notice( 'Setting logdir to %s' % logdir ) clip.logD.append(logdir) lastdir=logdir else: return S_ERROR( "Cannot discover the LogFilePath: No output files yet" ) return S_OK()
def __init__( self, *args, **kwargs ): """ c'tor :param self: self reference :param str agentName: name of agent :param bool baseAgentName: whatever :param dict properties: whatever else """ AgentModule.__init__( self, *args, **kwargs ) self.transClient = TransformationClient() self.transfStatuses = self.am_getOption( 'TransformationStatuses', ['Active', 'Stopped'] )
def __init__( self, transClient = None, logger = None ): if not transClient: from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient self.transClient = TransformationClient() else: self.transClient = transClient if not logger: self.log = gLogger.getSubLogger( 'TaskBase' ) else: self.log = logger
def initialize( self ): self.pluginLocation = self.am_getOption( 'PluginLocation', 'DIRAC.TransformationSystem.Agent.TransformationPlugin' ) self.checkCatalog = self.am_getOption( 'CheckCatalog', 'yes' ) # This sets the Default Proxy to used as that defined under # /Operations/Shifter/ProductionManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'ProductionManager' ) self.transDB = TransformationClient( 'TransformationDB' ) self.rm = ReplicaManager() return S_OK()
def __init__(self, *args, **kwargs): """c'tor""" AgentModule.__init__(self, *args, **kwargs) self.consistencyInspector = ConsistencyInspector() self.integrityClient = DataIntegrityClient() self.fc = FileCatalog() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() agentTSTypes = self.am_getOption("TransformationTypes", []) if agentTSTypes: self.transformationTypes = agentTSTypes else: self.transformationTypes = Operations().getValue( "Transformations/DataProcessing", ["MCSimulation", "Merge"] ) self.directoryLocations = sorted( self.am_getOption("DirectoryLocations", ["TransformationDB", "MetadataCatalog"]) ) self.transfidmeta = self.am_getOption("TransfIDMeta", "TransformationID") self.enableFlag = True
def submitTS(job, infileList): """ Create a transformation executing the job workflow """ t = Transformation() tc = TransformationClient() t.setType("SimtelMerging") t.setDescription("Runs merge_simtel for array 3HB8") t.setLongDescription("Merging array 3HB8 analysis") # mandatory t.setGroupSize(10) t.setBody(job.workflow.toXML()) res = t.addTransformation() # Transformation is created here if not res['OK']: print res['Message'] DIRAC.exit(-1) t.setStatus("Active") t.setAgentType("Automatic") transID = t.getTransformationID() tc.addFilesToTransformation(transID['Value'], infileList) # Files added here return res
def _getClients(self): """ returns the clients used in the threads - this is another function that should be extended. The clients provided here are defaults, and should be adapted """ threadTransformationClient = TransformationClient() threadTaskManager = WorkflowTasks( ) # this is for wms tasks, replace it with something else if needed threadTaskManager.pluginLocation = self.pluginLocation return { 'TransformationClient': threadTransformationClient, 'TaskManager': threadTaskManager }
def checkDatatype( self, prodID, datatype ): """ check if the datatype makes sense for given production """ from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient tClient = TransformationClient() cond = dict( TransformationID=prodID ) trafo = tClient.getTransformations( cond ) if not trafo['OK']: return trafo if len(trafo['Value']) != 1: return S_ERROR( "Did not get unique production for this prodID" ) trafoType = trafo['Value'][0]['Type'].split("_")[0] dataTypes = { 'MCGeneration': ['GEN'], 'Split': ['GEN'], 'MCSimulation': ['SIM'], 'MCReconstruction': ['REC', 'DST'], }.get( trafoType, [] ) if datatype not in dataTypes: return S_ERROR( "Datatype %s doesn't fit production type %s" %( datatype, trafoType ) ) return S_OK()
def submitTS(job, infileList): """ Create a transformation executing the job workflow """ t = Transformation() tc = TransformationClient() t.setType("DataReprocessing") t.setDescription("Mars example") t.setLongDescription("Mars analysis") # mandatory t.setGroupSize(10) # 1 for test t.setBody(job.workflow.toXML()) res = t.addTransformation() # Transformation is created here if not res['OK']: print res['Message'] DIRAC.exit(-1) t.setStatus("Active") t.setAgentType("Automatic") transID = t.getTransformationID() tc.addFilesToTransformation(transID['Value'], infileList) # Files added here return res
def __init__(self, plugin='Standard', transClient=None, dataManager=None, fc=None, debug=False, transInThread=None, transID=None): """ c'tor Setting defaults """ # clients if transClient is None: self.transClient = TransformationClient() else: self.transClient = transClient if dataManager is None: self.dm = DataManager() else: self.dm = dataManager if fc is None: self.fc = FileCatalog() else: self.fc = fc self.dmsHelper = DMSHelpers() self.plugin = plugin self.transID = transID self.params = {} self.groupSize = 0 self.maxFiles = 0 self.cachedLFNSize = {} self.transString = '' self.debug = debug if transInThread is None: self.transInThread = {} else: self.transInThread = transInThread self.log = gLogger.getSubLogger(plugin)
def initialize(self): """Agent initialization. The extensions MUST provide in the initialize method the following data members: - TransformationClient objects (self.transClient), - set the shifterProxy if different from the default one set here ('ProductionManager') - list of transformation types to be looked (self.transType) """ self.pluginLocation = self.am_getOption( "PluginLocation", "DIRAC.TransformationSystem.Client.TaskManagerPlugin") # Default clients self.transClient = TransformationClient() self.jobManagerClient = JobManagerClient() # Bulk submission flag self.bulkSubmissionFlag = self.am_getOption("BulkSubmission", self.bulkSubmissionFlag) # Shifter credentials to use, could replace the use of shifterProxy eventually self.shifterProxy = self.am_getOption("shifterProxy", self.shifterProxy) self.credentials = self.am_getOption("ShifterCredentials", self.credentials) resCred = self.__getCredentials() if not resCred["OK"]: return resCred # setting up the threading maxNumberOfThreads = self.am_getOption("maxNumberOfThreads", 15) self.log.verbose("Multithreaded with %d threads" % maxNumberOfThreads) self.threadPoolExecutor = concurrent.futures.ThreadPoolExecutor( max_workers=maxNumberOfThreads) return S_OK()
def __extendTransformation(self, transid): try: tasks = int(self.request.arguments["tasks"][-1]) except KeyError as excp: raise WErr(400, "Missing %s" % excp) gLogger.info("extend %s" % transid) tsClient = TransformationClient() gLogger.info("extendTransformation(%s,%s)" % (transid, tasks)) res = tsClient.extendTransformation(transid, tasks) if res["OK"]: resString = "%s extended by %s successfully" % (transid, tasks) else: resString = "%s failed to extend: %s" % (transid, res["Message"]) callback = { "success": "true", "showResult": [resString], "result": resString } gLogger.info("#######", res) return callback
def __init__( self, plugin, transClient = None, replicaManager = None ): self.params = False self.data = False self.plugin = plugin self.files = False if transClient == None: from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient self.transClient = TransformationClient() else: self.transClient = transClient if replicaManager == None: from DIRAC.DataManagementSystem.Client.ReplicaManager import ReplicaManager self.rm = ReplicaManager() else: self.rm = replicaManager
def initialize(self): """ Agent initialization. The extensions MUST provide in the initialize method the following data members: - TransformationClient objects (self.transClient), - set the shifterProxy if different from the default one set here ('ProductionManager') - list of transformation types to be looked (self.transType) """ gMonitor.registerActivity("SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks", gMonitor.OP_ACUM) self.pluginLocation = self.am_getOption('PluginLocation', 'DIRAC.TransformationSystem.Client.TaskManagerPlugin') # Default clients self.transClient = TransformationClient() self.jobManagerClient = JobManagerClient() # Bulk submission flag self.bulkSubmissionFlag = self.am_getOption('BulkSubmission', self.bulkSubmissionFlag) # Shifter credentials to use, could replace the use of shifterProxy eventually self.shifterProxy = self.am_getOption('shifterProxy', self.shifterProxy) self.credentials = self.am_getOption('ShifterCredentials', self.credentials) resCred = self.__getCredentials() if not resCred['OK']: return resCred # setting up the threading maxNumberOfThreads = self.am_getOption('maxNumberOfThreads', 15) threadPool = ThreadPool(maxNumberOfThreads, maxNumberOfThreads) self.log.verbose("Multithreaded with %d threads" % maxNumberOfThreads) for i in xrange(maxNumberOfThreads): threadPool.generateJobAndQueueIt(self._execute, [i]) return S_OK()
def applyInputDataQuery(self, metadata=None, prodid=None): """ Tell the production to update itself using the metadata query specified, i.e. submit new jobs if new files are added corresponding to same query. """ if not self.transfid and self.currtrans: self.transfid = self.currtrans.getTransformationID()['Value'] #pylint: disable=E1101 elif prodid: self.transfid = prodid if not self.transfid: print "Not transformation defined earlier" return S_ERROR("No transformation defined") if metadata: self.inputBKSelection = metadata client = TransformationClient() if not self.dryrun: res = client.createTransformationInputDataQuery( self.transfid, self.inputBKSelection) if not res['OK']: return res else: self.log.notice("Would use %s as metadata query for production" % str(self.inputBKSelection)) return S_OK()
def getRequestID(self, prod=None): """ Get the request ID for a single production """ from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient if not prod: prod = self.options.get('Productions', []) requestID = None if isinstance(prod, basestring): prods = [prod] else: prods = prod if len(prods) == 1 and str(prods[0]).upper() != 'ALL': res = TransformationClient().getTransformation(prods[0]) if res['OK']: requestID = int(res['Value']['TransformationFamily']) return requestID
def initialize( self ): """ Agent initialization. The extensions MUST provide in the initialize method the following data members: - TransformationClient objects (self.transClient), - set the shifterProxy if different from the default one set here ('ProductionManager') - list of transformation types to be looked (self.transType) """ gMonitor.registerActivity( "SubmittedTasks", "Automatically submitted tasks", "Transformation Monitoring", "Tasks", gMonitor.OP_ACUM ) # Default clients self.transClient = TransformationClient() #setting up the threading maxNumberOfThreads = self.am_getOption( 'maxNumberOfThreads', 15 ) threadPool = ThreadPool( maxNumberOfThreads, maxNumberOfThreads ) self.log.verbose( "Multithreaded with %d threads" % maxNumberOfThreads ) for i in xrange( maxNumberOfThreads ): threadPool.generateJobAndQueueIt( self._execute, [i] ) return S_OK()
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.integrityClient = DataIntegrityClient() self.replicaManager = ReplicaManager() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = agentTSTypes else: self.transformationTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge']) self.directoryLocations = sortList( self.am_getOption('DirectoryLocations', ['TransformationDB', 'MetadataCatalog'])) self.activeStorages = sortList(self.am_getOption('ActiveSEs', [])) self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID") self.enableFlag = True
class TaskBase( object ): def __init__( self, transClient = None, logger = None ): if not transClient: self.transClient = TransformationClient() else: self.transClient = transClient if not logger: self.log = gLogger.getSubLogger( 'TaskBase' ) else: self.log = logger def prepareTransformationTasks( self, transBody, taskDict, owner = '', ownerGroup = '', ownerDN = '' ): return S_ERROR( "Not implemented" ) def submitTransformationTasks( self, taskDict ): return S_ERROR( "Not implemented" ) def submitTasksToExternal( self, task ): return S_ERROR( "Not implemented" ) def updateDBAfterTaskSubmission( self, taskDict ): """ Sets tasks status after the submission to "Submitted", in case of success """ updated = 0 startTime = time.time() for taskID in sorted( taskDict ): transID = taskDict[taskID]['TransformationID'] if taskDict[taskID]['Success']: res = self.transClient.setTaskStatusAndWmsID( transID, taskID, 'Submitted', str( taskDict[taskID]['ExternalID'] ) ) if not res['OK']: self.log.warn( "updateDBAfterSubmission: Failed to update task status after submission" , "%s %s" % ( taskDict[taskID]['ExternalID'], res['Message'] ) ) updated += 1 self.log.info( "updateDBAfterSubmission: Updated %d tasks in %.1f seconds" % ( updated, time.time() - startTime ) ) return S_OK() def updateTransformationReservedTasks( self, taskDicts ): return S_ERROR( "Not implemented" ) def getSubmittedTaskStatus( self, taskDicts ): return S_ERROR( "Not implemented" ) def getSubmittedFileStatus( self, fileDicts ): return S_ERROR( "Not implemented" )
def web_executeOperation( self ): try: cmd = self.request.arguments[ 'action' ][-1] ids = self.request.arguments["ids"][0].split( "," ) ids = [int( i ) for i in ids ] except KeyError as excp: raise WErr( 400, "Missing %s" % excp ) tsClient = TransformationClient() agentType = 'Manual' if cmd == 'clean': status = 'Cleaning' elif cmd == 'start': status = 'Active' agentType = 'Automatic' elif cmd == 'flush': status = 'Flush' agentType = 'Automatic' elif cmd == 'stop': status = 'Stopped' elif cmd == 'complete': status = 'Completed' else: self.finish( {"success":"false", "error": "Unknown action"} ) callback = [] for i in ids: try: id = int( i ) result = yield self.threadTask( tsClient.setTransformationParameter, id, 'Status', status ) if result["OK"]: resString = "ProdID: %s set to %s successfully" % ( i, cmd ) result = yield self.threadTask( tsClient.setTransformationParameter, id, 'AgentType', agentType ) if not result["OK"]: resString = "ProdID: %s failed to set to %s: %s" % ( i, cmd, result["Message"] ) else: resString = "ProdID: %s failed due the reason: %s" % ( i, result["Message"] ) except: resString = "Unable to convert given ID %s to transformation ID" % i callback.append( resString ) callback = {"success":"true", "showResult":callback} gLogger.info( cmd, ids ) self.finish( callback )
def web_showFileStatus( self ): callback = {} start = int( self.request.arguments["start"][-1] ) limit = int( self.request.arguments["limit"][-1] ) try: id = self.request.arguments[ 'transformationId' ][-1] status = self.request.arguments[ 'status' ][-1] except KeyError as excp: raise WErr( 400, "Missing %s" % excp ) tsClient = TransformationClient() result = yield self.threadTask( tsClient.getTransformationFilesSummaryWeb, {'TransformationID':id, 'Status':status}, [["FileID", "ASC"]], start, limit ) if not result['OK']: callback = {"success":"false", "error":result["Message"]} else: result = result["Value"] if result.has_key( "TotalRecords" ) and result["TotalRecords"] > 0: if result.has_key( "ParameterNames" ) and result.has_key( "Records" ): if len( result["ParameterNames"] ) > 0: if len( result["Records"] ) > 0: callback = [] jobs = result["Records"] head = result["ParameterNames"] headLength = len( head ) for i in jobs: tmp = {} for j in range( 0, headLength ): tmp[head[j]] = i[j] callback.append( tmp ) total = result["TotalRecords"] timestamp = Time.dateTime().strftime( "%Y-%m-%d %H:%M [UTC]" ) if result.has_key( "Extras" ): extra = result["Extras"] callback = {"success":"true", "result":callback, "total":total, "extra":extra, "date":timestamp} else: callback = {"success":"true", "result":callback, "total":total, "date":timestamp} else: callback = {"success":"false", "result":"", "error":"There are no data to display"} else: callback = {"success":"false", "result":"", "error":"ParameterNames field is undefined"} else: callback = {"success":"false", "result":"", "error":"Data structure is corrupted"} else: callback = {"success":"false", "result":"", "error":"There were no data matching your selection"} self.finish( callback )
def __init__(self, plugin, transClient=None, dataManager=None): """ plugin name has to be passed in: it will then be executed as one of the functions below, e.g. plugin = 'BySize' will execute TransformationPlugin('BySize')._BySize() """ super(TransformationPlugin, self).__init__(plugin) self.data = {} self.files = False self.startTime = time.time() if transClient is None: transClient = TransformationClient() if dataManager is None: dataManager = DataManager() self.util = PluginUtilities(plugin, transClient, dataManager)
def __init__(self, plugin, transClient=None, replicaManager=None): """ plugin name has to be passed in: it will then be executed as one of the functions below, e.g. plugin = 'BySize' will execute TransformationPlugin('BySize')._BySize() """ self.params = {} self.data = {} self.plugin = plugin self.files = False if transClient is None: self.transClient = TransformationClient() else: self.transClient = transClient if replicaManager is None: self.rm = ReplicaManager() else: self.rm = replicaManager
class TaskBase: def __init__(self): self.transClient = TransformationClient() def prepareTransformationTasks(self, transBody, taskDict, owner='', ownerGroup=''): return S_ERROR("Not implemented") def submitTransformationTasks(self, taskDict): return S_ERROR("Not implemented") def submitTasksToExternal(self, task): return S_ERROR("Not implemented") def updateDBAfterTaskSubmission(self, taskDict): updated = 0 startTime = time.time() for taskID in sortList(taskDict.keys()): transID = taskDict[taskID]['TransformationID'] if taskDict[taskID]['Success']: res = self.transClient.setTaskStatusAndWmsID( transID, taskID, 'Submitted', str(taskDict[taskID]['ExternalID'])) if not res['OK']: gLogger.warn( "updateDBAfterSubmission: Failed to update task status after submission", "%s %s" % (taskDict[taskID]['ExternalID'], res['Message'])) updated += 1 gLogger.info( "updateDBAfterSubmission: Updated %d tasks in %.1f seconds" % (updated, time.time() - startTime)) return S_OK() def updateTransformationReservedTasks(self, taskDicts): return S_ERROR("Not implemented") def getSubmittedTaskStatus(self, taskDicts): return S_ERROR("Not implemented") def getSubmittedFileStatus(self, fileDicts): return S_ERROR("Not implemented")
def _getClients(self, ownerDN=None, ownerGroup=None): """Returns the clients used in the threads This is another function that should be extended. The clients provided here are defaults, and should be adapted If ownerDN and ownerGroup are not None the clients will delegate to these credentials :param str ownerDN: DN of the owner of the submitted jobs :param str ownerGroup: group of the owner of the submitted jobs :returns: dict of Clients """ threadTransformationClient = TransformationClient() threadTaskManager = WorkflowTasks(ownerDN=ownerDN, ownerGroup=ownerGroup) threadTaskManager.pluginLocation = self.pluginLocation return {"TransformationClient": threadTransformationClient, "TaskManager": threadTaskManager}
def setUp(self): self.prodClient = ProductionClient() self.transClient = TransformationClient() self.fc = FileCatalog() # ## Add metadata fields to the DFC self.MDFieldDict = { "particle": "VARCHAR(128)", "analysis_prog": "VARCHAR(128)", "tel_sim_prog": "VARCHAR(128)", "outputType": "VARCHAR(128)", "zenith": "int", "data_level": "int", } for MDField in self.MDFieldDict: MDFieldType = self.MDFieldDict[MDField] res = self.fc.addMetadataField(MDField, MDFieldType) self.assert_(res["OK"])
def setUp(self): self.prodClient = ProductionClient() self.transClient = TransformationClient() self.fc = FileCatalog() # ## Add metadata fields to the DFC self.MDFieldDict = { 'particle': 'VARCHAR(128)', 'analysis_prog': 'VARCHAR(128)', 'tel_sim_prog': 'VARCHAR(128)', 'outputType': 'VARCHAR(128)', 'zenith': 'int', 'data_level': 'int' } for MDField in self.MDFieldDict: MDFieldType = self.MDFieldDict[MDField] res = self.fc.addMetadataField(MDField, MDFieldType) self.assert_(res['OK'])
def web_setSite( self ): callback = {} try: transID = int( self.request.arguments[ 'TransformationId' ][-1] ) runID = int( self.request.arguments[ 'RunNumber' ][-1] ) site = self.request.arguments[ 'Site' ][-1] except KeyError as excp: raise WErr( 400, "Missing %s" % excp ) gLogger.info( "\033[0;31m setTransformationRunsSite(%s, %s, %s) \033[0m" % ( transID, runID, site ) ) tsClient = TransformationClient() result = yield self.threadTask( tsClient.setTransformationRunsSite, transID, runID, site ) if result["OK"]: callback = {"success":"true", "result":"true"} else: callback = {"success":"false", "error":result["Message"]} self.finish( callback )
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(["transID: transformation ID"]) _, args = Script.parseCommandLine() transIDs = [int(arg) for arg in args] from DIRAC.TransformationSystem.Agent.ValidateOutputDataAgent import ValidateOutputDataAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient agent = ValidateOutputDataAgent( "Transformation/ValidateOutputDataAgent", "Transformation/ValidateOutputDataAgent", "dirac-transformation-verify-outputdata", ) agent.initialize() client = TransformationClient() for transID in transIDs: agent.checkTransformationIntegrity(transID)
def main(): Script.parseCommandLine() args = Script.getPositionalArgs() if not args: Script.showHelp() transIDs = [int(arg) for arg in args] from DIRAC.TransformationSystem.Agent.ValidateOutputDataAgent import ValidateOutputDataAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient agent = ValidateOutputDataAgent('Transformation/ValidateOutputDataAgent', 'Transformation/ValidateOutputDataAgent', 'dirac-transformation-verify-outputdata') agent.initialize() client = TransformationClient() for transID in transIDs: agent.checkTransformationIntegrity(transID)
def main(): # Registering arguments will automatically add their description to the help menu Script.registerArgument(["transID: transformation ID"]) _, args = Script.parseCommandLine() transIDs = [int(arg) for arg in args] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient agent = TransformationCleaningAgent( "Transformation/TransformationCleaningAgent", "Transformation/TransformationCleaningAgent", "dirac-transformation-archive", ) agent.initialize() client = TransformationClient() for transID in transIDs: agent.archiveTransformation(transID)
def setUp(self): self.mockTransClient = MagicMock() self.mockTransClient.setTaskStatusAndWmsID.return_value = {'OK': True} self.WMSClientMock = MagicMock() self.jobMonitoringClient = MagicMock() self.mockReqClient = MagicMock() self.jobMock = MagicMock() self.jobMock2 = MagicMock() mockWF = MagicMock() mockPar = MagicMock() mockWF.findParameter.return_value = mockPar mockPar.getValue.return_value = 'MySite' self.jobMock2.workflow = mockWF self.jobMock2.setDestination.return_value = {'OK': True} self.jobMock.workflow.return_value = '' self.jobMock.return_value = self.jobMock2 self.reqValidatorMock = MagicMock() self.reqValidatorMock.validate.return_value = {'OK': True} self.taskBase = TaskBase(transClient=self.mockTransClient) self.pu = PluginUtilities(transClient=self.mockTransClient) self.wfTasks = WorkflowTasks( transClient=self.mockTransClient, submissionClient=self.WMSClientMock, jobMonitoringClient=self.jobMonitoringClient, outputDataModule="mock") self.requestTasks = RequestTasks(transClient=self.mockTransClient, requestClient=self.mockReqClient, requestValidator=reqValFake) self.tc = TransformationClient() self.transformation = Transformation() self.maxDiff = None gLogger.setLevel('DEBUG')
class SetFileStatus(OperationHandlerBase): """ .. class:: SetFileStatus SetFileStatus operation handler """ def __init__(self, operation=None, csPath=None): """c'tor :param self: self reference :param Operation operation: Operation instance :param str csPath: CS path for this handler """ OperationHandlerBase.__init__(self, operation, csPath) def __call__(self): """ It expects to find the arguments for tc.setFileStatusForTransformation in operation.Arguments """ try: setFileStatusDict = DEncode.decode(self.operation.Arguments)[0] self.log.debug("decoded filStatusDict=%s" % str(setFileStatusDict)) except ValueError, error: self.log.exception(error) self.operation.Error = str(error) self.operation.Status = "Failed" return S_ERROR(str(error)) tc = TransformationClient() setStatus = tc.setFileStatusForTransformation( setFileStatusDict['transformation'], setFileStatusDict['statusDict'], setFileStatusDict['force']) if not setStatus['OK']: errorStr = "failed to change status: %s" % setStatus['Message'] self.operation.Error = errorStr self.log.warn(errorStr) return S_ERROR(self.operation.Error) else: self.operation.Status = "Done" return S_OK()
def main(): Script.parseCommandLine() args = Script.getPositionalArgs() if not args: Script.showHelp() transIDs = [int(arg) for arg in args] from DIRAC.TransformationSystem.Agent.TransformationCleaningAgent import TransformationCleaningAgent from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient agent = TransformationCleaningAgent( 'Transformation/TransformationCleaningAgent', 'Transformation/TransformationCleaningAgent', 'dirac-transformation-remove-output') agent.initialize() client = TransformationClient() for transID in transIDs: agent.removeTransformationOutput(transID)