class ProductionJob(Job): #pylint: disable=too-many-public-methods, too-many-instance-attributes """ Production job class. Suitable for CLIC studies. Need to sub class and overload for other clients. """ def __init__(self, script=None): super(ProductionJob, self).__init__(script) self.prodVersion = __RCSID__ self.dryrun = False self.created = False self.checked = False self.call_finalization = False self.finalsdict = {} self.transfid = 0 self.type = 'Production' self.csSection = '/Production/Defaults' self.ops = Operations() self.fc = FileCatalogClient() self.trc = TransformationClient() self.defaultProdID = '12345' self.defaultProdJobID = '12345' self.jobFileGroupSize = 1 self.nbtasks = 1 self.slicesize = 0 self.basename = '' self.basepath = self.ops.getValue('/Production/CLIC/BasePath', '/ilc/prod/clic/') self.evttype = '' self.datatype = '' self.energycat = '' self.detector = '' self.currtrans = None self.description = '' self.finalpaths = [] self.finalMetaDict = defaultdict(dict) self.prodMetaDict = {} self.finalMetaDictNonSearch = {} self.metadict_external = {} self.outputStorage = '' self.proxyinfo = getProxyInfo() self.inputdataquery = False self.inputBKSelection = {} self.plugin = 'Standard' self.prodGroup = '' self.prodTypes = [ 'MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction', 'MCReconstruction_Overlay', 'Merge', 'Split', 'MCGeneration_ILD', 'MCSimulation_ILD', 'MCReconstruction_ILD', 'MCReconstruction_Overlay_ILD', 'Split_ILD' ] self.prodparameters = {} self.prodparameters['NbInputFiles'] = 1 self.prodparameters['nbevts'] = 0 #self.prodparameters["SWPackages"] = '' self._addParameter(self.workflow, "IS_PROD", 'JDL', True, "This job is a production job") if not script: self.__setDefaults() self._recBasePaths = {} ############################################################################# def __setDefaults(self): """Sets some default parameters. """ self.setPlatform( self.ops.getValue('%s/Platform' % (self.csSection), 'x86_64-slc5-gcc43-opt')) self.setCPUTime('300000') self.setLogLevel('verbose') self.setJobGroup('@{PRODUCTION_ID}') #version control self._setParameter('productionVersion', 'string', self.prodVersion, 'ProdAPIVersion') #General workflow parameters self._setParameter('PRODUCTION_ID', 'string', self.defaultProdID.zfill(8), 'ProductionID') self._setParameter('JOB_ID', 'string', self.defaultProdJobID.zfill(8), 'ProductionJobID') self._setParameter('Priority', 'JDL', '1', 'Priority') self._setParameter('emailAddress', 'string', '*****@*****.**', 'CrashEmailAddress') def _setParameter(self, name, parameterType, parameterValue, description): """Set parameters checking in CS in case some defaults need to be changed. """ if self.ops.getValue('%s/%s' % (self.csSection, name), ''): self.log.debug( 'Setting %s from CS defaults = %s' % (name, self.ops.getValue('%s/%s' % (self.csSection, name)))) self._addParameter( self.workflow, name, parameterType, self.ops.getValue('%s/%s' % (self.csSection, name), 'default'), description) else: self.log.debug('Setting parameter %s = %s' % (name, parameterValue)) self._addParameter(self.workflow, name, parameterType, parameterValue, description) def setConfig(self, version): """ Define the Configuration package to obtain """ appName = 'ILDConfig' self._addSoftware(appName.lower(), version) self.prodparameters['ILDConfigVersion'] = version self._addParameter(self.workflow, 'ILDConfigPackage', 'JDL', appName + version, 'ILDConfig package') return S_OK() def setClicConfig(self, version): """ Define the ClicConfig package to obtain """ appName = 'ClicConfig' self._addSoftware(appName.lower(), version) self._addParameter(self.workflow, 'ClicConfigPackage', 'JDL', appName + version, 'ClicConfig package') self.prodparameters['ClicConfigVersion'] = version return S_OK() def setDryRun(self, run): """ In case one wants to get all the info as if the prod was being submitted """ self.dryrun = run ############################################################################# def setProdGroup(self, group): """ Sets a user defined tag for the production as appears on the monitoring page """ self.prodGroup = group ############################################################################# def setProdPlugin(self, plugin): """ Sets the plugin to be used to creating the production jobs """ self.plugin = plugin ############################################################################# def setJobFileGroupSize(self, files): """ Sets the number of files to be input to each job created. """ if self.checked: return self._reportError( "This input is needed at the beginning of the production definition: it is \ needed for total number of evts.") self.jobFileGroupSize = files self.prodparameters['NbInputFiles'] = files def setNbEvtsPerSlice(self, nbevts): """ Define the number of events in a slice. """ self.slicesize = nbevts ############################################################################# def setProdType(self, prodType): """Set prod type. """ if prodType not in self.prodTypes: raise TypeError('Prod must be one of %s' % (', '.join(self.prodTypes))) self.setType(prodType) ############################################################################# def setWorkflowName(self, name): """Set workflow name. """ self.workflow.setName(name) self.name = name ############################################################################# def setWorkflowDescription(self, desc): """Set workflow name. """ self.workflow.setDescription(desc) ############################################################################# def createWorkflow(self): """ Create XML for local testing. """ name = '%s.xml' % self.name if os.path.exists(name): shutil.move(name, '%s.backup' % name) self.workflow.toXMLFile(name) ############################################################################# def setOutputSE(self, outputse): """ Define where the output file(s) will go. """ self.outputStorage = outputse return S_OK() ############################################################################# def setInputDataQuery(self, metadata): """ Define the input data query needed """ retMetaKey = self._checkMetaKeys(metadata.keys()) if not retMetaKey['OK']: return retMetaKey if "ProdID" not in metadata: return self._reportError( "Input metadata dictionary must contain at least a key 'ProdID' as reference" ) retDirs = self._checkFindDirectories(metadata) if not retDirs['OK']: return retDirs dirs = retDirs['Value'].values() for mdir in dirs: gLogger.notice("Directory: %s" % mdir) res = self.fc.getDirectoryUserMetadata(mdir) if not res['OK']: return self._reportError( "Error looking up the catalog for directory metadata") compatmeta = res['Value'] compatmeta.update(metadata) if 'EvtType' in compatmeta: self.evttype = JobHelpers.getValue(compatmeta['EvtType'], str, basestring) else: return self._reportError( "EvtType is not in the metadata, it has to be!") if 'NumberOfEvents' in compatmeta: self.nbevts = JobHelpers.getValue(compatmeta['NumberOfEvents'], int, None) self.basename = self.evttype gLogger.notice("MetaData: %s" % compatmeta) gLogger.notice("MetaData: %s" % metadata) if "Energy" in compatmeta: self.energycat = JobHelpers.getValue(compatmeta["Energy"], str, (int, long, basestring)) if self.energycat.count("tev"): self.energy = Decimal("1000.") * Decimal( self.energycat.split("tev")[0]) elif self.energycat.count("gev"): self.energy = Decimal("1.") * Decimal( self.energycat.split("gev")[0]) else: self.energy = Decimal("1.") * Decimal(self.energycat) gendata = False if 'Datatype' in compatmeta: self.datatype = JobHelpers.getValue(compatmeta['Datatype'], str, basestring) if self.datatype == 'gen': gendata = True if "DetectorType" in compatmeta and not gendata: self.detector = JobHelpers.getValue(compatmeta["DetectorType"], str, basestring) self.inputBKSelection = metadata self.inputdataquery = True self.prodparameters['nbevts'] = self.nbevts self.prodparameters["FCInputQuery"] = self.inputBKSelection return S_OK() def setDescription(self, desc): """ Set the production's description :param string desc: Description """ self.description = desc return S_OK() def getBasePath(self): """ Return the base path. Updated by :any:`setInputDataQuery`. """ return self.basepath def addFinalization(self, uploadData=False, registerData=False, uploadLog=False, sendFailover=False): """ Add finalization step :param bool uploadData: Upload or not the data to the storage :param bool uploadLog: Upload log file to storage (currently only available for admins, thus add them to OutputSandbox) :param bool sendFailover: Send Failover requests, and declare files as processed or unused in transfDB :param bool registerData: Register data in the file catalog """ #TODO: Do the registration only once, instead of once for each job self.call_finalization = True self.finalsdict['uploadData'] = uploadData self.finalsdict['registerData'] = registerData self.finalsdict['uploadLog'] = uploadLog self.finalsdict['sendFailover'] = sendFailover def _addRealFinalization(self): """ This is called at creation: now that the workflow is created at the last minute, we need to add this also at the last minute """ importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>' dataUpload = ModuleDefinition('UploadOutputData') dataUpload.setDescription('Uploads the output data') self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'UploadOutputData') dataUpload.setBody(body) failoverRequest = ModuleDefinition('FailoverRequest') failoverRequest.setDescription('Sends any failover requests') self._addParameter(failoverRequest, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'FailoverRequest') failoverRequest.setBody(body) registerdata = ModuleDefinition('RegisterOutputData') registerdata.setDescription( 'Module to add in the metadata catalog the relevant info about the files' ) self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'RegisterOutputData') registerdata.setBody(body) logUpload = ModuleDefinition('UploadLogFile') logUpload.setDescription('Uploads the output log files') self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'UploadLogFile') logUpload.setBody(body) finalization = StepDefinition('Job_Finalization') finalization.addModule(dataUpload) up = finalization.createModuleInstance('UploadOutputData', 'dataUpload') up.setValue("enable", self.finalsdict['uploadData']) finalization.addModule(registerdata) ro = finalization.createModuleInstance('RegisterOutputData', 'RegisterOutputData') ro.setValue("enable", self.finalsdict['registerData']) finalization.addModule(logUpload) ul = finalization.createModuleInstance('UploadLogFile', 'logUpload') ul.setValue("enable", self.finalsdict['uploadLog']) finalization.addModule(failoverRequest) fr = finalization.createModuleInstance('FailoverRequest', 'failoverRequest') fr.setValue("enable", self.finalsdict['sendFailover']) self.workflow.addStep(finalization) self.workflow.createStepInstance('Job_Finalization', 'finalization') return S_OK() def createProduction(self, name=None): """ Create production. """ if not self.proxyinfo['OK']: return S_ERROR( "Not allowed to create production, you need a ilc_prod proxy.") if 'group' in self.proxyinfo['Value']: group = self.proxyinfo['Value']['group'] if not group == "ilc_prod": return S_ERROR( "Not allowed to create production, you need a ilc_prod proxy." ) else: return S_ERROR( "Could not determine group, you do not have the right proxy.") if self.created: return S_ERROR("Production already created.") ###We need to add the applications to the workflow res = self._addToWorkflow() if not res['OK']: return res if self.call_finalization: self._addRealFinalization() workflowName = self.workflow.getName() fileName = '%s.xml' % workflowName self.log.verbose('Workflow XML file name is:', '%s' % fileName) try: self.createWorkflow() except Exception as x: self.log.error("Exception creating workflow", repr(x)) return S_ERROR('Could not create workflow') with open(fileName, 'r') as oFile: workflowXML = oFile.read() if not name: name = workflowName res = self.trc.getTransformationStats(name) if res['OK']: return self._reportError( "Transformation with name %s already exists! Cannot proceed." % name) ###Create Tranformation Trans = Transformation() Trans.setTransformationName(name) Trans.setDescription(self.description) Trans.setLongDescription(self.description) Trans.setType(self.type) self.prodparameters['JobType'] = self.type Trans.setPlugin(self.plugin) if self.inputdataquery: Trans.setGroupSize(self.jobFileGroupSize) Trans.setTransformationGroup(self.prodGroup) Trans.setBody(workflowXML) if not self.slicesize: Trans.setEventsPerTask(self.jobFileGroupSize * self.nbevts) else: Trans.setEventsPerTask(self.slicesize) self.currtrans = Trans if self.dryrun: self.log.notice('Would create prod called', name) self.transfid = 12345 else: res = Trans.addTransformation() if not res['OK']: print res['Message'] return res self.transfid = Trans.getTransformationID()['Value'] if self.inputBKSelection: res = self.applyInputDataQuery() if not self.dryrun: Trans.setAgentType("Automatic") Trans.setStatus("Active") finals = [] for finalpaths in self.finalpaths: finalpaths = finalpaths.rstrip("/") finalpaths += "/" + str(self.transfid).zfill(8) finals.append(finalpaths) self.finalMetaDict[finalpaths].update({"ProdID": self.transfid}) self.finalMetaDict[finalpaths].update(self.prodMetaDict) # if 'ILDConfigVersion' in self.prodparameters: # self.finalMetaDict[finalpaths].update({"ILDConfig":self.prodparameters['ILDConfigVersion']}) if self.nbevts: self.finalMetaDict[finalpaths].update( {'NumberOfEvents': self.jobFileGroupSize * self.nbevts}) self.finalpaths = finals self.created = True return S_OK() def setNbOfTasks(self, nbtasks): """ Define the number of tasks you want. Useful for generation jobs. """ if not self.currtrans: self.log.error("Not transformation defined earlier") return S_ERROR("No transformation defined") if self.inputBKSelection and self.plugin not in [ 'Limited', 'SlicedLimited' ]: self.log.error( "Meta data selection activated, should not specify the number of jobs" ) return S_ERROR() self.nbtasks = nbtasks self.currtrans.setMaxNumberOfTasks(self.nbtasks) #pylint: disable=E1101 return S_OK() def applyInputDataQuery(self, metadata=None, prodid=None): """ Tell the production to update itself using the metadata query specified, i.e. submit new jobs if new files are added corresponding to same query. """ if not self.transfid and self.currtrans: self.transfid = self.currtrans.getTransformationID()['Value'] #pylint: disable=E1101 elif prodid: self.transfid = prodid if not self.transfid: print "Not transformation defined earlier" return S_ERROR("No transformation defined") if metadata: self.inputBKSelection = metadata client = TransformationClient() if not self.dryrun: res = client.createTransformationInputDataQuery( self.transfid, self.inputBKSelection) if not res['OK']: return res else: self.log.notice("Would use %s as metadata query for production" % str(self.inputBKSelection)) return S_OK() def addMetadataToFinalFiles(self, metadict): """ Add additionnal non-query metadata """ self.metadict_external = metadict return S_OK() def finalizeProd(self, prodid=None, prodinfo=None): """ Finalize definition: submit to Transformation service and register metadata """ currtrans = 0 if self.currtrans: if not self.dryrun: currtrans = self.currtrans.getTransformationID()['Value'] #pylint: disable=E1101 else: currtrans = 12345 if prodid: currtrans = prodid if not currtrans: print "Not transformation defined earlier" return S_ERROR("No transformation defined") if prodinfo: self.prodparameters = prodinfo info = [] info.append('%s Production %s has following parameters:\n' % (self.prodparameters['JobType'], currtrans)) if "Process" in self.prodparameters: info.append('- Process %s' % self.prodparameters['Process']) if "Energy" in self.prodparameters: info.append('- Energy %s GeV' % self.prodparameters["Energy"]) if not self.slicesize: self.prodparameters['nbevts'] = self.jobFileGroupSize * self.nbevts else: self.prodparameters['nbevts'] = self.slicesize if self.prodparameters['nbevts']: info.append("- %s events per job" % (self.prodparameters['nbevts'])) if self.prodparameters.get('lumi', False): info.append(' corresponding to a luminosity %s fb' % (self.prodparameters['lumi'] * \ self.prodparameters['NbInputFiles'])) if 'FCInputQuery' in self.prodparameters: info.append('Using InputDataQuery :') for key, val in self.prodparameters['FCInputQuery'].iteritems(): info.append(' %s = %s' % (key, val)) if "SWPackages" in self.prodparameters: info.append('- SW packages %s' % self.prodparameters["SWPackages"]) if "SoftwareTag" in self.prodparameters: info.append('- SW tags %s' % self.prodparameters["SoftwareTag"]) if "ILDConfigVersion" in self.prodparameters: info.append('- ILDConfig %s' % self.prodparameters['ILDConfigVersion']) if 'ClicConfigVersion' in self.prodparameters: info.append('- ClicConfig %s' % self.prodparameters['ClicConfigVersion']) if 'extraCLIArguments' in self.prodparameters: info.append('- ExtraCLIArguments %s' % self.prodparameters['extraCLIArguments']) # as this is the very last call all applications are registered, so all software packages are known #add them the the metadata registration for finalpath in self.finalpaths: if finalpath not in self.finalMetaDictNonSearch: self.finalMetaDictNonSearch[finalpath] = {} if "SWPackages" in self.prodparameters: self.finalMetaDictNonSearch[finalpath][ "SWPackages"] = self.prodparameters["SWPackages"] if self.metadict_external: self.finalMetaDictNonSearch[finalpath].update( self.metadict_external) info.append('- Registered metadata: ') for path, metadata in sorted(self.finalMetaDict.iteritems()): info.append(' %s = %s' % (path, metadata)) info.append('- Registered non searchable metadata: ') for path, metadata in sorted(self.finalMetaDictNonSearch.iteritems()): info.append(' %s = %s' % (path, metadata)) pprint.pprint(info) infoString = '\n'.join(info) self.prodparameters['DetailedInfo'] = infoString for name, val in self.prodparameters.iteritems(): result = self._setProdParameter(currtrans, name, val) if not result['OK']: self.log.error(result['Message']) res = self._registerMetadata() if not res['OK']: self.log.error("Could not register the following directories :", "%s" % str(res)) return S_OK() ############################################################################# def _registerMetadata(self): """ Private method Register path and metadata before the production actually runs. This allows for the definition of the full chain in 1 go. """ prevent_registration = self.ops.getValue( "Production/PreventMetadataRegistration", False) if self.dryrun or prevent_registration: self.log.notice( "Would have created and registered the following\n", "\n ".join([ " * %s: %s" % (par, val) for par, val in self.finalMetaDict.iteritems() ])) self.log.notice("Would have set this as non searchable metadata", str(self.finalMetaDictNonSearch)) return S_OK() failed = [] for path, meta in self.finalMetaDict.items(): result = self.fc.createDirectory(path) if result['OK']: if result['Value']['Successful']: if path in result['Value']['Successful']: self.log.verbose("Successfully created directory:", "%s" % path) res = self.fc.changePathMode({path: 0o775}, False) if not res['OK']: self.log.error(res['Message']) failed.append(path) elif result['Value']['Failed']: if path in result['Value']['Failed']: self.log.error( 'Failed to create directory:', "%s" % str(result['Value']['Failed'][path])) failed.append(path) else: self.log.error('Failed to create directory:', result['Message']) failed.append(path) ## Get existing metadata, if it is the same don't set it again, otherwise throw error existingMetadata = self.fc.getDirectoryUserMetadata( path.rstrip("/")) metaCopy = dict(meta) if existingMetadata['OK']: failure = False for key, value in existingMetadata['Value'].iteritems(): if key in meta and meta[key] != value: self.log.error( "Metadata values for folder %s disagree for key %s: Existing(%r), new(%r)" % (path, key, value, meta[key])) failure = True elif key in meta and meta[key] == value: metaCopy.pop(key, None) if failure: return S_ERROR( "Error when setting new metadata, already existing metadata disagrees!" ) result = self.fc.setMetadata(path.rstrip("/"), metaCopy) if not result['OK']: self.log.error("Could not preset metadata", "%s" % str(metaCopy)) self.log.error("Could not preset metadata", "%s" % result['Message']) for path, meta in self.finalMetaDictNonSearch.items(): result = self.fc.createDirectory(path) if result['OK']: if result['Value']['Successful']: if path in result['Value']['Successful']: self.log.verbose("Successfully created directory:", "%s" % path) res = self.fc.changePathMode({path: 0o775}, False) if not res['OK']: self.log.error(res['Message']) failed.append(path) elif result['Value']['Failed']: if path in result['Value']['Failed']: self.log.error( 'Failed to create directory:', "%s" % str(result['Value']['Failed'][path])) failed.append(path) else: self.log.error('Failed to create directory:', result['Message']) failed.append(path) result = self.fc.setMetadata(path.rstrip("/"), meta) if not result['OK']: self.log.error("Could not preset metadata", "%s" % str(meta)) if failed: return {'OK': False, 'Failed': failed} return S_OK() def getMetadata(self): """ Return the corresponding metadata of the last step """ metadict = {} for meta in self.finalMetaDict.values(): metadict.update(meta) if 'NumberOfEvents' in metadict: del metadict[ 'NumberOfEvents'] #As this is not supposed to be a searchable thing return metadict def _setProdParameter(self, prodID, pname, pvalue): """ Set a production parameter. """ if isinstance(pvalue, list): pvalue = '\n'.join(pvalue) prodClient = RPCClient('Transformation/TransformationManager', timeout=120) if isinstance(pvalue, (int, long)): pvalue = str(pvalue) if not self.dryrun: result = prodClient.setTransformationParameter( int(prodID), str(pname), str(pvalue)) if not result['OK']: self.log.error( 'Problem setting parameter %s for production %s and value:\n%s' % (prodID, pname, pvalue)) else: self.log.notice("Adding %s=%s to transformation" % (str(pname), str(pvalue))) result = S_OK() return result def _jobSpecificParams(self, application): """ For production additional checks are needed: ask the user """ if self.created: return S_ERROR( "The production was created, you cannot add new applications to the job." ) if not application.logFile: logf = application.appname + "_" + application.version + "_@{STEP_ID}.log" res = application.setLogFile(logf) if not res['OK']: return res #in fact a bit more tricky as the log files have the prodID and jobID in them ### Retrieve from the application the essential info to build the prod info. if not self.nbevts and not self.slicesize: self.nbevts = application.numberOfEvents if not self.nbevts: return S_ERROR("Number of events to process is not defined.") elif not application.numberOfEvents: if not self.slicesize: res = application.setNumberOfEvents(self.jobFileGroupSize * self.nbevts) else: res = application.setNumberOfEvents(self.slicesize) if not res['OK']: return res if application.numberOfEvents > 0 and ( self.jobFileGroupSize * self.nbevts > application.numberOfEvents or self.slicesize > application.numberOfEvents): self.nbevts = application.numberOfEvents if not self.energy: if application.energy: self.energy = Decimal(str(application.energy)) else: return S_ERROR( "Could not find the energy defined, it is needed for the production definition." ) elif not application.energy: res = application.setEnergy(float(self.energy)) if not res['OK']: return res if self.energy: self._setParameter("Energy", "float", float(self.energy), "Energy used") self.prodparameters["Energy"] = float(self.energy) if not self.evttype: if hasattr(application, 'eventType'): self.evttype = application.eventType else: return S_ERROR( "Event type not found nor specified, it's mandatory for the production paths." ) self.prodparameters['Process'] = self.evttype if not self.outputStorage: return S_ERROR("You need to specify the Output storage element") curpackage = "%s.%s" % (application.appname, application.version) if "SWPackages" in self.prodparameters: if not self.prodparameters["SWPackages"].count(curpackage): self.prodparameters["SWPackages"] += ";%s" % (curpackage) else: self.prodparameters["SWPackages"] = "%s" % (curpackage) if not application.accountInProduction: res = self._updateProdParameters(application) if not res['OK']: return res self.checked = True return S_OK() res = application.setOutputSE(self.outputStorage) if not res['OK']: return res energypath = self.getEnergyPath() if not self.basename: self.basename = self.evttype evttypepath = '' if not self.evttype[-1] == '/': evttypepath = self.evttype + '/' path = self.basepath ###Need to resolve file names and paths if self.energy: self.finalMetaDict[self.basepath + energypath] = { "Energy": str(self.energy) } if hasattr(application, "setOutputRecFile") and not application.willBeCut: path = self.basepath + energypath + evttypepath + application.detectortype + "/REC" self.finalMetaDict[self.basepath + energypath + evttypepath] = { "EvtType": self.evttype } self.finalMetaDict[self.basepath + energypath + evttypepath + application.detectortype] = { "DetectorType": application.detectortype } self.finalMetaDict[self.basepath + energypath + evttypepath + application.detectortype + "/REC"] = { 'Datatype': "REC" } fname = self.basename + "_rec.slcio" application.setOutputRecFile(fname, path) self.log.info("Will store the files under", "%s" % path) self.finalpaths.append(path) path = self.basepath + energypath + evttypepath + application.detectortype + "/DST" self.finalMetaDict[self.basepath + energypath + evttypepath + application.detectortype + "/DST"] = { 'Datatype': "DST" } fname = self.basename + "_dst.slcio" application.setOutputDstFile(fname, path) self.log.info("Will store the files under", "%s" % path) self.finalpaths.append(path) elif hasattr(application, "outputFile") and hasattr( application, 'datatype' ) and not application.outputFile and not application.willBeCut: path = self.basepath + energypath + evttypepath self.finalMetaDict[path] = {"EvtType": self.evttype} if hasattr(application, "detectortype"): if application.detectortype: path += application.detectortype self.finalMetaDict[path] = { "DetectorType": application.detectortype } path += '/' elif self.detector: path += self.detector self.finalMetaDict[path] = {"DetectorType": self.detector} path += '/' if not application.datatype and self.datatype: application.datatype = self.datatype path += application.datatype self.finalMetaDict[path] = {'Datatype': application.datatype} self.log.info("Will store the files under", "%s" % path) self.finalpaths.append(path) extension = 'stdhep' if application.datatype in ['SIM', 'REC']: extension = 'slcio' fname = self.basename + "_%s" % ( application.datatype.lower()) + "." + extension application.setOutputFile(fname, path) self.basepath = path res = self._updateProdParameters(application) if not res['OK']: return res self.checked = True return S_OK() def _updateProdParameters(self, application): """ Update the prod parameters stored in the production parameters visible from the web """ try: self.prodparameters.update(application.prodparameters) except Exception as x: return S_ERROR("Exception: %r" % x) if hasattr(application, 'extraCLIArguments') and application.extraCLIArguments: self.prodparameters['extraCLIArguments'] = repr( application.extraCLIArguments) return S_OK() def _jobSpecificModules(self, application, step): return application._prodjobmodules(step) def getEnergyPath(self): """returns the energy path 250gev or 3tev or 1.4tev etc.""" energy = Decimal(str(self.energy)) tD = Decimal('1000.0') unit = 'gev' if energy < tD else 'tev' energy = energy if energy < tD else energy / tD if float(energy).is_integer(): energyPath = str(int(energy)) else: energyPath = "%1.1f" % energy energyPath = energyPath + unit + '/' self.log.info("Energy path is: ", energyPath) return energyPath def _checkMetaKeys(self, metakeys, extendFileMeta=False): """ check if metadata keys are allowed to be metadata :param list metakeys: metadata keys for production metadata :param bool extendFileMeta: also use FileMetaFields for checking meta keys :returns: S_OK, S_ERROR """ res = self.fc.getMetadataFields() if not res['OK']: print "Could not contact File Catalog" return S_ERROR("Could not contact File Catalog") metaFCkeys = res['Value']['DirectoryMetaFields'].keys() if extendFileMeta: metaFCkeys.extend(res['Value']['FileMetaFields'].keys()) for key in metakeys: for meta in metaFCkeys: if meta != key and meta.lower() == key.lower(): return self._reportError( "Key syntax error %r, should be %r" % (key, meta), name=self.__class__.__name__) if key not in metaFCkeys: return self._reportError( "Key %r not found in metadata keys, allowed are %r" % (key, metaFCkeys)) return S_OK() def _checkFindDirectories(self, metadata): """ find directories by metadata and check that there are directories found :param dict metadata: metadata dictionary :returns: S_OK, S_ERROR """ res = self.fc.findDirectoriesByMetadata(metadata) if not res['OK']: return self._reportError( "Error looking up the catalog for available directories") elif len(res['Value']) < 1: return self._reportError( 'Could not find any directories corresponding to the query issued' ) return res def setReconstructionBasePaths(self, recPath, dstPath): """ set the output Base paths for the reconstruction REC and DST files """ self._recBasePaths['REC'] = recPath self._recBasePaths['DST'] = dstPath
class TransformationCLI( cmd.Cmd, API ): def __init__( self ): self.server = TransformationClient() self.indentSpace = 4 cmd.Cmd.__init__( self ) API.__init__( self ) def printPair( self, key, value, separator = ":" ): valueList = value.split( "\n" ) print "%s%s%s %s" % ( key, " " * ( self.indentSpace - len( key ) ), separator, valueList[0].strip() ) for valueLine in valueList[ 1:-1 ]: print "%s %s" % ( " " * self.indentSpace, valueLine.strip() ) def do_exit( self, args ): """ Exits the shell. usage: exit """ sys.exit( 0 ) def do_quit( self, *args ): """ Exits the shell. Usage: quit """ sys.exit( 0 ) def do_help( self, args ): """ Default version of the help command Usage: help <command> OR use helpall to see description for all commans""" cmd.Cmd.do_help( self, args ) # overriting default help command def do_helpall( self, args ): """ Shows help information Usage: helpall <command> If no command is specified all commands are shown """ if len( args ) == 0: print "\nAvailable commands:\n" attrList = dir( self ) attrList.sort() for attribute in attrList: if attribute.find( "do_" ) == 0: self.printPair( attribute[ 3: ], getattr( self, attribute ).__doc__[ 1: ] ) print "" else: command = args.split()[0].strip() try: obj = getattr( self, "do_%s" % command ) except: print "There's no such %s command" % command return self.printPair( command, obj.__doc__[1:] ) def do_shell( self, args ): """Execute a shell command usage !<shell_command> """ comm = args res = shellCall( 0, comm ) if res['OK'] and res['Value'][0] == 0: returnCode, stdOut, stdErr = res['Value'] print "%s\n%s" % ( stdOut, stdErr ) else: print res['Message'] def check_params( self, args, num ): """Checks if the number of parameters correct""" argss = string.split( args ) length = len( argss ) if length < num: print "Error: Number of arguments provided %d less that required %d, please correct." % ( length, num ) return ( False, length ) return ( argss, length ) def check_id_or_name( self, id_or_name ): """resolve name or Id by converting type of argument """ if id_or_name.isdigit(): return long( id_or_name ) # its look like id return id_or_name def do_setServer( self, args ): """ Set the destination server usage: setServer serverURL """ argss = string.split( args ) if len( argss ) == 0: print "no server provided" self.serverURL = argss[0] self.server.setServer( self.serverURL ) #################################################################### # # These are the methods for transformation manipulation # def do_getall( self, args ): """Get transformation details usage: getall [Status] [Status] """ oTrans = Transformation() oTrans.setServer( self.serverURL ) oTrans.getTransformations( transStatus = string.split( args ), printOutput = True ) def do_getStatus( self, args ): """Get transformation details usage: getStatus <transName|ID> """ argss = string.split( args ) if not len( argss ) > 0: print "no transformation supplied" return for transName in argss: res = self.server.getTransformation( transName ) if not res['OK']: print "Getting status of %s failed: %s" % ( transName, res['Message'] ) else: print "%s: %s" % ( transName, res['Value']['Status'] ) def do_setStatus( self, args ): """Set transformation status usage: setStatus <Status> <transName|ID> Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'> """ argss = string.split( args ) if not len( argss ) > 1: print "transformation and status not supplied" return status = argss[0] transNames = argss[1:] for transName in transNames: res = self.server.setTransformationParameter( transName, 'Status', status ) if not res['OK']: print "Setting status of %s failed: %s" % ( transName, res['Message'] ) else: print "%s set to %s" % ( transName, status ) def do_start( self, args ): """Start transformation usage: start <transName|ID> """ argss = string.split( args ) if not len( argss ) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'Status', 'Active' ) if not res['OK']: print "Setting Status of %s failed: %s" % ( transName, res['Message'] ) else: res = self.server.setTransformationParameter( transName, 'AgentType', 'Automatic' ) if not res['OK']: print "Setting AgentType of %s failed: %s" % ( transName, res['Message'] ) else: print "%s started" % transName def do_stop( self, args ): """Stop transformation usage: stop <transID|ID> """ argss = string.split( args ) if not len( argss ) > 0: print "no transformation supplied" return if not len( argss ) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'AgentType', 'Manual' ) if not res['OK']: print "Stopping of %s failed: %s" % ( transName, res['Message'] ) else: print "%s stopped" % transName def do_flush( self, args ): """Flush transformation usage: flush <transName|ID> """ argss = string.split( args ) if not len( argss ) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'Status', 'Flush' ) if not res['OK']: print "Flushing of %s failed: %s" % ( transName, res['Message'] ) else: print "%s flushing" % transName def do_get( self, args ): """Get transformation definition usage: get <transName|ID> """ argss = string.split( args ) if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation( transName ) if not res['OK']: print "Failed to get %s: %s" % ( transName, res['Message'] ) else: res['Value'].pop( 'Body' ) printDict( res['Value'] ) def do_getBody( self, args ): """Get transformation body usage: getBody <transName|ID> """ argss = string.split( args ) if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation( transName ) if not res['OK']: print "Failed to get %s: %s" % ( transName, res['Message'] ) else: print res['Value']['Body'] def do_getFileStat( self, args ): """Get transformation file statistics usage: getFileStat <transName|ID> """ argss = string.split( args ) if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformationStats( transName ) if not res['OK']: print "Failed to get statistics for %s: %s" % ( transName, res['Message'] ) else: res['Value'].pop( 'Total' ) printDict( res['Value'] ) def do_modMask( self, args ): """Modify transformation input definition usage: modInput <mask> <transName|ID> """ argss = string.split( args ) if not len( argss ) > 0: print "no transformation supplied" return mask = argss[0] transNames = argss[1:] for transName in transNames: res = self.server.setTransformationParameter( transName, "FileMask", mask ) if not res['OK']: print "Failed to modify input file mask for %s: %s" % ( transName, res['Message'] ) else: print "Updated %s filemask" % transName def do_getFiles( self, args ): """Get files for the transformation (optionally with a given status) usage: getFiles <transName|ID> [Status] [Status] """ argss = string.split( args ) if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] status = argss[1:] res = self.server.getTransformation( transName ) if not res['OK']: print "Failed to get transformation information: %s" % res['Message'] else: selectDict = {'TransformationID':res['Value']['TransformationID']} if status: selectDict['Status'] = status res = self.server.getTransformationFiles( condDict = selectDict ) if not res['OK']: print "Failed to get transformation files: %s" % res['Message'] elif res['Value']: self._printFormattedDictList( res['Value'], ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'], 'LFN', 'LFN' ) else: print "No files found" def do_getFileStatus( self, args ): """Get file(s) status for the given transformation usage: getFileStatus <transName|ID> <lfn> [<lfn>...] """ argss = string.split( args ) if len( argss ) < 2: print "transformation and file not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.getTransformation( transName ) if not res['OK']: print "Failed to get transformation information: %s" % res['Message'] else: selectDict = {'TransformationID':res['Value']['TransformationID']} res = self.server.getTransformationFiles( condDict = selectDict ) if not res['OK']: print "Failed to get transformation files: %s" % res['Message'] elif res['Value']: filesList = [] for fileDict in res['Value']: if fileDict['LFN'] in lfns: filesList.append( fileDict ) if filesList: self._printFormattedDictList( filesList, ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'], 'LFN', 'LFN' ) else: print "Could not find any LFN in", lfns, "for transformation", transName else: print "No files found" def do_setFileStatus( self, args ): """Set file status for the given transformation usage: setFileStatus <transName|ID> <lfn> <status> """ argss = string.split( args ) if not len( argss ) == 3: print "transformation file and status not supplied" return transName = argss[0] lfn = argss[1] status = argss[2] res = self.server.setFileStatusForTransformation( transName, status, [lfn] ) if not res['OK']: print "Failed to update file status: %s" % res['Message'] else: print "Updated file status to %s" % status def do_resetFile( self, args ): """Reset file status for the given transformation usage: resetFile <transName|ID> <lfn> """ argss = string.split( args ) if not len( argss ) > 1: print "transformation and file(s) not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns ) if not res['OK']: print "Failed to reset file status: %s" % res['Message'] else: if res['Value']['Failed']: print "Could not reset some files: " for lfn, reason in res['Value']['Failed'].items(): print lfn, reason else: print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns ) def do_resetProcessedFile( self, args ): """ Reset file status for the given transformation usage: resetFile <transName|ID> <lfn> """ argss = string.split( args ) if not len( argss ) > 1: print "transformation and file(s) not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns, force = True ) if not res['OK']: print "Failed to reset file status: %s" % res['Message'] else: if res['Value']['Failed']: print "Could not reset some files: " for lfn, reason in res['Value']['Failed'].items(): print lfn, reason else: print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns ) #################################################################### # # These are the methods for file manipulation # def do_addDirectory( self, args ): """Add files from the given catalog directory usage: addDirectory <directory> [directory] """ argss = string.split( args ) if not len( argss ) > 0: print "no directory supplied" return for directory in argss: res = self.server.addDirectory( directory, force = True ) if not res['OK']: print 'failed to add directory %s: %s' % ( directory, res['Message'] ) else: print 'added %s files for %s' % ( res['Value'], directory ) def do_replicas( self, args ): """ Get replicas for <path> usage: replicas <lfn> [lfn] """ argss = string.split( args ) if not len( argss ) > 0: print "no files supplied" return res = self.server.getReplicas( argss ) if not res['OK']: print "failed to get any replica information: %s" % res['Message'] return for lfn in sortList( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to get replica information for %s: %s" % ( lfn, error ) for lfn in sortList( res['Value']['Successful'].keys() ): ses = sortList( res['Value']['Successful'][lfn].keys() ) outStr = "%s :" % lfn.ljust( 100 ) for se in ses: outStr = "%s %s" % ( outStr, se.ljust( 15 ) ) print outStr def do_addFile( self, args ): """Add new files to transformation DB usage: addFile <lfn> [lfn] """ argss = string.split( args ) if not len( argss ) > 0: print "no files supplied" return lfnDict = {} for lfn in argss: lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':'IGNORED-SE', 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'} res = self.server.addFile( lfnDict, force = True ) if not res['OK']: print "failed to add any files: %s" % res['Message'] return for lfn in sortList( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to add %s: %s" % ( lfn, error ) for lfn in sortList( res['Value']['Successful'].keys() ): print "added %s" % lfn def do_removeFile( self, args ): """Remove file from transformation DB usage: removeFile <lfn> [lfn] """ argss = string.split( args ) if not len( argss ) > 0: print "no files supplied" return res = self.server.removeFile( argss ) if not res['OK']: print "failed to remove any files: %s" % res['Message'] return for lfn in sortList( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to remove %s: %s" % ( lfn, error ) for lfn in sortList( res['Value']['Successful'].keys() ): print "removed %s" % lfn def do_addReplica( self, args ): """ Add new replica to the transformation DB usage: addReplica <lfn> <se> """ argss = string.split( args ) if not len( argss ) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'} res = self.server.addReplica( lfnDict, force = True ) if not res['OK']: print "failed to add replica: %s" % res['Message'] return for lfn in sortList( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to add replica: %s" % ( error ) for lfn in sortList( res['Value']['Successful'].keys() ): print "added %s" % lfn def do_removeReplica( self, args ): """Remove replica from the transformation DB usage: removeReplica <lfn> <se> """ argss = string.split( args ) if not len( argss ) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'} res = self.server.removeReplica( lfnDict ) if not res['OK']: print "failed to remove replica: %s" % res['Message'] return for lfn in sortList( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to remove replica: %s" % ( error ) for lfn in sortList( res['Value']['Successful'].keys() ): print "removed %s" % lfn def do_setReplicaStatus( self, args ): """Set replica status, usually used to mark a replica Problematic usage: setReplicaStatus <lfn> <status> <se> """ argss = string.split( args ) if not len( argss ) > 2: print "no file info supplied" return lfn = argss[0] status = argss[1] se = argss[2] lfnDict = {} lfnDict[lfn] = {'Status':status, 'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'} res = self.server.setReplicaStatus( lfnDict ) if not res['OK']: print "failed to set replica status: %s" % res['Message'] return for lfn in sortList( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to set replica status: %s" % ( error ) for lfn in sortList( res['Value']['Successful'].keys() ): print "updated replica status %s" % lfn
exit(1) lfns = res['Value'] if not len(lfns): print "No files found with this meta data query:%s" % meta exit(1) print "Example file: %s" % lfns[0] answer = raw_input('Proceed and submit replication? (Y/N): ') if not answer.lower() in ('y', 'yes'): print "Canceled" exit(1) trc = TransformationClient() res = trc.getTransformationStats(name_of_replication) if res['OK']: print "Replication with name %s already exists! Cannot proceed." % name_of_replication exit(1) Trans = Transformation() Trans.setTransformationName(name_of_replication) Trans.setDescription(description) Trans.setLongDescription(description) Trans.setType('Replication') Trans.setPlugin('Broadcast') #Trans.setFileMask(fmask) Trans.setSourceSE(source) Trans.setTargetSE(destination) res = Trans.addTransformation() if not res['OK']:
class TransformationCLI(CLI, API): def __init__(self): self.server = TransformationClient() self.indentSpace = 4 CLI.__init__(self) API.__init__(self) def printPair(self, key, value, separator=":"): valueList = value.split("\n") print "%s%s%s %s" % (key, " " * (self.indentSpace - len(key)), separator, valueList[0].strip()) for valueLine in valueList[1:-1]: print "%s %s" % (" " * self.indentSpace, valueLine.strip()) def do_help(self, args): """ Default version of the help command Usage: help <command> OR use helpall to see description for all commands""" CLI.do_help(self, args) # overriting default help command def do_helpall(self, args): """ Shows help information Usage: helpall <command> If no command is specified all commands are shown """ if len(args) == 0: print "\nAvailable commands:\n" attrList = dir(self) attrList.sort() for attribute in attrList: if attribute.find("do_") == 0: self.printPair(attribute[3:], getattr(self, attribute).__doc__[1:]) print "" else: command = args.split()[0].strip() try: obj = getattr(self, "do_%s" % command) except: print "There's no such %s command" % command return self.printPair(command, obj.__doc__[1:]) def do_shell(self, args): """Execute a shell command usage !<shell_command> """ comm = args res = shellCall(0, comm) if res['OK'] and res['Value'][0] == 0: _returnCode, stdOut, stdErr = res['Value'] print "%s\n%s" % (stdOut, stdErr) else: print res['Message'] def check_params(self, args, num): """Checks if the number of parameters correct""" argss = args.split() length = len(argss) if length < num: print "Error: Number of arguments provided %d less that required %d, please correct." % ( length, num) return (False, length) return (argss, length) def check_id_or_name(self, id_or_name): """resolve name or Id by converting type of argument """ if id_or_name.isdigit(): return long(id_or_name) # its look like id return id_or_name #################################################################### # # These are the methods for transformation manipulation # def do_getall(self, args): """Get transformation details usage: getall [Status] [Status] """ oTrans = Transformation() oTrans.getTransformations(transStatus=args.split(), printOutput=True) def do_getAllByUser(self, args): """Get all transformations created by a given user The first argument is the authorDN or username. The authorDN is preferred: it need to be inside quotes because contains white spaces. Only authorDN should be quoted. When the username is provided instead, the authorDN is retrieved from the uploaded proxy, so that the retrieved transformations are those created by the user who uploaded that proxy: that user could be different that the username provided to the function. usage: getAllByUser authorDN or username [Status] [Status] """ oTrans = Transformation() argss = args.split() username = "" author = "" status = [] if not len(argss) > 0: print self.do_getAllByUser.__doc__ return # if the user didnt quoted the authorDN ends if '=' in argss[0] and argss[0][0] not in ["'", '"']: print "AuthorDN need to be quoted (just quote that argument)" return if argss[0][0] in ["'", '"']: # authorDN given author = argss[0] status_idx = 1 for arg in argss[1:]: author += ' ' + arg status_idx += 1 if arg[-1] in ["'", '"']: break # At this point we should have something like 'author' if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']: print "AuthorDN need to be quoted (just quote that argument)" return else: author = author[1:-1] # throw away the quotes # the rest are the requested status status = argss[status_idx:] else: # username given username = argss[0] status = argss[1:] oTrans.getTransformationsByUser(authorDN=author, userName=username, transStatus=status, printOutput=True) def do_summaryTransformations(self, args): """Show the summary for a list of Transformations Fields starting with 'F' ('J') refers to files (jobs). Proc. stand for processed. Usage: summaryTransformations <ProdID> [<ProdID> ...] """ argss = args.split() if not len(argss) > 0: print self.do_summaryTransformations.__doc__ return transid = argss oTrans = Transformation() oTrans.getSummaryTransformations(transID=transid) def do_getStatus(self, args): """Get transformation details usage: getStatus <transName|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return for transName in argss: res = self.server.getTransformation(transName) if not res['OK']: print "Getting status of %s failed: %s" % (transName, res['Message']) else: print "%s: %s" % (transName, res['Value']['Status']) def do_setStatus(self, args): """Set transformation status usage: setStatus <Status> <transName|ID> Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'> """ argss = args.split() if not len(argss) > 1: print "transformation and status not supplied" return status = argss[0] transNames = argss[1:] for transName in transNames: res = self.server.setTransformationParameter( transName, 'Status', status) if not res['OK']: print "Setting status of %s failed: %s" % (transName, res['Message']) else: print "%s set to %s" % (transName, status) def do_start(self, args): """Start transformation usage: start <transName|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'Status', 'Active') if not res['OK']: print "Setting Status of %s failed: %s" % (transName, res['Message']) else: res = self.server.setTransformationParameter( transName, 'AgentType', 'Automatic') if not res['OK']: print "Setting AgentType of %s failed: %s" % ( transName, res['Message']) else: print "%s started" % transName def do_stop(self, args): """Stop transformation usage: stop <transID|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return if not len(argss) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'AgentType', 'Manual') if not res['OK']: print "Stopping of %s failed: %s" % (transName, res['Message']) else: print "%s stopped" % transName def do_flush(self, args): """Flush transformation usage: flush <transName|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'Status', 'Flush') if not res['OK']: print "Flushing of %s failed: %s" % (transName, res['Message']) else: print "%s flushing" % transName def do_get(self, args): """Get transformation definition usage: get <transName|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation(transName) if not res['OK']: print "Failed to get %s: %s" % (transName, res['Message']) else: res['Value'].pop('Body') printDict(res['Value']) def do_getBody(self, args): """Get transformation body usage: getBody <transName|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation(transName) if not res['OK']: print "Failed to get %s: %s" % (transName, res['Message']) else: print res['Value']['Body'] def do_getFileStat(self, args): """Get transformation file statistics usage: getFileStat <transName|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformationStats(transName) if not res['OK']: print "Failed to get statistics for %s: %s" % (transName, res['Message']) else: res['Value'].pop('Total') printDict(res['Value']) def do_modMask(self, args): """Modify transformation input definition usage: modInput <mask> <transName|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return mask = argss[0] transNames = argss[1:] for transName in transNames: res = self.server.setTransformationParameter( transName, "FileMask", mask) if not res['OK']: print "Failed to modify input file mask for %s: %s" % ( transName, res['Message']) else: print "Updated %s filemask" % transName def do_getFiles(self, args): """Get files for the transformation (optionally with a given status) usage: getFiles <transName|ID> [Status] [Status] """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] status = argss[1:] res = self.server.getTransformation(transName) if not res['OK']: print "Failed to get transformation information: %s" % res[ 'Message'] else: selectDict = {'TransformationID': res['Value']['TransformationID']} if status: selectDict['Status'] = status res = self.server.getTransformationFiles(condDict=selectDict) if not res['OK']: print "Failed to get transformation files: %s" % res['Message'] elif res['Value']: self._printFormattedDictList( res['Value'], ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'], 'LFN', 'LFN') else: print "No files found" def do_getFileStatus(self, args): """Get file(s) status for the given transformation usage: getFileStatus <transName|ID> <lfn> [<lfn>...] """ argss = args.split() if len(argss) < 2: print "transformation and file not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.getTransformation(transName) if not res['OK']: print "Failed to get transformation information: %s" % res[ 'Message'] else: selectDict = {'TransformationID': res['Value']['TransformationID']} res = self.server.getTransformationFiles(condDict=selectDict) if not res['OK']: print "Failed to get transformation files: %s" % res['Message'] elif res['Value']: filesList = [] for fileDict in res['Value']: if fileDict['LFN'] in lfns: filesList.append(fileDict) if filesList: self._printFormattedDictList(filesList, [ 'LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate' ], 'LFN', 'LFN') else: print "Could not find any LFN in", lfns, "for transformation", transName else: print "No files found" def do_getOutputFiles(self, args): """Get output files for the transformation usage: getOutputFiles <transName|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation(transName) if not res['OK']: print "Failed to get transformation information: %s" % res[ 'Message'] else: fc = FileCatalog() meta = {} meta['ProdID'] = transName res = fc.findFilesByMetadata(meta) if not res['OK']: print res['Message'] return if not len(res['Value']) > 0: print 'No output files yet for transformation %d' % int( transName) return else: for lfn in res['Value']: print lfn def do_getInputDataQuery(self, args): """Get input data query for the transformation usage: getInputDataQuery <transName|ID> """ argss = args.split() if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformationInputDataQuery(transName) if not res['OK']: print "Failed to get transformation input data query: %s" % res[ 'Message'] else: print res['Value'] def do_setFileStatus(self, args): """Set file status for the given transformation usage: setFileStatus <transName|ID> <lfn> <status> """ argss = args.split() if not len(argss) == 3: print "transformation file and status not supplied" return transName = argss[0] lfn = argss[1] status = argss[2] res = self.server.setFileStatusForTransformation( transName, status, [lfn]) if not res['OK']: print "Failed to update file status: %s" % res['Message'] else: print "Updated file status to %s" % status def do_resetFile(self, args): """Reset file status for the given transformation usage: resetFile <transName|ID> <lfns> """ argss = args.split() if not len(argss) > 1: print "transformation and file(s) not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns) if not res['OK']: print "Failed to reset file status: %s" % res['Message'] else: if 'Failed' in res['Value']: print "Could not reset some files: " for lfn, reason in res['Value']['Failed'].items(): print lfn, reason else: print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns) def do_resetProcessedFile(self, args): """ Reset file status for the given transformation usage: resetFile <transName|ID> <lfn> """ argss = args.split() if not len(argss) > 1: print "transformation and file(s) not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.setFileStatusForTransformation(transName, 'Unused', lfns, force=True) if not res['OK']: print "Failed to reset file status: %s" % res['Message'] else: if 'Failed' in res['Value'] and res['Value']['Failed']: print "Could not reset some files: " for lfn, reason in res['Value']['Failed'].items(): print lfn, reason else: print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns) #################################################################### # # These are the methods for file manipulation # def do_addDirectory(self, args): """Add files from the given catalog directory usage: addDirectory <directory> [directory] """ argss = args.split() if not len(argss) > 0: print "no directory supplied" return for directory in argss: res = self.server.addDirectory(directory, force=True) if not res['OK']: print 'failed to add directory %s: %s' % (directory, res['Message']) else: print 'added %s files for %s' % (res['Value'], directory) def do_replicas(self, args): """ Get replicas for <path> usage: replicas <lfn> [lfn] """ argss = args.split() if not len(argss) > 0: print "no files supplied" return res = self.server.getReplicas(argss) if not res['OK']: print "failed to get any replica information: %s" % res['Message'] return for lfn in sorted(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to get replica information for %s: %s" % (lfn, error) for lfn in sorted(res['Value']['Successful'].keys()): ses = sorted(res['Value']['Successful'][lfn].keys()) outStr = "%s :" % lfn.ljust(100) for se in ses: outStr = "%s %s" % (outStr, se.ljust(15)) print outStr def do_addFile(self, args): """Add new files to transformation DB usage: addFile <lfn> [lfn] """ argss = args.split() if not len(argss) > 0: print "no files supplied" return lfnDict = {} for lfn in argss: lfnDict[lfn] = { 'PFN': 'IGNORED-PFN', 'SE': 'IGNORED-SE', 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.addFile(lfnDict, force=True) if not res['OK']: print "failed to add any files: %s" % res['Message'] return for lfn in sorted(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to add %s: %s" % (lfn, error) for lfn in sorted(res['Value']['Successful'].keys()): print "added %s" % lfn def do_removeFile(self, args): """Remove file from transformation DB usage: removeFile <lfn> [lfn] """ argss = args.split() if not len(argss) > 0: print "no files supplied" return res = self.server.removeFile(argss) if not res['OK']: print "failed to remove any files: %s" % res['Message'] return for lfn in sorted(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to remove %s: %s" % (lfn, error) for lfn in sorted(res['Value']['Successful'].keys()): print "removed %s" % lfn def do_addReplica(self, args): """ Add new replica to the transformation DB usage: addReplica <lfn> <se> """ argss = args.split() if not len(argss) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = { 'PFN': 'IGNORED-PFN', 'SE': se, 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.addReplica(lfnDict, force=True) if not res['OK']: print "failed to add replica: %s" % res['Message'] return for lfn in sorted(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to add replica: %s" % (error) for lfn in sorted(res['Value']['Successful'].keys()): print "added %s" % lfn def do_removeReplica(self, args): """Remove replica from the transformation DB usage: removeReplica <lfn> <se> """ argss = args.split() if not len(argss) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = { 'PFN': 'IGNORED-PFN', 'SE': se, 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.removeReplica(lfnDict) if not res['OK']: print "failed to remove replica: %s" % res['Message'] return for lfn in sorted(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to remove replica: %s" % (error) for lfn in sorted(res['Value']['Successful'].keys()): print "removed %s" % lfn def do_setReplicaStatus(self, args): """Set replica status, usually used to mark a replica Problematic usage: setReplicaStatus <lfn> <status> <se> """ argss = args.split() if not len(argss) > 2: print "no file info supplied" return lfn = argss[0] status = argss[1] se = argss[2] lfnDict = {} lfnDict[lfn] = { 'Status': status, 'PFN': 'IGNORED-PFN', 'SE': se, 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.setReplicaStatus(lfnDict) if not res['OK']: print "failed to set replica status: %s" % res['Message'] return for lfn in sorted(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to set replica status: %s" % (error) for lfn in sorted(res['Value']['Successful'].keys()): print "updated replica status %s" % lfn
class TransformationCLI( cmd.Cmd, API ): def __init__( self ): self.server = TransformationClient() self.indentSpace = 4 cmd.Cmd.__init__( self ) API.__init__( self ) def printPair( self, key, value, separator = ":" ): valueList = value.split( "\n" ) print "%s%s%s %s" % ( key, " " * ( self.indentSpace - len( key ) ), separator, valueList[0].strip() ) for valueLine in valueList[ 1:-1 ]: print "%s %s" % ( " " * self.indentSpace, valueLine.strip() ) def do_exit( self, args ): """ Exits the shell. usage: exit """ sys.exit( 0 ) def do_quit( self, *args ): """ Exits the shell. Usage: quit """ sys.exit( 0 ) def do_help( self, args ): """ Default version of the help command Usage: help <command> OR use helpall to see description for all commands""" cmd.Cmd.do_help( self, args ) # overriting default help command def do_helpall( self, args ): """ Shows help information Usage: helpall <command> If no command is specified all commands are shown """ if len( args ) == 0: print "\nAvailable commands:\n" attrList = dir( self ) attrList.sort() for attribute in attrList: if attribute.find( "do_" ) == 0: self.printPair( attribute[ 3: ], getattr( self, attribute ).__doc__[ 1: ] ) print "" else: command = args.split()[0].strip() try: obj = getattr( self, "do_%s" % command ) except: print "There's no such %s command" % command return self.printPair( command, obj.__doc__[1:] ) def do_shell( self, args ): """Execute a shell command usage !<shell_command> """ comm = args res = shellCall( 0, comm ) if res['OK'] and res['Value'][0] == 0: _returnCode, stdOut, stdErr = res['Value'] print "%s\n%s" % ( stdOut, stdErr ) else: print res['Message'] def check_params( self, args, num ): """Checks if the number of parameters correct""" argss = args.split() length = len( argss ) if length < num: print "Error: Number of arguments provided %d less that required %d, please correct." % ( length, num ) return ( False, length ) return ( argss, length ) def check_id_or_name( self, id_or_name ): """resolve name or Id by converting type of argument """ if id_or_name.isdigit(): return long( id_or_name ) # its look like id return id_or_name #################################################################### # # These are the methods for transformation manipulation # def do_getall( self, args ): """Get transformation details usage: getall [Status] [Status] """ oTrans = Transformation() oTrans.getTransformations( transStatus = args.split(), printOutput = True ) def do_getAllByUser( self, args ): """Get all transformations created by a given user The first argument is the authorDN or username. The authorDN is preferred: it need to be inside quotes because contains white spaces. Only authorDN should be quoted. When the username is provided instead, the authorDN is retrieved from the uploaded proxy, so that the retrieved transformations are those created by the user who uploaded that proxy: that user could be different that the username provided to the function. usage: getAllByUser authorDN or username [Status] [Status] """ oTrans = Transformation() argss = args.split() username = "" author = "" status = [] if not len( argss ) > 0: print self.do_getAllByUser.__doc__ return # if the user didnt quoted the authorDN ends if '=' in argss[0] and argss[0][0] not in ["'", '"']: print "AuthorDN need to be quoted (just quote that argument)" return if argss[0][0] in ["'", '"']: # authorDN given author = argss[0] status_idx = 1 for arg in argss[1:]: author += ' ' + arg status_idx +=1 if arg[-1] in ["'", '"']: break # At this point we should have something like 'author' if not author[0] in ["'", '"'] or not author[-1] in ["'", '"']: print "AuthorDN need to be quoted (just quote that argument)" return else: author = author[1:-1] # throw away the quotes # the rest are the requested status status = argss[ status_idx: ] else: # username given username = argss[0] status = argss[ 1: ] oTrans.getTransformationsByUser( authorDN = author, userName = username, transStatus = status, printOutput = True ) def do_summaryTransformations( self, args ): """Show the summary for a list of Transformations Fields starting with 'F' ('J') refers to files (jobs). Proc. stand for processed. Usage: summaryTransformations <ProdID> [<ProdID> ...] """ argss = args.split() if not len( argss ) > 0: print self.do_summaryTransformations.__doc__ return transid = argss oTrans = Transformation() oTrans.getSummaryTransformations( transID = transid ) def do_getStatus( self, args ): """Get transformation details usage: getStatus <transName|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return for transName in argss: res = self.server.getTransformation( transName ) if not res['OK']: print "Getting status of %s failed: %s" % ( transName, res['Message'] ) else: print "%s: %s" % ( transName, res['Value']['Status'] ) def do_setStatus( self, args ): """Set transformation status usage: setStatus <Status> <transName|ID> Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'> """ argss = args.split() if not len( argss ) > 1: print "transformation and status not supplied" return status = argss[0] transNames = argss[1:] for transName in transNames: res = self.server.setTransformationParameter( transName, 'Status', status ) if not res['OK']: print "Setting status of %s failed: %s" % ( transName, res['Message'] ) else: print "%s set to %s" % ( transName, status ) def do_start( self, args ): """Start transformation usage: start <transName|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'Status', 'Active' ) if not res['OK']: print "Setting Status of %s failed: %s" % ( transName, res['Message'] ) else: res = self.server.setTransformationParameter( transName, 'AgentType', 'Automatic' ) if not res['OK']: print "Setting AgentType of %s failed: %s" % ( transName, res['Message'] ) else: print "%s started" % transName def do_stop( self, args ): """Stop transformation usage: stop <transID|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return if not len( argss ) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'AgentType', 'Manual' ) if not res['OK']: print "Stopping of %s failed: %s" % ( transName, res['Message'] ) else: print "%s stopped" % transName def do_flush( self, args ): """Flush transformation usage: flush <transName|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'Status', 'Flush' ) if not res['OK']: print "Flushing of %s failed: %s" % ( transName, res['Message'] ) else: print "%s flushing" % transName def do_get( self, args ): """Get transformation definition usage: get <transName|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation( transName ) if not res['OK']: print "Failed to get %s: %s" % ( transName, res['Message'] ) else: res['Value'].pop( 'Body' ) printDict( res['Value'] ) def do_getBody( self, args ): """Get transformation body usage: getBody <transName|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation( transName ) if not res['OK']: print "Failed to get %s: %s" % ( transName, res['Message'] ) else: print res['Value']['Body'] def do_getFileStat( self, args ): """Get transformation file statistics usage: getFileStat <transName|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformationStats( transName ) if not res['OK']: print "Failed to get statistics for %s: %s" % ( transName, res['Message'] ) else: res['Value'].pop( 'Total' ) printDict( res['Value'] ) def do_modMask( self, args ): """Modify transformation input definition usage: modInput <mask> <transName|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return mask = argss[0] transNames = argss[1:] for transName in transNames: res = self.server.setTransformationParameter( transName, "FileMask", mask ) if not res['OK']: print "Failed to modify input file mask for %s: %s" % ( transName, res['Message'] ) else: print "Updated %s filemask" % transName def do_getFiles( self, args ): """Get files for the transformation (optionally with a given status) usage: getFiles <transName|ID> [Status] [Status] """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] status = argss[1:] res = self.server.getTransformation( transName ) if not res['OK']: print "Failed to get transformation information: %s" % res['Message'] else: selectDict = {'TransformationID':res['Value']['TransformationID']} if status: selectDict['Status'] = status res = self.server.getTransformationFiles( condDict = selectDict ) if not res['OK']: print "Failed to get transformation files: %s" % res['Message'] elif res['Value']: self._printFormattedDictList( res['Value'], ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'], 'LFN', 'LFN' ) else: print "No files found" def do_getFileStatus( self, args ): """Get file(s) status for the given transformation usage: getFileStatus <transName|ID> <lfn> [<lfn>...] """ argss = args.split() if len( argss ) < 2: print "transformation and file not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.getTransformation( transName ) if not res['OK']: print "Failed to get transformation information: %s" % res['Message'] else: selectDict = {'TransformationID':res['Value']['TransformationID']} res = self.server.getTransformationFiles( condDict = selectDict ) if not res['OK']: print "Failed to get transformation files: %s" % res['Message'] elif res['Value']: filesList = [] for fileDict in res['Value']: if fileDict['LFN'] in lfns: filesList.append( fileDict ) if filesList: self._printFormattedDictList( filesList, ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'], 'LFN', 'LFN' ) else: print "Could not find any LFN in", lfns, "for transformation", transName else: print "No files found" def do_getOutputFiles( self, args ): """Get output files for the transformation usage: getOutputFiles <transName|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation( transName ) if not res['OK']: print "Failed to get transformation information: %s" % res['Message'] else: fc = FileCatalog() meta = {} meta ['ProdID'] = transName res = fc.findFilesByMetadata( meta ) if not res['OK']: print res['Message'] return if not len( res['Value'] ) > 0: print 'No output files yet for transformation %d' %int(transName) return else: for lfn in res['Value']: print lfn def do_getInputDataQuery( self, args ): """Get input data query for the transformation usage: getInputDataQuery <transName|ID> """ argss = args.split() if not len( argss ) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformationInputDataQuery( transName ) if not res['OK']: print "Failed to get transformation input data query: %s" % res['Message'] else: print res['Value'] def do_setFileStatus( self, args ): """Set file status for the given transformation usage: setFileStatus <transName|ID> <lfn> <status> """ argss = args.split() if not len( argss ) == 3: print "transformation file and status not supplied" return transName = argss[0] lfn = argss[1] status = argss[2] res = self.server.setFileStatusForTransformation( transName, status, [lfn] ) if not res['OK']: print "Failed to update file status: %s" % res['Message'] else: print "Updated file status to %s" % status def do_resetFile( self, args ): """Reset file status for the given transformation usage: resetFile <transName|ID> <lfns> """ argss = args.split() if not len( argss ) > 1: print "transformation and file(s) not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns ) if not res['OK']: print "Failed to reset file status: %s" % res['Message'] else: if 'Failed' in res['Value']: print "Could not reset some files: " for lfn, reason in res['Value']['Failed'].items(): print lfn, reason else: print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns ) def do_resetProcessedFile( self, args ): """ Reset file status for the given transformation usage: resetFile <transName|ID> <lfn> """ argss = args.split() if not len( argss ) > 1: print "transformation and file(s) not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns, force = True ) if not res['OK']: print "Failed to reset file status: %s" % res['Message'] else: if res['Value']['Failed']: print "Could not reset some files: " for lfn, reason in res['Value']['Failed'].items(): print lfn, reason else: print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns ) #################################################################### # # These are the methods for file manipulation # def do_addDirectory( self, args ): """Add files from the given catalog directory usage: addDirectory <directory> [directory] """ argss = args.split() if not len( argss ) > 0: print "no directory supplied" return for directory in argss: res = self.server.addDirectory( directory, force = True ) if not res['OK']: print 'failed to add directory %s: %s' % ( directory, res['Message'] ) else: print 'added %s files for %s' % ( res['Value'], directory ) def do_replicas( self, args ): """ Get replicas for <path> usage: replicas <lfn> [lfn] """ argss = args.split() if not len( argss ) > 0: print "no files supplied" return res = self.server.getReplicas( argss ) if not res['OK']: print "failed to get any replica information: %s" % res['Message'] return for lfn in sorted( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to get replica information for %s: %s" % ( lfn, error ) for lfn in sorted( res['Value']['Successful'].keys() ): ses = sorted( res['Value']['Successful'][lfn].keys() ) outStr = "%s :" % lfn.ljust( 100 ) for se in ses: outStr = "%s %s" % ( outStr, se.ljust( 15 ) ) print outStr def do_addFile( self, args ): """Add new files to transformation DB usage: addFile <lfn> [lfn] """ argss = args.split() if not len( argss ) > 0: print "no files supplied" return lfnDict = {} for lfn in argss: lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':'IGNORED-SE', 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'} res = self.server.addFile( lfnDict, force = True ) if not res['OK']: print "failed to add any files: %s" % res['Message'] return for lfn in sorted( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to add %s: %s" % ( lfn, error ) for lfn in sorted( res['Value']['Successful'].keys() ): print "added %s" % lfn def do_removeFile( self, args ): """Remove file from transformation DB usage: removeFile <lfn> [lfn] """ argss = args.split() if not len( argss ) > 0: print "no files supplied" return res = self.server.removeFile( argss ) if not res['OK']: print "failed to remove any files: %s" % res['Message'] return for lfn in sorted( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to remove %s: %s" % ( lfn, error ) for lfn in sorted( res['Value']['Successful'].keys() ): print "removed %s" % lfn def do_addReplica( self, args ): """ Add new replica to the transformation DB usage: addReplica <lfn> <se> """ argss = args.split() if not len( argss ) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'} res = self.server.addReplica( lfnDict, force = True ) if not res['OK']: print "failed to add replica: %s" % res['Message'] return for lfn in sorted( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to add replica: %s" % ( error ) for lfn in sorted( res['Value']['Successful'].keys() ): print "added %s" % lfn def do_removeReplica( self, args ): """Remove replica from the transformation DB usage: removeReplica <lfn> <se> """ argss = args.split() if not len( argss ) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = {'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'} res = self.server.removeReplica( lfnDict ) if not res['OK']: print "failed to remove replica: %s" % res['Message'] return for lfn in sorted( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to remove replica: %s" % ( error ) for lfn in sorted( res['Value']['Successful'].keys() ): print "removed %s" % lfn def do_setReplicaStatus( self, args ): """Set replica status, usually used to mark a replica Problematic usage: setReplicaStatus <lfn> <status> <se> """ argss = args.split() if not len( argss ) > 2: print "no file info supplied" return lfn = argss[0] status = argss[1] se = argss[2] lfnDict = {} lfnDict[lfn] = {'Status':status, 'PFN':'IGNORED-PFN', 'SE':se, 'Size':0, 'GUID':'IGNORED-GUID', 'Checksum':'IGNORED-CHECKSUM'} res = self.server.setReplicaStatus( lfnDict ) if not res['OK']: print "failed to set replica status: %s" % res['Message'] return for lfn in sorted( res['Value']['Failed'].keys() ): error = res['Value']['Failed'][lfn] print "failed to set replica status: %s" % ( error ) for lfn in sorted( res['Value']['Successful'].keys() ): print "updated replica status %s" % lfn
class ProductionJob(Job): #pylint: disable=too-many-public-methods, too-many-instance-attributes """ Production job class. Suitable for CLIC studies. Need to sub class and overload for other clients. """ def __init__(self, script = None): super(ProductionJob, self).__init__( script ) self.prodVersion = __RCSID__ self.dryrun = False self.created = False self.checked = False self.call_finalization = False self.finalsdict = {} self.transfid = 0 self.type = 'Production' self.csSection = '/Production/Defaults' self.ops = Operations() self.fc = FileCatalogClient() self.trc = TransformationClient() self.defaultProdID = '12345' self.defaultProdJobID = '12345' self.jobFileGroupSize = 1 self.nbtasks = 1 self.slicesize =0 self.basename = '' self.basepath = self.ops.getValue('/Production/CLIC/BasePath','/ilc/prod/clic/') self.evttype = '' self.datatype = '' self.energycat = '' self.detector = '' self.currtrans = None self.description = '' self.finalpaths = [] self.finalMetaDict = defaultdict( dict ) self.prodMetaDict = {} self.finalMetaDictNonSearch = {} self.metadict_external = {} self.outputStorage = '' self.proxyinfo = getProxyInfo() self.inputdataquery = False self.inputBKSelection = {} self.plugin = 'Standard' self.prodGroup = '' self.prodTypes = ['MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction', 'MCReconstruction_Overlay', 'Merge', 'Split', 'MCGeneration_ILD', 'MCSimulation_ILD', 'MCReconstruction_ILD', 'MCReconstruction_Overlay_ILD', 'Split_ILD' ] self.prodparameters = {} self.prodparameters['NbInputFiles'] = 1 self.prodparameters['nbevts'] = 0 #self.prodparameters["SWPackages"] = '' self._addParameter(self.workflow, "IS_PROD", 'JDL', True, "This job is a production job") if not script: self.__setDefaults() self._recBasePaths = {} self.maxFCFoldersToCheck = 100000 ############################################################################# def __setDefaults(self): """Sets some default parameters. """ self.setPlatform(self.ops.getValue('%s/Platform' % (self.csSection), 'x86_64-slc5-gcc43-opt')) self.setCPUTime('300000') self.setLogLevel('verbose') self.setJobGroup('@{PRODUCTION_ID}') #version control self._setParameter('productionVersion', 'string', self.prodVersion, 'ProdAPIVersion') #General workflow parameters self._setParameter('PRODUCTION_ID', 'string', self.defaultProdID.zfill(8), 'ProductionID') self._setParameter('JOB_ID', 'string', self.defaultProdJobID.zfill(8), 'ProductionJobID') self._setParameter('Priority', 'JDL', '1', 'Priority') self._setParameter('emailAddress', 'string', '*****@*****.**', 'CrashEmailAddress') def _setParameter(self, name, parameterType, parameterValue, description): """Set parameters checking in CS in case some defaults need to be changed. """ if self.ops.getValue('%s/%s' % (self.csSection, name), ''): LOG.debug('Setting %s from CS defaults = %s' % (name, self.ops.getValue('%s/%s' % (self.csSection, name)))) self._addParameter(self.workflow, name, parameterType, self.ops.getValue('%s/%s' % (self.csSection, name), 'default'), description) else: LOG.debug('Setting parameter %s = %s' % (name, parameterValue)) self._addParameter(self.workflow, name, parameterType, parameterValue, description) def setConfig(self,version): """ Define the Configuration package to obtain """ appName = 'ILDConfig' self._addSoftware(appName.lower(), version) self.prodparameters['ILDConfigVersion'] = version self._addParameter( self.workflow, 'ILDConfigPackage', 'JDL', appName+version, 'ILDConfig package' ) return S_OK() def setClicConfig(self, version): """Define the ClicConfig package to obtain.""" return self.setConfigPackage('ClicConfig', version) def setConfigPackage(self, appName, version): """Define the config package to obtain.""" self._addSoftware(appName.lower(), version) self._addParameter(self.workflow, appName + 'Package', 'JDL', appName + version, appName + 'package') self.prodparameters[appName + 'Version'] = version return S_OK() def setDryRun(self, run): """ In case one wants to get all the info as if the prod was being submitted """ self.dryrun = run ############################################################################# def setProdGroup(self, group): """ Sets a user defined tag for the production as appears on the monitoring page """ self.prodGroup = group ############################################################################# def setProdPlugin(self, plugin): """ Sets the plugin to be used to creating the production jobs """ self.plugin = plugin ############################################################################# def setJobFileGroupSize(self, files): """ Sets the number of files to be input to each job created. """ if self.checked: return self._reportError("This input is needed at the beginning of the production definition: it is \ needed for total number of evts.") self.jobFileGroupSize = files self.prodparameters['NbInputFiles'] = files def setNbEvtsPerSlice(self,nbevts): """ Define the number of events in a slice. """ self.slicesize = nbevts ############################################################################# def setProdType(self, prodType): """Set prod type. """ if prodType not in self.prodTypes: raise TypeError('Prod must be one of %s' % (', '.join(self.prodTypes))) self.setType(prodType) ############################################################################# def setWorkflowName(self, name): """Set workflow name. """ self.workflow.setName(name) self.name = name ############################################################################# def setWorkflowDescription(self, desc): """Set workflow name. """ self.workflow.setDescription(desc) ############################################################################# def createWorkflow(self): """ Create XML for local testing. """ name = '%s.xml' % self.name if os.path.exists(name): shutil.move(name,'%s.backup' % name) self.workflow.toXMLFile(name) ############################################################################# def setOutputSE(self, outputse): """ Define where the output file(s) will go. """ self.outputStorage = outputse return S_OK() ############################################################################# def setInputDataQuery(self, metadata): """ Define the input data query needed """ retMetaKey = self._checkMetaKeys( metadata.keys() ) if not retMetaKey['OK']: return retMetaKey if "ProdID" not in metadata: return self._reportError("Input metadata dictionary must contain at least a key 'ProdID' as reference") retDirs = self._checkFindDirectories( metadata ) if not retDirs['OK']: return retDirs dirs = retDirs['Value'].values() for mdir in dirs[:self.maxFCFoldersToCheck]: LOG.notice("Directory: %s" % mdir) res = self.fc.getDirectoryUserMetadata(mdir) if not res['OK']: return self._reportError("Error looking up the catalog for directory metadata") compatmeta = res['Value'] compatmeta.update(metadata) if 'EvtType' in compatmeta: self.evttype = JobHelpers.getValue( compatmeta['EvtType'], str, basestring ) else: return self._reportError("EvtType is not in the metadata, it has to be!") if 'NumberOfEvents' in compatmeta: self.nbevts = JobHelpers.getValue( compatmeta['NumberOfEvents'], int, None ) self.basename = self.evttype LOG.notice("MetaData: %s" % compatmeta) LOG.notice("MetaData: %s" % metadata) if "Energy" in compatmeta: self.energycat = JobHelpers.getValue( compatmeta["Energy"], str, (int, long, basestring) ) if self.energycat.count("tev"): self.energy = Decimal("1000.") * Decimal(self.energycat.split("tev")[0]) elif self.energycat.count("gev"): self.energy = Decimal("1.") * Decimal(self.energycat.split("gev")[0]) else: self.energy = Decimal("1.") * Decimal(self.energycat) gendata = False if 'Datatype' in compatmeta: self.datatype = JobHelpers.getValue( compatmeta['Datatype'], str, basestring ) if self.datatype == 'gen': gendata = True if "DetectorType" in compatmeta and not gendata: self.detector = JobHelpers.getValue( compatmeta["DetectorType"], str, basestring ) self.inputBKSelection = metadata self.inputdataquery = True self.prodparameters['nbevts'] = self.nbevts self.prodparameters["FCInputQuery"] = self.inputBKSelection return S_OK() def setDescription(self, desc): """ Set the production's description :param str desc: Description """ self.description = desc return S_OK() def getBasePath(self): """ Return the base path. Updated by :any:`setInputDataQuery`. """ return self.basepath def addFinalization(self, uploadData = False, registerData = False, uploadLog = False, sendFailover=False): """ Add finalization step :param bool uploadData: Upload or not the data to the storage :param bool uploadLog: Upload log file to storage (currently only available for admins, thus add them to OutputSandbox) :param bool sendFailover: Send Failover requests, and declare files as processed or unused in transfDB :param bool registerData: Register data in the file catalog """ #TODO: Do the registration only once, instead of once for each job self.call_finalization = True self.finalsdict['uploadData'] = uploadData self.finalsdict['registerData'] = registerData self.finalsdict['uploadLog'] = uploadLog self.finalsdict['sendFailover'] = sendFailover def _addRealFinalization(self): """ This is called at creation: now that the workflow is created at the last minute, we need to add this also at the last minute """ importLine = 'from ILCDIRAC.Workflow.Modules.<MODULE> import <MODULE>' dataUpload = ModuleDefinition('UploadOutputData') dataUpload.setDescription('Uploads the output data') self._addParameter(dataUpload, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'UploadOutputData') dataUpload.setBody(body) failoverRequest = ModuleDefinition('FailoverRequest') failoverRequest.setDescription('Sends any failover requests') self._addParameter(failoverRequest, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'FailoverRequest') failoverRequest.setBody(body) registerdata = ModuleDefinition('RegisterOutputData') registerdata.setDescription('Module to add in the metadata catalog the relevant info about the files') self._addParameter(registerdata, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'RegisterOutputData') registerdata.setBody(body) logUpload = ModuleDefinition('UploadLogFile') logUpload.setDescription('Uploads the output log files') self._addParameter(logUpload, 'enable', 'bool', False, 'EnableFlag') body = importLine.replace('<MODULE>', 'UploadLogFile') logUpload.setBody(body) errorReport = ModuleDefinition('ReportErrors') errorReport.setDescription('Reports errors at the end') body = importLine.replace('<MODULE>', 'ReportErrors') errorReport.setBody(body) finalization = StepDefinition('Job_Finalization') finalization.addModule(dataUpload) up = finalization.createModuleInstance('UploadOutputData', 'dataUpload') up.setValue("enable", self.finalsdict['uploadData']) finalization.addModule(registerdata) ro = finalization.createModuleInstance('RegisterOutputData', 'RegisterOutputData') ro.setValue("enable", self.finalsdict['registerData']) finalization.addModule(logUpload) ul = finalization.createModuleInstance('UploadLogFile', 'logUpload') ul.setValue("enable", self.finalsdict['uploadLog']) finalization.addModule(failoverRequest) fr = finalization.createModuleInstance('FailoverRequest', 'failoverRequest') fr.setValue("enable", self.finalsdict['sendFailover']) finalization.addModule(errorReport) fr = finalization.createModuleInstance('ReportErrors', 'reportErrors') self.workflow.addStep(finalization) self.workflow.createStepInstance('Job_Finalization', 'finalization') return S_OK() def createProduction(self, name = None): """ Create production. """ if not self.proxyinfo['OK']: return S_ERROR("Not allowed to create production, you need a production proxy.") if 'groupProperties' not in self.proxyinfo['Value']: return S_ERROR("Could not determine groupProperties, you do not have the right proxy.") groupProperties = self.proxyinfo['Value']['groupProperties'] if 'ProductionManagement' not in groupProperties: return S_ERROR("Not allowed to create production, you need a production proxy.") if self.created: return S_ERROR("Production already created.") ###We need to add the applications to the workflow res = self._addToWorkflow() if not res['OK']: return res if self.call_finalization: self._addRealFinalization() workflowName = self.workflow.getName() fileName = '%s.xml' % workflowName LOG.verbose('Workflow XML file name is:', '%s' % fileName) try: self.createWorkflow() except Exception as x: LOG.error("Exception creating workflow", repr(x)) return S_ERROR('Could not create workflow') with open(fileName, 'r') as oFile: workflowXML = oFile.read() if not name: name = workflowName res = self.trc.getTransformationStats(name) if res['OK']: return self._reportError("Transformation with name %s already exists! Cannot proceed." % name) ###Create Tranformation Trans = Transformation() Trans.setTransformationName(name) Trans.setDescription(self.description) Trans.setLongDescription(self.description) Trans.setType(self.type) self.prodparameters['JobType'] = self.type Trans.setPlugin(self.plugin) if self.inputdataquery: Trans.setGroupSize(self.jobFileGroupSize) Trans.setTransformationGroup(self.prodGroup) Trans.setBody(workflowXML) if not self.slicesize: Trans.setEventsPerTask(self.jobFileGroupSize * self.nbevts) else: Trans.setEventsPerTask(self.slicesize) self.currtrans = Trans if self.dryrun: LOG.notice('Would create prod called', name) self.transfid = 12345 else: res = Trans.addTransformation() if not res['OK']: LOG.error(res['Message']) return res self.transfid = Trans.getTransformationID()['Value'] if self.inputBKSelection: res = self.applyInputDataQuery() if not self.dryrun: Trans.setAgentType("Automatic") Trans.setStatus("Active") finals = [] for finalpaths in self.finalpaths: finalpaths = finalpaths.rstrip("/") finalpaths += "/"+str(self.transfid).zfill(8) finals.append(finalpaths) self.finalMetaDict[finalpaths].update( { "ProdID": self.transfid } ) self.finalMetaDict[finalpaths].update( self.prodMetaDict ) # if 'ILDConfigVersion' in self.prodparameters: # self.finalMetaDict[finalpaths].update({"ILDConfig":self.prodparameters['ILDConfigVersion']}) if self.nbevts: self.finalMetaDict[finalpaths].update({'NumberOfEvents' : self.jobFileGroupSize * self.nbevts}) self.finalpaths = finals self.created = True return S_OK() def setNbOfTasks(self, nbtasks): """ Define the number of tasks you want. Useful for generation jobs. """ if not self.currtrans: LOG.error("Not transformation defined earlier") return S_ERROR("No transformation defined") if self.inputBKSelection and self.plugin not in ['Limited', 'SlicedLimited']: LOG.error('Metadata selection activated, should not specify the number of jobs') return S_ERROR() self.nbtasks = nbtasks self.currtrans.setMaxNumberOfTasks(self.nbtasks) #pylint: disable=E1101 return S_OK() def applyInputDataQuery(self, metadata = None, prodid = None): """ Tell the production to update itself using the metadata query specified, i.e. submit new jobs if new files are added corresponding to same query. """ if not self.transfid and self.currtrans: self.transfid = self.currtrans.getTransformationID()['Value'] #pylint: disable=E1101 elif prodid: self.transfid = prodid if not self.transfid: LOG.error("Not transformation defined earlier") return S_ERROR("No transformation defined") if metadata: self.inputBKSelection = metadata if not self.dryrun: res = self.trc.createTransformationInputDataQuery(self.transfid, self.inputBKSelection) if not res['OK']: return res else: LOG.notice("Would use %s as metadata query for production" % str(self.inputBKSelection)) return S_OK() def addMetadataToFinalFiles(self, metadict): """ Add additionnal non-query metadata """ self.metadict_external = metadict return S_OK() def finalizeProd(self, prodid = None, prodinfo = None): """ Finalize definition: submit to Transformation service and register metadata """ currtrans = 0 if self.currtrans: if not self.dryrun: currtrans = self.currtrans.getTransformationID()['Value'] #pylint: disable=E1101 else: currtrans = 12345 if prodid: currtrans = prodid if not currtrans: LOG.error("Not transformation defined earlier") return S_ERROR("No transformation defined") if prodinfo: self.prodparameters = prodinfo info = [] info.append('%s Production %s has following parameters:\n' % (self.prodparameters['JobType'], currtrans)) if "Process" in self.prodparameters: info.append('- Process %s' % self.prodparameters['Process']) if "Energy" in self.prodparameters: info.append('- Energy %s GeV' % self.prodparameters["Energy"]) if not self.slicesize: self.prodparameters['nbevts'] = self.jobFileGroupSize * self.nbevts else: self.prodparameters['nbevts'] = self.slicesize if self.prodparameters['nbevts']: info.append("- %s events per job" % (self.prodparameters['nbevts'])) if self.prodparameters.get('lumi', False): info.append(' corresponding to a luminosity %s fb' % (self.prodparameters['lumi'] * \ self.prodparameters['NbInputFiles'])) if 'FCInputQuery' in self.prodparameters: info.append('Using InputDataQuery :') for key, val in self.prodparameters['FCInputQuery'].iteritems(): info.append(' %s = %s' % (key, val)) if "SWPackages" in self.prodparameters: info.append('- SW packages %s' % self.prodparameters["SWPackages"]) if "SoftwareTag" in self.prodparameters: info.append('- SW tags %s' % self.prodparameters["SoftwareTag"]) if "ILDConfigVersion" in self.prodparameters: info.append('- ILDConfig %s' % self.prodparameters['ILDConfigVersion']) if 'ClicConfigVersion' in self.prodparameters: info.append('- ClicConfig %s' % self.prodparameters['ClicConfigVersion'] ) if 'extraCLIArguments' in self.prodparameters: info.append('- ExtraCLIArguments %s' % self.prodparameters['extraCLIArguments'] ) # as this is the very last call all applications are registered, so all software packages are known #add them the the metadata registration for finalpath in self.finalpaths: if finalpath not in self.finalMetaDictNonSearch: self.finalMetaDictNonSearch[finalpath] = {} if "SWPackages" in self.prodparameters: self.finalMetaDictNonSearch[finalpath]["SWPackages"] = self.prodparameters["SWPackages"] if self.metadict_external: self.finalMetaDictNonSearch[finalpath].update(self.metadict_external) info.append('- Registered metadata: ') for path, metadata in sorted( self.finalMetaDict.iteritems() ): info.append(' %s = %s' % (path, metadata)) info.append('- Registered non searchable metadata: ') for path, metadata in sorted( self.finalMetaDictNonSearch.iteritems() ): info.append(' %s = %s' % (path, metadata)) infoString = '\n'.join(info) self.prodparameters['DetailedInfo'] = infoString for name, val in self.prodparameters.iteritems(): result = self._setProdParameter(currtrans, name, val) if not result['OK']: LOG.error(result['Message']) res = self._registerMetadata() if not res['OK']: LOG.error('Could not register the following directories:', res['Message']) return res return S_OK() def _createDirectory(self, path, failed, mode=0o775): """Create the directory at path if it does not exist. :param str path: path to check :param list failed: list of failed paths :param int mode: mode to set for directory """ exists = returnSingleResult(self.fc.isDirectory(path)) if exists['OK'] and exists['Value']: LOG.verbose('Directory already exists:', path) return S_OK() result = returnSingleResult(self.fc.createDirectory(path)) if not result['OK']: LOG.error('Failed to create directory:', '%s: %s' % (path, result['Message'])) failed[path].append(result['Message']) return S_ERROR() LOG.verbose('Successfully created directory:', path) res = self.fc.changePathMode({path: mode}, False) if not res['OK']: LOG.error(res['Message']) failed[path].append(res['Message']) return S_ERROR() LOG.verbose('Successfully changed mode:', path) return S_OK() def _checkMetadata(self, path, metaCopy): """Get existing metadata, if it is the same do not set it again, otherwise return error.""" existingMetadata = self.fc.getDirectoryUserMetadata(path.rstrip('/')) if not existingMetadata['OK']: return S_OK() failure = False for key, value in existingMetadata['Value'].iteritems(): if key in metaCopy and metaCopy[key] != value: LOG.error('Metadata values for folder %s disagree for key %s: Existing(%r), new(%r)' % (path, key, value, metaCopy[key])) failure = True elif key in metaCopy and metaCopy[key] == value: LOG.verbose('Meta entry is unchanged', '%s = %s' % (key, value)) metaCopy.pop(key, None) if failure: return S_ERROR('Error when setting new metadata, already existing metadata disagrees!') return S_OK() def _registerMetadata(self): """Set metadata for given folders. Register path and metadata before the production actually runs. This allows for the definition of the full chain in 1 go. """ prevent_registration = self.ops.getValue('Production/PreventMetadataRegistration', False) if self.dryrun or prevent_registration: LOG.notice('Would have created and registered the following\n', '\n '.join([' * %s: %s' % (fPath, val) for fPath, val in self.finalMetaDict.iteritems()])) LOG.notice('Would have set this as non searchable metadata', str(self.finalMetaDictNonSearch)) return S_OK() failed = defaultdict(list) for path, meta in sorted(self.finalMetaDict.items()): res = self._createDirectory(path, failed) if not res['OK']: continue LOG.verbose('Checking to set metadata:', meta) metaCopy = dict(meta) res = self._checkMetadata(path, metaCopy) if not res['OK']: return res if not metaCopy: LOG.verbose('No new metadata to set') continue LOG.verbose('Setting metadata information: ', '%s: %s' % (path, metaCopy)) result = self.fc.setMetadata(path.rstrip('/'), metaCopy) if not result['OK']: LOG.error('Could not preset metadata', str(metaCopy)) LOG.error('Could not preset metadata', result['Message']) failed[path].append(result['Message']) for path, meta in sorted(self.finalMetaDictNonSearch.items()): res = self._createDirectory(path, failed) if not res['OK']: continue LOG.verbose('Setting non searchable metadata information: ', '%s: %s' % (path, meta)) result = self.fc.setMetadata(path.rstrip('/'), meta) if not result['OK']: LOG.error('Could not preset non searchable metadata', str(meta)) LOG.error('Could not preset non searchable metadata', result['Message']) failed[path].append(result['Message']) if failed: return S_ERROR('Failed to register some metadata: %s' % dict(failed)) return S_OK() def getMetadata(self): """ Return the corresponding metadata of the last step """ metadict = {} for meta in self.finalMetaDict.values(): metadict.update(meta) if 'NumberOfEvents' in metadict: del metadict['NumberOfEvents'] #As this is not supposed to be a searchable thing return metadict def _setProdParameter(self, prodID, pname, pvalue): """ Set a production parameter. """ if isinstance( pvalue, list ): pvalue = '\n'.join(pvalue) if isinstance( pvalue, (int, long) ): pvalue = str(pvalue) if not self.dryrun: result = self.trc.setTransformationParameter(int(prodID), str(pname), str(pvalue)) if not result['OK']: LOG.error('Problem setting parameter %s for production %s and value:\n%s' % (prodID, pname, pvalue)) else: LOG.notice("Adding %s=%s to transformation" % (str(pname), str(pvalue))) result = S_OK() return result def _jobSpecificParams(self, application): """ For production additional checks are needed: ask the user """ if self.created: return S_ERROR("The production was created, you cannot add new applications to the job.") if not application.logFile: logf = application.appname + "_" + application.version + "_@{STEP_ID}.log" res = application.setLogFile(logf) if not res['OK']: return res #in fact a bit more tricky as the log files have the prodID and jobID in them ### Retrieve from the application the essential info to build the prod info. if not self.nbevts and not self.slicesize: self.nbevts = application.numberOfEvents if not self.nbevts: return S_ERROR("Number of events to process is not defined.") elif not application.numberOfEvents: if not self.slicesize: res = application.setNumberOfEvents(self.jobFileGroupSize * self.nbevts) else: res = application.setNumberOfEvents(self.slicesize) if not res['OK']: return res if application.numberOfEvents > 0 and (self.jobFileGroupSize * self.nbevts > application.numberOfEvents or self.slicesize > application.numberOfEvents): self.nbevts = application.numberOfEvents if not self.energy: if application.energy: self.energy = Decimal((("%1.2f" % float(application.energy)).rstrip('0').rstrip('.'))) else: return S_ERROR("Could not find the energy defined, it is needed for the production definition.") elif not application.energy: res = application.setEnergy(float(self.energy)) if not res['OK']: return res if self.energy: self._setParameter( "Energy", "float", float(self.energy), "Energy used") self.prodparameters["Energy"] = float(self.energy) if not self.evttype: if hasattr(application, 'eventType'): self.evttype = application.eventType else: return S_ERROR("Event type not found nor specified, it's mandatory for the production paths.") self.prodparameters['Process'] = self.evttype if not self.outputStorage: return S_ERROR("You need to specify the Output storage element") curpackage = "%s.%s" % (application.appname, application.version) if "SWPackages" in self.prodparameters: if not self.prodparameters["SWPackages"].count(curpackage): self.prodparameters["SWPackages"] += ";%s" % ( curpackage ) else : self.prodparameters["SWPackages"] = "%s" % (curpackage) if not application.accountInProduction: res = self._updateProdParameters(application) if not res['OK']: return res self.checked = True return S_OK() res = application.setOutputSE(self.outputStorage) if not res['OK']: return res energypath = self.getEnergyPath() if not self.basename: self.basename = self.evttype evttypepath = '' if not self.evttype[-1] == '/': evttypepath = self.evttype + '/' path = self.basepath ###Need to resolve file names and paths if self.energy: self.finalMetaDict[self.basepath + energypath] = {"Energy":str(self.energy)} if hasattr(application, "setOutputRecFile") and not application.willBeCut: evtPath = self.basepath + energypath + evttypepath self.finalMetaDict[evtPath] = {'EvtType': self.evttype} detPath = evtPath + application.detectortype self.finalMetaDict[detPath] = {'DetectorType': application.detectortype} if application.keepRecFile: path = self.basepath + energypath + evttypepath + application.detectortype + '/REC' self.finalMetaDict[path] = {'Datatype': 'REC'} fname = self.basename + '_rec.slcio' application.setOutputRecFile(fname, path) LOG.info('Will store the files under', path) self.finalpaths.append(path) path = self.basepath + energypath + evttypepath + application.detectortype + '/DST' self.finalMetaDict[path] = {'Datatype': 'DST'} fname = self.basename + '_dst.slcio' application.setOutputDstFile(fname, path) LOG.info('Will store the files under', path) self.finalpaths.append(path) elif hasattr(application, "outputFile") and hasattr(application, 'datatype') and not application.outputFile and not application.willBeCut: path = self.basepath + energypath + evttypepath self.finalMetaDict[path] = {"EvtType" : self.evttype} if hasattr(application, "detectortype"): if application.detectortype: path += application.detectortype self.finalMetaDict[path] = {"DetectorType" : application.detectortype} path += '/' elif self.detector: path += self.detector self.finalMetaDict[path] = {"DetectorType" : self.detector} path += '/' if not application.datatype and self.datatype: application.datatype = self.datatype path += application.datatype self.finalMetaDict[path] = {'Datatype' : application.datatype} LOG.info("Will store the files under", "%s" % path) self.finalpaths.append(path) extension = 'stdhep' if application.datatype in ['SIM', 'REC']: extension = 'slcio' fname = self.basename + "_%s" % (application.datatype.lower()) + "." + extension application.setOutputFile(fname, path) self.basepath = path res = self._updateProdParameters(application) if not res['OK']: return res self.checked = True return S_OK() def _updateProdParameters(self, application): """ Update the prod parameters stored in the production parameters visible from the web """ try: self.prodparameters.update(application.prodparameters) except Exception as x: return S_ERROR("Exception: %r" % x ) if hasattr( application, 'extraCLIArguments' ) and application.extraCLIArguments: self.prodparameters['extraCLIArguments'] = repr(application.extraCLIArguments) return S_OK() def _jobSpecificModules(self, application, step): return application._prodjobmodules(step) def getEnergyPath(self): """returns the energy path 250gev or 3tev or 1.4tev etc.""" energy = Decimal(str(self.energy)) tD = Decimal('1000.0') unit = 'gev' if energy < tD else 'tev' energy = energy if energy < tD else energy/tD energyPath = ("%1.2f" % energy).rstrip('0').rstrip('.') energyPath = energyPath+unit+'/' LOG.info("Energy path is: ", energyPath) return energyPath def _checkMetaKeys( self, metakeys, extendFileMeta=False ): """ check if metadata keys are allowed to be metadata :param list metakeys: metadata keys for production metadata :param bool extendFileMeta: also use FileMetaFields for checking meta keys :returns: S_OK, S_ERROR """ res = self.fc.getMetadataFields() if not res['OK']: LOG.error("Could not contact File Catalog") return S_ERROR("Could not contact File Catalog") metaFCkeys = res['Value']['DirectoryMetaFields'].keys() if extendFileMeta: metaFCkeys.extend( res['Value']['FileMetaFields'].keys() ) for key in metakeys: for meta in metaFCkeys: if meta != key and meta.lower() == key.lower(): return self._reportError("Key syntax error %r, should be %r" % (key, meta), name = self.__class__.__name__) if key not in metaFCkeys: return self._reportError("Key %r not found in metadata keys, allowed are %r" % (key, metaFCkeys)) return S_OK() def _checkFindDirectories( self, metadata ): """ find directories by metadata and check that there are directories found :param dict metadata: metadata dictionary :returns: S_OK, S_ERROR """ res = self.fc.findDirectoriesByMetadata(metadata) if not res['OK']: return self._reportError("Error looking up the catalog for available directories") elif len(res['Value']) < 1: return self._reportError('Could not find any directories corresponding to the query issued') return res def setReconstructionBasePaths( self, recPath, dstPath ): """ set the output Base paths for the reconstruction REC and DST files """ self._recBasePaths['REC'] = recPath self._recBasePaths['DST'] = dstPath
class TransformationCLI(cmd.Cmd, API): def __init__(self): self.server = TransformationClient() self.indentSpace = 4 cmd.Cmd.__init__(self) def printPair(self, key, value, separator=":"): valueList = value.split("\n") print "%s%s%s %s" % (key, " " * (self.indentSpace - len(key)), separator, valueList[0].strip()) for valueLine in valueList[1:-1]: print "%s %s" % (" " * self.indentSpace, valueLine.strip()) def do_exit(self, args): """ Exits the shell. usage: exit """ sys.exit(0) def do_quit(self, *args): """ Exits the shell. Usage: quit """ sys.exit(0) def do_help(self, args): """ Default version of the help command Usage: help <command> OR use helpall to see description for all commans""" cmd.Cmd.do_help(self, args) # overriting default help command def do_helpall(self, args): """ Shows help information Usage: helpall <command> If no command is specified all commands are shown """ if len(args) == 0: print "\nAvailable commands:\n" attrList = dir(self) attrList.sort() for attribute in attrList: if attribute.find("do_") == 0: self.printPair(attribute[3:], getattr(self, attribute).__doc__[1:]) print "" else: command = args.split()[0].strip() try: obj = getattr(self, "do_%s" % command) except: print "There's no such %s command" % command return self.printPair(command, obj.__doc__[1:]) def do_shell(self, args): """Execute a shell command usage !<shell_command> """ comm = args res = shellCall(0, comm) if res['OK'] and res['Value'][0] == 0: returnCode, stdOut, stdErr = res['Value'] print "%s\n%s" % (stdOut, stdErr) else: print res['Message'] def check_params(self, args, num): """Checks if the number of parameters correct""" argss = string.split(args) length = len(argss) if length < num: print "Error: Number of arguments provided %d less that required %d, please correct." % ( length, num) return (False, length) return (argss, length) def check_id_or_name(self, id_or_name): """resolve name or Id by converting type of argument """ if id_or_name.isdigit(): return long(id_or_name) # its look like id return id_or_name def do_setServer(self, args): """ Set the destination server usage: setServer serverURL """ argss = string.split(args) if len(argss) == 0: print "no server provided" self.serverURL = argss[0] self.server.setServer(self.serverURL) #################################################################### # # These are the methods for transformation manipulation # def do_getall(self, args): """Get transformation details usage: getall [Status] [Status] """ oTrans = Transformation() oTrans.setServer(self.serverURL) oTrans.getTransformations(transStatus=string.split(args), printOutput=True) def do_getStatus(self, args): """Get transformation details usage: getStatus <transName|ID> """ argss = string.split(args) if not len(argss) > 0: print "no transformation supplied" return for transName in argss: res = self.server.getTransformation(transName) if not res['OK']: print "Getting status of %s failed: %s" % (transName, res['Message']) else: print "%s: %s" % (transName, res['Value']['Status']) def do_setStatus(self, args): """Set transformation status usage: setStatus <Status> <transName|ID> Status <'New' 'Active' 'Stopped' 'Completed' 'Cleaning'> """ argss = string.split(args) if not len(argss) > 1: print "transformation and status not supplied" return status = argss[0] transNames = argss[1:] for transName in transNames: res = self.server.setTransformationParameter( transName, 'Status', status) if not res['OK']: print "Setting status of %s failed: %s" % (transName, res['Message']) else: print "%s set to %s" % (transName, status) def do_start(self, args): """Start transformation usage: start <transName|ID> """ argss = string.split(args) if not len(argss) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'Status', 'Active') if not res['OK']: print "Setting Status of %s failed: %s" % (transName, res['Message']) else: res = self.server.setTransformationParameter( transName, 'AgentType', 'Automatic') if not res['OK']: print "Setting AgentType of %s failed: %s" % ( transName, res['Message']) else: print "%s started" % transName def do_stop(self, args): """Stop transformation usage: stop <transID|ID> """ argss = string.split(args) if not len(argss) > 0: print "no transformation supplied" return if not len(argss) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'AgentType', 'Manual') if not res['OK']: print "Stopping of %s failed: %s" % (transName, res['Message']) else: print "%s stopped" % transName def do_flush(self, args): """Flush transformation usage: flush <transName|ID> """ argss = string.split(args) if not len(argss) > 0: print "no transformation supplied" return for transName in argss: res = self.server.setTransformationParameter( transName, 'Status', 'Flush') if not res['OK']: print "Flushing of %s failed: %s" % (transName, res['Message']) else: print "%s flushing" % transName def do_get(self, args): """Get transformation definition usage: get <transName|ID> """ argss = string.split(args) if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation(transName) if not res['OK']: print "Failed to get %s: %s" % (transName, res['Message']) else: res['Value'].pop('Body') printDict(res['Value']) def do_getBody(self, args): """Get transformation body usage: getBody <transName|ID> """ argss = string.split(args) if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformation(transName) if not res['OK']: print "Failed to get %s: %s" % (transName, res['Message']) else: print res['Value']['Body'] def do_getFileStat(self, args): """Get transformation file statistics usage: getFileStat <transName|ID> """ argss = string.split(args) if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] res = self.server.getTransformationStats(transName) if not res['OK']: print "Failed to get statistics for %s: %s" % (transName, res['Message']) else: res['Value'].pop('Total') printDict(res['Value']) def do_modMask(self, args): """Modify transformation input definition usage: modInput <mask> <transName|ID> """ argss = string.split(args) if not len(argss) > 0: print "no transformation supplied" return mask = argss[0] transNames = argss[1:] for transName in transNames: res = self.server.setTransformationParameter( transName, "FileMask", mask) if not res['OK']: print "Failed to modify input file mask for %s: %s" % ( transName, res['Message']) else: print "Updated %s filemask" % transName def do_getFiles(self, args): """Get files for the transformation (optionally with a given status) usage: getFiles <transName|ID> [Status] [Status] """ argss = string.split(args) if not len(argss) > 0: print "no transformation supplied" return transName = argss[0] status = argss[1:] res = self.server.getTransformation(transName) if not res['OK']: print "Failed to get transformation information: %s" % res[ 'Message'] else: selectDict = {'TransformationID': res['Value']['TransformationID']} if status: selectDict['Status'] = status res = self.server.getTransformationFiles(condDict=selectDict) if not res['OK']: print "Failed to get transformation files: %s" % res['Message'] elif res['Value']: self._printFormattedDictList( res['Value'], ['LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate'], 'LFN', 'LFN') else: print "No files found" def do_getFileStatus(self, args): """Get file(s) status for the given transformation usage: getFileStatus <transName|ID> <lfn> [<lfn>...] """ argss = string.split(args) if len(argss) < 2: print "transformation and file not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.getTransformation(transName) if not res['OK']: print "Failed to get transformation information: %s" % res[ 'Message'] else: selectDict = {'TransformationID': res['Value']['TransformationID']} res = self.server.getTransformationFiles(condDict=selectDict) if not res['OK']: print "Failed to get transformation files: %s" % res['Message'] elif res['Value']: filesList = [] for fileDict in res['Value']: if fileDict['LFN'] in lfns: filesList.append(fileDict) if filesList: self._printFormattedDictList(filesList, [ 'LFN', 'Status', 'ErrorCount', 'TargetSE', 'LastUpdate' ], 'LFN', 'LFN') else: print "Could not find any LFN in", lfns, "for transformation", transName else: print "No files found" def do_setFileStatus(self, args): """Set file status for the given transformation usage: setFileStatus <transName|ID> <lfn> <status> """ argss = string.split(args) if not len(argss) == 3: print "transformation file and status not supplied" return transName = argss[0] lfn = argss[1] status = argss[2] res = self.server.setFileStatusForTransformation( transName, status, [lfn]) if not res['OK']: print "Failed to update file status: %s" % res['Message'] else: print "Updated file status to %s" % status def do_resetFile(self, args): """Reset file status for the given transformation usage: setFileStatus <transName|ID> <lfn> """ argss = string.split(args) if not len(argss) > 1: print "transformation and file(s) not supplied" return transName = argss[0] lfns = argss[1:] res = self.server.setFileStatusForTransformation( transName, 'Unused', lfns) if not res['OK']: print "Failed to reset file status: %s" % res['Message'] else: print "Updated file statuses to 'Unused' for %d file(s)" % len( lfns) #################################################################### # # These are the methods for file manipulation # def do_addDirectory(self, args): """Add files from the given catalog directory usage: addDirectory <directory> [directory] """ argss = string.split(args) if not len(argss) > 0: print "no directory supplied" return for directory in argss: res = self.server.addDirectory(directory, force=True) if not res['OK']: print 'failed to add directory %s: %s' % (directory, res['Message']) else: print 'added %s files for %s' % (res['Value'], directory) def do_replicas(self, args): """ Get replicas for <path> usage: replicas <lfn> [lfn] """ argss = string.split(args) if not len(argss) > 0: print "no files supplied" return res = self.server.getReplicas(argss) if not res['OK']: print "failed to get any replica information: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to get replica information for %s: %s" % (lfn, error) for lfn in sortList(res['Value']['Successful'].keys()): ses = sortList(res['Value']['Successful'][lfn].keys()) outStr = "%s :" % lfn.ljust(100) for se in ses: outStr = "%s %s" % (outStr, se.ljust(15)) print outStr def do_addFile(self, args): """Add new files to transformation DB usage: addFile <lfn> [lfn] """ argss = string.split(args) if not len(argss) > 0: print "no files supplied" return lfnDict = {} for lfn in argss: lfnDict[lfn] = { 'PFN': 'IGNORED-PFN', 'SE': 'IGNORED-SE', 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.addFile(lfnDict, force=True) if not res['OK']: print "failed to add any files: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to add %s: %s" % (lfn, error) for lfn in sortList(res['Value']['Successful'].keys()): print "added %s" % lfn def do_removeFile(self, args): """Remove file from transformation DB usage: removeFile <lfn> [lfn] """ argss = string.split(args) if not len(argss) > 0: print "no files supplied" return res = self.server.removeFile(argss) if not res['OK']: print "failed to remove any files: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to remove %s: %s" % (lfn, error) for lfn in sortList(res['Value']['Successful'].keys()): print "removed %s" % lfn def do_addReplica(self, args): """ Add new replica to the transformation DB usage: addReplica <lfn> <se> """ argss = string.split(args) if not len(argss) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = { 'PFN': 'IGNORED-PFN', 'SE': se, 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.addReplica(lfnDict, force=True) if not res['OK']: print "failed to add replica: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to add replica: %s" % (error) for lfn in sortList(res['Value']['Successful'].keys()): print "added %s" % lfn def do_removeReplica(self, args): """Remove replica from the transformation DB usage: removeReplica <lfn> <se> """ argss = string.split(args) if not len(argss) == 2: print "no file info supplied" return lfn = argss[0] se = argss[1] lfnDict = {} lfnDict[lfn] = { 'PFN': 'IGNORED-PFN', 'SE': se, 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.removeReplica(lfnDict) if not res['OK']: print "failed to remove replica: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to remove replica: %s" % (error) for lfn in sortList(res['Value']['Successful'].keys()): print "removed %s" % lfn def do_setReplicaStatus(self, args): """Set replica status, usually used to mark a replica Problematic usage: setReplicaStatus <lfn> <status> <se> """ argss = string.split(args) if not len(argss) > 2: print "no file info supplied" return lfn = argss[0] status = argss[1] se = argss[2] lfnDict = {} lfnDict[lfn] = { 'Status': status, 'PFN': 'IGNORED-PFN', 'SE': se, 'Size': 0, 'GUID': 'IGNORED-GUID', 'Checksum': 'IGNORED-CHECKSUM' } res = self.server.setReplicaStatus(lfnDict) if not res['OK']: print "failed to set replica status: %s" % res['Message'] return for lfn in sortList(res['Value']['Failed'].keys()): error = res['Value']['Failed'][lfn] print "failed to set replica status: %s" % (error) for lfn in sortList(res['Value']['Successful'].keys()): print "updated replica status %s" % lfn
exit(1) lfns = res['Value'] if not len(lfns): print("No files found with this meta data query:%s" % meta) exit(1) print("Example file: %s" % lfns[0]) answer = raw_input('Proceed and submit replication? (Y/N): ') if not answer.lower() in ('y', 'yes'): print("Canceled") exit(1) trc = TransformationClient() res = trc.getTransformationStats(name_of_replication) if res['OK']: print("Replication with name %s already exists! Cannot proceed." % name_of_replication) exit(1) Trans = Transformation() Trans.setTransformationName(name_of_replication) Trans.setDescription(description) Trans.setLongDescription(description) Trans.setType('Replication') Trans.setPlugin('Broadcast') #Trans.setFileMask(fmask) Trans.setSourceSE(source) Trans.setTargetSE(destination) res = Trans.addTransformation() if not res['OK']:
class UpdateTransformationCounters( AgentModule ): """ This agent is doing what getTransformationSummaryWeb does, but can take the time it needs """ def __init__( self, *args, **kwargs ): """ c'tor :param self: self reference :param str agentName: name of agent :param bool baseAgentName: whatever :param dict properties: whatever else """ AgentModule.__init__( self, *args, **kwargs ) self.transClient = TransformationClient() self.transfStatuses = self.am_getOption( 'TransformationStatuses', ['Active', 'Stopped'] ) def initialize( self ): ''' Make the necessary initializations ''' gMonitor.registerActivity( "Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM ) return S_OK() def execute( self ): ''' Main execution method ''' gMonitor.addMark( 'Iteration', 1 ) # Get all the transformations result = self.transClient.getTransformations( condDict = {'Status': self.transfStatuses }, timeout = 320 ) if not result['OK']: gLogger.error( "UpdateTransformationCounters.execute: Failed to get transformations.", result['Message'] ) return S_OK() # Process each transformation jobsStates = self.transClient.getTransformationCountersStatuses( 'Tasks' )['Value'] filesStates = self.transClient.getTransformationCountersStatuses( 'Files' )['Value'] for transDict in result['Value']: transID = long( transDict['TransformationID'] ) gLogger.debug( "Looking at transformationID %d" % transID ) counterDict = {} counterDict['TransformationID'] = transID #Take care of the Tasks' states gLogger.verbose( "Getting the tasks stats for Transformation %s" % transID ) res = self.transClient.getTransformationTaskStats( transID ) if not res['OK']: gLogger.warn( "Could not get Transformation Task Stats for transformation %s : %s" % ( transID, res['Message'] ) ) break else: taskDict = {} if res['Value']: taskDict = res['Value'] gLogger.verbose( "Got %s tasks dict for transformation %s" % ( str( taskDict ), transID ) ) for state in jobsStates: counterDict[state] = taskDict.get( state, 0 ) else: gLogger.warn( "No Task Statuses found" ) break #Now look for the files' states gLogger.verbose( "Getting the files stats for Transformation %s" % transID ) res = self.transClient.getTransformationStats( transID ) if not res['OK']: gLogger.warn( "Could not get Transformation Stats for transformation %s : %s" % ( transID, res['Message'] ) ) break else: fileDict = {} if res['Value']: fileDict = res['Value'] gLogger.debug( "Got %s file dict for transformation %s" % ( str( fileDict ), transID ) ) for state in filesStates: counterDict[state] = fileDict.get( state, 0 ) else: gLogger.warn( "No File Statuses found" ) break gLogger.verbose( "Updating the counters for transformation %s" % transID ) res = self.transClient.updateTransformationCounters( counterDict ) if not res['OK']: gLogger.error( "Failed updating counters for transformation %s: %s" % ( transID, res['Message'] ) ) else: gLogger.verbose( "Updated the counters of transformation %s" % transID ) return S_OK()