def _getFileInfo(lfn): """ Retrieve the file info """ from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient from DIRAC.Core.Utilities import DEncode from DIRAC import gLogger fc = FileCatalogClient() lumi = 0 nbevts = 0 res = fc.getFileUserMetadata(lfn) if not res['OK']: gLogger.error("Failed to get metadata of %s" % lfn) return (0, 0, {}) if 'Luminosity' in res['Value']: lumi += float(res['Value']['Luminosity']) addinfo = {} if 'AdditionalInfo' in res['Value']: addinfo = res['Value']['AdditionalInfo'] if addinfo.count("{"): addinfo = eval(addinfo) else: addinfo = DEncode.decode(addinfo)[0] if "NumberOfEvents" in res['Value'].keys(): nbevts += int(res['Value']['NumberOfEvents']) return (float(lumi), int(nbevts), addinfo)
def initialize(self): """agent initialisation reading and setting config opts :param self: self reference """ # # shifter proxy # See cleanContent method: this proxy will be used ALSO when the file catalog used # is the DIRAC File Catalog (DFC). # This is possible because of unset of the "UseServerCertificate" option self.shifterProxy = self.am_getOption("shifterProxy", self.shifterProxy) # # transformations types self.dataProcTTypes = Operations().getValue( "Transformations/DataProcessing", self.dataProcTTypes) self.dataManipTTypes = Operations().getValue( "Transformations/DataManipulation", self.dataManipTTypes) agentTSTypes = self.am_getOption("TransformationTypes", []) if agentTSTypes: self.transformationTypes = sorted(agentTSTypes) else: self.transformationTypes = sorted(self.dataProcTTypes + self.dataManipTTypes) self.log.info("Will consider the following transformation types: %s" % str(self.transformationTypes)) # # directory locations self.directoryLocations = sorted( self.am_getOption("DirectoryLocations", self.directoryLocations)) self.log.info( "Will search for directories in the following locations: %s" % str(self.directoryLocations)) # # transformation metadata self.transfidmeta = self.am_getOption("TransfIDMeta", self.transfidmeta) self.log.info("Will use %s as metadata tag name for TransformationID" % self.transfidmeta) # # archive periof in days self.archiveAfter = self.am_getOption("ArchiveAfter", self.archiveAfter) # days self.log.info("Will archive Completed transformations after %d days" % self.archiveAfter) # # transformation log SEs self.logSE = Operations().getValue("/LogStorage/LogSE", self.logSE) self.log.info("Will remove logs found on storage element: %s" % self.logSE) # # transformation client self.transClient = TransformationClient() # # wms client self.wmsClient = WMSClient() # # request client self.reqClient = ReqClient() # # file catalog client self.metadataClient = FileCatalogClient() # # job monitoring client self.jobMonitoringClient = JobMonitoringClient() return S_OK()
def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) # # replica manager self.replicaManager = ReplicaManager() # # transformation client self.transClient = TransformationClient() # # wms client self.wmsClient = WMSClient() # # request client self.requestClient = RequestClient() # # file catalog clinet self.metadataClient = FileCatalogClient() # # placeholders for CS options # # transformations types self.transformationTypes = None # # directory locations self.directoryLocations = None # # transformation metadata self.transfidmeta = None # # archive periof in days self.archiveAfter = None # # active SEs self.activeStorages = None # # transformation log SEs self.logSE = None # # enable/disable execution self.enableFlag = None
def __init__(self, cpuTime=432000): """ Constructor Keyword arguments: cpuTime -- max cpu time allowed for the job """ Job.__init__(self) self.setCPUTime(cpuTime) # defaults self.setName('Evndisplay_Analysis') self.package = 'evndisplay' self.program_category = 'analysis' #'calibimgreco' self.version = 'prod3_d20150831b' self.prefix = 'CTA.prod3S' self.layout_list = '3HB1 3HB2 3HB3 3HD1 3HD2 3HI1' self.calibration_file = 'prod3.peds.20150820.dst.root' self.reconstructionparameter = 'EVNDISP.prod3.reconstruction.runparameter.NN' self.NNcleaninginputcard = 'EVNDISP.NNcleaning.dat' self.basepath = '/vo.cta.in2p3.fr/MC/PROD3/' self.outputpattern = './*evndisp.tar.gz' self.fcc = FileCatalogClient() self.metadata = collections.OrderedDict() self.filemetadata = {} self.catalogs = json.dumps(['DIRACFileCatalog', 'TSCatalog']) self.jobGroupID = 1
def _getProdInfoFromIDs(self): """get the processName, energy and eventsPerJob from the MetaData catalog :raises: AttributeError if some of the information cannot be found :returns: None """ if not self.prodIDs: raise AttributeError("No prodIDs defined") self.eventsPerJobs = [] self.processes = [] self.energies = [] from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient trc = TransformationClient() fc = FileCatalogClient() for prodID in self.prodIDs: gLogger.notice("Getting information for %s" % prodID) tRes = trc.getTransformation(str(prodID)) if not tRes['OK']: raise AttributeError("No prodInfo found for %s" % prodID) self.eventsPerJobs.append(int(tRes['Value']['EventsPerTask'])) lfnRes = fc.findFilesByMetadata({'ProdID': prodID}) if not lfnRes['OK'] or not lfnRes['Value']: raise AttributeError("Could not find files for %s: %s " % (prodID, lfnRes.get('Message', lfnRes.get('Value')))) path = os.path.dirname(lfnRes['Value'][0]) fileRes = fc.getDirectoryUserMetadata(path) self.processes.append(fileRes['Value']['EvtType']) self.energies.append(fileRes['Value']['Energy']) gLogger.notice("Found (Evts,Type,Energy): %s %s %s " % (self.eventsPerJobs[-1], self.processes[-1], self.energies[-1]))
def initialize(self): """ agent's initalisation :param self: self reference """ gLogger.info('Initialize') self.dataManager = DataManager() self.am_setOption('shifterProxy', 'DataManager') self.CopyToSE = (self.am_getOption("CopyToSE", 'PNNL-PIC-SRM-SE')) self.SEDataDirPath = (self.am_getOption("SEDataDirPath", '/project8/dirac/calib/')) self.LocalDataDirPath = (self.am_getOption("LocalDataDirPath", '/data_claude/')) self.DIRACCfgSEPath = 'Resources/StorageElements' self.dryRun = bool(self.am_getOption("DryRun", True)) self.fc = FileCatalogClient() gLogger.info("DryRun: " + str(self.dryRun)) gLogger.info("CopyToSE: " + str(self.CopyToSE)) ### This defines which calibration directories to consider self.calibDirs = ['rf_bkgd', 'esr'] ### This defines which sub dir under calib dir to replicate self.rawDataDir = 'raw' self.acceptableFileSuffix = [ '-esr.json', '.json', '.dpt', '.root', '.Setup' ] return S_OK()
def setupCatalogClient( self, catalogs ): """ Init FileCatalog client Ideally we would like to use only FileCatalog but it doesn't work for setMetadata because the returned value has a wrong format. So use of FileCatalogClient instead """ self.fc = FileCatalog( catalogs ) self.fcc = FileCatalogClient()
def __init__( self, cpuTime = 432000 ): """ Constructor Keyword arguments: cpuTime -- max cpu time allowed for the job """ Job.__init__( self ) self.setCPUTime( cpuTime ) # defaults self.setName( 'Mars_Analysis' ) self.package = 'chimp' self.version = 'prod3_xxx' self.PixelRequiredPhes = '-100.' self.outdir = './' self.MuonMode = '0' self.StarOutput = '-staroutput' # self.training_type = 'point-like' self.training_type = 'diffuse' #self.training_type = 'point-like diffuse' self.basepath = '/vo.cta.in2p3.fr/MC/PROD3/' #self.outputpattern = './stereo_All/*.root' # not used self.outputpattern = './data/*.tar.gz' # used for ctastereo #self.outputpattern = './*_I.root' # used for chimp self.fcc = FileCatalogClient() self.metadata = collections.OrderedDict() self.filemetadata = {} self.jobGroupID = 1
class TestUserMetadataBasicTestCase(unittest.TestCase): def setUp(self): self.dirac = Dirac() csAPI = CSAPI() self.lfn5 = os.path.join(DESTINATION_PATH, 'test_file_10MB_v5.bin') self.dir5 = os.path.dirname(self.lfn5) # local file, for now: self.fname = os.path.basename(self.lfn5) random_dd(self.fname, 10) self.diracSE = 'SE-1' try: self.fc = FileCatalogClient("DataManagement/MultiVOFileCatalog") except Exception: self.fail( " FileCatalog(['MultiVOFileCatalog']) raised Exception unexpectedly!\n" + traceback.format_exc()) return # add a replica self.fileadded = self.dirac.addFile(self.lfn5, self.fname, self.diracSE) self.assertTrue(self.fileadded['OK']) def tearDown(self): # meta index -r result = self.fc.deleteMetadataField('MetaInt6') self.assertTrue(result['OK']) result = self.fc.deleteMetadataField('TestDirectory6') self.assertTrue(result['OK']) # remove the MultiVOFileCatalog self.fc.removeCatalog('MultiVOFileCatalog') # delete a sole replica: dirac-dms-remove-files result = self.dirac.removeFile(self.lfn5) self.assertTrue(result['OK']) os.remove(self.fname)
def __init__(self): super(OverlayInput, self).__init__() self.enable = True self.STEP_NUMBER = '' self.log = gLogger.getSubLogger( "OverlayInput" ) self.applicationName = 'OverlayInput' self.curdir = os.getcwd() self.applicationLog = '' self.printoutflag = '' self.prodid = 0 self.detector = '' ##needed for backward compatibility self.detectormodel = "" self.energytouse = '' self.energy = 0 self.nbofeventsperfile = 100 self.lfns = [] self.nbfilestoget = 0 self.BkgEvtType = 'gghad' self.metaEventType = self.BkgEvtType self.BXOverlay = 0 self.ggtohadint = 3.2 self.nbsigeventsperfile = 0 self.nbinputsigfile = 1 self.NbSigEvtsPerJob = 0 self.datMan = DataManager() self.fcc = FileCatalogClient() self.site = DIRAC.siteName() self.useEnergyForFileLookup = True self.machine = 'clic_cdr' self.pathToOverlayFiles = '' self.processorName = ''
def getFileInfo(lfn): """ Retrieve the file info """ from DIRAC.Resources.Catalog.FileCatalogClient import FileCatalogClient from DIRAC.Core.Utilities import DEncode from DIRAC import gLogger fc = FileCatalogClient() lumi = 0 nbevts = 0 res = fc.getFileUserMetadata(lfn) if not res['OK']: gLogger.error("Failed to get metadata of %s" % lfn) return (0,0,{}) if res['Value'].has_key('Luminosity'): lumi += float(res['Value']['Luminosity']) addinfo = {} if 'AdditionalInfo' in res['Value']: addinfo = res['Value']['AdditionalInfo'] if addinfo.count("{"): addinfo = eval(addinfo) else: addinfo = DEncode.decode(addinfo)[0] if "NumberOfEvents" in res['Value'].keys(): nbevts += int(res['Value']['NumberOfEvents']) return (float(lumi),int(nbevts),addinfo)
def __init__(self, script=None): super(ProductionJob, self).__init__(script) self.prodVersion = __RCSID__ self.dryrun = False self.created = False self.checked = False self.call_finalization = False self.finalsdict = {} self.transfid = 0 self.type = 'Production' self.csSection = '/Production/Defaults' self.ops = Operations() self.fc = FileCatalogClient() self.trc = TransformationClient() self.defaultProdID = '12345' self.defaultProdJobID = '12345' self.jobFileGroupSize = 1 self.nbtasks = 1 self.slicesize = 0 self.basename = '' self.basepath = self.ops.getValue('/Production/CLIC/BasePath', '/ilc/prod/clic/') self.evttype = '' self.datatype = '' self.energycat = '' self.detector = '' self.currtrans = None self.description = '' self.finalpaths = [] self.finalMetaDict = defaultdict(dict) self.prodMetaDict = {} self.finalMetaDictNonSearch = {} self.metadict_external = {} self.outputStorage = '' self.proxyinfo = getProxyInfo() self.inputdataquery = False self.inputBKSelection = {} self.plugin = 'Standard' self.prodGroup = '' self.prodTypes = [ 'MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction', 'MCReconstruction_Overlay', 'Merge', 'Split', 'MCGeneration_ILD', 'MCSimulation_ILD', 'MCReconstruction_ILD', 'MCReconstruction_Overlay_ILD', 'Split_ILD' ] self.prodparameters = {} self.prodparameters['NbInputFiles'] = 1 self.prodparameters['nbevts'] = 0 #self.prodparameters["SWPackages"] = '' self._addParameter(self.workflow, "IS_PROD", 'JDL', True, "This job is a production job") if not script: self.__setDefaults() self._recBasePaths = {}
def initialize(self): """ agent initialisation reading and setting confing opts :param self: self reference """ # # shifter proxy self.am_setOption('shifterProxy', 'DataManager') # # transformations types self.dataProcTTypes = Operations().getValue( 'Transformations/DataProcessing', self.dataProcTTypes) self.dataManipTTypes = Operations().getValue( 'Transformations/DataManipulation', self.dataManipTTypes) agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = sorted(agentTSTypes) else: self.transformationTypes = sorted(self.dataProcTTypes + self.dataManipTTypes) self.log.info("Will consider the following transformation types: %s" % str(self.transformationTypes)) # # directory locations self.directoryLocations = sorted( self.am_getOption('DirectoryLocations', ['TransformationDB', 'MetadataCatalog'])) self.log.info( "Will search for directories in the following locations: %s" % str(self.directoryLocations)) # # transformation metadata self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID") self.log.info("Will use %s as metadata tag name for TransformationID" % self.transfidmeta) # # archive periof in days self.archiveAfter = self.am_getOption('ArchiveAfter', 7) # days self.log.info("Will archive Completed transformations after %d days" % self.archiveAfter) # # active SEs self.activeStorages = sorted(self.am_getOption('ActiveSEs', [])) self.log.info("Will check the following storage elements: %s" % str(self.activeStorages)) # # transformation log SEs self.logSE = Operations().getValue('/LogStorage/LogSE', 'LogSE') self.log.info("Will remove logs found on storage element: %s" % self.logSE) # # enable/disable execution, should be using CS option Status?? with default value as 'Active'?? self.enableFlag = self.am_getOption('EnableFlag', 'True') # # transformation client self.transClient = TransformationClient() # # wms client self.wmsClient = WMSClient() # # request client self.reqClient = ReqClient() # # file catalog client self.metadataClient = FileCatalogClient() return S_OK()
def __init__(self, fcClient=False): """Internal initialization of Badger API. """ if not fcClient: _fcType = 'DataManagement/FileCatalog' self.client = FileCatalogClient(_fcType) else: self.client = fcClient
def check_file_existence(directory, file, dontPromptMe=False): myLFN = 'LFN:' + STORAGE_USER_PATH + directory + '/' + file fcc = FileCatalogClient() result = fcc.exists(myLFN) if myLFN in result.get('Value', {}).get('Successful', {}) and result.get( 'Value', {}).get('Successful', {})[myLFN]: return True return False
def registerInputData(filepath, size, prefix='/cepc/lustre-ro'): infoDict = {} infoDict['PFN'] = '' infoDict['Size'] = size infoDict['SE'] = 'IHEP-STORM' infoDict['GUID'] = commands.getoutput('uuidgen') infoDict['Checksum'] = '' fileDict = {} lfn = prefix + filepath fileDict[lfn] = infoDict fcc = FileCatalogClient('DataManagement/FileCatalog') rm = ReplicaManager() result = {} result['lfn'] = lfn result['is_registered'] = False #查询 for repeatTimes in range(10): is_registered = fcc.isFile(lfn) if (is_registered['OK'] and is_registered['Value']['Successful'].has_key(lfn)): break # else: # continue if not is_registered['OK']: #查询失败 result['is_registered'] = 'querry error. unkown' print 'Failed to query %s in DFC. Error message is %s' % ( lfn, is_registered['Message']) if is_registered['Value']['Successful'][lfn]: #已注册 result['is_registered'] = True for repeatTimes in range(10): is_removed = rm.removeCatalogFile(lfn) #删除 if (is_removed['OK'] and is_removed['Value']['Successful'][lfn]['FileCatalog']): result['is_removed'] = True break # else: # continue if not is_removed['OK']: #删除失败 result['is_removed'] = 'remove error' print 'Failed to remove %s from DFC.' % lfn #add for repeatTimes in range(10): is_added = fcc.addFile(fileDict) #add/register if (is_added['OK'] and is_added['Value']['Successful'][lfn]): result['OK'] = True return result # else: # continue if not is_added['OK']: #add unsuccessfully result['OK'] = False result['Message'] = is_added['Message'] elif is_added['Value']['Failed']: result['OK'] = False result['Message'] = 'Failed to add file' + lfn return result
def __init__(self): super(DBDGenRegisterOutputData).__init__() self.version = "DBDGenRegisterOutputData v1" self.log = gLogger.getSubLogger( "DBDGenRegisterOutputData" ) self.commandTimeOut = 10 * 60 self.enable = True self.fcc = FileCatalogClient() self.nbofevents = 0 self.prodOutputLFNs = []
def _setMetaData(directory, meta): fcc = FileCatalogClient() res = fcc.createDirectory(directory) if not res['OK']: raise Exception('DFC createDirectory for "{0}" error: {1}'.format( directory, res['Message'])) res = fcc.setMetadata(directory, meta) if not res['OK']: raise Exception('DFC setMetadata for "{0}" error: {1}'.format( directory, res['Message']))
def __init__(self): super(ILDRegisterOutputData, self).__init__() self.version = "ILDRegisterOutputData v1" self.log = gLogger.getSubLogger("ILDRegisterOutputData") self.commandTimeOut = 10 * 60 self.enable = True self.prodOutputLFNs = [] self.nbofevents = 0 self.filecatalog = FileCatalogClient() self.ildconfig = '' self.swpackages = []
def registerInputData(filepath, size, prefix = '/cepc/lustre-ro'): infoDict = {} infoDict['PFN'] = '' infoDict['Size'] = size infoDict['SE'] = 'IHEP-STORM' infoDict['GUID'] = commands.getoutput('uuidgen') infoDict['Checksum'] = '' fileDict = {} lfn = prefix + filepath fileDict[lfn] = infoDict fcc = FileCatalogClient('DataManagement/FileCatalog') rm = ReplicaManager() result = {} result['lfn'] = lfn result['is_registered'] = False #查询 for repeatTimes in range(10): is_registered = fcc.isFile(lfn) if (is_registered['OK'] and is_registered['Value']['Successful'].has_key(lfn)): break # else: # continue if not is_registered['OK']:#查询失败 result['is_registered'] = 'querry error. unkown' print 'Failed to query %s in DFC. Error message is %s' %(lfn, is_registered['Message']) if is_registered['Value']['Successful'][lfn]:#已注册 result['is_registered'] = True for repeatTimes in range(10): is_removed = rm.removeCatalogFile(lfn)#删除 if (is_removed['OK'] and is_removed['Value']['Successful'][lfn]['FileCatalog']): result['is_removed'] = True break # else: # continue if not is_removed['OK']:#删除失败 result['is_removed'] = 'remove error' print 'Failed to remove %s from DFC.' %lfn #add for repeatTimes in range(10): is_added = fcc.addFile(fileDict)#add/register if (is_added['OK'] and is_added['Value']['Successful'][lfn]): result['OK'] = True return result # else: # continue if not is_added['OK']:#add unsuccessfully result['OK'] = False result['Message'] = is_added['Message'] elif is_added['Value']['Failed']: result['OK'] = False result['Message'] = 'Failed to add file' + lfn return result
def get_dataset_MQ(dataset_name): """ Return the Meta Query associated with a given dataset """ fc = FileCatalogClient() result = returnSingleResult(fc.getDatasetParameters(dataset_name)) if not result['OK']: DIRAC.gLogger.error("Failed to retrieved dataset:", result['Message']) DIRAC.exit(-1) else: DIRAC.gLogger.info("Successfully retrieved dataset: ", dataset_name) return result['Value']['MetaQuery']
def get_list_of_datasets(tag=''): fc = FileCatalogClient() dataset_tag = '*%s*' % tag result = fc.getDatasets(dataset_tag) if not result['OK']: print("ERROR: failed to get datasets") DIRAC.exit(-1) dataset_dict = result['Value'] dataset_list = list() for dataset_name in dataset_dict['Successful'][dataset_tag].keys(): dataset_list.append(dataset_name) return dataset_list
def get_dataset_info(dataset_name): fc = FileCatalogClient() result = fc.getDatasets(dataset_name) if not result['OK']: print("ERROR: failed to get datasets") DIRAC.exit(-1) dataset_dict = result['Value'] res = dataset_dict['Successful'][dataset_name][dataset_name] number_of_files = res['NumberOfFiles'] meta_query = res['MetaQuery'] total_size = res['TotalSize'] return (dataset_name, number_of_files, total_size, meta_query)
def _findInFC(): """Find something in the FileCatalog""" from DIRAC import exit as dexit clip = _Params() clip.registerSwitches() Script.parseCommandLine() args = Script.getPositionalArgs() if len(args) < 2: Script.showHelp('ERROR: Not enough arguments') gLogger.error("Run %s --help" % SCRIPTNAME) dexit(1) path = args[0] if path == '.': path = '/' ## Check that the first argument is not a MetaQuery if any(op in path for op in OPLIST): gLogger.error( "ERROR: Path '%s' is not a valid path! The first argument must be a path" % path) gLogger.error("Run %s --help" % SCRIPTNAME) dexit(1) gLogger.verbose("Path:", path) metaQuery = args[1:] metaDataDict = _createQueryDict(metaQuery) gLogger.verbose("Query:", str(metaDataDict)) if not metaDataDict: gLogger.info("No query") dexit(1) fc = FileCatalogClient() res = fc.findFilesByMetadata(metaDataDict, path) if not res['OK']: gLogger.error(res['Message']) dexit(1) if not res['Value']: gLogger.notice("No files found") listToPrint = None if clip.printOnlyDirectories: listToPrint = set("/".join(fullpath.split("/")[:-1]) for fullpath in res['Value']) else: listToPrint = res['Value'] for entry in listToPrint: print entry dexit(0)
def eventTypeCheck(eventType): entries = [] client = FileCatalogClient() dir = '/BES3/EventTypeList' result = client.listDirectory(dir) if result['OK']: for i, v in enumerate(result['Value']['Successful'][dir]['SubDirs']): if eventType == v.split('/')[2]: return True return False
def initialize(self): """ Make the necessary initializations """ self.fileLog = {} self.timeLog = {} self.fullTimeLog = {} self.pollingTime = self.am_getOption('PollingTime', 120) self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod', 86400) gMonitor.registerActivity("Iteration", "Agent Loops", AGENT_NAME, "Loops/min", gMonitor.OP_SUM) self.transClient = TransformationClient('TransformationDB') self.metadataClient = FileCatalogClient() return S_OK()
def __retrieveAllRemoteAttributes(self, lfnList): fc = FileCatalogClient('DataManagement/FileCatalog') result = fc.getFileMetadata(lfnList) if not result['OK']: raise Exception('getFileMetadata failed: %s' % result['Message']) attributes = {} for lfn in lfnList: if lfn in result['Value']['Successful']: attributes[lfn] = self.__parseMetadata(result['Value']['Successful'][lfn]) return attributes
def eventTypeCheck(eventType): entries = [] client = FileCatalogClient() dir = '/BES3/EventTypeList' result = client.listDirectory(dir) if result['OK']: for i,v in enumerate(result['Value']['Successful'][dir]['SubDirs']): if eventType == v.split('/')[2]: return True return False
def __init__(self): super(RegisterOutputData, self).__init__() self.version = "RegisterOutputData v1" self.log = gLogger.getSubLogger("RegisterOutputData") self.commandTimeOut = 10 * 60 self.enable = True self.prodOutputLFNs = [] self.nbofevents = 0 self.luminosity = 0 self.sel_eff = 0 self.cut_eff = 0 self.add_info = '' self.filecatalog = FileCatalogClient() self.filemeta = {}
def getFiles(): """ Get the lfns: This is not the point of this example, so keep it out of the main """ fc = FileCatalogClient() meta = {} meta['ProdID'] = 1543 meta["Datatype"] = "DST" result = fc.findFilesByMetadata(meta, "/ilc/prod/clic") if not result["OK"]: gLogger.error(result["Message"]) dexit(1) return result['Value']
def _findInFC(): """Find something in the FileCatalog""" from DIRAC import exit as dexit clip = _Params() clip.registerSwitches() Script.parseCommandLine() args = Script.getPositionalArgs() if len(args)<2: Script.showHelp('ERROR: Not enough arguments') gLogger.error("Run %s --help" % SCRIPTNAME ) dexit(1) path = args[0] if path == '.': path = '/' ## Check that the first argument is not a MetaQuery if any( op in path for op in OPLIST ): gLogger.error("ERROR: Path '%s' is not a valid path! The first argument must be a path" % path) gLogger.error("Run %s --help" % SCRIPTNAME ) dexit(1) gLogger.verbose("Path:", path) metaQuery = args[1:] metaDataDict = _createQueryDict(metaQuery) gLogger.verbose("Query:",str(metaDataDict)) if not metaDataDict: gLogger.info("No query") dexit(1) fc = FileCatalogClient() res = fc.findFilesByMetadata(metaDataDict, path) if not res['OK']: gLogger.error(res['Message']) dexit(1) if not res['Value']: gLogger.notice("No files found") listToPrint = None if clip.printOnlyDirectories: listToPrint = set( "/".join(fullpath.split("/")[:-1]) for fullpath in res['Value'] ) else: listToPrint = res['Value'] for entry in listToPrint: print entry dexit(0)
def addDataFiles(args): fcc = FileCatalogClient() dm = DataManager(['FileCatalog', 'TSCatalog']) outputPath = args[0] outputPattern = args[1] outputSE = args[2] metadata = args[3] metadata = json.loads(metadata) ## Create path res = fcc.createDirectory(outputPath) if not res['OK']: return res ## Set metadata for key, value in metadata.items(): res = fcc.setMetadata(outputPath, {key: value}) # terrible hack # if not res['OK']: # return res ## Upload data files all_files = glob.glob(outputPattern) ## Check that data files exist if len(all_files) == 0: return DIRAC.S_ERROR('No data files found') for one_file in all_files: lfn = os.path.join(outputPath, one_file) msg = 'Try to upload local file: %s \nwith LFN: %s \nto %s' % ( one_file, lfn, outputSE) DIRAC.gLogger.notice(msg) res = dm.putAndRegister(lfn, one_file, outputSE) ## Check if failed if not res['OK']: DIRAC.gLogger.error( 'Failed to putAndRegister %s \nto %s \nwith message: %s' % (lfn, outputSE, res['Message'])) return res elif res['Value']['Failed'].has_key(lfn): DIRAC.gLogger.error('Failed to putAndRegister %s to %s' % (lfn, outputSE)) return res return DIRAC.S_OK()
def initialize( self ): """Sets defaults """ self.replicaManager = ReplicaManager() self.transClient = TransformationClient() self.wmsClient = WMSClient() self.requestClient = RequestClient() self.metadataClient = FileCatalogClient() self.storageUsageClient = StorageUsageClient() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'DataManager' ) self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge', 'Replication'] ) ) gLogger.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) ) self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'StorageUsage', 'MetadataCatalog'] ) ) gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) ) self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" ) gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta ) self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days gLogger.info( "Will archive Completed transformations after %d days" % self.archiveAfter ) self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) ) gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) ) self.logSE = self.am_getOption( 'TransformationLogSE', 'LogSE' ) gLogger.info( "Will remove logs found on storage element: %s" % self.logSE ) return S_OK()
def __init__(self, script = None): super(ProductionJob, self).__init__( script ) self.prodVersion = __RCSID__ self.dryrun = False self.created = False self.checked = False self.call_finalization = False self.finalsdict = {} self.transfid = 0 self.type = 'Production' self.csSection = '/Production/Defaults' self.ops = Operations() self.fc = FileCatalogClient() self.trc = TransformationClient() self.defaultProdID = '12345' self.defaultProdJobID = '12345' self.jobFileGroupSize = 1 self.nbtasks = 1 self.slicesize =0 self.basename = '' self.basepath = self.ops.getValue('/Production/CLIC/BasePath','/ilc/prod/clic/') self.evttype = '' self.datatype = '' self.energycat = '' self.detector = '' self.currtrans = None self.description = '' self.finalpaths = [] self.finalMetaDict = defaultdict( dict ) self.prodMetaDict = {} self.finalMetaDictNonSearch = {} self.metadict_external = {} self.outputStorage = '' self.proxyinfo = getProxyInfo() self.inputdataquery = False self.inputBKSelection = {} self.plugin = 'Standard' self.prodGroup = '' self.prodTypes = ['MCGeneration', 'MCSimulation', 'Test', 'MCReconstruction', 'MCReconstruction_Overlay', 'Merge', 'Split', 'MCGeneration_ILD', 'MCSimulation_ILD', 'MCReconstruction_ILD', 'MCReconstruction_Overlay_ILD', 'Split_ILD' ] self.prodparameters = {} self.prodparameters['NbInputFiles'] = 1 self.prodparameters['nbevts'] = 0 #self.prodparameters["SWPackages"] = '' self._addParameter(self.workflow, "IS_PROD", 'JDL', True, "This job is a production job") if not script: self.__setDefaults() self._recBasePaths = {} self.maxFCFoldersToCheck = 100000
def __init__(self, cpuTime=432000): """ Constructor Keyword arguments: cpuTime -- max cpu time allowed for the job """ Job.__init__(self) self.setCPUTime(cpuTime) # defaults self.setName('Evndisplay_CalibReco') self.package = 'evndisplay' self.program_category = 'calibimgreco' self.version = 'prod3b_d20170602' # or later self.configuration_id = 0 self.output_data_level = 1 self.prefix = 'CTA.prod3Nb' self.layout = 'Baseline' self.calibration_file = 'gamma_20deg_180deg_run3___cta-prod3-lapalma3-2147m-LaPalma.ped.root' self.reconstructionparameter = 'EVNDISP.prod3.reconstruction.runparameter.NN' self.NNcleaninginputcard = 'EVNDISP.NNcleaning.dat' self.basepath = '/vo.cta.in2p3.fr/MC/PROD3/' self.fcc = FileCatalogClient() self.metadata = collections.OrderedDict() self.filemetadata = {} self.catalogs = json.dumps(['DIRACFileCatalog', 'TSCatalog2'])
def __init__(self, cpuTime=432000): """ Constructor Keyword arguments: cpuTime -- max cpu time allowed for the job """ Job.__init__(self) self.setCPUTime(cpuTime) # defaults self.setName('Evndisplay_Reco') self.package = 'evndisplay' self.program_category = 'reconstruction' self.version = 'prod3b_d20170602' # or later self.configuration_id = 0 self.output_data_level = 2 self.prefix = 'CTA.prod3Nb' self.layout = 'Baseline' self.pointing = '180' self.table_file = 'tables_CTA-prod3b-LaPalma-NNq05-NN-ID0_0deg-d20160925m4-Nb.3AL4-BN15.root' self.disp_subdir_name = 'BDTdisp.Nb.3AL4-BN15.T1' self.recid = '0,1,2' # 0 = all teltescopes, 1 = LST only, 2 = MST only self.basepath = '/vo.cta.in2p3.fr/MC/PROD3/' self.fcc = FileCatalogClient() self.metadata = collections.OrderedDict() self.filemetadata = {} self.catalogs = json.dumps(['DIRACFileCatalog', 'TSCatalog'])
def __init__(self, cpuTime=432000): """ Constructor Keyword arguments: cpuTime -- max cpu time allowed for the job """ Job.__init__(self) self.setCPUTime(cpuTime) # defaults self.setName('Evndisplay_CalibReco') self.package = 'evndisplay' self.program_category = 'calibimgreco' self.version = 'prod4_d20181110' self.configuration_id = 4 self.output_data_level = DATA_LEVEL_METADATA_ID['DL1'] self.N_output_files = 1 self.prefix = 'CTA.prod4S' self.layout = '3HB9-SST' self.calibration_file = 'prod4b-SST-IPR.root' self.reconstructionparameter = 'EVNDISP.prod4.reconstruction.runparameter.NN.noLL' self.base_path = '/vo.cta.in2p3.fr/MC/PROD4/' self.fcc = FileCatalogClient() self.metadata = collections.OrderedDict() self.filemetadata = {} self.catalogs = json.dumps(['DIRACFileCatalog', 'TSCatalog']) self.ts_task_id = 0
def __init__(self): super(OverlayInput, self).__init__() self.enable = True self.STEP_NUMBER = '' self.log = gLogger.getSubLogger( "OverlayInput" ) self.applicationName = 'OverlayInput' self.curdir = os.getcwd() self.applicationLog = '' self.printoutflag = '' self.prodid = 0 self.detector = '' ##needed for backward compatibility self.detectormodel = "" self.energytouse = '' self.energy = 0 self.nbofeventsperfile = 100 self.lfns = [] self.nbfilestoget = 0 self.BkgEvtType = 'gghad' self.BXOverlay = 0 self.ggtohadint = 3.2 self.nbsigeventsperfile = 0 self.nbinputsigfile = 1 self.NbSigEvtsPerJob = 0 self.rm = ReplicaManager() self.fc = FileCatalogClient() self.site = DIRAC.siteName() self.machine = 'clic_cdr'
def __init__(self): super(DBDGenRegisterOutputData).__init__() self.version = "DBDGenRegisterOutputData v1" self.commandTimeOut = 10 * 60 self.enable = True self.fcc = FileCatalogClient() self.nbofevents = 0 self.prodOutputLFNs = []
def __init__(self, fcClient=False): """Internal initialization of Badger API. """ if not fcClient: _fcType = "DataManagement/FileCatalog" self.client = FileCatalogClient(_fcType) else: self.client = fcClient
def runOne (): fc = FileCatalogClient() for mokkaJobNumber in range(39,45): for eventType in ['Z_uds','Photon','Kaon0L','Muon']: energies = [] if eventType == 'Z_uds': energies = [91,200,360,500,750,1000,2000,3000] elif eventType == 'Photon': energies = [10] elif eventType == 'Muon': energies = [10] elif eventType == 'Kaon0L': energies = [1,2,3,4,5,6,7,8,9,10,15,20,25,30,35,40,45,50] for energy in energies: path = '/ilc/user/s/sgreen/0804/HCalStudiessg568/GJN' + str(mokkaJobNumber) + '/' + eventType + '/' + str(energy) + 'GeV' pathdict = {'path':path, 'meta':{'MokkaJobNumber':mokkaJobNumber, 'Type':eventType, 'Energy':energy}} res = fc.setMetadata(pathdict['path'], pathdict['meta']) return
def compare(attributes,input): error_list = {} inDFC = {} #print "input keys:",input.keys() keys = sorted(input.keys()) #print "keys after being sorted:",keys expNum = _get_exp_num(attributes["expNum"]) eventType = _get_event_type(attributes["eventType"]) #dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"] dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"] if attributes["streamId"]=="stream0": dir = dir+ "/data"+"/"+eventType + "/"+expNum+"/"+attributes["LFN"] else: dir = dir+"/mc"+"/"+eventType+"/"+expNum+"/"+attributes["streamId"]+"/"+attributes["LFN"] client=FileCatalogClient() result = client.getFileMetadata(dir) file_exist = len(result['Value']['Successful']) if file_exist == 0: print "this file does't exist in DFC",attributes['LFN'] else: result = client.getDirectoryMetadata(dir) if result["OK"]: inDFC["resonance"] = result["Value"]["resonance"] inDFC["streamId"] = result["Value"]["streamId"] inDFC["eventType"] = result["Value"]["eventType"] inDFC["bossVer"] = result["Value"]["bossVer"] inDFC["expNum"] = result["Value"]["expNum"] for key in keys: if input[key] != inDFC[key]: error_list[key] = inDFC[key] if error_list is not None: return error_list
def __init__(self): super(ILDRegisterOutputData, self).__init__() self.version = "ILDRegisterOutputData v1" self.commandTimeOut = 10 * 60 self.enable = True self.prodOutputLFNs = [] self.nbofevents = 0 self.filecatalog = FileCatalogClient() self.ildconfig = '' self.swpackages = []
def initialize(self): """ Make the necessary initializations """ self.fileLog = {} self.timeLog = {} self.fullTimeLog = {} self.pollingTime = self.am_getOption('PollingTime',120) self.fullUpdatePeriod = self.am_getOption('FullUpdatePeriod',86400) gMonitor.registerActivity("Iteration","Agent Loops",AGENT_NAME,"Loops/min",gMonitor.OP_SUM) self.transClient = TransformationClient('TransformationDB') self.metadataClient = FileCatalogClient() return S_OK()
def __init__(self): super(SIDRegisterOutputData, self).__init__() self.version = "SIDRegisterOutputData v1" self.log = gLogger.getSubLogger( "SIDRegisterOutputData" ) self.commandTimeOut = 10 * 60 self.enable = True self.prodOutputLFNs = [] self.swpackages = [] self.nbofevents = 0 self.luminosity = 0 self.filecatalog = FileCatalogClient()
def __init__(self): super(RegisterOutputData, self).__init__() self.version = "RegisterOutputData v1" self.commandTimeOut = 10*60 self.enable = True self.prodOutputLFNs = [] self.nbofevents = 0 self.luminosity = 0 self.sel_eff = 0 self.cut_eff = 0 self.add_info = '' self.filecatalog = FileCatalogClient() self.filemeta = {}
def insert(attributes): dir = "/BES3/File/"+attributes["resonance"]+"/"+attributes["bossVer"] #get real eventType and expNum in catalog in amga eventType = _get_event_type(attributes["eventType"]) expNum = _get_exp_num(attributes["expNum"]) if attributes["streamId"]=="stream0": dir=dir+"/data" else: dir=dir+"/mc" client = FileCatalogClient() # make sure insertion directory exists in DFC insertDir = createCatalog(client,dir,eventType,expNum,attributes) #do we need to explicitly add LFN attr in DFC? # we do need to get the PFN, size, se for addFile (below) #TODO pfn = ? # need to add PFN attribute to the list, probably in control.py pfn = attributes["PFN"] lfn=insertDir+"/"+attributes["LFN"] attributes["LFN"] = lfn #TODO se = ? temporarily use a made-up SE name se = 'BES-TEST' # register files in DFC infoDict = {} #infoDict['PFN'] = attributes['PFN'] infoDict['PFN'] = pfn infoDict['Size'] = attributes['fileSize'] infoDict['SE'] = se # infoDict['guid'] = attributes['guid'] infoDict['Checksum'] = '' result = client.addFile({lfn:infoDict}) print result
def initialize( self ): """ agent initialisation reading and setting confing opts :param self: self reference """ # # shifter proxy self.am_setOption( 'shifterProxy', 'DataManager' ) # # transformations types self.dataProcTTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] ) self.dataManipTTypes = Operations().getValue( 'Transformations/DataManipulation', ['Replication', 'Removal'] ) agentTSTypes = self.am_getOption( 'TransformationTypes', [] ) if agentTSTypes: self.transformationTypes = sorted( agentTSTypes ) else: self.transformationTypes = sorted( self.dataProcTTypes + self.dataManipTTypes ) self.log.info( "Will consider the following transformation types: %s" % str( self.transformationTypes ) ) # # directory locations self.directoryLocations = sorted( self.am_getOption( 'DirectoryLocations', [ 'TransformationDB', 'MetadataCatalog' ] ) ) self.log.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) ) # # transformation metadata self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" ) self.log.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta ) # # archive periof in days self.archiveAfter = self.am_getOption( 'ArchiveAfter', 7 ) # days self.log.info( "Will archive Completed transformations after %d days" % self.archiveAfter ) # # active SEs self.activeStorages = sorted( self.am_getOption( 'ActiveSEs', [] ) ) self.log.info( "Will check the following storage elements: %s" % str( self.activeStorages ) ) # # transformation log SEs self.logSE = Operations().getValue( '/LogStorage/LogSE', 'LogSE' ) self.log.info( "Will remove logs found on storage element: %s" % self.logSE ) # # enable/disable execution, should be using CS option Status?? with default value as 'Active'?? self.enableFlag = self.am_getOption( 'EnableFlag', 'True' ) # # data manager # self.dm = DataManager() # # transformation client self.transClient = TransformationClient() # # wms client self.wmsClient = WMSClient() # # request client self.reqClient = ReqClient() # # file catalog client self.metadataClient = FileCatalogClient() return S_OK()
def initialize(self): """ agent initialisation reading and setting confing opts :param self: self reference """ # # shifter proxy # See cleanCatalogContents method: this proxy will be used ALSO when the file catalog used # is the DIRAC File Catalog (DFC). # This is possible because of unset of the "UseServerCertificate" option self.shifterProxy = self.am_getOption('shifterProxy', None) # # transformations types self.dataProcTTypes = Operations().getValue('Transformations/DataProcessing', self.dataProcTTypes) self.dataManipTTypes = Operations().getValue('Transformations/DataManipulation', self.dataManipTTypes) agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = sorted(agentTSTypes) else: self.transformationTypes = sorted(self.dataProcTTypes + self.dataManipTTypes) self.log.info("Will consider the following transformation types: %s" % str(self.transformationTypes)) # # directory locations self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', self.directoryLocations)) self.log.info("Will search for directories in the following locations: %s" % str(self.directoryLocations)) # # transformation metadata self.transfidmeta = self.am_getOption('TransfIDMeta', self.transfidmeta) self.log.info("Will use %s as metadata tag name for TransformationID" % self.transfidmeta) # # archive periof in days self.archiveAfter = self.am_getOption('ArchiveAfter', self.archiveAfter) # days self.log.info("Will archive Completed transformations after %d days" % self.archiveAfter) # # active SEs self.activeStorages = sorted(self.am_getOption('ActiveSEs', self.activeStorages)) if self.activeStorages: self.log.info("Will check the following storage elements: %s" % str(self.activeStorages)) # # transformation log SEs self.logSE = Operations().getValue('/LogStorage/LogSE', self.logSE) self.log.info("Will remove logs found on storage element: %s" % self.logSE) # # transformation client self.transClient = TransformationClient() # # wms client self.wmsClient = WMSClient() # # request client self.reqClient = ReqClient() # # file catalog client self.metadataClient = FileCatalogClient() return S_OK()
class DBDGenRegisterOutputData(ModuleBase): """ Normally, was supposed to be used to produce the DBD gen level files. Dropped in the end. """ def __init__(self): super(DBDGenRegisterOutputData).__init__() self.version = "DBDGenRegisterOutputData v1" self.log = gLogger.getSubLogger( "DBDGenRegisterOutputData" ) self.commandTimeOut = 10 * 60 self.enable = True self.fc = FileCatalogClient() self.nbofevents = 0 self.prodOutputLFNs = [] def applicationSpecificInputs(self): if self.workflow_commons.has_key('ProductionOutputData'): self.prodOutputLFNs = self.workflow_commons['ProductionOutputData'].split(";") if self.workflow_commons.has_key('NbOfEvts'): self.nbofevents = self.workflow_commons[ 'NbOfEvts'] return S_OK("Parameters resolved") def execute(self): self.log.info('Initializing %s' % self.version) result = self.resolveInputVariables() if not result['OK']: self.log.error(result['Message']) return result if not self.workflowStatus['OK'] or not self.stepStatus['OK']: self.log.verbose('Workflow status = %s, step status = %s' % (self.workflowStatus['OK'], self.stepStatus['OK'])) return S_OK('No registration of output data metadata attempted') if not len(self.prodOutputLFNs): self.log.info('No production data found, so no metadata registration to be done') return S_OK("No files' metadata to be registered") self.log.verbose("Will try to set the metadata for the following files: \n %s" % '\n'.join(self.prodOutputLFNs)) for files in self.prodOutputLFNs: metadict = {} metadict['NumberOfEvents'] = self.nbofevents path = files res = self.fc.setMetadata(files, metadict) if not res['OK']: self.log.error("Could not register %s for %s"%(metadict, path)) return res return S_OK()
def __init__( self, *args, **kwargs ): ''' c'tor ''' AgentModule.__init__( self, *args, **kwargs ) self.fileLog = {} self.timeLog = {} self.fullTimeLog = {} self.pollingTime = self.am_getOption( 'PollingTime', 120 ) self.fullUpdatePeriod = self.am_getOption( 'FullUpdatePeriod', 86400 ) self.refreshonly = self.am_getOption( 'RefreshOnly', False ) self.dateKey = self.am_getOption( 'DateKey', None ) self.transClient = TransformationClient() self.metadataClient = FileCatalogClient()
def __init__( self, *args, **kwargs ): """ c'tor """ AgentModule.__init__( self, *args, **kwargs ) self.integrityClient = DataIntegrityClient() self.replicaManager = ReplicaManager() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() agentTSTypes = self.am_getOption( 'TransformationTypes', [] ) if agentTSTypes: self.transformationTypes = agentTSTypes else: self.transformationTypes = Operations().getValue( 'Transformations/DataProcessing', ['MCSimulation', 'Merge'] ) self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'MetadataCatalog'] ) ) self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) ) self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" )
def __init__(self, *args, **kwargs): AgentModule.__init__(self, *args, **kwargs) self.name = 'FileStatusTransformationAgent' self.enabled = False self.shifterProxy = 'DataManager' self.transformationTypes = ["Replication"] self.transformationStatuses = ["Active"] self.transformationFileStatuses = ["Assigned", "Problematic", "Processed", "Unused"] self.addressTo = ["*****@*****.**"] self.addressFrom = "*****@*****.**" self.emailSubject = "FileStatusTransformationAgent" self.accounting = defaultdict(list) self.errors = [] self.fcClient = FileCatalogClient() self.tClient = TransformationClient() self.reqClient = ReqClient() self.nClient = NotificationClient()
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.consistencyInspector = ConsistencyInspector() self.integrityClient = DataIntegrityClient() self.fc = FileCatalog() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() agentTSTypes = self.am_getOption('TransformationTypes', []) if agentTSTypes: self.transformationTypes = agentTSTypes else: self.transformationTypes = Operations().getValue('Transformations/DataProcessing', ['MCSimulation', 'Merge']) self.directoryLocations = sorted(self.am_getOption('DirectoryLocations', ['TransformationDB', 'MetadataCatalog'])) self.transfidmeta = self.am_getOption('TransfIDMeta', "TransformationID") self.enableFlag = True
def initialize( self ): """Sets defaults """ self.integrityClient = DataIntegrityClient() self.replicaManager = ReplicaManager() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() # This sets the Default Proxy to used as that defined under # /Operations/Shifter/DataManager # the shifterProxy option in the Configuration can be used to change this default. self.am_setOption( 'shifterProxy', 'DataManager' ) self.transformationTypes = sortList( self.am_getOption( 'TransformationTypes', ['MCSimulation', 'DataReconstruction', 'DataStripping', 'MCStripping', 'Merge'] ) ) gLogger.info( "Will treat the following transformation types: %s" % str( self.transformationTypes ) ) self.directoryLocations = sortList( self.am_getOption( 'DirectoryLocations', ['TransformationDB', 'MetadataCatalog'] ) ) gLogger.info( "Will search for directories in the following locations: %s" % str( self.directoryLocations ) ) self.activeStorages = sortList( self.am_getOption( 'ActiveSEs', [] ) ) gLogger.info( "Will check the following storage elements: %s" % str( self.activeStorages ) ) self.transfidmeta = self.am_getOption( 'TransfIDMeta', "TransformationID" ) gLogger.info( "Will use %s as metadata tag name for TransformationID" % self.transfidmeta ) return S_OK()
def __init__( self, agentName, baseAgentName = False, properties = dict() ): """ c'tor :param self: self reference :param str agentName: name of agent :param bool baseAgentName: whatever :param dict properties: whatever else """ AgentModule.__init__( self, agentName, baseAgentName, properties ) ## replica manager self.replicaManager = ReplicaManager() ## transformation client self.transClient = TransformationClient() ## wms client self.wmsClient = WMSClient() ## request client self.requestClient = RequestClient() ## file catalog clinet self.metadataClient = FileCatalogClient() ## storage usage agent self.storageUsageClient = StorageUsageClient() ## placeholders for CS options ## transformations types self.transformationTypes = None ## directory locations self.directoryLocations = None ## transformation metadata self.transfidmeta = None ## archive periof in days self.archiveAfter = None ## active SEs self.activeStorages = None ## transformation log SEs self.logSE = None ## enable/disable execution self.enableFlag = None
def __init__(self, *args, **kwargs): """ c'tor """ AgentModule.__init__(self, *args, **kwargs) self.integrityClient = DataIntegrityClient() self.replicaManager = ReplicaManager() self.transClient = TransformationClient() self.fileCatalogClient = FileCatalogClient() agentTSTypes = self.am_getOption("TransformationTypes", []) if agentTSTypes: self.transformationTypes = agentTSTypes else: self.transformationTypes = Operations().getValue( "Transformations/DataProcessing", ["MCSimulation", "Merge"] ) self.directoryLocations = sortList( self.am_getOption("DirectoryLocations", ["TransformationDB", "MetadataCatalog"]) ) self.activeStorages = sortList(self.am_getOption("ActiveSEs", [])) self.transfidmeta = self.am_getOption("TransfIDMeta", "TransformationID") self.enableFlag = True