])) Script.parseCommandLine() args = Script.getPositionalArgs() if not len(args) == 2: Script.showHelp() try: prodID = int(args[0]) except: Script.showHelp() filetype = args[1] from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient client = BookkeepingClient() res = client.getProductionFiles(prodID, filetype) if not res['OK']: print 'ERROR: Failed to retrieve production files: %s' % res['Message'] else: if not res['Value']: print 'No files found for production %s with type %s' % (prodID, filetype) else: print '%s %s %s %s %s' % ('FileName'.ljust(100), 'Size'.ljust(10), 'GUID'.ljust(40), 'Replica'.ljust(8), 'Visible'.ljust(8)) for lfn in sorted(res['Value']): size = res['Value'][lfn]['FileSize'] guid = res['Value'][lfn]['GUID'] hasReplica = res['Value'][lfn]['GotReplica']
######################################################################## """ List simulation conditions from the Bookkeeping """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ...' % Script.scriptName ])) Script.parseCommandLine(ignoreErrors=True) from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bk = BookkeepingClient() exitCode = 0 res = bk.getSimConditions() if res['OK']: dbresult = res['Value'] for record in dbresult: print 'SimId: ' + str(record[0]).ljust(10) print ' SimDescription: ' + str(record[1]).ljust(10) print ' BeamCond: ' + str(record[2]).ljust(10) print ' BeamEnergy: ' + str(record[3]).ljust(10) print ' Generator: ' + str(record[4]).ljust(10) print ' MagneticField: ' + str(record[5]).ljust(10) print ' DetectorCond: ' + str(record[6]).ljust(10) print ' Luminosity: ' + str(record[7]).ljust(10) print ' G4settings: ' + str(record[8]).ljust(10)
class DiracLHCb(Dirac): ############################################################################# def __init__(self, withRepo=False, repoLocation='', operationsHelperIn=None): """Internal initialization of the DIRAC API. """ super(DiracLHCb, self).__init__(withRepo=withRepo, repoLocation=repoLocation) self.tier1s = [] if not operationsHelperIn: self.opsH = Operations() else: self.opsH = operationsHelperIn self._bkQueryTemplate = {'SimulationConditions': 'All', 'DataTakingConditions': 'All', 'ProcessingPass': '******', 'FileType': 'All', 'EventType': 'All', 'ConfigName': 'All', 'ConfigVersion': 'All', 'Production': 0, 'StartRun': 0, 'EndRun': 0, 'DataQuality': 'All', 'Visible': 'Yes'} self._bkClient = BookkeepingClient() # to expose all BK client methods indirectly ############################################################################# def addRootFile(self, lfn, fullPath, diracSE, printOutput=False): """ Add a Root file to Grid storage, an attempt is made to retrieve the POOL GUID of the file prior to upload. Example Usage: >>> print dirac.addFile('/lhcb/user/p/paterson/myRootFile.tar.gz','myFile.tar.gz','CERN-USER') {'OK': True, 'Value':{'Failed': {}, 'Successful': {'/lhcb/user/p/paterson/test/myRootFile.tar.gz': {'put': 64.246301889419556, 'register': 1.1102778911590576}}}} @param lfn: Logical File Name (LFN) @type lfn: string @param diracSE: DIRAC SE name e.g. CERN-USER @type diracSE: strin @param printOutput: Optional flag to print result @type printOutput: boolean @return: S_OK,S_ERROR """ return super(DiracLHCb, self).addFile(lfn, fullPath, diracSE, fileGuid=makeGuid(fullPath)[fullPath], printOutput=printOutput) def addFile(self, lfn, fullPath, diracSE, printOutput=False): # pylint: disable=arguments-differ """ Copy of addRootFile """ return super(DiracLHCb, self).addFile(lfn, fullPath, diracSE, fileGuid=makeGuid(fullPath)[fullPath], printOutput=printOutput) def getBKAncestors(self, lfns, depth=1, replica=True): """ This function allows to retrieve ancestor files from the Bookkeeping. Example Usage: >>> dirac.getBKAncestors('/lhcb/data/2009/DST/00005727/0000/00005727_00000042_1.dst',2) {'OK': True, 'Value': ['/lhcb/data/2009/DST/00005727/0000/00005727_00000042_1.dst', '/lhcb/data/2009/RAW/FULL/LHCb/COLLISION09/63807/063807_0000000004.raw']} @param lfn: Logical File Name (LFN) @type lfn: string or list @param depth: Ancestor depth @type depth: integer """ result = self._bkClient.getFileAncestors(lfns, depth, replica=replica) if not result['OK']: self.log.error('Could not get ancestors', result['Message']) return result ancestors = set(x['FileName'] for ancestors in result['Value']['Successful'].itervalues() for x in ancestors) return S_OK(lfns + list(ancestors)) ############################################################################# def bkQueryRunsByDate(self, bkPath, startDate, endDate, dqFlag='All', selection='Runs'): """ This function allows to create and perform a BK query given a supplied BK path. The following BK path convention is expected: /<ConfigurationName>/<Configuration Version>/<Condition Description><Processing Pass>/<Event Type>/<File Type> so an example for 2016 collisions data would be: /LHCb/Collision09//LHCb/Collision16/Beam6500GeV-VeloClosed-MagDown/Real Data/Reco16/Stripping26/90000000/EW.DST The startDate and endDate must be specified as yyyy-mm-dd. Runs can be selected based on their status e.g. the selection parameter has the following possible attributes: - Runs - data for all runs in the range are queried (default) - ProcessedRuns - data is retrieved for runs that are processed - NotProcessed - data is retrieved for runs that are not yet processed. Example Usage: >>> dirac.bkQueryRunsByDate('/LHCb/Collision16//Real Data/90000000/RAW', '2016-08-20','2016-08-22',dqFlag='OK',selection='Runs') {'OK': True, 'Value': [<LFN1>,<LFN2>]} dirac.bkQueryRunsByDate('/LHCb/Collision16/Beam6500GeV-VeloClosed-MagDown/Real' 'Data/Reco16/Stripping26/90000000/EW.DST', '2016-08-20','2016-08-22',dqFlag='OK',selection='Runs') @param bkPath: BK path as described above @type bkPath: string @param dqFlag: Optional Data Quality flag @type dqFlag: string @param startDate: Start date yyyy-mm-dd @param startDate: string @param endDate: End date yyyy-mm-dd @param endDate: string @param selection: Either Runs, ProcessedRuns or NotProcessed @param selection: string @return: S_OK,S_ERROR """ runSelection = ['Runs', 'ProcessedRuns', 'NotProcessed'] if selection not in runSelection: return S_ERROR('Expected one of %s not "%s" for selection' % (', '.join(runSelection), selection)) if not isinstance(bkPath, str): return S_ERROR('Expected string for bkPath') # remove any double slashes, spaces must be preserved # remove any empty components from leading and trailing slashes bkQuery = BKQuery().buildBKQuery(bkPath) if not bkQuery: return S_ERROR( 'Please provide a BK path: ' '/<ConfigurationName>/<Configuration Version>/<Condition Description>/<Processing Pass>' '/<Event Type>/<File Type>') if not startDate or not endDate: return S_ERROR('Expected both start and end dates to be defined in format: yyyy-mm-dd') if not isinstance(startDate, str) or not isinstance(endDate, str): return S_ERROR('Expected yyyy-mm-dd string for start and end dates') if not len(startDate.split('-')) == 3 or not len(endDate.split('-')) == 3: return S_ERROR('Expected yyyy-mm-dd string for start and end dates') start = time.time() result = self._bkClient.getRunsForAGivenPeriod({'StartDate': startDate, 'EndDate': endDate}) rtime = time.time() - start self.log.info('BK query time: %.2f sec' % rtime) if not result['OK']: self.log.info('Could not get runs with given dates from BK with result: "%s"' % result) return result if not result['Value']: self.log.info('No runs selected from BK for specified dates') return result if selection not in result['Value']: return S_ERROR('No %s runs for specified dates' % (selection)) runs = result['Value'][selection] self.log.info('Found the following %s runs:\n%s' % (len(runs), ', '.join([str(i) for i in runs]))) # temporary until we can query for a discrete list of runs selectedData = [] for run in runs: query = bkQuery.copy() query['StartRun'] = run query['EndRun'] = run query['CheckRunStatus'] = True if selection in ['ProcessedRuns', 'NotProcessed'] else False if dqFlag: check = self.__checkDQFlags(dqFlag) if not check['OK']: return check dqFlag = check['Value'] query['DataQuality'] = dqFlag start = time.time() result = self._bkClient.getVisibleFilesWithMetadata(query) rtime = time.time() - start self.log.info('BK query time: %.2f sec' % rtime) self.log.verbose(result) if not result['OK']: return result self.log.info('Selected %s files for run %s' % (len(result['Value']), run)) if result['Value']['LFNs']: selectedData += result['Value']['LFNs'].keys() self.log.info('Total files selected = %s' % (len(selectedData))) return S_OK(selectedData) ############################################################################# def bkQueryRun(self, bkPath, dqFlag='All'): """ This function allows to create and perform a BK query given a supplied BK path. The following BK path convention is expected: /<Run Number>/<Processing Pass>/<Event Type>/<File Type> so an example for 2009 collisions data would be: /63566/Real Data + RecoToDST-07/90000000/DST In addition users can specify a range of runs using the following convention: /<Run Number 1> - <Run Number 2>/<Processing Pass>/<Event Type>/<File Type> so extending the above example this would look like: /63566-63600/Real Data + RecoToDST-07/90000000/DST Example Usage: >>> dirac.bkQueryRun('/63566/Real Data/RecoToDST-07/90000000/DST') {'OK':True,'Value': ['/lhcb/data/2009/DST/00005842/0000/00005842_00000008_1.dst']} @param bkPath: BK path as described above @type bkPath: string @param dqFlag: Optional Data Quality flag @type dqFlag: string @return: S_OK,S_ERROR """ if not isinstance(bkPath, str): return S_ERROR('Expected string for bkPath') # remove any double slashes, spaces must be preserved # remove any empty components from leading and trailing slashes bkPath = translateBKPath(bkPath, procPassID=1) if not len(bkPath) == 4: return S_ERROR('Expected 4 components to the BK path: /<Run Number>/<Processing Pass>/<Event Type>/<File Type>') runNumberString = bkPath[0].replace('--', '-').replace(' ', '') startRun = 0 endRun = 0 if '-' in runNumberString: runs = runNumberString.split('-') if len(runs) != 2: return S_ERROR('Could not determine run range from "%s", try "<Run 1> - <Run2>"' % (runNumberString)) try: start = int(runs[0]) end = int(runs[1]) except Exception: return S_ERROR('Invalid run range: %s' % runNumberString) startRun = min(start, end) endRun = max(start, end) else: try: startRun = int(runNumberString) endRun = startRun except Exception: return S_ERROR('Invalid run number: %s' % runNumberString) query = self._bkQueryTemplate.copy() query['StartRun'] = startRun query['EndRun'] = endRun query['ProcessingPass'] = bkPath[1] query['EventType'] = bkPath[2] query['FileType'] = bkPath[3] if dqFlag: check = self.__checkDQFlags(dqFlag) if not check['OK']: return check dqFlag = check['Value'] query['DataQuality'] = dqFlag result = self.bkQuery(query) self.log.verbose(result) return result ############################################################################# def bkQueryProduction(self, bkPath, dqFlag='All'): """ This function allows to create and perform a BK query given a supplied BK path. The following BK path convention is expected: /<ProductionID>/[<Processing Pass>/<Event Type>/]<File Type> so an example for 2009 collisions data would be: /5842/Real Data + RecoToDST-07/90000000/DST Note that neither the processing pass nor the event type should be necessary. So either of them can be omitted a data quality flag can also optionally be provided, the full list of these is available via the getAllDQFlags() method. Example Usage: >>> dirac.bkQueryProduction('/5842/Real Data/RecoToDST-07/90000000/DST') {'OK': True, 'Value': [<LFN1>,<LFN2>]} @param bkPath: BK path as described above @type bkPath: string @param dqFlag: Optional Data Quality flag @type dqFlag: string @return: S_OK,S_ERROR """ if not isinstance(bkPath, str): return S_ERROR('Expected string for bkPath') # remove any double slashes, spaces must be preserved # remove any empty components from leading and trailing slashes bkPath = translateBKPath(bkPath, procPassID=1) if len(bkPath) < 2: return S_ERROR('Invalid bkPath: should at least contain /ProductionID/FileType') query = self._bkQueryTemplate.copy() try: query['Production'] = int(bkPath[0]) except Exception: return S_ERROR('Invalid production ID') query['FileType'] = bkPath[-1] if dqFlag: check = self.__checkDQFlags(dqFlag) if not check['OK']: return check dqFlag = check['Value'] query['DataQuality'] = dqFlag for key, val in query.items(): if isinstance(val, basestring) and val.lower() == 'all': query.pop(key) result = self.bkQuery(query) self.log.verbose(result) return result ############################################################################# def bkQueryPath(self, bkPath, dqFlag='All'): """ This function allows to create and perform a BK query given a supplied BK path. The following BK path convention is expected: /<ConfigurationName>/<Configuration Version>/<Sim or Data Taking Condition> /<Processing Pass>/<Event Type>/<File Type> so an example for 2009 collsions data would be: /LHCb/Collision09/Beam450GeV-VeloOpen-MagDown/Real Data + RecoToDST-07/90000000/DST or for MC09 simulated data: /MC/2010/Beam3500GeV-VeloClosed-MagDown-Nu1/2010-Sim01Reco01-withTruth/27163001/DST a data quality flag can also optionally be provided, the full list of these is available via the getAllDQFlags() method. Example Usage: >>> dirac.bkQueryPath('/MC/2010/Beam3500GeV-VeloClosed-MagDown-Nu1/Sim07/Reco06-withTruth/10012004/DST') {'OK': True, 'Value': [<LFN1>,<LFN2>]} @param bkPath: BK path as described above @type bkPath: string @param dqFlag: Optional Data Quality flag @type dqFlag: string @return: S_OK,S_ERROR """ if not isinstance(bkPath, str): return S_ERROR('Expected string for bkPath') # remove any double slashes, spaces must be preserved # remove any empty components from leading and trailing slashes bkPath = translateBKPath(bkPath, procPassID=3) if not len(bkPath) == 6: return S_ERROR('Expected 6 components to the BK path: ' '/<ConfigurationName>/<Configuration Version>/<Sim or Data Taking Condition>' '/<Processing Pass>/<Event Type>/<File Type>') query = self._bkQueryTemplate.copy() query['ConfigName'] = bkPath[0] query['ConfigVersion'] = bkPath[1] query['ProcessingPass'] = bkPath[3] query['EventType'] = bkPath[4] query['FileType'] = bkPath[5] if dqFlag: check = self.__checkDQFlags(dqFlag) if not check['OK']: return check dqFlag = check['Value'] query['DataQuality'] = dqFlag # The problem here is that we don't know if it's a sim or data taking condition, # assume that if configName=MC this is simulation if bkPath[0].lower() == 'mc': query['SimulationConditions'] = bkPath[2] else: query['DataTakingConditions'] = bkPath[2] result = self.bkQuery(query) self.log.verbose(result) return result ############################################################################# def bookkeepingQuery(self, SimulationConditions='All', DataTakingConditions='All', ProcessingPass='******', FileType='All', EventType='All', ConfigName='All', ConfigVersion='All', ProductionID=0, DataQuality='ALL'): """ This function will create and perform a BK query using the supplied arguments and return a list of LFNs. Example Usage: >>> dirac.bookkeepingQuery(ConfigName='LHCb',ConfigVersion='Collision09', EventType='90000000',ProcessingPass='******',DataTakingConditions='Beam450GeV-VeloOpen-MagDown') {'OK':True,'Value':<files>} @param ConfigName: BK ConfigName @type ConfigName: string @param EventType: BK EventType @type EventType: string @param FileType: BK FileType @type FileType: string @param ProcessingPass: BK ProcessingPass @type ProcessingPass: string @param ProductionID: BK ProductionID @type ProductionID: integer @param DataQuality: BK DataQuality @type DataQuality: string @param ConfigVersion: BK ConfigVersion @type ConfigVersion: string @param DataTakingConditions: BK DataTakingConditions @type DataTakingConditions: string @param SimulationConditions: BK SimulationConditions @type SimulationConditions: string @return: S_OK,S_ERROR """ query = self._bkQueryTemplate.copy() query['SimulationConditions'] = SimulationConditions query['DataTakingConditions'] = DataTakingConditions query['ProcessingPass'] = ProcessingPass query['FileType'] = FileType query['EventType'] = EventType query['ConfigName'] = ConfigName query['ConfigVersion'] = ConfigVersion query['Production'] = ProductionID query['DataQuality'] = DataQuality return self.bkQuery(query) ############################################################################# def bkQuery(self, bkQueryDict): """ Developer function. Perform a query to the LHCb Bookkeeping to return a list of LFN(s). This method takes a BK query dictionary. Example Usage: >>> print dirac.bkQuery(query) {'OK':True,'Value':<files>} @param bkQueryDict: BK query @type bkQueryDict: dictionary (see bookkeepingQuery() for keys) @return: S_OK,S_ERROR """ problematicFields = [] # Remove the Visible flag as anyway the method is for visible files ;-) # bkQueryDict.setdefault( 'Visible', 'Yes' ) for name, value in bkQueryDict.items(): if name not in self._bkQueryTemplate: problematicFields.append(name) if problematicFields: msg = 'The following fields are not valid for a BK query: %s\nValid fields include: %s' % \ (', '.join(problematicFields), ', '.join(self._bkQueryTemplate.keys())) return S_ERROR(msg) for name, value in bkQueryDict.items(): if name == "Production" or name == "EventType" or name == "StartRun" or name == "EndRun": if value == 0: del bkQueryDict[name] else: bkQueryDict[name] = str(value) elif name == "FileType": if value.lower() == "all": bkQueryDict[name] = 'ALL' else: if str(value).lower() == "all": del bkQueryDict[name] if 'Production' in bkQueryDict or 'StartRun' in bkQueryDict or 'EndRun' in bkQueryDict: self.log.verbose('Found a specific query so loosening some restrictions to prevent BK overloading') else: if 'SimulationConditions' not in bkQueryDict and 'DataTakingConditions' not in bkQueryDict: return S_ERROR('A Simulation or DataTaking Condition must be specified for a BK query.') if 'EventType' not in bkQueryDict and 'ConfigName' not in bkQueryDict and 'ConfigVersion' not in bkQueryDict: return S_ERROR( 'The minimal set of BK fields for a query is: EventType, ConfigName and ConfigVersion' ' in addition to a Simulation or DataTaking Condition') self.log.verbose('Final BK query dictionary is:') for item in bkQueryDict.iteritems(): self.log.verbose('%s : %s' % item) start = time.time() result = self._bkClient.getVisibleFilesWithMetadata(bkQueryDict) # result = bk.getFilesWithGivenDataSets(bkQueryDict) rtime = time.time() - start self.log.info('BK query time: %.2f sec' % rtime) if not result['OK']: return S_ERROR('BK query returned an error: "%s"' % (result['Message'])) if not result['Value']: return self._errorReport('No BK files selected') returnedFiles = len(result['Value']) self.log.verbose('%s files selected from the BK' % (returnedFiles)) return result ############################################################################# def __checkDQFlags(self, flags): """ Internal function. Checks the provided flags against the list of possible DQ flag statuses from the Bookkeeping. """ dqFlags = [] if isinstance(flags, list): dqFlags = flags else: dqFlags = [flags] bkFlags = self.getAllDQFlags() if not bkFlags['OK']: return bkFlags final = [] for flag in dqFlags: if flag.lower() == 'all': final.append(flag.upper()) else: flag = flag.upper() if flag not in bkFlags['Value']: msg = 'Specified DQ flag "%s" is not in allowed list: %s' % (flag, ', '.join(bkFlags['Value'])) self.log.error(msg) return S_ERROR(msg) else: final.append(flag) # when first coding this it was not possible to use a list ;) if len(final) == 1: final = final[0] return S_OK(final) ############################################################################# def getAllDQFlags(self, printOutput=False): """ Helper function. Returns the list of possible DQ flag statuses from the Bookkeeping. Example Usage: >>> print dirac.getAllDQFlags() {'OK':True,'Value':<flags>} @param printOutput: Optional flag to print result @type printOutput: boolean @return: S_OK,S_ERROR """ result = self._bkClient.getAvailableDataQuality() if not result['OK']: self.log.error('Could not obtain possible DQ flags from BK with result:\n%s' % (result)) return result if printOutput: flags = result['Value'] self.log.info('Possible DQ flags from BK are: %s' % (', '.join(flags))) return result ############################################################################# def getDataByRun(self, lfns, printOutput=False): """Sort the supplied lfn list by run. An S_OK object will be returned containing a dictionary of runs and the corresponding list of LFN(s) associated with them. Example usage: >>> print dirac.getDataByRun(lfns) {'OK': True, 'Value': {<RUN>:['<LFN>','<LFN>',...], <RUN>:['<LFN>',..]}} @param lfns: Logical File Name(s) @type lfns: list @param printOutput: Optional flag to print result @type printOutput: boolean @return: S_OK,S_ERROR """ if isinstance(lfns, str): lfns = [lfns.replace('LFN:', '')] elif isinstance(lfns, list): try: lfns = [str(lfn.replace('LFN:', '')) for lfn in lfns] except ValueError as x: return self._errorReport(str(x), 'Expected strings for LFNs') else: return self._errorReport('Expected single string or list of strings for LFN(s)') runDict = {} start = time.time() result = self._bkClient.getFileMetadata(lfns) self.log.verbose("Obtained BK file metadata in %.2f seconds" % (time.time() - start)) if not result['OK']: self.log.error('Failed to get bookkeeping metadata with result "%s"' % (result['Message'])) return result for lfn, metadata in result['Value']['Successful'].items(): if 'RunNumber' in metadata: runNumber = metadata['RunNumber'] runDict.setdefault(runNumber, []).append(lfn) else: self.log.warn('Could not find run number from BK for %s' % (lfn)) if printOutput: self.log.notice(self.pPrint.pformat(runDict)) return S_OK(runDict) ############################################################################# def bkMetadata(self, lfns, printOutput=False): """Return metadata for the supplied lfn list. An S_OK object will be returned containing a dictionary of LFN(s) and the corresponding metadata associated with them. Example usage: >>> print dirac.bkMetadata(lfns) {'OK': True, 'Value': {<LFN>:{'<Name>':'<Value>',...},...}} @param lfns: Logical File Name(s) @type lfns: list @param printOutput: Optional flag to print result @type printOutput: boolean @return: S_OK,S_ERROR """ if isinstance(lfns, str): lfns = [lfns.replace('LFN:', '')] elif isinstance(lfns, list): try: lfns = [str(lfn.replace('LFN:', '')) for lfn in lfns] except ValueError as x: return self._errorReport(str(x), 'Expected strings for LFNs') else: return self._errorReport('Expected single string or list of strings for LFN(s)') start = time.time() result = self._bkClient.getFileMetadata(lfns) self.log.verbose("Obtained BK file metadata in %.2f seconds" % (time.time() - start)) if not result['OK']: self.log.error('Failed to get bookkeeping metadata with result "%s"' % (result['Message'])) return result if printOutput: self.log.notice(self.pPrint.pformat(result['Value'])) return result ############################################################################# def lhcbProxyInit(self, *args): # pylint: disable=no-self-use """ just calling the dirac-proxy-init script """ os.system("dirac-proxy-init -o LogLevel=NOTICE -t --rfc %s" % "' '".join(args)) ############################################################################# def lhcbProxyInfo(self, *args): # pylint: disable=no-self-use """ just calling the dirac-proxy-info script """ os.system("dirac-proxy-info -o LogLevel=NOTICE %s" % "' '".join(args)) ############################################################################# def gridWeather(self, printOutput=False): """This method gives a snapshot of the current Grid weather from the perspective of the DIRAC site and SE masks. Tier-1 sites are returned with more detailed information. Example usage: >>> print dirac.gridWeather() {'OK': True, 'Value': {{'Sites':<siteInfo>,'SEs':<seInfo>,'Tier-1s':<tierInfo>}} @param printOutput: Optional flag to print result @type printOutput: boolean @return: S_OK,S_ERROR """ lcgSites = gConfig.getSections('/Resources/Sites/LCG') if not lcgSites['OK']: return lcgSites for lcgSite in lcgSites['Value']: tier = gConfig.getValue('/Resources/Sites/LCG/%s/MoUTierLevel' % lcgSite, 2) if tier in (0, 1): self.tier1s.append(lcgSite) siteInfo = self.checkSites() if not siteInfo['OK']: return siteInfo siteInfo = siteInfo['Value'] seInfo = self.checkSEs() if not seInfo['OK']: return seInfo seInfo = seInfo['Value'] tierSEs = {} for site in self.tier1s: tierSEs[site] = getSEsForSite(site)['Value'] tierInfo = {} for site, seList in tierSEs.items(): tierInfo[site] = {} for se in seList: if se in seInfo: tierSEInfo = seInfo[se] tierInfo[site][se] = tierSEInfo if site in siteInfo['AllowedSites']: tierInfo[site]['MaskStatus'] = 'Allowed' else: tierInfo[site]['MaskStatus'] = 'Banned' if printOutput: self.log.notice('========> Tier-1 status in DIRAC site and SE masks') for site in sorted(self.tier1s): self.log.notice('\n====> %s is %s in site mask\n' % (site, tierInfo[site]['MaskStatus'])) self.log.notice('%s %s %s' % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15))) for se in sorted(tierSEs[site]): if se in tierInfo[site]: self.log.notice('%s %s %s' % (se.ljust(25), tierInfo[site][se]['ReadStatus'].rjust(15), tierInfo[site][se]['WriteStatus'].rjust(15)) ) self.log.notice('\n========> Tier-2 status in DIRAC site mask\n') allowedSites = siteInfo['AllowedSites'] bannedSites = siteInfo['BannedSites'] for site in self.tier1s: if site in allowedSites: allowedSites.remove(site) if site in bannedSites: bannedSites.remove(site) self.log.notice(' %s sites are in the site mask, %s are banned.\n' % (len(allowedSites), len(bannedSites))) summary = {'Sites': siteInfo, 'SEs': seInfo, 'Tier-1s': tierInfo} return S_OK(summary) ############################################################################# def checkSites(self, printOutput=False): # pylint: disable=no-self-use """Return the list of sites in the DIRAC site mask and those which are banned. Example usage: >>> print dirac.checkSites() {'OK': True, 'Value': {'AllowedSites':['<Site>',...],'BannedSites':[]} @param printOutput: Optional flag to print result @type printOutput: boolean @return: S_OK,S_ERROR """ res = getSites() if not res['OK']: self.log.error('Could not get list of sites from CS', res['Message']) return res totalList = res['Value'] res = DiracAdmin().getSiteMask() if not res['OK']: return res sites = res['Value'] bannedSites = [] for site in totalList: if site not in sites: bannedSites.append(site) if printOutput: self.log.notice('\n========> Allowed Sites\n') self.log.notice('\n'.join(sites)) self.log.notice('\n========> Banned Sites\n') self.log.notice('\n'.join(bannedSites)) self.log.notice('\nThere is a total of %s allowed sites and %s banned sites in the system.' % (len(sites), len(bannedSites))) return S_OK({'AllowedSites': sites, 'BannedSites': bannedSites}) ############################################################################# def checkSEs(self, printOutput=False): # pylint: disable=no-self-use """Check the status of read and write operations in the DIRAC SE mask. Example usage: >>> print dirac.checkSEs() {'OK': True, 'Value': {<LFN>:{'<Name>':'<Value>',...},...}} @param printOutput: Optional flag to print result @type printOutput: boolean @return: S_OK,S_ERROR """ res = gConfig.getSections('/Resources/StorageElements', True) if not res['OK']: self.log.error('Failed to get storage element information', res['Message']) return res if printOutput: self.log.notice('%s %s %s' % ('Storage Element'.ljust(25), 'Read Status'.rjust(15), 'Write Status'.rjust(15))) seList = sorted(res['Value']) result = {} rss = ResourceStatus() for se in seList: res = rss.getElementStatus(se, 'StorageElement') if not res['OK']: self.log.error("Failed to get StorageElement status for %s" % se) else: readState = res['Value'].get('ReadAccess', 'Active') writeState = res['Value'].get('WriteAccess', 'Active') result[se] = {'ReadStatus': readState, 'WriteStatus': writeState} if printOutput: self.log.notice('%s %s %s' % (se.ljust(25), readState.rjust(15), writeState.rjust(15))) return S_OK(result) def splitInputDataBySize(self, lfns, maxSizePerJob=20, printOutput=False): """Split the supplied lfn list by the replicas present at the possible destination sites, based on a maximum size. An S_OK object will be returned containing a list of lists in order to create the jobs. Example usage: >>> d.splitInputDataBySize(lfns,10) {'OK': True, 'Value': [['<LFN>'], ['<LFN>']]} @param lfns: Logical File Name(s) to split @type lfns: list @param maxSizePerJob: Maximum size (in GB) per bunch @type maxSizePerJob: integer @param printOutput: Optional flag to print result @type printOutput: boolean @return: S_OK,S_ERROR """ sitesForSE = {} if isinstance(lfns, str): lfns = [lfns.replace('LFN:', '')] elif isinstance(lfns, list): try: lfns = [str(lfn.replace('LFN:', '')) for lfn in lfns] except TypeError as x: return self._errorReport(str(x), 'Expected strings for LFNs') else: return self._errorReport('Expected single string or list of strings for LFN(s)') if not isinstance(maxSizePerJob, int): try: maxSizePerJob = int(maxSizePerJob) except ValueError as x: return self._errorReport(str(x), 'Expected integer for maxSizePerJob') maxSizePerJob *= 1000 * 1000 * 1000 replicaDict = self.getReplicas(lfns) if not replicaDict['OK']: return replicaDict replicas = replicaDict['Value']['Successful'] if not replicas: return self._errorReport(replicaDict['Value']['Failed'].items()[0], 'Failed to get replica information') siteLfns = {} for lfn, reps in replicas.items(): possibleSites = set(site for se in reps for site in sitesForSE.setdefault(se, getSitesForSE(se).get('Value', []))) siteLfns.setdefault(','.join(sorted(possibleSites)), []).append(lfn) if '' in siteLfns: # Some files don't have active replicas return self._errorReport('No active replica found for', str(siteLfns[''])) # Get size of files metadataDict = self.getLfnMetadata(lfns, printOutput) if not metadataDict['OK']: return metadataDict fileSizes = dict((lfn, metadataDict['Value']['Successful'].get(lfn, {}).get('Size', maxSizePerJob)) for lfn in lfns) lfnGroups = [] # maxSize is in GB for files in siteLfns.values(): # Now get bunches of files, # Sort in decreasing size files.sort(cmp=(lambda f1, f2: fileSizes[f2] - fileSizes[f1])) while files: # print [( lfn, fileSizes[lfn] ) for lfn in files] group = [] sizeTot = 0 for lfn in list(files): size = fileSizes[lfn] if size >= maxSizePerJob: lfnGroups.append([lfn]) elif sizeTot + size < maxSizePerJob: sizeTot += size group.append(lfn) files.remove(lfn) if group: lfnGroups.append(group) if printOutput: self.log.notice(self.pPrint.pformat(lfnGroups)) return S_OK(lfnGroups) ############################################################################# def getAccessURL(self, lfn, storageElement, protocol=None, printOutput=False): """Allows to retrieve an access URL for an LFN replica given a valid DIRAC SE name. Contacts the file catalog and contacts the site SRM endpoint behind the scenes. Example Usage: >>> print dirac.getAccessURL('/lhcb/data/CCRC08/DST/00000151/0000/00000151_00004848_2.dst','CERN-RAW') {'OK': True, 'Value': {'Successful': {'srm://...': {'SRM2': 'rfio://...'}}, 'Failed': {}}} :param lfn: Logical File Name (LFN) :type lfn: str or python:list :param storageElement: DIRAC SE name e.g. CERN-RAW :type storageElement: string :param printOutput: Optional flag to print result :type printOutput: boolean :returns: S_OK,S_ERROR """ ret = self._checkFileArgument(lfn, 'LFN') if not ret['OK']: return ret lfn = ret['Value'] if isinstance(lfn, basestring): lfn = [lfn] results = getAccessURL(lfn, storageElement, protocol=protocol) if printOutput: printDMResult(results, empty="File not at SE", script="dirac-dms-lfn-accessURL") return results ############################################################################# def _getLocalInputData(self, parameters): """ LHCb extension of DIRAC API's _getLocalInputData. Only used for handling ancestors. """ inputData = parameters.get('InputData') if inputData: self.log.debug("DiracLHCb._getLocalInputData. InputData: %s" % inputData) if isinstance(inputData, basestring): inputData = inputData.split(';') inputData = [lfn.strip('LFN:') for lfn in inputData] ancestorsDepth = int(parameters.get('AncestorDepth', 0)) if ancestorsDepth: self.log.debug("DiracLHCb._getLocalInputData. ancestorsDepth: %d" % ancestorsDepth) res = self._bkClient.getFileAncestors(inputData, ancestorsDepth) if not res['OK']: self.log.error("Can't get ancestors", res['Message']) return res ancestorsLFNs = [] for ancestorsLFN in res['Value']['Successful'].itervalues(): ancestorsLFNs += [i['FileName'] for i in ancestorsLFN] self.log.info("DiracLHCb._getLocalInputData: adding %d ancestors" % len(ancestorsLFNs)) self.log.verbose("%s", ', '.join(ancestorsLFNs)) inputData += ancestorsLFNs return S_OK(inputData)
transformation.setPlugin(plugin) transformation.setBkQuery(bkQueryDict) from LHCbDIRAC.TransformationSystem.Agent.TransformationPlugin import TransformationPlugin transID = -9999 pluginParams['TransformationID'] = transID pluginParams['Status'] = "Active" pluginParams['Type'] = transType # Create a fake transformation client fakeClient = fakeClient(transformation, transID, lfns, asIfProd) from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient from DIRAC.DataManagementSystem.Client.DataManager import DataManager oplugin = TransformationPlugin(plugin, transClient=fakeClient, dataManager=DataManager(), bkClient=BookkeepingClient()) pluginParams['TransformationID'] = transID pluginParams.update(pluginSEParams) oplugin.setParameters(pluginParams) replicas = fakeClient.getReplicas() # Special case of RAW files registered in CERN-RDST... if plugin == "AtomicRun": for lfn in [lfn for lfn in replicas if "CERN-RDST" in replicas[lfn]]: ses = {} for se in replicas[lfn]: pfn = replicas[lfn][se] if se == "CERN-RDST": se = "CERN-RAW" ses[se] = pfn replicas[lfn] = ses files = fakeClient.getFiles()
import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... LFN|File Flag' % Script.scriptName, 'Arguments:', ' LFN: Logical File Name', ' File: Name of the file with a list of LFNs', ' Flag: Quality Flag' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bk = BookkeepingClient() if len(args) < 2: result = bk.getAvailableDataQuality() if not result['OK']: print 'ERROR: %s' % (result['Message']) DIRAC.exit(2) flags = result['Value'] print "Available Data Quality Flags" for flag in flags: print flag Script.showHelp() exitCode = 0 filename = args[0] flag = args[1]
" output file name [dirac-dms-chec-dir-cont.out]") Script.registerSwitch("v", "Verbose", " use this option for verbose output [False]") Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ...' % Script.scriptName, ])) Script.parseCommandLine(ignoreErrors=False) from DIRAC.Resources.Catalog.FileCatalog import FileCatalog from DIRAC.Resources.Storage.StorageElement import StorageElement from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bkClient = BookkeepingClient() outputFileName = 'dirac-dms-chec-dir-cont.out' for switch in Script.getUnprocessedSwitches(): if switch[0].lower() == "u" or switch[0].lower() == "unit": unit = switch[1] if switch[0] == "D" or switch[0].lower() == "dir": dir = switch[1] if switch[0] == "f" or switch[0].lower() == "output": outputFile = switch[1] outputFileName = outputFile if switch[0] == "v" or switch[0].lower() == "verbose": verbose = True if verbose:
#!/usr/bin/env python import sys import random import time from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient cl = BookkeepingClient() res = cl.getAvailableProductions() if not res['OK']: print res['Message'] sys.exit(0) allproductions = sorted([i[0] for i in res['Value']], reverse=True) productions = allproductions[:10000] class Transaction(object): def __init__(self): self.custom_timers = {} def run(self): # print len(datasets) i = random.randint(0, len(productions) - 1) production = productions[i] # print dataset
# DIRAC.exit(2) # # Processing pass needs to start as "/Real Data" for FULL stream flagging # if realData not in processing: print 'You forgot /Real Data in the processing pass: '******'OK']: gLogger.error('Cannot load the information for run %s' % (run)) gLogger.error(res['Message']) DIRAC.exit(2) dtd = res['Value']['DataTakingDescription'] configName = res['Value']['Configuration Name'] configVersion = res['Value']['Configuration Version'] bkDict = { 'ConfigName': configName, 'ConfigVersion': configVersion, 'ConditionDescription': dtd }
])) Script.parseCommandLine(ignoreErrors=False) dumpList = False for switch in Script.getUnprocessedSwitches(): if switch[0] == 'List': dumpList = True bkQuery = dmScript.getBKQuery() lfns = dmScript.getOption('LFNs', []) if not bkQuery and not lfns: gLogger.error("No BKQuery and no files given...") dExit(1) # Invert the visibility flag as want to set Invisible those that are visible and vice-versa from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bk = BookkeepingClient() visibilityFlag = dmScript.getOption('Visibility', None) if visibilityFlag is None: gLogger.error('Visibility option should be given') dExit(2) visibilityFlag = str(visibilityFlag).lower() == 'yes' if bkQuery: # Query with visibility opposite to what is requested to be set ;-) bkQuery.setOption('Visible', 'No' if visibilityFlag else 'Yes') gLogger.notice("BQ query:", bkQuery) lfns += bkQuery.getLFNs() if not lfns: gLogger.notice("No files found...") else: res = {'OK': True}
class ProcessingProgress( object ): def __init__( self, cacheFile = None ): if not cacheFile: self.prodStatFile = os.path.join( os.environ['HOME'], ".dirac/work", "dirac-production-stats.pkl" ) else: self.prodStatFile = cacheFile self.cacheVersion = '0.0' self.clearCache = [] self.cachedInfo = {} # Recuperate the previous cached information self.readCache() self.bk = BookkeepingClient() self.transClient = TransformationClient() def setClearCache( self, clearCache ): self.clearCache = clearCache def __getProdBkDict( self, prod ): res = self.transClient.getBookkeepingQuery( prod ) if not res['OK']: gLogger.error( "Couldn't get BK query on production %d" % prod ) return {} prodBKDict = res['Value'] return prodBKDict def getFullStats( self, bkQuery, printResult = False ): processingPass = bkQuery.getProcessingPass() if printResult: gLogger.info( "\nStatistics for processing %s, condition %s\n" % ( processingPass, bkQuery.getConditions() ) ) prodStats = [] processingPass = processingPass.split( '/' ) if len( processingPass ) != 4 or processingPass[1] != "Real Data": gLogger.error( "Processing pass should be /Real Data/<Reco>/<Stripping>" ) return [] # Get production numbers for the Reco recoBKQuery = BKQuery( bkQuery ) recoBKQuery.setProcessingPass( '/'.join( processingPass[0:3] ) ) recoList = recoBKQuery.getBKProductions( visible = False ) recoRunRanges = {} recoDQFlags = [] for prod in recoList: prodBKDict = self.__getProdBkDict( prod ) if prodBKDict: recoRunRanges[prod] = [prodBKDict.get( "StartRun", 0 ), prodBKDict.get( "EndRun", sys.maxint )] dqFlags = prodBKDict.get( "DataQualityFlag", ['UNCHECKED', 'EXPRESS_OK', 'OK'] ) if isinstance( dqFlags, basestring ): dqFlags = dqFlags.split( ',' ) recoDQFlags += [fl for fl in dqFlags if fl not in recoDQFlags] else: recoRunRanges[prod] = [0, 0] # Sort productions by runs try: recoList.sort( cmp = ( lambda p1, p2: int( recoRunRanges[p1][0] - recoRunRanges[p2][1] ) ) ) except: print "Exception in sorting productions:" for p in recoList: print p, recoRunRanges[p] gLogger.verbose( "Reconstruction productions found (%d): %s" % ( len( recoList ), str( sorted( recoList ) ) ) ) gLogger.verbose( "Reconstruction DQ flags: %s" % str( recoDQFlags ) ) # Get productions for merging mergeList = [] mergeStripProds = {} # Get stripping productions as parents of merging productions stripList = [] for prod in bkQuery.getBKProductions( visible = False ): prodBKDict = self.__getProdBkDict( prod ) gLogger.verbose( "BK query for production %s: %s" % ( prod, str( prodBKDict ) ) ) mergedTypes = prodBKDict.get( 'FileType' ) if type( mergedTypes ) != type( [] ): mergedTypes = [mergedTypes] if [ft for ft in bkQuery.getFileTypeList() if ft in mergedTypes] and 'ProductionID' in prodBKDict: mergeList.append( prod ) prods = prodBKDict['ProductionID'] if type( prods ) != type( [] ): prods = [prods] stripList += prods mergeStripProds[prod] = [str( p ) for p in prods] else: _msgTuple = ( str( bkQuery.getFileTypeList() ), prod, str( prodBKDict ) ) gLogger.verbose( "Could not find production or filetype %s in BKquery of production %d (%s)" % _msgTuple ) mergeList.sort( cmp = ( lambda p1, p2: int( mergeStripProds[p1][0] ) - int( mergeStripProds[p2][0] ) ) ) gLogger.verbose( "Merging productions found: %s" % str( mergeList ) ) # get list of stripping productions (from merging) stripRunRanges = {} for prod in stripList: prodBKDict = self.__getProdBkDict( prod ) if prodBKDict: stripRunRanges[prod] = [prodBKDict.get( "StartRun", 0 ), prodBKDict.get( "EndRun", sys.maxint )] else: stripRunRanges[prod] = [0, 0] # Sort productions by runs try: stripList.sort( cmp = ( lambda p1, p2: int( stripRunRanges[p1][0] - stripRunRanges[p2][1] ) ) ) except: print "Error when sorting stripping productions:" for prodStrip in stripList: print prodStrip, stripRunRanges[prodStrip] gLogger.verbose( "Stripping productions found (%d): %s" % ( len( stripList ), str( sorted( stripList ) ) ) ) # Get all runs corresponding to the run range used by the Reco productions rawBKQuery = BKQuery( bkQuery ) rawBKQuery.setProcessingPass( '/Real Data' ) rawBKQuery.setFileType( "RAW" ) # get the list of runs (-prodNum) fullRunList = rawBKQuery.getBKRuns() gLogger.verbose( "Initial list of runs: %s" % str( fullRunList ) ) recoRunList = [] openProd = False for prod in [p for p in recoList]: # Forget fully openProd productions # Don't consider productions without a BK query (these are individual files) if recoRunRanges[prod][1] == sys.maxint and recoRunRanges[prod][0] != -sys.maxint: openProd = True # Try and find if that open production overlaps with a closed one, in which case, remove it # Do nothing for derived productions for p in [p for p in recoList if p != prod and recoRunRanges[prod] != recoRunRanges[p]]: if recoRunRanges[prod][0] < recoRunRanges[p][1] and recoRunRanges[p][1] != sys.maxint: openProd = False gLogger.verbose( "Production %s was removed as redundant..." % str( prod ) ) recoList.remove( prod ) break if not openProd: continue recoRunList += [run for run in fullRunList if run not in recoRunList and run >= recoRunRanges[prod][0] and run <= recoRunRanges[prod][1]] gLogger.verbose( "List of runs matching Reco (%d): %s" % ( len( recoRunList ), str( sorted( recoRunList ) ) ) ) restrictToStripping = True if restrictToStripping and not openProd and stripList: runList = [] for prod in stripList: runList += [run for run in recoRunList if run not in runList and run >= stripRunRanges[prod][0] and run <= stripRunRanges[prod][1]] else: runList = recoRunList gLogger.verbose( "Final list of runs matching Reco and Stripping (%d): %s" % ( len( runList ), str( sorted( runList ) ) ) ) # Now get statistics from the runs info, runInfo = self._getStatsFromRuns( int( bkQuery.getEventTypeList()[0] ), runList, recoDQFlags ) rawInfo = StatInfo( processingPass[1], info ) prodStats.append( rawInfo ) if printResult: gLogger.info( "%s - All runs in Reco productions" % processingPass[1] ) for fileInfo in runInfo: if runInfo[fileInfo]: gLogger.info( "%s runs (%d): %s" % ( fileInfo, len( runInfo[fileInfo] ), str( runInfo[fileInfo] ) ) ) summStr = "%s files, " % rawInfo.getItemAsString( 'Files' ) summStr += "%s events in " % rawInfo.getItemAsString( 'Events' ) summStr += "%s runs, luminosity (pb-1):All=%s, Bad=%s, OK=%s" % ( rawInfo.getItemAsString( 'Runs' ), rawInfo.getItemAsString( 'Lumi' ), rawInfo.getItemAsString( 'BadLumi' ), rawInfo.getItemAsString( 'OKLumi' ) ) gLogger.info( summStr ) # Create the info for the 3 sets of productions prodSets = [] fileType = bkQuery.getFileTypeList()[0] prodSets.append( {'Name': processingPass[2], 'FileType': ['SDST', 'FULL.DST'], 'List':recoList, 'RunRange':recoRunRanges, 'MotherProds':None, 'AllReplicas':False } ) prodSets.append( {'Name': processingPass[3], 'FileType': fileType, 'List':stripList, 'RunRange':stripRunRanges, 'MotherProds':None, 'AllReplicas':True, 'StatForOK':False } ) prodSets.append( {'Name': "Merging (%s)" % fileType.split( '.' )[0], 'FileType': fileType, 'List':mergeList, 'RunRange':None, 'MotherProds':mergeStripProds, 'AllReplicas':False } ) prevInfo = rawInfo for prodSet in prodSets: info = StatInfo( prodSet['Name'], self._getProdInfo( prodSet, runList, printResult = printResult ) ) info.setRawInfo( rawInfo ) info.setPrevInfo( prevInfo ) prevInfo = info prodStats.append( info ) self.saveCache() return prodStats @staticmethod def __sumProdInfo( info, totInfo ): for inf in info: for flag in info[inf]: if inf == 'Runs': totInfo[inf][flag] = totInfo.setdefault( inf, {} ).setdefault( flag, [] ) + info[inf][flag] else: totInfo[inf][flag] = totInfo.setdefault( inf, {} ).setdefault( flag, 0 ) + info[inf][flag] return totInfo def _getProdInfo( self, prodSet, runList, printResult = False ): totInfo = {} if printResult: gLogger.info( "" ) for prod in prodSet['List']: info, runInfo = self._getStatsFromBK( prod, prodSet['FileType'], runList, prodSet['AllReplicas'] ) if info['Files'][''] == 0: continue if not prodSet.get( 'StatForOK', True ): for item in info: for fl in info[item]: if fl == 'OK': info[item][fl] = 0 if not item == 'Runs' else [] runRange = prodSet['RunRange'] if runRange and prod in runRange and runRange[prod][0] == 0 and runRange[prod][1] == 0: for flag in info['Runs']: info['Runs'][flag] = [] totInfo = self.__sumProdInfo( info, totInfo ) if printResult: summStr = "%s production %d -" % ( prodSet['Name'], prod ) if runRange and prod in runRange: firstRun = runRange[prod][0] lastRun = runRange[prod][1] if firstRun: summStr += " From run %d" % int( firstRun ) if lastRun and lastRun != sys.maxint: summStr += " Up to run %d" % int( lastRun ) if firstRun == 0 and lastRun == 0: summStr += "No run range specified" motherProds = prodSet['MotherProds'] if motherProds and prod in motherProds: summStr += " from productions %s" % motherProds[prod] gLogger.info( summStr ) for inf in runInfo: if runInfo[inf]: gLogger.info( "%s runs (%d): %s" % ( inf, len( runInfo[inf] ), str( runInfo[inf] ) ) ) summStr = "%d files, " % info['Files'][''] if info['Events']: summStr += "%d events in " % info['Events'][''] _msgTuple = ( len( info['Runs'][''] ), info['Lumi'][''], info['Lumi']['Bad'], info['Lumi']['OK'] ) summStr += "%d runs, luminosity (pb-1): All=%.3f, Bad=%.3f, OK=%.3f" % _msgTuple gLogger.info( summStr ) for flag in totInfo.get( 'Runs', [] ): totInfo['Runs'][flag] = len( totInfo['Runs'][flag] ) return totInfo @staticmethod def outputResults( conditions, processingPass, prodStats ): outputString = "" _msgTuple = ( conditions, ",", processingPass, "on", time.ctime( time.time() ) ) outputString += "\nProduction progress for %s %s %s %s %s\n" % _msgTuple if len( prodStats ) < 4: outputString += "No statistics found for this BK query" return outputString for i in xrange( 4 ): info = prodStats[i] if not info: continue name = info.getName() outputString += "\nSummary for %s\n" % name outputString += "%d files, " % info.getItem( 'Files' ) if info.getItem( 'Events' ): outputString += "%d events in " % info.getItem( 'Events' ) _msgTuple = ( info.getItem( 'Runs' ), info.getItem( 'Lumi' ), info.getItem( 'BadLumi' ), info.getItem( 'OKLumi' ) ) outputString += "%d runs, luminosity (pb-1): All=%.3f, Bad=%.3f, OK=%.3f\n" % _msgTuple prevStats = prodStats[:i] prevStats.reverse() for prevInfo in prevStats: name = prevInfo.getName() if prevInfo.getItem( 'Runs' ) == 0: outputString += "From %s : - No runs...\n" % name else: outputString += "From %s : %.1f%% files, " % ( name, 100.*info.getItem( 'Files' ) / prevInfo.getItem( 'Files' ) ) if info.getItem( 'Events' ) and prevInfo.getItem( 'Events' ): outputString += "%.1f%% events\n" % ( 100.*info.getItem( 'Events' ) / prevInfo.getItem( 'Events' ) ) outputString += "%.1f%% runs, %.1f%% luminosity\n" \ % ( 100. * info.getItem( 'Runs' ) / prevInfo.getItem( 'Runs' ), 100. * info.getItem( 'Lumi' ) / prevInfo.getItem( 'Lumi' ) ) return outputString def __getRunsDQFlag( self, runList, evtType ): res = self.bk.getRunFilesDataQuality( runList ) runFlags = {} if res['OK']: dqFlags = res['Value'] for dq in dqFlags: if dq[2] == evtType: runFlags.setdefault( dq[0], [] ).append( dq[1] ) runDQFlags = {} flags = ( 'BAD', 'OK', 'EXPRESS_OK', 'UNCHECKED' ) for run in runFlags: for fl in flags: if fl in runFlags[run]: runDQFlags[run] = fl break return runDQFlags def _getStatsFromRuns( self, evtType, runList, recoDQFlags ): info = dict.fromkeys( ( 'Events', 'Runs', 'Files', 'Lumi' ), {} ) for inf in info: info[inf] = dict.fromkeys( ( 'Bad', 'OK', '' ), 0 ) now = datetime.datetime.utcnow() # Set to True to renew the cache clearCache = 'RAW' in self.clearCache newRuns = [ run for run in runList if clearCache or run not in self.cachedInfo or 'DQFlag' not in self.cachedInfo[run] or ( now - self.cachedInfo[run]['Time'] ) < datetime.timedelta( days = 2 ) ] if newRuns: runFlags = self.__getRunsDQFlag( newRuns, evtType ) else: runFlags = {} runsByDQFlag = {} runInfo = {} for run in runList: cached = self.cachedInfo.get( run, {} ) cachedTime = cached.get( 'Time', None ) if run not in newRuns: cachedFiles = cached.get( 'Files', 0 ) cachedEvents = cached.get( 'EventStat', 0 ) cachedLumi = cached.get( 'Luminosity', 0 ) dqFlag = cached.get( 'DQFlag', None ) else: res = self.bk.getRunInformations( run ) if res['OK']: val = res['Value'] ind = val['Stream'].index( 90000000 ) cachedFiles = val['Number of file'][ind] cachedEvents = val['Number of events'][ind] cachedLumi = val['luminosity'][ind] cachedTime = val['RunStart'] else: gLogger.error( "Unable to get run information for run %s" % str( run ) ) continue dqFlag = runFlags[run] self.cachedInfo[run] = { 'Time':cachedTime, 'Files':cachedFiles, 'EventStat': cachedEvents, 'Luminosity': cachedLumi, 'DQFlag':dqFlag } runsByDQFlag[dqFlag] = runsByDQFlag.setdefault( dqFlag, 0 ) + 1 if dqFlag == "BAD": runInfo.setdefault( 'BAD', [] ).append( run ) elif dqFlag not in recoDQFlags and dqFlag != 'OK' : runInfo.setdefault( 'Untagged', [] ).append( run ) # Now count... flags = [] if dqFlag != 'BAD': flags.append( '' ) # OK in recoDQFlags means we take everything that is not BAD (reprocessing or new convention) if dqFlag in recoDQFlags or dqFlag == 'OK': flags.append( 'OK' ) else: flags.append( 'Bad' ) for flag in flags: info['Runs'][flag] += 1 info['Files'][flag] += cachedFiles info['Events'][flag] += cachedEvents info['Lumi'][flag] += cachedLumi # Set lumi in pb-1 for flag in info['Lumi']: info['Lumi'][flag] /= 1000000. gLogger.info( "Runs per flag:" ) for key in runsByDQFlag: gLogger.info( "%s : %d" % ( key, runsByDQFlag[key] ) ) for flag in runInfo: runInfo[flag].sort() return info, runInfo def __getLfnsMetadata( self, lfns ): lfnDict = {} if len( lfns ): gLogger.verbose( "Getting metadata for %d files" % len( lfns ) ) for lfnChunk in breakListIntoChunks( lfns, 1000 ): while True: res = self.bk.getFileMetadata( lfnChunk ) if not res['OK']: gLogger.error( "Error getting files metadata, retrying...", res['Message'] ) else: break metadata = res['Value']['Successful'] for lfn in lfnChunk: lfnDict[lfn] = {} for meta in ( 'EventStat', 'Luminosity', 'DQFlag', 'RunNumber' ): lfnDict[lfn][meta] = metadata[lfn][meta] return lfnDict def _getStatsFromBK( self, prod, fileType, runList, allReplicas ): bkQueryDict = { "ProductionID": prod, "FileType": fileType } bkStr = str( bkQueryDict ) bkQuery = BKQuery( bkQueryDict, visible = not allReplicas ) if allReplicas: bkQuery.setOption( 'ReplicaFlag', "All" ) cached = self.cachedInfo.get( bkStr, {} ) cachedTime = cached.get( 'Time', None ) cachedLfns = cached.get( 'Lfns', {} ) if isinstance( fileType, basestring ): fileType = [fileType] if set( fileType ).intersection( set( self.clearCache ) ): cachedTime = datetime.datetime.utcnow() - datetime.timedelta( days = 8 ) cachedTime = None cachedLfns = {} gLogger.verbose( "Cleared cache for production %s, file type %s" % ( str( prod ), fileType ) ) # Update if needed the cached information on LFNs if cachedLfns: lfns = [lfn for lfn in cachedLfns if cachedLfns[lfn].get( 'DQFlag' ) not in ( 'OK', 'BAD' )] for lfnChunk in breakListIntoChunks( lfns, 1000 ): # get the DQFlag of files that are not yet OK while True: res = self.bk.getFileMetadata( lfnChunk ) if not res['OK']: gLogger.error( "Error getting files metadata for cached files, bkQuery %s: %s" % ( bkStr, res['Message'] ) ) else: metadata = res['Value']['Successful'] for lfn in lfnChunk: cachedLfns[lfn]['DQFlag'] = metadata[lfn]['DQFlag'] break # Now get the new files since last time... if cachedTime: bkQuery.setOption( 'StartDate', cachedTime.strftime( '%Y-%m-%d %H:%M:%S' ) ) gLogger.verbose( "Getting files for BKQuery %s" % str( bkQuery ) ) cachedTime = datetime.datetime.utcnow() lfns = [lfn for lfn in bkQuery.getLFNs( printOutput = False ) if lfn not in cachedLfns] gLogger.verbose( "Returned %d files" % len( lfns ) ) cachedLfns.update( self.__getLfnsMetadata( lfns ) ) self.cachedInfo[bkStr] = { 'Time':cachedTime, 'Lfns':cachedLfns } # Now sum up all information for the files info = dict.fromkeys( ( 'Events', 'Runs', 'Files', 'Lumi' ), {} ) for inf in info: if inf == 'Runs': for flag in ( 'Bad', 'OK', '' ): info[inf][flag] = [] else: info[inf] = dict.fromkeys( ( 'Bad', 'OK', '' ), 0 ) for lfn in cachedLfns: lfnInfo = cachedLfns[lfn] run = lfnInfo['RunNumber'] if run in runList and run in self.cachedInfo and self.cachedInfo[run]['DQFlag'] != 'BAD': dqFlag = cachedLfns[lfn]['DQFlag'] flags = [] if dqFlag != 'BAD': flags.append( '' ) else: flags.append( 'Bad' ) if dqFlag == 'OK': flags.append( 'OK' ) for flag in flags: if run not in info['Runs'][flag]: info['Runs'][flag].append( run ) info['Files'][flag] += 1 info['Events'][flag] += lfnInfo['EventStat'] info['Lumi'][flag] += lfnInfo['Luminosity'] runInfo = {} if 'BAD' in info['Runs']: runInfo['BAD'] = info['Runs']['BAD'] runInfo['BAD'].sort() else: runInfo['BAD'] = [] if '' in info['Runs'] and 'OK' in info['Runs']: runInfo['Untagged'] = [run for run in info['Runs'][''] if run not in info['Runs']['OK']] runInfo['Untagged'].sort() else: runInfo['Untagged'] = [] # for f in info['Runs']: # info['Runs'][f] = len( info['Runs'][f] ) for flag in info['Lumi']: info['Lumi'][flag] /= 1000000. return info, runInfo def getPreviousStats( self, processingPass ): prevStats = self.cachedInfo.get( 'ProdStats', {} ).get( processingPass ) if prevStats: try: _name = prevStats['ProdStats'][0][0].getName() except: prevStats = None return prevStats def setPreviousStats( self, processingPass, prevStats ): self.cachedInfo.setdefault( 'ProdStats', {} )[processingPass] = prevStats self.saveCache() def readCache( self ): if not os.path.exists( self.prodStatFile ): gLogger.info( "Created cached file %s" % self.prodStatFile ) self.cachedInfo = {} self.saveCache() return fileRead = False while not fileRead: try: with FileLock( self.prodStatFile ): lFile = open( self.prodStatFile, 'r' ) cachedVersion = pickle.load( lFile ) startTime = time.time() if cachedVersion == self.cacheVersion: self.cachedInfo = pickle.load( lFile ) _msgTuple = ( self.prodStatFile, time.time() - startTime ) gLogger.info( "Loaded cached information from %s in %.3f seconds" % _msgTuple ) else: _msgTuple = ( cachedVersion, self.cacheVersion ) gLogger.info( "Incompatible versions of cache, reset information (%s, expect %s)" % _msgTuple ) self.cachedInfo = {} lFile.close() fileRead = True except FileLockException, error: gLogger.error( "Lock exception: %s while reading pickle file %s" % ( error, self.prodStatFile ) ) except:
inputFiles = True outputFiles = True jobidList = [] for jobid in args: if os.path.exists(jobid): bkScript.setJobidsFromFile(jobid) else: jobidList += jobid.split(',') jobidList += bkScript.getOption('JobIDs', []) if not jobidList: print "No jobID provided!" Script.showHelp() DIRAC.exit(0) from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient retVal = BookkeepingClient().getJobInputOutputFiles(jobidList) if retVal['OK']: success = retVal['Value']['Successful'] for job in success: # Remove from input the files that are also output! This happens because the output of step 1 can be the input of step 2... # only worth if input files are requested though if inputFiles: success[job]['InputFiles'] = sorted( set(success[job]['InputFiles']) - set(success[job]['OutputFiles'])) if not inputFiles or not outputFiles: success[job].pop( 'InputFiles' if not inputFiles else 'OutputFiles') printDMResult(retVal, empty="File does not exists in the Bookkeeping")
def execute(): """ Parse the options and execute the script """ bkQuery = dmScript.getBKQuery() fileType = bkQuery.getFileTypeList() if not set(fileType) & {'FULL.DST', 'RDST', 'SDST'}: gLogger.error("Please provide a reconstruction BK path") DIRAC.exit(1) from LHCbDIRAC.TransformationSystem.Client.TransformationClient import TransformationClient from DIRAC.DataManagementSystem.Client.DataManager import DataManager from DIRAC.Core.Utilities.List import breakListIntoChunks from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient from DIRAC.DataManagementSystem.Utilities.DMSHelpers import DMSHelpers, resolveSEGroup bk = BookkeepingClient() tr = TransformationClient() dm = DataManager() dmsHelper = DMSHelpers() bkQueryDict = bkQuery.getQueryDict() gLogger.notice("For BK Query:", str(bkQueryDict)) progressBar = ProgressBar(1, title="Running BK query...", step=1) res = bk.getFilesWithMetadata(bkQueryDict) if not res['OK']: gLogger.error("Error getting files from BK", res['Message']) DIRAC.exit(2) if 'ParameterNames' in res.get('Value', {}): parameterNames = res['Value']['ParameterNames'] info = res['Value']['Records'] progressBar.endLoop("Obtained %d files" % len(info)) else: gLogger.error('\nNo metadata found') DIRAC.exit(3) lfns = [] runLFNs = {} for item in info: metadata = dict(zip(parameterNames, item)) lfn = metadata['FileName'] lfns.append(lfn) runLFNs.setdefault(metadata['RunNumber'], []).append(lfn) chunkSize = 1000 progressBar = ProgressBar(len(lfns), title='Getting replicas of %d files' % len(lfns), chunk=chunkSize) replicas = {} errors = {} for lfnChunk in breakListIntoChunks(lfns, chunkSize): progressBar.loop() res = dm.getReplicas(lfnChunk, getUrl=False) if not res['OK']: errors.setdefault(res['Message'], []).extend(lfnChunk) else: replicas.update(res['Value']['Successful']) for lfn, error in res['Value']['Failed'].iteritems(): errors.setdefault(error, []).append(lfn) progressBar.endLoop() for error, lfns in errors.iteritems(): gLogger.error(error, 'for %d files' % len(lfns)) tier1RDST = set(resolveSEGroup('Tier1-RDST')) setOK = 0 errors = {} progressBar = ProgressBar(len(runLFNs), title='Defining destination for %d runs' % len(runLFNs), step=10) for run, lfns in runLFNs.iteritems(): progressBar.loop() res = tr.getDestinationForRun(run) if res.get('Value'): errors.setdefault('Destination already set', []).append(str(run)) continue # print 'Run', run, len( lfns ), 'Files', lfns[:3] seCounts = {} for lfn in lfns: for se in tier1RDST.intersection(replicas.get(lfn, [])): seCounts[se] = seCounts.setdefault(se, 0) + 1 # print seCounts maxi = 0 seMax = None for se, count in seCounts.iteritems(): if count > maxi: seMax = se maxi = count if not seMax: errors.setdefault('No SE found, use CERN-RDST', []).append(str(run)) seMax = 'CERN-RDST' # SE found, get its site res = dmsHelper.getLocalSiteForSE(seMax) if res['OK']: site = res['Value'] res = tr.setDestinationForRun(run, site) if not res['OK']: errors.setdefault(res['Message'], []).append(str(run)) else: setOK += 1 progressBar.endLoop('Successfully set destination for %d runs' % setOK) for error, runs in errors.iteritems(): gLogger.error(error, 'for runs %s' % ','.join(runs))
# Author : Zoltan Mathe ######################################################################## """ List file types from the Bookkeeping """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile]' % Script.scriptName ])) Script.parseCommandLine(ignoreErrors=True) from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bk = BookkeepingClient() exitCode = 0 mfiletypes = [] res = bk.getAvailableFileTypes() if res['OK']: dbresult = res['Value'] print 'Filetypes:' for record in dbresult['Records']: print str(record[0]).ljust(30) + str(record[1]) DIRAC.exit(exitCode)
######################################################################## """ Insert a new set of simulation conditions in the Bookkeeping """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ...' % Script.scriptName ])) Script.parseCommandLine(ignoreErrors=True) from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bk = BookkeepingClient() exitCode = 0 desc = raw_input("SimDescription: ") beamcond = raw_input("BeamCond: ") beamEnergy = raw_input("BeamEnergy: ") generator = raw_input("Generator: ") magneticField = raw_input("MagneticField: ") detectorCond = raw_input("DetectorCond: ") luminosity = raw_input("Luminosity: ") g4settings = raw_input("G4settings: ") print 'Do you want to add these new simulation conditions? (yes or no)' value = raw_input('Choice:') choice = value.lower() if choice in ['yes', 'y']:
class BookkeepingDB: def __init__(self): self.bkClient = BookkeepingClient() self.tfClient = TransformationClient() # self.err = err def MakeRunLFN(self, runNmbr, cfgVersion, prodId): try: padding = "%08d" % int(prodId) lfn = LFN_FORMAT_STRING %( cfgVersion, runNmbr, runNmbr, padding) return lfn except Exception as inst: # self.err.rethrowException(inst) return None def getTCK(self, runNmbr): try: print runNmbr res = self.getRunsMetadata(runNmbr) pprint.pprint(res) if res != None and hasattr(res, 'Value') \ and hasattr(res['Value'], runNmbr) \ and hasattr(res['Value'][runNmbr], "TCK"): return res['Value'][runNmbr]["TCK"] else: return None except Exception as inst: # self.err.rethrowException(inst) return None def getRunsMetadata(self, runNmbr): try: res = self.tfClient.getRunsMetadata(int(runNmbr)) if res['OK']: return res else: return None except Exception as inst: # self.err.rethrowException(inst) return None def getInformation(self, run): try: res = self.bkClient.getRunInformations(run) if res['OK']: result = dict() val = res['Value'] result = {"runstart": val.get('RunStart', 'Unknown'), "runend": val.get('RunEnd', 'Unknown'), "configname": val.get('Configuration Name', 'Unknown'), "configversion": val.get('Configuration Version', 'Unknown'), "fillnb" : val.get('FillNumber', 'Unknown'), "datataking" : val.get('DataTakingDescription', 'Unknown'), "datataking" : val.get('DataTakingDescription', 'Unknown'), "processing" : val.get('ProcessingPass', 'Unknown'), "stream" : val.get('Stream', 'Unknown'), "fullstat" : val.get('FullStat', 'Unknown'), "nbofe" : val.get('Number of events', 'Unknown'), "nboff" : val.get('Number of file', 'Unknown'), "fsize" : val.get('File size', 'Unknown') } return result else: self.errorMessage("error in bkClient Connection") return None except Exception as inst: # self.err.rethrowException(inst) return None def getListOfRecos(self, runNmbr): try: d = {'RunNumber' : runNmbr} res = self.bkClient.getRunAndProcessingPass(d) results = list() if res['OK'] == True: recosList = res["Value"] for recoEntry in recosList: recoPath = recoEntry[1] if recoEntry[0] == runNmbr \ and recoPath.count("/") == 2 \ and "Reco" in recoPath : results.append(recoPath) return results else: pprint.pprint(res) self.errorMessage("error in bkClient Connection") return None except Exception as inst: # self.err.rethrowException(inst) return None def getProcessId(self, runNmbr, recoVersion): try: d = {'RunNumber' : runNmbr, 'ProcessingPass': recoVersion} res = self.bkClient.getProductionsFromView(d) if res["OK"] == True: return res["Value"][0][0] else: self.errorMessage("error in bkClient Connection") return None except Exception as inst: # self.err.rethrowException(inst) return None #recoVersion is just Reco13 and not the full path!! def makeReferenceROOTFileName(self, recoVersion, runNmbr): try: basePath = REFERENCE_BASE_PATH + recoVersion +"/" #nasty stuff!! #the problem is tck retrieved from db #0x790038 #but in the file it looks like #TCK_0x00760037_1.root #so do padding here tck = self.getTCK(runNmbr) #sometimes no tck set, then take default file if tck != None: tckDecimal = int(tck, 0) tckHexPaddedFileName = "TCK_0x" + str(format(tckDecimal, '08x')) + "_" #if we have multiple files like #TCK_0x00790038_1.root #TCK_0x00790038_2.root #we want the file with the highest subindex, so in this example _2 possibleTCKList = list() #store all possible files for file in os.listdir(basePath): if tck != None \ and file.endswith(".root") \ and file != "default_1.root" \ and tckHexPaddedFileName in file: possibleTCKList.append(file) #if we haven't foun anything, look for the default files and choose the one with the highest index if len(possibleTCKList) == 0: #store all possible files for file in os.listdir(basePath): if file.endswith(".root") \ and "default_" in file: possibleTCKList.append(file) #now sort this list, to find the highest subindex possibleTCKList.sort() return basePath+possibleTCKList.pop() except Exception as inst: # self.err.rethrowException(inst) return None
Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... ProdID' % Script.scriptName, 'Arguments:', ' ProdID: Production ID' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() if len(args) < 1: Script.showHelp() exitCode = 0 from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bk = BookkeepingClient() try: prod = long(args[0]) except: Script.showHelp() DIRAC.exit(1) res = bk.getNbOfJobsBySites(prod) if res['OK']: if not res['Value']: print "No jobs for production", prod DIRAC.exit(0) sites = dict([(site, num) for num, site in res['Value']]) shift = 0 for site in sites:
def __init__(self): self.bkClient = BookkeepingClient() self.tfClient = TransformationClient()
class ProductionRequest(object): """ Production request class - objects are usually created starting from a production request """ def __init__(self, bkkClientIn=None, diracProdIn=None): """ c'tor Some variables are defined here. A production request is made of: stepsList, productionsTypes, and various parameters of those productions """ if bkkClientIn is None: self.bkkClient = BookkeepingClient() else: self.bkkClient = bkkClientIn if diracProdIn is None: self.diracProduction = DiracProduction() else: self.diracProduction = diracProdIn self.rpcProductionRequest = RPCClient( 'ProductionManagement/ProductionRequest') self.tc = TransformationClient() self.logger = gLogger.getSubLogger('ProductionRequest') self.opsH = Operations() # parameters of the request self.requestID = 0 self.parentRequestID = 0 self.appendName = '1' self.outConfigName = '' self.prodsToLaunch = [] # productions to launch self.stepsListDict = [] # list of dict of steps self.stepsInProds = [] # a list of lists # parameters of the input data self.processingPass = '' self.dataTakingConditions = '' self.eventType = '' self.bkFileType = [] self.dqFlag = '' self.startRun = 1 self.endRun = 2 self.runsList = '' self.configName = 'test' self.configVersion = 'certification' # parameters of the first production self.publishFlag = True self.testFlag = False self.derivedProduction = 0 self.previousProdID = 0 # optional prod from which to start self.fullListOfOutputFileTypes = [] # parameters that are the same for each productions self.prodGroup = '' # This ends up being 'ProcessingType', workflow parameter, and it is used for accounting self.visibility = '' # For BKQuery self.fractionToProcess = 0 self.minFilesToProcess = 0 self.modulesList = None # Usually: # ['GaudiApplication', 'AnalyseXMLSummary', # 'ErrorLogging', 'BookkeepingReport', 'StepAccounting' ] # parameters of each production (the length of each list has to be the same as the number of productions self.events = [] self.stepsList = [] self.extraOptions = {} self.prodsTypeList = [] self.bkQueries = [] # list of bk queries self.removeInputsFlags = [] self.priorities = [] self.cpus = [] self.inputs = [] # list of lists self.outputModes = [] self.targets = [] self.outputFileMasks = [] self.outputFileSteps = [] self.groupSizes = [] self.plugins = [] self.inputDataPolicies = [] self.previousProds = [ None ] # list of productions from which to take the inputs (the first is always None) self.multicore = [ ] # list of flags to override the multi core flags of the steps self.outputSEs = [] # a list of StorageElements self.specialOutputSEs = [] # a list of dictionaries - might be empty self.outputSEsPerFileType = [] # a list of dictionaries - filled later self.ancestorDepths = [] self.compDict = { 'HIGH': 'Compression-LZMA-4', 'LOW': 'Compression-ZLIB-1' } self.compressionLvl = [''] # List: one compression level each step self.appConfig = '$APPCONFIGOPTS/Persistency/' # Default location of the compression level configuration files # # These lists define the visibility of the output files produced by each step. For MC productions, the visibility # is tied to compression level. VIsible files are compressed at the highest level # self.outputVisFlag = [ ] # List of dictionary with default visibility flag of the output files per single step self.specialOutputVisFlag = [ ] # List of dictionaries with special visibility flag for given file type ############################################################################# def resolveSteps(self): """ Given a list of steps in strings, some of which might be missing, resolve it into a list of dictionary of steps (self.stepsListDict) """ outputVisFlag = dict( [k, v] for el in self.outputVisFlag for k, v in el.iteritems() ) # Transform the list of dictionaries in a dictionary specialOutputVisFlag = dict([k, v] for el in self.specialOutputVisFlag for k, v in el.iteritems()) count = 0 # Needed to add correctly the optionFiles to the list of dictonaries of steps for stepID in self.stepsList: stepDict = self.bkkClient.getAvailableSteps({'StepId': stepID}) if not stepDict['OK']: raise ValueError(stepDict['Message']) else: stepDict = stepDict['Value'] stepsListDictItem = {} s_in = self.bkkClient.getStepInputFiles(stepID) if not s_in['OK']: raise ValueError(s_in['Message']) stepsListDictItem['fileTypesIn'] = [ fileType[0].strip() for fileType in s_in['Value']['Records'] ] s_out = self.bkkClient.getStepOutputFiles(stepID) if not s_out['OK']: raise ValueError(s_out['Message']) fileTypesList = [ fileType[0].strip() for fileType in s_out['Value']['Records'] ] self.fullListOfOutputFileTypes = self.fullListOfOutputFileTypes + fileTypesList stepsListDictItem['fileTypesOut'] = fileTypesList for parameter, value in itertools.izip(stepDict['ParameterNames'], stepDict['Records'][0]): if parameter.lower() in ['conddb', 'dddb', 'dqtag'] and value: if value.lower() == 'frompreviousstep': value = self.stepsListDict[-1][parameter] if parameter == 'OptionFiles': # Modifying the OptionFiles (for setting the compression level) if 'MDF' not in stepsListDictItem[ 'fileTypesOut']: # certain MC produce MDF, which shouldn't be compressed # # If the prod manager sets a compression level for a particular step, either we append the option file # or we overwrite the existing one inherited with the step # if len(self.compressionLvl ) > count and self.compressionLvl[count] != '': persist = re.compile('Compression-[A-Z]{4}-[1-9]') # self.compressionLvl[count] = self.appConfig + self.compressionLvl[count] + '.py' self.compressionLvl[ count] = self.appConfig + self.compDict[ self.compressionLvl[count].upper()] + '.py' if not persist.search(value): if value == '': value = self.compressionLvl[count] else: value = ";".join( (value, self.compressionLvl[count])) else: value = persist.sub( persist.search( self.compressionLvl[count]).group(), value) # # If instead the prod manager doesn't declare a compression level, e.g. for intermediate steps, # we check if there is one in the options and in case we delete it. This leaves the default zip level # defined inside Gaudi # elif len( self.compressionLvl ) > count and self.compressionLvl[count] == '': persist = re.compile( r'\$\w+/Persistency/Compression-[A-Z]{4}-[1-9].py;?' ) if persist.search(value): value = persist.sub('', value) if parameter == 'SystemConfig' and value is not None and re.search( 'slc5', value): p = re.compile( r'\$\w+/Persistency/Compression-[A-Z]{4}-[1-9].py;?') if p.search(stepsListDictItem['OptionFiles']): stepsListDictItem['OptionFiles'] = p.sub( '', stepsListDictItem['OptionFiles']) stepsListDictItem[parameter] = value # Fixing what decided if stepsListDictItem['StepId'] in self.extraOptions: stepsListDictItem['ExtraOptions'] = self.extraOptions[ stepsListDictItem['StepId']] else: stepsListDictItem['ExtraOptions'] = '' stepsListDictItem['prodStepID'] = str(stepID) + str( stepsListDictItem['fileTypesIn']) if 'isMulticore' not in stepsListDictItem: stepsListDictItem['isMulticore'] = 'N' if 'SystemConfig' not in stepsListDictItem: stepsListDictItem['SystemConfig'] = '' if 'mcTCK' not in stepsListDictItem: stepsListDictItem['mcTCK'] = '' # Add visibility info during step resolution if 'visibilityFlag' not in stepsListDictItem: outputVisList = list({ 'Visible': outputVisFlag[str(count + 1)], 'FileType': ftype } for ftype in stepsListDictItem['fileTypesOut']) if str(count + 1) in specialOutputVisFlag: for it in outputVisList: if it['FileType'] in specialOutputVisFlag[str(count + 1)]: it['Visible'] = specialOutputVisFlag[str( count + 1)][it['FileType']] stepsListDictItem['visibilityFlag'] = outputVisList self.stepsListDict.append(stepsListDictItem) count += 1 ############################################################################# def buildAndLaunchRequest(self): """ uses _applyOptionalCorrections, _getProdsDescriptionDict, _buildProduction, and DiracProduction.launchProduction """ if not self.stepsListDict: self.resolveSteps() self._applyOptionalCorrections() self._determineOutputSEs() prodsDict = self._getProdsDescriptionDict() stepsListDict = list(self.stepsListDict) fromProd = self.previousProdID prodsLaunched = [] self.logger.debug(prodsDict) # now we build and launch each productions for prodIndex, prodDict in prodsDict.items(): if self.prodsToLaunch: if prodIndex not in self.prodsToLaunch: continue # build the list of steps in a production stepsInProd = [] for stepID in prodDict['stepsInProd-ProdName']: for step in stepsListDict: if step['prodStepID'] == stepID: stepsInProd.append( stepsListDict.pop(stepsListDict.index(step))) # NOT READY (alternative to previous 5 lines) # build the DAG of steps in a production # stepsInProdDAG = self._getStepsInProdDAG( prodDict, stepsListDict ) # Here, for today it is just convert to a list # TODO: fix this in order to properly use DAGs (now it's only sequential) # FIXME: using getIndexNodes we can't assure the order is respected # stepsInProd = stepsInProdDAG.getIndexNodes() if prodDict['previousProd'] is not None: fromProd = prodsLaunched[prodDict['previousProd'] - 1] self.previousProdID = fromProd prod = self._buildProduction( prodType=prodDict['productionType'], stepsInProd=stepsInProd, outputSE=prodDict['outputSE'], priority=prodDict['priority'], cpu=prodDict['cpu'], inputDataList=prodDict['input'], outputMode=prodDict['outputMode'], inputDataPolicy=prodDict['inputDataPolicy'], outputFileMask=prodDict['outputFileMask'], outputFileStep=prodDict['outputFileStep'], target=prodDict['target'], removeInputData=prodDict['removeInputsFlag'], groupSize=prodDict['groupSize'], bkQuery=prodDict['bkQuery'], plugin=prodDict['plugin'], previousProdID=fromProd, derivedProdID=prodDict['derivedProduction'], transformationFamily=prodDict['transformationFamily'], events=prodDict['events'], multicore=prodDict['multicore'], ancestorDepth=prodDict['ancestorDepth']) # if the production is a simulation production type, submit it to the automated testing if prodDict['productionType'] in self.opsH.getValue( 'Transformations/ExtendableTransfTypes', ['MCSimulation']): prodID = self._mcSpecialCase(prod, prodDict) else: res = self.diracProduction.launchProduction( prod=prod, publishFlag=self.publishFlag, testFlag=self.testFlag, requestID=self.requestID, tracking=prodDict['tracking']) if not res['OK']: raise RuntimeError(res['Message']) prodID = res['Value'] prodsLaunched.append(prodID) if self.publishFlag: self.logger.notice( "For request %d, submitted Production %d, of type %s, ID = %s" % (self.requestID, prodIndex, prodDict['productionType'], str(prodID))) return S_OK(prodsLaunched) ############################################################################# def _getStepsInProdDAG(self, prodDict, stepsListDict, stepsOrder='sequential'): """ Builds the DAG of steps in a production :params dict prodDict: dictionary representing one production :params list stepsListDict: list of steps (which are dictionaries) that should be in the production :returns: stepsInProd (DAG) """ stepsInProd = DAG() inserted = None for stepID in prodDict['stepsInProd-ProdName']: for step in stepsListDict: if step['prodStepID'] == stepID: ind = stepsListDict.index(step) step = stepsListDict.pop(ind) stepsInProd.addNode(step) if inserted and stepsOrder == 'sequential': stepsInProd.addEdge(inserted, step) inserted = step return stepsInProd def _mcSpecialCase(self, prod, prodDict): """ Treating the MC special case for putting MC productions in status "Testing" """ # save the original xml before it is edited for testing prod._lastParameters() # pylint: disable=protected-access # launchProduction adds extra parameters, as we 'hot swap' the xml, we # need to get these parameters for the un-edited version originalProcessingType = prod.prodGroup originalPriority = prod.priority prodXML = prod.LHCbJob.workflow.toXML() prodID = self._modifyAndLaunchMCXML(prod, prodDict) # load a production from the original xml to save the priority and processing type workflowToSave = fromXMLString(prodXML) prod.LHCbJob.workflow = workflowToSave prod.setParameter('ProcessingType', 'JDL', str(originalProcessingType), 'ProductionGroupOrType') prod.setParameter('Priority', 'JDL', str(originalPriority), 'Job Priority') # original xml to save descriptionToStore = prod.LHCbJob.workflow.toXML() # saving the original xml in the StoredJobDescription table. res = self.tc.addStoredJobDescription(prodID, descriptionToStore) if not res['OK']: self.logger.error("Error calling addStoredJobDescription", res['Message']) self.logger.info("Cleaning created production and exiting") self.diracProduction.production(res['Value'], 'cleaning') raise RuntimeError(res['Message']) return prodID def _modifyAndLaunchMCXML(self, prod, prodDict): """ Apply modifications to the workflow XML for MC testing case :param Production prod: Production object :param dict prodDict: dictionary with production info :returns: res['OK'] or res['ERROR'] """ # set the destination and number of events for testing destination = self.opsH.getValue( "Productions/MCTesting/MCTestingDestination", 'DIRAC.Test.ch') numberOfEvents = self.opsH.getValue( "Productions/MCTesting/numberOfEvents", '500') extendBy = self.opsH.getValue("Productions/MCTesting/extendBy", 20) prod.setJobParameters({'Destination': destination}) prod.LHCbJob.workflow.removeParameter('BannedSites') prod.setParameter('numberOfEvents', 'string', str(numberOfEvents), 'Number of events to test') # add '1' to the stepMask and add GAUSSHIST to the fileMask fileTypesOutLastStep = prod.LHCbJob.workflow.step_instances[ -2].findParameter('listoutput').getValue()[0]['outputDataType'] newFileMask = ['GAUSSHIST'] + [ ftOut.upper() for ftOut in fileTypesOutLastStep.split(';') ] stepMaskParameter = prod.LHCbJob.workflow.findParameter( 'outputDataStep') if stepMaskParameter: stepMask = stepMaskParameter.getValue().replace(' ', '').split(';') newOutputFileStep = ';'.join( sorted(list(set(['1']).union(set(stepMask))))) else: newOutputFileStep = '1' prod.setFileMask(newFileMask, newOutputFileStep) # find the file types out already built, append GAUSSHIST and set the new listoutput fileTypesOut = prod.LHCbJob.workflow.step_instances[0].findParameter( 'listoutput').getValue()[0]['outputDataType'] fileTypesOut = fileTypesOut.split(', ') fileTypesOut.append('GAUSSHIST') outputFilesList = prod._constructOutputFilesList(fileTypesOut) # pylint: disable=protected-access prod.LHCbJob.workflow.step_instances[0].setValue( 'listoutput', outputFilesList) # increase the priority to 10 prod.priority = 10 # launch the test production res = self.diracProduction.launchProduction( prod=prod, publishFlag=self.publishFlag, testFlag=self.testFlag, requestID=self.requestID, extend=extendBy, tracking=prodDict['tracking'], MCsimflag=True) if not res['OK']: self.logger.error("Error launching production", res['Message']) raise RuntimeError(res['Message']) return res['Value'] def _determineOutputSEs(self): """ Fill outputSEsPerFileType based on outputSEs, fullListOfOutputFileTypes and specialOutputSEs """ for outputSE, specialOutputSEs in itertools.izip( self.outputSEs, self.specialOutputSEs): outputSEDict = {} if not self.fullListOfOutputFileTypes: raise ValueError("No steps defined") outputSEDict = dict([(fType, outputSE) for fType in self.fullListOfOutputFileTypes]) if specialOutputSEs: outputSEDict.update(specialOutputSEs) self.outputSEsPerFileType.append(outputSEDict) def _applyOptionalCorrections(self): """ if needed, calls _splitIntoProductionSteps. It also applies other changes """ if len(self.bkQueries) != len(self.prodsTypeList): self.bkQueries += ['fromPreviousProd'] * (len(self.prodsTypeList) - len(self.bkQueries)) if len(self.previousProds) != len(self.prodsTypeList): self.previousProds += xrange(1, len(self.prodsTypeList)) if len(self.events) != len(self.prodsTypeList): self.events += ['-1' ] * (len(self.prodsTypeList) - len(self.events)) if not self.removeInputsFlags: removeInputsFlags = [] for prodType in self.prodsTypeList: if prodType.lower() == 'merge': removeInputsFlags.append(True) else: removeInputsFlags.append(False) self.removeInputsFlags = removeInputsFlags if not self.outputFileMasks: self.outputFileMasks = [''] * len(self.prodsTypeList) if not self.outputFileSteps: self.outputFileSteps = [''] * len(self.prodsTypeList) if not self.inputs: self.inputs = [[]] * len(self.prodsTypeList) if not self.outputModes: self.outputModes = ['Any'] * len(self.prodsTypeList) if not self.targets: self.targets = [''] * len(self.prodsTypeList) if not self.inputDataPolicies: self.inputDataPolicies = ['download'] * len(self.prodsTypeList) if not self.multicore: self.multicore = ['True'] * len(self.prodsTypeList) if not self.specialOutputSEs: self.specialOutputSEs = [{}] * len(self.prodsTypeList) if not self.ancestorDepths: self.ancestorDepths = [0] * len(self.prodsTypeList) # Checking if we need to split the merging step into many productions if 'merge' in [pt.lower() for pt in self.prodsTypeList]: i = 0 indexes = [] for pt in self.prodsTypeList: if pt.lower() == 'merge': indexes.append(i) i += 1 for index in indexes: # In this case and only in this case I have to split the merging in many productions plugin = self.plugins[index] outputSE = self.outputSEs[index] specialOutputSE = self.specialOutputSEs[index] priority = self.priorities[index] cpu = self.cpus[index] bkQuery = self.bkQueries[index] groupSize = self.groupSizes[index] preProd = self.previousProds[index] removeInputsFlag = self.removeInputsFlags[index] outputFileMask = self.outputFileMasks[index] outputFileStep = self.outputFileSteps[index] inputs = self.inputs[index] idp = self.inputDataPolicies[index] stepID = self.stepsList[index] events = self.events[index] targets = self.targets[index] multicore = self.multicore[index] outputMode = self.outputModes[index] ancestorDepth = self.ancestorDepths[index] if plugin.lower( ) != 'byrunfiletypesizewithflush' and 'rootmerging' not in plugin.lower( ): stepToSplit = self.stepsListDict[index] numberOfProdsToInsert = len(stepToSplit['fileTypesOut']) self.prodsTypeList.remove('Merge') self.plugins.pop(index) self.outputSEs.pop(index) self.specialOutputSEs.pop(index) self.priorities.pop(index) self.cpus.pop(index) self.bkQueries.pop(index) self.previousProds.pop(index) self.groupSizes.pop(index) self.removeInputsFlags.pop(index) self.outputFileMasks.pop(index) self.outputFileSteps.pop(index) self.inputs.pop(index) self.inputDataPolicies.pop(index) self.stepsList.pop(index) self.events.pop(index) self.targets.pop(index) self.multicore.pop(index) self.outputModes.pop(index) self.ancestorDepths.pop(index) newSteps = _splitIntoProductionSteps(stepToSplit) newSteps.reverse() self.stepsListDict.remove(stepToSplit) last = self.stepsInProds.pop(index)[0] for x in xrange(numberOfProdsToInsert): self.prodsTypeList.insert(index, 'Merge') self.plugins.insert(index, plugin) self.outputSEs.insert(index, outputSE) self.specialOutputSEs.insert(index, specialOutputSE) self.priorities.insert(index, priority) self.cpus.insert(index, cpu) self.bkQueries.insert(index, bkQuery) self.groupSizes.insert(index, groupSize) self.removeInputsFlags.insert(index, removeInputsFlag) self.outputFileMasks.insert(index, outputFileMask) self.outputFileSteps.insert(index, outputFileStep) self.inputs.insert(index, inputs) self.inputDataPolicies.insert(index, idp) self.stepsList.insert(index, stepID) self.previousProds.insert(index, preProd) self.stepsListDict.insert(index, newSteps[x]) self.stepsInProds.insert(index + x, [last + x]) self.events.insert(index, events) self.targets.insert(index, targets) self.multicore.insert(index, multicore) self.outputModes.insert(index, outputMode) self.ancestorDepths.insert(index, ancestorDepth) correctedStepsInProds = [] toInsert = self.stepsInProds[0][0] lengths = [len(x) for x in self.stepsInProds] for length in lengths: li = [toInsert + x for x in xrange(length)] toInsert += length correctedStepsInProds.append(li) self.stepsInProds = correctedStepsInProds ############################################################################# def _getProdsDescriptionDict(self): """ Returns a dictionary representing the description of the request (of all the productions in it) """ prodsDict = {} prodNumber = 1 for prodType, stepsInProd, bkQuery, removeInputsFlag, outputSE, priority, \ cpu, inputD, outputMode, outFileMask, outFileStep, target, groupSize, plugin, idp, \ previousProd, events, multicore, ancestorDepth in itertools.izip(self.prodsTypeList, self.stepsInProds, self.bkQueries, self.removeInputsFlags, self.outputSEsPerFileType, self.priorities, self.cpus, self.inputs, self.outputModes, self.outputFileMasks, self.outputFileSteps, self.targets, self.groupSizes, self.plugins, self.inputDataPolicies, self.previousProds, self.events, self.multicore, self.ancestorDepths): if not self.parentRequestID and self.requestID: transformationFamily = self.requestID else: transformationFamily = self.parentRequestID stepsInProdProdNameList = [ str(self.stepsList[index - 1]) + str(self.stepsListDict[index - 1]['fileTypesIn']) for index in stepsInProd ] prodsDict[prodNumber] = { 'productionType': prodType, 'stepsInProd': [self.stepsList[index - 1] for index in stepsInProd], 'bkQuery': bkQuery, 'removeInputsFlag': removeInputsFlag, 'tracking': 0, 'outputSE': outputSE, 'priority': priority, 'cpu': cpu, 'input': inputD, 'outputMode': outputMode, 'outputFileMask': outFileMask, 'outputFileStep': outFileStep, 'target': target, 'groupSize': groupSize, 'plugin': plugin, 'inputDataPolicy': idp, 'derivedProduction': 0, 'transformationFamily': transformationFamily, 'previousProd': previousProd, 'stepsInProd-ProdName': stepsInProdProdNameList, 'events': events, 'multicore': multicore, 'ancestorDepth': ancestorDepth } prodNumber += 1 # tracking the last production(s) prodsDict[prodNumber - 1]['tracking'] = 1 typeOfLastProd = prodsDict[prodNumber - 1]['productionType'] index = 2 try: while prodsDict[prodNumber - index]['productionType'] == typeOfLastProd: prodsDict[prodNumber - index]['tracking'] = 1 index += 1 except KeyError: pass # production derivation, if necessary if self.derivedProduction: prodsDict[1]['derivedProduction'] = self.derivedProduction return prodsDict ############################################################################# def _buildProduction(self, prodType, stepsInProd, outputSE, priority, cpu, inputDataList=None, outputMode='Any', inputDataPolicy='download', outputFileMask='', outputFileStep='', target='', removeInputData=False, groupSize=1, bkQuery=None, plugin='', previousProdID=0, derivedProdID=0, transformationFamily=0, events=-1, multicore='True', ancestorDepth=0): """ Wrapper around Production API to build a production, given the needed parameters Args: prodType (str): production type (e.g. 'DataStripping') stepsInProd (list): list of steps in the production outputSE (dict): dictionary that holds relation between file type and output SE priority (int): production priority cpu (int): CPU time, in HS06s for jobs in this production Returns: prod: a Production object """ prod = Production() # non optional parameters prod.LHCbJob.setType(prodType) try: fTypeIn = [ft.upper() for ft in stepsInProd[0]['fileTypesIn']] except IndexError: fTypeIn = [] prod.LHCbJob.workflow.setName( 'Request_%d_%s_%s_EventType_%s_%s_%s' % (self.requestID, prodType, self.prodGroup, self.eventType, ''.join( [x.split('.')[0] for x in fTypeIn]), self.appendName)) prod.setBKParameters(configName=self.outConfigName, configVersion=self.configVersion, groupDescriptionOrStepsList=stepsInProd, conditions=self.dataTakingConditions) prod.setParameter('eventType', 'string', self.eventType, 'Event Type of the production') prod.setParameter('numberOfEvents', 'string', str(events), 'Number of events requested') prod.setParameter('multicore', 'string', multicore, 'Flag for enabling gaudi parallel') prod.prodGroup = self.prodGroup prod.priority = priority prod.LHCbJob.workflow.setDescrShort('prodDescription') prod.LHCbJob.workflow.setDescription('prodDescription') prod.LHCbJob.setCPUTime(cpu) prod.plugin = plugin # optional parameters prod.jobFileGroupSize = groupSize if inputDataPolicy: prod.LHCbJob.setInputDataPolicy(inputDataPolicy) prod.setOutputMode(outputMode) if outputFileMask: outputFileMask = [ m.lower() for m in outputFileMask.replace(' ', '').split(',') ] if outputFileStep: if isinstance(outputFileStep, str): outputFileStep = [ m.lower() for m in outputFileStep.replace(' ', '').split(',') ] prod.setFileMask(outputFileMask, outputFileStep) if target: if target == 'Tier2': prod.banTier1s() elif 'BAN' in target: sitesToBan = target.split(':') if len(sitesToBan) > 1: prod.banSites(sitesToBan[1:]) elif target != 'ALL': prod.LHCbJob.setDestination(target) prod.LHCbJob.setInputData(inputDataList) if ancestorDepth: prod.LHCbJob.setAncestorDepth(ancestorDepth) if derivedProdID: prod.ancestorProduction = derivedProdID if transformationFamily: prod.transformationFamily = transformationFamily if self.fractionToProcess: prod.setParameter('FractionToProcess', 'string', str(self.fractionToProcess), 'Fraction to process') if self.minFilesToProcess: prod.setParameter('MinFilesToProcess', 'string', str(self.minFilesToProcess), 'Min N of Files to process') if self.processingPass: prod.setParameter('processingPass', 'string', self.processingPass, 'Processing pass of input for the request') # Adding optional input BK query if bkQuery: if bkQuery.lower() == 'full': prod.inputBKSelection = self._getBKKQuery() elif bkQuery.lower() == 'frompreviousprod': prod.inputBKSelection = self._getBKKQuery( 'frompreviousprod', fTypeIn, previousProdID) self.logger.verbose('Launching with BK selection %s' % prod.inputBKSelection) prod = self._addStepsToProd(prod, stepsInProd, removeInputData=removeInputData) for ft, oSE in outputSE.items(): prod.outputSEs.setdefault(ft, oSE) prod.LHCbJob.setDIRACPlatform() return prod ############################################################################# def _addStepsToProd(self, prod, stepsInProd, stepsSequence='sequential', removeInputData=False): """ Given a Production object, add requested steps (application and finalization) Args: prod (Production): the Production object to which the steps are added stepsInProd (DAG): DAG of steps in a production stepsSequence (str or dict): applications steps sequence removeInputData (bool): flag that indicates if the input data should be removed (for the finalization step) Returns: prod with steps added """ # Adding the application steps firstStep = stepsInProd.pop(0) stepName = prod.addApplicationStep(stepDict=firstStep, modulesList=self.modulesList) prod.gaudiSteps.append(stepName) for step in stepsInProd: stepName = prod.addApplicationStep(stepDict=step, inputData='previousStep', modulesList=self.modulesList) prod.gaudiSteps.append(stepName) # Adding the finalization step if removeInputData: prod.addFinalizationStep([ 'UploadOutputData', 'RemoveInputData', 'UploadLogFile', 'UploadMC', 'FailoverRequest' ]) else: prod.addFinalizationStep() return prod def _getBKKQuery(self, mode='full', fileType=None, previousProdID=0): """ simply creates the bkk query dictionary """ if fileType is None: fileType = [] if mode.lower() == 'full': bkQuery = { 'FileType': ';;;'.join(self.bkFileType), 'EventType': str(self.eventType), 'ConfigName': self.configName, 'ConfigVersion': self.configVersion } if self.dataTakingConditions: bkQuery['DataTakingConditions'] = self.dataTakingConditions if self.processingPass: bkQuery['ProcessingPass'] = self.processingPass if self.dqFlag: bkQuery['DataQualityFlag'] = self.dqFlag.replace( ',', ';;;').replace(' ', '') if self.startRun and self.runsList or self.endRun and self.runsList: raise ValueError( "Please don't mix runs list with start/end run") if self.endRun and self.startRun: if self.endRun < self.startRun: gLogger.error( "Your end run '%d' should be more than your start run '%d'!" % (self.endRun, self.startRun)) raise ValueError("Error setting start or end run") if self.startRun: bkQuery['StartRun'] = self.startRun if self.endRun: bkQuery['EndRun'] = self.endRun if self.runsList: bkQuery['RunNumbers'] = self.runsList.replace(',', ';;;').replace( ' ', '') if self.visibility: bkQuery['Visible'] = self.visibility elif mode.lower() == 'frompreviousprod': bkQuery = { 'FileType': ';;;'.join(fileType).replace(' ', ''), 'ProductionID': int(previousProdID) } if self.eventType: bkQuery['EventType'] = str(self.eventType) if self.dqFlag: bkQuery['DataQualityFlag'] = self.dqFlag.replace( ',', ';;;').replace(' ', '') return bkQuery ################################################################################ # properties def set_stepsList(self, value): listInt = [] i = 0 while True: try: listInt.append(int(value[i])) i = i + 1 except ValueError: break except IndexError: break self._stepsList = listInt def get_stepsList(self): return self._stepsList stepsList = property(get_stepsList, set_stepsList) def set_startRun(self, value): if isinstance(value, str): value = int(value) if value < 0: raise ValueError("startRun can not be negative") self._startRun = value def get_startRun(self): return self._startRun startRun = property(get_startRun, set_startRun) def set_endRun(self, value): if isinstance(value, str): value = int(value) if value < 0: raise ValueError("endRun can not be negative") self._endRun = value def get_endRun(self): return self._endRun endRun = property(get_endRun, set_endRun) def set_requestID(self, value): if value == '': value = 0 if isinstance(value, str): value = int(value) if value < 0: raise ValueError("requestID can not be negative") self._requestID = value def get_requestID(self): return self._requestID requestID = property(get_requestID, set_requestID) def set_parentRequestID(self, value): if value == '': value = 0 if isinstance(value, str): value = int(value) if value < 0: raise ValueError("parentRequestID can not be negative") self._parentRequestID = value def get_parentRequestID(self): return self._parentRequestID parentRequestID = property(get_parentRequestID, set_parentRequestID) def set_bkFileType(self, value): if isinstance(value, str): value = value.replace(' ', '').split(',') self._bkFileType = value def get_bkFileType(self): return self._bkFileType bkFileType = property(get_bkFileType, set_bkFileType)
""" This is a very simple bkk performance test. It calls the service with a message. The service return the message. """ import time from DIRAC.Core.Base.Script import parseCommandLine parseCommandLine() from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient cl = BookkeepingClient() class Transaction(object): def __init__(self): self.custom_timers = {} def run(self): start_time = time.time() retVal = cl.echo("simple test") if not retVal['OK']: print 'ERROR', retVal['Message'] end_time = time.time() self.custom_timers['Bkk_ResponseTime'] = end_time - start_time self.custom_timers['Bkk_Echo'] = end_time - start_time if __name__ == '__main__': trans = Transaction()
def __init__(self, bkkClientIn=None, diracProdIn=None): """ c'tor Some variables are defined here. A production request is made of: stepsList, productionsTypes, and various parameters of those productions """ if bkkClientIn is None: self.bkkClient = BookkeepingClient() else: self.bkkClient = bkkClientIn if diracProdIn is None: self.diracProduction = DiracProduction() else: self.diracProduction = diracProdIn self.rpcProductionRequest = RPCClient( 'ProductionManagement/ProductionRequest') self.tc = TransformationClient() self.logger = gLogger.getSubLogger('ProductionRequest') self.opsH = Operations() # parameters of the request self.requestID = 0 self.parentRequestID = 0 self.appendName = '1' self.outConfigName = '' self.prodsToLaunch = [] # productions to launch self.stepsListDict = [] # list of dict of steps self.stepsInProds = [] # a list of lists # parameters of the input data self.processingPass = '' self.dataTakingConditions = '' self.eventType = '' self.bkFileType = [] self.dqFlag = '' self.startRun = 1 self.endRun = 2 self.runsList = '' self.configName = 'test' self.configVersion = 'certification' # parameters of the first production self.publishFlag = True self.testFlag = False self.derivedProduction = 0 self.previousProdID = 0 # optional prod from which to start self.fullListOfOutputFileTypes = [] # parameters that are the same for each productions self.prodGroup = '' # This ends up being 'ProcessingType', workflow parameter, and it is used for accounting self.visibility = '' # For BKQuery self.fractionToProcess = 0 self.minFilesToProcess = 0 self.modulesList = None # Usually: # ['GaudiApplication', 'AnalyseXMLSummary', # 'ErrorLogging', 'BookkeepingReport', 'StepAccounting' ] # parameters of each production (the length of each list has to be the same as the number of productions self.events = [] self.stepsList = [] self.extraOptions = {} self.prodsTypeList = [] self.bkQueries = [] # list of bk queries self.removeInputsFlags = [] self.priorities = [] self.cpus = [] self.inputs = [] # list of lists self.outputModes = [] self.targets = [] self.outputFileMasks = [] self.outputFileSteps = [] self.groupSizes = [] self.plugins = [] self.inputDataPolicies = [] self.previousProds = [ None ] # list of productions from which to take the inputs (the first is always None) self.multicore = [ ] # list of flags to override the multi core flags of the steps self.outputSEs = [] # a list of StorageElements self.specialOutputSEs = [] # a list of dictionaries - might be empty self.outputSEsPerFileType = [] # a list of dictionaries - filled later self.ancestorDepths = [] self.compDict = { 'HIGH': 'Compression-LZMA-4', 'LOW': 'Compression-ZLIB-1' } self.compressionLvl = [''] # List: one compression level each step self.appConfig = '$APPCONFIGOPTS/Persistency/' # Default location of the compression level configuration files # # These lists define the visibility of the output files produced by each step. For MC productions, the visibility # is tied to compression level. VIsible files are compressed at the highest level # self.outputVisFlag = [ ] # List of dictionary with default visibility flag of the output files per single step self.specialOutputVisFlag = [ ] # List of dictionaries with special visibility flag for given file type
class StorageHistoryAgent( AgentModule ): def initialize( self ): '''Sets defaults ''' self.am_setOption( 'PollingTime', 43200 ) if self.am_getOption( 'DirectDB', False ): from LHCbDIRAC.DataManagementSystem.DB.StorageUsageDB import StorageUsageDB self.__stDB = StorageUsageDB() else: from DIRAC.Core.DISET.RPCClient import RPCClient self.__stDB = RPCClient( 'DataManagement/StorageUsage' ) self.__workDirectory = self.am_getOption( "WorkDirectory" ) mkDir( self.__workDirectory ) self.log.info( "Working directory is %s" % self.__workDirectory ) self.__ignoreDirsList = self.am_getOption( 'Ignore', [] ) self.log.info( "List of directories to ignore for the DataStorage accounting: %s " % self.__ignoreDirsList ) self.__bkClient = BookkeepingClient() self.__dataUsageClient = DataUsageClient() self.cachedMetadata = {} # build a dictionary with Event Type descriptions (to be send to accounting, instead of number Event Type ID) self.eventTypeDescription = { 'na':'na', 'notInBkk':'notInBkk', 'FailedBkkQuery':'FailedBkkQuery', 'None':'None'} self.limitForCommit = self.am_getOption( "LimitForCommit", 1000 ) self.callsToGetSummary = 0 self.callsToDirectorySummary = 0 self.callsToDudbForMetadata = 0 # keep count of calls to Bkk self.callsToBkkgetDirectoryMetadata = 0 self.callsToBkkGetEvtType = 0 return S_OK() def userStorageAccounting( self ): self.log.notice( "-------------------------------------------------------------------------------------\n" ) self.log.notice( "Generate accounting records for user directories " ) self.log.notice( "-------------------------------------------------------------------------------------\n" ) result = self.__stDB.getUserSummary() if not result[ 'OK' ]: return result userCatalogData = result[ 'Value' ] print userCatalogData self.log.notice( "Got summary for %s users" % ( len( userCatalogData ) ) ) result = self.__stDB.getUserSummaryPerSE() if not result[ 'OK' ]: return result userSEData = result[ 'Value' ] self.log.notice( "Got SE summary for %s users" % ( len( userSEData ) ) ) now = Time.dateTime() numRows = 0 for user in sorted( userSEData ): if user not in userCatalogData: self.log.error( "User has SE data but not Catalog data!", user ) continue for se in sorted( userSEData[ user ] ): seData = userSEData[ user ][ se ] usRecord = UserStorage() usRecord.setStartTime( now ) usRecord.setEndTime( now ) usRecord.setValueByKey( "User", user ) usRecord.setValueByKey( "StorageElement", se ) usRecord.setValueByKey( "LogicalSize", userCatalogData[ user ][ 'Size' ] ) usRecord.setValueByKey( "LogicalFiles", userCatalogData[ user ][ 'Files' ] ) usRecord.setValueByKey( "PhysicalSize", seData[ 'Size' ] ) usRecord.setValueByKey( "PhysicalFiles", seData[ 'Files' ] ) usRecord.setValueByKey( "StorageSize", 0 ) usRecord.setValueByKey( "StorageFiles", 0 ) gDataStoreClient.addRegister( usRecord ) numRows += 1 self.log.notice( " User %s is using %.2f GiB (%s files)" % ( user, userCatalogData[ user ][ 'Size' ] / ( 1024.0 ** 3 ), userCatalogData[ user ][ 'Files' ] ) ) self.log.notice( "Sending %s records to accounting for user storage" % numRows ) res = gDataStoreClient.commit() if not res[ 'OK' ]: self.log.notice( "ERROR: committing UserStorage records: %s " % res ) return S_ERROR( res ) else: self.log.notice( "%s records for UserStorage type successfully committed" % numRows ) def topDirectoryAccounting( self ): self.log.notice( "-------------------------------------------------------------------------------------\n" ) self.log.notice( "Generate accounting records for top directories " ) self.log.notice( "-------------------------------------------------------------------------------------\n" ) ftb = 1.0e12 # get info from the DB about the LOGICAL STORAGE USAGE (from the su_Directory table): result = self.__stDB.getSummary( '/lhcb/' ) if not result[ 'OK' ]: return result logicalUsage = result['Value'] topDirLogicalUsage = {} # build the list of first level directories for row in logicalUsage: # d, size, files = row splitDir = row.split( "/" ) if len( splitDir ) > 3: # skip the root directory "/lhcb/" firstLevelDir = '/' + splitDir[1] + '/' + splitDir[2] + '/' topDirLogicalUsage.setdefault( firstLevelDir, {'Files':0, 'Size':0} ) topDirLogicalUsage[ firstLevelDir ][ 'Files' ] += logicalUsage[ row ][ 'Files' ] topDirLogicalUsage[ firstLevelDir ][ 'Size' ] += logicalUsage[ row ][ 'Size' ] self.log.notice( "Summary on logical usage of top directories: " ) for row in topDirLogicalUsage: self.log.notice( "dir: %s size: %.4f TB files: %d" % ( row, topDirLogicalUsage[row]['Size'] / ftb, topDirLogicalUsage[row]['Files'] ) ) # loop on top level directories (/lhcb/data, /lhcb/user/, /lhcb/MC/, etc..) # to get the summary in terms of PHYSICAL usage grouped by SE: seData = {} for directory in topDirLogicalUsage: result = self.__stDB.getDirectorySummaryPerSE( directory ) # retrieve the PHYSICAL usage if not result[ 'OK' ]: return result seData[ directory ] = result[ 'Value' ] self.log.notice( "Got SE summary for %s directories " % ( len( seData ) ) ) self.log.debug( "SEData: %s" % seData ) # loop on top level directories to send the accounting records numRows = 0 now = Time.dateTime() for directory in seData: self.log.debug( "dir: %s SEData: %s " % ( directory, seData[ directory ] ) ) if directory not in topDirLogicalUsage: self.log.error( "Dir %s is in the summary per SE, but it is not in the logical files summary!" % directory ) continue for se in sorted( seData[ directory ] ): storageRecord = Storage() storageRecord.setStartTime( now ) storageRecord.setEndTime( now ) storageRecord.setValueByKey( "Directory", directory ) storageRecord.setValueByKey( "StorageElement", se ) storageRecord.setValueByKey( "LogicalFiles", topDirLogicalUsage[ directory ][ 'Files' ] ) storageRecord.setValueByKey( "LogicalSize", topDirLogicalUsage[ directory ][ 'Size' ] ) try: physicalFiles = seData[ directory ][ se ][ 'Files' ] except: self.log.error( "WARNING! no files replicas for directory %s on SE %s" % ( directory, se ) ) physicalFiles = 0 try: physicalSize = seData[ directory ][ se ][ 'Size' ] except: self.log.error( "WARNING! no size for replicas for directory %s on SE %s" % ( directory, se ) ) physicalSize = 0 storageRecord.setValueByKey( "PhysicalFiles", physicalFiles ) storageRecord.setValueByKey( "PhysicalSize", physicalSize ) gDataStoreClient.addRegister( storageRecord ) numRows += 1 self.log.debug( "Directory: %s SE: %s physical size: %.4f TB (%d files)" % ( directory, se, physicalSize / ftb, physicalFiles ) ) self.log.notice( "Sending %s records to accounting for top level directories storage" % numRows ) res = gDataStoreClient.commit() if not res[ 'OK' ]: self.log.notice( "ERROR: committing Storage records: %s " % res ) return S_ERROR( res ) else: self.log.notice( "%s records for Storage type successfully committed" % numRows ) def bkPathAccounting( self ): self.log.notice( "-------------------------------------------------------------------------------------\n" ) self.log.notice( "Generate accounting records for DataStorage type " ) self.log.notice( "-------------------------------------------------------------------------------------\n" ) # counter for DataStorage records, commit to the accounting in bunches of self.limitForCommit records self.totalRecords = 0 self.recordsToCommit = 0 self.log.notice( " Call the function to create the StorageUsageDB dump.." ) res = self.generateStorageUsagePerDir() if not res[ 'OK' ]: self.log.error( "ERROR generating the StorageUsageDB dump per directory" ) return S_ERROR() # Keep a list of all directories in FC that are not found in the Bkk self.directoriesNotInBkk = [] # for debugging purposes build dictionaries with storage usage to compare with the accounting plots self.debug_seUsage = {} self.debug_seUsage_acc = {} # set the time for the accounting records (same time for all records) now = Time.dateTime() # Get the directory metadata in a bulk query metaForList = self.__getMetadataForAcc( self.dirDict.values() ) # loop on all directories to get the bkk metadata for dirLfn, fullDirectory in self.dirDict.iteritems(): if dirLfn not in fullDirectory: self.log.error( "ERROR: fullDirectory should include the dirname: %s %s " % ( fullDirectory, dirLfn ) ) continue self.log.info( "Processing directory %s " % dirLfn ) if dirLfn not in self.pfnUsage: self.log.error( "ERROR: directory does not have PFN usage %s " % dirLfn ) continue self.log.verbose( "PFN usage: %s " % self.pfnUsage[ dirLfn ] ) if dirLfn not in self.lfnUsage: self.log.error( "ERROR: directory does not have LFN usage %s " % dirLfn ) continue self.log.verbose( "LFN usage: %s " % self.lfnUsage[ dirLfn ] ) # for DEBUGGING: for se in self.pfnUsage[ dirLfn ]: self.debug_seUsage.setdefault( se, {'Files': 0 , 'Size' : 0 } ) self.debug_seUsage[ se ][ 'Files' ] += self.pfnUsage[ dirLfn ][ se ][ 'Files' ] self.debug_seUsage[ se ][ 'Size' ] += self.pfnUsage[ dirLfn ][ se ][ 'Size' ] # end of DEBUGGING # get metadata for this directory metaForDir = metaForList.get( fullDirectory, {} ) if not metaForDir: self.log.warn( "Metadata not available for directory %s" % fullDirectory ) continue # Fill in the accounting record self.log.info( "Fill the record for %s and metadata: %s " % ( dirLfn, metaForDir ) ) res = self.fillAndSendAccountingRecord( dirLfn, metaForDir, now ) if not res['OK']: return res for se in self.pfnUsage[ dirLfn ]: self.debug_seUsage_acc.setdefault( se, { 'Files': 0 , 'Size': 0 } ) self.debug_seUsage_acc[ se ][ 'Files' ] += self.pfnUsage[ dirLfn ][ se ][ 'Files' ] self.debug_seUsage_acc[ se ][ 'Size' ] += self.pfnUsage[ dirLfn ][ se ][ 'Size' ] # Don't forget to commit the remaining records! self.__commitRecords() def execute( self ): if self.am_getOption( "CleanBefore", False ): self.log.notice( "Cleaning the DB" ) result = self.__stDB.purgeOutdatedEntries( "/lhcb/user", self.am_getOption( "OutdatedSeconds", 86400 * 10 ) ) if not result[ 'OK' ]: return result self.log.notice( "Purged %s outdated records" % result[ 'Value' ] ) # User accounting (per user and SE) self.userStorageAccounting() # Accounting per top directory self.topDirectoryAccounting() # full production data accounting self.bkPathAccounting() self.log.notice( "-------------------------------------------------------------------------------------\n" ) self.log.notice( "------ End of cycle report for DataStorage accounting--------------------------------\n" ) self.log.notice( "Total directories found in FC: %d " % len( self.dirDict ) ) totalCallsToStorageUsage = self.callsToGetSummary + self.callsToDirectorySummary self.log.notice( "Total calls to StorageUsage: %d , took: %d s " % ( totalCallsToStorageUsage, self.genTotalTime ) ) totalCallsToBkk = self.callsToBkkgetDirectoryMetadata + self.callsToBkkGetEvtType self.log.notice( "Total calls to DataUsage for cache: %d" % self.callsToDudbForMetadata ) self.log.notice( "Total calls to Bookkeeping: %d (getDirectoryMetadata: %d, getEventType: %d)" % ( totalCallsToBkk, self.callsToBkkgetDirectoryMetadata, self.callsToBkkGetEvtType ) ) self.log.notice( "Total records sent to accounting for DataStorage: %d " % self.totalRecords ) self.log.notice( "Directories not found in Bookkeeping: %d " % ( len( self.directoriesNotInBkk ) ) ) fileName = os.path.join( self.__workDirectory, "directoriesNotInBkk.txt" ) self.log.notice( "written to file: %s " % fileName ) f = open( fileName, "w" ) for d in self.directoriesNotInBkk: f.write( "%s\n" % d ) f.close() # for DEBUG only self.log.info( "Summary of StorageUsage: files size " ) for se in sorted( self.debug_seUsage ): self.log.info( "all: %s %d %d Bytes ( %.2f TB ) " % ( se, self.debug_seUsage[ se ]['Files'], self.debug_seUsage[ se ]['Size'], self.debug_seUsage[ se ]['Size'] / 1.0e12 ) ) if se in self.debug_seUsage_acc: self.log.info( "acc: %s %d %d Bytes ( %.2f TB ) " % ( se, self.debug_seUsage_acc[ se ]['Files'], self.debug_seUsage_acc[ se ]['Size'], self.debug_seUsage_acc[ se ]['Size'] / 1.0e12 ) ) else: self.log.info( "SE not in self.debug_seUsage_acc keys" ) return S_OK() def __getMetadataForAcc( self, dirList ): """ Get metadata for a directory either from memory, from the storageDB or from BK """ # Try and get the metadata from memory cache notFound = [] metaForList = {} for dirName in dirList: metaForList[dirName] = self.cachedMetadata.get( dirName, {} ) if not metaForList[dirName]: notFound.append( dirName ) notInCache = [] if notFound: self.log.info( "Memory metadata cache missed for %d directories" % len( notFound ) ) self.log.verbose( "call getDirMetadata for (first 10): %s " % str( notFound[0:10] ) ) for dirChunk in breakListIntoChunks( notFound, 10000 ): self.callsToDudbForMetadata += 1 res = self.__dataUsageClient.getDirMetadata( dirChunk ) # this could be a bulk query for a list of directories if not res[ 'OK' ]: self.log.error( "Error retrieving %d directories meta-data %s " % ( len( dirChunk ), res['Message'] ) ) # this usually happens when directories are removed from FC between the StorageUsageDB dump and this call, # if the Db is refreshed exactly in this time interval. Not really a problem. #######################3 just a try ##############################################3 notInCache += dirChunk continue self.log.verbose( "getDirMetadata returned: %s " % str( res['Value'] ) ) for dirName in dirChunk: # Compatibility with old (list for single file) and new (dictionary) service if type( res['Value'] ) == type( {} ): metaTuple = res['Value'].get( dirName, () ) elif len( dirList ) == 1 and res['Value']: metaTuple = res['Value'][0] else: metaTuple = () if metaTuple and metaTuple[3] is not None: metaForDir = metaForList[dirName] _dirID, metaForDir[ 'DataType' ], metaForDir[ 'Activity' ], metaForDir[ 'Conditions' ], metaForDir[ 'ProcessingPass' ], \ metaForDir[ 'EventType' ], metaForDir[ 'FileType' ], metaForDir[ 'Production' ], metaForDir['Visibility'] = metaTuple else: notInCache.append( dirName ) failedBK = [] if notInCache: cachedFromBK = [] self.log.info( "Directory metadata cache missed for %d directories => query BK and cache" % len( notInCache ) ) for dirChunk in breakListIntoChunks( notInCache, 200 ): self.callsToBkkgetDirectoryMetadata += 1 res = self.__bkClient.getDirectoryMetadata( dirChunk ) if not res[ 'OK' ]: self.log.error( "Totally failed to query Bookkeeping", res[ 'Message' ] ) failedBK += dirChunk for dirName in dirChunk: metaForDir = metaForList[dirName] _fillMetadata( metaForDir, 'FailedBkkQuery' ) else: bkMetadata = res['Value'] self.log.debug( "Successfully queried Bookkeeping, result: %s " % bkMetadata ) for dirName in dirChunk: metaForDir = metaForList[dirName] # BK returns a list of metadata, chose the first one... metadata = bkMetadata['Successful'].get( dirName, [{}] )[0] if metadata and metadata.get( 'ConditionDescription' ) is not None: metadata['Visibility'] = metadata.pop( 'VisibilityFlag', metadata.get( 'Visibility', 'na' ) ) # All is OK, directory found _fillMetadata( metaForDir, metadata ) self.log.verbose( "Cache entry %s in DirMetadata table.." % dirName ) resInsert = self.__dataUsageClient.insertToDirMetadata( { dirName: metadata} ) if not resInsert[ 'OK' ]: self.log.error( "Failed to cache metadata in DirMetadata table! %s " % resInsert[ 'Message' ] ) else: cachedFromBK.append( dirName ) self.log.verbose( "Successfully cached metadata for %s : %s" % ( dirName, str( metadata ) ) ) self.log.debug( "result: %s " % str( resInsert ) ) else: # Directory not found self.log.verbose( "Directory %s not registered in Bookkeeping!" % dirName ) _fillMetadata( metaForDir, 'notInBkk' ) failedBK.append( dirName ) self.directoriesNotInBkk.append( dirName ) # Translate a few keys for accounting for bkName, accName in ( ( 'ConfigName', 'DataType' ), ( 'ConfigVersion', 'Activity' ), ( 'ConditionDescription', 'Conditions' ) ): metaForDir[accName] = metaForDir.pop( bkName, 'na' ) self.log.info( 'Successfully cached %d directories from BK' % len( cachedFromBK ) ) if self.directoriesNotInBkk: self.log.warn( '%d directories not found in BK' % len( self.directoriesNotInBkk ) ) # cache locally the metadata for dirName in [dn for dn in notFound if dn not in failedBK]: metaForDir = metaForList[dirName] # Translate the numerical event type to a description string metaForDir['EventType'] = self.__getEventTypeDescription( metaForDir.pop( 'EventType', 'na' ) ) self.cachedMetadata[ dirName] = metaForDir.copy() else: self.log.info( "Memory metadata cache hit for %d directories" % len( dirList ) ) return metaForList def __commitRecords( self ): if self.recordsToCommit: res = gDataStoreClient.commit() if not res[ 'OK' ]: self.log.error( "Accounting ERROR: commit returned %s" % res ) else: self.log.notice( "%d records committed " % self.recordsToCommit ) self.recordsToCommit = 0 self.log.notice( "commit for DataStorage returned: %s" % res ) def generateStorageUsagePerDir( self ): '''Generate a dump of the StorageUsageDB and keep it in memory in a dictionary (new version of Apr 2012) ''' start = time.time() self.log.notice( 'Starting from path: /lhcb/' ) res = self.__stDB.getStorageDirectories( '/lhcb/' ) if not res['OK']: return S_ERROR() totalDirList = res['Value'] self.log.info( "Total directories retrieved from StorageUsageDB: %d " % len( totalDirList ) ) # select only terminal directories (directories without sub-directories) # mc directory structure: /lhcb/MC/[year]/[file type]/[prod]/0000/ => len = 7 # raw data: /lhcb/data/2011/RAW/FULL/LHCb/COLLISION11/99983 # => len 9 (under /lhcb/data/ from 2011 only RAW, before 2011 also other file types) # processed data: under both /lhcb/data and /lhcb/LHCb/ # /lhcb/data/2010/DST/00009300/0000 # data: /lhcb/LHCb/Collision12/ICHEP.DST/00017399/0000/ self.dirDict = {} ignoredDirectories = dict.fromkeys( self.__ignoreDirsList, 0 ) self.log.info( "Directories to be ignored: %s " % str( sorted( ignoredDirectories ) ) ) for dirItem in totalDirList: # make sure that last character is a '/' dirItem = _standardDirectory( dirItem ) splitDir = dirItem.split( '/' ) if len( splitDir ) < 4: # avoid picking up intermediate directories which don't contain files, like /lhcb/ self.log.warn( "Directory %s skipped, as top directory" % dirItem ) continue secDir = splitDir[ 2 ] if secDir in ignoredDirectories: self.log.verbose( "Directory to be ignored, skipped: %s " % dirItem ) ignoredDirectories[ secDir ] += 1 continue # for each type of directory (MC, reconstructed data and runs) check the format, in order not to count more than # once the productions with more than one sub-directory # for MC directories: # example: '/lhcb/MC/MC10/ALLSTREAMS.DST/00010908/0000/', # or /lhcb/MC/2011/DST/00010870/0000 # one directory for each file type # for data # /lhcb/LHCb/Collision11/SWIMSTRIPPINGD02KSPIPI.MDST/00019088/0000/ # for raw data: /lhcb/data/2012/RAW/FULL/LHCb/COLLISION12/133784/ try: dataType = splitDir[ -6 ] if dataType == "RAW": self.log.verbose( "RAW DATA directory: %s" % splitDir ) directory = '/'.join( splitDir[:-1] ) fullDirectory = directory else: suffix = splitDir[ -2 ] # is the sub-directory suffix 0000, 0001, etc... self.log.verbose( "MC or reconstructed data directory: %s" % splitDir ) if splitDir[-3] == 'HIST': directory = '/'.join( splitDir[:-1] ) fullDirectory = directory self.log.verbose( "histo dir: %s " % directory ) else: directory = '/'.join( splitDir[:-2] ) fullDirectory = os.path.join( directory, suffix ) directory = _standardDirectory( directory ) fullDirectory = _standardDirectory( fullDirectory ) if directory not in self.dirDict: self.dirDict[ directory ] = fullDirectory self.log.verbose( "Directory contains production files: %s " % directory ) except: self.log.warn( "The directory has unexpected format: %s " % splitDir ) self.lfnUsage = {} self.pfnUsage = {} totalDiscardedDirs = 0 self.log.info( "Directories that have been discarded:" ) for dd in ignoredDirectories: self.log.info( "/lhcb/%s - %d " % ( dd, ignoredDirectories[ dd ] ) ) totalDiscardedDirs += ignoredDirectories[ dd ] self.log.info( "Total discarded directories: %d " % totalDiscardedDirs ) self.log.info( "Retrieved %d dirs from StorageUsageDB containing prod files" % len( self.dirDict ) ) self.log.info( "Getting the number of files and size from StorageUsage service" ) for d in self.dirDict: self.log.verbose( "Get storage usage for directory %s " % d ) res = self.__stDB.getDirectorySummaryPerSE( d ) self.callsToDirectorySummary += 1 if not res[ 'OK' ]: self.log.error( "Cannot retrieve PFN usage %s" % res['Message'] ) continue if d not in self.pfnUsage: self.pfnUsage[ d ] = res['Value'] self.log.verbose( "Get logical usage for directory %s " % d ) res = self.__stDB.getSummary( d ) self.callsToGetSummary += 1 if not res[ 'OK' ]: self.log.error( "Cannot retrieve LFN usage %s" % res['Message'] ) continue if not res['Value']: self.log.error( "For dir %s getSummary returned an empty value: %s " % ( d, str( res ) ) ) continue self.lfnUsage.setdefault( d, {} ) for retDir, dirInfo in res['Value'].iteritems(): if d in retDir: self.lfnUsage[ d ][ 'LfnSize' ] = dirInfo['Size'] self.lfnUsage[ d ][ 'LfnFiles'] = dirInfo['Files'] self.log.verbose( "PFN usage: %s" % self.pfnUsage[ d ] ) self.log.verbose( "LFN usage: %s" % self.lfnUsage[ d ] ) end = time.time() self.genTotalTime = end - start self.log.info( "StorageUsageDB dump completed in %d s" % self.genTotalTime ) return S_OK() def __getEventTypeDescription( self, eventType ): # convert eventType to string: try: eventType = int( eventType ) except: pass # check that the event type description is in the cached dictionary, and otherwise query the Bkk if eventType not in self.eventTypeDescription: self.log.notice( "Event type description not available for eventTypeID %s, getting from Bkk" % eventType ) res = self.__bkClient.getAvailableEventTypes() self.callsToBkkGetEvtType += 1 if not res['OK']: self.log.error( "Error querying the Bkk: %s" % res['Message'] ) else: self.eventTypeDescription.update( dict( res['Value'] ) ) self.log.verbose( "Updated self.eventTypeDescription dict: %s " % str( self.eventTypeDescription ) ) # If still not found, log it! if eventType not in self.eventTypeDescription: self.log.error( "EventType %s is not in cached dictionary" % str( eventType ) ) return self.eventTypeDescription.get( eventType, 'na' ) def fillAndSendAccountingRecord( self, lfnDir, metadataDict, now ): ''' Create, fill and send to accounting a record for the DataStorage type. ''' dataRecord = DataStorage() dataRecord.setStartTime( now ) dataRecord.setEndTime( now ) logicalSize = self.lfnUsage[ lfnDir ][ 'LfnSize' ] logicalFiles = self.lfnUsage[ lfnDir ][ 'LfnFiles' ] dataRecord.setValueByKey( "LogicalSize", logicalSize ) dataRecord.setValueByKey( "LogicalFiles", logicalFiles ) for key in ( 'DataType', 'Activity', 'FileType', 'Production', 'ProcessingPass', 'Conditions', 'EventType' ): dataRecord.setValueByKey( key, metadataDict.get( key, 'na' ) ) self.log.verbose( ">>> Send DataStorage record to accounting:" ) self.log.verbose( "\tlfnFiles: %d lfnSize: %d " % ( logicalFiles, logicalSize ) ) for se in self.pfnUsage[ lfnDir ]: self.log.verbose( "Filling accounting record for se %s" % se ) physicalSize = self.pfnUsage[ lfnDir ][ se ][ 'Size' ] physicalFiles = self.pfnUsage[ lfnDir ][ se ][ 'Files' ] dataRecord.setValueByKey( "StorageElement", se ) dataRecord.setValueByKey( "PhysicalSize", physicalSize ) dataRecord.setValueByKey( "PhysicalFiles", physicalFiles ) self.log.verbose( "\t\tStorageElement: %s --> physFiles: %d physSize: %d " % ( se, physicalFiles, physicalSize ) ) # addRegister is NOT making a copy, therefore all records are otherwise overwritten res = gDataStoreClient.addRegister( copy.deepcopy( dataRecord ) ) if not res[ 'OK']: self.log.error( "addRegister returned: %s" % res ) return S_ERROR( "addRegister returned: %s" % res ) # Reset logical information to zero in order to send it only once! dataRecord.setValueByKey( "LogicalSize", 0 ) dataRecord.setValueByKey( "LogicalFiles", 0 ) self.totalRecords += 1 self.recordsToCommit += 1 # Commit if necessary if self.recordsToCommit > self.limitForCommit: self.__commitRecords() return S_OK()
class TransformationCleaningAgent(DiracTCAgent): """ .. class:: TransformationCleaningAgent """ def __init__(self, *args, **kwargs): """ c'tor """ DiracTCAgent.__init__(self, *args, **kwargs) self.directoryLocations = ['TransformationDB', 'StorageUsage'] self.archiveAfter = 7 self.fileTypesToKeep = ['GAUSSHIST'] self.bkClient = None self.transClient = None self.storageUsageClient = None ############################################################################# def initialize(self): """ Standard initialize method for agents """ DiracTCAgent.initialize(self) self.directoryLocations = sorted( self.am_getOption('DirectoryLocations', self.directoryLocations)) self.archiveAfter = self.am_getOption('ArchiveAfter', self.archiveAfter) # days self.fileTypesToKeep = Operations().getValue( 'Transformations/FileTypesToKeep', self.fileTypesToKeep) self.bkClient = BookkeepingClient() self.transClient = TransformationClient() self.storageUsageClient = StorageUsageClient() return S_OK() def cleanMetadataCatalogFiles(self, transID): """ clean the metadata using BKK and Data Manager. This method is a replacement of the one from base class :param self: self reference :param int transID: transformation ID """ res = self.bkClient.getProductionFiles(transID, 'ALL', 'Yes') if not res['OK']: return res bkMetadata = res['Value'] fileToRemove = [] yesReplica = [] self.log.info( "Found a total of %d files in the BK for transformation %d" % (len(bkMetadata), transID)) for lfn, metadata in bkMetadata.iteritems(): if metadata['FileType'] != 'LOG': fileToRemove.append(lfn) if metadata['GotReplica'] == 'Yes': yesReplica.append(lfn) if fileToRemove: self.log.info( "Attempting to remove %d possible remnants from the catalog and storage" % len(fileToRemove)) # Executing with shifter proxy gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'false') res = DataManager().removeFile(fileToRemove, force=True) gConfigurationData.setOptionInCFG( '/DIRAC/Security/UseServerCertificate', 'true') if not res['OK']: return res for lfn, reason in res['Value']['Failed'].iteritems(): self.log.error("Failed to remove file found in BK", "%s %s" % (lfn, reason)) if res['Value']['Failed']: return S_ERROR("Failed to remove all files found in the BK") if yesReplica: self.log.info( "Ensuring that %d files are removed from the BK" % (len(yesReplica))) res = FileCatalog( catalogs=['BookkeepingDB']).removeFile(yesReplica) if not res['OK']: return res for lfn, reason in res['Value']['Failed'].iteritems(): self.log.error("Failed to remove file from BK", "%s %s" % (lfn, reason)) if res['Value']['Failed']: return S_ERROR("Failed to remove all files from the BK") self.log.info("Successfully removed all files found in the BK") return S_OK() def getTransformationDirectories(self, transID): """ get the directories for the supplied transformation from the transformation system :param self: self reference :param int transID: transformation ID """ res = DiracTCAgent.getTransformationDirectories(self, transID) if not res['OK']: return res directories = res['Value'] if isinstance(directories, basestring): # Check for (stupid) formats directories = ast.literal_eval(directories) if not isinstance(directories, list): return S_ERROR("Wrong format of output directories") if 'StorageUsage' in self.directoryLocations: res = self.storageUsageClient.getStorageDirectories( '', '', transID, []) if not res['OK']: self.log.error("Failed to obtain storage usage directories", res['Message']) return res transDirectories = res['Value'] directories = self._addDirs(transID, transDirectories, directories) if not directories: self.log.info("No output directories found") # We should be removing from the list of directories # those directories created for file types that are part of those: # - uploaded (as output files) # - not merged by subsequent steps # but this is pretty difficult to identify at run time, so we better remove the "RemovingFiles" production status # and replace it with a flush (this applies only to MC). # So we just have a created list. fileTypesToKeepDirs = [] for fileTypeToKeep in self.fileTypesToKeep: fileTypesToKeepDirs.extend( [x for x in directories if fileTypeToKeep in x]) directories = list( set(directories).difference(set(fileTypesToKeepDirs))) directories = sorted(directories) return S_OK(directories)
class fakeClient: def __init__(self, trans, transID, lfns, asIfProd): self.trans = trans self.transID = transID from DIRAC.TransformationSystem.Client.TransformationClient import TransformationClient self.transClient = TransformationClient() from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient self.bk = BookkeepingClient() from DIRAC.DataManagementSystem.Client.DataManager import DataManager self.dm = DataManager() self.asIfProd = asIfProd (self.transFiles, self.transReplicas) = self.prepareForPlugin(lfns) def addFilesToTransformation(self, transID, lfns): return S_OK({ 'Failed': {}, 'Successful': dict([(lfn, 'Added') for lfn in lfns]) }) def getTransformation(self, transID, extraParams=False): if transID == self.transID and self.asIfProd: transID = self.asIfProd if transID != self.transID: return self.transClient.getTransformation(transID) res = self.trans.getType() return DIRAC.S_OK({'Type': res['Value']}) def getReplicas(self): return self.transReplicas def getFiles(self): return self.transFiles def getCounters(self, table, attrList, condDict): if condDict['TransformationID'] == self.transID and self.asIfProd: condDict['TransformationID'] = self.asIfProd if condDict['TransformationID'] != self.transID: return self.transClient.getCounters(table, attrList, condDict) possibleTargets = [ 'CERN-RAW', 'CNAF-RAW', 'GRIDKA-RAW', 'IN2P3-RAW', 'SARA-RAW', 'PIC-RAW', 'RAL-RAW', 'RRCKI-RAW' ] counters = [] for se in possibleTargets: counters.append(({'UsedSE': se}, 0)) return DIRAC.S_OK(counters) def getBookkeepingQuery(self, transID): if transID == self.transID and self.asIfProd: return self.transClient.getBookkeepingQuery(asIfProd) return self.trans.getBkQuery() def insertTransformationRun(self, transID, runID, xx): return DIRAC.S_OK() def getTransformationRuns(self, condDict): if condDict['TransformationID'] == self.transID and self.asIfProd: condDict['TransformationID'] = self.asIfProd if condDict['TransformationID'] == self.transID: transRuns = [] runs = condDict.get('RunNumber', []) if not runs and self.transFiles: res = self.bk.getFileMetadata( [fileDict['LFN'] for fileDict in self.transFiles]) if not res['OK']: return res runs = list( set(meta['RunNumber'] for meta in res['Value']['Successful'].itervalues())) for run in runs: transRuns.append({ 'RunNumber': run, 'Status': "Active", "SelectedSite": None }) return DIRAC.S_OK(transRuns) else: return self.transClient.getTransformationRuns(condDict) def getTransformationFiles(self, condDict=None): if condDict.get('TransformationID') == self.transID and self.asIfProd: condDict['TransformationID'] = self.asIfProd if condDict.get('TransformationID') == self.transID: transFiles = [] if 'Status' in condDict and 'Unused' not in condDict['Status']: return DIRAC.S_OK(transFiles) runs = None if 'RunNumber' in condDict: runs = condDict['RunNumber'] if not isinstance(runs, list): runs = [runs] for fileDict in self.transFiles: if not runs or fileDict['RunNumber'] in runs: transFiles.append({ 'LFN': fileDict['LFN'], 'Status': 'Unused', 'RunNumber': fileDict['RunNumber'] }) return DIRAC.S_OK(transFiles) else: return self.transClient.getTransformationFiles(condDict=condDict) def setParameterToTransformationFiles(self, transID, lfnDict): """ Update the transFiles with some parameters """ if transID == self.transID: for fileDict in self.transFiles: fileDict.update(lfnDict.get(fileDict['LFN'], {})) return S_OK() else: return self.transClient.setParameterToTransformationFiles( transID, lfnDict) def getTransformationFilesCount(self, transID, field, selection=None): if selection is None: selection = {} if transID == self.transID or selection.get( 'TransformationID') == self.transID: runs = selection.get('RunNumber') if runs and not isinstance(runs, list): runs = [runs] if field == 'Status': counters = {'Unused': 0} for fileDict in self.transFiles: if not runs or fileDict['RunNumber'] in runs: counters['Unused'] += 1 elif field == 'RunNumber': counters = {} for fileDict in self.transFiles: runID = fileDict['RunNumber'] if not runs or runID in runs: counters.setdefault(runID, 0) counters[runID] += 1 else: return DIRAC.S_ERROR('Not implemented for field ' + field) counters['Total'] = sum(count for count in counters.itervalues()) return DIRAC.S_OK(counters) else: return self.transClient.getTransformationFilesCount( transID, field, selection=selection) def getTransformationRunStats(self, transIDs): counters = {} for transID in transIDs: if transID == self.transID: for fileDict in self.transFiles: runID = fileDict['RunNumber'] counters[transID][runID]['Unused'] = counters.setdefault( transID, {}).setdefault(runID, {}).setdefault( 'Unused', 0) + 1 for runID in counters[transID]: counters[transID][runID]['Total'] = counters[transID][ runID]['Unused'] else: res = self.transClient.getTransformationRunStats(transIDs) if res['OK']: counters.update(res['Value']) else: return res return DIRAC.S_OK(counters) def addRunsMetadata(self, runID, val): return self.transClient.addRunsMetadata(runID, val) def getRunsMetadata(self, runID): return self.transClient.getRunsMetadata(runID) def setTransformationRunStatus(self, transID, runID, status): return DIRAC.S_OK() def setTransformationRunsSite(self, transID, runID, site): return DIRAC.S_OK() def setFileStatusForTransformation(self, transID, status, lfns): return DIRAC.S_OK() def addTransformationRunFiles(self, transID, run, lfns): return DIRAC.S_OK() def setDestinationForRun(self, runID, site): return DIRAC.S_OK() def getDestinationForRun(self, runID): return self.transClient.getDestinationForRun(runID) def prepareForPlugin(self, lfns): import time print "Preparing the plugin input data (%d files)" % len(lfns) type = self.trans.getType()['Value'] if not lfns: return (None, None) res = self.bk.getFileMetadata(lfns) if res['OK']: files = [] for lfn, metadata in res['Value']['Successful'].iteritems(): runID = metadata.get('RunNumber', 0) runDict = {"RunNumber": runID, "LFN": lfn} files.append(runDict) else: print "Error getting BK metadata", res['Message'] return ([], {}) replicas = {} startTime = time.time() from DIRAC.Core.Utilities.List import breakListIntoChunks for lfnChunk in breakListIntoChunks(lfns, 200): # print lfnChunk if type.lower() in ("replication", "removal"): res = self.dm.getReplicas(lfnChunk, getUrl=False) else: res = self.dm.getReplicasForJobs(lfnChunk, getUrl=False) # print res if res['OK']: for lfn, ses in res['Value']['Successful'].iteritems(): if ses: replicas[lfn] = sorted(ses) else: print "Error getting replicas of %d files:" % len( lfns), res['Message'] print "Obtained replicas of %d files in %.3f seconds" % ( len(lfns), time.time() - startTime) return (files, replicas)
######################################################################## """ Insert new file types in the Bookkeeping """ __RCSID__ = "$Id$" import DIRAC from DIRAC.Core.Base import Script Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile]' % Script.scriptName ])) Script.parseCommandLine(ignoreErrors=True) from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bk = BookkeepingClient() exitCode = 0 ftype = raw_input("FileType: ") desc = raw_input("Description: ") version = raw_input("File type version: ") print 'Do you want to add this new file type? (yes or no)' value = raw_input('Choice:') choice = value.lower() if choice in ['yes', 'y']: res = bk.insertFileTypes(ftype.upper(), desc, version) if res['OK']: print 'The file types added successfully!' else: print "Error discovered!", res['Message']
Script.setUsageMessage('\n'.join([ __doc__.split('\n')[1], 'Usage:', ' %s [option|cfgfile] ... ProdID' % Script.scriptName, 'Arguments:', ' ProdID: Production ID' ])) Script.parseCommandLine(ignoreErrors=True) args = Script.getPositionalArgs() import types if len(args) < 1: Script.showHelp() exitCode = 0 from LHCbDIRAC.BookkeepingSystem.Client.BookkeepingClient import BookkeepingClient bk = BookkeepingClient() prod = long(args[0]) res = bk.getProductionInformations(prod) if res['OK']: val = res['Value'] print "Production Info: " infs = val["Production informations"] if infs != None: for inf in infs: if inf[2] != None: print ' Configuration Name:', inf[0] print ' Configuration Version:', inf[1] print ' Event type:', inf[2]
class PopularityAgent(AgentModule): """ .. class:: PopularityAgent """ # # DataUsageClient __dataUsageClient = None # # StorageUsageDB instance or DMS/DataUsage RPS client __stDB = None # # BKK Client __bkClient = None # # work directory __workDirectory = None # # counter for records to be sent to the accouting numPopRows = None def initialize(self): """ agent initialisation """ self.am_setOption('PollingTime', 43200) if self.am_getOption('DirectDB', False): self.__stDB = StorageUsageDB() # self.bkClient = BookkeepingClient()#the necessary method is still not available in Bookk. client else: self.__stDB = RPCClient('DataManagement/DataUsage') timeout = 600 self.__bkClient = BookkeepingClient() self.__dataUsageClient = DataUsageClient() self.__workDirectory = self.am_getOption("WorkDirectory") mkDir(self.__workDirectory) self.log.info("Working directory is %s" % self.__workDirectory) # by default, collects raw records from Popularity table inserted in the last day self.timeInterval = self.am_getOption( 'timeIntervalForPopularityRecords', 1) self.queryTimeout = self.am_getOption('queryTimeout', 3600) self.cacheMetadata = {} self.limitForCommit = self.am_getOption("LimitForCommit", 1000) return S_OK() # ......................................................................................... def execute(self): """ Main loop of Popularity agent """ now = datetime.now() endTime = datetime(now.year, now.month, now.day, 0, 0, 0) startTime = endTime - timedelta(days=self.timeInterval) endTimeQuery = endTime.isoformat() startTimeQuery = startTime.isoformat() # query all traces in popularity in the time rage startTime,endTime and status =new # the condition to get th etraces is the AND of the time range and the status new self.log.info( "Querying Pop db to retrieve entries in time range %s - %s " % (startTimeQuery, endTimeQuery)) status = 'New' res = self.__dataUsageClient.getDataUsageSummary( startTimeQuery, endTimeQuery, status, timeout=self.queryTimeout) if not res['OK']: self.log.error("Error querying Popularity table.. %s" % res['Message']) return S_ERROR(res['Message']) val = res['Value'] self.log.info("Retrieved %d entries from Popularity table" % len(val)) # Build popularity report, and store the Ids in a list: idList = set() traceDict = {} for row in val: self.log.debug("row: %s" % str(row)) rowId, dirLfn, site, count, insertTime = row if rowId not in idList: idList.add(rowId) else: self.log.error("Same Id found twice! %d " % rowId) continue if dirLfn.startswith('/lhcb/user/'): self.log.verbose( "Private user directory. No metadata stored in Bkk %s " % dirLfn) continue # get the day (to do ) dayBin = (insertTime - startTime).days traceDict[dayBin][dirLfn][site] = \ traceDict.setdefault(dayBin, {}).setdefault(dirLfn, {}).setdefault(site, 0) + count # print a summary dayList = sorted(traceDict) for day in dayList: self.log.info(" ###### day %s (starting from %s ) " % (day, startTimeQuery)) self.log.info("---- %d directories touched:" % len(traceDict[day])) for lfn in traceDict[day]: self.log.verbose(" ---- lfn %s " % lfn) for site in traceDict[day][lfn]: self.log.verbose(" -------- site %s count: %d " % (site, traceDict[day][lfn][site])) self.log.info("Retrieve meta-data information for each directory ") now = Time.dateTime() self.numPopRows = 0 # keep a counter of the records to send to accounting data-store for day in traceDict: timeForAccounting = self.computeTimeForAccounting(startTime, day) self.log.info("Processing day %s - time for accounting %s " % (day, timeForAccounting)) for dirLfn in traceDict[day]: # did = configName = configVersion = conditions = processingPass = eventType = fileType = production = "na" # retrieve the directory meta-data from the DirMetadata table self.log.info("Processing dir %s " % dirLfn) metaForDir = self.cacheMetadata.get(dirLfn) if not metaForDir: dirList = [dirLfn] # this could be done in a bulk query for a list of directories... TBD res = self.__dataUsageClient.getDirMetadata(dirList) if not res['OK']: self.log.error( "Error retrieving directory meta-data %s " % res['Message']) continue dirMetadata = res['Value'].get(dirLfn) if not res['Value'] or not dirMetadata: self.log.info( "Cache missed: query BK to retrieve '%s' metadata and store cache" % dirList) res = self.__bkClient.getDirectoryMetadata(dirList) if not res['OK']: self.log.error("Failed to query Bookkeeping %s" % res['Message']) metadata = None else: self.log.verbose( "Successfully queried Bookkeeping, result: %s " % res) metadata = res['Value'].get('Successful', {}).get(dirLfn, [{}])[0] if not metadata: self.log.warn( "Directory is not registered in Bookkeeping! %s " % dirLfn) configName = configVersion = conditions = processingPass = eventType = fileType = production = "na" else: metadata['Visibility'] = metadata.pop( 'VisibilityFlag', metadata.get('Visibility', 'na')) configName = metadata['ConfigName'] configVersion = metadata['ConfigVersion'] conditions = metadata['ConditionDescription'] processingPass = metadata['ProcessingPass'] eventType = metadata['EventType'] fileType = metadata['FileType'] production = metadata['Production'] self.log.info( "Cache this entry in DirMetadata table..") res = self.__dataUsageClient.insertToDirMetadata( {dirLfn: metadata}) if not res['OK']: self.log.error( "Failed to insert metadata in DirMetadata table! %s " % res['Message']) else: self.log.info( "Successfully inserted metadata for directory %s in DirMetadata table " % dirLfn) self.log.verbose("result: %s " % res) else: self.log.info( "Directory %s was cached in DirMetadata table" % dirLfn) try: __did, configName, configVersion, conditions, \ processingPass, eventType, fileType, production = dirMetadata[0:8] except BaseException: self.log.error( "Error decoding directory cached information", dirMetadata) continue self.cacheMetadata[dirLfn] = (configName, configVersion, conditions, processingPass, eventType, fileType, production) else: configName, configVersion, conditions, processingPass, eventType, fileType, production = metaForDir for site in traceDict[day][dirLfn]: usage = traceDict[day][dirLfn][site] # compute the normalized usage, dividing by the number of files in the directory: normUsage = usage # to be done! after we have decided how to normalize # Build record for the accounting popRecord = Popularity() popRecord.setStartTime(timeForAccounting) popRecord.setEndTime(timeForAccounting) popRecord.setValueByKey("DataType", configName) popRecord.setValueByKey("Activity", configVersion) popRecord.setValueByKey("FileType", fileType) popRecord.setValueByKey("Production", production) popRecord.setValueByKey("ProcessingPass", processingPass) popRecord.setValueByKey("Conditions", conditions) popRecord.setValueByKey("EventType", eventType) popRecord.setValueByKey("StorageElement", site) popRecord.setValueByKey("Usage", usage) popRecord.setValueByKey("NormalizedUsage", normUsage) res = gDataStoreClient.addRegister(popRecord) if not res['OK']: self.log.error("ERROR: addRegister returned: %s" % res['Message']) continue self.numPopRows += 1 self.log.info( ">>> Sending record to accounting for: %s %s %s %s %s %s %s %s %s %d %d " % (timeForAccounting, configName, configVersion, fileType, production, processingPass, conditions, eventType, site, usage, normUsage)) if self.numPopRows > self.limitForCommit: res = self.__commitAccounting() if not res['OK']: return res # then set the status to Used res = self.__commitAccounting() if not res['OK']: return res self.log.info("Set the status to Used for %d entries" % len(idList)) from DIRAC.Core.Utilities.List import breakListIntoChunks for idChunk in breakListIntoChunks(list(idList), 1000): res = self.__dataUsageClient.updatePopEntryStatus( list(idChunk), 'Used', timeout=self.queryTimeout) if not res['OK']: self.log.error( "Error to update status in Popularity table.. %s" % res['Message']) return res self.log.info("Status updated to Used correctly for %s entries " % len(idList)) return S_OK() # ......................................................................................... def __commitAccounting(self): res = gDataStoreClient.commit() if not res['OK']: self.log.error( "while committing %d Popularity records" % self.numPopRows, res['Message']) else: self.log.info( "%s records for Popularity type successfully committed" % self.numPopRows) self.numPopRows = 0 return res def computeTimeForAccounting(self, startTime, day): """ Compute the time for the accounting record, starting from the start time of the query and the day bin """ self.log.verbose( "find time for accounting for startTime: %s + day %s " % (startTime, day)) daysToAdd = timedelta( days=day, hours=12 ) # add 12h just to put the value in the middle of time bin self.log.verbose("timedelta to add: %s " % daysToAdd) accTime = startTime + daysToAdd self.log.verbose("accTime = %s " % accTime) return accTime