def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl): """ pileupDict is a Python dictionary containing particular pileup configuration information. Query DBS on given dataset contained now in both input defaultArguments as well as in the pileupDict and compare values. """ reader = DBS3Reader(dbsUrl) phedex = PhEDEx() inputArgs = defaultArguments["PileupConfig"] self.assertEqual(len(inputArgs), len(pileupDict), "Number of pileup types different.") for pileupType in inputArgs: m = ("pileup type '%s' not in PileupFetcher-produced pileup " "configuration: '%s'" % (pileupType, pileupDict)) self.assertTrue(pileupType in pileupDict, m) # now query DBS for compare actual results on files lists for each # pileup type and dataset and location (storage element names) # pileupDict is saved in the file and now comparing items of this # configuration with actual DBS results, the structure of pileupDict: # {"pileupTypeA": {"BlockA": {"FileList": [], "PhEDExNodeNames": []}, # "BlockB": {"FileList": [], "PhEDExNodeNames": []}, ....} for pileupType, datasets in inputArgs.items(): # this is from the pileup configuration produced by PileupFetcher blockDict = pileupDict[pileupType] for dataset in datasets: dbsFileBlocks = reader.listFileBlocks(dataset=dataset) blocksLocation = phedex.getReplicaPhEDExNodesForBlocks( dataset=dataset, complete='y') for dbsFileBlockName in dbsFileBlocks: fileList = [] pnns = set() for pnn in blocksLocation[dbsFileBlockName]: pnns.add(pnn) # now get list of files in the block dbsFiles = reader.listFilesInBlock(dbsFileBlockName) for dbsFile in dbsFiles: fileList.append(dbsFile["LogicalFileName"]) # now compare the sets: m = ("PNNs don't agree for pileup type '%s', " "dataset '%s' in configuration: '%s'" % (pileupType, dataset, pileupDict)) self.assertEqual( set(blockDict[dbsFileBlockName]["PhEDExNodeNames"]), pnns, m) m = ( "FileList don't agree for pileup type '%s', dataset '%s' " " in configuration: '%s'" % (pileupType, dataset, pileupDict)) storedFileList = [ item['logical_file_name'] for item in blockDict[dbsFileBlockName]["FileList"] ] self.assertItemsEqual(storedFileList, fileList, m)
def _queryDbsAndGetPileupConfig(self, stepHelper, dbsReader): """ Method iterates over components of the pileup configuration input and queries DBS. Then iterates over results from DBS. There needs to be a list of files and their locations for each dataset name. Use dbsReader the result data structure is a Python dict following dictionary: FileList is a list of LFNs {"pileupTypeA": {"BlockA": {"FileList": [], "PhEDExNodeNames": []}, "BlockB": {"FileList": [], "PhEDExNodeName": []}, ....} this structure preserves knowledge of where particular files of dataset are physically (list of PNNs) located. DBS only lists sites which have all files belonging to blocks but e.g. BlockA of dataset DS1 may be located at site1 and BlockB only at site2 - it's possible that only a subset of the blocks in a dataset will be at a site. """ # only production PhEDEx is connected (This can be moved to init method phedex = PhEDEx() node_filter = set(['UNKNOWN', None]) resultDict = {} # iterate over input pileup types (e.g. "cosmics", "minbias") for pileupType in stepHelper.data.pileup.listSections_(): # the format here is: step.data.pileup.cosmics.dataset = [/some/data/set] datasets = getattr(getattr(stepHelper.data.pileup, pileupType), "dataset") # each dataset input can generally be a list, iterate over dataset names blockDict = {} for dataset in datasets: blockFileInfo = dbsReader.getFileListByDataset(dataset=dataset, detail=True) for fileInfo in blockFileInfo: blockDict.setdefault(fileInfo['block_name'], { 'FileList': [], 'NumberOfEvents': 0, 'PhEDExNodeNames': [] }) blockDict[fileInfo['block_name']]['FileList'].append( {'logical_file_name': fileInfo['logical_file_name']}) blockDict[fileInfo['block_name']][ 'NumberOfEvents'] += fileInfo['event_count'] blockReplicasInfo = phedex.getReplicaPhEDExNodesForBlocks( dataset=dataset, complete='y') for block in blockReplicasInfo: nodes = set(blockReplicasInfo[block]) - node_filter blockDict[block]['PhEDExNodeNames'] = list(nodes) blockDict[block]['FileList'] = sorted( blockDict[block]['FileList']) resultDict[pileupType] = blockDict return resultDict
def _queryDbsAndGetPileupConfig(self, stepHelper, dbsReader, fakeSites): """ Method iterates over components of the pileup configuration input and queries DBS. Then iterates over results from DBS. There needs to be a list of files and their locations for each dataset name. Use dbsReader the result data structure is a Python dict following dictionary: FileList is a list of LFNs {"pileupTypeA": {"BlockA": {"FileList": [], "PhEDExNodeNames": []}, "BlockB": {"FileList": [], "PhEDExNodeName": []}, ....} this structure preserves knowledge of where particular files of dataset are physically (list of PNNs) located. DBS only lists sites which have all files belonging to blocks but e.g. BlockA of dataset DS1 may be located at site1 and BlockB only at site2 - it's possible that only a subset of the blocks in a dataset will be at a site. """ # only production PhEDEx is connected (This can be moved to init method phedex = PhEDEx() node_filter = set(['UNKNOWN', None]) # convert the siteWhitelist into SE list and add SEs to the pileup location list fakePNNs = [] if fakeSites: fakePNNs = mapSitetoPNN(fakeSites) resultDict = {} # iterate over input pileup types (e.g. "cosmics", "minbias") for pileupType in stepHelper.data.pileup.listSections_(): # the format here is: step.data.pileup.cosmics.dataset = [/some/data/set] datasets = getattr(getattr(stepHelper.data.pileup, pileupType), "dataset") # each dataset input can generally be a list, iterate over dataset names blockDict = {} for dataset in datasets: blockFileInfo = dbsReader.getFileListByDataset(dataset=dataset, detail=True) for fileInfo in blockFileInfo: blockDict.setdefault(fileInfo['block_name'], {'FileList': [], 'NumberOfEvents': 0, 'PhEDExNodeNames': []}) blockDict[fileInfo['block_name']]['FileList'].append( {'logical_file_name': fileInfo['logical_file_name']}) blockDict[fileInfo['block_name']]['NumberOfEvents'] += fileInfo['event_count'] blockReplicasInfo = phedex.getReplicaPhEDExNodesForBlocks(dataset=dataset, complete='y') for block in blockReplicasInfo: nodes = set(blockReplicasInfo[block]) - node_filter | set(fakePNNs) blockDict[block]['PhEDExNodeNames'] = list(nodes) blockDict[block]['FileList'] = sorted(blockDict[block]['FileList']) resultDict[pileupType] = blockDict return resultDict
def _queryAndCompareWithDBS(self, pileupDict, defaultArguments, dbsUrl): """ pileupDict is a Python dictionary containing particular pileup configuration information. Query DBS on given dataset contained now in both input defaultArguments as well as in the pileupDict and compare values. """ reader = DBS3Reader(dbsUrl) phedex = PhEDEx() inputArgs = defaultArguments["PileupConfig"] self.assertEqual(len(inputArgs), len(pileupDict), "Number of pileup types different.") for pileupType in inputArgs: m = ("pileup type '%s' not in PileupFetcher-produced pileup " "configuration: '%s'" % (pileupType, pileupDict)) self.assertTrue(pileupType in pileupDict, m) # now query DBS for compare actual results on files lists for each # pileup type and dataset and location (storage element names) # pileupDict is saved in the file and now comparing items of this # configuration with actual DBS results, the structure of pileupDict: # {"pileupTypeA": {"BlockA": {"FileList": [], "PhEDExNodeNames": []}, # "BlockB": {"FileList": [], "PhEDExNodeNames": []}, ....} for pileupType, datasets in inputArgs.items(): # this is from the pileup configuration produced by PileupFetcher blockDict = pileupDict[pileupType] for dataset in datasets: dbsFileBlocks = reader.listFileBlocks(dataset=dataset) blocksLocation = phedex.getReplicaPhEDExNodesForBlocks(dataset=dataset, complete='y') for dbsFileBlockName in dbsFileBlocks: fileList = [] pnns = set() for pnn in blocksLocation[dbsFileBlockName]: pnns.add(pnn) # now get list of files in the block dbsFiles = reader.listFilesInBlock(dbsFileBlockName) for dbsFile in dbsFiles: fileList.append(dbsFile["LogicalFileName"]) # now compare the sets: m = ("PNNs don't agree for pileup type '%s', " "dataset '%s' in configuration: '%s'" % (pileupType, dataset, pileupDict)) self.assertEqual(set(blockDict[dbsFileBlockName]["PhEDExNodeNames"]), pnns, m) m = ("FileList don't agree for pileup type '%s', dataset '%s' " " in configuration: '%s'" % (pileupType, dataset, pileupDict)) storedFileList = [item['logical_file_name'] for item in blockDict[dbsFileBlockName]["FileList"]] self.assertItemsEqual(storedFileList, fileList, m)
class DBS3Reader(object): """ _DBSReader_ General API for reading data from DBS """ # cache all the datatiers known by DBS _datatiers = {} def __init__(self, url, **contact): # instantiate dbs api object try: self.dbsURL = url self.dbs = DbsApi(url, **contact) except dbsClientException as ex: msg = "Error in DBSReader with DbsApi\n" msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # connection to PhEDEx (Use default endpoint url) self.phedex = PhEDEx(responseType="json") def _getLumiList(self, blockName=None, lfns=None, validFileOnly=1): """ currently only take one lfn but dbs api need be updated """ try: if blockName: lumiLists = self.dbs.listFileLumis(block_name=blockName, validFileOnly=validFileOnly) elif lfns: lumiLists = [] for slfn in grouper(lfns, 50): lumiLists.extend( self.dbs.listFileLumiArray(logical_file_name=slfn)) else: # shouldn't call this with both blockName and lfns empty # but still returns empty dict for that case return {} except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFileLumiArray(%s)\n" % lfns msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) lumiDict = {} for lumisItem in lumiLists: lumiDict.setdefault(lumisItem['logical_file_name'], []) item = {} item["RunNumber"] = lumisItem['run_num'] item['LumiSectionNumber'] = lumisItem['lumi_section_num'] if lumisItem.get('event_count', None) is not None: item['EventCount'] = lumisItem['event_count'] lumiDict[lumisItem['logical_file_name']].append(item) # TODO: add key for lumi and event pair. return lumiDict def checkDBSServer(self): """ check whether dbs server is up and running returns {"dbs_instance": "prod/global", "dbs_version": "3.3.144"} """ try: return self.dbs.serverinfo() except dbsClientException as ex: msg = "Error in " msg += "DBS server is not up: %s" % self.dbsURL msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listPrimaryDatasets(self, match='*'): """ _listPrimaryDatasets_ return a list of primary datasets, The full dataset name must be provided pattern based mathcing is no longer supported. If no expression is provided, all datasets are returned """ try: result = self.dbs.listPrimaryDatasets(primary_ds_name=match) except dbsClientException as ex: msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % match msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['primary_ds_name'] for x in result] return result def matchProcessedDatasets(self, primary, tier, process): """ _matchProcessedDatasets_ return a list of Processed datasets """ result = [] try: datasets = self.dbs.listDatasets(primary_ds_name=primary, data_tier_name=tier, detail=True) except dbsClientException as ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) for dataset in datasets: dataset = remapDBS3Keys(dataset, processed_ds_name='Name') dataset['PathList'] = [dataset['dataset']] if dataset['Name'] == process: result.append(dataset) return result def listRuns(self, dataset=None, block=None): """ it gets list of DbsRun object but for our purpose only list of number is collected. DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ runs = [] try: if block: results = self.dbs.listRuns(block_name=block) else: results = self.dbs.listRuns(dataset=dataset) except dbsClientException as ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) [runs.extend(x['run_num']) for x in results] return runs def listRunLumis(self, dataset=None, block=None): """ It gets a list of DBSRun objects and returns the number of lumisections per run DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ # Pointless code in python3 if isinstance(block, str): block = unicode(block) if isinstance(dataset, str): dataset = unicode(dataset) try: if block: results = self.dbs.listRuns(block_name=block) else: results = self.dbs.listRuns(dataset=dataset) except dbsClientException as ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # send runDict format as result, this format is for sync with dbs2 call # which has {run_number: num_lumis} but dbs3 call doesn't return num Lumis # So it returns {run_number: None} # TODO: After DBS2 is completely removed change the return format more sensible one runDict = {} for x in results: for runNumber in x["run_num"]: runDict[runNumber] = None return runDict def listProcessedDatasets(self, primary, dataTier='*'): """ _listProcessedDatasets_ return a list of Processed datasets for the primary and optional data tier value """ try: result = self.dbs.listDatasets(primary_ds_name=primary, data_tier_name=dataTier) except dbsClientException as ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['dataset'].split('/')[2] for x in result] return result def listDatasetFiles(self, datasetPath): """ _listDatasetFiles_ Get list of files for dataset """ return [ x['logical_file_name'] for x in self.dbs.listFileArray(dataset=datasetPath) ] @staticmethod def listDatatiers(dbsUrl=None): """ _listDatatiers_ Get a list of datatiers known by DBS. """ if dbsUrl is None: msg = "Error in DBSReader.listDatatiers(). DBS Url not set." raise DBSReaderError(msg) timenow = int(time.time()) if DBS3Reader._datatiers and timenow - 7200 < DBS3Reader._datatiers[ 'ts']: return DBS3Reader._datatiers['tiers'] try: DBS3Reader._setDatatiersCache(timenow, dbsUrl) except Exception as ex: if not DBS3Reader._datatiers: msg = "Error in DBSReader.listDatatiers\n%s" % formatEx3(ex) raise DBSReaderError(msg) return DBS3Reader._datatiers['tiers'] @staticmethod def _setDatatiersCache(ts, dbsUrl): """ Set a timestamp and update the list of datatiers cached in the class property """ dbs = DbsApi(dbsUrl) DBS3Reader._datatiers['ts'] = ts DBS3Reader._datatiers['tiers'] = [ tier['data_tier_name'] for tier in dbs.listDataTiers() ] return def listDatasetFileDetails(self, datasetPath, getParents=False, validFileOnly=1): """ TODO: This is completely wrong need to be redone. or be removed - getting dataset altogether might be to costly _listDatasetFileDetails_ Get list of lumis, events, and parents for each file in a dataset Return a dict where the keys are the files, and for each file we have something like: { 'NumberOfEvents': 545, 'BlockName': '/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace', 'Lumis': {173658: [8, 12, 9, 14, 19, 109, 105]}, 'Parents': [], 'Checksum': '22218315', 'Adler32': 'a41a1446', 'FileSize': 286021145, 'ValidFile': 1 } """ fileDetails = self.getFileListByDataset(dataset=datasetPath, validFileOnly=validFileOnly, detail=True) blocks = set() # the set of blocks of the dataset # Iterate over the files and prepare the set of blocks and a dict where the keys are the files files = {} for f in fileDetails: blocks.add(f['block_name']) files[f['logical_file_name']] = remapDBS3Keys(f, stringify=True) files[f['logical_file_name']]['ValidFile'] = f['is_file_valid'] files[f['logical_file_name']]['Lumis'] = {} files[f['logical_file_name']]['Parents'] = [] # Iterate over the blocks and get parents and lumis for blockName in blocks: # get the parents if getParents: parents = self.dbs.listFileParents(block_name=blockName) for p in parents: if p['logical_file_name'] in files: # invalid files are not there if validFileOnly=1 files[p['logical_file_name']]['Parents'].extend( p['parent_logical_file_name']) # get the lumis file_lumis = self.dbs.listFileLumis(block_name=blockName) for f in file_lumis: if f['logical_file_name'] in files: # invalid files are not there if validFileOnly=1 if f['run_num'] in files[f['logical_file_name']]['Lumis']: files[f['logical_file_name']]['Lumis'][ f['run_num']].extend(f['lumi_section_num']) else: files[f['logical_file_name']]['Lumis'][ f['run_num']] = f['lumi_section_num'] return files def crossCheck(self, datasetPath, *lfns): """ _crossCheck_ For the dataset provided, check that the lfns listed all exist in the dataset. Return the list of lfns that are in the dataset """ allLfns = self.dbs.listFileArray(dataset=datasetPath, validFileOnly=1, detail=False) setOfAllLfns = set(allLfns) setOfKnownLfns = set(lfns) return list(setOfAllLfns.intersection(setOfKnownLfns)) def crossCheckMissing(self, datasetPath, *lfns): """ _crossCheckMissing_ As cross check, but return value is a list of files that are *not* known by DBS """ allLfns = self.dbs.listFileArray(dataset=datasetPath, validFileOnly=1, detail=False) setOfAllLfns = set(allLfns) setOfKnownLfns = set(lfns) knownFiles = setOfAllLfns.intersection(setOfKnownLfns) unknownFiles = setOfKnownLfns.difference(knownFiles) return list(unknownFiles) def getDBSSummaryInfo(self, dataset=None, block=None): """ Get dataset summary includes # of files, events, blocks and total size """ # FIXME: Doesnt raise exceptions on missing data as old api did if dataset: self.checkDatasetPath(dataset) try: if block: summary = self.dbs.listFileSummaries(block_name=block, validFileOnly=1) else: # dataset case dataset shouldn't be None summary = self.dbs.listFileSummaries(dataset=dataset, validFileOnly=1) except Exception as ex: msg = "Error in DBSReader.getDBSSummaryInfo(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if not summary or summary[0].get( 'file_size') is None: # appears to indicate missing dataset msg = "DBSReader.listDatasetSummary(%s, %s): No matching data" raise DBSReaderError(msg % (dataset, block)) result = remapDBS3Keys(summary[0], stringify=True) result['path'] = dataset if dataset else '' result['block'] = block if block else '' return result def getFileBlocksInfo(self, dataset, onlyClosedBlocks=False, blockName=None, locations=True): """ """ self.checkDatasetPath(dataset) args = {'dataset': dataset, 'detail': True} if blockName: args['block_name'] = blockName try: blocks = self.dbs.listBlocks(**args) except Exception as ex: msg = "Error in DBSReader.getFileBlocksInfo(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) blocks = [ remapDBS3Keys(block, stringify=True, block_name='Name') for block in blocks ] # only raise if blockName not specified - mimic dbs2 error handling if not blocks and not blockName: msg = "DBSReader.getFileBlocksInfo(%s, %s): No matching data" raise DBSReaderError(msg % (dataset, blockName)) if locations: for block in blocks: block['PhEDExNodeList'] = [{ 'Name': x } for x in self.listFileBlockLocation(block['Name'])] if onlyClosedBlocks: return [x for x in blocks if str(x['OpenForWriting']) != "1"] return blocks def listFileBlocks(self, dataset, onlyClosedBlocks=False, blockName=None): """ _listFileBlocks_ Retrieve a list of fileblock names for a dataset """ self.checkDatasetPath(dataset) args = {'dataset': dataset, 'detail': False} if blockName: args['block_name'] = blockName if onlyClosedBlocks: args['detail'] = True try: blocks = self.dbs.listBlocks(**args) except dbsClientException as ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if onlyClosedBlocks: result = [ x['block_name'] for x in blocks if str(x['open_for_writing']) != "1" ] else: result = [x['block_name'] for x in blocks] return result def listOpenFileBlocks(self, dataset): """ _listOpenFileBlocks_ Retrieve a list of open fileblock names for a dataset """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset=dataset, detail=True) except dbsClientException as ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [ x['block_name'] for x in blocks if str(x['open_for_writing']) == "1" ] return result def blockExists(self, fileBlockName): """ _blockExists_ Check to see if block with name provided exists in the DBS Instance. Return True if exists, False if not """ self.checkBlockName(fileBlockName) try: blocks = self.dbs.listBlocks(block_name=fileBlockName) except Exception as ex: msg = "Error in " msg += "DBSReader.blockExists(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if len(blocks) == 0: return False return True def listFilesInBlock(self, fileBlockName, lumis=True, validFileOnly=1): """ _listFilesInBlock_ Get a list of files in the named fileblock TODO: lumis can be false when lumi splitting is not required However WMBSHelper expect file['LumiList'] to get the run number so for now it will be always true. We need to clean code up when dbs2 is completely deprecated. calling lumis for run number is expensive. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: files = self.dbs.listFileArray(block_name=fileBlockName, validFileOnly=validFileOnly, detail=True) except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if lumis: lumiDict = self._getLumiList(blockName=fileBlockName, validFileOnly=validFileOnly) result = [] for fileInfo in files: if lumis: fileInfo["LumiList"] = lumiDict[fileInfo['logical_file_name']] result.append(remapDBS3Keys(fileInfo, stringify=True)) return result def listFilesInBlockWithParents(self, fileBlockName, lumis=True, validFileOnly=1): """ _listFilesInBlockWithParents_ Get a list of files in the named fileblock including the parents of that file. TODO: lumis can be false when lumi splitting is not required However WMBSHelper expect file['LumiList'] to get the run number so for now it will be always true. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlockWithParents(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: # TODO: shoud we get only valid block for this? files = self.dbs.listFileParents(block_name=fileBlockName) fileDetails = self.listFilesInBlock(fileBlockName, lumis, validFileOnly) except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % ( fileBlockName, ) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) childByParents = defaultdict(list) for f in files: # Probably a child can have more than 1 parent file for fp in f['parent_logical_file_name']: childByParents[fp].append(f['logical_file_name']) parentsLFNs = childByParents.keys() if len(parentsLFNs) == 0: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n There is no parents files" % ( fileBlockName) raise DBSReaderError(msg) parentFilesDetail = [] # TODO: slicing parentLFNs util DBS api is handling that. # Remove slicing if DBS api handles for pLFNs in grouper(parentsLFNs, 50): parentFilesDetail.extend( self.dbs.listFileArray(logical_file_name=pLFNs, detail=True)) if lumis: parentLumis = self._getLumiList(lfns=parentsLFNs) parentsByLFN = defaultdict(list) for pf in parentFilesDetail: parentLFN = pf['logical_file_name'] dbsFile = remapDBS3Keys(pf, stringify=True) if lumis: dbsFile["LumiList"] = parentLumis[parentLFN] for childLFN in childByParents[parentLFN]: parentsByLFN[childLFN].append(dbsFile) for fileInfo in fileDetails: fileInfo["ParentList"] = parentsByLFN[ fileInfo['logical_file_name']] return fileDetails def lfnsInBlock(self, fileBlockName): """ _lfnsInBlock_ LFN list only for block, details = False => faster query """ if not self.blockExists(fileBlockName): msg = "DBSReader.lfnsInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: lfns = self.dbs.listFileArray(block_name=fileBlockName, validFileOnly=1, detail=False) return lfns except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listFileBlockLocation(self, fileBlockNames, dbsOnly=False): """ _listFileBlockLocation_ Get origin_site_name of a block """ singleBlockName = None if isinstance(fileBlockNames, basestring): singleBlockName = fileBlockNames fileBlockNames = [fileBlockNames] for block in fileBlockNames: self.checkBlockName(block) locations = {} node_filter = set(['UNKNOWN', None]) if dbsOnly: blocksInfo = {} try: for block in fileBlockNames: blocksInfo.setdefault(block, []) # there should be only one element with a single origin site string ... for blockInfo in self.dbs.listBlockOrigin( block_name=block): blocksInfo[block].append(blockInfo['origin_site_name']) except dbsClientException as ex: msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockNames msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) else: try: blocksInfo = self.phedex.getReplicaPhEDExNodesForBlocks( block=fileBlockNames, complete='y') except Exception as ex: msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockNames msg += "%s\n" % str(ex) raise Exception(msg) for block in fileBlockNames: valid_nodes = set(blocksInfo.get(block, [])) - node_filter locations[block] = list(valid_nodes) # returning single list if a single block is passed if singleBlockName: return locations[singleBlockName] return locations def getFileBlock(self, fileBlockName, dbsOnly=False): """ _getFileBlock_ dbsOnly flag is mostly meant for StoreResults, since there is no data in TMDB. return a dictionary: { blockName: { "PhEDExNodeNames" : [<pnn list>], "Files" : { LFN : Events }, } } """ # Pointless code in python3 if isinstance(fileBlockName, str): fileBlockName = unicode(fileBlockName) if not self.blockExists(fileBlockName): msg = "DBSReader.getFileBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) result = { fileBlockName: { "PhEDExNodeNames": self.listFileBlockLocation(fileBlockName, dbsOnly), "Files": self.listFilesInBlock(fileBlockName), "IsOpen": self.blockIsOpen(fileBlockName) } } return result def getFileBlockWithParents(self, fileBlockName): """ _getFileBlockWithParents_ return a dictionary: { blockName: { "PhEDExNodeNames" : [<pnn list>], "Files" : dictionaries representing each file } } files """ if isinstance(fileBlockName, str): fileBlockName = unicode(fileBlockName) if not self.blockExists(fileBlockName): msg = "DBSReader.getFileBlockWithParents(%s): No matching data" raise DBSReaderError(msg % fileBlockName) result = { fileBlockName: { "PhEDExNodeNames": self.listFileBlockLocation(fileBlockName), "Files": self.listFilesInBlockWithParents(fileBlockName), "IsOpen": self.blockIsOpen(fileBlockName) } } return result def getFiles(self, dataset, onlyClosedBlocks=False): """ _getFiles_ Returns a dictionary of block names for the dataset where each block constists of a dictionary containing the PhEDExNodeNames for that block and the files in that block by LFN mapped to NEvents """ result = {} blocks = self.listFileBlocks(dataset, onlyClosedBlocks) [result.update(self.getFileBlock(x)) for x in blocks] return result def listBlockParents(self, blockName): """Get parent blocks for block""" result = [] self.checkBlockName(blockName) blocks = self.dbs.listBlockParents(block_name=blockName) for block in blocks: toreturn = {'Name': block['parent_block_name']} toreturn['PhEDExNodeList'] = self.listFileBlockLocation( toreturn['Name']) result.append(toreturn) return result def blockIsOpen(self, blockName): """ _blockIsOpen_ Return True if named block is open, false if not, or if block doenst exist """ self.checkBlockName(blockName) blockInstance = self.dbs.listBlocks(block_name=blockName, detail=True) if len(blockInstance) == 0: return False blockInstance = blockInstance[0] isOpen = blockInstance.get('open_for_writing', 1) if isOpen == 0: return False return True def blockToDatasetPath(self, blockName): """ _blockToDatasetPath_ Given a block name, get the dataset Path associated with that Block. Returns the dataset path, or None if not found """ self.checkBlockName(blockName) try: blocks = self.dbs.listBlocks(block_name=blockName, detail=True) except Exception as ex: msg = "Error in " msg += "DBSReader.blockToDatasetPath(%s)\n" % blockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if blocks == []: return None pathname = blocks[-1].get('dataset', None) return pathname def listDatasetLocation(self, datasetName, dbsOnly=False): """ _listDatasetLocation_ List the origin SEs where there is at least a block of the given dataset. """ self.checkDatasetPath(datasetName) locations = set() if dbsOnly: try: blocksInfo = self.dbs.listBlockOrigin(dataset=datasetName) except dbsClientException as ex: msg = "Error in DBSReader: dbsApi.listBlocks(dataset=%s)\n" % datasetName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if not blocksInfo: # no data location from dbs return list() for blockInfo in blocksInfo: locations.update(blockInfo['origin_site_name']) locations.difference_update( ['UNKNOWN', None]) # remove entry when SE name is 'UNKNOWN' else: try: blocksInfo = self.phedex.getReplicaPhEDExNodesForBlocks( dataset=[datasetName], complete='y') except Exception as ex: msg = "Error while getting block location from PhEDEx for dataset=%s)\n" % datasetName msg += "%s\n" % str(ex) raise Exception(msg) if blocksInfo: for blockSites in blocksInfo.values(): locations.update(blockSites) return list(locations) def checkDatasetPath(self, pathName): """ _checkDatasetPath_ """ if pathName in ("", None): raise DBSReaderError("Invalid Dataset Path name: => %s <=" % pathName) else: try: result = self.dbs.listDatasets(dataset=pathName, dataset_access_type='*') if len(result) == 0: raise DBSReaderError("Dataset %s doesn't exist in DBS %s" % (pathName, self.dbsURL)) except (dbsClientException, HTTPError) as ex: msg = "Error in " msg += "DBSReader.checkDatasetPath(%s)\n" % pathName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) return def checkBlockName(self, blockName): """ _checkBlockName_ """ if blockName in ("", "*", None): raise DBSReaderError("Invalid Block name: => %s <=" % blockName) def getFileListByDataset(self, dataset, validFileOnly=1, detail=True): """ _getFileListByDataset_ Given a dataset, retrieves all blocks, lfns and number of events (among other not really important info). Returns a list of dict. """ try: fileList = self.dbs.listFileArray(dataset=dataset, validFileOnly=validFileOnly, detail=detail) return fileList except dbsClientException as ex: msg = "Error in " msg += "DBSReader.getFileListByDataset(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listDatasetParents(self, childDataset): """ list the the parents dataset path given childDataset """ try: parentList = self.dbs.listDatasetParents(dataset=childDataset) return parentList except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listDatasetParents(%s)\n" % childDataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg)
class DBS3Reader(object): """ _DBSReader_ General API for reading data from DBS """ def __init__(self, url, logger=None, **contact): # instantiate dbs api object try: self.dbsURL = url self.dbs = DbsApi(url, **contact) self.logger = logger or logging.getLogger(self.__class__.__name__) except dbsClientException as ex: msg = "Error in DBSReader with DbsApi\n" msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # connection to PhEDEx (Use default endpoint url) self.phedex = PhEDEx(responseType="json", dbsUrl=self.dbsURL) def _getLumiList(self, blockName=None, lfns=None, validFileOnly=1): """ currently only take one lfn but dbs api need be updated """ try: if blockName: lumiLists = self.dbs.listFileLumis(block_name=blockName, validFileOnly=validFileOnly) elif lfns: lumiLists = [] for slfn in grouper(lfns, 50): lumiLists.extend(self.dbs.listFileLumiArray(logical_file_name=slfn)) else: # shouldn't call this with both blockName and lfns empty # but still returns empty dict for that case return {} except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFileLumiArray(%s)\n" % lfns msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) lumiDict = {} for lumisItem in lumiLists: lumiDict.setdefault(lumisItem['logical_file_name'], []) item = {} item["RunNumber"] = lumisItem['run_num'] item['LumiSectionNumber'] = lumisItem['lumi_section_num'] if lumisItem.get('event_count', None) is not None: item['EventCount'] = lumisItem['event_count'] lumiDict[lumisItem['logical_file_name']].append(item) # TODO: add key for lumi and event pair. return lumiDict def checkDBSServer(self): """ check whether dbs server is up and running returns {"dbs_instance": "prod/global", "dbs_version": "3.3.144"} """ try: return self.dbs.serverinfo() except dbsClientException as ex: msg = "Error in " msg += "DBS server is not up: %s" % self.dbsURL msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listPrimaryDatasets(self, match='*'): """ _listPrimaryDatasets_ return a list of primary datasets, The full dataset name must be provided pattern based mathcing is no longer supported. If no expression is provided, all datasets are returned """ try: result = self.dbs.listPrimaryDatasets(primary_ds_name=match) except dbsClientException as ex: msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % match msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['primary_ds_name'] for x in result] return result def matchProcessedDatasets(self, primary, tier, process): """ _matchProcessedDatasets_ return a list of Processed datasets """ result = [] try: datasets = self.dbs.listDatasets(primary_ds_name=primary, data_tier_name=tier, detail=True) except dbsClientException as ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) for dataset in datasets: dataset = remapDBS3Keys(dataset, processed_ds_name='Name') dataset['PathList'] = [dataset['dataset']] if dataset['Name'] == process: result.append(dataset) return result def listRuns(self, dataset=None, block=None): """ it gets list of DbsRun object but for our purpose only list of number is collected. DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ runs = [] try: if block: results = self.dbs.listRuns(block_name=block) else: results = self.dbs.listRuns(dataset=dataset) except dbsClientException as ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) for x in results: runs.extend(x['run_num']) return runs def listRunLumis(self, dataset=None, block=None): """ It gets a list of DBSRun objects and returns the number of lumisections per run DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ # Pointless code in python3 if isinstance(block, str): block = unicode(block) if isinstance(dataset, str): dataset = unicode(dataset) try: if block: results = self.dbs.listRuns(block_name=block) else: results = self.dbs.listRuns(dataset=dataset) except dbsClientException as ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # send runDict format as result, this format is for sync with dbs2 call # which has {run_number: num_lumis} but dbs3 call doesn't return num Lumis # So it returns {run_number: None} # TODO: After DBS2 is completely removed change the return format more sensible one runDict = {} for x in results: for runNumber in x["run_num"]: runDict[runNumber] = None return runDict def listProcessedDatasets(self, primary, dataTier='*'): """ _listProcessedDatasets_ return a list of Processed datasets for the primary and optional data tier value """ try: result = self.dbs.listDatasets(primary_ds_name=primary, data_tier_name=dataTier) except dbsClientException as ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['dataset'].split('/')[2] for x in result] return result def listDatasetFiles(self, datasetPath): """ _listDatasetFiles_ Get list of files for dataset """ return [x['logical_file_name'] for x in self.dbs.listFileArray(dataset=datasetPath)] def listDatatiers(self): """ _listDatatiers_ Get a list of datatiers known by DBS. """ return [tier['data_tier_name'] for tier in self.dbs.listDataTiers()] def listDatasetFileDetails(self, datasetPath, getParents=False, getLumis=True, validFileOnly=1): """ TODO: This is completely wrong need to be redone. or be removed - getting dataset altogether might be to costly _listDatasetFileDetails_ Get list of lumis, events, and parents for each file in a dataset Return a dict where the keys are the files, and for each file we have something like: { 'NumberOfEvents': 545, 'BlockName': '/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace', 'Lumis': {173658: [8, 12, 9, 14, 19, 109, 105]}, 'Parents': [], 'Checksum': '22218315', 'Adler32': 'a41a1446', 'FileSize': 286021145, 'ValidFile': 1 } """ fileDetails = self.getFileListByDataset(dataset=datasetPath, validFileOnly=validFileOnly, detail=True) blocks = set() # the set of blocks of the dataset # Iterate over the files and prepare the set of blocks and a dict where the keys are the files files = {} for f in fileDetails: blocks.add(f['block_name']) files[f['logical_file_name']] = remapDBS3Keys(f, stringify=True) files[f['logical_file_name']]['ValidFile'] = f['is_file_valid'] files[f['logical_file_name']]['Lumis'] = {} files[f['logical_file_name']]['Parents'] = [] # Iterate over the blocks and get parents and lumis for blockName in blocks: # get the parents if getParents: parents = self.dbs.listFileParents(block_name=blockName) for p in parents: if p['logical_file_name'] in files: # invalid files are not there if validFileOnly=1 files[p['logical_file_name']]['Parents'].extend(p['parent_logical_file_name']) if getLumis: # get the lumis file_lumis = self.dbs.listFileLumis(block_name=blockName) for f in file_lumis: if f['logical_file_name'] in files: # invalid files are not there if validFileOnly=1 if f['run_num'] in files[f['logical_file_name']]['Lumis']: files[f['logical_file_name']]['Lumis'][f['run_num']].extend(f['lumi_section_num']) else: files[f['logical_file_name']]['Lumis'][f['run_num']] = f['lumi_section_num'] return files def crossCheck(self, datasetPath, *lfns): """ _crossCheck_ For the dataset provided, check that the lfns listed all exist in the dataset. Return the list of lfns that are in the dataset """ allLfns = self.dbs.listFileArray(dataset=datasetPath, validFileOnly=1, detail=False) setOfAllLfns = set(allLfns) setOfKnownLfns = set(lfns) return list(setOfAllLfns.intersection(setOfKnownLfns)) def crossCheckMissing(self, datasetPath, *lfns): """ _crossCheckMissing_ As cross check, but return value is a list of files that are *not* known by DBS """ allLfns = self.dbs.listFileArray(dataset=datasetPath, validFileOnly=1, detail=False) setOfAllLfns = set(allLfns) setOfKnownLfns = set(lfns) knownFiles = setOfAllLfns.intersection(setOfKnownLfns) unknownFiles = setOfKnownLfns.difference(knownFiles) return list(unknownFiles) def getDBSSummaryInfo(self, dataset=None, block=None): """ Get dataset summary includes # of files, events, blocks and total size """ if dataset: self.checkDatasetPath(dataset) try: if block: summary = self.dbs.listFileSummaries(block_name=block, validFileOnly=1) else: summary = self.dbs.listFileSummaries(dataset=dataset, validFileOnly=1) except Exception as ex: msg = "Error in DBSReader.getDBSSummaryInfo(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if not summary: # missing data or all files invalid return {} result = remapDBS3Keys(summary[0], stringify=True) result['path'] = dataset if dataset else '' result['block'] = block if block else '' return result def getFileBlocksInfo(self, dataset, onlyClosedBlocks=False, blockName=None, locations=True): """ """ self.checkDatasetPath(dataset) args = {'dataset': dataset, 'detail': True} if blockName: args['block_name'] = blockName try: blocks = self.dbs.listBlocks(**args) except Exception as ex: msg = "Error in DBSReader.getFileBlocksInfo(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) blocks = [remapDBS3Keys(block, stringify=True, block_name='Name') for block in blocks] # only raise if blockName not specified - mimic dbs2 error handling if not blocks and not blockName: msg = "DBSReader.getFileBlocksInfo(%s, %s): No matching data" raise DBSReaderError(msg % (dataset, blockName)) if locations: for block in blocks: block['PhEDExNodeList'] = [{'Name': x} for x in self.listFileBlockLocation(block['Name'])] if onlyClosedBlocks: return [x for x in blocks if str(x['OpenForWriting']) != "1"] return blocks def listFileBlocks(self, dataset, onlyClosedBlocks=False, blockName=None): """ _listFileBlocks_ Retrieve a list of fileblock names for a dataset """ self.checkDatasetPath(dataset) args = {'dataset': dataset, 'detail': False} if blockName: args['block_name'] = blockName if onlyClosedBlocks: args['detail'] = True try: blocks = self.dbs.listBlocks(**args) except dbsClientException as ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if onlyClosedBlocks: result = [x['block_name'] for x in blocks if str(x['open_for_writing']) != "1"] else: result = [x['block_name'] for x in blocks] return result def listOpenFileBlocks(self, dataset): """ _listOpenFileBlocks_ Retrieve a list of open fileblock names for a dataset """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset=dataset, detail=True) except dbsClientException as ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['block_name'] for x in blocks if str(x['open_for_writing']) == "1"] return result def blockExists(self, fileBlockName): """ _blockExists_ Check to see if block with name provided exists in the DBS Instance. Return True if exists, False if not """ self.checkBlockName(fileBlockName) try: blocks = self.dbs.listBlocks(block_name=fileBlockName) except Exception as ex: msg = "Error in " msg += "DBSReader.blockExists(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if len(blocks) == 0: return False return True def listFilesInBlock(self, fileBlockName, lumis=True, validFileOnly=1): """ _listFilesInBlock_ Get a list of files in the named fileblock TODO: lumis can be false when lumi splitting is not required However WMBSHelper expect file['LumiList'] to get the run number so for now it will be always true. We need to clean code up when dbs2 is completely deprecated. calling lumis for run number is expensive. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: files = self.dbs.listFileArray(block_name=fileBlockName, validFileOnly=validFileOnly, detail=True) except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if lumis: lumiDict = self._getLumiList(blockName=fileBlockName, validFileOnly=validFileOnly) result = [] for fileInfo in files: if lumis: fileInfo["LumiList"] = lumiDict[fileInfo['logical_file_name']] result.append(remapDBS3Keys(fileInfo, stringify=True)) return result def listFilesInBlockWithParents(self, fileBlockName, lumis=True, validFileOnly=1): """ _listFilesInBlockWithParents_ Get a list of files in the named fileblock including the parents of that file. TODO: lumis can be false when lumi splitting is not required However WMBSHelper expect file['LumiList'] to get the run number so for now it will be always true. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlockWithParents(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: # TODO: shoud we get only valid block for this? files = self.dbs.listFileParents(block_name=fileBlockName) fileDetails = self.listFilesInBlock(fileBlockName, lumis, validFileOnly) except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % ( fileBlockName,) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) childByParents = defaultdict(list) for f in files: # Probably a child can have more than 1 parent file for fp in f['parent_logical_file_name']: childByParents[fp].append(f['logical_file_name']) parentsLFNs = childByParents.keys() if len(parentsLFNs) == 0: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n There is no parents files" % ( fileBlockName) raise DBSReaderError(msg) parentFilesDetail = [] # TODO: slicing parentLFNs util DBS api is handling that. # Remove slicing if DBS api handles for pLFNs in grouper(parentsLFNs, 50): parentFilesDetail.extend(self.dbs.listFileArray(logical_file_name=pLFNs, detail=True)) if lumis: parentLumis = self._getLumiList(lfns=parentsLFNs) parentsByLFN = defaultdict(list) for pf in parentFilesDetail: parentLFN = pf['logical_file_name'] dbsFile = remapDBS3Keys(pf, stringify=True) if lumis: dbsFile["LumiList"] = parentLumis[parentLFN] for childLFN in childByParents[parentLFN]: parentsByLFN[childLFN].append(dbsFile) for fileInfo in fileDetails: fileInfo["ParentList"] = parentsByLFN[fileInfo['logical_file_name']] return fileDetails def lfnsInBlock(self, fileBlockName): """ _lfnsInBlock_ LFN list only for block, details = False => faster query """ if not self.blockExists(fileBlockName): msg = "DBSReader.lfnsInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: lfns = self.dbs.listFileArray(block_name=fileBlockName, validFileOnly=1, detail=False) return lfns except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listFileBlockLocation(self, fileBlockNames, dbsOnly=False): """ _listFileBlockLocation_ Get origin_site_name of a block """ singleBlockName = None if isinstance(fileBlockNames, basestring): singleBlockName = fileBlockNames fileBlockNames = [fileBlockNames] for block in fileBlockNames: self.checkBlockName(block) locations = {} node_filter = set(['UNKNOWN', None]) if dbsOnly: blocksInfo = {} try: for block in fileBlockNames: blocksInfo.setdefault(block, []) # there should be only one element with a single origin site string ... for blockInfo in self.dbs.listBlockOrigin(block_name=block): blocksInfo[block].append(blockInfo['origin_site_name']) except dbsClientException as ex: msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockNames msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) else: try: blocksInfo = self.phedex.getReplicaPhEDExNodesForBlocks(block=fileBlockNames, complete='y') except Exception as ex: msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockNames msg += "%s\n" % str(ex) raise Exception(msg) for block in fileBlockNames: valid_nodes = set(blocksInfo.get(block, [])) - node_filter locations[block] = list(valid_nodes) # returning single list if a single block is passed if singleBlockName: return locations[singleBlockName] return locations def getFileBlock(self, fileBlockName, dbsOnly=False): """ _getFileBlock_ dbsOnly flag is mostly meant for StoreResults, since there is no data in TMDB. return a dictionary: { blockName: { "PhEDExNodeNames" : [<pnn list>], "Files" : { LFN : Events }, } } """ # Pointless code in python3 if isinstance(fileBlockName, str): fileBlockName = unicode(fileBlockName) if not self.blockExists(fileBlockName): msg = "DBSReader.getFileBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) result = {fileBlockName: { "PhEDExNodeNames": self.listFileBlockLocation(fileBlockName, dbsOnly), "Files": self.listFilesInBlock(fileBlockName), "IsOpen": self.blockIsOpen(fileBlockName) } } return result def getFileBlockWithParents(self, fileBlockName): """ _getFileBlockWithParents_ return a dictionary: { blockName: { "PhEDExNodeNames" : [<pnn list>], "Files" : dictionaries representing each file } } files """ if isinstance(fileBlockName, str): fileBlockName = unicode(fileBlockName) if not self.blockExists(fileBlockName): msg = "DBSReader.getFileBlockWithParents(%s): No matching data" raise DBSReaderError(msg % fileBlockName) result = {fileBlockName: { "PhEDExNodeNames": self.listFileBlockLocation(fileBlockName), "Files": self.listFilesInBlockWithParents(fileBlockName), "IsOpen": self.blockIsOpen(fileBlockName) } } return result def getFiles(self, dataset, onlyClosedBlocks=False): """ _getFiles_ Returns a dictionary of block names for the dataset where each block constists of a dictionary containing the PhEDExNodeNames for that block and the files in that block by LFN mapped to NEvents """ result = {} blocks = self.listFileBlocks(dataset, onlyClosedBlocks) for x in blocks: result.update(self.getFileBlock(x)) return result def listBlockParents(self, blockName): """Get parent blocks for block""" result = [] self.checkBlockName(blockName) blocks = self.dbs.listBlockParents(block_name=blockName) for block in blocks: toreturn = {'Name': block['parent_block_name']} toreturn['PhEDExNodeList'] = self.listFileBlockLocation(toreturn['Name']) result.append(toreturn) return result def blockIsOpen(self, blockName): """ _blockIsOpen_ Return True if named block is open, false if not, or if block doenst exist """ self.checkBlockName(blockName) blockInstance = self.dbs.listBlocks(block_name=blockName, detail=True) if len(blockInstance) == 0: return False blockInstance = blockInstance[0] isOpen = blockInstance.get('open_for_writing', 1) if isOpen == 0: return False return True def blockToDatasetPath(self, blockName): """ _blockToDatasetPath_ Given a block name, get the dataset Path associated with that Block. Returns the dataset path, or None if not found """ self.checkBlockName(blockName) try: blocks = self.dbs.listBlocks(block_name=blockName, detail=True) except Exception as ex: msg = "Error in " msg += "DBSReader.blockToDatasetPath(%s)\n" % blockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if blocks == []: return None pathname = blocks[-1].get('dataset', None) return pathname def listDatasetLocation(self, datasetName, dbsOnly=False): """ _listDatasetLocation_ List the origin SEs where there is at least a block of the given dataset. """ self.checkDatasetPath(datasetName) locations = set() if dbsOnly: try: blocksInfo = self.dbs.listBlockOrigin(dataset=datasetName) except dbsClientException as ex: msg = "Error in DBSReader: dbsApi.listBlocks(dataset=%s)\n" % datasetName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if not blocksInfo: # no data location from dbs return list() for blockInfo in blocksInfo: locations.update(blockInfo['origin_site_name']) locations.difference_update(['UNKNOWN', None]) # remove entry when SE name is 'UNKNOWN' else: try: blocksInfo = self.phedex.getReplicaPhEDExNodesForBlocks(dataset=[datasetName], complete='y') except Exception as ex: msg = "Error while getting block location from PhEDEx for dataset=%s)\n" % datasetName msg += "%s\n" % str(ex) raise Exception(msg) if blocksInfo: for blockSites in blocksInfo.values(): locations.update(blockSites) return list(locations) def checkDatasetPath(self, pathName): """ _checkDatasetPath_ """ if pathName in ("", None): raise DBSReaderError("Invalid Dataset Path name: => %s <=" % pathName) else: try: result = self.dbs.listDatasets(dataset=pathName, dataset_access_type='*') if len(result) == 0: raise DBSReaderError("Dataset %s doesn't exist in DBS %s" % (pathName, self.dbsURL)) except (dbsClientException, HTTPError) as ex: msg = "Error in " msg += "DBSReader.checkDatasetPath(%s)\n" % pathName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) return def checkBlockName(self, blockName): """ _checkBlockName_ """ if blockName in ("", "*", None): raise DBSReaderError("Invalid Block name: => %s <=" % blockName) def getFileListByDataset(self, dataset, validFileOnly=1, detail=True): """ _getFileListByDataset_ Given a dataset, retrieves all blocks, lfns and number of events (among other not really important info). Returns a list of dict. """ try: fileList = self.dbs.listFileArray(dataset=dataset, validFileOnly=validFileOnly, detail=detail) return fileList except dbsClientException as ex: msg = "Error in " msg += "DBSReader.getFileListByDataset(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listDatasetParents(self, childDataset): """ list the the parents dataset path given childDataset """ try: parentList = self.dbs.listDatasetParents(dataset=childDataset) return parentList except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listDatasetParents(%s)\n" % childDataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # def getListFilesByLumiAndDataset(self, dataset, files): # "Unsing pycurl to get all the child parents pair for given dataset" # # urls = ['%s/data/dbs/fileparentbylumis?block_name=%s' % ( # self.dbsURL, b["block_name"]) for b in self.dbs.listBlocks(dataset=dataset)] # # data = multi_getdata(urls, ckey(), cert()) # rdict = {} # for row in data: # try: # data = json.loads(row['data']) # rdict[req] = data['result'][0] # we get back {'result': [workflow]} dict # except Exception as exp: # print("ERROR: fail to load data as json record, error=%s" % str(exp)) # print(row) # return rdict def getParentFilesGivenParentDataset(self, parentDataset, childLFNs): """ returns parent files for given childLFN when DBS doesn't have direct parent child relationship in DB Only use this for finding missing parents :param parentDataset: parent dataset for childLFN :param childLFN: a file in child dataset :return: set of parent files for childLFN """ fInfo = self.dbs.listFileLumiArray(logical_file_name=childLFNs) parentFiles = defaultdict(set) for f in fInfo: pFileList = self.dbs.listFiles(dataset=parentDataset, run_num=f['run_num'], lumi_list=f['lumi_section_num']) pFiles = set([x['logical_file_name'] for x in pFileList]) parentFiles[f['logical_file_name']] = parentFiles[f['logical_file_name']].union(pFiles) return parentFiles def getParentFilesByLumi(self, childLFN): """ get the parent file's lfns by lumi (This might not be the actual parentage relations in DBS just parentage by Lumis). use for only specific lfn for validating purpose, for the parentage fix use findAndInsertMissingParentage :param childLFN: :return: list of dictionary with parent files for given child LFN and parent dataset [{"ParentDataset": /abc/bad/ddd, "ParentFiles": [alf, baf, ...]] """ childDatasets = self.dbs.listDatasets(logical_file_name=childLFN) result = [] for i in childDatasets: parents = self.dbs.listDatasetParents(dataset=i["dataset"]) for parent in parents: parentFiles = self.getParentFilesGivenParentDataset(parent['parent_dataset'], childLFN) result.append({"ParentDataset": parent['parent_dataset'], "ParentFiles": list(parentFiles)}) return result def listParentsByLumi(self, childBlockName, childLFNs=None): """ :param childBlockName: child block name :param childLFNs: list of child lfns if it is not specified, all the file in the block will be used, if specified, dbs validate child lfns from the childBlockName :return: list of list with child and parent id pair. [[1,2], [3,4]...] """ childLFNs = childLFNs or [] return self.dbs.listFileParentsByLumi(block_name=childBlockName, logical_file_name=childLFNs) def insertFileParents(self, childBlockName, childParentsIDPairs): """ :param childBlockName: child block name :param childParentsIDPairs: list of list child and parent file ids, i.e. [[1,2], [3,4]...] dbs validate child ids from the childBlockName :return: None """ return self.dbs.insertFileParents({"block_name": childBlockName, "child_parent_id_list": childParentsIDPairs}) def findAndInsertMissingParentage(self, childBlockName, childLFNs=None, insertFlag=True): """ :param childBlockName: child block name :param childLFNs: list of child lfns if it is not specified, all the file in the block will be used, if specified, dbs validate child lfns from the childBlockName :return: number of file parents pair inserted """ childLFNs = childLFNs or [] fileParents = self.dbs.listFileParentsByLumi(block_name=childBlockName, logical_file_name=childLFNs) childParentsIDPairs = fileParents[0]["child_parent_id_list"] if insertFlag: self.dbs.insertFileParents({"block_name": childBlockName, "child_parent_id_list": childParentsIDPairs}) return len(childParentsIDPairs) def listBlocksWithNoParents(self, childDataset): """ :param childDataset: child dataset for :return: set of child blocks with no parentBlock """ allBlocks = self.dbs.listBlocks(dataset=childDataset) blockNames = [] for block in allBlocks: blockNames.append(block['block_name']) parentBlocks = self.dbs.listBlockParents(block_name=blockNames) cblock = set() for pblock in parentBlocks: cblock.add(pblock['this_block_name']) noParentBlocks = set(blockNames) - cblock return noParentBlocks def listFilesWithNoParents(self, childBlockName): """ :param childBlockName: :return: """ allFiles = self.dbs.listFiles(block_name=childBlockName) parentFiles = self.dbs.listFileParents(block_name=childBlockName) allFileNames = set() for fInfo in allFiles: allFileNames.add(fInfo['logical_file_name']) cfile = set() for pFile in parentFiles: cfile.add(pFile['logical_file_name']) noParentFiles = allFileNames - cfile return list(noParentFiles) def fixMissingParentageDatasets(self, childDataset, insertFlag=True): """ :param childDataset: child dataset need to set the parentage correctly. :return: blocks which failed to insert parentage. for retry """ pDatasets = self.listDatasetParents(childDataset) # print("parent datasets %s\n" % pDatasets) # pDatasets format is # [{'this_dataset': '/SingleMuon/Run2016D-03Feb2017-v1/MINIAOD', 'parent_dataset_id': 13265209, 'parent_dataset': '/SingleMuon/Run2016D-23Sep2016-v1/AOD'}] if not pDatasets: self.logger.warning("No parent dataset found for child dataset %s", childDataset) return {} blocks = self.listBlocksWithNoParents(childDataset) failedBlocks = [] for blockName in blocks: try: numFiles = self.findAndInsertMissingParentage(blockName, insertFlag=insertFlag) self.logger.debug("%s file parentage added for block %s" % (numFiles, blockName)) except Exception as ex: self.logger.exception("Parentage updated failed for block %s", blockName) failedBlocks.append(blockName) return failedBlocks def insertMissingParentageForAllFiles(self, childDataset, filterFilesWithParents=True, insertFlag=False): """ :param childDataset: child dataset need to set the parentage correctly. :param filterFilesWithParents: if True, only select files without parents, if False all the files in the dataset :param insertFlag: if True, insert to DBS, if False just get the list of the file parentage without insert :return: blocks which failed to insert parentage. should be used for retrying """ blocks = [b['block_name'] for b in self.dbs.listBlocks(dataset=childDataset)] failedBlocks = [] print("Handling %d blocks" % len(blocks)) totalFiles = 0 for blockName in blocks: try: if filterFilesWithParents: childLFNs = self.listFilesWithNoParents(blockName) if len(childLFNs) == 0: continue else: childLFNs = [] numFiles = self.findAndInsertMissingParentage(blockName, childLFNs=childLFNs, insertFlag=insertFlag) print("%s file parentage added for block %s" % (numFiles, blockName)) totalFiles += numFiles except Exception as e: print(traceback.format_exc()) failedBlocks.append(blockName) print("Total pairs: ", totalFiles) return failedBlocks
class DBS3Reader(object): """ _DBSReader_ General API for reading data from DBS """ # cache all the datatiers known by DBS _datatiers = {} def __init__(self, url, **contact): # instantiate dbs api object try: self.dbsURL = url self.dbs = DbsApi(url, **contact) except dbsClientException as ex: msg = "Error in DBSReader with DbsApi\n" msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # connection to PhEDEx (Use default endpoint url) self.phedex = PhEDEx(responseType="json") def _getLumiList(self, blockName=None, lfns=None, validFileOnly=1): """ currently only take one lfn but dbs api need be updated """ try: if blockName: lumiLists = self.dbs.listFileLumis(block_name=blockName, validFileOnly=validFileOnly) elif lfns: lumiLists = [] for slfn in grouper(lfns, 50): lumiLists.extend(self.dbs.listFileLumiArray(logical_file_name = slfn)) else: # shouldn't call this with both blockName and lfns empty # but still returns empty dict for that case return {} except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFileLumiArray(%s)\n" % lfns msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) lumiDict = {} for lumisItem in lumiLists: lumiDict.setdefault(lumisItem['logical_file_name'], []) item = {} item["RunNumber"] = lumisItem['run_num'] item['LumiSectionNumber'] = lumisItem['lumi_section_num'] lumiDict[lumisItem['logical_file_name']].append(item) return lumiDict def checkDBSServer(self): """ check whether dbs server is up and running returns {"dbs_instance": "prod/global", "dbs_version": "3.3.144"} """ try: return self.dbs.serverinfo() except dbsClientException as ex: msg = "Error in " msg += "DBS server is not up: %s" % self.dbsURL msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listPrimaryDatasets(self, match='*'): """ _listPrimaryDatasets_ return a list of primary datasets, The full dataset name must be provided pattern based mathcing is no longer supported. If no expression is provided, all datasets are returned """ try: result = self.dbs.listPrimaryDatasets(primary_ds_name=match) except dbsClientException as ex: msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % match msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['primary_ds_name'] for x in result] return result def matchProcessedDatasets(self, primary, tier, process): """ _matchProcessedDatasets_ return a list of Processed datasets """ result = [] try: datasets = self.dbs.listDatasets(primary_ds_name=primary, data_tier_name=tier, detail=True) except dbsClientException as ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) for dataset in datasets: dataset = remapDBS3Keys(dataset, processed_ds_name='Name') dataset['PathList'] = [dataset['dataset']] if dataset['Name'] == process: result.append(dataset) return result def listRuns(self, dataset=None, block=None): """ it gets list of DbsRun object but for our purpose only list of number is collected. DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ runs = [] try: if block: results = self.dbs.listRuns(block_name=block) else: results = self.dbs.listRuns(dataset=dataset) except dbsClientException as ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) [runs.extend(x['run_num']) for x in results] return runs def listRunLumis(self, dataset=None, block=None): """ It gets a list of DBSRun objects and returns the number of lumisections per run DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ # Pointless code in python3 if isinstance(block, str): block = unicode(block) if isinstance(dataset, str): dataset = unicode(dataset) try: if block: results = self.dbs.listRuns(block_name=block) else: results = self.dbs.listRuns(dataset=dataset) except dbsClientException as ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # send runDict format as result, this format is for sync with dbs2 call # which has {run_number: num_lumis} but dbs3 call doesn't return num Lumis # So it returns {run_number: None} # TODO: After DBS2 is completely removed change the return format more sensible one runDict = {} for x in results: for runNumber in x["run_num"]: runDict[runNumber] = None return runDict def listProcessedDatasets(self, primary, dataTier='*'): """ _listProcessedDatasets_ return a list of Processed datasets for the primary and optional data tier value """ try: result = self.dbs.listDatasets(primary_ds_name=primary, data_tier_name=dataTier) except dbsClientException as ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['dataset'].split('/')[2] for x in result] return result def listDatasetFiles(self, datasetPath): """ _listDatasetFiles_ Get list of files for dataset """ return [x['logical_file_name'] for x in self.dbs.listFileArray(dataset=datasetPath)] @staticmethod def listDatatiers(dbsUrl=None): """ _listDatatiers_ Get a list of datatiers known by DBS. """ if dbsUrl is None: msg = "Error in DBSReader.listDatatiers(). DBS Url not set." raise DBSReaderError(msg) timenow = int(time.time()) if DBS3Reader._datatiers and timenow - 7200 < DBS3Reader._datatiers['ts']: return DBS3Reader._datatiers['tiers'] try: DBS3Reader._setDatatiersCache(timenow, dbsUrl) except Exception as ex: if not DBS3Reader._datatiers: msg = "Error in DBSReader.listDatatiers\n%s" % formatEx3(ex) raise DBSReaderError(msg) return DBS3Reader._datatiers['tiers'] @staticmethod def _setDatatiersCache(ts, dbsUrl): """ Set a timestamp and update the list of datatiers cached in the class property """ dbs = DbsApi(dbsUrl) DBS3Reader._datatiers['ts'] = ts DBS3Reader._datatiers['tiers'] = [tier['data_tier_name'] for tier in dbs.listDataTiers()] return def listDatasetFileDetails(self, datasetPath, getParents=False, validFileOnly=1): """ TODO: This is completely wrong need to be redone. or be removed - getting dataset altogether might be to costly _listDatasetFileDetails_ Get list of lumis, events, and parents for each file in a dataset Return a dict where the keys are the files, and for each file we have something like: { 'NumberOfEvents': 545, 'BlockName': '/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace', 'Lumis': {173658: [8, 12, 9, 14, 19, 109, 105]}, 'Parents': [], 'Checksum': '22218315', 'Adler32': 'a41a1446', 'FileSize': 286021145, 'ValidFile': 1 } """ fileDetails = self.getFileListByDataset(dataset=datasetPath, validFileOnly=validFileOnly, detail=True) blocks = set() # the set of blocks of the dataset # Iterate over the files and prepare the set of blocks and a dict where the keys are the files files = {} for f in fileDetails: blocks.add(f['block_name']) files[f['logical_file_name']] = remapDBS3Keys(f, stringify=True) files[f['logical_file_name']]['ValidFile'] = f['is_file_valid'] files[f['logical_file_name']]['Lumis'] = {} files[f['logical_file_name']]['Parents'] = [] # Iterate over the blocks and get parents and lumis for blockName in blocks: # get the parents if getParents: parents = self.dbs.listFileParents(block_name=blockName) for p in parents: if p['logical_file_name'] in files: # invalid files are not there if validFileOnly=1 files[p['logical_file_name']]['Parents'].extend(p['parent_logical_file_name']) # get the lumis file_lumis = self.dbs.listFileLumis(block_name=blockName) for f in file_lumis: if f['logical_file_name'] in files: # invalid files are not there if validFileOnly=1 if f['run_num'] in files[f['logical_file_name']]['Lumis']: files[f['logical_file_name']]['Lumis'][f['run_num']].extend(f['lumi_section_num']) else: files[f['logical_file_name']]['Lumis'][f['run_num']] = f['lumi_section_num'] return files def crossCheck(self, datasetPath, *lfns): """ _crossCheck_ For the dataset provided, check that the lfns listed all exist in the dataset. Return the list of lfns that are in the dataset """ allLfns = self.dbs.listFileArray(dataset=datasetPath, validFileOnly=1, detail=False) setOfAllLfns = set(allLfns) setOfKnownLfns = set(lfns) return list(setOfAllLfns.intersection(setOfKnownLfns)) def crossCheckMissing(self, datasetPath, *lfns): """ _crossCheckMissing_ As cross check, but return value is a list of files that are *not* known by DBS """ allLfns = self.dbs.listFileArray(dataset=datasetPath, validFileOnly=1, detail=False) setOfAllLfns = set(allLfns) setOfKnownLfns = set(lfns) knownFiles = setOfAllLfns.intersection(setOfKnownLfns) unknownFiles = setOfKnownLfns.difference(knownFiles) return list(unknownFiles) def getDBSSummaryInfo(self, dataset=None, block=None): """ Get dataset summary includes # of files, events, blocks and total size """ # FIXME: Doesnt raise exceptions on missing data as old api did if dataset: self.checkDatasetPath(dataset) try: if block: summary = self.dbs.listFileSummaries(block_name=block, validFileOnly=1) else: # dataset case dataset shouldn't be None summary = self.dbs.listFileSummaries(dataset=dataset, validFileOnly=1) except Exception as ex: msg = "Error in DBSReader.getDBSSummaryInfo(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if not summary or summary[0].get('file_size') is None: # appears to indicate missing dataset msg = "DBSReader.listDatasetSummary(%s, %s): No matching data" raise DBSReaderError(msg % (dataset, block)) result = remapDBS3Keys(summary[0], stringify=True) result['path'] = dataset if dataset else '' result['block'] = block if block else '' return result def getFileBlocksInfo(self, dataset, onlyClosedBlocks=False, blockName=None, locations=True): """ """ self.checkDatasetPath(dataset) args = {'dataset': dataset, 'detail': True} if blockName: args['block_name'] = blockName try: blocks = self.dbs.listBlocks(**args) except Exception as ex: msg = "Error in DBSReader.getFileBlocksInfo(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) blocks = [remapDBS3Keys(block, stringify=True, block_name='Name') for block in blocks] # only raise if blockName not specified - mimic dbs2 error handling if not blocks and not blockName: msg = "DBSReader.getFileBlocksInfo(%s, %s): No matching data" raise DBSReaderError(msg % (dataset, blockName)) if locations: for block in blocks: block['PhEDExNodeList'] = [{'Name': x} for x in self.listFileBlockLocation(block['Name'])] if onlyClosedBlocks: return [x for x in blocks if str(x['OpenForWriting']) != "1"] return blocks def listFileBlocks(self, dataset, onlyClosedBlocks=False, blockName=None): """ _listFileBlocks_ Retrieve a list of fileblock names for a dataset """ self.checkDatasetPath(dataset) args = {'dataset': dataset, 'detail': False} if blockName: args['block_name'] = blockName if onlyClosedBlocks: args['detail'] = True try: blocks = self.dbs.listBlocks(**args) except dbsClientException as ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if onlyClosedBlocks: result = [x['block_name'] for x in blocks if str(x['open_for_writing']) != "1"] else: result = [x['block_name'] for x in blocks] return result def listOpenFileBlocks(self, dataset): """ _listOpenFileBlocks_ Retrieve a list of open fileblock names for a dataset """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset=dataset, detail=True) except dbsClientException as ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['block_name'] for x in blocks if str(x['open_for_writing']) == "1"] return result def blockExists(self, fileBlockName): """ _blockExists_ Check to see if block with name provided exists in the DBS Instance. Return True if exists, False if not """ self.checkBlockName(fileBlockName) try: blocks = self.dbs.listBlocks(block_name=fileBlockName) except Exception as ex: msg = "Error in " msg += "DBSReader.blockExists(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if len(blocks) == 0: return False return True def listFilesInBlock(self, fileBlockName, lumis=True, validFileOnly=1): """ _listFilesInBlock_ Get a list of files in the named fileblock TODO: lumis can be false when lumi splitting is not required However WMBSHelper expect file['LumiList'] to get the run number so for now it will be always true. We need to clean code up when dbs2 is completely deprecated. calling lumis for run number is expensive. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: files = self.dbs.listFileArray(block_name=fileBlockName, validFileOnly=validFileOnly, detail=True) except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if lumis: lumiDict = self._getLumiList(blockName=fileBlockName, validFileOnly=validFileOnly) result = [] for fileInfo in files: if lumis: fileInfo["LumiList"] = lumiDict[fileInfo['logical_file_name']] result.append(remapDBS3Keys(fileInfo, stringify=True)) return result def listFilesInBlockWithParents(self, fileBlockName, lumis=True, validFileOnly=1): """ _listFilesInBlockWithParents_ Get a list of files in the named fileblock including the parents of that file. TODO: lumis can be false when lumi splitting is not required However WMBSHelper expect file['LumiList'] to get the run number so for now it will be always true. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlockWithParents(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: # TODO: shoud we get only valid block for this? files = self.dbs.listFileParents(block_name=fileBlockName) fileDetails = self.listFilesInBlock(fileBlockName, lumis, validFileOnly) except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % ( fileBlockName,) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) childByParents = defaultdict(list) for f in files: # Probably a child can have more than 1 parent file for fp in f['parent_logical_file_name']: childByParents[fp].append(f['logical_file_name']) parentsLFNs = childByParents.keys() if len(parentsLFNs) == 0: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n There is no parents files" % ( fileBlockName) raise DBSReaderError(msg) parentFilesDetail = [] # TODO: slicing parentLFNs util DBS api is handling that. # Remove slicing if DBS api handles for pLFNs in grouper(parentsLFNs, 50): parentFilesDetail.extend(self.dbs.listFileArray(logical_file_name=pLFNs, detail=True)) if lumis: parentLumis = self._getLumiList(lfns=parentsLFNs) parentsByLFN = defaultdict(list) for pf in parentFilesDetail: parentLFN = pf['logical_file_name'] dbsFile = remapDBS3Keys(pf, stringify=True) if lumis: dbsFile["LumiList"] = parentLumis[parentLFN] for childLFN in childByParents[parentLFN]: parentsByLFN[childLFN].append(dbsFile) for fileInfo in fileDetails: fileInfo["ParentList"] = parentsByLFN[fileInfo['logical_file_name']] return fileDetails def lfnsInBlock(self, fileBlockName): """ _lfnsInBlock_ LFN list only for block, details = False => faster query """ if not self.blockExists(fileBlockName): msg = "DBSReader.lfnsInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: lfns = self.dbs.listFileArray(block_name=fileBlockName, validFileOnly=1, detail=False) return lfns except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listFileBlockLocation(self, fileBlockNames, dbsOnly=False): """ _listFileBlockLocation_ Get origin_site_name of a block """ singleBlockName = None if isinstance(fileBlockNames, basestring): singleBlockName = fileBlockNames fileBlockNames = [fileBlockNames] for block in fileBlockNames: self.checkBlockName(block) locations = {} node_filter = set(['UNKNOWN', None]) if dbsOnly: blocksInfo = {} try: for block in fileBlockNames: blocksInfo.setdefault(block, []) # there should be only one element with a single origin site string ... for blockInfo in self.dbs.listBlockOrigin(block_name=block): blocksInfo[block].append(blockInfo['origin_site_name']) except dbsClientException as ex: msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockNames msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) else: try: blocksInfo = self.phedex.getReplicaPhEDExNodesForBlocks(block=fileBlockNames, complete='y') except Exception as ex: msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockNames msg += "%s\n" % str(ex) raise Exception(msg) for block in fileBlockNames: valid_nodes = set(blocksInfo.get(block, [])) - node_filter locations[block] = list(valid_nodes) # returning single list if a single block is passed if singleBlockName: return locations[singleBlockName] return locations def getFileBlock(self, fileBlockName, dbsOnly=False): """ _getFileBlock_ dbsOnly flag is mostly meant for StoreResults, since there is no data in TMDB. return a dictionary: { blockName: { "PhEDExNodeNames" : [<pnn list>], "Files" : { LFN : Events }, } } """ # Pointless code in python3 if isinstance(fileBlockName, str): fileBlockName = unicode(fileBlockName) if not self.blockExists(fileBlockName): msg = "DBSReader.getFileBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) result = {fileBlockName: { "PhEDExNodeNames": self.listFileBlockLocation(fileBlockName, dbsOnly), "Files": self.listFilesInBlock(fileBlockName), "IsOpen": self.blockIsOpen(fileBlockName) } } return result def getFileBlockWithParents(self, fileBlockName): """ _getFileBlockWithParents_ return a dictionary: { blockName: { "PhEDExNodeNames" : [<pnn list>], "Files" : dictionaries representing each file } } files """ if isinstance(fileBlockName, str): fileBlockName = unicode(fileBlockName) if not self.blockExists(fileBlockName): msg = "DBSReader.getFileBlockWithParents(%s): No matching data" raise DBSReaderError(msg % fileBlockName) result = {fileBlockName: { "PhEDExNodeNames": self.listFileBlockLocation(fileBlockName), "Files": self.listFilesInBlockWithParents(fileBlockName), "IsOpen": self.blockIsOpen(fileBlockName) } } return result def getFiles(self, dataset, onlyClosedBlocks=False): """ _getFiles_ Returns a dictionary of block names for the dataset where each block constists of a dictionary containing the PhEDExNodeNames for that block and the files in that block by LFN mapped to NEvents """ result = {} blocks = self.listFileBlocks(dataset, onlyClosedBlocks) [result.update(self.getFileBlock(x)) for x in blocks] return result def listBlockParents(self, blockName): """Get parent blocks for block""" result = [] self.checkBlockName(blockName) blocks = self.dbs.listBlockParents(block_name=blockName) for block in blocks: toreturn = {'Name': block['parent_block_name']} toreturn['PhEDExNodeList'] = self.listFileBlockLocation(toreturn['Name']) result.append(toreturn) return result def blockIsOpen(self, blockName): """ _blockIsOpen_ Return True if named block is open, false if not, or if block doenst exist """ self.checkBlockName(blockName) blockInstance = self.dbs.listBlocks(block_name=blockName, detail=True) if len(blockInstance) == 0: return False blockInstance = blockInstance[0] isOpen = blockInstance.get('open_for_writing', 1) if isOpen == 0: return False return True def blockToDatasetPath(self, blockName): """ _blockToDatasetPath_ Given a block name, get the dataset Path associated with that Block. Returns the dataset path, or None if not found """ self.checkBlockName(blockName) try: blocks = self.dbs.listBlocks(block_name=blockName, detail=True) except Exception as ex: msg = "Error in " msg += "DBSReader.blockToDatasetPath(%s)\n" % blockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if blocks == []: return None pathname = blocks[-1].get('dataset', None) return pathname def listDatasetLocation(self, datasetName, dbsOnly=False): """ _listDatasetLocation_ List the origin SEs where there is at least a block of the given dataset. """ self.checkDatasetPath(datasetName) locations = set() if dbsOnly: try: blocksInfo = self.dbs.listBlockOrigin(dataset=datasetName) except dbsClientException as ex: msg = "Error in DBSReader: dbsApi.listBlocks(dataset=%s)\n" % datasetName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if not blocksInfo: # no data location from dbs return list() for blockInfo in blocksInfo: locations.update(blockInfo['origin_site_name']) locations.difference_update(['UNKNOWN', None]) # remove entry when SE name is 'UNKNOWN' else: try: blocksInfo = self.phedex.getReplicaPhEDExNodesForBlocks(dataset=[datasetName], complete='y') except Exception as ex: msg = "Error while getting block location from PhEDEx for dataset=%s)\n" % datasetName msg += "%s\n" % str(ex) raise Exception(msg) if blocksInfo: for blockSites in blocksInfo.values(): locations.update(blockSites) return list(locations) def checkDatasetPath(self, pathName): """ _checkDatasetPath_ """ if pathName in ("", None): raise DBSReaderError("Invalid Dataset Path name: => %s <=" % pathName) else: try: result = self.dbs.listDatasets(dataset=pathName, dataset_access_type='*') if len(result) == 0: raise DBSReaderError("Dataset %s doesn't exist in DBS %s" % (pathName, self.dbsURL)) except (dbsClientException, HTTPError) as ex: msg = "Error in " msg += "DBSReader.checkDatasetPath(%s)\n" % pathName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) return def checkBlockName(self, blockName): """ _checkBlockName_ """ if blockName in ("", "*", None): raise DBSReaderError("Invalid Block name: => %s <=" % blockName) def getFileListByDataset(self, dataset, validFileOnly=1, detail=True): """ _getFileListByDataset_ Given a dataset, retrieves all blocks, lfns and number of events (among other not really important info). Returns a list of dict. """ try: fileList = self.dbs.listFileArray(dataset=dataset, validFileOnly=validFileOnly, detail=detail) return fileList except dbsClientException as ex: msg = "Error in " msg += "DBSReader.getFileListByDataset(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listDatasetParents(self, childDataset): """ list the the parents dataset path given childDataset """ try: parentList = self.dbs.listDatasetParents(dataset=childDataset) return parentList except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listDatasetParents(%s)\n" % childDataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg)
class DBS3Reader(object): """ _DBSReader_ General API for reading data from DBS """ # cache all the datatiers known by DBS _datatiers = {} def __init__(self, url, logger=None, **contact): # instantiate dbs api object try: self.dbsURL = url self.dbs = DbsApi(url, **contact) self.logger = logger or logging.getLogger(self.__class__.__name__) except dbsClientException as ex: msg = "Error in DBSReader with DbsApi\n" msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # connection to PhEDEx (Use default endpoint url) self.phedex = PhEDEx(responseType="json", dbsUrl=self.dbsURL) def _getLumiList(self, blockName=None, lfns=None, validFileOnly=1): """ currently only take one lfn but dbs api need be updated """ try: if blockName: lumiLists = self.dbs.listFileLumis(block_name=blockName, validFileOnly=validFileOnly) elif lfns: lumiLists = [] for slfn in grouper(lfns, 50): lumiLists.extend(self.dbs.listFileLumiArray(logical_file_name=slfn)) else: # shouldn't call this with both blockName and lfns empty # but still returns empty dict for that case return {} except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFileLumiArray(%s)\n" % lfns msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) lumiDict = {} for lumisItem in lumiLists: lumiDict.setdefault(lumisItem['logical_file_name'], []) item = {} item["RunNumber"] = lumisItem['run_num'] item['LumiSectionNumber'] = lumisItem['lumi_section_num'] if lumisItem.get('event_count', None) is not None: item['EventCount'] = lumisItem['event_count'] lumiDict[lumisItem['logical_file_name']].append(item) # TODO: add key for lumi and event pair. return lumiDict def checkDBSServer(self): """ check whether dbs server is up and running returns {"dbs_instance": "prod/global", "dbs_version": "3.3.144"} """ try: return self.dbs.serverinfo() except dbsClientException as ex: msg = "Error in " msg += "DBS server is not up: %s" % self.dbsURL msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listPrimaryDatasets(self, match='*'): """ _listPrimaryDatasets_ return a list of primary datasets, The full dataset name must be provided pattern based mathcing is no longer supported. If no expression is provided, all datasets are returned """ try: result = self.dbs.listPrimaryDatasets(primary_ds_name=match) except dbsClientException as ex: msg = "Error in DBSReader.listPrimaryDataset(%s)\n" % match msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['primary_ds_name'] for x in result] return result def matchProcessedDatasets(self, primary, tier, process): """ _matchProcessedDatasets_ return a list of Processed datasets """ result = [] try: datasets = self.dbs.listDatasets(primary_ds_name=primary, data_tier_name=tier, detail=True) except dbsClientException as ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) for dataset in datasets: dataset = remapDBS3Keys(dataset, processed_ds_name='Name') dataset['PathList'] = [dataset['dataset']] if dataset['Name'] == process: result.append(dataset) return result def listRuns(self, dataset=None, block=None): """ it gets list of DbsRun object but for our purpose only list of number is collected. DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ runs = [] try: if block: results = self.dbs.listRuns(block_name=block) else: results = self.dbs.listRuns(dataset=dataset) except dbsClientException as ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) for x in results: runs.extend(x['run_num']) return runs def listRunLumis(self, dataset=None, block=None): """ It gets a list of DBSRun objects and returns the number of lumisections per run DbsRun (RunNumber, NumberOfEvents, NumberOfLumiSections, TotalLuminosity, StoreNumber, StartOfRungetLong, EndOfRun, CreationDate, CreatedBy, LastModificationDate, LastModifiedBy ) """ # Pointless code in python3 if isinstance(block, str): block = unicode(block) if isinstance(dataset, str): dataset = unicode(dataset) try: if block: results = self.dbs.listRuns(block_name=block) else: results = self.dbs.listRuns(dataset=dataset) except dbsClientException as ex: msg = "Error in DBSReader.listRuns(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # send runDict format as result, this format is for sync with dbs2 call # which has {run_number: num_lumis} but dbs3 call doesn't return num Lumis # So it returns {run_number: None} # TODO: After DBS2 is completely removed change the return format more sensible one runDict = {} for x in results: for runNumber in x["run_num"]: runDict[runNumber] = None return runDict def listProcessedDatasets(self, primary, dataTier='*'): """ _listProcessedDatasets_ return a list of Processed datasets for the primary and optional data tier value """ try: result = self.dbs.listDatasets(primary_ds_name=primary, data_tier_name=dataTier) except dbsClientException as ex: msg = "Error in DBSReader.listProcessedDatasets(%s)\n" % primary msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['dataset'].split('/')[2] for x in result] return result def listDatasetFiles(self, datasetPath): """ _listDatasetFiles_ Get list of files for dataset """ return [x['logical_file_name'] for x in self.dbs.listFileArray(dataset=datasetPath)] @staticmethod def listDatatiers(dbsUrl=None): """ _listDatatiers_ Get a list of datatiers known by DBS. """ if dbsUrl is None: msg = "Error in DBSReader.listDatatiers(). DBS Url not set." raise DBSReaderError(msg) timenow = int(time.time()) if DBS3Reader._datatiers and timenow - 7200 < DBS3Reader._datatiers['ts']: return DBS3Reader._datatiers['tiers'] try: DBS3Reader._setDatatiersCache(timenow, dbsUrl) except Exception as ex: if not DBS3Reader._datatiers: msg = "Error in DBSReader.listDatatiers\n%s" % formatEx3(ex) raise DBSReaderError(msg) return DBS3Reader._datatiers['tiers'] @staticmethod def _setDatatiersCache(ts, dbsUrl): """ Set a timestamp and update the list of datatiers cached in the class property """ dbs = DbsApi(dbsUrl) DBS3Reader._datatiers['ts'] = ts DBS3Reader._datatiers['tiers'] = [tier['data_tier_name'] for tier in dbs.listDataTiers()] return def listDatasetFileDetails(self, datasetPath, getParents=False, getLumis=True, validFileOnly=1): """ TODO: This is completely wrong need to be redone. or be removed - getting dataset altogether might be to costly _listDatasetFileDetails_ Get list of lumis, events, and parents for each file in a dataset Return a dict where the keys are the files, and for each file we have something like: { 'NumberOfEvents': 545, 'BlockName': '/HighPileUp/Run2011A-v1/RAW#dd6e0796-cbcc-11e0-80a9-003048caaace', 'Lumis': {173658: [8, 12, 9, 14, 19, 109, 105]}, 'Parents': [], 'Checksum': '22218315', 'Adler32': 'a41a1446', 'FileSize': 286021145, 'ValidFile': 1 } """ fileDetails = self.getFileListByDataset(dataset=datasetPath, validFileOnly=validFileOnly, detail=True) blocks = set() # the set of blocks of the dataset # Iterate over the files and prepare the set of blocks and a dict where the keys are the files files = {} for f in fileDetails: blocks.add(f['block_name']) files[f['logical_file_name']] = remapDBS3Keys(f, stringify=True) files[f['logical_file_name']]['ValidFile'] = f['is_file_valid'] files[f['logical_file_name']]['Lumis'] = {} files[f['logical_file_name']]['Parents'] = [] # Iterate over the blocks and get parents and lumis for blockName in blocks: # get the parents if getParents: parents = self.dbs.listFileParents(block_name=blockName) for p in parents: if p['logical_file_name'] in files: # invalid files are not there if validFileOnly=1 files[p['logical_file_name']]['Parents'].extend(p['parent_logical_file_name']) if getLumis: # get the lumis file_lumis = self.dbs.listFileLumis(block_name=blockName) for f in file_lumis: if f['logical_file_name'] in files: # invalid files are not there if validFileOnly=1 if f['run_num'] in files[f['logical_file_name']]['Lumis']: files[f['logical_file_name']]['Lumis'][f['run_num']].extend(f['lumi_section_num']) else: files[f['logical_file_name']]['Lumis'][f['run_num']] = f['lumi_section_num'] return files def crossCheck(self, datasetPath, *lfns): """ _crossCheck_ For the dataset provided, check that the lfns listed all exist in the dataset. Return the list of lfns that are in the dataset """ allLfns = self.dbs.listFileArray(dataset=datasetPath, validFileOnly=1, detail=False) setOfAllLfns = set(allLfns) setOfKnownLfns = set(lfns) return list(setOfAllLfns.intersection(setOfKnownLfns)) def crossCheckMissing(self, datasetPath, *lfns): """ _crossCheckMissing_ As cross check, but return value is a list of files that are *not* known by DBS """ allLfns = self.dbs.listFileArray(dataset=datasetPath, validFileOnly=1, detail=False) setOfAllLfns = set(allLfns) setOfKnownLfns = set(lfns) knownFiles = setOfAllLfns.intersection(setOfKnownLfns) unknownFiles = setOfKnownLfns.difference(knownFiles) return list(unknownFiles) def getDBSSummaryInfo(self, dataset=None, block=None): """ Get dataset summary includes # of files, events, blocks and total size """ if dataset: self.checkDatasetPath(dataset) try: if block: summary = self.dbs.listFileSummaries(block_name=block, validFileOnly=1) else: summary = self.dbs.listFileSummaries(dataset=dataset, validFileOnly=1) except Exception as ex: msg = "Error in DBSReader.getDBSSummaryInfo(%s, %s)\n" % (dataset, block) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if not summary: # missing data or all files invalid return {} result = remapDBS3Keys(summary[0], stringify=True) result['path'] = dataset if dataset else '' result['block'] = block if block else '' return result def getFileBlocksInfo(self, dataset, onlyClosedBlocks=False, blockName=None, locations=True): """ """ self.checkDatasetPath(dataset) args = {'dataset': dataset, 'detail': True} if blockName: args['block_name'] = blockName try: blocks = self.dbs.listBlocks(**args) except Exception as ex: msg = "Error in DBSReader.getFileBlocksInfo(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) blocks = [remapDBS3Keys(block, stringify=True, block_name='Name') for block in blocks] # only raise if blockName not specified - mimic dbs2 error handling if not blocks and not blockName: msg = "DBSReader.getFileBlocksInfo(%s, %s): No matching data" raise DBSReaderError(msg % (dataset, blockName)) if locations: for block in blocks: block['PhEDExNodeList'] = [{'Name': x} for x in self.listFileBlockLocation(block['Name'])] if onlyClosedBlocks: return [x for x in blocks if str(x['OpenForWriting']) != "1"] return blocks def listFileBlocks(self, dataset, onlyClosedBlocks=False, blockName=None): """ _listFileBlocks_ Retrieve a list of fileblock names for a dataset """ self.checkDatasetPath(dataset) args = {'dataset': dataset, 'detail': False} if blockName: args['block_name'] = blockName if onlyClosedBlocks: args['detail'] = True try: blocks = self.dbs.listBlocks(**args) except dbsClientException as ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if onlyClosedBlocks: result = [x['block_name'] for x in blocks if str(x['open_for_writing']) != "1"] else: result = [x['block_name'] for x in blocks] return result def listOpenFileBlocks(self, dataset): """ _listOpenFileBlocks_ Retrieve a list of open fileblock names for a dataset """ self.checkDatasetPath(dataset) try: blocks = self.dbs.listBlocks(dataset=dataset, detail=True) except dbsClientException as ex: msg = "Error in DBSReader.listFileBlocks(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) result = [x['block_name'] for x in blocks if str(x['open_for_writing']) == "1"] return result def blockExists(self, fileBlockName): """ _blockExists_ Check to see if block with name provided exists in the DBS Instance. Return True if exists, False if not """ self.checkBlockName(fileBlockName) try: blocks = self.dbs.listBlocks(block_name=fileBlockName) except Exception as ex: msg = "Error in " msg += "DBSReader.blockExists(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if len(blocks) == 0: return False return True def listFilesInBlock(self, fileBlockName, lumis=True, validFileOnly=1): """ _listFilesInBlock_ Get a list of files in the named fileblock TODO: lumis can be false when lumi splitting is not required However WMBSHelper expect file['LumiList'] to get the run number so for now it will be always true. We need to clean code up when dbs2 is completely deprecated. calling lumis for run number is expensive. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: files = self.dbs.listFileArray(block_name=fileBlockName, validFileOnly=validFileOnly, detail=True) except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if lumis: lumiDict = self._getLumiList(blockName=fileBlockName, validFileOnly=validFileOnly) result = [] for fileInfo in files: if lumis: fileInfo["LumiList"] = lumiDict[fileInfo['logical_file_name']] result.append(remapDBS3Keys(fileInfo, stringify=True)) return result def listFilesInBlockWithParents(self, fileBlockName, lumis=True, validFileOnly=1): """ _listFilesInBlockWithParents_ Get a list of files in the named fileblock including the parents of that file. TODO: lumis can be false when lumi splitting is not required However WMBSHelper expect file['LumiList'] to get the run number so for now it will be always true. """ if not self.blockExists(fileBlockName): msg = "DBSReader.listFilesInBlockWithParents(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: # TODO: shoud we get only valid block for this? files = self.dbs.listFileParents(block_name=fileBlockName) fileDetails = self.listFilesInBlock(fileBlockName, lumis, validFileOnly) except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n" % ( fileBlockName,) msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) childByParents = defaultdict(list) for f in files: # Probably a child can have more than 1 parent file for fp in f['parent_logical_file_name']: childByParents[fp].append(f['logical_file_name']) parentsLFNs = childByParents.keys() if len(parentsLFNs) == 0: msg = "Error in " msg += "DBSReader.listFilesInBlockWithParents(%s)\n There is no parents files" % ( fileBlockName) raise DBSReaderError(msg) parentFilesDetail = [] # TODO: slicing parentLFNs util DBS api is handling that. # Remove slicing if DBS api handles for pLFNs in grouper(parentsLFNs, 50): parentFilesDetail.extend(self.dbs.listFileArray(logical_file_name=pLFNs, detail=True)) if lumis: parentLumis = self._getLumiList(lfns=parentsLFNs) parentsByLFN = defaultdict(list) for pf in parentFilesDetail: parentLFN = pf['logical_file_name'] dbsFile = remapDBS3Keys(pf, stringify=True) if lumis: dbsFile["LumiList"] = parentLumis[parentLFN] for childLFN in childByParents[parentLFN]: parentsByLFN[childLFN].append(dbsFile) for fileInfo in fileDetails: fileInfo["ParentList"] = parentsByLFN[fileInfo['logical_file_name']] return fileDetails def lfnsInBlock(self, fileBlockName): """ _lfnsInBlock_ LFN list only for block, details = False => faster query """ if not self.blockExists(fileBlockName): msg = "DBSReader.lfnsInBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) try: lfns = self.dbs.listFileArray(block_name=fileBlockName, validFileOnly=1, detail=False) return lfns except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listFilesInBlock(%s)\n" % fileBlockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listFileBlockLocation(self, fileBlockNames, dbsOnly=False): """ _listFileBlockLocation_ Get origin_site_name of a block """ singleBlockName = None if isinstance(fileBlockNames, basestring): singleBlockName = fileBlockNames fileBlockNames = [fileBlockNames] for block in fileBlockNames: self.checkBlockName(block) locations = {} node_filter = set(['UNKNOWN', None]) if dbsOnly: blocksInfo = {} try: for block in fileBlockNames: blocksInfo.setdefault(block, []) # there should be only one element with a single origin site string ... for blockInfo in self.dbs.listBlockOrigin(block_name=block): blocksInfo[block].append(blockInfo['origin_site_name']) except dbsClientException as ex: msg = "Error in DBS3Reader: self.dbs.listBlockOrigin(block_name=%s)\n" % fileBlockNames msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) else: try: blocksInfo = self.phedex.getReplicaPhEDExNodesForBlocks(block=fileBlockNames, complete='y') except Exception as ex: msg = "Error while getting block location from PhEDEx for block_name=%s)\n" % fileBlockNames msg += "%s\n" % str(ex) raise Exception(msg) for block in fileBlockNames: valid_nodes = set(blocksInfo.get(block, [])) - node_filter locations[block] = list(valid_nodes) # returning single list if a single block is passed if singleBlockName: return locations[singleBlockName] return locations def getFileBlock(self, fileBlockName, dbsOnly=False): """ _getFileBlock_ dbsOnly flag is mostly meant for StoreResults, since there is no data in TMDB. return a dictionary: { blockName: { "PhEDExNodeNames" : [<pnn list>], "Files" : { LFN : Events }, } } """ # Pointless code in python3 if isinstance(fileBlockName, str): fileBlockName = unicode(fileBlockName) if not self.blockExists(fileBlockName): msg = "DBSReader.getFileBlock(%s): No matching data" raise DBSReaderError(msg % fileBlockName) result = {fileBlockName: { "PhEDExNodeNames": self.listFileBlockLocation(fileBlockName, dbsOnly), "Files": self.listFilesInBlock(fileBlockName), "IsOpen": self.blockIsOpen(fileBlockName) } } return result def getFileBlockWithParents(self, fileBlockName): """ _getFileBlockWithParents_ return a dictionary: { blockName: { "PhEDExNodeNames" : [<pnn list>], "Files" : dictionaries representing each file } } files """ if isinstance(fileBlockName, str): fileBlockName = unicode(fileBlockName) if not self.blockExists(fileBlockName): msg = "DBSReader.getFileBlockWithParents(%s): No matching data" raise DBSReaderError(msg % fileBlockName) result = {fileBlockName: { "PhEDExNodeNames": self.listFileBlockLocation(fileBlockName), "Files": self.listFilesInBlockWithParents(fileBlockName), "IsOpen": self.blockIsOpen(fileBlockName) } } return result def getFiles(self, dataset, onlyClosedBlocks=False): """ _getFiles_ Returns a dictionary of block names for the dataset where each block constists of a dictionary containing the PhEDExNodeNames for that block and the files in that block by LFN mapped to NEvents """ result = {} blocks = self.listFileBlocks(dataset, onlyClosedBlocks) for x in blocks: result.update(self.getFileBlock(x)) return result def listBlockParents(self, blockName): """Get parent blocks for block""" result = [] self.checkBlockName(blockName) blocks = self.dbs.listBlockParents(block_name=blockName) for block in blocks: toreturn = {'Name': block['parent_block_name']} toreturn['PhEDExNodeList'] = self.listFileBlockLocation(toreturn['Name']) result.append(toreturn) return result def blockIsOpen(self, blockName): """ _blockIsOpen_ Return True if named block is open, false if not, or if block doenst exist """ self.checkBlockName(blockName) blockInstance = self.dbs.listBlocks(block_name=blockName, detail=True) if len(blockInstance) == 0: return False blockInstance = blockInstance[0] isOpen = blockInstance.get('open_for_writing', 1) if isOpen == 0: return False return True def blockToDatasetPath(self, blockName): """ _blockToDatasetPath_ Given a block name, get the dataset Path associated with that Block. Returns the dataset path, or None if not found """ self.checkBlockName(blockName) try: blocks = self.dbs.listBlocks(block_name=blockName, detail=True) except Exception as ex: msg = "Error in " msg += "DBSReader.blockToDatasetPath(%s)\n" % blockName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if blocks == []: return None pathname = blocks[-1].get('dataset', None) return pathname def listDatasetLocation(self, datasetName, dbsOnly=False): """ _listDatasetLocation_ List the origin SEs where there is at least a block of the given dataset. """ self.checkDatasetPath(datasetName) locations = set() if dbsOnly: try: blocksInfo = self.dbs.listBlockOrigin(dataset=datasetName) except dbsClientException as ex: msg = "Error in DBSReader: dbsApi.listBlocks(dataset=%s)\n" % datasetName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) if not blocksInfo: # no data location from dbs return list() for blockInfo in blocksInfo: locations.update(blockInfo['origin_site_name']) locations.difference_update(['UNKNOWN', None]) # remove entry when SE name is 'UNKNOWN' else: try: blocksInfo = self.phedex.getReplicaPhEDExNodesForBlocks(dataset=[datasetName], complete='y') except Exception as ex: msg = "Error while getting block location from PhEDEx for dataset=%s)\n" % datasetName msg += "%s\n" % str(ex) raise Exception(msg) if blocksInfo: for blockSites in blocksInfo.values(): locations.update(blockSites) return list(locations) def checkDatasetPath(self, pathName): """ _checkDatasetPath_ """ if pathName in ("", None): raise DBSReaderError("Invalid Dataset Path name: => %s <=" % pathName) else: try: result = self.dbs.listDatasets(dataset=pathName, dataset_access_type='*') if len(result) == 0: raise DBSReaderError("Dataset %s doesn't exist in DBS %s" % (pathName, self.dbsURL)) except (dbsClientException, HTTPError) as ex: msg = "Error in " msg += "DBSReader.checkDatasetPath(%s)\n" % pathName msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) return def checkBlockName(self, blockName): """ _checkBlockName_ """ if blockName in ("", "*", None): raise DBSReaderError("Invalid Block name: => %s <=" % blockName) def getFileListByDataset(self, dataset, validFileOnly=1, detail=True): """ _getFileListByDataset_ Given a dataset, retrieves all blocks, lfns and number of events (among other not really important info). Returns a list of dict. """ try: fileList = self.dbs.listFileArray(dataset=dataset, validFileOnly=validFileOnly, detail=detail) return fileList except dbsClientException as ex: msg = "Error in " msg += "DBSReader.getFileListByDataset(%s)\n" % dataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) def listDatasetParents(self, childDataset): """ list the the parents dataset path given childDataset """ try: parentList = self.dbs.listDatasetParents(dataset=childDataset) return parentList except dbsClientException as ex: msg = "Error in " msg += "DBSReader.listDatasetParents(%s)\n" % childDataset msg += "%s\n" % formatEx3(ex) raise DBSReaderError(msg) # def getListFilesByLumiAndDataset(self, dataset, files): # "Unsing pycurl to get all the child parents pair for given dataset" # # urls = ['%s/data/dbs/fileparentbylumis?block_name=%s' % ( # self.dbsURL, b["block_name"]) for b in self.dbs.listBlocks(dataset=dataset)] # # data = multi_getdata(urls, ckey(), cert()) # rdict = {} # for row in data: # try: # data = json.loads(row['data']) # rdict[req] = data['result'][0] # we get back {'result': [workflow]} dict # except Exception as exp: # print("ERROR: fail to load data as json record, error=%s" % str(exp)) # print(row) # return rdict def getParentFilesGivenParentDataset(self, parentDataset, childLFNs): """ returns parent files for given childLFN when DBS doesn't have direct parent child relationship in DB Only use this for finding missing parents :param parentDataset: parent dataset for childLFN :param childLFN: a file in child dataset :return: set of parent files for childLFN """ fInfo = self.dbs.listFileLumiArray(logical_file_name=childLFNs) parentFiles = defaultdict(set) for f in fInfo: pFileList = self.dbs.listFiles(dataset=parentDataset, run_num=f['run_num'], lumi_list=f['lumi_section_num']) pFiles = set([x['logical_file_name'] for x in pFileList]) parentFiles[f['logical_file_name']] = parentFiles[f['logical_file_name']].union(pFiles) return parentFiles def getParentFilesByLumi(self, childLFN): """ get the parent file's lfns by lumi (This might not be the actual parentage relations in DBS just parentage by Lumis). use for only specific lfn for validating purpose, for the parentage fix use findAndInsertMissingParentage :param childLFN: :return: list of dictionary with parent files for given child LFN and parent dataset [{"ParentDataset": /abc/bad/ddd, "ParentFiles": [alf, baf, ...]] """ childDatasets = self.dbs.listDatasets(logical_file_name=childLFN) result = [] for i in childDatasets: parents = self.dbs.listDatasetParents(dataset=i["dataset"]) for parent in parents: parentFiles = self.getParentFilesGivenParentDataset(parent['parent_dataset'], childLFN) result.append({"ParentDataset": parent['parent_dataset'], "ParentFiles": list(parentFiles)}) return result def listParentsByLumi(self, childBlockName, childLFNs=None): """ :param childBlockName: child block name :param childLFNs: list of child lfns if it is not specified, all the file in the block will be used, if specified, dbs validate child lfns from the childBlockName :return: list of list with child and parent id pair. [[1,2], [3,4]...] """ childLFNs = childLFNs or [] return self.dbs.listFileParentsByLumi(block_name=childBlockName, logical_file_name=childLFNs) def insertFileParents(self, childBlockName, childParentsIDPairs): """ :param childBlockName: child block name :param childParentsIDPairs: list of list child and parent file ids, i.e. [[1,2], [3,4]...] dbs validate child ids from the childBlockName :return: None """ return self.dbs.insertFileParents({"block_name": childBlockName, "child_parent_id_list": childParentsIDPairs}) def findAndInsertMissingParentage(self, childBlockName, childLFNs=None, insertFlag=True): """ :param childBlockName: child block name :param childLFNs: list of child lfns if it is not specified, all the file in the block will be used, if specified, dbs validate child lfns from the childBlockName :return: number of file parents pair inserted """ childLFNs = childLFNs or [] fileParents = self.dbs.listFileParentsByLumi(block_name=childBlockName, logical_file_name=childLFNs) childParentsIDPairs = fileParents[0]["child_parent_id_list"] if insertFlag: self.dbs.insertFileParents({"block_name": childBlockName, "child_parent_id_list": childParentsIDPairs}) return len(childParentsIDPairs) def listBlocksWithNoParents(self, childDataset): """ :param childDataset: child dataset for :return: set of child blocks with no parentBlock """ allBlocks = self.dbs.listBlocks(dataset=childDataset) blockNames = [] for block in allBlocks: blockNames.append(block['block_name']) parentBlocks = self.dbs.listBlockParents(block_name=blockNames) cblock = set() for pblock in parentBlocks: cblock.add(pblock['this_block_name']) noParentBlocks = set(blockNames) - cblock return noParentBlocks def listFilesWithNoParents(self, childBlockName): """ :param childBlockName: :return: """ allFiles = self.dbs.listFiles(block_name=childBlockName) parentFiles = self.dbs.listFileParents(block_name=childBlockName) allFileNames = set() for fInfo in allFiles: allFileNames.add(fInfo['logical_file_name']) cfile = set() for pFile in parentFiles: cfile.add(pFile['logical_file_name']) noParentFiles = allFileNames - cfile return list(noParentFiles) def fixMissingParentageDatasets(self, childDataset, insertFlag=True): """ :param childDataset: child dataset need to set the parentage correctly. :return: blocks which failed to insert parentage. for retry """ pDatasets = self.listDatasetParents(childDataset) # print("parent datasets %s\n" % pDatasets) # pDatasets format is # [{'this_dataset': '/SingleMuon/Run2016D-03Feb2017-v1/MINIAOD', 'parent_dataset_id': 13265209, 'parent_dataset': '/SingleMuon/Run2016D-23Sep2016-v1/AOD'}] if not pDatasets: self.logger.warning("No parent dataset found for child dataset %s", childDataset) return {} blocks = self.listBlocksWithNoParents(childDataset) failedBlocks = [] for blockName in blocks: try: numFiles = self.findAndInsertMissingParentage(blockName, insertFlag=insertFlag) self.logger.debug("%s file parentage added for block %s" % (numFiles, blockName)) except Exception as ex: self.logger.exception("Parentage updated failed for block %s", blockName) failedBlocks.append(blockName) return failedBlocks def insertMissingParentageForAllFiles(self, childDataset, filterFilesWithParents=True, insertFlag=False): """ :param childDataset: child dataset need to set the parentage correctly. :param filterFilesWithParents: if True, only select files without parents, if False all the files in the dataset :param insertFlag: if True, insert to DBS, if False just get the list of the file parentage without insert :return: blocks which failed to insert parentage. should be used for retrying """ blocks = [b['block_name'] for b in self.dbs.listBlocks(dataset=childDataset)] failedBlocks = [] print("Handling %d blocks" % len(blocks)) totalFiles = 0 for blockName in blocks: try: if filterFilesWithParents: childLFNs = self.listFilesWithNoParents(blockName) if len(childLFNs) == 0: continue else: childLFNs = [] numFiles = self.findAndInsertMissingParentage(blockName, childLFNs=childLFNs, insertFlag=insertFlag) print("%s file parentage added for block %s" % (numFiles, blockName)) totalFiles += numFiles except Exception as e: print(traceback.format_exc()) failedBlocks.append(blockName) print("Total pairs: ", totalFiles) return failedBlocks
class PileupFetcher(FetcherInterface): """ Pull dataset block/SE : LFN list from DBS for the pileup datasets required by the steps in the job. Save these maps as files in the sandbox """ def __init__(self): """ Prepare module setup """ super(PileupFetcher, self).__init__() if usingRucio(): # Too much work to pass the rucio account name all the way to here # just use the production rucio account for resolving pileup location self.rucio = Rucio("wma_prod", configDict={'phedexCompatible': False}) else: self.phedex = PhEDEx() # this will go away eventually def _queryDbsAndGetPileupConfig(self, stepHelper, dbsReader): """ Method iterates over components of the pileup configuration input and queries DBS. Then iterates over results from DBS. There needs to be a list of files and their locations for each dataset name. Use dbsReader the result data structure is a Python dict following dictionary: FileList is a list of LFNs {"pileupTypeA": {"BlockA": {"FileList": [], "PhEDExNodeNames": []}, "BlockB": {"FileList": [], "PhEDExNodeName": []}, ....} this structure preserves knowledge of where particular files of dataset are physically (list of PNNs) located. DBS only lists sites which have all files belonging to blocks but e.g. BlockA of dataset DS1 may be located at site1 and BlockB only at site2 - it's possible that only a subset of the blocks in a dataset will be at a site. """ resultDict = {} # iterate over input pileup types (e.g. "cosmics", "minbias") for pileupType in stepHelper.data.pileup.listSections_(): # the format here is: step.data.pileup.cosmics.dataset = [/some/data/set] datasets = getattr(getattr(stepHelper.data.pileup, pileupType), "dataset") # each dataset input can generally be a list, iterate over dataset names blockDict = {} for dataset in datasets: blockFileInfo = dbsReader.getFileListByDataset(dataset=dataset, detail=True) for fileInfo in blockFileInfo: blockDict.setdefault(fileInfo['block_name'], { 'FileList': [], 'NumberOfEvents': 0, 'PhEDExNodeNames': [] }) blockDict[fileInfo['block_name']]['FileList'].append( {'logical_file_name': fileInfo['logical_file_name']}) blockDict[fileInfo['block_name']][ 'NumberOfEvents'] += fileInfo['event_count'] self._getDatasetLocation(dataset, blockDict) resultDict[pileupType] = blockDict return resultDict def _getDatasetLocation(self, dset, blockDict): """ Given a dataset name, query PhEDEx or Rucio and resolve the block location :param dset: string with the dataset name :param blockDict: dictionary with DBS summary info :return: update blockDict in place """ node_filter = set(['UNKNOWN', None]) if hasattr(self, "rucio"): # then it's Rucio!! blockReplicasInfo = self.rucio.getReplicaInfoForBlocks( dataset=dset) for item in blockReplicasInfo: block = item['name'] try: blockDict[block]['PhEDExNodeNames'] = item['replica'] blockDict[block]['FileList'] = sorted( blockDict[block]['FileList']) except KeyError: logging.warning( "Block '%s' does not have any complete Rucio replica", block) else: blockReplicasInfo = self.phedex.getReplicaPhEDExNodesForBlocks( dataset=dset, complete='y') for block in blockReplicasInfo: nodes = set(blockReplicasInfo[block]) - node_filter try: blockDict[block]['PhEDExNodeNames'] = list(nodes) blockDict[block]['FileList'] = sorted( blockDict[block]['FileList']) except KeyError: logging.warning( "Block '%s' does not have any complete PhEDEx replica", block) def _getCacheFilePath(self, stepHelper): fileName = "" for pileupType in stepHelper.data.pileup.listSections_(): datasets = getattr(getattr(stepHelper.data.pileup, pileupType), "dataset") fileName += ("_").join(datasets) # TODO cache is not very effective if the dataset combination is different between workflow # here is possibility of hash value collision cacheFile = "%s/pileupconf-%s.json" % (self.cacheDirectory(), hash(fileName)) return cacheFile def _getStepFilePath(self, stepHelper): stepPath = "%s/%s" % (self.workingDirectory(), stepHelper.name()) fileName = "%s/%s" % (stepPath, "pileupconf.json") return fileName def _writeFile(self, filePath, jsonPU): directory = filePath.rsplit('/', 1)[0] if not os.path.exists(directory): os.mkdir(directory) try: with open(filePath, 'w') as f: f.write(jsonPU) except IOError: m = "Could not save pileup JSON configuration file: '%s'" % filePath raise RuntimeError(m) def _copyFile(self, src, dest): directory = dest.rsplit('/', 1)[0] if not os.path.exists(directory): os.mkdir(directory) shutil.copyfile(src, dest) def _isCacheExpired(self, cacheFilePath, delta=24): """Is the cache expired? At delta hours (default 24) in the future. """ # cache can either be a file name or an already opened file object if not os.path.exists(cacheFilePath): return True delta = datetime.timedelta(hours=delta) t = datetime.datetime.now() - delta # cache file mtime has been set to cache expiry time if os.path.getmtime(cacheFilePath) < time.mktime(t.timetuple()): return True return False def _isCacheValid(self, stepHelper): """ Check whether cache is exits TODO: if the cacheDirectory is not inside the Sandbox it should not autormatically deleted. We can add cache refresh policy here """ cacheFile = self._getCacheFilePath(stepHelper) if not self._isCacheExpired( cacheFile, delta=0.5) and os.path.getsize(cacheFile) > 0: # if file already exist don't make a new dbs call and overwrite the file. # just return fileName = self._getStepFilePath(stepHelper) if not os.path.isfile(fileName) or os.path.getsize( fileName) != os.path.getsize(cacheFile): self._copyFile(cacheFile, fileName) return True else: return False def _saveFile(self, stepHelper, jsonPU): cacheFile = self._getCacheFilePath(stepHelper) self._writeFile(cacheFile, jsonPU) fileName = self._getStepFilePath(stepHelper) self._copyFile(cacheFile, fileName) def createPileupConfigFile(self, helper): """ Stores pileup JSON configuration file in the working directory / sandbox. """ if self._isCacheValid(helper): # if file already exist don't make a new dbs call and overwrite the file. # just return return encoder = JSONEncoder() # this should have been set in CMSSWStepHelper along with # the pileup configuration url = helper.data.dbsUrl dbsReader = DBSReader(url) configDict = self._queryDbsAndGetPileupConfig(helper, dbsReader) # create JSON and save into a file jsonPU = encoder.encode(configDict) self._saveFile(helper, jsonPU) def __call__(self, wmTask): """ Method is called when WorkQueue creates the sandbox for a job. Need to look at the pileup configuration in the spec and query dbs to determine the lfns for the files in the datasets and what sites they're located at (WQ creates the job sandbox). wmTask is instance of WMTask.WMTaskHelper """ for step in wmTask.steps().nodeIterator(): helper = WMStep.WMStepHelper(step) # returns e.g. instance of CMSSWHelper # doesn't seem to be necessary ... strangely (some inheritance involved?) # typeHelper = helper.getTypeHelper() if hasattr(helper.data, "pileup"): self.createPileupConfigFile(helper)
class WMBSHelperTest(EmulatedUnitTestCase): def setUp(self): """ _setUp_ """ super(WMBSHelperTest, self).setUp() self.testInit = TestInitCouchApp(__file__) self.testInit.setLogging() self.testInit.setDatabaseConnection(destroyAllDatabase=True) self.testInit.setupCouch("wmbshelper_t/jobs", "JobDump") self.testInit.setupCouch("wmbshelper_t/fwjrs", "FWJRDump") self.testInit.setupCouch("config_test", "GroupUser", "ConfigCache") os.environ["COUCHDB"] = "wmbshelper_t" self.testInit.setSchema(customModules=[ "WMCore.WMBS", "WMComponent.DBS3Buffer", "WMCore.BossAir", "WMCore.ResourceControl" ], useDefault=False) self.workDir = self.testInit.generateWorkDir() self.wmspec = self.createWMSpec() self.topLevelTask = getFirstTask(self.wmspec) self.inputDataset = self.topLevelTask.inputDataset() self.dataset = self.topLevelTask.getInputDatasetPath() self.dbs = DBSReader(self.inputDataset.dbsurl) self.phedex = PhEDEx() self.daoFactory = DAOFactory(package="WMCore.WMBS", logger=threading.currentThread().logger, dbinterface=threading.currentThread().dbi) self.configFile = EmulatorSetup.setupWMAgentConfig() self.config = loadConfigurationFile(self.configFile) self.config.component_("JobSubmitter") self.config.JobSubmitter.submitDir = self.workDir self.config.JobSubmitter.submitScript = os.path.join( getTestBase(), 'WMComponent_t/JobSubmitter_t', 'submit.sh') return def tearDown(self): """ _tearDown_ Clear out the database. """ self.testInit.clearDatabase() self.testInit.tearDownCouch() self.testInit.delWorkDir() EmulatorSetup.deleteConfig(self.configFile) super(WMBSHelperTest, self).tearDown() return def setupForKillTest(self, baAPI=None): """ _setupForKillTest_ Inject a workflow into WMBS that has a processing task, a merge task and a cleanup task. Inject files into the various tasks at various processing states (acquired, complete, available...). Also create jobs for each subscription in various states. """ myThread = threading.currentThread() daoFactory = DAOFactory(package="WMCore.WMBS", logger=myThread.logger, dbinterface=myThread.dbi) dummyLocationAction = daoFactory(classname="Locations.New") changeStateAction = daoFactory(classname="Jobs.ChangeState") resourceControl = ResourceControl() resourceControl.insertSite(siteName='site1', pnn='goodse.cern.ch', ceName='site1', plugin="TestPlugin") resourceControl.insertThreshold(siteName='site1', taskType='Processing', \ maxSlots=10000, pendingSlots=10000) userDN = 'someDN' userAction = daoFactory(classname="Users.New") userAction.execute(dn=userDN, group_name='DEFAULT', role_name='DEFAULT') inputFileset = Fileset("input") inputFileset.create() inputFileA = File("lfnA", locations="goodse.cern.ch") inputFileB = File("lfnB", locations="goodse.cern.ch") inputFileC = File("lfnC", locations="goodse.cern.ch") inputFileA.create() inputFileB.create() inputFileC.create() inputFileset.addFile(inputFileA) inputFileset.addFile(inputFileB) inputFileset.addFile(inputFileC) inputFileset.commit() unmergedOutputFileset = Fileset("unmerged") unmergedOutputFileset.create() unmergedFileA = File("ulfnA", locations="goodse.cern.ch") unmergedFileB = File("ulfnB", locations="goodse.cern.ch") unmergedFileC = File("ulfnC", locations="goodse.cern.ch") unmergedFileA.create() unmergedFileB.create() unmergedFileC.create() unmergedOutputFileset.addFile(unmergedFileA) unmergedOutputFileset.addFile(unmergedFileB) unmergedOutputFileset.addFile(unmergedFileC) unmergedOutputFileset.commit() mainProcWorkflow = Workflow(spec="spec1", owner="Steve", name="Main", task="Proc") mainProcWorkflow.create() mainProcMergeWorkflow = Workflow(spec="spec1", owner="Steve", name="Main", task="ProcMerge") mainProcMergeWorkflow.create() mainCleanupWorkflow = Workflow(spec="spec1", owner="Steve", name="Main", task="Cleanup") mainCleanupWorkflow.create() self.mainProcSub = Subscription(fileset=inputFileset, workflow=mainProcWorkflow, type="Processing") self.mainProcSub.create() self.mainProcSub.acquireFiles(inputFileA) self.mainProcSub.completeFiles(inputFileB) procJobGroup = JobGroup(subscription=self.mainProcSub) procJobGroup.create() self.procJobA = Job(name="ProcJobA") self.procJobA["state"] = "new" self.procJobA["location"] = "site1" self.procJobB = Job(name="ProcJobB") self.procJobB["state"] = "executing" self.procJobB["location"] = "site1" self.procJobC = Job(name="ProcJobC") self.procJobC["state"] = "complete" self.procJobC["location"] = "site1" self.procJobA.create(procJobGroup) self.procJobB.create(procJobGroup) self.procJobC.create(procJobGroup) self.mainMergeSub = Subscription(fileset=unmergedOutputFileset, workflow=mainProcMergeWorkflow, type="Merge") self.mainMergeSub.create() self.mainMergeSub.acquireFiles(unmergedFileA) self.mainMergeSub.failFiles(unmergedFileB) mergeJobGroup = JobGroup(subscription=self.mainMergeSub) mergeJobGroup.create() self.mergeJobA = Job(name="MergeJobA") self.mergeJobA["state"] = "exhausted" self.mergeJobA["location"] = "site1" self.mergeJobB = Job(name="MergeJobB") self.mergeJobB["state"] = "cleanout" self.mergeJobB["location"] = "site1" self.mergeJobC = Job(name="MergeJobC") self.mergeJobC["state"] = "new" self.mergeJobC["location"] = "site1" self.mergeJobA.create(mergeJobGroup) self.mergeJobB.create(mergeJobGroup) self.mergeJobC.create(mergeJobGroup) self.mainCleanupSub = Subscription(fileset=unmergedOutputFileset, workflow=mainCleanupWorkflow, type="Cleanup") self.mainCleanupSub.create() self.mainCleanupSub.acquireFiles(unmergedFileA) self.mainCleanupSub.completeFiles(unmergedFileB) cleanupJobGroup = JobGroup(subscription=self.mainCleanupSub) cleanupJobGroup.create() self.cleanupJobA = Job(name="CleanupJobA") self.cleanupJobA["state"] = "new" self.cleanupJobA["location"] = "site1" self.cleanupJobB = Job(name="CleanupJobB") self.cleanupJobB["state"] = "executing" self.cleanupJobB["location"] = "site1" self.cleanupJobC = Job(name="CleanupJobC") self.cleanupJobC["state"] = "complete" self.cleanupJobC["location"] = "site1" self.cleanupJobA.create(cleanupJobGroup) self.cleanupJobB.create(cleanupJobGroup) self.cleanupJobC.create(cleanupJobGroup) jobList = [ self.procJobA, self.procJobB, self.procJobC, self.mergeJobA, self.mergeJobB, self.mergeJobC, self.cleanupJobA, self.cleanupJobB, self.cleanupJobC ] changeStateAction.execute(jobList) if baAPI: for job in jobList: job['plugin'] = 'TestPlugin' job['userdn'] = userDN job['usergroup'] = 'DEFAULT' job['userrole'] = 'DEFAULT' job['custom']['location'] = 'site1' baAPI.createNewJobs(wmbsJobs=jobList) # We'll create an unrelated workflow to verify that it isn't affected # by the killing code. bogusFileset = Fileset("dontkillme") bogusFileset.create() bogusFileA = File("bogus/lfnA", locations="goodse.cern.ch") bogusFileA.create() bogusFileset.addFile(bogusFileA) bogusFileset.commit() bogusWorkflow = Workflow(spec="spec2", owner="Steve", name="Bogus", task="Proc") bogusWorkflow.create() self.bogusSub = Subscription(fileset=bogusFileset, workflow=bogusWorkflow, type="Processing") self.bogusSub.create() self.bogusSub.acquireFiles(bogusFileA) return def verifyFileKillStatus(self): """ _verifyFileKillStatus_ Verify that all files were killed correctly. The status of files in Cleanup and LogCollect subscriptions isn't modified. Status of already completed and failed files is not modified. Also verify that the bogus subscription is untouched. """ failedFiles = self.mainProcSub.filesOfStatus("Failed") acquiredFiles = self.mainProcSub.filesOfStatus("Acquired") completedFiles = self.mainProcSub.filesOfStatus("Completed") availableFiles = self.mainProcSub.filesOfStatus("Available") bogusAcquiredFiles = self.bogusSub.filesOfStatus("Acquired") self.assertEqual(len(availableFiles), 0, \ "Error: There should be no available files.") self.assertEqual(len(acquiredFiles), 0, \ "Error: There should be no acquired files.") self.assertEqual(len(bogusAcquiredFiles), 1, \ "Error: There should be one acquired file.") self.assertEqual(len(completedFiles), 3, \ "Error: There should be only one completed file.") goldenLFNs = ["lfnA", "lfnB", "lfnC"] for completedFile in completedFiles: self.assertTrue(completedFile["lfn"] in goldenLFNs, \ "Error: Extra completed file.") goldenLFNs.remove(completedFile["lfn"]) self.assertEqual(len(failedFiles), 0, \ "Error: There should be no failed files.") self.assertEqual(len(goldenLFNs), 0, \ "Error: Missing LFN") failedFiles = self.mainMergeSub.filesOfStatus("Failed") acquiredFiles = self.mainMergeSub.filesOfStatus("Acquired") completedFiles = self.mainMergeSub.filesOfStatus("Completed") availableFiles = self.mainMergeSub.filesOfStatus("Available") self.assertEqual(len(acquiredFiles), 0, \ "Error: Merge subscription should have 0 acq files.") self.assertEqual(len(availableFiles), 0, \ "Error: Merge subscription should have 0 avail files.") self.assertEqual(len(failedFiles), 1, \ "Error: Merge subscription should have 1 failed files.") self.assertEqual( list(failedFiles)[0]["lfn"], "ulfnB", "Error: Wrong failed file.") self.assertEqual(len(completedFiles), 2, \ "Error: Merge subscription should have 2 compl files.") goldenLFNs = ["ulfnA", "ulfnC"] for completedFile in completedFiles: self.assertTrue(completedFile["lfn"] in goldenLFNs, \ "Error: Extra complete file.") goldenLFNs.remove(completedFile["lfn"]) self.assertEqual(len(goldenLFNs), 0, \ "Error: Missing LFN") failedFiles = self.mainCleanupSub.filesOfStatus("Failed") acquiredFiles = self.mainCleanupSub.filesOfStatus("Acquired") completedFiles = self.mainCleanupSub.filesOfStatus("Completed") availableFiles = self.mainCleanupSub.filesOfStatus("Available") self.assertEqual(len(failedFiles), 0, \ "Error: Cleanup subscription should have 0 fai files.") self.assertEqual(len(acquiredFiles), 1, \ "Error: There should be only one acquired file.") self.assertEqual(list(acquiredFiles)[0]["lfn"], "ulfnA", \ "Error: Wrong acquired LFN.") self.assertEqual(len(completedFiles), 1, \ "Error: There should be only one completed file.") self.assertEqual(list(completedFiles)[0]["lfn"], "ulfnB", \ "Error: Wrong completed LFN.") self.assertEqual(len(availableFiles), 1, \ "Error: There should be only one available file.") self.assertEqual(list(availableFiles)[0]["lfn"], "ulfnC", \ "Error: Wrong completed LFN.") return def verifyJobKillStatus(self): """ _verifyJobKillStatus_ Verify that jobs are killed correctly. Jobs belonging to Cleanup and LogCollect subscriptions are not killed. The status of jobs that have already finished running is not changed. """ self.procJobA.load() self.procJobB.load() self.procJobC.load() self.assertEqual(self.procJobA["state"], "killed", \ "Error: Proc job A should be killed.") self.assertEqual(self.procJobB["state"], "killed", \ "Error: Proc job B should be killed.") self.assertEqual(self.procJobC["state"], "complete", \ "Error: Proc job C should be complete.") self.mergeJobA.load() self.mergeJobB.load() self.mergeJobC.load() self.assertEqual(self.mergeJobA["state"], "exhausted", \ "Error: Merge job A should be exhausted.") self.assertEqual(self.mergeJobB["state"], "cleanout", \ "Error: Merge job B should be cleanout.") self.assertEqual(self.mergeJobC["state"], "killed", \ "Error: Merge job C should be killed.") self.cleanupJobA.load() self.cleanupJobB.load() self.cleanupJobC.load() self.assertEqual(self.cleanupJobA["state"], "new", \ "Error: Cleanup job A should be new.") self.assertEqual(self.cleanupJobB["state"], "executing", \ "Error: Cleanup job B should be executing.") self.assertEqual(self.cleanupJobC["state"], "complete", \ "Error: Cleanup job C should be complete.") return def createTestWMSpec(self): """ _createTestWMSpec_ Create a WMSpec that has a processing, merge, cleanup and skims tasks that can be used by the subscription creation test. """ testWorkload = WMWorkloadHelper(WMWorkload("TestWorkload")) testWorkload.setDashboardActivity("TestReReco") testWorkload.setSpecUrl("/path/to/workload") testWorkload.setOwnerDetails("sfoulkes", "DMWM", {'dn': 'MyDN'}) procTask = testWorkload.newTask("ProcessingTask") procTask.setTaskType("Processing") procTask.setSplittingAlgorithm("FileBased", files_per_job=1) procTaskCMSSW = procTask.makeStep("cmsRun1") procTaskCMSSW.setStepType("CMSSW") procTaskCMSSWHelper = procTaskCMSSW.getTypeHelper() procTask.setTaskType("Processing") procTask.setSiteWhitelist(["site1"]) procTask.setSiteBlacklist(["site2"]) procTask.applyTemplates() procTaskCMSSWHelper.addOutputModule("OutputA", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) mergeTask = procTask.addTask("MergeTask") mergeTask.setInputReference(procTaskCMSSW, outputModule="OutputA", dataTier='DataTierA') mergeTask.setTaskType("Merge") mergeTask.setSplittingAlgorithm("WMBSMergeBySize", min_merge_size=1, max_merge_size=2, max_merge_events=3) mergeTaskCMSSW = mergeTask.makeStep("cmsRun1") mergeTaskCMSSW.setStepType("CMSSW") mergeTaskCMSSWHelper = mergeTaskCMSSW.getTypeHelper() mergeTask.setTaskType("Merge") mergeTask.applyTemplates() mergeTaskCMSSWHelper.addOutputModule("Merged", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) cleanupTask = procTask.addTask("CleanupTask") cleanupTask.setInputReference(procTaskCMSSW, outputModule="OutputA", dataTier="DataTierA") cleanupTask.setTaskType("Merge") cleanupTask.setSplittingAlgorithm("SiblingProcessingBased", files_per_job=50) cleanupTaskCMSSW = cleanupTask.makeStep("cmsRun1") cleanupTaskCMSSW.setStepType("CMSSW") cleanupTask.setTaskType("Cleanup") cleanupTask.applyTemplates() skimTask = mergeTask.addTask("SkimTask") skimTask.setTaskType("Skim") skimTask.setInputReference(mergeTaskCMSSW, outputModule="Merged", dataTier="DataTierA") skimTask.setSplittingAlgorithm("FileBased", files_per_job=1, include_parents=True) skimTaskCMSSW = skimTask.makeStep("cmsRun1") skimTaskCMSSW.setStepType("CMSSW") skimTaskCMSSWHelper = skimTaskCMSSW.getTypeHelper() skimTask.setTaskType("Skim") skimTask.applyTemplates() skimTaskCMSSWHelper.addOutputModule("SkimOutputA", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierA", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) skimTaskCMSSWHelper.addOutputModule("SkimOutputB", primaryDataset="bogusPrimary", processedDataset="bogusProcessed", dataTier="DataTierB", lfnBase="bogusUnmerged", mergedLFNBase="bogusMerged", filterName=None) return testWorkload def setupMCWMSpec(self): """Setup MC workflow""" self.wmspec = self.createMCWMSpec() self.topLevelTask = getFirstTask(self.wmspec) self.inputDataset = self.topLevelTask.inputDataset() self.dataset = self.topLevelTask.getInputDatasetPath() self.dbs = None # add sites that would normally be added by operator via resource_control locationDAO = self.daoFactory(classname="Locations.New") self.pnns = [] for site in ['T2_XX_SiteA', 'T2_XX_SiteB']: locationDAO.execute(siteName=site, pnn=site) self.pnns.append(site) def createWMSpec(self, name='ReRecoWorkload'): factory = ReRecoWorkloadFactory() rerecoArgs["ConfigCacheID"] = createConfig(rerecoArgs["CouchDBName"]) wmspec = factory.factoryWorkloadConstruction(name, rerecoArgs) wmspec.setSpecUrl("/path/to/workload") wmspec.setSubscriptionInformation(custodialSites=[], nonCustodialSites=[], autoApproveSites=[], priority="Low", custodialSubType="Move") return wmspec def createMCWMSpec(self, name='MonteCarloWorkload'): mcArgs = TaskChainWorkloadFactory.getTestArguments() mcArgs["CouchDBName"] = rerecoArgs["CouchDBName"] mcArgs["Task1"]["ConfigCacheID"] = createConfig(mcArgs["CouchDBName"]) wmspec = taskChainWorkload(name, mcArgs) wmspec.setSpecUrl("/path/to/workload") getFirstTask(wmspec).addProduction(totalevents=10000) return wmspec def getDBS(self, wmspec): topLevelTask = getFirstTask(wmspec) inputDataset = topLevelTask.inputDataset() dbs = DBSReader(inputDataset.dbsurl) # dbsDict = {self.inputDataset.dbsurl : self.dbs} return dbs def createWMBSHelperWithTopTask(self, wmspec, block, mask=None, parentFlag=False, detail=False, commonLocation=None): topLevelTask = getFirstTask(wmspec) wmbs = WMBSHelper(wmspec, topLevelTask.name(), block, mask, cachepath=self.workDir, commonLocation=commonLocation) if block: blockName = block if parentFlag: block = self.dbs.getFileBlockWithParents(blockName) location = self.phedex.getReplicaPhEDExNodesForBlocks( block=[blockName], complete='y') block['PhEDExNodeNames'] = location[blockName] else: block = self.dbs.getFileBlock(blockName) location = self.phedex.getReplicaPhEDExNodesForBlocks( block=[blockName], complete='y') block['PhEDExNodeNames'] = location[blockName] sub, files = wmbs.createSubscriptionAndAddFiles(block=block) if detail: return wmbs, sub, files else: return wmbs def testKillWorkflow(self): """ _testKillWorkflow_ Verify that workflow killing works correctly. """ baAPI = BossAirAPI(config=self.config, insertStates=True) # Create nine jobs self.setupForKillTest(baAPI=baAPI) self.assertEqual(len(baAPI._listRunJobs()), 9) killWorkflow("Main", self.config, self.config) self.verifyFileKillStatus() self.verifyJobKillStatus() self.assertEqual(len(baAPI._listRunJobs()), 8) return def testCreateSubscription(self): """ _testCreateSubscription_ Verify that the subscription creation code works correctly. """ resourceControl = ResourceControl() resourceControl.insertSite(siteName='site1', pnn='goodse.cern.ch', ceName='site1', plugin="TestPlugin") resourceControl.insertSite(siteName='site2', pnn='goodse2.cern.ch', ceName='site2', plugin="TestPlugin") testWorkload = self.createTestWMSpec() testTopLevelTask = getFirstTask(testWorkload) testWMBSHelper = WMBSHelper(testWorkload, testTopLevelTask.name(), "SomeBlock", cachepath=self.workDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testTopLevelTask, testWMBSHelper.topLevelFileset) procWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/ProcessingTask") procWorkflow.load() self.assertEqual(procWorkflow.owner, "sfoulkes", "Error: Wrong owner: %s" % procWorkflow.owner) self.assertEqual(procWorkflow.group, "DMWM", "Error: Wrong group: %s" % procWorkflow.group) self.assertEqual(procWorkflow.wfType, "TestReReco", "Error: Wrong type.") self.assertEqual( procWorkflow.spec, os.path.join(self.workDir, procWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(procWorkflow.outputMap), 1, "Error: Wrong number of WF outputs.") mergedProcOutput = procWorkflow.outputMap["OutputADataTierA"][0][ "merged_output_fileset"] unmergedProcOutput = procWorkflow.outputMap["OutputADataTierA"][0][ "output_fileset"] mergedProcOutput.loadData() unmergedProcOutput.loadData() self.assertEqual( mergedProcOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-MergedDataTierA", "Error: Merged output fileset is wrong.") self.assertEqual( unmergedProcOutput.name, "/TestWorkload/ProcessingTask/unmerged-OutputADataTierA", "Error: Unmerged output fileset is wrong.") mergeWorkflow = Workflow(name="TestWorkload", task="/TestWorkload/ProcessingTask/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap), 1, "Error: Wrong number of WF outputs.") cleanupWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/ProcessingTask/CleanupTask") cleanupWorkflow.load() self.assertEqual(cleanupWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( cleanupWorkflow.spec, os.path.join(self.workDir, cleanupWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(cleanupWorkflow.outputMap), 0, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["MergedDataTierA"][0][ "output_fileset"] unmergedMergeOutput.loadData() self.assertEqual( unmergedMergeOutput.name, "/TestWorkload/ProcessingTask/MergeTask/merged-MergedDataTierA", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow( name="TestWorkload", task="/TestWorkload/ProcessingTask/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputADataTierA"][0][ "merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputADataTierA"][ 0]["output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputBDataTierB"][0][ "merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputBDataTierB"][ 0]["output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual( mergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputADataTierA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual( unmergedSkimOutputA.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputADataTierA", "Error: Unmerged output fileset is wrong.") self.assertEqual( mergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputBDataTierB", "Error: Merged output fileset is wrong.") self.assertEqual( unmergedSkimOutputB.name, "/TestWorkload/ProcessingTask/MergeTask/SkimTask/unmerged-SkimOutputBDataTierB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset(name="TestWorkload-ProcessingTask-SomeBlock") topLevelFileset.loadData() procSubscription = Subscription(fileset=topLevelFileset, workflow=procWorkflow) procSubscription.loadData() self.assertEqual(len(procSubscription.getWhiteBlackList()), 2, "Error: Wrong site white/black list for proc sub.") for site in procSubscription.getWhiteBlackList(): if site["site_name"] == "site1": self.assertEqual(site["valid"], 1, "Error: Site should be white listed.") else: self.assertEqual(site["valid"], 0, "Error: Site should be black listed.") self.assertEqual(procSubscription["type"], "Processing", "Error: Wrong subscription type.") self.assertEqual(procSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") mergeSubscription = Subscription(fileset=unmergedProcOutput, workflow=mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset=unmergedMergeOutput, workflow=skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return def testTruncatedWFInsertion(self): """ _testTruncatedWFInsertion_ """ resourceControl = ResourceControl() resourceControl.insertSite(siteName='site1', pnn='goodse.cern.ch', ceName='site1', plugin="TestPlugin") resourceControl.insertSite(siteName='site2', pnn='goodse2.cern.ch', ceName='site2', plugin="TestPlugin") testWorkload = self.createTestWMSpec() testTopLevelTask = getFirstTask(testWorkload) testWMBSHelper = WMBSHelper(testWorkload, testTopLevelTask.name(), "SomeBlock", cachepath=self.workDir) testWMBSHelper.createTopLevelFileset() testWMBSHelper._createSubscriptionsInWMBS( testTopLevelTask, testWMBSHelper.topLevelFileset) testWorkload.truncate("ResubmitTestWorkload", "/TestWorkload/ProcessingTask/MergeTask", "someserver", "somedatabase") # create the subscription for multiple top task (MergeTask and CleanupTask for the same block) for task in testWorkload.getTopLevelTask(): testResubmitWMBSHelper = WMBSHelper(testWorkload, task.name(), "SomeBlock2", cachepath=self.workDir) testResubmitWMBSHelper.createTopLevelFileset() testResubmitWMBSHelper._createSubscriptionsInWMBS( task, testResubmitWMBSHelper.topLevelFileset) mergeWorkflow = Workflow(name="ResubmitTestWorkload", task="/ResubmitTestWorkload/MergeTask") mergeWorkflow.load() self.assertEqual(mergeWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( mergeWorkflow.spec, os.path.join(self.workDir, mergeWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(mergeWorkflow.outputMap), 1, "Error: Wrong number of WF outputs.") unmergedMergeOutput = mergeWorkflow.outputMap["MergedDataTierA"][0][ "output_fileset"] unmergedMergeOutput.loadData() self.assertEqual( unmergedMergeOutput.name, "/ResubmitTestWorkload/MergeTask/merged-MergedDataTierA", "Error: Unmerged output fileset is wrong.") skimWorkflow = Workflow( name="ResubmitTestWorkload", task="/ResubmitTestWorkload/MergeTask/SkimTask") skimWorkflow.load() self.assertEqual(skimWorkflow.owner, "sfoulkes", "Error: Wrong owner.") self.assertEqual( skimWorkflow.spec, os.path.join(self.workDir, skimWorkflow.name, "WMSandbox", "WMWorkload.pkl"), "Error: Wrong spec URL") self.assertEqual(len(skimWorkflow.outputMap), 2, "Error: Wrong number of WF outputs.") mergedSkimOutputA = skimWorkflow.outputMap["SkimOutputADataTierA"][0][ "merged_output_fileset"] unmergedSkimOutputA = skimWorkflow.outputMap["SkimOutputADataTierA"][ 0]["output_fileset"] mergedSkimOutputB = skimWorkflow.outputMap["SkimOutputBDataTierB"][0][ "merged_output_fileset"] unmergedSkimOutputB = skimWorkflow.outputMap["SkimOutputBDataTierB"][ 0]["output_fileset"] mergedSkimOutputA.loadData() mergedSkimOutputB.loadData() unmergedSkimOutputA.loadData() unmergedSkimOutputB.loadData() self.assertEqual( mergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputADataTierA", "Error: Merged output fileset is wrong: %s" % mergedSkimOutputA.name) self.assertEqual( unmergedSkimOutputA.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputADataTierA", "Error: Unmerged output fileset is wrong.") self.assertEqual( mergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputBDataTierB", "Error: Merged output fileset is wrong.") self.assertEqual( unmergedSkimOutputB.name, "/ResubmitTestWorkload/MergeTask/SkimTask/unmerged-SkimOutputBDataTierB", "Error: Unmerged output fileset is wrong.") topLevelFileset = Fileset( name="ResubmitTestWorkload-MergeTask-SomeBlock2") topLevelFileset.loadData() mergeSubscription = Subscription(fileset=topLevelFileset, workflow=mergeWorkflow) mergeSubscription.loadData() self.assertEqual(len(mergeSubscription.getWhiteBlackList()), 0, "Error: Wrong white/black list for merge sub.") self.assertEqual(mergeSubscription["type"], "Merge", "Error: Wrong subscription type.") self.assertEqual(mergeSubscription["split_algo"], "WMBSMergeBySize", "Error: Wrong split algo.") skimSubscription = Subscription(fileset=unmergedMergeOutput, workflow=skimWorkflow) skimSubscription.loadData() self.assertEqual(skimSubscription["type"], "Skim", "Error: Wrong subscription type.") self.assertEqual(skimSubscription["split_algo"], "FileBased", "Error: Wrong split algo.") return def testReReco(self): """ReReco workflow""" # create workflow block = self.dataset + "#" + BLOCK1 wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 5) def testReRecoBlackRunRestriction(self): """ReReco workflow with Run restrictions""" block = self.dataset + "#" + BLOCK2 self.topLevelTask.setInputRunBlacklist( [181183]) # Set run blacklist to only run in the block wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 0) def testReRecoWhiteRunRestriction(self): block = self.dataset + "#" + BLOCK2 self.topLevelTask.setInputRunWhitelist( [181183]) # Set run whitelist to only run in the block wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 1) def testLumiMaskRestrictionsOK(self): block = self.dataset + "#" + BLOCK1 self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = ['181367'] self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = ['57,80'] wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 1) def testLumiMaskRestrictionsKO(self): block = self.dataset + "#" + BLOCK1 self.wmspec.getTopLevelTask()[0].data.input.splitting.runs = [ '123454321' ] self.wmspec.getTopLevelTask()[0].data.input.splitting.lumis = [ '123,123' ] wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) files = wmbs.validFiles(self.dbs.getFileBlock(block)['Files']) self.assertEqual(len(files), 0) def testDuplicateFileInsert(self): # using default wmspec block = self.dataset + "#" + BLOCK1 wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) wmbs.topLevelFileset.loadData() numOfFiles = len(wmbs.topLevelFileset.files) # check initially inserted files. dbsFiles = self.dbs.getFileBlock(block)['Files'] self.assertEqual(numOfFiles, len(dbsFiles)) firstFileset = wmbs.topLevelFileset wmbsDao = wmbs.daofactory(classname="Files.InFileset") numOfFiles = len(wmbsDao.execute(firstFileset.id)) self.assertEqual(numOfFiles, len(dbsFiles)) # use the new spec with same inputdataset block = self.dataset + "#" + BLOCK1 wmspec = self.createWMSpec("TestSpec1") dbs = self.getDBS(wmspec) wmbs = self.createWMBSHelperWithTopTask(wmspec, block) # check duplicate insert dbsFiles = dbs.getFileBlock(block) location = self.phedex.getReplicaPhEDExNodesForBlocks(block=[block], complete='y') dbsFiles['PhEDExNodeNames'] = location[block] numOfFiles = wmbs.addFiles(dbsFiles) self.assertEqual(numOfFiles, 0) secondFileset = wmbs.topLevelFileset wmbsDao = wmbs.daofactory(classname="Files.InFileset") numOfFiles = len(wmbsDao.execute(secondFileset.id)) self.assertEqual(numOfFiles, len(dbsFiles['Files'])) self.assertNotEqual(firstFileset.id, secondFileset.id) def testDuplicateSubscription(self): """Can't duplicate subscriptions""" siteWhitelist = ["T2_XX_SiteA", "T2_XX_SiteB"] # using default wmspec block = self.dataset + "#" + BLOCK1 wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) wmbs.topLevelFileset.loadData() numOfFiles = len(wmbs.topLevelFileset.files) filesetId = wmbs.topLevelFileset.id subId = wmbs.topLevelSubscription['id'] # check initially inserted files. dbsFiles = self.dbs.getFileBlock(block)['Files'] self.assertEqual(numOfFiles, len(dbsFiles)) # Not clear what's supposed to happen here, 2nd test is completely redundant dummyFirstFileset = wmbs.topLevelFileset self.assertEqual(numOfFiles, len(dbsFiles)) # reinsert subscription - shouldn't create anything new wmbs = self.createWMBSHelperWithTopTask(self.wmspec, block) wmbs.topLevelFileset.loadData() self.assertEqual(numOfFiles, len(wmbs.topLevelFileset.files)) self.assertEqual(filesetId, wmbs.topLevelFileset.id) self.assertEqual(subId, wmbs.topLevelSubscription['id']) # now do a montecarlo workflow self.setupMCWMSpec() mask = Mask(FirstRun=12, FirstLumi=1234, FirstEvent=12345, LastEvent=999995, LastLumi=12345, LastRun=12) wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask, commonLocation=siteWhitelist) wmbs.topLevelFileset.loadData() numOfFiles = len(wmbs.topLevelFileset.files) filesetId = wmbs.topLevelFileset.id subId = wmbs.topLevelSubscription['id'] # check initially inserted files. # Not clear what's supposed to happen here, 2nd test is completely redundant numDbsFiles = 1 self.assertEqual(numOfFiles, numDbsFiles) dummyFirstFileset = wmbs.topLevelFileset self.assertEqual(numOfFiles, numDbsFiles) # reinsert subscription - shouldn't create anything new wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask, commonLocation=siteWhitelist) wmbs.topLevelFileset.loadData() self.assertEqual(numOfFiles, len(wmbs.topLevelFileset.files)) self.assertEqual(filesetId, wmbs.topLevelFileset.id) self.assertEqual(subId, wmbs.topLevelSubscription['id']) def testParentage(self): """ 1. check whether parent files are created in wmbs. 2. check parent files are associated to child. 3. When 2 specs with the same input data (one with parent processing, one without it) is inserted, if one without parent processing inserted first then the other with parent processing insert, it still needs to create parent files although child files are duplicate """ # Swap out the dataset for one that has parents task = next(self.wmspec.taskIterator()) oldDS = task.inputDataset( ) # Copy the old dataset, only will use DBS URL from it task.addInputDataset(name="/Cosmics/ComissioningHI-PromptReco-v1/RECO", primary='Cosmics', processed='ComissioningHI-PromptReco-v1', tier='RECO', dbsurl=oldDS.dbsurl) block = '/Cosmics/ComissioningHI-PromptReco-v1/RECO' + '#5b89ba9c-0dbf-11e1-9b6c-003048caaace' # File creation without parents wmbs, _, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block, parentFlag=False, detail=True) self.assertEqual(8, numFiles) wmbs.topLevelFileset.loadData() for child in wmbs.topLevelFileset.files: self.assertEqual(len(child["parents"]), 0) # no parents per child # File creation with parents wmbs, _, numFiles = self.createWMBSHelperWithTopTask(self.wmspec, block, parentFlag=True, detail=True) self.assertEqual(8, numFiles) wmbs.topLevelFileset.loadData() for child in wmbs.topLevelFileset.files: self.assertEqual(len(child["parents"]), 1) # one parent per child def testMCFakeFileInjection(self): """Inject fake Monte Carlo files into WMBS""" # This test is failing because the name of the couch DB is set to None # in BasicProductionWorkload.getProdArgs() but changing it to # "reqmgr_config_cache_t" from StdBase test arguments does not fix the # situation. testDuplicateSubscription probably has the same issue siteWhitelist = ["T2_XX_SiteA", "T2_XX_SiteB"] self.setupMCWMSpec() mask = Mask(FirstRun=12, FirstLumi=1234, FirstEvent=12345, LastEvent=999995, LastLumi=12345, LastRun=12) wmbs = self.createWMBSHelperWithTopTask(self.wmspec, None, mask, commonLocation=siteWhitelist) subscription = wmbs.topLevelSubscription self.assertEqual(1, subscription.exists()) fileset = subscription['fileset'] self.assertEqual(1, fileset.exists()) fileset.loadData() # need to refresh from database self.assertEqual(len(fileset.files), 1) self.assertEqual(len(fileset.parents), 0) self.assertFalse(fileset.open) firstFile = list(fileset.files)[0] self.assertEqual(firstFile['events'], mask['LastEvent'] - mask['FirstEvent'] + 1) # inclusive range self.assertEqual(firstFile['merged'], False) # merged files get added to dbs self.assertEqual(len(firstFile['parents']), 0) # firstFile.loadData() self.assertEqual(sorted(firstFile['locations']), sorted(self.pnns)) self.assertEqual(len(firstFile.getParentLFNs()), 0) self.assertEqual(len(firstFile.getRuns()), 1) run = firstFile.getRuns()[0] self.assertEqual(run.run, mask['FirstRun']) self.assertEqual(run.lumis[0], mask['FirstLumi']) self.assertEqual(run.lumis[-1], mask['LastLumi']) self.assertEqual(len(run.lumis), mask['LastLumi'] - mask['FirstLumi'] + 1)
class StartPolicyInterface(PolicyInterface): """Interface for start policies""" def __init__(self, **args): PolicyInterface.__init__(self, **args) self.workQueueElements = [] self.wmspec = None self.team = None self.initialTask = None self.splitParams = None self.dbs_pool = {} self.data = {} self.lumi = None self.couchdb = None self.rejectedWork = [] # List of inputs that were rejected self.badWork = [ ] # list of bad work unit (e.g. without any valid files) self.pileupData = {} self.cric = CRIC() if usingRucio(): self.rucio = Rucio(self.args['rucioAcct'], configDict={'logger': self.logger}) else: self.phedex = PhEDEx() # this will go away eventually def split(self): """Apply policy to spec""" raise NotImplementedError def validate(self): """Check params and spec are appropriate for the policy""" raise NotImplementedError def validateCommon(self): """Common validation stuff""" try: Lexicon.requestName(self.wmspec.name()) except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc. error = WorkQueueWMSpecError( self.wmspec, "Workflow name validation error: %s" % str(ex)) raise error if self.initialTask.siteWhitelist(): if isinstance(self.initialTask.siteWhitelist(), basestring): error = WorkQueueWMSpecError( self.wmspec, 'Invalid site whitelist: Must be tuple/list but is %s' % type(self.initialTask.siteWhitelist())) raise error try: [ Lexicon.cmsname(site) for site in self.initialTask.siteWhitelist() ] except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc. error = WorkQueueWMSpecError( self.wmspec, "Site whitelist validation error: %s" % str(ex)) raise error else: error = WorkQueueWMSpecError( self.wmspec, "Site whitelist validation error: Empty site whitelist") raise error if self.initialTask.siteBlacklist(): if isinstance(self.initialTask.siteBlacklist(), basestring): error = WorkQueueWMSpecError( self.wmspec, 'Invalid site blacklist: Must be tuple/list but is %s' % type(self.initialTask.siteBlacklist())) raise error try: [ Lexicon.cmsname(site) for site in self.initialTask.siteBlacklist() ] except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc. error = WorkQueueWMSpecError( self.wmspec, "Site blacklist validation error: %s" % str(ex)) raise error # splitter settings if self.args.get('SliceSize', 1) <= 0: error = WorkQueueWMSpecError( self.wmspec, 'Zero or negative SliceSize parameter') raise error if self.args.get('SubSliceSize', 1) <= 0: error = WorkQueueWMSpecError( self.wmspec, 'Zero or negative SubSliceSize parameter') raise error # check input dataset is valid try: if self.initialTask.getInputDatasetPath(): Lexicon.dataset(self.initialTask.getInputDatasetPath()) except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc. error = WorkQueueWMSpecError( self.wmspec, "Dataset validation error: %s" % str(ex)) raise error # if pileup is found, check that they are valid datasets try: pileupDatasets = self.wmspec.listPileupDatasets() for dbsUrl in pileupDatasets: for dataset in pileupDatasets[dbsUrl]: Lexicon.dataset(dataset) except Exception as ex: # can throw many errors e.g. AttributeError, AssertionError etc. error = WorkQueueWMSpecError( self.wmspec, "Pileup dataset validation error: %s" % str(ex)) raise error def newQueueElement(self, **args): # DBS Url may not be available in the initial task # but in the pileup data (MC pileup) dbsUrl = self.initialTask.dbsUrl() if dbsUrl is None and self.pileupData: # Get the first DBS found dbsUrl = self.wmspec.listPileupDatasets().keys()[0] args.setdefault('Status', 'Available') args.setdefault('WMSpec', self.wmspec) args.setdefault('Task', self.initialTask) args.setdefault('RequestName', self.wmspec.name()) args.setdefault('TaskName', self.initialTask.name()) args.setdefault('Dbs', dbsUrl) args.setdefault('SiteWhitelist', self.initialTask.siteWhitelist()) args.setdefault('SiteBlacklist', self.initialTask.siteBlacklist()) args.setdefault('StartPolicy', self.wmspec.startPolicy()) args.setdefault('EndPolicy', self.wmspec.endPolicyParameters()) args.setdefault('Priority', self.wmspec.priority()) args.setdefault('PileupData', self.pileupData) if not args['Priority']: args['Priority'] = 0 ele = WorkQueueElement(**args) for data, sites in ele['Inputs'].items(): if not sites: raise WorkQueueWMSpecError( self.wmspec, 'Input data has no locations "%s"' % data) # catch infinite splitting loops if len(self.workQueueElements) > self.args.get('maxRequestSize', 1e8): raise WorkQueueWMSpecError( self.wmspec, 'Too many elements (%d)' % self.args.get('MaxRequestElements', 1e8)) self.workQueueElements.append(ele) def __call__(self, wmspec, task, data=None, mask=None, team=None, continuous=False): self.wmspec = wmspec # bring in spec specific settings self.args.update(self.wmspec.startPolicyParameters()) self.initialTask = task if data: self.data = data self.mask = mask self.validate() try: pileupDatasets = self.wmspec.listPileupDatasets() if pileupDatasets: self.pileupData = self.getDatasetLocations(pileupDatasets) self.split() # For known exceptions raise custom error that will fail the workflow. except dbsClientException as ex: # A dbs configuration error implies the spec is invalid error = WorkQueueWMSpecError(self.wmspec, "DBS config error: %s" % str(ex)) raise error except AssertionError as ex: # Assertion generally means validation of an input field failed error = WorkQueueWMSpecError(self.wmspec, "Assertion error: %s" % str(ex)) raise error except DBSReaderError as ex: # Hacky way of identifying non-existant data, DbsBadRequest chomped by DBSReader if 'Invalid parameters' in str(ex): data = task.data.input.pythonise_( ) if task.data.input else 'None' msg = """data: %s, mask: %s, pileup: %s. %s""" % ( str(data), str(mask), str(pileupDatasets), str(ex)) error = WorkQueueNoWorkError(self.wmspec, msg) raise error raise # propagate other dbs errors # if we have no new elements and we are not adding work to request # already running, then raise exception if not self.workQueueElements and not continuous: data = task.data.input.pythonise_() if task.data.input else 'None' msg = "Failed to add work. Input data: %s, mask: %s." % (str(data), str(mask)) error = WorkQueueNoWorkError(self.wmspec, msg) raise error return self.workQueueElements, self.rejectedWork, self.badWork def dbs(self, dbs_url=None): """Get DBSReader""" from WMCore.WorkQueue.WorkQueueUtils import get_dbs if dbs_url is None: dbs_url = self.initialTask.dbsUrl() return get_dbs(dbs_url) @staticmethod def supportsWorkAddition(): """Indicates if a given policy supports addition of new work""" return False def getMaskedBlocks(self, task, dbs, datasetPath): """ Get the blocks which pass the lumi mask restrictions. For each block return the list of lumis which were ok (given the lumi mask). The data structure returned is the following: { "block1" : {"file1" : LumiList(), "file5" : LumiList(), ...} "block2" : {"file2" : LumiList(), "file7" : LumiList(), ...} } """ # Get the task mask as a LumiList object to make operations easier maskedBlocks = {} taskMask = task.getLumiMask() # for performance reasons, we first get all the blocknames blocks = [ x['block_name'] for x in dbs.dbs.listBlocks(dataset=datasetPath) ] for block in blocks: fileLumis = dbs.dbs.listFileLumis(block_name=block, validFileOnly=1) for fileLumi in fileLumis: lfn = fileLumi['logical_file_name'] runNumber = str(fileLumi['run_num']) lumis = fileLumi['lumi_section_num'] fileMask = LumiList(runsAndLumis={runNumber: lumis}) commonMask = taskMask & fileMask if commonMask: maskedBlocks.setdefault(block, {}) maskedBlocks[block].setdefault(lfn, LumiList()) maskedBlocks[block][lfn] += commonMask return maskedBlocks def modifyPolicyForWorkAddition(self, inboxElement): """Set modifiers to the policy based on the inboxElement information so that after a splitting pass with this policy strictly new work is returned, the inbox element must have information about already existing work""" raise NotImplementedError( "This can't be called on a base StartPolicyInterface object") def newDataAvailable(self, task, inbound): """ Returns True if there is data in the future could be included as an element for the inbound parent. However it doesn't guarantee that the new data will be included if the inbound element is split (i.e. the new data could be open blocks for the Block policy). """ raise NotImplementedError( "This can't be called on a base StartPolicyInterface object") def getDatasetLocations(self, datasets): """ Returns a dictionary with the location of the datasets according to Rucio The definition of "location" here is a union of all sites holding at least part of the dataset (defined by the DATASET grouping). :param datasets: dictionary with a list of dataset names (key'ed by the DBS URL) :return: a dictionary of dataset locations, key'ed by the dataset name """ result = {} for dbsUrl in datasets: for datasetPath in datasets[dbsUrl]: if hasattr(self, "rucio"): locations = self.rucio.getDataLockedAndAvailable( name=datasetPath, account=self.args['rucioAcct']) else: locations = set() resp = self.phedex.getReplicaPhEDExNodesForBlocks( dataset=[datasetPath], complete='y') for blockSites in resp.values(): locations.update(blockSites) result[datasetPath] = self.cric.PNNstoPSNs(locations) return result def blockLocationRucioPhedex(self, blockName): """ Wrapper around Rucio and PhEDEx systems. Fetch the current location of the block name (if Rucio, also consider the locks made on that block) :param blockName: string with the block name :return: a list of RSEs """ if hasattr(self, "rucio"): location = self.rucio.getDataLockedAndAvailable( name=blockName, account=self.args['rucioAcct']) else: location = self.phedex.getReplicaPhEDExNodesForBlocks( block=[blockName], complete='y')[blockName] return location