def printInfoForJob(splitInfo): if splitInfo.get(DataSplitter.Invalid, False): utils.vprint(' Status: Invalidated splitting', -1) utils.vprint(('Dataset: %s' % splitInfo[DataSplitter.Dataset]).ljust(50), -1, newline = False) utils.vprint(('Events: %d' % splitInfo[DataSplitter.NEntries]).ljust(20), -1, newline = False) utils.vprint(' ID: %s' % splitInfo.get(DataSplitter.DatasetID, 0), -1) utils.vprint((' Block: %s' % splitInfo.get(DataSplitter.BlockName, 0)).ljust(50), -1, newline = False) utils.vprint((' Skip: %d' % splitInfo.get(DataSplitter.Skipped, 0)).ljust(20), -1, newline = False) if splitInfo.get(DataSplitter.Nickname): utils.vprint('Nick: %s' % splitInfo[DataSplitter.Nickname], -1) else: utils.vprint('', -1) if splitInfo.get(DataSplitter.Locations) != None: utils.vprint('Locations: %s' % utils.wrapList(splitInfo[DataSplitter.Locations], 70, ',\n '), -1) for idx, head in enumerate(splitInfo.get(DataSplitter.MetadataHeader, [])): oneFileMetadata = map(lambda x: repr(x[idx]), splitInfo[DataSplitter.Metadata]) utils.vprint('%7s: %s' % (head, utils.wrapList(oneFileMetadata, 70, ',\n ')), -1) utils.vprint(' Files: ', -1, newline = False) if utils.verbosity() > 2: utils.vprint(str.join('\n ', splitInfo[DataSplitter.FileList]), -1) else: utils.vprint('%d files selected' % len(splitInfo[DataSplitter.FileList]), -1)
def __init__(self, config, datasetExpr, datasetNick, datasetID = 0): DataProvider.__init__(self, config, datasetExpr, datasetNick, datasetID) # PhEDex blacklist: 'T1_DE_KIT', 'T1_US_FNAL' and '*_Disk' allow user jobs - other T1's dont! self.phedexBL = config.getList('phedex sites', ['-T3_US_FNALLPC']) self.phedexWL = config.getList('phedex t1 accept', ['T1_DE_KIT', 'T1_US_FNAL']) self.phedexT1 = config.get('phedex t1 mode', 'disk').lower() self.onlyComplete = config.getBool('only complete sites', True) self.locationFormat = config.get('location format', 'hostname').lower() # hostname or sitedb if self.locationFormat not in ['hostname', 'sitedb', 'both']: raise ConfigError('Invalid location format: %s' % self.locationFormat) (self.datasetPath, self.url, self.datasetBlock) = utils.optSplit(datasetExpr, '@#') self.url = QM(self.url, self.url, config.get('dbs instance', '')) self.datasetBlock = QM(self.datasetBlock, self.datasetBlock, 'all') self.includeLumi = config.getBool('keep lumi metadata', False) self.onlyValid = config.getBool('only valid', True) self.checkUnique = config.getBool('check unique', True) # This works in tandem with active task module (cmssy.py supports only [section] lumi filter!) self.selectedLumis = parseLumiFilter(config.get('lumi filter', '')) if self.selectedLumis: utils.vprint('Runs/lumi section filter enabled! (%d entries)' % len(self.selectedLumis), -1, once = True) utils.vprint('\tThe following runs and lumi sections are selected:', 1, once = True) utils.vprint('\t' + utils.wrapList(formatLumi(self.selectedLumis), 65, ',\n\t'), 1, once = True)