def setup_dataset(config, dataset):
if dataset.lower() == 'true':
log.info('Registering dummy data provider data')
dataSplitter = DummySplitter()
else:
dataSplitter = DataSplitter.loadPartitionsForScript(dataset)
config = config.changeView(setSections = None)
partProcessor = config.getCompositePlugin('partition processor',
'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor BasicPartitionProcessor',
'MultiPartitionProcessor', cls = 'PartitionProcessor', onChange = None)
ParameterSource.createInstance('DataParameterSource', config.getWorkPath(), 'data',
None, dataSplitter, partProcessor, repository)
def main(opts, args): if opts.gc or opts.json or opts.full: return lumi_expr(opts, args) if opts.job_json or opts.job_gc or opts.job_events: (config, jobDB) = initGC(args) workDir = config.getWorkPath() splitter = None try: splitter = DataSplitter.loadPartitionsForScript(os.path.join(workDir, 'datamap.tar')) except Exception: clear_current_exception() return lumi_calc(opts, workDir, sorted(jobDB.getJobs(ClassSelector(JobClass.SUCCESS))), splitter)
def main(opts, args):
if opts.gc or opts.json or opts.full:
return lumi_expr(opts, args)
if opts.job_json or opts.job_gc or opts.job_events:
(config, jobDB) = initGC(args)
workDir = config.getWorkPath()
splitter = None
try:
splitter = DataSplitter.loadPartitionsForScript(
os.path.join(workDir, 'datamap.tar'))
except Exception:
clear_current_exception()
return lumi_calc(
opts, workDir,
sorted(jobDB.getJobs(ClassSelector(JobClass.SUCCESS))), splitter)
def setup_dataset(config, dataset):
if dataset.lower() == 'true':
utils.vprint('Registering dummy data provider data')
dataSplitter = DummySplitter()
else:
dataSplitter = DataSplitter.loadPartitionsForScript(dataset)
config = config.changeView(setSections=None)
partProcessor = config.getCompositePlugin(
'partition processor',
'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor BasicPartitionProcessor',
'MultiPartitionProcessor',
cls='PartitionProcessor',
onChange=None)
DataParameterSource = ParameterSource.getClass('DataParameterSource')
DataParameterSource.datasetsAvailable['data'] = DataParameterSource(
config.getWorkPath(), 'data', None, dataSplitter, partProcessor)
def setup_dataset(config, dataset):
if dataset.lower() == 'true':
log.info('Registering dummy data provider data')
dataSplitter = DummySplitter()
else:
dataSplitter = DataSplitter.loadPartitionsForScript(dataset)
config = config.changeView(setSections=None)
partProcessor = config.getCompositePlugin(
'partition processor',
'TFCPartitionProcessor LocationPartitionProcessor MetaPartitionProcessor BasicPartitionProcessor',
'MultiPartitionProcessor',
cls='PartitionProcessor',
onChange=None)
ParameterSource.createInstance('DataParameterSource', config.getWorkPath(),
'data', None, dataSplitter, partProcessor,
repository)
def printError(curJ, curS, msg):
if curJ != curS:
logging.warning('%s in job %d (j:%s != s:%s)', msg, jobNum, curJ, curS)
fail.add(jobNum)
printError(events, splitInfo[DataSplitter.NEntries], 'Inconsistent number of events')
printError(skip, splitInfo[DataSplitter.Skipped], 'Inconsistent number of skipped events')
printError(files, splitInfo[DataSplitter.FileList], 'Inconsistent list of files')
except Exception:
logging.warning('Job %d was never initialized!', jobNum)
if fail:
logging.warning('Failed: ' + str.join('\n', imap(str, fail)))
if (opts.partition_list is not None) or opts.partition_list_invalid or opts.partition_check:
if len(args) != 1:
utils.exitWithUsage(parser.usage('part'))
splitter = DataSplitter.loadPartitionsForScript(args[0])
if opts.partition_list_invalid:
utils.printTabular([(0, 'Job')], partition_invalid(splitter))
if opts.partition_list is not None:
if opts.partition_list in ('', 'all'):
keyStrings = DataSplitter.enumNames
else:
keyStrings = opts.partition_list.split(',')
keyList = lmap(DataSplitter.str2enum, keyStrings)
if None in keyList:
logging.warning('Available keys: %r', DataSplitter.enumNames)
utils.printTabular([('jobNum', 'Job')] + lzip(keyList, keyStrings), partition_list(splitter, keyList))
if opts.partition_check:
