# | limitations under the License.
import sys
from datasetListFromX import addDatasetListOptions, discoverDataset
from gcSupport import Options, scriptOptions, utils
parser = Options(usage = '%s [OPTIONS] <config file / work directory>')
parser.addText(None, 'J', 'job-selector', dest = 'external job selector', default = '',
help = 'Specify which jobs to process')
parser.addText(None, 'i', 'info-scanner',
help = 'Specify which info scanner to run')
parser.addText(None, 'm', 'event-mode', dest = 'mode', default = 'CMSSW-Out',
help = 'Specify how to determine events - available: [CMSSW-Out], CMSSW-In, DataMod')
parser.addText(None, 'l', 'lfn', dest = 'lfn marker', default = '/store/',
help = 'Assume everything starting with marker to be a logical file name')
parser.addBool(None, 'c', 'config', dest = 'include config infos', default = False,
help = 'CMSSW specific: Add configuration data to metadata')
parser.addBool(None, 'p', 'parents', dest = 'include parent infos', default = False,
help = 'CMSSW specific: Add parent infos to metadata')
addDatasetListOptions(parser)
options = scriptOptions(parser, arg_keys = ['dataset'])
# Positional parameters override options
if len(options.args) == 0:
utils.exitWithUsage(parser.usage())
tmp = {'cmssw-out': 'CMSSW_EVENTS_WRITE', 'cmssw-in': 'CMSSW_EVENTS_READ', 'datamod': 'MAX_EVENTS'}
if options.opts.info_scanner:
options.config_dict['scanner'] = options.opts.info_scanner.replace(',', ' ')
options.config_dict['events key'] = tmp.get(options.config_dict['mode'].lower(), '')
sys.exit(discoverDataset('GCProvider', options.config_dict))
files = lmap(lambda x: x.strip().strip(','), files.split())
def printError(curJ, curS, msg):
if curJ != curS:
logging.warning('%s in job %d (j:%s != s:%s)', msg, jobNum, curJ, curS)
fail.add(jobNum)
printError(events, splitInfo[DataSplitter.NEntries], 'Inconsistent number of events')
printError(skip, splitInfo[DataSplitter.Skipped], 'Inconsistent number of skipped events')
printError(files, splitInfo[DataSplitter.FileList], 'Inconsistent list of files')
except Exception:
logging.warning('Job %d was never initialized!', jobNum)
if fail:
logging.warning('Failed: ' + str.join('\n', imap(str, fail)))
if (opts.partition_list is not None) or opts.partition_list_invalid or opts.partition_check:
if len(args) != 1:
utils.exitWithUsage(parser.usage('part'))
splitter = DataSplitter.loadPartitionsForScript(args[0])
if opts.partition_list_invalid:
utils.printTabular([(0, 'Job')], partition_invalid(splitter))
if opts.partition_list is not None:
if opts.partition_list in ('', 'all'):
keyStrings = DataSplitter.enumNames
else:
keyStrings = opts.partition_list.split(',')
keyList = lmap(DataSplitter.str2enum, keyStrings)
if None in keyList:
logging.warning('Available keys: %r', DataSplitter.enumNames)
utils.printTabular([('jobNum', 'Job')] + lzip(keyList, keyStrings), partition_list(splitter, keyList))
parser = Options(usage = '%s [OPTIONS] <config file>')
parser.addBool(None, 'L', 'report-list', default = False, help = 'List available report classes')
parser.addBool(None, 'T', 'use-task', default = False, help = 'Forward task information to report')
parser.addText(None, 'R', 'report', default = 'GUIReport')
parser.addText(None, 'J', 'job-selector', default = None)
parser.addText(None, ' ', 'string', default = '')
options = scriptOptions(parser)
Report = Plugin.getClass('Report')
if options.opts.report_list:
sys.stderr.write('Available report classes:\n')
displayPluginList(getPluginList('Report'))
if len(options.args) != 1:
utils.exitWithUsage(parser.usage())
def main(opts, args):
# try to open config file
config = getConfig(args[0], section = 'global')
# Initialise task module
task = None
if opts.use_task:
task = config.getPlugin('workflow', 'Workflow:global', cls = 'Workflow', pargs = ('task',)).task
# Initialise job database
jobDB = config.getPlugin('job database', 'TextFileJobDB', cls = 'JobDB')
activity = Activity('Filtering job entries')
selected = jobDB.getJobs(JobSelector.create(opts.job_selector, task = task))
activity.finish()