def setup_parser():
parser = Options(usage = '%s [OPTIONS] <config file / work directory>')
parser.section('disc', 'Discovery options - ignored in case dbs input file is specified')
# options that are used as config settings for InfoScanners
parser.addText('disc', 'n', 'dataset-name-pattern', default = '',
help = 'Specify dbs path name - Example: DataSet_@NICK@_@VAR@')
parser.addText('disc', 'H', 'dataset-hash-keys', default = '',
help = 'Included additional variables in dataset hash calculation')
parser.addText('disc', 'J', 'source-job-selector', default = '',
help = 'Specify dataset(s) to process')
parser.addBool('disc', 'm', 'merge-parents', default = False,
help = 'Merge output files from different parent blocks into a single block [Default: Keep boundaries]')
parser.addText('disc', 'P', 'parent-source', default = '',
help = 'Override parent information source - to bootstrap a reprocessing on local files')
# options directly used by this script
parser.addText('disc', 'T', 'datatype', default = None,
help = 'Supply dataset type in case cmssw report did not specify it - valid values: "mc" or "data"')
parser.addBool('disc', 'j', 'jobhash', default = False,
help = 'Use hash of all config files in job for dataset key calculation')
parser.addBool('disc', 'u', 'unique-cfg', default = False,
help = 'Circumvent edmConfigHash collisions so each dataset is stored with unique config information')
parser.addText('disc', 'G', 'globaltag', default = 'crab2_tag',
help = 'Specify global tag')
parser.section('proc', 'Processing mode')
parser.addBool('proc', 'd', 'discovery', default = False,
help = 'Enable discovery mode - just collect file information and exit')
parser.addText('proc', ' ', 'tempdir', default = '',
help = 'Override temp directory')
parser.addBool('proc', 'i', 'no-import', default = True, dest = 'do_import',
help = 'Disable import of new datasets into target DBS instance - only temporary json files are created')
parser.addBool('proc', 'I', 'incremental', default = False,
help = 'Skip import of existing files - Warning: this destroys coherent block structure!')
parser.addBool('proc', 'o', 'open-blocks', default = True, dest = 'do_close_blocks',
help = 'Keep blocks open for addition of further files [Default: Close blocks]')
# parser.addBool('proc', 'b', 'batch', default = False,
# help = 'Enable non-interactive batch mode [Default: Interactive mode]')
parser.section('dbsi', 'DBS instance handling')
parser.addText('dbsi', 't', 'target-instance', default = 'https://cmsweb.cern.ch/dbs/prod/phys03',
help = 'Specify target dbs instance url')
parser.addText('dbsi', 's', 'source-instance', default = 'https://cmsweb.cern.ch/dbs/prod/global',
help = 'Specify source dbs instance url(s), where parent datasets are taken from')
parser.addText(None, 'F', 'input-file', default = None,
help = 'Specify dbs input file to use instead of scanning job output')
parser.addBool(None, 'c', 'continue-migration', default = False,
help = 'Continue an already started migration')
# parser.addText(None, 'D', 'display-dataset', default = None,
# help = 'Display information associated with dataset key(s) (accepts "all")')
# parser.addText(None, 'C', 'display-config', default = None,
# help = 'Display information associated with config hash(es) (accepts "all")')
# parser.addText(None, 'k', 'dataset-key-select', default = '',
# help = 'Specify dataset keys to process')
return scriptOptions(parser)
def setup_parser():
parser = Options(usage = '%s [OPTIONS] <config file / work directory>')
parser.section('disc', 'Discovery options - ignored in case dbs input file is specified')
# options that are used as config settings for InfoScanners
parser.addText('disc', 'n', 'dataset-name-pattern', default = '',
help = 'Specify dbs path name - Example: DataSet_@NICK@_@VAR@')
parser.addText('disc', 'H', 'dataset-hash-keys', default = '',
help = 'Included additional variables in dataset hash calculation')
parser.addText('disc', 'J', 'source-job-selector', default = '',
help = 'Specify dataset(s) to process')
parser.addBool('disc', 'm', 'merge-parents', default = False,
help = 'Merge output files from different parent blocks into a single block [Default: Keep boundaries]')
parser.addText('disc', 'P', 'parent-source', default = '',
help = 'Override parent information source - to bootstrap a reprocessing on local files')
# options directly used by this script
parser.addText('disc', 'T', 'datatype', default = None,
help = 'Supply dataset type in case cmssw report did not specify it - valid values: "mc" or "data"')
parser.addBool('disc', 'j', 'jobhash', default = False,
help = 'Use hash of all config files in job for dataset key calculation')
parser.addBool('disc', 'u', 'unique-cfg', default = False,
help = 'Circumvent edmConfigHash collisions so each dataset is stored with unique config information')
parser.addText('disc', 'G', 'globaltag', default = 'crab2_tag',
help = 'Specify global tag')
parser.section('proc', 'Processing mode')
parser.addBool('proc', 'd', 'discovery', default = False,
help = 'Enable discovery mode - just collect file information and exit')
parser.addText('proc', ' ', 'tempdir', default = '',
help = 'Override temp directory')
parser.addBool('proc', 'i', 'no-import', default = True, dest = 'do_import',
help = 'Disable import of new datasets into target DBS instance - only temporary json files are created')
parser.addBool('proc', 'I', 'incremental', default = False,
help = 'Skip import of existing files - Warning: this destroys coherent block structure!')
parser.addBool('proc', 'o', 'open-blocks', default = True, dest = 'do_close_blocks',
help = 'Keep blocks open for addition of further files [Default: Close blocks]')
# parser.addBool('proc', 'b', 'batch', default = False,
# help = 'Enable non-interactive batch mode [Default: Interactive mode]')
parser.section('dbsi', 'DBS instance handling')
parser.addText('dbsi', 't', 'target-instance', default = 'https://cmsweb.cern.ch/dbs/prod/phys03',
help = 'Specify target dbs instance url')
parser.addText('dbsi', 's', 'source-instance', default = 'https://cmsweb.cern.ch/dbs/prod/global',
help = 'Specify source dbs instance url(s), where parent datasets are taken from')
parser.addText(None, 'F', 'input-file', default = None,
help = 'Specify dbs input file to use instead of scanning job output')
parser.addBool(None, 'c', 'continue-migration', default = False,
help = 'Continue an already started migration')
# parser.addText(None, 'D', 'display-dataset', default = None,
# help = 'Display information associated with dataset key(s) (accepts "all")')
# parser.addText(None, 'C', 'display-config', default = None,
# help = 'Display information associated with config hash(es) (accepts "all")')
# parser.addText(None, 'k', 'dataset-key-select', default = '',
# help = 'Specify dataset keys to process')
return scriptOptions(parser)
parser.addText('part', '', 'partition-list', default=None, help='Select dataset partition information to display')
parser.addBool('part', '', 'partition-list-invalid', default=False, help='List invalidated dataset partitions')
parser.addBool('part', '', 'partition-check', default=None, help='Check dataset partition in specified work directory')
parser.section('jobs', 'Jobs debugging', '%s <config file / job file> ... ')
parser.addText('jobs', '', 'job-selector', default='', help='Display jobs matching selector')
parser.addBool('jobs', '', 'job-reset-attempts', default=False, help='Reset the attempt counter')
parser.addText('jobs', '', 'job-force-state', default='', help='Force new job state')
parser.addText('jobs', '', 'job-show-jdl', default='', help='Show JDL file if available')
parser.section('data', 'Dataset debugging', '%s <dataset file> <dataset file> ...')
parser.addText('data', '', 'dataset-show-diff', default='', help='Show difference between datasets')
parser.addText('data', '', 'dataset-show-removed', default='', help='Find removed dataset blocks')
parser.addText(None, 'd', 'logfile-decode', default='', help='Decode log files')
options = scriptOptions(parser)
(opts, args) = (options.opts, options.args)
########################################################
# BACKEND
if opts.backend_list_nodes or opts.backend_list_queues:
config = getConfig()
backend = str.join(' ', args) or 'local'
wms = Plugin.getClass('WMS').createInstance(backend, config, backend)
if opts.backend_list_nodes:
logging.info(repr(wms.getNodes()))
if opts.backend_list_queues:
logging.info(repr(wms.getQueues()))
########################################################
# | limitations under the License.
import sys
from datasetListFromX import addDatasetListOptions, discoverDataset
from gcSupport import Options, scriptOptions, utils
parser = Options(usage = '%s [OPTIONS] <config file / work directory>')
parser.addText(None, 'J', 'job-selector', dest = 'external job selector', default = '',
help = 'Specify which jobs to process')
parser.addText(None, 'i', 'info-scanner',
help = 'Specify which info scanner to run')
parser.addText(None, 'm', 'event-mode', dest = 'mode', default = 'CMSSW-Out',
help = 'Specify how to determine events - available: [CMSSW-Out], CMSSW-In, DataMod')
parser.addText(None, 'l', 'lfn', dest = 'lfn marker', default = '/store/',
help = 'Assume everything starting with marker to be a logical file name')
parser.addBool(None, 'c', 'config', dest = 'include config infos', default = False,
help = 'CMSSW specific: Add configuration data to metadata')
parser.addBool(None, 'p', 'parents', dest = 'include parent infos', default = False,
help = 'CMSSW specific: Add parent infos to metadata')
addDatasetListOptions(parser)
options = scriptOptions(parser, arg_keys = ['dataset'])
# Positional parameters override options
if len(options.args) == 0:
utils.exitWithUsage(parser.usage())
tmp = {'cmssw-out': 'CMSSW_EVENTS_WRITE', 'cmssw-in': 'CMSSW_EVENTS_READ', 'datamod': 'MAX_EVENTS'}
if options.opts.info_scanner:
options.config_dict['scanner'] = options.opts.info_scanner.replace(',', ' ')
options.config_dict['events key'] = tmp.get(options.config_dict['mode'].lower(), '')
sys.exit(discoverDataset('GCProvider', options.config_dict))
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import sys
from datasetListFromX import addDatasetListOptions, discoverDataset
from gcSupport import Options, scriptOptions, utils
parser = Options(usage = '%s [OPTIONS] <config file / work directory>')
parser.addText(None, 'J', 'job-selector', dest = 'external job selector', default = '',
help = 'Specify which jobs to process')
parser.addText(None, 'm', 'event-mode', dest = 'mode', default = 'CMSSW-Out',
help = 'Specify how to determine events - available: [CMSSW-Out], CMSSW-In, DataMod')
parser.addText(None, 'l', 'lfn', dest = 'lfn marker', default = '/store/',
help = 'Assume everything starting with marker to be a logical file name')
parser.addBool(None, 'c', 'config', dest = 'include config infos', default = False,
help = 'CMSSW specific: Add configuration data to metadata')
parser.addBool(None, 'p', 'parents', dest = 'include parent infos', default = False,
help = 'CMSSW specific: Add parent infos to metadata')
addDatasetListOptions(parser)
options = scriptOptions(parser, arg_keys = ['dataset'])
# Positional parameters override options
if len(options.args) == 0:
utils.exitWithUsage(parser.usage())
tmp = {'cmssw-out': 'CMSSW_EVENTS_WRITE', 'cmssw-in': 'CMSSW_EVENTS_READ', 'datamod': 'MAX_EVENTS'}
options.config_dict['events key'] = tmp.get(options.config_dict['mode'].lower(), '')
sys.exit(discoverDataset('GCProvider', options.config_dict))
# | Copyright 2010-2016 Karlsruhe Institute of Technology
# |
# | Licensed under the Apache License, Version 2.0 (the "License");
# | you may not use this file except in compliance with the License.
# | You may obtain a copy of the License at
# |
# | http://www.apache.org/licenses/LICENSE-2.0
# |
# | Unless required by applicable law or agreed to in writing, software
# | distributed under the License is distributed on an "AS IS" BASIS,
# | WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# | See the License for the specific language governing permissions and
# | limitations under the License.
import sys, logging
from datasetListFromX import addDatasetListOptions, discoverDataset
from gcSupport import Options, scriptOptions
parser = Options(usage = '%s [OPTIONS] <data path> <dataset name> <pattern (*.root) / files>')
parser.addText(None, 'p', 'path', dest = 'dataset', default = '.', help = 'Path to dataset files')
addDatasetListOptions(parser)
options = scriptOptions(parser, arg_keys = ['dataset', 'dataset name pattern', 'filename filter'])
logging.getLogger('user').setLevel(logging.CRITICAL)
def conditionalSet(name, source, sourceKey):
if options.config_dict.get(source) and not options.config_dict.get(name):
options.config_dict[name] = options.config_dict[source]
conditionalSet('dataset name pattern', 'delimeter dataset key', '/PRIVATE/@DELIMETER_DS@')
conditionalSet('block name pattern', 'delimeter block key', '@DELIMETER_B@')
sys.exit(discoverDataset('ScanProvider', options.config_dict))