def parse_cmd_line():
help_msg = '\n\nDEFAULT: The default is to download the SE file and check them with MD5 hashes.'
help_msg += '\n * In case all files are transferred sucessfully, the job is marked'
help_msg += '\n as already downloaded, so that the files are not copied again.'
help_msg += '\n * Failed transfer attempts will mark the job as failed, so that it'
help_msg += '\n can be resubmitted.'
parser = Options(usage = '%s [OPTIONS] <config file>' + help_msg)
def addBoolOpt(group, short_pair, option_base, help_base, default = False,
option_prefix_pair = ('', 'no'), help_prefix_pair = ('', 'do not '), dest = None):
def create_opt(idx):
return str.join('-', option_prefix_pair[idx].split() + option_base.split())
def create_help(idx):
help_def = ''
if (default and (idx == 0)) or ((not default) and (idx == 1)):
help_def = ' [Default]'
return help_prefix_pair[idx] + help_base + help_def
parser.addFlag(group, short_pair, (create_opt(0), create_opt(1)), default = default, dest = dest,
help_pair = (create_help(0), create_help(1)))
addBoolOpt(None, 'v ', 'verify-md5', default = True, help_base = 'MD5 verification of SE files',
help_prefix_pair = ('enable ', 'disable '))
addBoolOpt(None, 'l ', 'loop', default = False, help_base = 'loop over jobs until all files are successfully processed')
addBoolOpt(None, 'L ', 'infinite', default = False, help_base = 'process jobs in an infinite loop')
addBoolOpt(None, ' ', 'shuffle', default = False, help_base = 'shuffle download order')
addBoolOpt(None, ' ', '', default = False, help_base = 'files which are already on local disk',
option_prefix_pair = ('skip-existing', 'overwrite'), help_prefix_pair = ('skip ', 'overwrite '), dest = 'skip_existing')
parser.section('jobs', 'Job state / flag handling')
addBoolOpt('jobs', ' ', 'mark-dl', default = True, help_base = 'mark sucessfully downloaded jobs as such')
addBoolOpt('jobs', ' ', 'mark-dl', default = False, help_base = 'mark about sucessfully downloaded jobs',
option_prefix_pair = ('ignore', 'use'), help_prefix_pair = ('ignore ', 'use '), dest = 'mark_ignore_dl')
addBoolOpt('jobs', ' ', 'mark-fail', default = True, help_base = 'mark jobs failing verification as such')
addBoolOpt('jobs', ' ', 'mark-empty-fail', default = False, help_base = 'mark jobs without any files as failed')
parser.section('file', 'Local / SE file handling')
for (option, help_base) in [
('local-ok', 'files of successful jobs in local directory'),
('local-fail', 'files of failed jobs in local directory'),
('se-ok', 'files of successful jobs on SE'),
('se-fail', 'files of failed jobs on the SE'),
]:
addBoolOpt('file', ' ', option, default = False, help_base = help_base,
option_prefix_pair = ('rm', 'keep'), help_prefix_pair = ('remove ', 'keep '))
parser.addText(None, 'o', 'output', default = None,
help = 'specify the local output directory')
parser.addText(None, 'T', 'token', default = 'VomsProxy',
help = 'specify the access token used to determine ability to download - VomsProxy or TrivialAccessToken')
parser.addList(None, 'S', 'selectSE', default = None,
help = 'specify the SE paths to process')
parser.addText(None, 'r', 'retry',
help = 'how often should a transfer be attempted [Default: 0]')
parser.addText(None, 't', 'threads', default = 0,
help = 'how many parallel download threads should be used to download files [Default: no multithreading]')
parser.addText(None, ' ', 'slowdown', default = 2,
help = 'specify time between downloads [Default: 2 sec]')
parser.addBool(None, ' ', 'show-host', default = False,
help = 'show SE hostname during download')
parser.section('short', 'Shortcuts')
parser.addFSet('short', 'm', 'move', help = 'Move files from SE - shorthand for:'.ljust(100) + '%s',
flag_set = '--verify-md5 --overwrite --mark-dl --use-mark-dl --mark-fail --rm-se-fail --rm-local-fail --rm-se-ok --keep-local-ok')
parser.addFSet('short', 'c', 'copy', help = 'Copy files from SE - shorthand for:'.ljust(100) + '%s',
flag_set = '--verify-md5 --overwrite --mark-dl --use-mark-dl --mark-fail --rm-se-fail --rm-local-fail --keep-se-ok --keep-local-ok')
parser.addFSet('short', 'j', 'just-copy', help = 'Just copy files from SE - shorthand for:'.ljust(100) + '%s',
flag_set = '--verify-md5 --skip-existing --no-mark-dl --ignore-mark-dl --no-mark-fail --keep-se-fail --keep-local-fail --keep-se-ok --keep-local-ok')
parser.addFSet('short', 's', 'smart-copy',
help = 'Copy correct files from SE, but remember already downloaded files and delete corrupt files - shorthand for: '.ljust(100) + '%s',
flag_set = '--verify-md5 --mark-dl --mark-fail --rm-se-fail --rm-local-fail --keep-se-ok --keep-local-ok')
parser.addFSet('short', 'V', 'just-verify', help = 'Just verify files on SE - shorthand for:'.ljust(100) + '%s',
flag_set = '--verify-md5 --no-mark-dl --keep-se-fail --rm-local-fail --keep-se-ok --rm-local-ok --ignore-mark-dl')
parser.addFSet('short', 'D', 'just-delete', help = 'Just delete all finished files on SE - shorthand for:'.ljust(100) + '%s',
flag_set = '--skip-existing --rm-se-fail --rm-se-ok --rm-local-fail --keep-local-ok --no-mark-dl --ignore-mark-dl')
return parser.parse()
from python_compat import ifilter, imap, irange, izip, lfilter, lmap, md5_hex, set, sorted random.seed(0) parser = Options(usage = '%s [OPTIONS] <parameter definition>') parser.addAccu(None, 'c', 'collapse', default = 0, help = 'Do not collapse dataset infos in display') parser.addBool(None, 'a', 'active', default = False, help = 'Show activity state') parser.addBool(None, 'd', 'disabled', default = False, help = 'Show disabled parameter sets') parser.addBool(None, 'f', 'force-intervention', default = False, help = 'Simulate dataset intervention') parser.addBool(None, 'I', 'intervention', default = False, help = 'Display intervention tasks') parser.addBool(None, 'l', 'list-parameters', default = False, help = 'Display parameter list') parser.addBool(None, 'L', 'show-sources', default = False, help = 'Show parameter sources') parser.addBool(None, 's', 'static', default = False, help = 'Assume a static parameterset') parser.addBool(None, 't', 'untracked', default = False, help = 'Display untracked variables') parser.addBool(None, 'T', 'persistent', default = False, help = 'Work with persistent paramters') parser.addList(None, 'p', 'parameter', default = [], help = 'Specify parameters') parser.addText(None, 'D', 'dataset', default = '', help = 'Add dataset splitting (use "True" to simulate a dataset)') parser.addText(None, 'M', 'manager', default = None, help = 'Select parameter source manager') parser.addText(None, 'o', 'output', default = '', help = 'Show only specified parameters') parser.addText(None, 'S', 'save', default = '', help = 'Saves information to specified file') parser.addText(None, 'V', 'visible', default = '', help = 'Set visible variables') options = scriptOptions(parser) if len(options.args) != 1: utils.exitWithUsage(parser.usage()) # Create dataset parameter source class DummySplitter: def getMaxJobs(self): return 3 def getSplitInfo(self, pNum):
parser.addBool(None,
'L',
'show-sources',
default=False,
help='Show parameter sources')
parser.addBool(None,
't',
'untracked',
default=False,
help='Display untracked variables')
parser.addBool(None,
'T',
'persistent',
default=False,
help='Work with persistent paramters')
parser.addList(None, 'p', 'parameter', default=[], help='Specify parameters')
parser.addText(None,
'D',
'dataset',
default='',
help='Add dataset splitting (use "True" to simulate a dataset)')
parser.addText(
None,
'j',
'job',
default=None,
help='Select job to display (used for unbounded parameter spaces)')
parser.addText(None,
'F',
'factory',
default=None,
def parse_cmd_line():
help_msg = '\n\nDEFAULT: The default is to download the SE file and check them with MD5 hashes.'
help_msg += '\n * In case all files are transferred sucessfully, the job is marked'
help_msg += '\n as already downloaded, so that the files are not copied again.'
help_msg += '\n * Failed transfer attempts will mark the job as failed, so that it'
help_msg += '\n can be resubmitted.'
parser = Options(usage='%s [OPTIONS] <config file>' + help_msg)
def addBoolOpt(group,
short_pair,
option_base,
help_base,
default=False,
option_prefix_pair=('', 'no'),
help_prefix_pair=('', 'do not '),
dest=None):
def create_opt(idx):
return str.join(
'-', option_prefix_pair[idx].split() + option_base.split())
def create_help(idx):
help_def = ''
if (default and (idx == 0)) or ((not default) and (idx == 1)):
help_def = ' [Default]'
return help_prefix_pair[idx] + help_base + help_def
parser.addFlag(group,
short_pair, (create_opt(0), create_opt(1)),
default=default,
dest=dest,
help_pair=(create_help(0), create_help(1)))
addBoolOpt(None,
'v ',
'verify-md5',
default=True,
help_base='MD5 verification of SE files',
help_prefix_pair=('enable ', 'disable '))
addBoolOpt(
None,
'l ',
'loop',
default=False,
help_base='loop over jobs until all files are successfully processed')
addBoolOpt(None,
'L ',
'infinite',
default=False,
help_base='process jobs in an infinite loop')
addBoolOpt(None,
' ',
'shuffle',
default=False,
help_base='shuffle download order')
addBoolOpt(None,
' ',
'',
default=False,
help_base='files which are already on local disk',
option_prefix_pair=('skip-existing', 'overwrite'),
help_prefix_pair=('skip ', 'overwrite '),
dest='skip_existing')
parser.section('jobs', 'Job state / flag handling')
addBoolOpt('jobs',
' ',
'mark-dl',
default=True,
help_base='mark sucessfully downloaded jobs as such')
addBoolOpt('jobs',
' ',
'mark-dl',
default=False,
help_base='mark about sucessfully downloaded jobs',
option_prefix_pair=('ignore', 'use'),
help_prefix_pair=('ignore ', 'use '),
dest='mark_ignore_dl')
addBoolOpt('jobs',
' ',
'mark-fail',
default=True,
help_base='mark jobs failing verification as such')
addBoolOpt('jobs',
' ',
'mark-empty-fail',
default=False,
help_base='mark jobs without any files as failed')
parser.section('file', 'Local / SE file handling')
for (option, help_base) in [
('local-ok', 'files of successful jobs in local directory'),
('local-fail', 'files of failed jobs in local directory'),
('se-ok', 'files of successful jobs on SE'),
('se-fail', 'files of failed jobs on the SE'),
]:
addBoolOpt('file',
' ',
option,
default=False,
help_base=help_base,
option_prefix_pair=('rm', 'keep'),
help_prefix_pair=('remove ', 'keep '))
parser.addText(None,
'o',
'output',
default=None,
help='specify the local output directory')
parser.addText(
None,
'T',
'token',
default='VomsProxy',
help=
'specify the access token used to determine ability to download - VomsProxy or TrivialAccessToken'
)
parser.addList(None,
'S',
'selectSE',
default=None,
help='specify the SE paths to process')
parser.addText(
None,
'r',
'retry',
help='how often should a transfer be attempted [Default: 0]')
parser.addText(
None,
't',
'threads',
default=0,
help=
'how many parallel download threads should be used to download files [Default: no multithreading]'
)
parser.addText(None,
' ',
'slowdown',
default=2,
help='specify time between downloads [Default: 2 sec]')
parser.addBool(None,
' ',
'show-host',
default=False,
help='show SE hostname during download')
parser.section('short', 'Shortcuts')
parser.addFSet(
'short',
'm',
'move',
help='Move files from SE - shorthand for:'.ljust(100) + '%s',
flag_set=
'--verify-md5 --overwrite --mark-dl --use-mark-dl --mark-fail --rm-se-fail --rm-local-fail --rm-se-ok --keep-local-ok'
)
parser.addFSet(
'short',
'c',
'copy',
help='Copy files from SE - shorthand for:'.ljust(100) + '%s',
flag_set=
'--verify-md5 --overwrite --mark-dl --use-mark-dl --mark-fail --rm-se-fail --rm-local-fail --keep-se-ok --keep-local-ok'
)
parser.addFSet(
'short',
'j',
'just-copy',
help='Just copy files from SE - shorthand for:'.ljust(100) + '%s',
flag_set=
'--verify-md5 --skip-existing --no-mark-dl --ignore-mark-dl --no-mark-fail --keep-se-fail --keep-local-fail --keep-se-ok --keep-local-ok'
)
parser.addFSet(
'short',
's',
'smart-copy',
help=
'Copy correct files from SE, but remember already downloaded files and delete corrupt files - shorthand for: '
.ljust(100) + '%s',
flag_set=
'--verify-md5 --mark-dl --mark-fail --rm-se-fail --rm-local-fail --keep-se-ok --keep-local-ok'
)
parser.addFSet(
'short',
'V',
'just-verify',
help='Just verify files on SE - shorthand for:'.ljust(100) + '%s',
flag_set=
'--verify-md5 --no-mark-dl --keep-se-fail --rm-local-fail --keep-se-ok --rm-local-ok --ignore-mark-dl'
)
parser.addFSet(
'short',
'D',
'just-delete',
help='Just delete all finished files on SE - shorthand for:'.ljust(100)
+ '%s',
flag_set=
'--skip-existing --rm-se-fail --rm-se-ok --rm-local-fail --keep-local-ok --no-mark-dl --ignore-mark-dl'
)
return parser.parse()