def _parse_cmd_line(): parser = ScriptOptions(usage='%s [OPTIONS] <parameter definition>') parser.add_accu(None, 'c', 'collapse', default=0, help='Do not collapse dataset infos in display') parser.add_bool(None, 'a', 'active', default=False, help='Show activity state') parser.add_bool(None, 'd', 'disabled', default=False, help='Show disabled parameter sets') parser.add_bool(None, 'l', 'list-parameters', default=False, help='Display parameter list') parser.add_bool(None, 'L', 'show-sources', default=False, help='Show parameter sources') parser.add_bool(None, 't', 'untracked', default=False, help='Display untracked variables') parser.add_bool(None, 'T', 'persistent', default=False, help='Work with persistent parameters') parser.add_list(None, 'p', 'parameter', default=[], help='Specify parameters') parser.add_text(None, 'D', 'dataset', default='', help='Add dataset splitting (use "True" to simulate a dataset)') parser.add_text(None, 'j', 'job', default=None, help='Select job to display (used for unbounded parameter spaces)') parser.add_text(None, 'F', 'factory', default=None, help='Select parameter source factory') parser.add_text(None, 'o', 'output', default='', help='Show only specified parameters') parser.add_text(None, 'S', 'save', default='', help='Saves information to specified file') parser.add_text(None, 'V', 'visible', default='', help='Set visible variables') options = parser.script_parse() if len(options.args) != 1: parser.exit_with_usage() return options
def _parse_cmd_line():
help_msg = '\n\nDEFAULT: The default is to download the SE file and check them with MD5 hashes.'
help_msg += '\n * In case all files are transferred sucessfully, the job is marked'
help_msg += '\n as already downloaded, so that the files are not copied again.'
help_msg += '\n * Failed transfer attempts will mark the job as failed, so that it'
help_msg += '\n can be resubmitted.\n'
parser = ScriptOptions(usage='%s [OPTIONS] <config file>' + help_msg)
def _add_bool_opt(group, short_pair, option_base, help_base, default=False,
option_prefix_pair=('', 'no'), help_prefix_pair=('', 'do not '), dest=None):
def _create_help(idx):
help_def = ''
if (default and (idx == 0)) or ((not default) and (idx == 1)):
help_def = ' [Default]'
return help_prefix_pair[idx] + help_base + help_def
def _create_opt(idx):
return str.join('-', option_prefix_pair[idx].split() + option_base.split())
parser.add_flag(group, short_pair or ' ', (_create_opt(0), _create_opt(1)),
default=default, dest=dest, help_pair=(_create_help(0), _create_help(1)))
_add_bool_opt(None, 'l ', 'loop', default=False,
help_base='loop over jobs until all files are successfully processed')
_add_bool_opt(None, 'L ', 'infinite', default=False,
help_base='process jobs in an infinite loop')
_add_bool_opt(None, None, 'shuffle', default=False,
help_base='shuffle job processing order')
parser.add_text(None, 'J', 'job-selector', default=None,
help='specify the job selector')
parser.add_text(None, 'T', 'token', default='VomsProxy',
help='specify the access token used to determine ability to download / delete' +
' - VomsProxy or TrivialAccessToken')
parser.add_list(None, 'S', 'select-se', default=None,
help='specify the SE paths to process')
parser.add_bool(None, 'd', 'delete', default=False,
help='perform file deletions')
parser.add_bool(None, 'R', 'hide-results', default=False,
help='specify if the transfer overview should be hidden')
parser.add_text(None, 't', 'threads', default=0,
help='how many jobs should be processed in parallel [Default: no multithreading]')
parser.add_text(None, None, 'slowdown', default=2,
help='specify delay between processing jobs [Default: 2 sec]')
parser.section('jobs', 'Job state / flag handling')
_add_bool_opt('jobs', None, 'job-success', default=True,
help_base='only select successful jobs')
_add_bool_opt('jobs', None, 'mark-rm', default=False,
option_prefix_pair=('ignore', 'use'), dest='mark_ignore_rm',
help_base='mark about successfully removed jobs', help_prefix_pair=('ignore ', 'use '))
_add_bool_opt('jobs', None, 'mark-dl', default=True,
help_base='mark successfully downloaded jobs as such')
_add_bool_opt('jobs', None, 'mark-dl', default=False,
option_prefix_pair=('ignore', 'use'), dest='mark_ignore_dl',
help_base='mark about successfully downloaded jobs', help_prefix_pair=('ignore ', 'use '))
_add_bool_opt('jobs', None, 'mark-fail', default=True,
help_base='mark jobs failing verification as such')
_add_bool_opt('jobs', None, 'mark-empty-fail', default=False,
help_base='mark jobs without any files as failed')
parser.section('down', 'Download options')
_add_bool_opt('down', 'v ', 'verify-md5', default=True,
help_base='MD5 verification of SE files', help_prefix_pair=('enable ', 'disable '))
_add_bool_opt('down', None, '', default=False,
option_prefix_pair=('skip-existing', 'overwrite'), dest='skip_existing',
help_base='files which are already on local disk', help_prefix_pair=('skip ', 'overwrite '))
parser.add_text('down', 'o', 'output', default=None,
help='specify the local output directory')
parser.add_text('down', 'O', 'tmp-dir', default='/tmp',
help='specify the local tmp directory')
parser.add_text('down', 'r', 'retry',
help='how often should a transfer be attempted [Default: 0]')
parser.section('file', 'Local / SE file handling during download')
option_help_base_list = [
('local-ok', 'files of successful jobs in local directory'),
('local-fail', 'files of failed jobs in local directory'),
('se-ok', 'files of successful jobs on SE'),
('se-fail', 'files of failed jobs on the SE'),
]
for (option, help_base) in option_help_base_list:
_add_bool_opt('file', None, option, default=False, option_prefix_pair=('rm', 'keep'),
help_base=help_base, help_prefix_pair=('remove ', 'keep '))
parser.section('short_delete', 'Shortcuts for delete options')
parser.add_fset('short_delete', 'D', 'just-delete',
help='Delete files from SE and local area - shorthand for:'.ljust(100) + '%s',
flag_set='--delete --use-mark-rm --ignore-mark-dl ' +
'--rm-se-fail --rm-local-fail --rm-se-ok --rm-local-ok')
parser.section('short_down', 'Shortcuts for download options')
parser.add_fset('short_down', 'm', 'move',
help='Move files from SE - shorthand for:'.ljust(100) + '%s',
flag_set='--verify-md5 --overwrite --mark-dl --use-mark-dl --mark-fail ' +
'--rm-se-fail --rm-local-fail --rm-se-ok --keep-local-ok')
parser.add_fset('short_down', 'c', 'copy',
help='Copy files from SE - shorthand for:'.ljust(100) + '%s',
flag_set='--verify-md5 --overwrite --mark-dl --use-mark-dl --mark-fail ' +
'--rm-se-fail --rm-local-fail --keep-se-ok --keep-local-ok')
parser.add_fset('short_down', 'j', 'just-copy',
help='Just copy files from SE - shorthand for:'.ljust(100) + '%s',
flag_set='--verify-md5 --skip-existing --no-mark-dl --ignore-mark-dl --no-mark-fail ' +
'--keep-se-fail --keep-local-fail --keep-se-ok --keep-local-ok')
parser.add_fset('short_down', 's', 'smart-copy',
help='Copy correct files from SE, but remember already downloaded ' +
'files and delete corrupt files - shorthand for: '.ljust(100) + '%s',
flag_set='--verify-md5 --mark-dl --mark-fail --rm-se-fail ' +
'--rm-local-fail --keep-se-ok --keep-local-ok')
parser.add_fset('short_down', 'V', 'just-verify',
help='Just verify files on SE - shorthand for:'.ljust(100) + '%s',
flag_set='--verify-md5 --no-mark-dl --keep-se-fail ' +
'--rm-local-fail --keep-se-ok --rm-local-ok --ignore-mark-dl')
options = parser.script_parse(verbose_short=None)
if len(options.args) != 1: # we need exactly one positional argument (config file)
parser.exit_with_usage(msg='Config file not specified!')
options.opts.threads = int(options.opts.threads)
return options
def _parse_cmd_line():
help_msg = '\n\nDEFAULT: The default is to download the SE file and check them with MD5 hashes.'
help_msg += '\n * In case all files are transferred sucessfully, the job is marked'
help_msg += '\n as already downloaded, so that the files are not copied again.'
help_msg += '\n * Failed transfer attempts will mark the job as failed, so that it'
help_msg += '\n can be resubmitted.\n'
parser = ScriptOptions(usage='%s [OPTIONS] <config file>' + help_msg)
def _add_bool_opt(group,
short_pair,
option_base,
help_base,
default=False,
option_prefix_pair=('', 'no'),
help_prefix_pair=('', 'do not '),
dest=None):
def _create_help(idx):
help_def = ''
if (default and (idx == 0)) or ((not default) and (idx == 1)):
help_def = ' [Default]'
return help_prefix_pair[idx] + help_base + help_def
def _create_opt(idx):
return str.join(
'-', option_prefix_pair[idx].split() + option_base.split())
parser.add_flag(group,
short_pair or ' ', (_create_opt(0), _create_opt(1)),
default=default,
dest=dest,
help_pair=(_create_help(0), _create_help(1)))
_add_bool_opt(
None,
'l ',
'loop',
default=False,
help_base='loop over jobs until all files are successfully processed')
_add_bool_opt(None,
'L ',
'infinite',
default=False,
help_base='process jobs in an infinite loop')
_add_bool_opt(None,
None,
'shuffle',
default=False,
help_base='shuffle job processing order')
parser.add_text(None,
'J',
'job-selector',
default=None,
help='specify the job selector')
parser.add_text(
None,
'T',
'token',
default='VomsProxy',
help=
'specify the access token used to determine ability to download / delete'
+ ' - VomsProxy or TrivialAccessToken')
parser.add_list(None,
'S',
'select-se',
default=None,
help='specify the SE paths to process')
parser.add_bool(None,
'd',
'delete',
default=False,
help='perform file deletions')
parser.add_bool(None,
'R',
'hide-results',
default=False,
help='specify if the transfer overview should be hidden')
parser.add_text(
None,
't',
'threads',
default=0,
help=
'how many jobs should be processed in parallel [Default: no multithreading]'
)
parser.add_text(
None,
None,
'slowdown',
default=2,
help='specify delay between processing jobs [Default: 2 sec]')
parser.section('jobs', 'Job state / flag handling')
_add_bool_opt('jobs',
None,
'job-success',
default=True,
help_base='only select successful jobs')
_add_bool_opt('jobs',
None,
'mark-rm',
default=False,
option_prefix_pair=('ignore', 'use'),
dest='mark_ignore_rm',
help_base='mark about successfully removed jobs',
help_prefix_pair=('ignore ', 'use '))
_add_bool_opt('jobs',
None,
'mark-dl',
default=True,
help_base='mark successfully downloaded jobs as such')
_add_bool_opt('jobs',
None,
'mark-dl',
default=False,
option_prefix_pair=('ignore', 'use'),
dest='mark_ignore_dl',
help_base='mark about successfully downloaded jobs',
help_prefix_pair=('ignore ', 'use '))
_add_bool_opt('jobs',
None,
'mark-fail',
default=True,
help_base='mark jobs failing verification as such')
_add_bool_opt('jobs',
None,
'mark-empty-fail',
default=False,
help_base='mark jobs without any files as failed')
parser.section('down', 'Download options')
_add_bool_opt('down',
'v ',
'verify-md5',
default=True,
help_base='MD5 verification of SE files',
help_prefix_pair=('enable ', 'disable '))
_add_bool_opt('down',
None,
'',
default=False,
option_prefix_pair=('skip-existing', 'overwrite'),
dest='skip_existing',
help_base='files which are already on local disk',
help_prefix_pair=('skip ', 'overwrite '))
parser.add_text('down',
'o',
'output',
default=None,
help='specify the local output directory')
parser.add_text('down',
'O',
'tmp-dir',
default='/tmp',
help='specify the local tmp directory')
parser.add_text(
'down',
'r',
'retry',
help='how often should a transfer be attempted [Default: 0]')
parser.section('file', 'Local / SE file handling during download')
option_help_base_list = [
('local-ok', 'files of successful jobs in local directory'),
('local-fail', 'files of failed jobs in local directory'),
('se-ok', 'files of successful jobs on SE'),
('se-fail', 'files of failed jobs on the SE'),
]
for (option, help_base) in option_help_base_list:
_add_bool_opt('file',
None,
option,
default=False,
option_prefix_pair=('rm', 'keep'),
help_base=help_base,
help_prefix_pair=('remove ', 'keep '))
parser.section('short_delete', 'Shortcuts for delete options')
parser.add_fset(
'short_delete',
'D',
'just-delete',
help='Delete files from SE and local area - shorthand for:'.ljust(100)
+ '%s',
flag_set='--delete --use-mark-rm --ignore-mark-dl ' +
'--rm-se-fail --rm-local-fail --rm-se-ok --rm-local-ok')
parser.section('short_down', 'Shortcuts for download options')
parser.add_fset(
'short_down',
'm',
'move',
help='Move files from SE - shorthand for:'.ljust(100) + '%s',
flag_set='--verify-md5 --overwrite --mark-dl --use-mark-dl --mark-fail '
+ '--rm-se-fail --rm-local-fail --rm-se-ok --keep-local-ok')
parser.add_fset(
'short_down',
'c',
'copy',
help='Copy files from SE - shorthand for:'.ljust(100) + '%s',
flag_set='--verify-md5 --overwrite --mark-dl --use-mark-dl --mark-fail '
+ '--rm-se-fail --rm-local-fail --keep-se-ok --keep-local-ok')
parser.add_fset(
'short_down',
'j',
'just-copy',
help='Just copy files from SE - shorthand for:'.ljust(100) + '%s',
flag_set=
'--verify-md5 --skip-existing --no-mark-dl --ignore-mark-dl --no-mark-fail '
+ '--keep-se-fail --keep-local-fail --keep-se-ok --keep-local-ok')
parser.add_fset(
'short_down',
's',
'smart-copy',
help='Copy correct files from SE, but remember already downloaded ' +
'files and delete corrupt files - shorthand for: '.ljust(100) + '%s',
flag_set='--verify-md5 --mark-dl --mark-fail --rm-se-fail ' +
'--rm-local-fail --keep-se-ok --keep-local-ok')
parser.add_fset(
'short_down',
'V',
'just-verify',
help='Just verify files on SE - shorthand for:'.ljust(100) + '%s',
flag_set='--verify-md5 --no-mark-dl --keep-se-fail ' +
'--rm-local-fail --keep-se-ok --rm-local-ok --ignore-mark-dl')
options = parser.script_parse(verbose_short=None)
if len(options.args
) != 1: # we need exactly one positional argument (config file)
parser.exit_with_usage(msg='Config file not specified!')
options.opts.threads = int(options.opts.threads)
return options
def _parse_cmd_line():
parser = ScriptOptions(usage='%s [OPTIONS] <parameter definition>')
parser.add_accu(None,
'c',
'collapse',
default=0,
help='Do not collapse dataset infos in display')
parser.add_bool(None,
'a',
'active',
default=False,
help='Show activity state')
parser.add_bool(None,
'd',
'disabled',
default=False,
help='Show disabled parameter sets')
parser.add_bool(None,
'l',
'list-parameters',
default=False,
help='Display parameter list')
parser.add_bool(None,
'L',
'show-sources',
default=False,
help='Show parameter sources')
parser.add_bool(None,
't',
'untracked',
default=False,
help='Display untracked variables')
parser.add_bool(None,
'T',
'persistent',
default=False,
help='Work with persistent parameters')
parser.add_list(None,
'p',
'parameter',
default=[],
help='Specify parameters')
parser.add_text(
None,
'D',
'dataset',
default='',
help='Add dataset splitting (use "True" to simulate a dataset)')
parser.add_text(
None,
'j',
'job',
default=None,
help='Select job to display (used for unbounded parameter spaces)')
parser.add_text(None,
'F',
'factory',
default=None,
help='Select parameter source factory')
parser.add_text(None,
'o',
'output',
default='',
help='Show only specified parameters')
parser.add_text(None,
'S',
'save',
default='',
help='Saves information to specified file')
parser.add_text(None,
'V',
'visible',
default='',
help='Set visible variables')
options = parser.script_parse()
if len(options.args) != 1:
parser.exit_with_usage()
return options