def setup_parser():
# setup cmdline args parser
# main parser
parser = argparse.ArgumentParser(
fromfile_prefix_chars='@',
# usage="%(prog)s ...",
description=dedent_docstring("""\
DataLad provides a unified data distribution with the convenience of git-annex
repositories as a backend. datalad command line tool allows to manipulate
(obtain, create, update, publish, etc.) datasets and their collections."""),
epilog='"Control Your Data"',
formatter_class=argparse.RawDescriptionHelpFormatter,
add_help=False)
# common options
helpers.parser_add_common_opt(parser, 'help')
helpers.parser_add_common_opt(parser, 'log_level')
helpers.parser_add_common_opt(
parser,
'version',
version='datalad %s\n\n%s' % (datalad.__version__, _license_info()))
if __debug__:
parser.add_argument(
'--dbg', action='store_true', dest='common_debug',
help="do not catch exceptions and show exception traceback")
parser.add_argument(
'-C', action='append', dest='change_path', metavar='PATH',
help="""Run as if datalad was started in <path> instead
of the current working directory. When multiple -C options are given,
each subsequent non-absolute -C <path> is interpreted relative to the
preceding -C <path>. This option affects the interpretations of the
path names in that they are made relative to the working directory
caused by the -C option.""")
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# subparsers
subparsers = parser.add_subparsers()
# auto detect all available interfaces and generate a function-based
# API from them
grp_short_descriptions = []
interface_groups = get_interface_groups()
for grp_name, grp_descr, _interfaces in interface_groups:
# for all subcommand modules it can find
cmd_short_descriptions = []
for _intfcls in _interfaces:
_intf = _intfcls()
cmd_name = _intf.__module__.split('.')[-1].replace('_', '-')
# deal with optional parser args
if hasattr(_intf, 'parser_args'):
parser_args = _intf.parser_args
else:
parser_args = dict(formatter_class=argparse.RawDescriptionHelpFormatter)
# use class description, if no explicit description is available
parser_args['description'] = dedent_docstring(_intf.__doc__)
# create subparser, use module suffix as cmd name
subparser = subparsers.add_parser(cmd_name, add_help=False, **parser_args)
# all subparser can report the version
helpers.parser_add_common_opt(
subparser, 'version',
version='datalad %s %s\n\n%s' % (cmd_name, datalad.__version__,
_license_info()))
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
helpers.parser_add_common_opt(subparser, 'log_level')
# let module configure the parser
_intf.setup_parser(subparser)
# logger for command
# configure 'run' function for this command
subparser.set_defaults(func=_intf.call_from_parser,
logger=logging.getLogger(_intf.__module__))
# store short description for later
sdescr = getattr(_intf, 'short_description',
parser_args['description'].split('\n')[0])
cmd_short_descriptions.append((cmd_name, sdescr))
grp_short_descriptions.append(cmd_short_descriptions)
# create command summary
cmd_summary = []
for i, grp in enumerate(interface_groups):
grp_descr = grp[1]
grp_cmds = grp_short_descriptions[i]
cmd_summary.append('\n*%s*\n' % (grp_descr,))
for cd in grp_cmds:
cmd_summary.append(' - %s: %s'
% (cd[0],
textwrap.fill(
cd[1].rstrip(' .'),
75,
#initial_indent=' ' * 4,
subsequent_indent=' ' * 8)))
# we need one last formal section to not have the trailed be
# confused with the last command group
cmd_summary.append('\n*General information*\n')
parser.description = '%s\n%s\n\n%s' \
% (parser.description,
'\n'.join(cmd_summary),
textwrap.fill(dedent_docstring("""\
Detailed usage information for individual commands is
available via command-specific --help, i.e.:
datalad <command> --help"""),
75, initial_indent='', subsequent_indent=''))
return parser
def setup_parser():
# setup cmdline args parser
# main parser
parser = argparse.ArgumentParser(
fromfile_prefix_chars='@',
# usage="%(prog)s ...",
description="""\
DataLad aims to expose (scientific) data available online as a unified data distribution with the convenience of git-annex repositories as a backend.
datalad command line tool facilitates initial construction and update of harvested online datasets. It supports following commands
""",
epilog='"Geet My Data"',
formatter_class=argparse.RawDescriptionHelpFormatter,
add_help=False)
# common options
helpers.parser_add_common_opt(parser, 'help')
helpers.parser_add_common_opt(parser, 'log_level')
helpers.parser_add_common_opt(parser,
'version',
version='datalad %s\n\n%s' %
(datalad.__version__, _license_info()))
if __debug__:
parser.add_argument(
'--dbg',
action='store_true',
dest='common_debug',
help="do not catch exceptions and show exception traceback")
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# subparsers
subparsers = parser.add_subparsers()
# for all subcommand modules it can find
cmd_short_description = []
for cmd in get_commands():
cmd_name = cmd[4:]
subcmdmod = getattr(
__import__('datalad.cmdline', globals(), locals(), [cmd], -1), cmd)
# deal with optional parser args
if 'parser_args' in subcmdmod.__dict__:
parser_args = subcmdmod.parser_args
else:
parser_args = dict()
# use module description, if no explicit description is available
if not 'description' in parser_args:
parser_args['description'] = subcmdmod.__doc__
# create subparser, use module suffix as cmd name
subparser = subparsers.add_parser(cmd_name,
add_help=False,
**parser_args)
# all subparser can report the version
helpers.parser_add_common_opt(
subparser,
'version',
version='datalad %s %s\n\n%s' %
(cmd_name, datalad.__version__, _license_info()))
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
helpers.parser_add_common_opt(subparser, 'log_level')
# let module configure the parser
subcmdmod.setup_parser(subparser)
# logger for command
# configure 'run' function for this command
subparser.set_defaults(func=subcmdmod.run,
logger=logging.getLogger('datalad.%s' % cmd))
# store short description for later
sdescr = getattr(subcmdmod, 'short_description',
parser_args['description'].split('\n')[0])
cmd_short_description.append((cmd_name, sdescr))
# create command summary
cmd_summary = []
for cd in cmd_short_description:
cmd_summary.append('%s\n%s\n\n' \
% (cd[0],
textwrap.fill(cd[1], 75,
initial_indent=' ' * 4,
subsequent_indent=' ' * 4)))
parser.description = '%s\n%s\n\n%s' \
% (parser.description,
'\n'.join(cmd_summary),
textwrap.fill("""\
Detailed usage information for individual commands is
available via command-specific help options, i.e.:
%s <command> --help""" % sys.argv[0],
75, initial_indent='',
subsequent_indent=''))
return parser
def setup_parser(
cmdlineargs,
formatter_class=argparse.RawDescriptionHelpFormatter,
return_subparsers=False):
lgr.log(5, "Starting to setup_parser")
# delay since it can be a heavy import
from ..interface.base import dedent_docstring, get_interface_groups, \
get_cmdline_command_name, alter_interface_docs_for_cmdline
# setup cmdline args parser
parts = {}
# main parser
parser = ArgumentParserDisableAbbrev(
# cannot use '@' because we need to input JSON-LD properties (which might come wit @ prefix)
# MH: question, do we need this at all?
fromfile_prefix_chars=':',
# usage="%(prog)s ...",
description=dedent_docstring("""\
DataLad provides a unified data distribution with the convenience of git-annex
repositories as a backend. DataLad command line tools allow to manipulate
(obtain, create, update, publish, etc.) datasets and their collections."""),
epilog='"Control Your Data"',
formatter_class=formatter_class,
add_help=False)
# common options
helpers.parser_add_common_opt(parser, 'log_level')
helpers.parser_add_common_opt(parser, 'pbs_runner')
helpers.parser_add_common_opt(parser, 'change_path')
helpers.parser_add_common_opt(
parser,
'version',
version='datalad %s\n\n%s' % (datalad.__version__, _license_info()))
if __debug__:
parser.add_argument(
'--dbg', action='store_true', dest='common_debug',
help="enter Python debugger when uncaught exception happens")
parser.add_argument(
'--idbg', action='store_true', dest='common_idebug',
help="enter IPython debugger when uncaught exception happens")
parser.add_argument(
'-c', action='append', dest='cfg_overrides', metavar='KEY=VALUE',
help="""configuration variable setting. Overrides any configuration
read from a file, but is potentially overridden itself by configuration
variables in the process environment.""")
parser.add_argument(
'-f', '--output-format', dest='common_output_format',
default='default',
type=assure_unicode,
metavar="{default,json,json_pp,tailored,'<template>'}",
help="""select format for returned command results. 'default' give one line
per result reporting action, status, path and an optional message;
'json' renders a JSON object with all properties for each result (one per
line); 'json_pp' pretty-prints JSON spanning multiple lines; 'tailored'
enables a command-specific rendering style that is typically
tailored to human consumption (no result output otherwise),
'<template>' reports any value(s) of any result properties in any format
indicated by the template (e.g. '{path}'; compare with JSON
output for all key-value choices). The template syntax follows the Python
"format() language". It is possible to report individual
dictionary values, e.g. '{metadata[name]}'. If a 2nd-level key contains
a colon, e.g. 'music:Genre', ':' must be substituted by '#' in the template,
like so: '{metadata[music#Genre]}'.""")
parser.add_argument(
'--report-status', dest='common_report_status',
choices=['success', 'failure', 'ok', 'notneeded', 'impossible', 'error'],
help="""constrain command result report to records matching the given
status. 'success' is a synonym for 'ok' OR 'notneeded', 'failure' stands
for 'impossible' OR 'error'.""")
parser.add_argument(
'--report-type', dest='common_report_type',
choices=['dataset', 'file'],
action='append',
help="""constrain command result report to records matching the given
type. Can be given more than once to match multiple types.""")
parser.add_argument(
'--on-failure', dest='common_on_failure',
choices=['ignore', 'continue', 'stop'],
# no default: better be configure per-command
help="""when an operation fails: 'ignore' and continue with remaining
operations, the error is logged but does not lead to a non-zero exit code
of the command; 'continue' works like 'ignore', but an error causes a
non-zero exit code; 'stop' halts on first failure and yields non-zero exit
code. A failure is any result with status 'impossible' or 'error'.""")
parser.add_argument(
'--run-before', dest='common_run_before',
nargs='+',
action='append',
metavar=('<PROCEDURE NAME>', 'ARGS'),
help="""Dataset procedure to run before the main command (see run-procedure
command for details). This option can be given more than once to run
multiple procedures in the order in which they were given.
It is important to specify the target dataset via the --dataset argument
of the main command."""),
parser.add_argument(
'--run-after', dest='common_run_after',
nargs='+',
action='append',
metavar=('<PROCEDURE NAME>', 'ARGS'),
help="""Like --run-before, but procedures are executed after the main command
has finished."""),
parser.add_argument(
'--cmd', dest='_', action='store_true',
help="""syntactical helper that can be used to end the list of global
command line options before the subcommand label. Options like
--run-before can take an arbitrary number of arguments and may require
to be followed by a single --cmd in order to enable identification
of the subcommand.""")
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# Before doing anything additional and possibly expensive see may be that
# we have got the command already
need_single_subparser = False if return_subparsers else None
fail_handler = (lambda *a, **kw: True) \
if return_subparsers else fail_with_short_help
try:
parsed_args, unparsed_args = parser._parse_known_args(
cmdlineargs[1:], argparse.Namespace())
if not unparsed_args:
fail_handler(parser, msg="too few arguments", exit_code=2)
lgr.debug("Command line args 1st pass. Parsed: %s Unparsed: %s",
parsed_args, unparsed_args)
except Exception as exc:
lgr.debug("Early parsing failed with %s", exc_str(exc))
need_single_subparser = False
unparsed_args = cmdlineargs[1:] # referenced before assignment otherwise
interface_groups = get_interface_groups()
# TODO: see if we could retain "generator" for plugins
# ATM we need to make it explicit so we could check the command(s) below
# It could at least follow the same destiny as extensions so we would
# just do more iterative "load ups"
interface_groups.append(('plugins', 'Plugins', list(_get_plugins())))
# First unparsed could be either unknown option to top level "datalad"
# or a command. Among unknown could be --help/--help-np which would
# need to be dealt with
unparsed_arg = unparsed_args[0] if unparsed_args else None
if need_single_subparser is not None \
or unparsed_arg in ('--help', '--help-np', '-h'):
need_single_subparser = False
add_entrypoints_to_interface_groups(interface_groups)
elif unparsed_arg.startswith('-'): # unknown option
fail_with_short_help(parser,
msg="unrecognized argument %s" % unparsed_arg,
exit_code=2)
# if we could get a list of options known to parser,
# we could suggest them
# known=get_all_options(parser), provided=unparsed_arg)
else: # the command to handle
known_commands = get_commands_from_groups(interface_groups)
if unparsed_arg not in known_commands:
# need to load all the extensions and try again
add_entrypoints_to_interface_groups(interface_groups)
known_commands = get_commands_from_groups(interface_groups)
if unparsed_arg not in known_commands:
# check if might be coming from known extensions
from ..interface import _known_extension_commands
extension_commands = {
c: e
for e, commands in _known_extension_commands.items()
for c in commands
}
hint = None
if unparsed_arg in extension_commands:
hint = "Command %s is provided by (not installed) extension %s." \
% (unparsed_arg, extension_commands[unparsed_arg])
fail_with_short_help(
parser,
hint=hint,
provided=unparsed_arg,
known=list(known_commands.keys()) + list(extension_commands.keys())
)
if need_single_subparser is None:
need_single_subparser = unparsed_arg
# --help specification was delayed since it causes immediate printout of
# --help output before we setup --help for each command
helpers.parser_add_common_opt(parser, 'help')
grp_short_descriptions = []
# create subparser, use module suffix as cmd name
subparsers = parser.add_subparsers()
for _, _, _interfaces \
in sorted(interface_groups, key=lambda x: x[1]):
# for all subcommand modules it can find
cmd_short_descriptions = []
for _intfspec in _interfaces:
cmd_name = get_cmdline_command_name(_intfspec)
if need_single_subparser and cmd_name != need_single_subparser:
continue
if isinstance(_intfspec[1], dict):
# plugin
_intf = _load_plugin(_intfspec[1]['file'], fail=False)
if _intf is None:
# TODO: add doc why we could skip this one... makes this
# loop harder to extract into a dedicated function
continue
else:
# turn the interface spec into an instance
lgr.log(5, "Importing module %s " % _intfspec[0])
try:
_mod = import_module(_intfspec[0], package='datalad')
except Exception as e:
lgr.error("Internal error, cannot import interface '%s': %s",
_intfspec[0], exc_str(e))
continue
_intf = getattr(_mod, _intfspec[1])
# deal with optional parser args
if hasattr(_intf, 'parser_args'):
parser_args = _intf.parser_args
else:
parser_args = dict(formatter_class=formatter_class)
# use class description, if no explicit description is available
intf_doc = '' if _intf.__doc__ is None else _intf.__doc__.strip()
if hasattr(_intf, '_docs_'):
# expand docs
intf_doc = intf_doc.format(**_intf._docs_)
parser_args['description'] = alter_interface_docs_for_cmdline(
intf_doc)
subparser = subparsers.add_parser(cmd_name, add_help=False, **parser_args)
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
# let module configure the parser
_intf.setup_parser(subparser)
# logger for command
# configure 'run' function for this command
plumbing_args = dict(
func=_intf.call_from_parser,
logger=logging.getLogger(_intf.__module__),
subparser=subparser)
if hasattr(_intf, 'result_renderer_cmdline'):
plumbing_args['result_renderer'] = _intf.result_renderer_cmdline
subparser.set_defaults(**plumbing_args)
parts[cmd_name] = subparser
# store short description for later
sdescr = getattr(_intf, 'short_description',
parser_args['description'].split('\n')[0])
cmd_short_descriptions.append((cmd_name, sdescr))
grp_short_descriptions.append(cmd_short_descriptions)
# create command summary
if '--help' in cmdlineargs or '--help-np' in cmdlineargs:
parser.description = get_description_with_cmd_summary(
grp_short_descriptions,
interface_groups,
parser.description)
parts['datalad'] = parser
lgr.log(5, "Finished setup_parser")
if return_subparsers:
return parts
else:
return parser
def setup_parser(formatter_class=argparse.RawDescriptionHelpFormatter,
return_subparsers=False):
lgr.log(5, "Starting to setup_parser")
# delay since it can be a heavy import
from ..interface.base import dedent_docstring, get_interface_groups, \
get_cmdline_command_name, alter_interface_docs_for_cmdline
# setup cmdline args parser
parts = {}
# main parser
parser = argparse.ArgumentParser(
# cannot use '@' because we need to input JSON-LD properties (which might come wit @ prefix)
# MH: question, do we need this at all?
fromfile_prefix_chars=':',
# usage="%(prog)s ...",
description=dedent_docstring("""\
DataLad provides a unified data distribution with the convenience of git-annex
repositories as a backend. DataLad command line tools allow to manipulate
(obtain, create, update, publish, etc.) datasets and their collections."""
),
epilog='"Control Your Data"',
formatter_class=formatter_class,
add_help=False)
# common options
helpers.parser_add_common_opt(parser, 'help')
helpers.parser_add_common_opt(parser, 'log_level')
helpers.parser_add_common_opt(parser, 'pbs_runner')
helpers.parser_add_common_opt(parser,
'version',
version='datalad %s\n\n%s' %
(datalad.__version__, _license_info()))
if __debug__:
parser.add_argument(
'--dbg',
action='store_true',
dest='common_debug',
help="enter Python debugger when uncaught exception happens")
parser.add_argument(
'--idbg',
action='store_true',
dest='common_idebug',
help="enter IPython debugger when uncaught exception happens")
parser.add_argument('-C',
action='append',
dest='change_path',
metavar='PATH',
help="""run as if datalad was started in <path> instead
of the current working directory. When multiple -C options are given,
each subsequent non-absolute -C <path> is interpreted relative to the
preceding -C <path>. This option affects the interpretations of the
path names in that they are made relative to the working directory
caused by the -C option""")
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# subparsers
subparsers = parser.add_subparsers()
# auto detect all available interfaces and generate a function-based
# API from them
grp_short_descriptions = []
interface_groups = get_interface_groups()
for grp_name, grp_descr, _interfaces in interface_groups:
# for all subcommand modules it can find
cmd_short_descriptions = []
for _intfspec in _interfaces:
# turn the interface spec into an instance
lgr.log(5, "Importing module %s " % _intfspec[0])
_mod = import_module(_intfspec[0], package='datalad')
_intf = getattr(_mod, _intfspec[1])
cmd_name = get_cmdline_command_name(_intfspec)
# deal with optional parser args
if hasattr(_intf, 'parser_args'):
parser_args = _intf.parser_args
else:
parser_args = dict(formatter_class=formatter_class)
# use class description, if no explicit description is available
parser_args['description'] = alter_interface_docs_for_cmdline(
_intf.__doc__.strip())
# create subparser, use module suffix as cmd name
subparser = subparsers.add_parser(cmd_name,
add_help=False,
**parser_args)
# all subparser can report the version
helpers.parser_add_common_opt(
subparser,
'version',
version='datalad %s %s\n\n%s' %
(cmd_name, datalad.__version__, _license_info()))
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
helpers.parser_add_common_opt(subparser, 'log_level')
helpers.parser_add_common_opt(subparser, 'pbs_runner')
# let module configure the parser
_intf.setup_parser(subparser)
# logger for command
# configure 'run' function for this command
plumbing_args = dict(func=_intf.call_from_parser,
logger=logging.getLogger(_intf.__module__),
subparser=subparser)
if hasattr(_intf, 'result_renderer_cmdline'):
plumbing_args[
'result_renderer'] = _intf.result_renderer_cmdline
subparser.set_defaults(**plumbing_args)
# store short description for later
sdescr = getattr(_intf, 'short_description',
parser_args['description'].split('\n')[0])
cmd_short_descriptions.append((cmd_name, sdescr))
parts[cmd_name] = subparser
grp_short_descriptions.append(cmd_short_descriptions)
# create command summary
cmd_summary = []
for i, grp in enumerate(interface_groups):
grp_descr = grp[1]
grp_cmds = grp_short_descriptions[i]
cmd_summary.append('\n*%s*\n' % (grp_descr, ))
for cd in grp_cmds:
cmd_summary.append(' - %-20s %s' % ((
cd[0] + ':',
textwrap.fill(
cd[1].rstrip(' .'),
75,
#initial_indent=' ' * 4,
subsequent_indent=' ' * 8))))
# we need one last formal section to not have the trailed be
# confused with the last command group
cmd_summary.append('\n*General information*\n')
parser.description = '%s\n%s\n\n%s' \
% (parser.description,
'\n'.join(cmd_summary),
textwrap.fill(dedent_docstring("""\
Detailed usage information for individual commands is
available via command-specific --help, i.e.:
datalad <command> --help"""),
75, initial_indent='', subsequent_indent=''))
parts['datalad'] = parser
lgr.log(5, "Finished setup_parser")
if return_subparsers:
return parts
else:
return parser
def setup_parser(formatter_class=argparse.RawDescriptionHelpFormatter,
return_subparsers=False):
lgr.log(5, "Starting to setup_parser")
# delay since it can be a heavy import
from ..interface.base import dedent_docstring, get_interface_groups, \
get_cmdline_command_name, alter_interface_docs_for_cmdline
# setup cmdline args parser
parts = {}
# main parser
parser = argparse.ArgumentParser(
# cannot use '@' because we need to input JSON-LD properties (which might come wit @ prefix)
# MH: question, do we need this at all?
fromfile_prefix_chars=':',
# usage="%(prog)s ...",
description=dedent_docstring("""\
DataLad provides a unified data distribution with the convenience of git-annex
repositories as a backend. DataLad command line tools allow to manipulate
(obtain, create, update, publish, etc.) datasets and their collections."""
),
epilog='"Control Your Data"',
formatter_class=formatter_class,
add_help=False)
# common options
helpers.parser_add_common_opt(parser, 'help')
helpers.parser_add_common_opt(parser, 'log_level')
helpers.parser_add_common_opt(parser, 'pbs_runner')
helpers.parser_add_common_opt(parser, 'change_path')
helpers.parser_add_common_opt(parser,
'version',
version='datalad %s\n\n%s' %
(datalad.__version__, _license_info()))
if __debug__:
parser.add_argument(
'--dbg',
action='store_true',
dest='common_debug',
help="enter Python debugger when uncaught exception happens")
parser.add_argument(
'--idbg',
action='store_true',
dest='common_idebug',
help="enter IPython debugger when uncaught exception happens")
parser.add_argument(
'-c',
action='append',
dest='cfg_overrides',
metavar='KEY=VALUE',
help="""configuration variable setting. Overrides any configuration
read from a file, but is potentially overridden itself by configuration
variables in the process environment.""")
parser.add_argument(
'--output-format',
dest='common_output_format',
default='default',
metavar="{default,json,json_pp,tailored,'<template>'",
help=
"""select format for returned command results. 'default' give one line
per result reporting action, status, path and an optional message;
'json' renders a JSON object with all properties for each result (one per
line); 'json_pp' pretty-prints JSON spanning multiple lines; 'tailored'
enables a command-specific rendering style that is typically
tailored to human consumption (no result output otherwise),
'<template>' reports any value(s) of any result properties in any format
indicated by the template (e.g. '{path}', compare with JSON
output for all key-value choices).""")
parser.add_argument(
'--report-status',
dest='common_report_status',
choices=[
'success', 'failure', 'ok', 'notneeded', 'impossible', 'error'
],
help="""constrain command result report to records matching the given
status. 'success' is a synonym for 'ok' OR 'notneeded', 'failure' stands
for 'impossible' OR 'error'.""")
parser.add_argument(
'--report-type',
dest='common_report_type',
choices=['dataset', 'file'],
action='append',
help="""constrain command result report to records matching the given
type. Can be given more than once to match multiple types.""")
parser.add_argument(
'--on-failure',
dest='common_on_failure',
choices=['ignore', 'continue', 'stop'],
# no default: better be configure per-command
help="""when an operation fails: 'ignore' and continue with remaining
operations, the error is logged but does not lead to a non-zero exit code
of the command; 'continue' works like 'ignore', but an error causes a
non-zero exit code; 'stop' halts on first failure and yields non-zero exit
code. A failure is any result with status 'impossible' or 'error'.""")
parser.add_argument(
'--run-before',
dest='common_run_before',
nargs='+',
action='append',
metavar='PLUGINSPEC',
help="""DataLad plugin to run after the command. PLUGINSPEC is a list
comprised of a plugin name plus optional `key=value` pairs with arguments
for the plugin call (see `plugin` command documentation for details).
This option can be given more than once to run multiple plugins
in the order in which they were given.
For running plugins that require a --dataset argument it is important
to provide the respective dataset as the --dataset argument of the main
command, if it is not in the list of plugin arguments."""),
parser.add_argument(
'--run-after',
dest='common_run_after',
nargs='+',
action='append',
metavar='PLUGINSPEC',
help=
"""Like --run-before, but plugins are executed after the main command
has finished."""),
parser.add_argument(
'--cmd',
dest='_',
action='store_true',
help="""syntactical helper that can be used to end the list of global
command line options before the subcommand label. Options like
--run-before can take an arbitray number of arguments and may require
to be followed by a single --cmd in order to enable identification
of the subcommand.""")
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# subparsers
subparsers = parser.add_subparsers()
# auto detect all available interfaces and generate a function-based
# API from them
grp_short_descriptions = []
interface_groups = get_interface_groups()
for grp_name, grp_descr, _interfaces \
in sorted(interface_groups, key=lambda x: x[1]):
# for all subcommand modules it can find
cmd_short_descriptions = []
for _intfspec in _interfaces:
# turn the interface spec into an instance
lgr.log(5, "Importing module %s " % _intfspec[0])
_mod = import_module(_intfspec[0], package='datalad')
_intf = getattr(_mod, _intfspec[1])
cmd_name = get_cmdline_command_name(_intfspec)
# deal with optional parser args
if hasattr(_intf, 'parser_args'):
parser_args = _intf.parser_args
else:
parser_args = dict(formatter_class=formatter_class)
# use class description, if no explicit description is available
intf_doc = _intf.__doc__.strip()
if hasattr(_intf, '_docs_'):
# expand docs
intf_doc = intf_doc.format(**_intf._docs_)
parser_args['description'] = alter_interface_docs_for_cmdline(
intf_doc)
# create subparser, use module suffix as cmd name
subparser = subparsers.add_parser(cmd_name,
add_help=False,
**parser_args)
# all subparser can report the version
helpers.parser_add_common_opt(
subparser,
'version',
version='datalad %s %s\n\n%s' %
(cmd_name, datalad.__version__, _license_info()))
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
helpers.parser_add_common_opt(subparser, 'log_level')
helpers.parser_add_common_opt(subparser, 'pbs_runner')
# let module configure the parser
_intf.setup_parser(subparser)
# logger for command
# configure 'run' function for this command
plumbing_args = dict(func=_intf.call_from_parser,
logger=logging.getLogger(_intf.__module__),
subparser=subparser)
if hasattr(_intf, 'result_renderer_cmdline'):
plumbing_args[
'result_renderer'] = _intf.result_renderer_cmdline
subparser.set_defaults(**plumbing_args)
# store short description for later
sdescr = getattr(_intf, 'short_description',
parser_args['description'].split('\n')[0])
cmd_short_descriptions.append((cmd_name, sdescr))
parts[cmd_name] = subparser
grp_short_descriptions.append(cmd_short_descriptions)
# create command summary
cmd_summary = []
console_width = shutil.get_terminal_size()[0] \
if hasattr(shutil, 'get_terminal_size') else 80
for i, grp in enumerate(sorted(interface_groups, key=lambda x: x[1])):
grp_descr = grp[1]
grp_cmds = grp_short_descriptions[i]
cmd_summary.append('\n*%s*\n' % (grp_descr, ))
for cd in grp_cmds:
cmd_summary.append(' %s\n%s' %
((cd[0],
textwrap.fill(cd[1].rstrip(' .'),
console_width - 5,
initial_indent=' ' * 6,
subsequent_indent=' ' * 6))))
# we need one last formal section to not have the trailed be
# confused with the last command group
cmd_summary.append('\n*General information*\n')
parser.description = '%s\n%s\n\n%s' \
% (parser.description,
'\n'.join(cmd_summary),
textwrap.fill(dedent_docstring("""\
Detailed usage information for individual commands is
available via command-specific --help, i.e.:
datalad <command> --help"""),
console_width - 5, initial_indent='', subsequent_indent=''))
parts['datalad'] = parser
lgr.log(5, "Finished setup_parser")
if return_subparsers:
return parts
else:
return parser
def setup_parser(
formatter_class=argparse.RawDescriptionHelpFormatter,
return_subparsers=False):
lgr.log(5, "Starting to setup_parser")
# delay since it can be a heavy import
from ..interface.base import dedent_docstring, get_interface_groups, \
get_cmdline_command_name, alter_interface_docs_for_cmdline
# setup cmdline args parser
parts = {}
# main parser
parser = argparse.ArgumentParser(
# cannot use '@' because we need to input JSON-LD properties (which might come wit @ prefix)
# MH: question, do we need this at all?
fromfile_prefix_chars=':',
# usage="%(prog)s ...",
description=dedent_docstring("""\
DataLad provides a unified data distribution with the convenience of git-annex
repositories as a backend. DataLad command line tools allow to manipulate
(obtain, create, update, publish, etc.) datasets and their collections."""),
epilog='"Control Your Data"',
formatter_class=formatter_class,
add_help=False)
# common options
helpers.parser_add_common_opt(parser, 'help')
helpers.parser_add_common_opt(parser, 'log_level')
helpers.parser_add_common_opt(parser, 'pbs_runner')
helpers.parser_add_common_opt(
parser,
'version',
version='datalad %s\n\n%s' % (datalad.__version__, _license_info()))
if __debug__:
parser.add_argument(
'--dbg', action='store_true', dest='common_debug',
help="enter Python debugger when uncaught exception happens")
parser.add_argument(
'--idbg', action='store_true', dest='common_idebug',
help="enter IPython debugger when uncaught exception happens")
parser.add_argument(
'-C', action='append', dest='change_path', metavar='PATH',
help="""run as if datalad was started in <path> instead
of the current working directory. When multiple -C options are given,
each subsequent non-absolute -C <path> is interpreted relative to the
preceding -C <path>. This option affects the interpretations of the
path names in that they are made relative to the working directory
caused by the -C option""")
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# subparsers
subparsers = parser.add_subparsers()
# auto detect all available interfaces and generate a function-based
# API from them
grp_short_descriptions = []
interface_groups = get_interface_groups()
for grp_name, grp_descr, _interfaces in interface_groups:
# for all subcommand modules it can find
cmd_short_descriptions = []
for _intfspec in _interfaces:
# turn the interface spec into an instance
lgr.log(5, "Importing module %s " % _intfspec[0])
_mod = import_module(_intfspec[0], package='datalad')
_intf = getattr(_mod, _intfspec[1])
cmd_name = get_cmdline_command_name(_intfspec)
# deal with optional parser args
if hasattr(_intf, 'parser_args'):
parser_args = _intf.parser_args
else:
parser_args = dict(formatter_class=formatter_class)
# use class description, if no explicit description is available
parser_args['description'] = alter_interface_docs_for_cmdline(
_intf.__doc__.strip())
# create subparser, use module suffix as cmd name
subparser = subparsers.add_parser(cmd_name, add_help=False, **parser_args)
# all subparser can report the version
helpers.parser_add_common_opt(
subparser, 'version',
version='datalad %s %s\n\n%s' % (cmd_name, datalad.__version__,
_license_info()))
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
helpers.parser_add_common_opt(subparser, 'log_level')
helpers.parser_add_common_opt(subparser, 'pbs_runner')
# let module configure the parser
_intf.setup_parser(subparser)
# logger for command
# configure 'run' function for this command
plumbing_args = dict(
func=_intf.call_from_parser,
logger=logging.getLogger(_intf.__module__),
subparser=subparser)
if hasattr(_intf, 'result_renderer_cmdline'):
plumbing_args['result_renderer'] = _intf.result_renderer_cmdline
subparser.set_defaults(**plumbing_args)
# store short description for later
sdescr = getattr(_intf, 'short_description',
parser_args['description'].split('\n')[0])
cmd_short_descriptions.append((cmd_name, sdescr))
parts[cmd_name] = subparser
grp_short_descriptions.append(cmd_short_descriptions)
# create command summary
cmd_summary = []
console_width = shutil.get_terminal_size()[0] \
if hasattr(shutil, 'get_terminal_size') else 80
for i, grp in enumerate(interface_groups):
grp_descr = grp[1]
grp_cmds = grp_short_descriptions[i]
cmd_summary.append('\n*%s*\n' % (grp_descr,))
for cd in grp_cmds:
cmd_summary.append(' %s\n%s'
% ((cd[0],
textwrap.fill(
cd[1].rstrip(' .'),
console_width - 5,
initial_indent=' ' * 6,
subsequent_indent=' ' * 6))))
# we need one last formal section to not have the trailed be
# confused with the last command group
cmd_summary.append('\n*General information*\n')
parser.description = '%s\n%s\n\n%s' \
% (parser.description,
'\n'.join(cmd_summary),
textwrap.fill(dedent_docstring("""\
Detailed usage information for individual commands is
available via command-specific --help, i.e.:
datalad <command> --help"""),
console_width - 5, initial_indent='', subsequent_indent=''))
parts['datalad'] = parser
lgr.log(5, "Finished setup_parser")
if return_subparsers:
return parts
else:
return parser
def setup_parser():
# setup cmdline args parser
# main parser
parser = argparse.ArgumentParser(
fromfile_prefix_chars='@',
# usage="%(prog)s ...",
description="""\
DataLad aims to expose (scientific) data available online as a unified data distribution with the convenience of git-annex repositories as a backend.
datalad command line tool facilitates initial construction and update of harvested online datasets. It supports following commands
""",
epilog='"Geet My Data"',
formatter_class=argparse.RawDescriptionHelpFormatter,
add_help=False
)
# common options
helpers.parser_add_common_opt(parser, 'help')
helpers.parser_add_common_opt(parser, 'log_level')
helpers.parser_add_common_opt(parser,
'version',
version='datalad %s\n\n%s' % (datalad.__version__,
_license_info()))
if __debug__:
parser.add_argument(
'--dbg', action='store_true', dest='common_debug',
help="do not catch exceptions and show exception traceback")
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# subparsers
subparsers = parser.add_subparsers()
# for all subcommand modules it can find
cmd_short_description = []
for cmd in get_commands():
cmd_name = cmd[4:]
subcmdmod = getattr(__import__('datalad.cmdline',
globals(), locals(),
[cmd], -1),
cmd)
# deal with optional parser args
if 'parser_args' in subcmdmod.__dict__:
parser_args = subcmdmod.parser_args
else:
parser_args = dict()
# use module description, if no explicit description is available
if not 'description' in parser_args:
parser_args['description'] = subcmdmod.__doc__
# create subparser, use module suffix as cmd name
subparser = subparsers.add_parser(cmd_name, add_help=False, **parser_args)
# all subparser can report the version
helpers.parser_add_common_opt(
subparser, 'version',
version='datalad %s %s\n\n%s' % (cmd_name, datalad.__version__,
_license_info()))
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
helpers.parser_add_common_opt(subparser, 'log_level')
# let module configure the parser
subcmdmod.setup_parser(subparser)
# logger for command
# configure 'run' function for this command
subparser.set_defaults(func=subcmdmod.run,
logger=logging.getLogger('datalad.%s' % cmd))
# store short description for later
sdescr = getattr(subcmdmod, 'short_description',
parser_args['description'].split('\n')[0])
cmd_short_description.append((cmd_name, sdescr))
# create command summary
cmd_summary = []
for cd in cmd_short_description:
cmd_summary.append('%s\n%s\n\n' \
% (cd[0],
textwrap.fill(cd[1], 75,
initial_indent=' ' * 4,
subsequent_indent=' ' * 4)))
parser.description = '%s\n%s\n\n%s' \
% (parser.description,
'\n'.join(cmd_summary),
textwrap.fill("""\
Detailed usage information for individual commands is
available via command-specific help options, i.e.:
%s <command> --help""" % sys.argv[0],
75, initial_indent='',
subsequent_indent=''))
return parser
def setup_parser(
cmdlineargs,
formatter_class=argparse.RawDescriptionHelpFormatter,
return_subparsers=False,
# Was this triggered by argparse?
completing=False,
# prevent loading of extension entrypoints when --help is requested
# this is enabled when building docs to avoid pollution of generated
# manpages with extensions commands (that should appear in their own
# docs, but not in the core datalad package docs)
help_ignore_extensions=False):
lgr.log(5, "Starting to setup_parser")
# delay since it can be a heavy import
from ..interface.base import dedent_docstring, get_interface_groups, \
get_cmdline_command_name, alter_interface_docs_for_cmdline, \
load_interface, get_cmd_doc, get_cmd_ex
# setup cmdline args parser
parts = {}
# main parser
parser = ArgumentParserDisableAbbrev(
fromfile_prefix_chars=None,
# usage="%(prog)s ...",
description=dedent_docstring("""\
Comprehensive data management solution
DataLad provides a unified data distribution system built on the Git
and Git-annex. DataLad command line tools allow to manipulate (obtain,
create, update, publish, etc.) datasets and provide a comprehensive
toolbox for joint management of data and code. Compared to Git/annex
it primarily extends their functionality to transparently and
simultaneously work with multiple inter-related repositories."""),
epilog='"Be happy!"',
formatter_class=formatter_class,
add_help=False)
# common options
helpers.parser_add_common_options(parser)
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# Before doing anything additional and possibly expensive see may be that
# we have got the command already
need_single_subparser = False if return_subparsers else None
fail_handler = (lambda *a, **kw: True) \
if return_subparsers else fail_with_short_help
try:
parsed_args, unparsed_args = parser._parse_known_args(
cmdlineargs[1:], argparse.Namespace())
# before anything handle possible datalad --version
if not unparsed_args and getattr(parsed_args, 'version', None):
parsed_args.version() # will exit with 0
if not (completing or unparsed_args):
fail_handler(parser, msg="too few arguments", exit_code=2)
lgr.debug(
"Command line args 1st pass for DataLad %s. Parsed: %s Unparsed: %s",
__full_version__, parsed_args, unparsed_args)
except Exception as exc:
lgr.debug("Early parsing failed with %s", exc_str(exc))
need_single_subparser = False
unparsed_args = cmdlineargs[
1:] # referenced before assignment otherwise
interface_groups = get_interface_groups()
# First unparsed could be either unknown option to top level "datalad"
# or a command. Among unknown could be --help/--help-np which would
# need to be dealt with
unparsed_arg = unparsed_args[0] if unparsed_args else None
if need_single_subparser is not None \
or unparsed_arg in ('--help', '--help-np', '-h'):
need_single_subparser = False
if not help_ignore_extensions:
add_entrypoints_to_interface_groups(interface_groups)
elif not completing and unparsed_arg.startswith('-'): # unknown option
fail_with_short_help(parser,
msg="unrecognized argument %s" % unparsed_arg,
exit_code=2)
# if we could get a list of options known to parser,
# we could suggest them
# known=get_all_options(parser), provided=unparsed_arg)
else: # the command to handle
known_commands = get_commands_from_groups(interface_groups)
if unparsed_arg not in known_commands:
# need to load all the extensions and try again
add_entrypoints_to_interface_groups(interface_groups)
known_commands = get_commands_from_groups(interface_groups)
if unparsed_arg not in known_commands:
# check if might be coming from known extensions
from ..interface import (
_known_extension_commands,
_deprecated_commands,
)
extension_commands = {
c: e
for e, commands in _known_extension_commands.items()
for c in commands
}
hint = None
if unparsed_arg in extension_commands:
hint = "Command %s is provided by (not installed) extension %s." \
% (unparsed_arg, extension_commands[unparsed_arg])
elif unparsed_arg in _deprecated_commands:
hint_cmd = _deprecated_commands[unparsed_arg]
hint = "Command %r was deprecated" % unparsed_arg
hint += (" in favor of %r command." %
hint_cmd) if hint_cmd else '.'
if not completing:
fail_with_short_help(parser,
hint=hint,
provided=unparsed_arg,
known=list(known_commands.keys()) +
list(extension_commands.keys()))
if need_single_subparser is None:
need_single_subparser = unparsed_arg
# --help specification was delayed since it causes immediate printout of
# --help output before we setup --help for each command
helpers.parser_add_common_opt(parser, 'help')
grp_short_descriptions = defaultdict(list)
# create subparser, use module suffix as cmd name
subparsers = parser.add_subparsers()
for group_name, _, _interfaces \
in sorted(interface_groups, key=lambda x: x[1]):
for _intfspec in _interfaces:
cmd_name = get_cmdline_command_name(_intfspec)
if need_single_subparser and cmd_name != need_single_subparser:
continue
_intf = load_interface(_intfspec)
if _intf is None:
# TODO(yoh): add doc why we could skip this one... makes this
# loop harder to extract into a dedicated function
continue
# deal with optional parser args
if hasattr(_intf, 'parser_args'):
parser_args = _intf.parser_args
else:
parser_args = dict(formatter_class=formatter_class)
# use class description, if no explicit description is available
intf_doc = get_cmd_doc(_intf)
parser_args['description'] = alter_interface_docs_for_cmdline(
intf_doc)
if hasattr(_intf, '_examples_'):
intf_ex = alter_interface_docs_for_cmdline(
get_cmd_ex(_intf))
parser_args['description'] += intf_ex
subparser = subparsers.add_parser(cmd_name,
add_help=False,
**parser_args)
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
# let module configure the parser
_intf.setup_parser(subparser)
# and we would add custom handler for --version
helpers.parser_add_version_opt(subparser,
_intf.__module__.split('.', 1)[0],
include_name=True)
# logger for command
# configure 'run' function for this command
plumbing_args = dict(func=_intf.call_from_parser,
logger=logging.getLogger(_intf.__module__),
subparser=subparser)
if hasattr(_intf, 'result_renderer_cmdline'):
plumbing_args[
'result_renderer'] = _intf.result_renderer_cmdline
subparser.set_defaults(**plumbing_args)
parts[cmd_name] = subparser
# store short description for later
sdescr = getattr(_intf, 'short_description',
parser_args['description'].split('\n')[0])
grp_short_descriptions[group_name].append((cmd_name, sdescr))
# create command summary
if '--help' in cmdlineargs or '--help-np' in cmdlineargs:
parser.description = get_description_with_cmd_summary(
grp_short_descriptions, interface_groups, parser.description)
parts['datalad'] = parser
lgr.log(5, "Finished setup_parser")
if return_subparsers:
return parts
else:
return parser
def setup_parser(
cmdlineargs,
formatter_class=argparse.RawDescriptionHelpFormatter,
return_subparsers=False,
# prevent loading of extension entrypoints when --help is requested
# this is enabled when building docs to avoid pollution of generated
# manpages with extensions commands (that should appear in their own
# docs, but not in the core datalad package docs)
help_ignore_extensions=False):
lgr.log(5, "Starting to setup_parser")
# delay since it can be a heavy import
from ..interface.base import dedent_docstring, get_interface_groups, \
get_cmdline_command_name, alter_interface_docs_for_cmdline, \
load_interface, get_cmd_doc
# setup cmdline args parser
parts = {}
# main parser
parser = ArgumentParserDisableAbbrev(
fromfile_prefix_chars=None,
# usage="%(prog)s ...",
description=dedent_docstring("""\
Comprehensive data management solution
DataLad provides a unified data distribution system built on the Git
and Git-annex. DataLad command line tools allow to manipulate (obtain,
create, update, publish, etc.) datasets and provide a comprehensive
toolbox for joint management of data and code. Compared to Git/annex
it primarly extends their functionality to transparently and
simultaneously work with multiple inter-related repositories."""),
epilog='"Be happy!"',
formatter_class=formatter_class,
add_help=False)
# common options
helpers.parser_add_common_opt(parser, 'log_level')
helpers.parser_add_common_opt(parser, 'pbs_runner')
helpers.parser_add_common_opt(parser, 'change_path')
helpers.parser_add_common_opt(
parser,
'version',
version='datalad %s\n' % datalad.__version__)
if __debug__:
parser.add_argument(
'--dbg', action='store_true', dest='common_debug',
help="enter Python debugger when uncaught exception happens")
parser.add_argument(
'--idbg', action='store_true', dest='common_idebug',
help="enter IPython debugger when uncaught exception happens")
parser.add_argument(
'-c', action='append', dest='cfg_overrides', metavar='KEY=VALUE',
help="""configuration variable setting. Overrides any configuration
read from a file, but is potentially overridden itself by configuration
variables in the process environment.""")
parser.add_argument(
'-f', '--output-format', dest='common_output_format',
default='default',
type=assure_unicode,
metavar="{default,json,json_pp,tailored,'<template>'}",
help="""select format for returned command results. 'default' give one line
per result reporting action, status, path and an optional message;
'json' renders a JSON object with all properties for each result (one per
line); 'json_pp' pretty-prints JSON spanning multiple lines; 'tailored'
enables a command-specific rendering style that is typically
tailored to human consumption (no result output otherwise),
'<template>' reports any value(s) of any result properties in any format
indicated by the template (e.g. '{path}'; compare with JSON
output for all key-value choices). The template syntax follows the Python
"format() language". It is possible to report individual
dictionary values, e.g. '{metadata[name]}'. If a 2nd-level key contains
a colon, e.g. 'music:Genre', ':' must be substituted by '#' in the template,
like so: '{metadata[music#Genre]}'.""")
parser.add_argument(
'--report-status', dest='common_report_status',
choices=['success', 'failure', 'ok', 'notneeded', 'impossible', 'error'],
help="""constrain command result report to records matching the given
status. 'success' is a synonym for 'ok' OR 'notneeded', 'failure' stands
for 'impossible' OR 'error'.""")
parser.add_argument(
'--report-type', dest='common_report_type',
choices=['dataset', 'file'],
action='append',
help="""constrain command result report to records matching the given
type. Can be given more than once to match multiple types.""")
parser.add_argument(
'--on-failure', dest='common_on_failure',
choices=['ignore', 'continue', 'stop'],
# no default: better be configure per-command
help="""when an operation fails: 'ignore' and continue with remaining
operations, the error is logged but does not lead to a non-zero exit code
of the command; 'continue' works like 'ignore', but an error causes a
non-zero exit code; 'stop' halts on first failure and yields non-zero exit
code. A failure is any result with status 'impossible' or 'error'.""")
parser.add_argument(
'--proc-pre', dest='common_proc_pre',
nargs='+',
action='append',
metavar=('<PROCEDURE NAME>', 'ARGS'),
help="""Dataset procedure to run before the main command (see run-procedure
command for details). This option can be given more than once to run
multiple procedures in the order in which they were given.
It is important to specify the target dataset via the --dataset argument
of the main command."""),
parser.add_argument(
'--proc-post', dest='common_proc_post',
nargs='+',
action='append',
metavar=('<PROCEDURE NAME>', 'ARGS'),
help="""Like --proc-pre, but procedures are executed after the main command
has finished."""),
parser.add_argument(
'--cmd', dest='_', action='store_true',
help="""syntactical helper that can be used to end the list of global
command line options before the subcommand label. Options like
--proc-pre can take an arbitrary number of arguments and may require
to be followed by a single --cmd in order to enable identification
of the subcommand.""")
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# Before doing anything additional and possibly expensive see may be that
# we have got the command already
need_single_subparser = False if return_subparsers else None
fail_handler = (lambda *a, **kw: True) \
if return_subparsers else fail_with_short_help
try:
parsed_args, unparsed_args = parser._parse_known_args(
cmdlineargs[1:], argparse.Namespace())
if not unparsed_args:
fail_handler(parser, msg="too few arguments", exit_code=2)
lgr.debug("Command line args 1st pass. Parsed: %s Unparsed: %s",
parsed_args, unparsed_args)
except Exception as exc:
lgr.debug("Early parsing failed with %s", exc_str(exc))
need_single_subparser = False
unparsed_args = cmdlineargs[1:] # referenced before assignment otherwise
interface_groups = get_interface_groups(include_plugins=True)
# First unparsed could be either unknown option to top level "datalad"
# or a command. Among unknown could be --help/--help-np which would
# need to be dealt with
unparsed_arg = unparsed_args[0] if unparsed_args else None
if need_single_subparser is not None \
or unparsed_arg in ('--help', '--help-np', '-h'):
need_single_subparser = False
if not help_ignore_extensions:
add_entrypoints_to_interface_groups(interface_groups)
elif unparsed_arg.startswith('-'): # unknown option
fail_with_short_help(parser,
msg="unrecognized argument %s" % unparsed_arg,
exit_code=2)
# if we could get a list of options known to parser,
# we could suggest them
# known=get_all_options(parser), provided=unparsed_arg)
else: # the command to handle
known_commands = get_commands_from_groups(interface_groups)
if unparsed_arg not in known_commands:
# need to load all the extensions and try again
add_entrypoints_to_interface_groups(interface_groups)
known_commands = get_commands_from_groups(interface_groups)
if unparsed_arg not in known_commands:
# check if might be coming from known extensions
from ..interface import _known_extension_commands
extension_commands = {
c: e
for e, commands in _known_extension_commands.items()
for c in commands
}
hint = None
if unparsed_arg in extension_commands:
hint = "Command %s is provided by (not installed) extension %s." \
% (unparsed_arg, extension_commands[unparsed_arg])
fail_with_short_help(
parser,
hint=hint,
provided=unparsed_arg,
known=list(known_commands.keys()) + list(extension_commands.keys())
)
if need_single_subparser is None:
need_single_subparser = unparsed_arg
# --help specification was delayed since it causes immediate printout of
# --help output before we setup --help for each command
helpers.parser_add_common_opt(parser, 'help')
grp_short_descriptions = defaultdict(list)
# create subparser, use module suffix as cmd name
subparsers = parser.add_subparsers()
for group_name, _, _interfaces \
in sorted(interface_groups, key=lambda x: x[1]):
for _intfspec in _interfaces:
cmd_name = get_cmdline_command_name(_intfspec)
if need_single_subparser and cmd_name != need_single_subparser:
continue
_intf = load_interface(_intfspec)
if _intf is None:
# TODO(yoh): add doc why we could skip this one... makes this
# loop harder to extract into a dedicated function
continue
# deal with optional parser args
if hasattr(_intf, 'parser_args'):
parser_args = _intf.parser_args
else:
parser_args = dict(formatter_class=formatter_class)
# use class description, if no explicit description is available
intf_doc = get_cmd_doc(_intf)
parser_args['description'] = alter_interface_docs_for_cmdline(
intf_doc)
subparser = subparsers.add_parser(cmd_name, add_help=False, **parser_args)
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
# let module configure the parser
_intf.setup_parser(subparser)
# logger for command
# configure 'run' function for this command
plumbing_args = dict(
func=_intf.call_from_parser,
logger=logging.getLogger(_intf.__module__),
subparser=subparser)
if hasattr(_intf, 'result_renderer_cmdline'):
plumbing_args['result_renderer'] = _intf.result_renderer_cmdline
subparser.set_defaults(**plumbing_args)
parts[cmd_name] = subparser
# store short description for later
sdescr = getattr(_intf, 'short_description',
parser_args['description'].split('\n')[0])
grp_short_descriptions[group_name].append((cmd_name, sdescr))
# create command summary
if '--help' in cmdlineargs or '--help-np' in cmdlineargs:
parser.description = get_description_with_cmd_summary(
grp_short_descriptions,
interface_groups,
parser.description)
parts['datalad'] = parser
lgr.log(5, "Finished setup_parser")
if return_subparsers:
return parts
else:
return parser
def setup_parser(
cmdlineargs,
formatter_class=argparse.RawDescriptionHelpFormatter,
return_subparsers=False,
# Was this triggered by argparse?
completing=False,
# prevent loading of extension entrypoints when --help is requested
# this is enabled when building docs to avoid pollution of generated
# manpages with extensions commands (that should appear in their own
# docs, but not in the core datalad package docs)
help_ignore_extensions=False):
lgr.log(5, "Starting to setup_parser")
# delay since it can be a heavy import
from ..interface.base import dedent_docstring, get_interface_groups, \
get_cmdline_command_name, alter_interface_docs_for_cmdline, \
load_interface, get_cmd_doc, get_cmd_ex
# setup cmdline args parser
parts = {}
# main parser
parser = ArgumentParserDisableAbbrev(
fromfile_prefix_chars=None,
# usage="%(prog)s ...",
description=dedent_docstring("""\
Comprehensive data management solution
DataLad provides a unified data distribution system built on the Git
and Git-annex. DataLad command line tools allow to manipulate (obtain,
create, update, publish, etc.) datasets and provide a comprehensive
toolbox for joint management of data and code. Compared to Git/annex
it primarly extends their functionality to transparently and
simultaneously work with multiple inter-related repositories."""),
epilog='"Be happy!"',
formatter_class=formatter_class,
add_help=False)
# common options
helpers.parser_add_common_opt(parser, 'log_level')
helpers.parser_add_common_opt(parser, 'pbs_runner')
helpers.parser_add_common_opt(parser, 'change_path')
helpers.parser_add_common_opt(parser,
'version',
version='datalad %s\n' % datalad.__version__)
if __debug__:
parser.add_argument(
'--dbg',
action='store_true',
dest='common_debug',
help="enter Python debugger when uncaught exception happens")
parser.add_argument(
'--idbg',
action='store_true',
dest='common_idebug',
help="enter IPython debugger when uncaught exception happens")
parser.add_argument(
'-c',
action='append',
dest='cfg_overrides',
metavar='KEY=VALUE',
help="""configuration variable setting. Overrides any configuration
read from a file, but is potentially overridden itself by configuration
variables in the process environment.""")
parser.add_argument(
'-f',
'--output-format',
dest='common_output_format',
default='default',
type=assure_unicode,
metavar="{default,json,json_pp,tailored,'<template>'}",
help=
"""select format for returned command results. 'default' give one line
per result reporting action, status, path and an optional message;
'json' renders a JSON object with all properties for each result (one per
line); 'json_pp' pretty-prints JSON spanning multiple lines; 'tailored'
enables a command-specific rendering style that is typically
tailored to human consumption (no result output otherwise),
'<template>' reports any value(s) of any result properties in any format
indicated by the template (e.g. '{path}'; compare with JSON
output for all key-value choices). The template syntax follows the Python
"format() language". It is possible to report individual
dictionary values, e.g. '{metadata[name]}'. If a 2nd-level key contains
a colon, e.g. 'music:Genre', ':' must be substituted by '#' in the template,
like so: '{metadata[music#Genre]}'.""")
parser.add_argument(
'--report-status',
dest='common_report_status',
choices=[
'success', 'failure', 'ok', 'notneeded', 'impossible', 'error'
],
help="""constrain command result report to records matching the given
status. 'success' is a synonym for 'ok' OR 'notneeded', 'failure' stands
for 'impossible' OR 'error'.""")
parser.add_argument(
'--report-type',
dest='common_report_type',
choices=['dataset', 'file'],
action='append',
help="""constrain command result report to records matching the given
type. Can be given more than once to match multiple types.""")
parser.add_argument(
'--on-failure',
dest='common_on_failure',
choices=['ignore', 'continue', 'stop'],
# no default: better be configure per-command
help="""when an operation fails: 'ignore' and continue with remaining
operations, the error is logged but does not lead to a non-zero exit code
of the command; 'continue' works like 'ignore', but an error causes a
non-zero exit code; 'stop' halts on first failure and yields non-zero exit
code. A failure is any result with status 'impossible' or 'error'.""")
parser.add_argument(
'--cmd',
dest='_',
action='store_true',
help="""syntactical helper that can be used to end the list of global
command line options before the subcommand label. Options taking
an arbitrary number of arguments may require to be followed by a single
--cmd in order to enable identification of the subcommand.""")
# yoh: atm we only dump to console. Might adopt the same separation later on
# and for consistency will call it --verbose-level as well for now
# log-level is set via common_opts ATM
# parser.add_argument('--log-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_log_level',
# help="""level of verbosity in log files. By default
# everything, including debug messages is logged.""")
#parser.add_argument('-l', '--verbose-level',
# choices=('critical', 'error', 'warning', 'info', 'debug'),
# dest='common_verbose_level',
# help="""level of verbosity of console output. By default
# only warnings and errors are printed.""")
# Before doing anything additional and possibly expensive see may be that
# we have got the command already
need_single_subparser = False if return_subparsers else None
fail_handler = (lambda *a, **kw: True) \
if return_subparsers else fail_with_short_help
try:
parsed_args, unparsed_args = parser._parse_known_args(
cmdlineargs[1:], argparse.Namespace())
if not (completing or unparsed_args):
fail_handler(parser, msg="too few arguments", exit_code=2)
lgr.debug("Command line args 1st pass. Parsed: %s Unparsed: %s",
parsed_args, unparsed_args)
except Exception as exc:
lgr.debug("Early parsing failed with %s", exc_str(exc))
need_single_subparser = False
unparsed_args = cmdlineargs[
1:] # referenced before assignment otherwise
interface_groups = get_interface_groups(include_plugins=True)
# First unparsed could be either unknown option to top level "datalad"
# or a command. Among unknown could be --help/--help-np which would
# need to be dealt with
unparsed_arg = unparsed_args[0] if unparsed_args else None
if need_single_subparser is not None \
or unparsed_arg in ('--help', '--help-np', '-h'):
need_single_subparser = False
if not help_ignore_extensions:
add_entrypoints_to_interface_groups(interface_groups)
elif not completing and unparsed_arg.startswith('-'): # unknown option
fail_with_short_help(parser,
msg="unrecognized argument %s" % unparsed_arg,
exit_code=2)
# if we could get a list of options known to parser,
# we could suggest them
# known=get_all_options(parser), provided=unparsed_arg)
else: # the command to handle
known_commands = get_commands_from_groups(interface_groups)
if unparsed_arg not in known_commands:
# need to load all the extensions and try again
add_entrypoints_to_interface_groups(interface_groups)
known_commands = get_commands_from_groups(interface_groups)
if unparsed_arg not in known_commands:
# check if might be coming from known extensions
from ..interface import (
_known_extension_commands,
_deprecated_commands,
)
extension_commands = {
c: e
for e, commands in _known_extension_commands.items()
for c in commands
}
hint = None
if unparsed_arg in extension_commands:
hint = "Command %s is provided by (not installed) extension %s." \
% (unparsed_arg, extension_commands[unparsed_arg])
elif unparsed_arg in _deprecated_commands:
hint_cmd = _deprecated_commands[unparsed_arg]
hint = "Command %r was deprecated" % unparsed_arg
hint += (" in favor of %r command." %
hint_cmd) if hint_cmd else '.'
if not completing:
fail_with_short_help(parser,
hint=hint,
provided=unparsed_arg,
known=list(known_commands.keys()) +
list(extension_commands.keys()))
if need_single_subparser is None:
need_single_subparser = unparsed_arg
# --help specification was delayed since it causes immediate printout of
# --help output before we setup --help for each command
helpers.parser_add_common_opt(parser, 'help')
grp_short_descriptions = defaultdict(list)
# create subparser, use module suffix as cmd name
subparsers = parser.add_subparsers()
for group_name, _, _interfaces \
in sorted(interface_groups, key=lambda x: x[1]):
for _intfspec in _interfaces:
cmd_name = get_cmdline_command_name(_intfspec)
if need_single_subparser and cmd_name != need_single_subparser:
continue
_intf = load_interface(_intfspec)
if _intf is None:
# TODO(yoh): add doc why we could skip this one... makes this
# loop harder to extract into a dedicated function
continue
# deal with optional parser args
if hasattr(_intf, 'parser_args'):
parser_args = _intf.parser_args
else:
parser_args = dict(formatter_class=formatter_class)
# use class description, if no explicit description is available
intf_doc = get_cmd_doc(_intf)
parser_args['description'] = alter_interface_docs_for_cmdline(
intf_doc)
if hasattr(_intf, '_examples_'):
intf_ex = alter_interface_docs_for_cmdline(
get_cmd_ex(_intf))
parser_args['description'] += intf_ex
subparser = subparsers.add_parser(cmd_name,
add_help=False,
**parser_args)
# our own custom help for all commands
helpers.parser_add_common_opt(subparser, 'help')
# let module configure the parser
_intf.setup_parser(subparser)
# logger for command
# configure 'run' function for this command
plumbing_args = dict(func=_intf.call_from_parser,
logger=logging.getLogger(_intf.__module__),
subparser=subparser)
if hasattr(_intf, 'result_renderer_cmdline'):
plumbing_args[
'result_renderer'] = _intf.result_renderer_cmdline
subparser.set_defaults(**plumbing_args)
parts[cmd_name] = subparser
# store short description for later
sdescr = getattr(_intf, 'short_description',
parser_args['description'].split('\n')[0])
grp_short_descriptions[group_name].append((cmd_name, sdescr))
# create command summary
if '--help' in cmdlineargs or '--help-np' in cmdlineargs:
parser.description = get_description_with_cmd_summary(
grp_short_descriptions, interface_groups, parser.description)
parts['datalad'] = parser
lgr.log(5, "Finished setup_parser")
if return_subparsers:
return parts
else:
return parser
