def register_show_param(cls, parser):
uuid_attr_metavar = 'UUID[.ATTRIBUTE]'
add_argument(
parser,
'--show',
action='append',
default=[],
type=cls._parse_uuid_attr,
metavar=uuid_attr_metavar,
help=
"""Show the attribute value with given UUID, or one of its attribute."""
)
add_argument(
parser,
'--show-yaml',
action='append',
default=[],
type=cls._parse_uuid_attr,
metavar=uuid_attr_metavar,
help=
"""Show the YAML dump of value with given UUID, or one of its attributes."""
)
add_argument(parser,
'--serialize',
nargs=2,
action='append',
default=[],
metavar=(uuid_attr_metavar, 'PATH'),
help="""Serialize the value of given UUID to PATH.""")
def register_run_param(parser):
add_argument(
parser,
'--conf',
action='append',
default=[],
help=
"LISA configuration file. If multiple configurations of a given type are found, they are merged (last one can override keys in previous ones). Only load trusted files as it can lead to arbitrary code execution."
)
add_argument(
parser,
'--inject',
action='append',
metavar='SERIALIZED_OBJECT_PATH',
default=[],
help="Serialized object to inject when building expressions")
# Create an empty TargetConf, so we are able to get the list of tests
# as if we were going to execute them using a target.
# note: that is only used for generating the documentation.
add_argument(parser,
'--inject-empty-target-conf',
action='store_true',
help=argparse.SUPPRESS)
def register_compare_param(parser):
add_argument(parser, '--alpha', type=float,
default=5,
help="""Alpha risk for Fisher exact test in percents.""")
add_argument(parser, '--non-significant', action='store_true',
help="""Also show non-significant changes of failure rate.""")
add_argument(parser, '--remove-tag', action='append',
default=[],
help="""Remove the given tags in the testcase IDs before
comparison. Can be repeated.""")
def _main(argv):
parser = argparse.ArgumentParser(
description="""
Test runner
PATTERNS
All patterns are fnmatch pattern, following basic shell globbing syntax.
A pattern starting with "!" is used as a negative pattern.
""",
formatter_class=argparse.RawTextHelpFormatter)
add_argument(
parser,
'--debug',
action='store_true',
help="""Show complete Python backtrace when exekall crashes.""")
subparsers = parser.add_subparsers(title='subcommands', dest='subcommand')
run_parser = subparsers.add_parser(
'run',
description="""
Run expressions
Note that the adaptor in the customization module is able to add more
parameters to ``exekall run``. In order to get the complete set of options,
please run ``exekall run YOUR_SOURCES_OR_MODULES --help``.
""",
formatter_class=argparse.RawTextHelpFormatter)
# It is not possible to give a default value to positional options,
# otherwise adaptor-specific options' values will be picked up as Python
# sources, and importing the modules will therefore fail with unknown files
# error.
add_argument(
run_parser,
'python_files',
nargs='+',
metavar='PYTHON_MODULES',
help=
"""Python modules files or module names. If passed a folder, all contained files recursively are selected. By default, the current directory is selected."""
)
add_argument(
run_parser,
'-s',
'--select',
action='append',
metavar='ID_PATTERN',
default=[],
help=
"""Only run the expressions with an ID matching any of the supplied pattern. A pattern starting with "!" can be used to exclude IDs matching it."""
)
# Same as --select, but allows multiple patterns without needing to
# repeat the option. This is mostly available to support wrapper
# scripts, and is not recommended for direct use since it can lead to
# some parsing ambiguities.
add_argument(
run_parser,
'--select-multiple',
nargs='*',
default=[],
help=argparse.SUPPRESS,
)
add_argument(
run_parser,
'--list',
action='store_true',
help="""List the expressions that will be run without running them.""")
add_argument(run_parser,
'-n',
type=int,
default=1,
help="""Run the tests for a number of iterations.""")
add_argument(
run_parser,
'--load-db',
action='append',
default=[],
help=
"""Reload a database to use some of its objects. The DB and its artifact directory will be merged in the produced DB at the end of the execution, to form a self-contained artifact directory."""
)
add_argument(
run_parser,
'--load-type',
action='append',
metavar='TYPE_PATTERN',
default=[],
help=
"""Load the (indirect) instances of the given class from the database instead of the root objects."""
)
run_uuid_group = run_parser.add_mutually_exclusive_group()
add_argument(
run_uuid_group,
'--replay',
help=
"""Replay the execution of the given UUID, loading as much prerequisite from the DB as possible. This implies --pdb for convenience."""
)
add_argument(run_uuid_group,
'--load-uuid',
action='append',
default=[],
help="""Load the given UUID from the database.""")
# Load the parameters that were used to compute the value with the given
# UUID from the database. This can be used as a more flexible form of
# --replay that does not imply restricting the selection
add_argument(run_uuid_group, '--load-uuid-args', help=argparse.SUPPRESS)
run_artifact_dir_group = run_parser.add_mutually_exclusive_group()
add_argument(
run_artifact_dir_group,
'--artifact-dir',
default=os.getenv('EXEKALL_ARTIFACT_DIR'),
help=
"""Folder in which the artifacts will be stored. Defaults to EXEKALL_ARTIFACT_DIR env var."""
)
add_argument(
run_artifact_dir_group,
'--artifact-root',
default=os.getenv('EXEKALL_ARTIFACT_ROOT', 'artifacts'),
help=
"Root folder under which the artifact folders will be created. Defaults to EXEKALL_ARTIFACT_ROOT env var."
)
run_advanced_group = run_parser.add_argument_group(
title='advanced arguments',
description='Options not needed for every-day use')
add_argument(
run_advanced_group,
'--no-save-value-db',
action='store_false',
dest='save_value_db',
help=
"""Do not create a VALUE_DB.pickle.xz file in the artifact folder. This avoids a costly serialization of the results, but prevents partial re-execution of expressions."""
)
add_argument(
run_advanced_group,
'--verbose',
'-v',
action='count',
default=0,
help=
"""More verbose output. Can be repeated for even more verbosity. This only impacts exekall output, --log-level for more global settings."""
)
add_argument(
run_advanced_group,
'--pdb',
action='store_true',
help=
"""If an exception occurs in the code ran by ``exekall``, drops into a debugger shell."""
)
add_argument(
run_advanced_group,
'--log-level',
default='info',
choices=('debug', 'info', 'warn', 'error', 'critical'),
help="""Change the default log level of the standard logging module."""
)
add_argument(run_advanced_group,
'--param',
nargs=3,
action='append',
default=[],
metavar=('CALLABLE_PATTERN', 'PARAM', 'VALUE'),
help="""Set a function parameter. It needs three fields:
* pattern matching qualified name of the callable
* name of the parameter
* value""")
add_argument(
run_advanced_group,
'--sweep',
nargs=5,
action='append',
default=[],
metavar=('CALLABLE_PATTERN', 'PARAM', 'START', 'STOP', 'STEP'),
help="""Parametric sweep on a function parameter. It needs five fields:
* pattern matching qualified name of the callable
* name of the parameter
* start value
* stop value
* step size.""")
add_argument(
run_advanced_group,
'--share',
action='append',
metavar='TYPE_PATTERN',
default=[],
help="""Class name pattern to share between multiple iterations.""")
add_argument(
run_advanced_group,
'--random-order',
action='store_true',
help=
"""Run the expressions in a random order, instead of sorting by name."""
)
add_argument(run_advanced_group,
'--symlink-artifact-dir-to',
type=pathlib.Path,
help="""Create a symlink pointing at the artifact dir.""")
# Show the list of expressions in reStructuredText format, suitable for
# inclusion in Sphinx documentation
add_argument(run_advanced_group,
'--rst-list',
action='store_true',
help=argparse.SUPPRESS)
add_argument(
run_advanced_group,
'--restrict',
action='append',
metavar='CALLABLE_PATTERN',
default=[],
help=
"""Callable names patterns. Types produced by these callables will only be produced by these (other callables will be excluded)."""
)
add_argument(
run_advanced_group,
'--forbid',
action='append',
metavar='TYPE_PATTERN',
default=[],
help=
"""Fully qualified type names patterns. Callable returning these types or any subclass will not be called."""
)
add_argument(
run_advanced_group,
'--allow',
action='append',
metavar='CALLABLE_PATTERN',
default=[],
help=
"""Allow using callable with a fully qualified name matching these patterns, even if they have been not selected for various reasons."""
)
run_goal_group = run_advanced_group.add_mutually_exclusive_group()
add_argument(
run_goal_group,
'--goal',
action='append',
metavar='TYPE_PATTERN',
default=[],
help=
"""Compute expressions leading to an instance of a class with name matching this pattern (or a subclass of it)."""
)
add_argument(
run_goal_group,
'--callable-goal',
action='append',
metavar='CALLABLE_PATTERN',
default=[],
help=
"""Compute expressions ending with a callable which name is matching this pattern."""
)
add_argument(
run_advanced_group,
'--template-scripts',
metavar='SCRIPT_FOLDER',
help=
"""Only create the template scripts of the expressions without running them."""
)
add_argument(
run_advanced_group,
'--adaptor',
help=
"""Adaptor to use from the customization module, if there is more than one to choose from."""
)
merge_parser = subparsers.add_parser(
'merge',
description="""
Merge artifact directories of "exekall run" executions.
By default, it will use hardlinks instead of copies to improve speed and
avoid eating up large amount of space, but that means that artifact
directories should be treated as read-only.
""",
formatter_class=argparse.RawTextHelpFormatter)
add_argument(
merge_parser,
'artifact_dirs',
nargs='+',
help=
"""Artifact directories created using "exekall run", or value databases to merge."""
)
add_argument(merge_parser,
'-o',
'--output',
required=True,
help="""
Output merged artifacts directory or value database. If the
output already exists, the merged DB will only contain the same roots
as this one. This allows patching-up a pruned DB with other DBs that
contains subexpression's values.
""")
add_argument(merge_parser,
'--copy',
action='store_true',
help="""Force copying files, instead of using hardlinks.""")
compare_parser = subparsers.add_parser(
'compare',
description="""
Compare two DBs produced by exekall run.
Note that the adaptor in the customization module recorded in the database
is able to add more parameters to ``exekall compare``. In order to get the
complete set of options, please run ``exekall compare DB1 DB2 --help``.
Options part of a custom group will need to be passed after positional
arguments.
""",
formatter_class=argparse.RawTextHelpFormatter)
add_argument(compare_parser,
'db',
nargs=2,
help="""DBs created using exekall run to compare.""")
show_parser = subparsers.add_parser(
'show',
description="""
Show the content of a ValueDB created by exekall ``run``
Note that the adaptor in the customization module recorded in the database
is able to add more parameters to ``exekall show``. In order to get the
complete set of options, please run ``exekall show DB --help``.
Options part of a custom group will need to be passed after positional
arguments.
""",
formatter_class=argparse.RawTextHelpFormatter)
add_argument(show_parser,
'db',
help="""DB created using exekall run to show.""")
# Avoid showing help message on the incomplete parser. Instead, we carry on
# and the help will be displayed after the parser customization of run
# subcommand has a chance to take place.
help_options = ('-h', '--help')
no_help_argv = [arg for arg in argv if arg not in help_options]
try:
# Silence argparse until we know what is going on
stream = io.StringIO()
with contextlib.redirect_stderr(stream):
args, _ = parser.parse_known_args(no_help_argv)
# If it fails, that may be because of an incomplete command line with just
# --help for example. If it was for another reason, it will fail again and
# show the message.
except SystemExit:
parser.parse_known_args(argv)
# That should never be reached
assert False
if not args.subcommand:
parser.print_help()
return 2
global show_traceback
show_traceback = args.debug
# Some subcommands need not parser customization, in which case we more
# strictly parse the command line
if args.subcommand not in ('run', 'compare', 'show'):
parser.parse_args(argv)
if args.subcommand == 'run':
# do_run needs to reparse the CLI, so it needs the parser and argv
return do_run(args, parser, run_parser, argv)
elif args.subcommand == 'merge':
return do_merge(
artifact_dirs=args.artifact_dirs,
output_dir=args.output,
use_hardlink=(not args.copy),
)
elif args.subcommand == 'compare':
return do_compare(
parser=parser,
compare_parser=compare_parser,
argv=argv,
db_path_list=args.db,
)
elif args.subcommand == 'show':
return do_show(
parser=parser,
show_parser=show_parser,
argv=argv,
db_path=args.db,
)
