def test_run(args): """Run tests for the specified installed packages. If no specs are listed, run tests for all packages in the current environment or all installed packages if there is no active environment. """ # cdash help option if args.help_cdash: parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, epilog=textwrap.dedent('''\ environment variables: SPACK_CDASH_AUTH_TOKEN authentication token to present to CDash ''')) arguments.add_cdash_args(parser, True) parser.print_help() return # set config option for fail-fast if args.fail_fast: spack.config.set('config:fail_fast', True, scope='command_line') # Get specs to test env = ev.get_env(args, 'test') hashes = env.all_hashes() if env else None specs = spack.cmd.parse_specs(args.specs) if args.specs else [None] specs_to_test = [] for spec in specs: matching = spack.store.db.query_local(spec, hashes=hashes) if spec and not matching: tty.warn("No installed packages match spec %s" % spec) specs_to_test.extend(matching) # test_stage_dir test_suite = spack.install_test.TestSuite(specs_to_test, args.alias) test_suite.ensure_stage() tty.msg("Spack test %s" % test_suite.name) # Set up reporter setattr(args, 'package', [s.format() for s in test_suite.specs]) reporter = spack.report.collect_info(spack.package.PackageBase, 'do_test', args.log_format, args) if not reporter.filename: if args.log_file: if os.path.isabs(args.log_file): log_file = args.log_file else: log_dir = os.getcwd() log_file = os.path.join(log_dir, args.log_file) else: log_file = os.path.join(os.getcwd(), 'test-%s' % test_suite.name) reporter.filename = log_file reporter.specs = specs_to_test with reporter('test', test_suite.stage): test_suite(remove_directory=not args.keep_stage, dirty=args.dirty, fail_first=args.fail_first)
def setup_parser(subparser): sp = subparser.add_subparsers(metavar='SUBCOMMAND', dest='test_command') # Run run_parser = sp.add_parser('run', description=test_run.__doc__, help=first_line(test_run.__doc__)) alias_help_msg = "Provide an alias for this test-suite" alias_help_msg += " for subsequent access." run_parser.add_argument('--alias', help=alias_help_msg) run_parser.add_argument( '--fail-fast', action='store_true', help="Stop tests for each package after the first failure.") run_parser.add_argument('--fail-first', action='store_true', help="Stop after the first failed package.") run_parser.add_argument('--keep-stage', action='store_true', help='Keep testing directory for debugging') run_parser.add_argument('--log-format', default=None, choices=spack.report.valid_formats, help="format to be used for log files") run_parser.add_argument( '--log-file', default=None, help="filename for the log file. if not passed a default will be used") arguments.add_cdash_args(run_parser, False) run_parser.add_argument('--help-cdash', action='store_true', help="Show usage instructions for CDash reporting") cd_group = run_parser.add_mutually_exclusive_group() arguments.add_common_arguments(cd_group, ['clean', 'dirty']) arguments.add_common_arguments(run_parser, ['installed_specs']) # List list_parser = sp.add_parser('list', description=test_list.__doc__, help=first_line(test_list.__doc__)) list_parser.add_argument( "-a", "--all", action="store_true", dest="list_all", help="list all packages with tests (not just installed)") # Find find_parser = sp.add_parser('find', description=test_find.__doc__, help=first_line(test_find.__doc__)) find_parser.add_argument( 'filter', nargs=argparse.REMAINDER, help='optional case-insensitive glob patterns to filter results.') # Status status_parser = sp.add_parser('status', description=test_status.__doc__, help=first_line(test_status.__doc__)) status_parser.add_argument('names', nargs=argparse.REMAINDER, help="Test suites for which to print status") # Results results_parser = sp.add_parser('results', description=test_results.__doc__, help=first_line(test_results.__doc__)) results_parser.add_argument( '-l', '--logs', action='store_true', help="print the test log for each matching package") results_parser.add_argument( '-f', '--failed', action='store_true', help="only show results for failed tests of matching packages") results_parser.add_argument( 'names', nargs=argparse.REMAINDER, metavar='[name(s)] [-- installed_specs]...', help="suite names and installed package constraints") results_parser.epilog = 'Test results will be filtered by space-' \ 'separated suite name(s) and installed\nspecs when provided. '\ 'If names are provided, then only results for those test\nsuites '\ 'will be shown. If installed specs are provided, then ony results'\ '\nmatching those specs will be shown.' # Remove remove_parser = sp.add_parser('remove', description=test_remove.__doc__, help=first_line(test_remove.__doc__)) arguments.add_common_arguments(remove_parser, ['yes_to_all']) remove_parser.add_argument('names', nargs=argparse.REMAINDER, help="Test suites to remove from test stage")
def install(parser, args, **kwargs): if args.help_cdash: parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, epilog=textwrap.dedent('''\ environment variables: SPACK_CDASH_AUTH_TOKEN authentication token to present to CDash ''')) arguments.add_cdash_args(parser, True) parser.print_help() return reporter = spack.report.collect_info(spack.package.PackageInstaller, '_install_task', args.log_format, args) if args.log_file: reporter.filename = args.log_file if not args.spec and not args.specfiles: # if there are no args but an active environment # then install the packages from it. env = ev.get_env(args, 'install') if env: if not args.only_concrete: with env.write_transaction(): concretized_specs = env.concretize() ev.display_specs(concretized_specs) # save view regeneration for later, so that we only do it # once, as it can be slow. env.write(regenerate_views=False) specs = env.all_specs() if not args.log_file and not reporter.filename: reporter.filename = default_log_file(specs[0]) reporter.specs = specs tty.msg("Installing environment {0}".format(env.name)) with reporter: env.install_all(args, **kwargs) tty.debug("Regenerating environment views for {0}".format( env.name)) with env.write_transaction(): # It is not strictly required to synchronize view regeneration # but doing so can prevent redundant work in the filesystem. env.regenerate_views() return else: msg = "install requires a package argument or active environment" if 'spack.yaml' in os.listdir(os.getcwd()): # There's a spack.yaml file in the working dir, the user may # have intended to use that msg += "\n\n" msg += "Did you mean to install using the `spack.yaml`" msg += " in this directory? Try: \n" msg += " spack env activate .\n" msg += " spack install\n" msg += " OR\n" msg += " spack --env . install" tty.die(msg) if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') # Parse cli arguments and construct a dictionary # that will be passed to the package installer update_kwargs_from_args(args, kwargs) if args.run_tests: tty.warn("Deprecated option: --run-tests: use --test=all instead") # 1. Abstract specs from cli abstract_specs = spack.cmd.parse_specs(args.spec) tests = False if args.test == 'all' or args.run_tests: tests = True elif args.test == 'root': tests = [spec.name for spec in abstract_specs] kwargs['tests'] = tests try: specs = spack.cmd.parse_specs(args.spec, concretize=True, tests=tests) except SpackError as e: tty.debug(e) reporter.concretization_report(e.message) raise # 2. Concrete specs from yaml files for file in args.specfiles: with open(file, 'r') as f: s = spack.spec.Spec.from_yaml(f) concretized = s.concretized() if concretized.dag_hash() != s.dag_hash(): msg = 'skipped invalid file "{0}". ' msg += 'The file does not contain a concrete spec.' tty.warn(msg.format(file)) continue abstract_specs.append(s) specs.append(concretized) if len(specs) == 0: tty.die('The `spack install` command requires a spec to install.') if not args.log_file and not reporter.filename: reporter.filename = default_log_file(specs[0]) reporter.specs = specs with reporter('build'): if args.overwrite: installed = list( filter(lambda x: x, map(spack.store.db.query_one, specs))) if not args.yes_to_all: display_args = { 'long': True, 'show_flags': True, 'variants': True } if installed: tty.msg('The following package specs will be ' 'reinstalled:\n') spack.cmd.display_specs(installed, **display_args) not_installed = list( filter(lambda x: x not in installed, specs)) if not_installed: tty.msg('The following package specs are not installed and' ' the --overwrite flag was given. The package spec' ' will be newly installed:\n') spack.cmd.display_specs(not_installed, **display_args) # We have some specs, so one of the above must have been true answer = tty.get_yes_or_no('Do you want to proceed?', default=False) if not answer: tty.die('Reinstallation aborted.') # overwrite all concrete explicit specs from this build kwargs['overwrite'] = [spec.dag_hash() for spec in specs] install_specs(args, kwargs, zip(abstract_specs, specs))
def setup_parser(subparser): subparser.add_argument('--only', default='package,dependencies', dest='things_to_install', choices=['package', 'dependencies'], help="""select the mode of installation. the default is to install the package along with all its dependencies. alternatively one can decide to install only the package or only the dependencies""") subparser.add_argument( '-u', '--until', type=str, dest='until', default=None, help="phase to stop after when installing (default None)") arguments.add_common_arguments(subparser, ['jobs']) subparser.add_argument( '--overwrite', action='store_true', help="reinstall an existing spec, even if it has dependents") subparser.add_argument( '--fail-fast', action='store_true', help="stop all builds if any build fails (default is best effort)") subparser.add_argument( '--keep-prefix', action='store_true', help="don't remove the install prefix if installation fails") subparser.add_argument( '--keep-stage', action='store_true', help="don't remove the build stage if installation succeeds") subparser.add_argument( '--dont-restage', action='store_true', help="if a partial install is detected, don't delete prior state") cache_group = subparser.add_mutually_exclusive_group() cache_group.add_argument( '--use-cache', action='store_true', dest='use_cache', default=True, help="check for pre-built Spack packages in mirrors (default)") cache_group.add_argument( '--no-cache', action='store_false', dest='use_cache', default=True, help="do not check for pre-built Spack packages in mirrors") cache_group.add_argument('--cache-only', action='store_true', dest='cache_only', default=False, help="only install package from binary mirrors") subparser.add_argument('--no-check-signature', action='store_true', dest='unsigned', default=False, help="do not check signatures of binary packages") subparser.add_argument('--require-full-hash-match', action='store_true', dest='full_hash_match', default=False, help="""when installing from binary mirrors, do not install binary package unless the full hash of the remote spec matches that of the local spec""") subparser.add_argument( '--show-log-on-error', action='store_true', help="print full build log to stderr if build fails") subparser.add_argument('--source', action='store_true', dest='install_source', help="install source files in prefix") arguments.add_common_arguments(subparser, ['no_checksum']) subparser.add_argument( '-v', '--verbose', action='store_true', help="display verbose build output while installing") subparser.add_argument('--fake', action='store_true', help="fake install for debug purposes.") subparser.add_argument( '--only-concrete', action='store_true', default=False, help='(with environment) only install already concretized specs') subparser.add_argument( '-f', '--file', action='append', default=[], dest='specfiles', metavar='SPEC_YAML_FILE', help="install from file. Read specs to install from .yaml files") cd_group = subparser.add_mutually_exclusive_group() arguments.add_common_arguments(cd_group, ['clean', 'dirty']) testing = subparser.add_mutually_exclusive_group() testing.add_argument('--test', default=None, choices=['root', 'all'], help="""If 'root' is chosen, run package tests during installation for top-level packages (but skip tests for dependencies). if 'all' is chosen, run package tests during installation for all packages. If neither are chosen, don't run tests for any packages.""") testing.add_argument( '--run-tests', action='store_true', help='run package tests during installation (same as --test=all)') subparser.add_argument('--log-format', default=None, choices=spack.report.valid_formats, help="format to be used for log files") subparser.add_argument( '--log-file', default=None, help="filename for the log file. if not passed a default will be used") subparser.add_argument('--help-cdash', action='store_true', help="Show usage instructions for CDash reporting") arguments.add_cdash_args(subparser, False) arguments.add_common_arguments(subparser, ['yes_to_all', 'spec'])
def install(parser, args, **kwargs): if args.help_cdash: parser = argparse.ArgumentParser( formatter_class=argparse.RawDescriptionHelpFormatter, epilog=textwrap.dedent('''\ environment variables: SPACK_CDASH_AUTH_TOKEN authentication token to present to CDash ''')) arguments.add_cdash_args(parser, True) parser.print_help() return # The user wants to monitor builds using github.com/spack/spack-monitor if args.use_monitor: monitor = spack.monitor.get_client( host=args.monitor_host, prefix=args.monitor_prefix, disable_auth=args.monitor_disable_auth, tags=args.monitor_tags, save_local=args.monitor_save_local, ) reporter = spack.report.collect_info(spack.package.PackageInstaller, '_install_task', args.log_format, args) if args.log_file: reporter.filename = args.log_file if args.run_tests: tty.warn("Deprecated option: --run-tests: use --test=all instead") def get_tests(specs): if args.test == 'all' or args.run_tests: return True elif args.test == 'root': return [spec.name for spec in specs] else: return False # Parse cli arguments and construct a dictionary # that will be passed to the package installer update_kwargs_from_args(args, kwargs) if not args.spec and not args.specfiles: # if there are no args but an active environment # then install the packages from it. env = ev.active_environment() if env: tests = get_tests(env.user_specs) kwargs['tests'] = tests if not args.only_concrete: with env.write_transaction(): concretized_specs = env.concretize(tests=tests) ev.display_specs(concretized_specs) # save view regeneration for later, so that we only do it # once, as it can be slow. env.write(regenerate=False) specs = env.all_specs() if not args.log_file and not reporter.filename: reporter.filename = default_log_file(specs[0]) reporter.specs = specs # Tell the monitor about the specs if args.use_monitor and specs: monitor.new_configuration(specs) tty.msg("Installing environment {0}".format(env.name)) with reporter('build'): env.install_all(**kwargs) tty.debug("Regenerating environment views for {0}".format( env.name)) with env.write_transaction(): # write env to trigger view generation and modulefile # generation env.write() return else: msg = "install requires a package argument or active environment" if 'spack.yaml' in os.listdir(os.getcwd()): # There's a spack.yaml file in the working dir, the user may # have intended to use that msg += "\n\n" msg += "Did you mean to install using the `spack.yaml`" msg += " in this directory? Try: \n" msg += " spack env activate .\n" msg += " spack install\n" msg += " OR\n" msg += " spack --env . install" tty.die(msg) if args.no_checksum: spack.config.set('config:checksum', False, scope='command_line') if args.deprecated: spack.config.set('config:deprecated', True, scope='command_line') # 1. Abstract specs from cli abstract_specs = spack.cmd.parse_specs(args.spec) tests = get_tests(abstract_specs) kwargs['tests'] = tests try: specs = spack.cmd.parse_specs(args.spec, concretize=True, tests=tests) except SpackError as e: tty.debug(e) reporter.concretization_report(e.message) raise # 2. Concrete specs from yaml files for file in args.specfiles: with open(file, 'r') as f: if file.endswith('yaml') or file.endswith('yml'): s = spack.spec.Spec.from_yaml(f) else: s = spack.spec.Spec.from_json(f) concretized = s.concretized() if concretized.dag_hash() != s.dag_hash(): msg = 'skipped invalid file "{0}". ' msg += 'The file does not contain a concrete spec.' tty.warn(msg.format(file)) continue abstract_specs.append(s) specs.append(concretized) if len(specs) == 0: tty.die('The `spack install` command requires a spec to install.') if not args.log_file and not reporter.filename: reporter.filename = default_log_file(specs[0]) reporter.specs = specs with reporter('build'): if args.overwrite: installed = list( filter(lambda x: x, map(spack.store.db.query_one, specs))) if not args.yes_to_all: display_args = { 'long': True, 'show_flags': True, 'variants': True } if installed: tty.msg('The following package specs will be ' 'reinstalled:\n') spack.cmd.display_specs(installed, **display_args) not_installed = list( filter(lambda x: x not in installed, specs)) if not_installed: tty.msg('The following package specs are not installed and' ' the --overwrite flag was given. The package spec' ' will be newly installed:\n') spack.cmd.display_specs(not_installed, **display_args) # We have some specs, so one of the above must have been true answer = tty.get_yes_or_no('Do you want to proceed?', default=False) if not answer: tty.die('Reinstallation aborted.') # overwrite all concrete explicit specs from this build kwargs['overwrite'] = [spec.dag_hash() for spec in specs] # Update install_args with the monitor args, needed for build task kwargs.update({ "monitor_disable_auth": args.monitor_disable_auth, "monitor_keep_going": args.monitor_keep_going, "monitor_host": args.monitor_host, "use_monitor": args.use_monitor, "monitor_prefix": args.monitor_prefix, }) # If we are using the monitor, we send configs. and create build # The full_hash is the main package id, the build_hash for others if args.use_monitor and specs: monitor.new_configuration(specs) install_specs(args, kwargs, zip(abstract_specs, specs))