def run_contrib_checks(ecs): """Run contribution check on specified easyconfigs.""" def print_result(checks_passed, label): """Helper function to print result of last group of checks.""" if checks_passed: print_msg("\n>> All %s checks PASSed!" % label, prefix=False) else: print_msg("\n>> One or more %s checks FAILED!" % label, prefix=False) # start by running style checks style_check_ok = cmdline_easyconfigs_style_check(ecs) print_result(style_check_ok, "style") # check whether SHA256 checksums are in place print_msg("\nChecking for SHA256 checksums in %d easyconfig(s)...\n" % len(ecs), prefix=False) sha256_checksums_ok = True for ec in ecs: sha256_checksum_fails = check_sha256_checksums([ec]) if sha256_checksum_fails: sha256_checksums_ok = False msgs = ['[FAIL] %s' % ec.path] + sha256_checksum_fails else: msgs = ['[PASS] %s' % ec.path] print_msg('\n'.join(msgs), prefix=False) print_result(sha256_checksums_ok, "SHA256 checksums") return style_check_ok and sha256_checksums_ok
def run_contrib_checks(ecs): """Run contribution check on specified easyconfigs.""" def print_result(checks_passed, label): """Helper function to print result of last group of checks.""" if checks_passed: print_msg("\n>> All %s checks PASSed!" % label, prefix=False) else: print_msg("\n>> One or more %s checks FAILED!" % label, prefix=False) # start by running style checks style_check_ok = cmdline_easyconfigs_style_check(ecs) print_result(style_check_ok, "style") # check whether SHA256 checksums are in place print_msg("\nChecking for SHA256 checksums in %d easyconfig(s)...\n" % len(ecs), prefix=False) sha256_checksums_ok = True for ec in ecs: sha256_checksum_fails = check_sha256_checksums([ec]) if sha256_checksum_fails: sha256_checksums_ok = False msgs = ['[FAIL] %s' % ec.path] + sha256_checksum_fails else: msgs = ['[PASS] %s' % ec.path] print_msg('\n'.join(msgs), prefix=False) print_result(sha256_checksums_ok, "SHA256 checksums") return style_check_ok and sha256_checksums_ok
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set by option parsers via set_tmpdir eb_tmpdir = tempfile.gettempdir() search_query = options.search or options.search_filename or options.search_short # initialise logging for main global _log _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=(testing or options.terse or search_query), colorize=options.color) # disallow running EasyBuild as root if os.getuid() == 0: raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here.") # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot) _log.debug("Full robot path: %s" % robot_path) # configure & initialize build options config_options_dict = eb_go.get_options_by_section('config') build_options = { 'build_specs': build_specs, 'command_line': eb_cmd_line, 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata), 'pr_path': pr_path, 'robot_path': robot_path, 'silent': testing, 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } # initialise the EasyBuild configuration & build options config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) if modtool is None: modtool = modules_tool(testing=testing) if options.last_log: # print location to last log file, and exit last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) # check whether packaging is supported when it's being used if options.package: check_pkg_support() else: _log.debug("Packaging not enabled, so not checking for packaging support.") # search for easyconfigs, if a query is specified if search_query: search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() elif options.install_github_token: install_github_token(options.github_user, silent=build_option('silent')) elif options.review_pr: print review_pr(options.review_pr, colored=use_color(options.color)) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' print list_software(output_format=options.output_format, detailed=detailed, only_installed=True) elif options.list_software: print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed') # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, options.install_github_token, options.list_installed_software, options.list_software, options.review_pr, options.terse, search_query, ] if any(early_stop_options): cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = modtool.list() # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") if options.install_latest_eb_release: if orig_paths: raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified no_ec_opts = [options.aggregate_regtest, options.new_pr, options.regtest, options.update_pr, search_query] # determine paths to easyconfigs paths = det_easyconfig_paths(categorized_paths['easyconfigs']) if paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in paths] else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif not any(no_ec_opts): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if options.check_style: _log.debug("Running style check...") if cmdline_easyconfigs_style_check([path[0] for path in paths]): print_msg("All style checks passed!", prefix=False) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) else: raise EasyBuildError("One or more style checks FAILED!") # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) dry_run_mode = options.dry_run or options.dry_run_short new_update_pr = options.new_pr or options.update_pr # skip modules that are already installed unless forced if not (options.force or options.rebuild or dry_run_mode or options.extended_dry_run or new_update_pr): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled if options.robot and (not dry_run_mode or new_update_pr): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs elif new_update_pr: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # creating/updating PRs if new_update_pr: if options.new_pr: new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) else: update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg) cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) elif options.check_conflicts: if check_conflicts(easyconfigs, modtool): print_error("One or more conflicts detected!") sys.exit(1) else: print_msg("\nNo conflicts detected!\n", prefix=False) # dump source script to set up build environment elif options.dump_env_script: dump_env_script(easyconfigs) # cleanup and exit after dry run, searching easyconfigs or submitting regression test if any(no_ec_opts + [options.check_conflicts, dry_run_mode, options.dump_env_script]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set by option parsers via set_tmpdir eb_tmpdir = tempfile.gettempdir() search_query = options.search or options.search_filename or options.search_short # initialise logging for main global _log _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=(testing or options.terse or search_query), colorize=options.color) # disallow running EasyBuild as root (by default) check_root_usage( allow_use_as_root=options.allow_use_as_root_and_accept_consequences) # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot) _log.debug("Full robot path: %s" % robot_path) # configure & initialize build options config_options_dict = eb_go.get_options_by_section('config') build_options = { 'build_specs': build_specs, 'command_line': eb_cmd_line, 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata), 'pr_path': pr_path, 'robot_path': robot_path, 'silent': testing, 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } # initialise the EasyBuild configuration & build options config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) if modtool is None: modtool = modules_tool(testing=testing) # check whether any (EasyBuild-generated) modules are loaded already in the current session modtool.check_loaded_modules() if options.last_log: # print location to last log file, and exit last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) # check whether packaging is supported when it's being used if options.package: check_pkg_support() else: _log.debug( "Packaging not enabled, so not checking for packaging support.") # search for easyconfigs, if a query is specified if search_query: search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() elif options.install_github_token: install_github_token(options.github_user, silent=build_option('silent')) elif options.merge_pr: merge_pr(options.merge_pr) elif options.review_pr: print review_pr(pr=options.review_pr, colored=use_color(options.color)) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' print list_software(output_format=options.output_format, detailed=detailed, only_installed=True) elif options.list_software: print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed') # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, options.install_github_token, options.list_installed_software, options.list_software, options.merge_pr, options.review_pr, options.terse, search_query, ] if any(early_stop_options): cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = modtool.list( ) # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning( "Failed to determine install path for easybuild-easyconfigs package." ) if options.install_latest_eb_release: if orig_paths: raise EasyBuildError( "Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr no_ec_opts = [ options.aggregate_regtest, options.regtest, search_query, new_update_preview_pr ] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) if determined_paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in determined_paths] else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif any(no_ec_opts): paths = determined_paths else: print_error(( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if options.check_style: _log.debug("Running style check...") if cmdline_easyconfigs_style_check([path[0] for path in paths]): print_msg("All style checks passed!", prefix=False) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) else: raise EasyBuildError("One or more style checks FAILED!") # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs( paths, validate=not options.inject_checksums) # verify easyconfig filenames, if desired if options.verify_easyconfig_filenames: _log.info("Verifying easyconfig filenames...") for easyconfig in easyconfigs: verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec']) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) if options.containerize: containerize(easyconfigs) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ ec for ec in easyconfigs if ec not in retained_ecs ]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled if options.robot and (not dry_run_mode or new_update_preview_pr): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs elif new_update_preview_pr: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # creating/updating PRs if new_update_preview_pr: if options.new_pr: new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) elif options.preview_pr: print review_pr(paths=determined_paths, colored=use_color(options.color)) else: update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg) # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) elif options.check_conflicts: if check_conflicts(easyconfigs, modtool): print_error("One or more conflicts detected!") sys.exit(1) else: print_msg("\nNo conflicts detected!\n", prefix=False) # dump source script to set up build environment elif options.dump_env_script: dump_env_script(easyconfigs) elif options.inject_checksums: inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [ options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums ] if any(no_ec_opts) or any(stop_options): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) hooks = load_hooks(options.hooks) ecs_with_res = build_and_install_software( ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, hooks=hooks) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False) ]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)