def missing_deps(easyconfigs, modtool): """ Determine subset of easyconfigs for which no module is installed yet. """ ordered_ecs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False) missing = skip_available(ordered_ecs, modtool) if missing: lines = [ '', "%d out of %d required modules missing:" % (len(missing), len(ordered_ecs)), '' ] for ec in [x['ec'] for x in missing]: if ec.short_mod_name != ec.full_mod_name: modname = '%s | %s' % (ec.mod_subdir, ec.short_mod_name) else: modname = ec.full_mod_name lines.append("* %s (%s)" % (modname, os.path.basename(ec.path))) lines.append('') else: lines = ['', "No missing modules!", ''] return '\n'.join(lines)
def dry_run(easyconfigs, modtool, short=False): """ Compose dry run overview for supplied easyconfigs: * [ ] for unavailable * [x] for available * [F] for forced * [R] for rebuild :param easyconfigs: list of parsed easyconfigs (EasyConfig instances) :param modtool: ModulesTool instance to use :param short: use short format for overview: use a variable for common prefixes """ lines = [] if build_option('robot_path') is None: lines.append("Dry run: printing build status of easyconfigs") all_specs = easyconfigs else: lines.append( "Dry run: printing build status of easyconfigs and dependencies") all_specs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) unbuilt_specs = skip_available(all_specs, modtool) dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) listed_ec_paths = [spec['spec'] for spec in easyconfigs] var_name = 'CFGS' common_prefix = det_common_path_prefix( [spec['spec'] for spec in all_specs]) # only allow short if common prefix is long enough short = short and common_prefix is not None and len( common_prefix) > len(var_name) * 2 for spec in all_specs: if spec in unbuilt_specs: ans = ' ' elif build_option('force') and spec['spec'] in listed_ec_paths: ans = 'F' elif build_option('rebuild') and spec['spec'] in listed_ec_paths: ans = 'R' else: ans = 'x' if spec['ec'].short_mod_name != spec['ec'].full_mod_name: mod = "%s | %s" % (spec['ec'].mod_subdir, spec['ec'].short_mod_name) else: mod = spec['ec'].full_mod_name if short: item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:]) else: item = spec['spec'] lines.append(dry_run_fmt % (ans, item, mod)) if short: # insert after 'Dry run:' message lines.insert(1, "%s=%s" % (var_name, common_prefix)) return '\n'.join(lines)
def dry_run(easyconfigs, modtool, short=False): """ Compose dry run overview for supplied easyconfigs: * [ ] for unavailable * [x] for available * [F] for forced * [R] for rebuild :param easyconfigs: list of parsed easyconfigs (EasyConfig instances) :param modtool: ModulesTool instance to use :param short: use short format for overview: use a variable for common prefixes """ lines = [] if build_option('robot_path') is None: lines.append("Dry run: printing build status of easyconfigs") all_specs = easyconfigs else: lines.append("Dry run: printing build status of easyconfigs and dependencies") all_specs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False) unbuilt_specs = skip_available(all_specs, modtool) dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) listed_ec_paths = [spec['spec'] for spec in easyconfigs] var_name = 'CFGS' common_prefix = det_common_path_prefix([spec['spec'] for spec in all_specs if spec['spec'] is not None]) # only allow short if common prefix is long enough short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2 for spec in all_specs: if spec in unbuilt_specs: ans = ' ' elif build_option('force') and spec['spec'] in listed_ec_paths: ans = 'F' elif build_option('rebuild') and spec['spec'] in listed_ec_paths: ans = 'R' else: ans = 'x' if spec['ec'] is not None and spec['ec'].short_mod_name != spec['ec'].full_mod_name: mod = "%s | %s" % (spec['ec'].mod_subdir, spec['ec'].short_mod_name) else: mod = spec['full_mod_name'] if spec['spec'] is None: item = "(no easyconfig file found)" elif short: item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:]) else: item = spec['spec'] lines.append(dry_run_fmt % (ans, item, mod)) if short: # insert after 'Dry run:' message lines.insert(1, "%s=%s" % (var_name, common_prefix)) return '\n'.join(lines)
def dry_run(easyconfigs, modtool, short=False): """ Compose dry run overview for supplied easyconfigs: * [ ] for unavailable * [x] for available * [F] for forced * [R] for rebuild @param easyconfigs: list of parsed easyconfigs (EasyConfig instances) @param modtool: ModulesTool instance to use @param short: use short format for overview: use a variable for common prefixes """ lines = [] if build_option("robot_path") is None: lines.append("Dry run: printing build status of easyconfigs") all_specs = easyconfigs else: lines.append("Dry run: printing build status of easyconfigs and dependencies") all_specs = resolve_dependencies(easyconfigs, modtool, retain_all_deps=True) unbuilt_specs = skip_available(all_specs, modtool) dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) listed_ec_paths = [spec["spec"] for spec in easyconfigs] var_name = "CFGS" common_prefix = det_common_path_prefix([spec["spec"] for spec in all_specs]) # only allow short if common prefix is long enough short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2 for spec in all_specs: if spec in unbuilt_specs: ans = " " elif build_option("force") and spec["spec"] in listed_ec_paths: ans = "F" elif build_option("rebuild") and spec["spec"] in listed_ec_paths: ans = "R" else: ans = "x" if spec["ec"].short_mod_name != spec["ec"].full_mod_name: mod = "%s | %s" % (spec["ec"].mod_subdir, spec["ec"].short_mod_name) else: mod = spec["ec"].full_mod_name if short: item = os.path.join("$%s" % var_name, spec["spec"][len(common_prefix) + 1 :]) else: item = spec["spec"] lines.append(dry_run_fmt % (ans, item, mod)) if short: # insert after 'Dry run:' message lines.insert(1, "%s=%s" % (var_name, common_prefix)) return "\n".join(lines)
def dry_run(easyconfigs, short=False, build_specs=None): """ Compose dry run overview for supplied easyconfigs ([ ] for unavailable, [x] for available, [F] for forced) @param easyconfigs: list of parsed easyconfigs (EasyConfig instances) @param short: use short format for overview: use a variable for common prefixes @param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ lines = [] if build_option('robot_path') is None: lines.append("Dry run: printing build status of easyconfigs") all_specs = easyconfigs else: lines.append("Dry run: printing build status of easyconfigs and dependencies") all_specs = resolve_dependencies(easyconfigs, build_specs=build_specs, retain_all_deps=True) unbuilt_specs = skip_available(all_specs) dry_run_fmt = " * [%1s] %s (module: %s)" # markdown compatible (list of items with checkboxes in front) listed_ec_paths = [spec['spec'] for spec in easyconfigs] var_name = 'CFGS' common_prefix = det_common_path_prefix([spec['spec'] for spec in all_specs]) # only allow short if common prefix is long enough short = short and common_prefix is not None and len(common_prefix) > len(var_name) * 2 for spec in all_specs: if spec in unbuilt_specs: ans = ' ' elif build_option('force') and spec['spec'] in listed_ec_paths: ans = 'F' else: ans = 'x' if spec['ec'].short_mod_name != spec['ec'].full_mod_name: mod = "%s | %s" % (spec['ec'].mod_subdir, spec['ec'].short_mod_name) else: mod = spec['ec'].full_mod_name if short: item = os.path.join('$%s' % var_name, spec['spec'][len(common_prefix) + 1:]) else: item = spec['spec'] lines.append(dry_run_fmt % (ans, item, mod)) if short: # insert after 'Dry run:' message lines.insert(1, "%s=%s" % (var_name, common_prefix)) return '\n'.join(lines)
def regtest(easyconfig_paths, modtool, build_specs=None): """ Run regression test, using easyconfigs available in given path :param easyconfig_paths: path of easyconfigs to run regtest on :param modtool: ModulesTool instance to use :param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ cur_dir = os.getcwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: output_file = os.path.join( aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest)) aggregate_xml_in_dirs(aggregate_regtest, output_file) _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) # create base directory, which is used to place all log files and the test output as xml regtest_output_dir = build_option('regtest_output_dir') testoutput = build_option('testoutput') if regtest_output_dir is not None: output_dir = regtest_output_dir elif testoutput is not None: output_dir = os.path.abspath(testoutput) else: # default: current dir + easybuild-test-[timestamp] dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") output_dir = os.path.join(cur_dir, dirname) mkdir(output_dir, parents=True) # find all easyconfigs ecfiles = [] if easyconfig_paths: for path in easyconfig_paths: ecfiles += find_easyconfigs( path, ignore_dirs=build_option('ignore_dirs')) else: raise EasyBuildError("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend( process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError as err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug( "Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs, modtool) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, modtool) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % { 'cmd': cmd } build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) _log.info("Submitted regression test as jobs, results in %s" % output_dir) return True # success
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set by option parsers via set_tmpdir eb_tmpdir = tempfile.gettempdir() search_query = options.search or options.search_filename or options.search_short # initialise logging for main global _log _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=(testing or options.terse or search_query), colorize=options.color) # disallow running EasyBuild as root if os.getuid() == 0: raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here.") # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot) _log.debug("Full robot path: %s" % robot_path) # configure & initialize build options config_options_dict = eb_go.get_options_by_section('config') build_options = { 'build_specs': build_specs, 'command_line': eb_cmd_line, 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata), 'pr_path': pr_path, 'robot_path': robot_path, 'silent': testing, 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } # initialise the EasyBuild configuration & build options config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) if modtool is None: modtool = modules_tool(testing=testing) if options.last_log: # print location to last log file, and exit last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) # check whether packaging is supported when it's being used if options.package: check_pkg_support() else: _log.debug("Packaging not enabled, so not checking for packaging support.") # search for easyconfigs, if a query is specified if search_query: search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() elif options.install_github_token: install_github_token(options.github_user, silent=build_option('silent')) elif options.review_pr: print review_pr(options.review_pr, colored=use_color(options.color)) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' print list_software(output_format=options.output_format, detailed=detailed, only_installed=True) elif options.list_software: print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed') # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, options.install_github_token, options.list_installed_software, options.list_software, options.review_pr, options.terse, search_query, ] if any(early_stop_options): cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = modtool.list() # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") if options.install_latest_eb_release: if orig_paths: raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified no_ec_opts = [options.aggregate_regtest, options.new_pr, options.regtest, options.update_pr, search_query] # determine paths to easyconfigs paths = det_easyconfig_paths(categorized_paths['easyconfigs']) if paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in paths] else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif not any(no_ec_opts): print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if options.check_style: _log.debug("Running style check...") if cmdline_easyconfigs_style_check([path[0] for path in paths]): print_msg("All style checks passed!", prefix=False) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) else: raise EasyBuildError("One or more style checks FAILED!") # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) dry_run_mode = options.dry_run or options.dry_run_short new_update_pr = options.new_pr or options.update_pr # skip modules that are already installed unless forced if not (options.force or options.rebuild or dry_run_mode or options.extended_dry_run or new_update_pr): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled if options.robot and (not dry_run_mode or new_update_pr): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs elif new_update_pr: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # creating/updating PRs if new_update_pr: if options.new_pr: new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) else: update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg) cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) elif options.check_conflicts: if check_conflicts(easyconfigs, modtool): print_error("One or more conflicts detected!") sys.exit(1) else: print_msg("\nNo conflicts detected!\n", prefix=False) # dump source script to set up build environment elif options.dump_env_script: dump_env_script(easyconfigs) # cleanup and exit after dry run, searching easyconfigs or submitting regression test if any(no_ec_opts + [options.check_conflicts, dry_run_mode, options.dump_env_script]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
def test_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ self.install_mock_module() base_easyconfig_dir = find_full_path(os.path.join('test', 'framework', 'easyconfigs', 'test_ecs')) self.assertTrue(base_easyconfig_dir) easyconfig = { 'spec': '_', 'full_mod_name': 'name/version', 'short_mod_name': 'name/version', 'dependencies': [] } build_options = { 'allow_modules_tool_mismatch': True, 'external_modules_metadata': ConfigObj(), 'robot_path': None, 'validate': False, } init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig)], self.modtool) self.assertEqual([easyconfig], res) easyconfig_dep = { 'ec': { 'name': 'foo', 'version': '1.2.3', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, }, 'spec': '_', 'short_mod_name': 'foo/1.2.3', 'full_mod_name': 'foo/1.2.3', 'dependencies': [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': False, }], 'parsed': True, } build_options.update({'robot': True, 'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool) # dependency should be found, order should be correct self.assertEqual(len(res), 2) self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) # hidden dependencies are found too, but only retained if they're not available (or forced to be retained hidden_dep = { 'name': 'toy', 'version': '0.0', 'versionsuffix': '-deps', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': True, } easyconfig_moredeps = deepcopy(easyconfig_dep) easyconfig_moredeps['dependencies'].append(hidden_dep) easyconfig_moredeps['hiddendependencies'] = [hidden_dep] # toy/.0.0-deps is available and thus should be omitted res = resolve_dependencies([deepcopy(easyconfig_moredeps)], self.modtool) self.assertEqual(len(res), 2) full_mod_names = [ec['full_mod_name'] for ec in res] self.assertFalse('toy/.0.0-deps' in full_mod_names) res = resolve_dependencies([deepcopy(easyconfig_moredeps)], self.modtool, retain_all_deps=True) self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1) self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) full_mod_names = [ec['full_mod_name'] for ec in res] self.assertTrue('toy/.0.0-deps' in full_mod_names) self.assertTrue('ictce/4.1.13' in full_mod_names) # here we have included a dependency in the easyconfig list easyconfig['full_mod_name'] = 'gzip/1.4' ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] build_options.update({'robot_path': None}) init_config(build_options=build_options) res = resolve_dependencies(ecs, self.modtool) # all dependencies should be resolved self.assertEqual(0, sum(len(ec['dependencies']) for ec in res)) # this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled ecs = [deepcopy(easyconfig_dep)] msg = "Irresolvable dependencies encountered" self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs, self.modtool) # test if dependencies of an automatically found file are also loaded easyconfig_dep['dependencies'] = [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}, 'dummy': True, 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] build_options.update({'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool) # GCC should be first (required by gzip dependency) self.assertEqual('GCC/4.6.3', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) # make sure that only missing stuff is built, and that available modules are not rebuilt # monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'FFTW/3.3.3-gompi-1.4.10', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs, self.modtool) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 2) # goolf should be first, the software itself second self.assertEqual('goolf/1.4.10', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[1]['full_mod_name']) # force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt build_options.update({'force': True}) init_config(build_options=build_options) easyconfig['full_mod_name'] = 'this/is/already/there' MockModule.avail_modules.append('this/is/already/there') ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] res = resolve_dependencies(ecs, self.modtool) # there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available) self.assertEqual(len(res), 3) # goolf should be first, the software itself second self.assertEqual('this/is/already/there', res[0]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[1]['full_mod_name']) self.assertEqual('foo/1.2.3', res[2]['full_mod_name']) # build that are listed but already have a module available are not retained without force build_options.update({'force': False}) init_config(build_options=build_options) newecs = skip_available(ecs, self.modtool) # skip available builds since force is not enabled res = resolve_dependencies(newecs, self.modtool) self.assertEqual(len(res), 2) self.assertEqual('goolf/1.4.10', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[1]['full_mod_name']) # with retain_all_deps enabled, all dependencies ae retained build_options.update({'retain_all_deps': True}) init_config(build_options=build_options) ecs = [deepcopy(easyconfig_dep)] newecs = skip_available(ecs, self.modtool) # skip available builds since force is not enabled res = resolve_dependencies(newecs, self.modtool) self.assertEqual(len(res), 9) self.assertEqual('GCC/4.7.2', res[0]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[-2]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) build_options.update({'retain_all_deps': False}) init_config(build_options=build_options) # provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'gompi/1.4.10', 'FFTW/3.3.3-gompi-1.4.10', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)], self.modtool) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 4) # goolf should be first, the software itself second self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['full_mod_name']) self.assertEqual('ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[2]['full_mod_name']) self.assertEqual('foo/1.2.3', res[3]['full_mod_name'])
# process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend( process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError, err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug( "Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % { 'cmd': cmd } build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
def xtest_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ # replace Modules class with something we have control over config.modules_tool = mock_module ectools.modules_tool = mock_module robot.modules_tool = mock_module os.environ['module'] = "() { eval `/bin/echo $*`\n}" base_easyconfig_dir = find_full_path(os.path.join("test", "framework", "easyconfigs")) self.assertTrue(base_easyconfig_dir) easyconfig = { 'spec': '_', 'full_mod_name': 'name/version', 'short_mod_name': 'name/version', 'dependencies': [] } build_options = { 'allow_modules_tool_mismatch': True, 'robot_path': None, 'validate': False, } init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig)]) self.assertEqual([easyconfig], res) easyconfig_dep = { 'ec': { 'name': 'foo', 'version': '1.2.3', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, }, 'spec': '_', 'short_mod_name': 'foo/1.2.3', 'full_mod_name': 'foo/1.2.3', 'dependencies': [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': False, }], 'parsed': True, } build_options.update({'robot': True, 'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) # dependency should be found, order should be correct self.assertEqual(len(res), 2) self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) # hidden dependencies are found too, but only retained if they're not available (or forced to be retained hidden_dep = { 'name': 'toy', 'version': '0.0', 'versionsuffix': '-deps', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': True, } easyconfig_moredeps = deepcopy(easyconfig_dep) easyconfig_moredeps['dependencies'].append(hidden_dep) easyconfig_moredeps['hiddendependencies'] = [hidden_dep] # toy/.0.0-deps is available and thus should be omitted res = resolve_dependencies([deepcopy(easyconfig_moredeps)]) self.assertEqual(len(res), 2) full_mod_names = [ec['full_mod_name'] for ec in res] self.assertFalse('toy/.0.0-deps' in full_mod_names) res = resolve_dependencies([deepcopy(easyconfig_moredeps)], retain_all_deps=True) self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1) self.assertEqual('gzip/1.4', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) full_mod_names = [ec['full_mod_name'] for ec in res] self.assertTrue('toy/.0.0-deps' in full_mod_names) self.assertTrue('ictce/4.1.13' in full_mod_names) # here we have included a dependency in the easyconfig list easyconfig['full_mod_name'] = 'gzip/1.4' ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] build_options.update({'robot_path': None}) init_config(build_options=build_options) res = resolve_dependencies(ecs) # all dependencies should be resolved self.assertEqual(0, sum(len(ec['dependencies']) for ec in res)) # this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled ecs = [deepcopy(easyconfig_dep)] msg = "Irresolvable dependencies encountered" self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs) # test if dependencies of an automatically found file are also loaded easyconfig_dep['dependencies'] = [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}, 'dummy': True, 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] build_options.update({'robot_path': base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) # GCC should be first (required by gzip dependency) self.assertEqual('GCC/4.6.3', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) # make sure that only missing stuff is built, and that available modules are not rebuilt # monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'FFTW/3.3.3-gompi-1.4.10', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 2) # goolf should be first, the software itself second self.assertEqual('goolf/1.4.10', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[1]['full_mod_name']) # force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt build_options.update({'force': True}) init_config(build_options=build_options) easyconfig['full_mod_name'] = 'this/is/already/there' MockModule.avail_modules.append('this/is/already/there') ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] res = resolve_dependencies(ecs) # there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available) self.assertEqual(len(res), 3) # goolf should be first, the software itself second self.assertEqual('this/is/already/there', res[0]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[1]['full_mod_name']) self.assertEqual('foo/1.2.3', res[2]['full_mod_name']) # build that are listed but already have a module available are not retained without force build_options.update({'force': False}) init_config(build_options=build_options) newecs = skip_available(ecs) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 2) self.assertEqual('goolf/1.4.10', res[0]['full_mod_name']) self.assertEqual('foo/1.2.3', res[1]['full_mod_name']) # with retain_all_deps enabled, all dependencies ae retained build_options.update({'retain_all_deps': True}) init_config(build_options=build_options) ecs = [deepcopy(easyconfig_dep)] newecs = skip_available(ecs) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 9) self.assertEqual('GCC/4.7.2', res[0]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[-2]['full_mod_name']) self.assertEqual('foo/1.2.3', res[-1]['full_mod_name']) build_options.update({'retain_all_deps': False}) init_config(build_options=build_options) # provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'gompi/1.4.10', 'FFTW/3.3.3-gompi-1.4.10', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, 'hidden': False, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)]) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 4) # goolf should be first, the software itself second self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['full_mod_name']) self.assertEqual('ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['full_mod_name']) self.assertEqual('goolf/1.4.10', res[2]['full_mod_name']) self.assertEqual('foo/1.2.3', res[3]['full_mod_name']) config.modules_tool = ORIG_MODULES_TOOL ectools.modules_tool = ORIG_ECTOOLS_MODULES_TOOL robot.modules_tool = ORIG_ROBOT_MODULES_TOOL if ORIG_MODULE_FUNCTION is not None: os.environ['module'] = ORIG_MODULE_FUNCTION else: if 'module' in os.environ: del os.environ['module']
# dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: print_dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) if any([ options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short ]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: easyconfigs = skip_available(easyconfigs, testing=testing) # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) sys.exit(0)
def main(testing_data=(None, None, None)): """ Main function: parse command line options, and act accordingly. @param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # steer behavior when testing main testing = testing_data[0] is not None args, logfile, do_build = testing_data # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set temporary directory to use eb_tmpdir = set_tmpdir(options.tmpdir) # initialise logging for main global _log _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, testing=testing) # disallow running EasyBuild as root if os.getuid() == 0: raise EasyBuildError( "You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here.") # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot) _log.debug("Full robot path: %s" % robot_path) # configure & initialize build options config_options_dict = eb_go.get_options_by_section('config') build_options = { 'build_specs': build_specs, 'command_line': eb_cmd_line, 'pr_path': pr_path, 'robot_path': robot_path, 'silent': testing, 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } # initialise the EasyBuild configuration & build options config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = session_module_list( testing=testing ) # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # search for easyconfigs, if a query is specified query = options.search or options.search_short if query: search_easyconfigs(query, short=not options.search) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning( "Failed to determine install path for easybuild-easyconfigs package." ) # determine paths to easyconfigs paths = det_easyconfig_paths(orig_paths) if paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in paths] else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif not any([ options.aggregate_regtest, options.search, options.search_short, options.regtest ]): print_error(( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path) # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: txt = dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) print_msg(txt, log=_log, silent=testing, prefix=False) # cleanup and exit after dry run, searching easyconfigs or submitting regression test if any([ options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short ]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: retained_ecs = skip_available(easyconfigs) if not testing: for skipped_ec in [ ec for ec in easyconfigs if ec not in retained_ecs ]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: if options.robot: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: ordered_ecs = easyconfigs else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) sys.exit(0) # submit build as job(s), clean up and exit if options.job: job_info_txt = submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now: %s" % job_info_txt) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): ecs_with_res = build_and_install_software( ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False) ]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
def test_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ easyconfig = { 'spec': '_', 'module': 'name/version', 'dependencies': [] } build_options = { 'ignore_osdeps': True, 'robot_path': None, 'validate': False, } res = resolve_dependencies([deepcopy(easyconfig)], build_options=build_options) self.assertEqual([easyconfig], res) easyconfig_dep = { 'ec': { 'name': 'foo', 'version': '1.2.3', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, }, 'spec': '_', 'module': 'foo/1.2.3', 'dependencies': [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, }], 'parsed': True, } build_options.update({'robot_path': self.base_easyconfig_dir}) res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options) # dependency should be found, order should be correct self.assertEqual(len(res), 2) self.assertEqual('gzip/1.4', res[0]['module']) self.assertEqual('foo/1.2.3', res[-1]['module']) # here we have include a Dependency in the easyconfig list easyconfig['module'] = 'gzip/1.4' ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] build_options.update({'robot_path': None}) res = resolve_dependencies(ecs, build_options=build_options) # all dependencies should be resolved self.assertEqual(0, sum(len(ec['dependencies']) for ec in res)) # this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled ecs = [deepcopy(easyconfig_dep)] msg = "Irresolvable dependencies encountered" self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs, build_options=build_options) # test if dependencies of an automatically found file are also loaded easyconfig_dep['dependencies'] = [{ 'name': 'gzip', 'version': '1.4', 'versionsuffix': '', 'toolchain': {'name': 'GCC', 'version': '4.6.3'}, 'dummy': True, }] ecs = [deepcopy(easyconfig_dep)] build_options.update({'robot_path': self.base_easyconfig_dir}) res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options) # GCC should be first (required by gzip dependency) self.assertEqual('GCC/4.6.3', res[0]['module']) self.assertEqual('foo/1.2.3', res[-1]['module']) # make sure that only missing stuff is built, and that available modules are not rebuilt # monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', 'FFTW/3.3.3-gompi-1.4.10', 'ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs, build_options=build_options) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 2) # goolf should be first, the software itself second self.assertEqual('goolf/1.4.10', res[0]['module']) self.assertEqual('foo/1.2.3', res[1]['module']) # force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt build_options.update({'force': True}) easyconfig['module'] = 'this/is/already/there' MockModule.avail_modules.append('this/is/already/there') ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] res = resolve_dependencies(ecs, build_options=build_options) # there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available) self.assertEqual(len(res), 3) # goolf should be first, the software itself second self.assertEqual('this/is/already/there', res[0]['module']) self.assertEqual('goolf/1.4.10', res[1]['module']) self.assertEqual('foo/1.2.3', res[2]['module']) # build that are listed but already have a module available are not retained without force build_options.update({'force': False}) newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled res = resolve_dependencies(newecs, build_options=build_options) self.assertEqual(len(res), 2) self.assertEqual('goolf/1.4.10', res[0]['module']) self.assertEqual('foo/1.2.3', res[1]['module']) # with retain_all_deps enabled, all dependencies ae retained build_options.update({'retain_all_deps': True}) ecs = [deepcopy(easyconfig_dep)] newecs = skip_available(ecs, testing=True) # skip available builds since force is not enabled res = resolve_dependencies(newecs, build_options=build_options) self.assertEqual(len(res), 9) self.assertEqual('GCC/4.7.2', res[0]['module']) self.assertEqual('goolf/1.4.10', res[-2]['module']) self.assertEqual('foo/1.2.3', res[-1]['module']) build_options.update({'retain_all_deps': False}) # provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up MockModule.avail_modules = [ 'GCC/4.7.2', 'OpenMPI/1.6.4-GCC-4.7.2', 'gompi/1.4.10', 'FFTW/3.3.3-gompi-1.4.10', ] easyconfig_dep['dependencies'] = [{ 'name': 'goolf', 'version': '1.4.10', 'versionsuffix': '', 'toolchain': {'name': 'dummy', 'version': 'dummy'}, 'dummy': True, }] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)], build_options=build_options) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 4) # goolf should be first, the software itself second self.assertEqual('OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2', res[0]['module']) self.assertEqual('ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2', res[1]['module']) self.assertEqual('goolf/1.4.10', res[2]['module']) self.assertEqual('foo/1.2.3', res[3]['module'])
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set by option parsers via set_tmpdir eb_tmpdir = tempfile.gettempdir() search_query = options.search or options.search_filename or options.search_short # initialise logging for main global _log _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=(testing or options.terse or search_query), colorize=options.color) # disallow running EasyBuild as root (by default) check_root_usage( allow_use_as_root=options.allow_use_as_root_and_accept_consequences) # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot) _log.debug("Full robot path: %s" % robot_path) # configure & initialize build options config_options_dict = eb_go.get_options_by_section('config') build_options = { 'build_specs': build_specs, 'command_line': eb_cmd_line, 'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata), 'pr_path': pr_path, 'robot_path': robot_path, 'silent': testing, 'try_to_generate': try_to_generate, 'valid_stops': [x[0] for x in EasyBlock.get_steps()], } # initialise the EasyBuild configuration & build options config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) if modtool is None: modtool = modules_tool(testing=testing) # check whether any (EasyBuild-generated) modules are loaded already in the current session modtool.check_loaded_modules() if options.last_log: # print location to last log file, and exit last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) # check whether packaging is supported when it's being used if options.package: check_pkg_support() else: _log.debug( "Packaging not enabled, so not checking for packaging support.") # search for easyconfigs, if a query is specified if search_query: search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() elif options.install_github_token: install_github_token(options.github_user, silent=build_option('silent')) elif options.merge_pr: merge_pr(options.merge_pr) elif options.review_pr: print review_pr(pr=options.review_pr, colored=use_color(options.color)) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' print list_software(output_format=options.output_format, detailed=detailed, only_installed=True) elif options.list_software: print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed') # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, options.install_github_token, options.list_installed_software, options.list_software, options.merge_pr, options.review_pr, options.terse, search_query, ] if any(early_stop_options): cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = modtool.list( ) # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning( "Failed to determine install path for easybuild-easyconfigs package." ) if options.install_latest_eb_release: if orig_paths: raise EasyBuildError( "Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr no_ec_opts = [ options.aggregate_regtest, options.regtest, search_query, new_update_preview_pr ] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) if determined_paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in determined_paths] else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif any(no_ec_opts): paths = determined_paths else: print_error(( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 if options.check_style: _log.debug("Running style check...") if cmdline_easyconfigs_style_check([path[0] for path in paths]): print_msg("All style checks passed!", prefix=False) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) else: raise EasyBuildError("One or more style checks FAILED!") # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs( paths, validate=not options.inject_checksums) # verify easyconfig filenames, if desired if options.verify_easyconfig_filenames: _log.info("Verifying easyconfig filenames...") for easyconfig in easyconfigs: verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec']) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) if options.containerize: containerize(easyconfigs) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ ec for ec in easyconfigs if ec not in retained_ecs ]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled if options.robot and (not dry_run_mode or new_update_preview_pr): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs elif new_update_preview_pr: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # creating/updating PRs if new_update_preview_pr: if options.new_pr: new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) elif options.preview_pr: print review_pr(paths=determined_paths, colored=use_color(options.color)) else: update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg) # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) elif options.check_conflicts: if check_conflicts(easyconfigs, modtool): print_error("One or more conflicts detected!") sys.exit(1) else: print_msg("\nNo conflicts detected!\n", prefix=False) # dump source script to set up build environment elif options.dump_env_script: dump_env_script(easyconfigs) elif options.inject_checksums: inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [ options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums ] if any(no_ec_opts) or any(stop_options): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) hooks = load_hooks(options.hooks) ecs_with_res = build_and_install_software( ordered_ecs, init_session_state, exit_on_failure=exit_on_failure, hooks=hooks) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False) ]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args global _log (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings # load hook implementations (if any) hooks = load_hooks(options.hooks) run_hook(START, hooks) if modtool is None: modtool = modules_tool(testing=testing) # check whether any (EasyBuild-generated) modules are loaded already in the current session modtool.check_loaded_modules() if options.last_log: # print location to last log file, and exit last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) # check whether packaging is supported when it's being used if options.package: check_pkg_support() else: _log.debug("Packaging not enabled, so not checking for packaging support.") # search for easyconfigs, if a query is specified if search_query: search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() elif options.install_github_token: install_github_token(options.github_user, silent=build_option('silent')) elif options.close_pr: close_pr(options.close_pr, motivation_msg=options.close_pr_msg) elif options.list_prs: print(list_prs(options.list_prs)) elif options.merge_pr: merge_pr(options.merge_pr) elif options.review_pr: print(review_pr(pr=options.review_pr, colored=use_color(options.color))) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True)) elif options.list_software: print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')) # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, options.install_github_token, options.list_installed_software, options.list_software, options.close_pr, options.list_prs, options.merge_pr, options.review_pr, options.terse, search_query, ] if any(early_stop_options): clean_exit(logfile, eb_tmpdir, testing, silent=True) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = modtool.list() # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") if options.install_latest_eb_release: if orig_paths: raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr no_ec_opts = [options.aggregate_regtest, options.regtest, search_query, new_update_preview_pr] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) if determined_paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in determined_paths] else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif any(no_ec_opts): paths = determined_paths else: print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums) # handle --check-contrib & --check-style options if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style): clean_exit(logfile, eb_tmpdir, testing) # verify easyconfig filenames, if desired if options.verify_easyconfig_filenames: _log.info("Verifying easyconfig filenames...") for easyconfig in easyconfigs: verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec']) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) if options.containerize: # if --containerize/-C create a container recipe (and optionally container image), and stop containerize(easyconfigs) clean_exit(logfile, eb_tmpdir, testing) forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled if options.robot and (not dry_run_mode or new_update_preview_pr): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs elif new_update_preview_pr: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # creating/updating PRs if new_update_preview_pr: if options.new_pr: new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) elif options.preview_pr: print(review_pr(paths=determined_paths, colored=use_color(options.color))) else: update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg) # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) elif options.check_conflicts: if check_conflicts(easyconfigs, modtool): print_error("One or more conflicts detected!") sys.exit(1) else: print_msg("\nNo conflicts detected!\n", prefix=False) # dump source script to set up build environment elif options.dump_env_script: dump_env_script(easyconfigs) elif options.inject_checksums: inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] if any(no_ec_opts) or any(stop_options): clean_exit(logfile, eb_tmpdir, testing) # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) clean_exit(logfile, eb_tmpdir, testing, silent=True) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") clean_exit(logfile, eb_tmpdir, testing) # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) run_hook(END, hooks) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
_log.error("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError, err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, build_specs=build_specs) cmd = "eb %(spec)s --regtest --sequential -ld" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd} jobs = build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) print "List of submitted jobs:"
def test_resolve_dependencies(self): """ Test with some basic testcases (also check if he can find dependencies inside the given directory """ easyconfig = { "spec": "_", "full_mod_name": "name/version", "short_mod_name": "name/version", "dependencies": [], } build_options = {"allow_modules_tool_mismatch": True, "robot_path": None, "validate": False} init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig)]) self.assertEqual([easyconfig], res) easyconfig_dep = { "ec": { "name": "foo", "version": "1.2.3", "versionsuffix": "", "toolchain": {"name": "dummy", "version": "dummy"}, }, "spec": "_", "short_mod_name": "foo/1.2.3", "full_mod_name": "foo/1.2.3", "dependencies": [ { "name": "gzip", "version": "1.4", "versionsuffix": "", "toolchain": {"name": "dummy", "version": "dummy"}, "dummy": True, "hidden": False, } ], "parsed": True, } build_options.update({"robot": True, "robot_path": self.base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) # dependency should be found, order should be correct self.assertEqual(len(res), 2) self.assertEqual("gzip/1.4", res[0]["full_mod_name"]) self.assertEqual("foo/1.2.3", res[-1]["full_mod_name"]) # hidden dependencies are found too, but only retained if they're not available (or forced to be retained hidden_dep = { "name": "toy", "version": "0.0", "versionsuffix": "-deps", "toolchain": {"name": "dummy", "version": "dummy"}, "dummy": True, "hidden": True, } easyconfig_moredeps = deepcopy(easyconfig_dep) easyconfig_moredeps["dependencies"].append(hidden_dep) easyconfig_moredeps["hiddendependencies"] = [hidden_dep] # toy/.0.0-deps is available and thus should be omitted res = resolve_dependencies([deepcopy(easyconfig_moredeps)]) self.assertEqual(len(res), 2) full_mod_names = [ec["full_mod_name"] for ec in res] self.assertFalse("toy/.0.0-deps" in full_mod_names) res = resolve_dependencies([deepcopy(easyconfig_moredeps)], retain_all_deps=True) self.assertEqual(len(res), 4) # hidden dep toy/.0.0-deps (+1) depends on (fake) ictce/4.1.13 (+1) self.assertEqual("gzip/1.4", res[0]["full_mod_name"]) self.assertEqual("foo/1.2.3", res[-1]["full_mod_name"]) full_mod_names = [ec["full_mod_name"] for ec in res] self.assertTrue("toy/.0.0-deps" in full_mod_names) self.assertTrue("ictce/4.1.13" in full_mod_names) # here we have included a dependency in the easyconfig list easyconfig["full_mod_name"] = "gzip/1.4" ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] build_options.update({"robot_path": None}) init_config(build_options=build_options) res = resolve_dependencies(ecs) # all dependencies should be resolved self.assertEqual(0, sum(len(ec["dependencies"]) for ec in res)) # this should not resolve (cannot find gzip-1.4.eb), both with and without robot enabled ecs = [deepcopy(easyconfig_dep)] msg = "Irresolvable dependencies encountered" self.assertErrorRegex(EasyBuildError, msg, resolve_dependencies, ecs) # test if dependencies of an automatically found file are also loaded easyconfig_dep["dependencies"] = [ { "name": "gzip", "version": "1.4", "versionsuffix": "", "toolchain": {"name": "GCC", "version": "4.6.3"}, "dummy": True, "hidden": False, } ] ecs = [deepcopy(easyconfig_dep)] build_options.update({"robot_path": self.base_easyconfig_dir}) init_config(build_options=build_options) res = resolve_dependencies([deepcopy(easyconfig_dep)]) # GCC should be first (required by gzip dependency) self.assertEqual("GCC/4.6.3", res[0]["full_mod_name"]) self.assertEqual("foo/1.2.3", res[-1]["full_mod_name"]) # make sure that only missing stuff is built, and that available modules are not rebuilt # monkey patch MockModule to pretend that all ingredients required for goolf-1.4.10 toolchain are present MockModule.avail_modules = [ "GCC/4.7.2", "OpenMPI/1.6.4-GCC-4.7.2", "OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2", "FFTW/3.3.3-gompi-1.4.10", "ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2", ] easyconfig_dep["dependencies"] = [ { "name": "goolf", "version": "1.4.10", "versionsuffix": "", "toolchain": {"name": "dummy", "version": "dummy"}, "dummy": True, "hidden": False, } ] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies(ecs) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 2) # goolf should be first, the software itself second self.assertEqual("goolf/1.4.10", res[0]["full_mod_name"]) self.assertEqual("foo/1.2.3", res[1]["full_mod_name"]) # force doesn't trigger rebuild of all deps, but listed easyconfigs for which a module is available are rebuilt build_options.update({"force": True}) init_config(build_options=build_options) easyconfig["full_mod_name"] = "this/is/already/there" MockModule.avail_modules.append("this/is/already/there") ecs = [deepcopy(easyconfig_dep), deepcopy(easyconfig)] res = resolve_dependencies(ecs) # there should only be three retained builds, foo + goolf dep and the additional build (even though a module is available) self.assertEqual(len(res), 3) # goolf should be first, the software itself second self.assertEqual("this/is/already/there", res[0]["full_mod_name"]) self.assertEqual("goolf/1.4.10", res[1]["full_mod_name"]) self.assertEqual("foo/1.2.3", res[2]["full_mod_name"]) # build that are listed but already have a module available are not retained without force build_options.update({"force": False}) init_config(build_options=build_options) newecs = skip_available(ecs) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 2) self.assertEqual("goolf/1.4.10", res[0]["full_mod_name"]) self.assertEqual("foo/1.2.3", res[1]["full_mod_name"]) # with retain_all_deps enabled, all dependencies ae retained build_options.update({"retain_all_deps": True}) init_config(build_options=build_options) ecs = [deepcopy(easyconfig_dep)] newecs = skip_available(ecs) # skip available builds since force is not enabled res = resolve_dependencies(newecs) self.assertEqual(len(res), 9) self.assertEqual("GCC/4.7.2", res[0]["full_mod_name"]) self.assertEqual("goolf/1.4.10", res[-2]["full_mod_name"]) self.assertEqual("foo/1.2.3", res[-1]["full_mod_name"]) build_options.update({"retain_all_deps": False}) init_config(build_options=build_options) # provide even less goolf ingredients (no OpenBLAS/ScaLAPACK), make sure the numbers add up MockModule.avail_modules = ["GCC/4.7.2", "OpenMPI/1.6.4-GCC-4.7.2", "gompi/1.4.10", "FFTW/3.3.3-gompi-1.4.10"] easyconfig_dep["dependencies"] = [ { "name": "goolf", "version": "1.4.10", "versionsuffix": "", "toolchain": {"name": "dummy", "version": "dummy"}, "dummy": True, "hidden": False, } ] ecs = [deepcopy(easyconfig_dep)] res = resolve_dependencies([deepcopy(easyconfig_dep)]) # there should only be two retained builds, i.e. the software itself and the goolf toolchain as dep self.assertEqual(len(res), 4) # goolf should be first, the software itself second self.assertEqual("OpenBLAS/0.2.6-gompi-1.4.10-LAPACK-3.4.2", res[0]["full_mod_name"]) self.assertEqual("ScaLAPACK/2.0.2-gompi-1.4.10-OpenBLAS-0.2.6-LAPACK-3.4.2", res[1]["full_mod_name"]) self.assertEqual("goolf/1.4.10", res[2]["full_mod_name"]) self.assertEqual("foo/1.2.3", res[3]["full_mod_name"])
raise EasyBuildError("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError, err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs, modtool) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, modtool) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd} build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) _log.info("Submitted regression test as jobs, results in %s" % output_dir)
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode """ register_lock_cleanup_signal_handlers() # if $CDPATH is set, unset it, it'll only cause trouble... # see https://github.com/easybuilders/easybuild-framework/issues/2944 if 'CDPATH' in os.environ: del os.environ['CDPATH'] # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args global _log (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings # load hook implementations (if any) hooks = load_hooks(options.hooks) run_hook(START, hooks) if modtool is None: modtool = modules_tool(testing=testing) # check whether any (EasyBuild-generated) modules are loaded already in the current session modtool.check_loaded_modules() if options.last_log: # print location to last log file, and exit last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) # check whether packaging is supported when it's being used if options.package: check_pkg_support() else: _log.debug("Packaging not enabled, so not checking for packaging support.") # search for easyconfigs, if a query is specified if search_query: search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() elif options.install_github_token: install_github_token(options.github_user, silent=build_option('silent')) elif options.close_pr: close_pr(options.close_pr, motivation_msg=options.close_pr_msg) elif options.list_prs: print(list_prs(options.list_prs)) elif options.merge_pr: merge_pr(options.merge_pr) elif options.review_pr: print(review_pr(pr=options.review_pr, colored=use_color(options.color))) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True)) elif options.list_software: print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')) elif options.create_index: print_msg("Creating index for %s..." % options.create_index, prefix=False) index_fp = dump_index(options.create_index, max_age_sec=options.index_max_age) index = load_index(options.create_index) print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False) # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, options.create_index, options.install_github_token, options.list_installed_software, options.list_software, options.close_pr, options.list_prs, options.merge_pr, options.review_pr, options.terse, search_query, ] if any(early_stop_options): clean_exit(logfile, eb_tmpdir, testing, silent=True) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = modtool.list() # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") if options.install_latest_eb_release: if orig_paths: raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) if len(orig_paths) == 1: # if only one easyconfig file is specified, use current directory as target directory target_path = os.getcwd() elif orig_paths: # last path is target when --copy-ec is used, so remove that from the list target_path = orig_paths.pop() if options.copy_ec else None categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr pr_options = pr_options or options.sync_branch_with_develop or options.sync_pr_with_develop pr_options = pr_options or options.update_branch_github or options.update_pr no_ec_opts = [options.aggregate_regtest, options.regtest, pr_options, search_query] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: if len(determined_paths) == 1: copy_file(determined_paths[0], target_path) print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=False) elif len(determined_paths) > 1: copy_files(determined_paths, target_path) print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False) else: raise EasyBuildError("One of more files to copy should be specified!") elif options.fix_deprecated_easyconfigs: fix_deprecated_easyconfigs(determined_paths) elif options.show_ec: for path in determined_paths: print_msg("Contents of %s:" % path) print_msg(read_file(path), prefix=False) clean_exit(logfile, eb_tmpdir, testing) if determined_paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in determined_paths] elif 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif any(no_ec_opts): paths = determined_paths else: print_error("Please provide one or multiple easyconfig files, or use software build " + "options to make EasyBuild search for easyconfigs", log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s", paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums) # handle --check-contrib & --check-style options if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style): clean_exit(logfile, eb_tmpdir, testing) # verify easyconfig filenames, if desired if options.verify_easyconfig_filenames: _log.info("Verifying easyconfig filenames...") for easyconfig in easyconfigs: verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec']) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) if options.containerize: # if --containerize/-C create a container recipe (and optionally container image), and stop containerize(easyconfigs) clean_exit(logfile, eb_tmpdir, testing) forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not (forced or dry_run_mode or options.extended_dry_run or pr_options or options.inject_checksums): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # keep track for which easyconfigs we should set the corresponding module as default if options.set_default_module: for easyconfig in easyconfigs: easyconfig['ec'].set_default_module = True # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled if options.robot and (not dry_run_mode or pr_options): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs elif pr_options: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] if options.copy_ec and tweaked_ecs_paths: all_specs = [spec['spec'] for spec in resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False)] tweaked_ecs_in_all_ecs = [path for path in all_specs if any(tweaked_ecs_path in path for tweaked_ecs_path in tweaked_ecs_paths)] if tweaked_ecs_in_all_ecs: # Clean them, then copy them clean_up_easyconfigs(tweaked_ecs_in_all_ecs) copy_files(tweaked_ecs_in_all_ecs, target_path) print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False) # creating/updating PRs if pr_options: if options.new_pr: new_pr(categorized_paths, ordered_ecs) elif options.new_branch_github: new_branch_github(categorized_paths, ordered_ecs) elif options.new_pr_from_branch: new_pr_from_branch(options.new_pr_from_branch) elif options.preview_pr: print(review_pr(paths=determined_paths, colored=use_color(options.color))) elif options.sync_branch_with_develop: sync_branch_with_develop(options.sync_branch_with_develop) elif options.sync_pr_with_develop: sync_pr_with_develop(options.sync_pr_with_develop) elif options.update_branch_github: update_branch(options.update_branch_github, categorized_paths, ordered_ecs) elif options.update_pr: update_pr(options.update_pr, categorized_paths, ordered_ecs) else: raise EasyBuildError("Unknown PR option!") # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: if options.missing_modules: txt = missing_deps(easyconfigs, modtool) else: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) elif options.check_conflicts: if check_conflicts(easyconfigs, modtool): print_error("One or more conflicts detected!") sys.exit(1) else: print_msg("\nNo conflicts detected!\n", prefix=False) # dump source script to set up build environment elif options.dump_env_script: dump_env_script(easyconfigs) elif options.inject_checksums: inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] if any(no_ec_opts) or any(stop_options): clean_exit(logfile, eb_tmpdir, testing) # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) clean_exit(logfile, eb_tmpdir, testing, silent=True) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") clean_exit(logfile, eb_tmpdir, testing) # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) run_hook(END, hooks) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
def main(args=None, logfile=None, do_build=None, testing=False): """ Main function: parse command line options, and act accordingly. @param args: command line arguments to use @param logfile: log file to use @param do_build: whether or not to actually perform the build @param testing: enable testing mode """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() # initialise options eb_go = eboptions.parse_options(args=args) options = eb_go.options orig_paths = eb_go.args # set umask (as early as possible) if options.umask is not None: new_umask = int(options.umask, 8) old_umask = os.umask(new_umask) # set by option parsers via set_tmpdir eb_tmpdir = tempfile.gettempdir() # initialise logging for main global _log _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=testing or options.last_log) # disallow running EasyBuild as root if os.getuid() == 0: raise EasyBuildError( "You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here." ) # log startup info eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args log_start(eb_cmd_line, eb_tmpdir) if options.umask is not None: _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask))) # process software build specifications (if any), i.e. # software name/version, toolchain name/version, extra patches, ... (try_to_generate, build_specs) = process_software_build_specs(options) # determine robot path # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs tweaked_ecs = try_to_generate and build_specs tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr) auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot) _log.debug("Full robot path: %s" % robot_path) # configure & initialize build options config_options_dict = eb_go.get_options_by_section("config") build_options = { "build_specs": build_specs, "command_line": eb_cmd_line, "pr_path": pr_path, "robot_path": robot_path, "silent": testing, "try_to_generate": try_to_generate, "valid_stops": [x[0] for x in EasyBlock.get_steps()], } # initialise the EasyBuild configuration & build options config.init(options, config_options_dict) config.init_build_options(build_options=build_options, cmdline_options=options) if options.last_log: # print location to last log file, and exit last_log = find_last_log(logfile) or "(none)" print_msg(last_log, log=_log, prefix=False) cleanup(logfile, eb_tmpdir, testing, silent=True) sys.exit(0) # check whether packaging is supported when it's being used if options.package: check_pkg_support() else: _log.debug("Packaging not enabled, so not checking for packaging support.") # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works init_session_state.update({"easybuild_configuration": eb_config}) init_session_state.update({"module_list": modlist}) _log.debug("Initial session state: %s" % init_session_state) # GitHub integration if options.review_pr or options.new_pr or options.update_pr: if options.review_pr: print review_pr(options.review_pr, colored=options.color) elif options.new_pr: new_pr(orig_paths, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) elif options.update_pr: update_pr(options.update_pr, orig_paths, commit_msg=options.pr_commit_msg) cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # search for easyconfigs, if a query is specified query = options.search or options.search_short if query: search_easyconfigs(query, short=not options.search) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") # command line options that do not require any easyconfigs to be specified no_ec_opts = [ options.aggregate_regtest, options.new_pr, options.review_pr, options.search, options.search_short, options.regtest, options.update_pr, ] # determine paths to easyconfigs paths = det_easyconfig_paths(orig_paths) if paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in paths] else: if "name" in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif not any(no_ec_opts): print_error( ( "Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs" ), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing, ) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path) # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: txt = dry_run(easyconfigs, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) # cleanup and exit after dry run, searching easyconfigs or submitting regression test if any(no_ec_opts + [options.dry_run, options.dry_run_short]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not (options.force or options.rebuild or options.extended_dry_run): retained_ecs = skip_available(easyconfigs) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: print_msg("%s is already installed (module found), skipping" % skipped_ec["full_mod_name"]) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: if options.robot: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies( easyconfigs, minimal_toolchains=build_option("minimal_toolchains"), use_existing_modules=build_option("use_existing_modules"), ) else: ordered_ecs = easyconfigs else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) sys.exit(0) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # build software, will exit when errors occurs (except when testing) exit_on_failure = not options.dump_test_report and not options.upload_test_report if not testing or (testing and do_build): ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get("success", False)]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if "original_spec" in ec and os.path.isfile(ec["spec"]): os.remove(ec["spec"]) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
# before building starts, take snapshot of environment (watch out -t option!) orig_environ = copy.deepcopy(os.environ) os.chdir(os.environ['PWD']) # dry_run: print all easyconfigs and dependencies, and whether they are already built if options.dry_run or options.dry_run_short: print_dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs) if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]): cleanup(logfile, eb_tmpdir, testing) sys.exit(0) # skip modules that are already installed unless forced if not options.force: easyconfigs = skip_available(easyconfigs, testing=testing) # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs) else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) sys.exit(0)
def regtest(easyconfig_paths, modtool, build_specs=None): """ Run regression test, using easyconfigs available in given path :param easyconfig_paths: path of easyconfigs to run regtest on :param modtool: ModulesTool instance to use :param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...) """ cur_dir = os.getcwd() aggregate_regtest = build_option('aggregate_regtest') if aggregate_regtest is not None: output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest)) aggregate_xml_in_dirs(aggregate_regtest, output_file) _log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file)) sys.exit(0) # create base directory, which is used to place all log files and the test output as xml regtest_output_dir = build_option('regtest_output_dir') testoutput = build_option('testoutput') if regtest_output_dir is not None: output_dir = regtest_output_dir elif testoutput is not None: output_dir = os.path.abspath(testoutput) else: # default: current dir + easybuild-test-[timestamp] dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S") output_dir = os.path.join(cur_dir, dirname) mkdir(output_dir, parents=True) # find all easyconfigs ecfiles = [] if easyconfig_paths: for path in easyconfig_paths: ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs')) else: raise EasyBuildError("No easyconfig paths specified.") test_results = [] # process all the found easyconfig files easyconfigs = [] for ecfile in ecfiles: try: easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs)) except EasyBuildError as err: test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log)) # skip easyconfigs for which a module is already available, unless forced if not build_option('force'): _log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs) easyconfigs = skip_available(easyconfigs, modtool) _log.debug("Retained easyconfigs after skipping: %s" % easyconfigs) if build_option('sequential'): return build_easyconfigs(easyconfigs, output_dir, test_results) else: resolved = resolve_dependencies(easyconfigs, modtool) cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s" command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd) # retry twice in case of failure, to avoid fluke errors command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd} build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir) _log.info("Submitted regression test as jobs, results in %s" % output_dir) return True # success