def configure(self): """Utility function to set up EasyBuild configuration.""" # wipe BuildOption singleton instance, so it gets re-created when set_up_configuration is called del BuildOptions._instances[BuildOptions] self.assertFalse(BuildOptions in BuildOptions._instances) set_up_configuration(silent=True) self.assertTrue(BuildOptions in BuildOptions._instances)
def configure(self): """Utility function to set up EasyBuild configuration.""" # wipe BuildOption singleton instance, so it gets re-created when set_up_configuration is called if BuildOptions in BuildOptions._instances: del BuildOptions._instances[BuildOptions] self.assertFalse(BuildOptions in BuildOptions._instances) set_up_configuration(silent=True) self.assertTrue(BuildOptions in BuildOptions._instances)
description: stdout of the eb process (not fill if retruncode is zero) type: str stderr: description: stderr of the eb process (not fill if retruncode is zero) type: str ''' import os import subprocess from ansible.module_utils.basic import AnsibleModule # to allow to use easybuild as an API, you need to do something like import easybuild from easybuild.tools.options import set_up_configuration; os.chdir('/tmp') set_up_configuration(silent=True) def run_module(): # define the available arguments/parameters that a user can pass to # the module module_args = dict( package=dict(type='str', required=False, default=''), installpath_modules=dict(type='str', required=True), installpath_software=dict(type='str', required=False, default=''), installpath_source=dict(type='str', required=False, default=''), buildpath=dict(type='str', required=False, default=''), robot=dict(type='bool',required=False, default=True), additional_options=dict(type='str', required=False, default=''), force=dict(type='bool', required=False, default=False),
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode """ register_lock_cleanup_signal_handlers() # if $CDPATH is set, unset it, it'll only cause trouble... # see https://github.com/easybuilders/easybuild-framework/issues/2944 if 'CDPATH' in os.environ: del os.environ['CDPATH'] # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args global _log (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings # load hook implementations (if any) hooks = load_hooks(options.hooks) run_hook(START, hooks) if modtool is None: modtool = modules_tool(testing=testing) # check whether any (EasyBuild-generated) modules are loaded already in the current session modtool.check_loaded_modules() if options.last_log: # print location to last log file, and exit last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) # check whether packaging is supported when it's being used if options.package: check_pkg_support() else: _log.debug("Packaging not enabled, so not checking for packaging support.") # search for easyconfigs, if a query is specified if search_query: search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() elif options.install_github_token: install_github_token(options.github_user, silent=build_option('silent')) elif options.close_pr: close_pr(options.close_pr, motivation_msg=options.close_pr_msg) elif options.list_prs: print(list_prs(options.list_prs)) elif options.merge_pr: merge_pr(options.merge_pr) elif options.review_pr: print(review_pr(pr=options.review_pr, colored=use_color(options.color))) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True)) elif options.list_software: print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')) elif options.create_index: print_msg("Creating index for %s..." % options.create_index, prefix=False) index_fp = dump_index(options.create_index, max_age_sec=options.index_max_age) index = load_index(options.create_index) print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False) # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, options.create_index, options.install_github_token, options.list_installed_software, options.list_software, options.close_pr, options.list_prs, options.merge_pr, options.review_pr, options.terse, search_query, ] if any(early_stop_options): clean_exit(logfile, eb_tmpdir, testing, silent=True) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = modtool.list() # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") if options.install_latest_eb_release: if orig_paths: raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) if len(orig_paths) == 1: # if only one easyconfig file is specified, use current directory as target directory target_path = os.getcwd() elif orig_paths: # last path is target when --copy-ec is used, so remove that from the list target_path = orig_paths.pop() if options.copy_ec else None categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr pr_options = pr_options or options.sync_branch_with_develop or options.sync_pr_with_develop pr_options = pr_options or options.update_branch_github or options.update_pr no_ec_opts = [options.aggregate_regtest, options.regtest, pr_options, search_query] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec: if options.copy_ec: if len(determined_paths) == 1: copy_file(determined_paths[0], target_path) print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=False) elif len(determined_paths) > 1: copy_files(determined_paths, target_path) print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False) else: raise EasyBuildError("One of more files to copy should be specified!") elif options.fix_deprecated_easyconfigs: fix_deprecated_easyconfigs(determined_paths) elif options.show_ec: for path in determined_paths: print_msg("Contents of %s:" % path) print_msg(read_file(path), prefix=False) clean_exit(logfile, eb_tmpdir, testing) if determined_paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in determined_paths] elif 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif any(no_ec_opts): paths = determined_paths else: print_error("Please provide one or multiple easyconfig files, or use software build " + "options to make EasyBuild search for easyconfigs", log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s", paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums) # handle --check-contrib & --check-style options if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style): clean_exit(logfile, eb_tmpdir, testing) # verify easyconfig filenames, if desired if options.verify_easyconfig_filenames: _log.info("Verifying easyconfig filenames...") for easyconfig in easyconfigs: verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec']) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) if options.containerize: # if --containerize/-C create a container recipe (and optionally container image), and stop containerize(easyconfigs) clean_exit(logfile, eb_tmpdir, testing) forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not (forced or dry_run_mode or options.extended_dry_run or pr_options or options.inject_checksums): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # keep track for which easyconfigs we should set the corresponding module as default if options.set_default_module: for easyconfig in easyconfigs: easyconfig['ec'].set_default_module = True # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled if options.robot and (not dry_run_mode or pr_options): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs elif pr_options: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] if options.copy_ec and tweaked_ecs_paths: all_specs = [spec['spec'] for spec in resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False)] tweaked_ecs_in_all_ecs = [path for path in all_specs if any(tweaked_ecs_path in path for tweaked_ecs_path in tweaked_ecs_paths)] if tweaked_ecs_in_all_ecs: # Clean them, then copy them clean_up_easyconfigs(tweaked_ecs_in_all_ecs) copy_files(tweaked_ecs_in_all_ecs, target_path) print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False) # creating/updating PRs if pr_options: if options.new_pr: new_pr(categorized_paths, ordered_ecs) elif options.new_branch_github: new_branch_github(categorized_paths, ordered_ecs) elif options.new_pr_from_branch: new_pr_from_branch(options.new_pr_from_branch) elif options.preview_pr: print(review_pr(paths=determined_paths, colored=use_color(options.color))) elif options.sync_branch_with_develop: sync_branch_with_develop(options.sync_branch_with_develop) elif options.sync_pr_with_develop: sync_pr_with_develop(options.sync_pr_with_develop) elif options.update_branch_github: update_branch(options.update_branch_github, categorized_paths, ordered_ecs) elif options.update_pr: update_pr(options.update_pr, categorized_paths, ordered_ecs) else: raise EasyBuildError("Unknown PR option!") # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: if options.missing_modules: txt = missing_deps(easyconfigs, modtool) else: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) elif options.check_conflicts: if check_conflicts(easyconfigs, modtool): print_error("One or more conflicts detected!") sys.exit(1) else: print_msg("\nNo conflicts detected!\n", prefix=False) # dump source script to set up build environment elif options.dump_env_script: dump_env_script(easyconfigs) elif options.inject_checksums: inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] if any(no_ec_opts) or any(stop_options): clean_exit(logfile, eb_tmpdir, testing) # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) clean_exit(logfile, eb_tmpdir, testing, silent=True) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") clean_exit(logfile, eb_tmpdir, testing) # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) run_hook(END, hooks) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
def main(): """Main function.""" if len(sys.argv) == 3: ec = sys.argv[1] tc = sys.argv[2] else: error("Usage %s <easyconfig> <toolchain> [<name=version>]" % sys.argv[0]) tc_name, tc_ver = tc.split('/') print("Updating %s for %s toolchain version %s..." % (ec, tc_name, tc_ver)) set_up_configuration(silent=True) modtool = modules_tool() robot_path = build_option('robot_path') ec_path = det_easyconfig_paths([ec])[0] print("Found %s easyconfig file at %s" % (ec, ec_path)) parsed_ecs, _ = parse_easyconfigs([(ec_path, False)], validate=False) print("Resolving dependencies... ", end='') ecs = resolve_dependencies(parsed_ecs, modtool, retain_all_deps=True) print("found stack of %d easyconfigs" % len(ecs)) print("Filtering toolchain and its dependencies...") ec_tc = parsed_ecs[0]['ec']['toolchain'] ecs_to_remove = [{ 'name': ec_tc['name'], 'version': ec_tc['version'], 'toolchain': { 'name': SYSTEM_TOOLCHAIN_NAME } }] updated_ecs = {} # if GCCcore is used as toolchain, determine binutils version to use if tc_name == 'GCCcore': binutils_pattern = '^binutils.*-%s-%s.*.eb$' % (tc_name, tc_ver) _, res = search_file(robot_path, binutils_pattern) if res: if len(res) == 1: parsed_ecs, _ = parse_easyconfigs([(res[0], False)]) binutils_ec = parsed_ecs[0] tc = copy.copy(binutils_ec['ec']['toolchain']) ecs_to_remove.append({ 'name': 'binutils', 'version': binutils_ec['ec'].version, 'toolchain': tc }) else: error("Found more than one easyconfig matching '%s': %s" % (binutils_pattern, res)) else: error("No easyconfig file found for binutils using pattern '%s'" % binutils_pattern) while (ecs_to_remove): to_remove = ecs_to_remove.pop(0) print("Removing %(name)s/%(version)s (toolchain: %(toolchain)s)" % to_remove) for ec in ecs: if ec['ec'].name == to_remove['name'] and ec['ec'].version == to_remove['version'] and \ ec['ec']['toolchain']['name'] == to_remove['toolchain']['name']: ecs.remove(ec) ecs_to_remove.extend(dep for dep in ec['ec']['dependencies'] + ec['ec']['builddependencies']) updated_ecs[ec['full_mod_name']] = { 'builddependencies': [], 'dependencies': [], 'toolchain': copy.copy(ec['ec']['toolchain']), 'version': ec['ec'].version, } break ecs_to_write = [] for ec in ecs: ec_fn = os.path.basename(ec['spec']) print(term.bold("Determining version for %s..." % ec_fn)) full_mod_name = ec['full_mod_name'] ec_tc = copy.copy(ec['ec']['toolchain']) # update toolchain (unless it's SYSTEM) if ec_tc['name'] != SYSTEM_TOOLCHAIN_NAME: if ec_tc['name'] == tc_name: ec_tc['version'] = tc_ver else: error("Don't know how to update toolchain %s" % ec_tc['name']) # update (build) dependencies build_deps = [] for dep in ec['ec']['builddependencies']: new_dep_ver = updated_ecs[dep['full_mod_name']]['version'] build_deps.append((dep['name'], new_dep_ver)) deps = [] for dep in ec['ec']['dependencies']: new_dep_ver = updated_ecs[dep['full_mod_name']]['version'] deps.append((dep['name'], new_dep_ver)) # determine software version to use; # first, try searching for an existing easyconfig with specified toolchain; # if that fails, try to determine latest upstream version ec_pattern = '^%s.*-%s-%s.*.eb$' % (ec['ec'].name, tc_name, tc_ver) _, res = search_file(robot_path, ec_pattern) if res: if len(res) == 1: parsed_ecs, _ = parse_easyconfigs([(res[0], False)]) ec = parsed_ecs[0] new_version = ec['ec'].version print( term.green( "Found existing easyconfig, sticking to version %s" % new_version)) else: error("Multiple hits found using '%s': %s" % (res, ec_pattern)) else: new_version = update_version(ec['ec']) ecs_to_write.append(ec) if new_version is None: print( term.yellow( "No new version found for %s, using existing version" % full_mod_name)) new_version = ec['ec'].version updated_ecs[full_mod_name] = { 'builddependencies': build_deps, 'dependencies': deps, 'toolchain': ec_tc, 'version': new_version, } for ec in ecs_to_write: full_mod_name = ec['full_mod_name'] pprint.pprint(full_mod_name) ec = ec['ec'] ectxt = ec.rawtxt key_pattern = r'^%s\s*=.*' list_key_pattern = r'^%s\s*=\s*\[([^\]]|\n)*\s*\]' new_version = updated_ecs[full_mod_name]['version'] if ec.version != new_version: regex = re.compile(key_pattern % 'version', re.M) ectxt = regex.sub("version = '%s'" % new_version, ectxt) # if version got updated, also wipe the checksums regex = re.compile(list_key_pattern % 'checksums', re.M) ectxt = regex.sub("checksums = []", ectxt) # toolchain tc_str = "toolchain = {'name': '%(name)s', 'version': '%(version)s'}" % updated_ecs[ full_mod_name]['toolchain'] regex = re.compile(key_pattern % 'toolchain', re.M) ectxt = regex.sub(tc_str, ectxt) # dependencies for key in ('builddependencies', 'dependencies'): deps_str = '%s = [\n' % key for dep in updated_ecs[full_mod_name][key]: deps_str += ' ' + str(dep) + ',\n' deps_str += ']' regex = re.compile(list_key_pattern % key, re.M) ectxt = regex.sub(deps_str, ectxt) specs = { 'name': ec.name, 'toolchain': updated_ecs[full_mod_name]['toolchain'], 'version': new_version, 'versionsuffix': ec['versionsuffix'], } ec_fn = '%s-%s.eb' % (ec.name, det_full_ec_version(specs)) write_file(ec_fn, ectxt) print(term.green("%s written" % ec_fn))
if res: key, val = res.group('key'), res.group('val') return key, val # ================================================================================================= if len(sys.argv) < 2: error("Usage: %s <space-separated set of values for easyconfig parameters>" % os.path.basename(sys.argv[0])) specs = sys.argv[1:] cfg = {} set_up_configuration() known_module_classes = module_classes() tc_names = sorted([tc.NAME for tc in search_toolchain('')[1]], key=len, reverse=True) for spec in specs: info("Processing spec: %s" % spec) key = None key, val = parse_explicit_param(spec) if key: info("Found explicit parameter spec for '%s': %s" % (key, val)) spec = val if key in ['builddependencies', 'dependencies']:
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None): """ Main function: parse command line options, and act accordingly. :param args: command line arguments to use :param logfile: log file to use :param do_build: whether or not to actually perform the build :param testing: enable testing mode """ # purposely session state very early, to avoid modules loaded by EasyBuild meddling in init_session_state = session_state() eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing) options, orig_paths = eb_go.options, eb_go.args global _log (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings # load hook implementations (if any) hooks = load_hooks(options.hooks) run_hook(START, hooks) if modtool is None: modtool = modules_tool(testing=testing) # check whether any (EasyBuild-generated) modules are loaded already in the current session modtool.check_loaded_modules() if options.last_log: # print location to last log file, and exit last_log = find_last_log(logfile) or '(none)' print_msg(last_log, log=_log, prefix=False) # check whether packaging is supported when it's being used if options.package: check_pkg_support() else: _log.debug("Packaging not enabled, so not checking for packaging support.") # search for easyconfigs, if a query is specified if search_query: search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename, terse=options.terse) # GitHub options that warrant a silent cleanup & exit if options.check_github: check_github() elif options.install_github_token: install_github_token(options.github_user, silent=build_option('silent')) elif options.close_pr: close_pr(options.close_pr, motivation_msg=options.close_pr_msg) elif options.list_prs: print(list_prs(options.list_prs)) elif options.merge_pr: merge_pr(options.merge_pr) elif options.review_pr: print(review_pr(pr=options.review_pr, colored=use_color(options.color))) elif options.list_installed_software: detailed = options.list_installed_software == 'detailed' print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True)) elif options.list_software: print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')) # non-verbose cleanup after handling GitHub integration stuff or printing terse info early_stop_options = [ options.check_github, options.install_github_token, options.list_installed_software, options.list_software, options.close_pr, options.list_prs, options.merge_pr, options.review_pr, options.terse, search_query, ] if any(early_stop_options): clean_exit(logfile, eb_tmpdir, testing, silent=True) # update session state eb_config = eb_go.generate_cmd_line(add_default=True) modlist = modtool.list() # build options must be initialized first before 'module list' works init_session_state.update({'easybuild_configuration': eb_config}) init_session_state.update({'module_list': modlist}) _log.debug("Initial session state: %s" % init_session_state) # determine easybuild-easyconfigs package install path easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR) if not easyconfigs_pkg_paths: _log.warning("Failed to determine install path for easybuild-easyconfigs package.") if options.install_latest_eb_release: if orig_paths: raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing " "other easyconfigs") else: eb_file = find_easybuild_easyconfig() orig_paths.append(eb_file) categorized_paths = categorize_files_by_type(orig_paths) # command line options that do not require any easyconfigs to be specified new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr no_ec_opts = [options.aggregate_regtest, options.regtest, search_query, new_update_preview_pr] # determine paths to easyconfigs determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs']) if determined_paths: # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated paths = [(p, False) for p in determined_paths] else: if 'name' in build_specs: # try to obtain or generate an easyconfig file via build specifications if a software name is provided paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing) elif any(no_ec_opts): paths = determined_paths else: print_error(("Please provide one or multiple easyconfig files, or use software build " "options to make EasyBuild search for easyconfigs"), log=_log, opt_parser=eb_go.parser, exit_on_error=not testing) _log.debug("Paths: %s" % paths) # run regtest if options.regtest or options.aggregate_regtest: _log.info("Running regression test") # fallback: easybuild-easyconfigs install path regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool) if not regtest_ok: _log.info("Regression test failed (partially)!") sys.exit(31) # exit -> 3x1t -> 31 # read easyconfig files easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums) # handle --check-contrib & --check-style options if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style): clean_exit(logfile, eb_tmpdir, testing) # verify easyconfig filenames, if desired if options.verify_easyconfig_filenames: _log.info("Verifying easyconfig filenames...") for easyconfig in easyconfigs: verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec']) # tweak obtained easyconfig files, if requested # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail # if easyconfig files for the dependencies are not available if try_to_generate and build_specs and not generated_ecs: easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths) if options.containerize: # if --containerize/-C create a container recipe (and optionally container image), and stop containerize(easyconfigs) clean_exit(logfile, eb_tmpdir, testing) forced = options.force or options.rebuild dry_run_mode = options.dry_run or options.dry_run_short # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums): retained_ecs = skip_available(easyconfigs, modtool) if not testing: for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]: print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name']) easyconfigs = retained_ecs # determine an order that will allow all specs in the set to build if len(easyconfigs) > 0: # resolve dependencies if robot is enabled, except in dry run mode # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled if options.robot and (not dry_run_mode or new_update_preview_pr): print_msg("resolving dependencies ...", log=_log, silent=testing) ordered_ecs = resolve_dependencies(easyconfigs, modtool) else: ordered_ecs = easyconfigs elif new_update_preview_pr: ordered_ecs = None else: print_msg("No easyconfigs left to be built.", log=_log, silent=testing) ordered_ecs = [] # creating/updating PRs if new_update_preview_pr: if options.new_pr: new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg) elif options.preview_pr: print(review_pr(paths=determined_paths, colored=use_color(options.color))) else: update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg) # dry_run: print all easyconfigs and dependencies, and whether they are already built elif dry_run_mode: txt = dry_run(easyconfigs, modtool, short=not options.dry_run) print_msg(txt, log=_log, silent=testing, prefix=False) elif options.check_conflicts: if check_conflicts(easyconfigs, modtool): print_error("One or more conflicts detected!") sys.exit(1) else: print_msg("\nNo conflicts detected!\n", prefix=False) # dump source script to set up build environment elif options.dump_env_script: dump_env_script(easyconfigs) elif options.inject_checksums: inject_checksums(ordered_ecs, options.inject_checksums) # cleanup and exit after dry run, searching easyconfigs or submitting regression test stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums] if any(no_ec_opts) or any(stop_options): clean_exit(logfile, eb_tmpdir, testing) # create dependency graph and exit if options.dep_graph: _log.info("Creating dependency graph %s" % options.dep_graph) dep_graph(options.dep_graph, ordered_ecs) clean_exit(logfile, eb_tmpdir, testing, silent=True) # submit build as job(s), clean up and exit if options.job: submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing) if not testing: print_msg("Submitted parallel build jobs, exiting now") clean_exit(logfile, eb_tmpdir, testing) # build software, will exit when errors occurs (except when testing) if not testing or (testing and do_build): exit_on_failure = not (options.dump_test_report or options.upload_test_report) ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure) else: ecs_with_res = [(ec, {}) for ec in ordered_ecs] correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)]) overall_success = correct_builds_cnt == len(ordered_ecs) success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs)) repo = init_repository(get_repository(), get_repositorypath()) repo.cleanup() # dump/upload overall test report test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state) if test_report_msg is not None: print_msg(test_report_msg) print_msg(success_msg, log=_log, silent=testing) # cleanup and spec files for ec in easyconfigs: if 'original_spec' in ec and os.path.isfile(ec['spec']): os.remove(ec['spec']) run_hook(END, hooks) # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir) stop_logging(logfile, logtostdout=options.logtostdout) if overall_success: cleanup(logfile, eb_tmpdir, testing)
walk = os.walk join = os.path.join read = lambda ec_file: ec_file log.info('parsing easyconfigs from location %s' % options.path) else: fs = Githubfs(options.username, options.repo, options.branch) walk = Githubfs(options.username, options.repo, options.branch).walk join = fs.join read = lambda ec_file: fs.read(ec_file, api=False) log.info('parsing easyconfigs from user %s reponame %s' % (options.username, options.repo)) # configure EasyBuild, by parsing options set_up_configuration(args=args + ['--force']) configs = [] names = [] # fs.walk yields the same results as os.walk, so should be interchangable # same for fs.join and os.path.join for root, subfolders, files in walk(options.path): if '.git' in subfolders: log.info("found .git subfolder, ignoring it") subfolders.remove('.git') for ec_file in files: if not ec_file.endswith('.eb') or ec_file in ["TEMPLATE.eb"]: