Example #1
0
    def test_generaloption_overrides_legacy(self):
        """Test whether generaloption overrides legacy configuration."""
        self.purge_environment()
        # if both legacy and generaloption style configuration is mixed, generaloption wins
        legacy_prefix = os.path.join(self.tmpdir, 'legacy')
        go_prefix = os.path.join(self.tmpdir, 'generaloption')

        # legacy env vars
        os.environ['EASYBUILDPREFIX'] = legacy_prefix
        os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'build')
        # generaloption env vars
        os.environ['EASYBUILD_INSTALLPATH'] = go_prefix
        init_config()
        self.assertEqual(build_path(), os.path.join(legacy_prefix, 'build'))
        self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertEqual(repo.repo, os.path.join(legacy_prefix, 'ebfiles_repo'))
        del os.environ['EASYBUILDPREFIX']

        # legacy env vars
        os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'buildagain')
        # generaloption env vars
        os.environ['EASYBUILD_PREFIX'] = go_prefix
        init_config()
        self.assertEqual(build_path(), os.path.join(go_prefix, 'build'))
        self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertEqual(repo.repo, os.path.join(go_prefix, 'ebfiles_repo'))
        del os.environ['EASYBUILDBUILDPATH']
Example #2
0
    def test_generaloption_overrides_legacy(self):
        """Test whether generaloption overrides legacy configuration."""
        self.purge_environment()
        # if both legacy and generaloption style configuration is mixed, generaloption wins
        legacy_prefix = os.path.join(self.tmpdir, 'legacy')
        go_prefix = os.path.join(self.tmpdir, 'generaloption')

        # legacy env vars
        os.environ['EASYBUILDPREFIX'] = legacy_prefix
        os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'build')
        # generaloption env vars
        os.environ['EASYBUILD_INSTALLPATH'] = go_prefix
        init_config()
        self.assertEqual(build_path(), os.path.join(legacy_prefix, 'build'))
        self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertEqual(repo.repo, os.path.join(legacy_prefix,
                                                 'ebfiles_repo'))
        del os.environ['EASYBUILDPREFIX']

        # legacy env vars
        os.environ['EASYBUILDBUILDPATH'] = os.path.join(
            legacy_prefix, 'buildagain')
        # generaloption env vars
        os.environ['EASYBUILD_PREFIX'] = go_prefix
        init_config()
        self.assertEqual(build_path(), os.path.join(go_prefix, 'build'))
        self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertEqual(repo.repo, os.path.join(go_prefix, 'ebfiles_repo'))
        del os.environ['EASYBUILDBUILDPATH']
Example #3
0
def create_job(build_command,
               easyconfig,
               output_dir=None,
               conn=None,
               ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    @param build_command: format string for command, full path to an easyconfig file will be substituted in it
    @param easyconfig: easyconfig as processed by process_easyconfig
    @param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
    @param conn: open connection to PBS server
    @param ppn: ppn setting to use (# 'processors' (cores) per node to use)
    returns the job
    """
    if output_dir is None:
        output_dir = 'easybuild-build'

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" %
              easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'],
                det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    # create command based on build_command template
    command = build_command % {
        'spec': easyconfig['spec'],
        'output_dir': os.path.join(os.path.abspath(output_dir), name),
    }

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command,
                 name,
                 easybuild_vars,
                 resources=resources,
                 conn=conn,
                 ppn=ppn)
    job.module = easyconfig['ec'].full_mod_name

    return job
Example #4
0
def create_job(job_backend,
               build_command,
               easyconfig,
               output_dir='easybuild-build'):
    """
    Creates a job to build a *single* easyconfig.

    :param job_backend: A factory object for querying server parameters and creating actual job objects
    :param build_command: format string for command, full path to an easyconfig file will be substituted in it
    :param easyconfig: easyconfig as processed by process_easyconfig
    :param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable

    returns the job
    """
    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    for env_var in ["PYTHONPATH", "MODULEPATH"]:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" %
              easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'],
                det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    # determine whether additional options need to be passed to the 'eb' command
    add_opts = ''
    if easyconfig['hidden']:
        add_opts += ' --hidden'

    # create command based on build_command template
    command = build_command % {
        'add_opts': add_opts,
        'output_dir': os.path.join(os.path.abspath(output_dir), name),
        'spec': easyconfig['spec'],
    }

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    extra = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        extra['hours'] = int(math.ceil(previous_time * 2 / 60))

    if build_option('job_cores'):
        extra['cores'] = build_option('job_cores')

    job = job_backend.make_job(command, name, easybuild_vars, **extra)
    job.module = easyconfig['ec'].full_mod_name

    return job
Example #5
0
def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build'):
    """
    Creates a job to build a *single* easyconfig.

    :param job_backend: A factory object for querying server parameters and creating actual job objects
    :param build_command: format string for command, full path to an easyconfig file will be substituted in it
    :param easyconfig: easyconfig as processed by process_easyconfig
    :param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable

    returns the job
    """
    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    for env_var in ["PYTHONPATH", "MODULEPATH"]:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    # determine whether additional options need to be passed to the 'eb' command
    add_opts = ''
    if easyconfig['hidden']:
        add_opts += ' --hidden'

    # create command based on build_command template
    command = build_command % {
        'add_opts': add_opts,
        'output_dir': os.path.join(os.path.abspath(output_dir), name),
        'spec': easyconfig['spec'],
    }

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    extra = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        extra['hours'] = int(math.ceil(previous_time * 2 / 60))

    if build_option('job_cores'):
        extra['cores'] = build_option('job_cores')

    job = job_backend.make_job(command, name, easybuild_vars, **extra)
    job.module = easyconfig['ec'].full_mod_name

    return job
Example #6
0
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    build_command is a format string in which a full path to an eb file will be substituted
    easyconfig should be in the format as processEasyConfig returns them
    output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
    returns the job
    """
    # create command based on build_command template
    command = build_command % {'spec': easyconfig['spec']}

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" %
              easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'],
                det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    var = config.oldstyle_environment_variables['test_output_path']
    easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command,
                 name,
                 easybuild_vars,
                 resources=resources,
                 conn=conn,
                 ppn=ppn)
    job.module = det_full_module_name(easyconfig['ec'])

    return job
def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    @param build_command: format string for command, full path to an easyconfig file will be substituted in it
    @param easyconfig: easyconfig as processed by process_easyconfig
    @param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
    @param conn: open connection to PBS server
    @param ppn: ppn setting to use (# 'processors' (cores) per node to use)
    returns the job
    """
    if output_dir is None:
        output_dir = 'easybuild-build'

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    # create command based on build_command template
    command = build_command % {
        'spec': easyconfig['spec'],
        'output_dir': os.path.join(os.path.abspath(output_dir), name),
    }

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
    job.module = easyconfig['ec'].full_mod_name

    return job
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    build_command is a format string in which a full path to an eb file will be substituted
    easyconfig should be in the format as processEasyConfig returns them
    output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
    returns the job
    """
    # create command based on build_command template
    command = build_command % {'spec': easyconfig['spec']}

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)

    # obtain unique name based on name/easyconfig version tuple
    ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
    name = '-'.join(ec_tuple)

    var = config.oldstyle_environment_variables['test_output_path']
    easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    repo = init_repository(get_repository(), get_repositorypath())
    buildstats = repo.get_buildstats(*ec_tuple)
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]['build_time']
        resources['hours'] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
    job.module = det_full_module_name(easyconfig['ec'])

    return job
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
    """
    Creates a job, to build a *single* easyconfig
    build_command is a format string in which a full path to an eb file will be substituted
    easyconfig should be in the format as processEasyConfig returns them
    output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
    returns the job
    """
    # create command based on build_command template
    command = build_command % {"spec": easyconfig["spec"]}

    # capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
    easybuild_vars = {}
    for name in os.environ:
        if name.startswith("EASYBUILD"):
            easybuild_vars[name] = os.environ[name]

    others = ["PYTHONPATH", "MODULEPATH"]

    for env_var in others:
        if env_var in os.environ:
            easybuild_vars[env_var] = os.environ[env_var]

    _log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)

    # create unique name based on module name
    name = "%s-%s" % easyconfig["module"]

    var = config.oldstyle_environment_variables["test_output_path"]
    easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)

    # just use latest build stats
    buildstats = get_repository().get_buildstats(*easyconfig["module"])
    resources = {}
    if buildstats:
        previous_time = buildstats[-1]["build_time"]
        resources["hours"] = int(math.ceil(previous_time * 2 / 60))

    job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
    job.module = easyconfig["module"]

    return job
Example #10
0
def main(args=None, logfile=None, do_build=None, testing=False):
    """
    Main function: parse command line options, and act accordingly.
    @param args: command line arguments to use
    @param logfile: log file to use
    @param do_build: whether or not to actually perform the build
    @param testing: enable testing mode
    """
    # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
    init_session_state = session_state()

    # initialise options
    eb_go = eboptions.parse_options(args=args)
    options = eb_go.options
    orig_paths = eb_go.args

    # set umask (as early as possible)
    if options.umask is not None:
        new_umask = int(options.umask, 8)
        old_umask = os.umask(new_umask)

    # set by option parsers via set_tmpdir
    eb_tmpdir = tempfile.gettempdir()

    # initialise logging for main
    global _log
    _log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=testing or options.last_log)

    # disallow running EasyBuild as root
    if os.getuid() == 0:
        raise EasyBuildError(
            "You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here."
        )

    # log startup info
    eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
    log_start(eb_cmd_line, eb_tmpdir)

    if options.umask is not None:
        _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))

    # process software build specifications (if any), i.e.
    # software name/version, toolchain name/version, extra patches, ...
    (try_to_generate, build_specs) = process_software_build_specs(options)

    # determine robot path
    # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
    tweaked_ecs = try_to_generate and build_specs
    tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
    auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short
    robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot)
    _log.debug("Full robot path: %s" % robot_path)

    # configure & initialize build options
    config_options_dict = eb_go.get_options_by_section("config")
    build_options = {
        "build_specs": build_specs,
        "command_line": eb_cmd_line,
        "pr_path": pr_path,
        "robot_path": robot_path,
        "silent": testing,
        "try_to_generate": try_to_generate,
        "valid_stops": [x[0] for x in EasyBlock.get_steps()],
    }
    # initialise the EasyBuild configuration & build options
    config.init(options, config_options_dict)
    config.init_build_options(build_options=build_options, cmdline_options=options)

    if options.last_log:
        # print location to last log file, and exit
        last_log = find_last_log(logfile) or "(none)"
        print_msg(last_log, log=_log, prefix=False)
        cleanup(logfile, eb_tmpdir, testing, silent=True)
        sys.exit(0)

    # check whether packaging is supported when it's being used
    if options.package:
        check_pkg_support()
    else:
        _log.debug("Packaging not enabled, so not checking for packaging support.")

    # update session state
    eb_config = eb_go.generate_cmd_line(add_default=True)
    modlist = session_module_list(testing=testing)  # build options must be initialized first before 'module list' works
    init_session_state.update({"easybuild_configuration": eb_config})
    init_session_state.update({"module_list": modlist})
    _log.debug("Initial session state: %s" % init_session_state)

    # GitHub integration
    if options.review_pr or options.new_pr or options.update_pr:
        if options.review_pr:
            print review_pr(options.review_pr, colored=options.color)

        elif options.new_pr:
            new_pr(orig_paths, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg)

        elif options.update_pr:
            update_pr(options.update_pr, orig_paths, commit_msg=options.pr_commit_msg)

        cleanup(logfile, eb_tmpdir, testing)
        sys.exit(0)

    # search for easyconfigs, if a query is specified
    query = options.search or options.search_short
    if query:
        search_easyconfigs(query, short=not options.search)

    # determine easybuild-easyconfigs package install path
    easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
    if not easyconfigs_pkg_paths:
        _log.warning("Failed to determine install path for easybuild-easyconfigs package.")

    # command line options that do not require any easyconfigs to be specified
    no_ec_opts = [
        options.aggregate_regtest,
        options.new_pr,
        options.review_pr,
        options.search,
        options.search_short,
        options.regtest,
        options.update_pr,
    ]

    # determine paths to easyconfigs
    paths = det_easyconfig_paths(orig_paths)
    if paths:
        # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
        paths = [(p, False) for p in paths]
    else:
        if "name" in build_specs:
            # try to obtain or generate an easyconfig file via build specifications if a software name is provided
            paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
        elif not any(no_ec_opts):
            print_error(
                (
                    "Please provide one or multiple easyconfig files, or use software build "
                    "options to make EasyBuild search for easyconfigs"
                ),
                log=_log,
                opt_parser=eb_go.parser,
                exit_on_error=not testing,
            )
    _log.debug("Paths: %s" % paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        _log.info("Running regression test")
        # fallback: easybuild-easyconfigs install path
        regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths)
        if not regtest_ok:
            _log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    # read easyconfig files
    easyconfigs, generated_ecs = parse_easyconfigs(paths)

    # tweak obtained easyconfig files, if requested
    # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
    # if easyconfig files for the dependencies are not available
    if try_to_generate and build_specs and not generated_ecs:
        easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path)

    # dry_run: print all easyconfigs and dependencies, and whether they are already built
    if options.dry_run or options.dry_run_short:
        txt = dry_run(easyconfigs, short=not options.dry_run)
        print_msg(txt, log=_log, silent=testing, prefix=False)

    # cleanup and exit after dry run, searching easyconfigs or submitting regression test
    if any(no_ec_opts + [options.dry_run, options.dry_run_short]):
        cleanup(logfile, eb_tmpdir, testing)
        sys.exit(0)

    # skip modules that are already installed unless forced
    if not (options.force or options.rebuild or options.extended_dry_run):
        retained_ecs = skip_available(easyconfigs)
        if not testing:
            for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
                print_msg("%s is already installed (module found), skipping" % skipped_ec["full_mod_name"])
        easyconfigs = retained_ecs

    # determine an order that will allow all specs in the set to build
    if len(easyconfigs) > 0:
        if options.robot:
            print_msg("resolving dependencies ...", log=_log, silent=testing)
            ordered_ecs = resolve_dependencies(
                easyconfigs,
                minimal_toolchains=build_option("minimal_toolchains"),
                use_existing_modules=build_option("use_existing_modules"),
            )
        else:
            ordered_ecs = easyconfigs
    else:
        print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
        ordered_ecs = []

    # create dependency graph and exit
    if options.dep_graph:
        _log.info("Creating dependency graph %s" % options.dep_graph)
        dep_graph(options.dep_graph, ordered_ecs)
        sys.exit(0)

    # submit build as job(s), clean up and exit
    if options.job:
        submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
        if not testing:
            print_msg("Submitted parallel build jobs, exiting now")
            cleanup(logfile, eb_tmpdir, testing)
            sys.exit(0)

    # build software, will exit when errors occurs (except when testing)
    exit_on_failure = not options.dump_test_report and not options.upload_test_report
    if not testing or (testing and do_build):
        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
    else:
        ecs_with_res = [(ec, {}) for ec in ordered_ecs]

    correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get("success", False)])
    overall_success = correct_builds_cnt == len(ordered_ecs)
    success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # dump/upload overall test report
    test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
    if test_report_msg is not None:
        print_msg(test_report_msg)

    print_msg(success_msg, log=_log, silent=testing)

    # cleanup and spec files
    for ec in easyconfigs:
        if "original_spec" in ec and os.path.isfile(ec["spec"]):
            os.remove(ec["spec"])

    # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
    stop_logging(logfile, logtostdout=options.logtostdout)
    if overall_success:
        cleanup(logfile, eb_tmpdir, testing)
Example #11
0
            sys.exit(0)

    # build software, will exit when errors occurs (except when regtesting)
    correct_built_cnt = 0
    all_built_cnt = 0
    if not testing or (testing and do_build):
        for ec in ordered_ecs:
            (success, _) = build_and_install_software(ec, orig_environ)
            if success:
                correct_built_cnt += 1
            all_built_cnt += 1

    print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # cleanup and spec files
    for ec in easyconfigs:
        if 'original_spec' in ec and os.path.isfile(ec['spec']):
            os.remove(ec['spec'])

    # cleanup tmp log file (all is well, all modules have their own log file)
    if options.logtostdout:
        fancylogger.logToScreen(enable=False, stdout=True)
    else:
        fancylogger.logToFile(logfile, enable=False)
    cleanup(logfile, eb_tmpdir, testing)

Example #12
0
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
    """
    Main function: parse command line options, and act accordingly.
    :param args: command line arguments to use
    :param logfile: log file to use
    :param do_build: whether or not to actually perform the build
    :param testing: enable testing mode
    """

    register_lock_cleanup_signal_handlers()

    # if $CDPATH is set, unset it, it'll only cause trouble...
    # see https://github.com/easybuilders/easybuild-framework/issues/2944
    if 'CDPATH' in os.environ:
        del os.environ['CDPATH']

    # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
    init_session_state = session_state()

    eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
    options, orig_paths = eb_go.options, eb_go.args

    global _log
    (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings

    # load hook implementations (if any)
    hooks = load_hooks(options.hooks)

    run_hook(START, hooks)

    if modtool is None:
        modtool = modules_tool(testing=testing)

    # check whether any (EasyBuild-generated) modules are loaded already in the current session
    modtool.check_loaded_modules()

    if options.last_log:
        # print location to last log file, and exit
        last_log = find_last_log(logfile) or '(none)'
        print_msg(last_log, log=_log, prefix=False)

    # check whether packaging is supported when it's being used
    if options.package:
        check_pkg_support()
    else:
        _log.debug("Packaging not enabled, so not checking for packaging support.")

    # search for easyconfigs, if a query is specified
    if search_query:
        search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
                           terse=options.terse)

    # GitHub options that warrant a silent cleanup & exit
    if options.check_github:
        check_github()

    elif options.install_github_token:
        install_github_token(options.github_user, silent=build_option('silent'))

    elif options.close_pr:
        close_pr(options.close_pr, motivation_msg=options.close_pr_msg)

    elif options.list_prs:
        print(list_prs(options.list_prs))

    elif options.merge_pr:
        merge_pr(options.merge_pr)

    elif options.review_pr:
        print(review_pr(pr=options.review_pr, colored=use_color(options.color)))

    elif options.list_installed_software:
        detailed = options.list_installed_software == 'detailed'
        print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True))

    elif options.list_software:
        print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed'))

    elif options.create_index:
        print_msg("Creating index for %s..." % options.create_index, prefix=False)
        index_fp = dump_index(options.create_index, max_age_sec=options.index_max_age)
        index = load_index(options.create_index)
        print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False)

    # non-verbose cleanup after handling GitHub integration stuff or printing terse info
    early_stop_options = [
        options.check_github,
        options.create_index,
        options.install_github_token,
        options.list_installed_software,
        options.list_software,
        options.close_pr,
        options.list_prs,
        options.merge_pr,
        options.review_pr,
        options.terse,
        search_query,
    ]
    if any(early_stop_options):
        clean_exit(logfile, eb_tmpdir, testing, silent=True)

    # update session state
    eb_config = eb_go.generate_cmd_line(add_default=True)
    modlist = modtool.list()  # build options must be initialized first before 'module list' works
    init_session_state.update({'easybuild_configuration': eb_config})
    init_session_state.update({'module_list': modlist})
    _log.debug("Initial session state: %s" % init_session_state)

    # determine easybuild-easyconfigs package install path
    easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
    if not easyconfigs_pkg_paths:
        _log.warning("Failed to determine install path for easybuild-easyconfigs package.")

    if options.install_latest_eb_release:
        if orig_paths:
            raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
                                 "other easyconfigs")
        else:
            eb_file = find_easybuild_easyconfig()
            orig_paths.append(eb_file)

    if len(orig_paths) == 1:
        # if only one easyconfig file is specified, use current directory as target directory
        target_path = os.getcwd()
    elif orig_paths:
        # last path is target when --copy-ec is used, so remove that from the list
        target_path = orig_paths.pop() if options.copy_ec else None

    categorized_paths = categorize_files_by_type(orig_paths)

    # command line options that do not require any easyconfigs to be specified
    pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr
    pr_options = pr_options or options.sync_branch_with_develop or options.sync_pr_with_develop
    pr_options = pr_options or options.update_branch_github or options.update_pr
    no_ec_opts = [options.aggregate_regtest, options.regtest, pr_options, search_query]

    # determine paths to easyconfigs
    determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])

    if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec:

        if options.copy_ec:
            if len(determined_paths) == 1:
                copy_file(determined_paths[0], target_path)
                print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=False)
            elif len(determined_paths) > 1:
                copy_files(determined_paths, target_path)
                print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False)
            else:
                raise EasyBuildError("One of more files to copy should be specified!")

        elif options.fix_deprecated_easyconfigs:
            fix_deprecated_easyconfigs(determined_paths)

        elif options.show_ec:
            for path in determined_paths:
                print_msg("Contents of %s:" % path)
                print_msg(read_file(path), prefix=False)

        clean_exit(logfile, eb_tmpdir, testing)

    if determined_paths:
        # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
        paths = [(p, False) for p in determined_paths]
    elif 'name' in build_specs:
        # try to obtain or generate an easyconfig file via build specifications if a software name is provided
        paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
    elif any(no_ec_opts):
        paths = determined_paths
    else:
        print_error("Please provide one or multiple easyconfig files, or use software build " +
                    "options to make EasyBuild search for easyconfigs",
                    log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
    _log.debug("Paths: %s", paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        _log.info("Running regression test")
        # fallback: easybuild-easyconfigs install path
        regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool)
        if not regtest_ok:
            _log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    # read easyconfig files
    easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums)

    # handle --check-contrib & --check-style options
    if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style):
        clean_exit(logfile, eb_tmpdir, testing)

    # verify easyconfig filenames, if desired
    if options.verify_easyconfig_filenames:
        _log.info("Verifying easyconfig filenames...")
        for easyconfig in easyconfigs:
            verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec'])

    # tweak obtained easyconfig files, if requested
    # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
    # if easyconfig files for the dependencies are not available
    if try_to_generate and build_specs and not generated_ecs:
        easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)

    if options.containerize:
        # if --containerize/-C create a container recipe (and optionally container image), and stop
        containerize(easyconfigs)
        clean_exit(logfile, eb_tmpdir, testing)

    forced = options.force or options.rebuild
    dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules

    # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
    if not (forced or dry_run_mode or options.extended_dry_run or pr_options or options.inject_checksums):
        retained_ecs = skip_available(easyconfigs, modtool)
        if not testing:
            for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
                print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
        easyconfigs = retained_ecs

    # keep track for which easyconfigs we should set the corresponding module as default
    if options.set_default_module:
        for easyconfig in easyconfigs:
            easyconfig['ec'].set_default_module = True

    # determine an order that will allow all specs in the set to build
    if len(easyconfigs) > 0:
        # resolve dependencies if robot is enabled, except in dry run mode
        # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
        if options.robot and (not dry_run_mode or pr_options):
            print_msg("resolving dependencies ...", log=_log, silent=testing)
            ordered_ecs = resolve_dependencies(easyconfigs, modtool)
        else:
            ordered_ecs = easyconfigs
    elif pr_options:
        ordered_ecs = None
    else:
        print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
        ordered_ecs = []

    if options.copy_ec and tweaked_ecs_paths:
        all_specs = [spec['spec'] for spec in
                     resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False)]
        tweaked_ecs_in_all_ecs = [path for path in all_specs if
                                  any(tweaked_ecs_path in path for tweaked_ecs_path in tweaked_ecs_paths)]
        if tweaked_ecs_in_all_ecs:
            # Clean them, then copy them
            clean_up_easyconfigs(tweaked_ecs_in_all_ecs)
            copy_files(tweaked_ecs_in_all_ecs, target_path)
            print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False)

    # creating/updating PRs
    if pr_options:
        if options.new_pr:
            new_pr(categorized_paths, ordered_ecs)
        elif options.new_branch_github:
            new_branch_github(categorized_paths, ordered_ecs)
        elif options.new_pr_from_branch:
            new_pr_from_branch(options.new_pr_from_branch)
        elif options.preview_pr:
            print(review_pr(paths=determined_paths, colored=use_color(options.color)))
        elif options.sync_branch_with_develop:
            sync_branch_with_develop(options.sync_branch_with_develop)
        elif options.sync_pr_with_develop:
            sync_pr_with_develop(options.sync_pr_with_develop)
        elif options.update_branch_github:
            update_branch(options.update_branch_github, categorized_paths, ordered_ecs)
        elif options.update_pr:
            update_pr(options.update_pr, categorized_paths, ordered_ecs)
        else:
            raise EasyBuildError("Unknown PR option!")

    # dry_run: print all easyconfigs and dependencies, and whether they are already built
    elif dry_run_mode:
        if options.missing_modules:
            txt = missing_deps(easyconfigs, modtool)
        else:
            txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
        print_msg(txt, log=_log, silent=testing, prefix=False)

    elif options.check_conflicts:
        if check_conflicts(easyconfigs, modtool):
            print_error("One or more conflicts detected!")
            sys.exit(1)
        else:
            print_msg("\nNo conflicts detected!\n", prefix=False)

    # dump source script to set up build environment
    elif options.dump_env_script:
        dump_env_script(easyconfigs)

    elif options.inject_checksums:
        inject_checksums(ordered_ecs, options.inject_checksums)

    # cleanup and exit after dry run, searching easyconfigs or submitting regression test
    stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]
    if any(no_ec_opts) or any(stop_options):
        clean_exit(logfile, eb_tmpdir, testing)

    # create dependency graph and exit
    if options.dep_graph:
        _log.info("Creating dependency graph %s" % options.dep_graph)
        dep_graph(options.dep_graph, ordered_ecs)
        clean_exit(logfile, eb_tmpdir, testing, silent=True)

    # submit build as job(s), clean up and exit
    if options.job:
        submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
        if not testing:
            print_msg("Submitted parallel build jobs, exiting now")
            clean_exit(logfile, eb_tmpdir, testing)

    # build software, will exit when errors occurs (except when testing)
    if not testing or (testing and do_build):
        exit_on_failure = not (options.dump_test_report or options.upload_test_report)

        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
    else:
        ecs_with_res = [(ec, {}) for ec in ordered_ecs]

    correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
    overall_success = correct_builds_cnt == len(ordered_ecs)
    success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # dump/upload overall test report
    test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
    if test_report_msg is not None:
        print_msg(test_report_msg)

    print_msg(success_msg, log=_log, silent=testing)

    # cleanup and spec files
    for ec in easyconfigs:
        if 'original_spec' in ec and os.path.isfile(ec['spec']):
            os.remove(ec['spec'])

    run_hook(END, hooks)

    # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
    stop_logging(logfile, logtostdout=options.logtostdout)
    if overall_success:
        cleanup(logfile, eb_tmpdir, testing)
Example #13
0
    def test_legacy_config_file(self):
        """Test finding/using legacy configuration files."""
        self.purge_environment()

        cfg_fn = self.configure(args=[])
        self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py'))

        configtxt = """
build_path = '%(buildpath)s'
source_path = '%(sourcepath)s'
install_path = '%(installpath)s'
repository_path = '%(repopath)s'
repository = FileRepository(repository_path)
log_format = ('%(logdir)s', '%(logtmpl)s')
log_dir = '%(tmplogdir)s'
software_install_suffix = '%(softsuffix)s'
modules_install_suffix = '%(modsuffix)s'
"""

        buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path')
        sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path')
        installpath = os.path.join(self.tmpdir, 'my', 'test', 'install', 'path')
        repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path')
        logdir = 'somedir'
        logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log'
        tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir')
        softsuffix = 'myfavoritesoftware'
        modsuffix = 'modulesgohere'

        configdict = {
            'buildpath': buildpath,
            'sourcepath': sourcepath,
            'installpath': installpath,
            'repopath': repopath,
            'logdir': logdir,
            'logtmpl': logtmpl,
            'tmplogdir': tmplogdir,
            'softsuffix': softsuffix,
            'modsuffix': modsuffix
        }

        # create user config file on default location
        myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py')
        if not os.path.exists(os.path.dirname(myconfigfile)):
            os.makedirs(os.path.dirname(myconfigfile))
        write_file(myconfigfile, configtxt % configdict)

        # redefine home so we can test user config file on default location
        home = os.environ.get('HOME', None)
        os.environ['HOME'] = self.tmpdir
        init_config()
        cfg_fn = self.configure(args=[])
        if home is not None:
            os.environ['HOME'] = home

        # check finding and use of config file
        self.assertEqual(cfg_fn, myconfigfile)
        self.assertEqual(build_path(), buildpath)
        self.assertEqual(source_paths()[0], sourcepath)
        self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
        self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, repopath)
        self.assertEqual(log_file_format(return_directory=True), logdir)
        self.assertEqual(log_file_format(), logtmpl)
        self.assertEqual(get_build_log_path(), tmplogdir)

        # redefine config file entries for proper testing below
        buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build', 'path')
        sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'source', 'path')
        installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'install', 'path')
        repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo', 'path')
        logdir = 'somedir_custom'
        logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log'
        tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'tmplogdir')
        softsuffix = 'myfavoritesoftware_custom'
        modsuffix = 'modulesgohere_custom'

        configdict = {
            'buildpath': buildpath,
            'sourcepath': sourcepath,
            'installpath': installpath,
            'repopath': repopath,
            'logdir': logdir,
            'logtmpl': logtmpl,
            'tmplogdir': tmplogdir,
            'softsuffix': softsuffix,
            'modsuffix': modsuffix }

        # create custom config file, and point to it
        mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py')
        if not os.path.exists(os.path.dirname(mycustomconfigfile)):
            os.makedirs(os.path.dirname(mycustomconfigfile))
        write_file(mycustomconfigfile, configtxt % configdict)
        os.environ['EASYBUILDCONFIG'] = mycustomconfigfile

        # reconfigure
        init_config()
        cfg_fn = self.configure(args=[])

        # verify configuration
        self.assertEqual(cfg_fn, mycustomconfigfile)
        self.assertEqual(build_path(), buildpath)
        self.assertEqual(source_paths()[0], sourcepath)
        self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
        self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, repopath)
        self.assertEqual(log_file_format(return_directory=True), logdir)
        self.assertEqual(log_file_format(), logtmpl)
        self.assertEqual(get_build_log_path(), tmplogdir)
Example #14
0
    def test_legacy_env_vars(self):
        """Test legacy environment variables."""
        self.purge_environment()

        # build path
        test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
        os.environ['EASYBUILDBUILDPATH'] = test_buildpath
        self.configure(args=[])
        self.assertEqual(build_path(), test_buildpath)
        del os.environ['EASYBUILDBUILDPATH']

        # source path(s)
        test_sourcepaths = [
            os.path.join(self.tmpdir, 'source', 'path'),
            ':'.join([
                os.path.join(self.tmpdir, 'source', 'path1'),
                os.path.join(self.tmpdir, 'source', 'path2'),
            ]),
            ':'.join([
                os.path.join(self.tmpdir, 'source', 'path1'),
                os.path.join(self.tmpdir, 'source', 'path2'),
                os.path.join(self.tmpdir, 'source', 'path3'),
            ]),
        ]
        for test_sourcepath in test_sourcepaths:
            init_config()
            os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
            self.configure(args=[])
            self.assertEqual(build_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
                                                        DEFAULT_PATH_SUBDIRS['buildpath']))
            self.assertEqual(source_paths(), test_sourcepath.split(':'))
            del os.environ['EASYBUILDSOURCEPATH']

        test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')

        # install path
        init_config()
        test_installpath = os.path.join(self.tmpdir, 'install', 'path')
        os.environ['EASYBUILDINSTALLPATH'] = test_installpath
        self.configure(args=[])
        self.assertEqual(source_paths()[0], os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
                                                          DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
                                                                 DEFAULT_PATH_SUBDIRS['subdir_modules']))
        del os.environ['EASYBUILDINSTALLPATH']

        # prefix: should change build/install/source/repo paths
        init_config()
        test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
        os.environ['EASYBUILDPREFIX'] = test_prefixpath
        self.configure(args=[])
        self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
                                                               DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))

        # build/source/install path overrides prefix
        init_config()
        os.environ['EASYBUILDBUILDPATH'] = test_buildpath
        self.configure(args=[])
        self.assertEqual(build_path(), test_buildpath)
        self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
                                                               DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
        del os.environ['EASYBUILDBUILDPATH']

        init_config()
        os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
        self.configure(args=[])
        self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_paths()[0], test_sourcepath)
        self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
                                                               DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
        del os.environ['EASYBUILDSOURCEPATH']

        init_config()
        os.environ['EASYBUILDINSTALLPATH'] = test_installpath
        self.configure(args=[])
        self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
                                                               DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
Example #15
0
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
    """
    Main function: parse command line options, and act accordingly.
    :param args: command line arguments to use
    :param logfile: log file to use
    :param do_build: whether or not to actually perform the build
    :param testing: enable testing mode
    """
    # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
    init_session_state = session_state()

    # initialise options
    eb_go = eboptions.parse_options(args=args)
    options = eb_go.options
    orig_paths = eb_go.args

    # set umask (as early as possible)
    if options.umask is not None:
        new_umask = int(options.umask, 8)
        old_umask = os.umask(new_umask)

    # set by option parsers via set_tmpdir
    eb_tmpdir = tempfile.gettempdir()

    search_query = options.search or options.search_filename or options.search_short

    # initialise logging for main
    global _log
    _log, logfile = init_logging(logfile, logtostdout=options.logtostdout,
                                 silent=(testing or options.terse or search_query), colorize=options.color)

    # disallow running EasyBuild as root
    if os.getuid() == 0:
        raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, "
                             "so let's end this here.")

    # log startup info
    eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
    log_start(eb_cmd_line, eb_tmpdir)

    if options.umask is not None:
        _log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))

    # process software build specifications (if any), i.e.
    # software name/version, toolchain name/version, extra patches, ...
    (try_to_generate, build_specs) = process_software_build_specs(options)

    # determine robot path
    # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
    tweaked_ecs = try_to_generate and build_specs
    tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
    auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
    robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot)
    _log.debug("Full robot path: %s" % robot_path)

    # configure & initialize build options
    config_options_dict = eb_go.get_options_by_section('config')
    build_options = {
        'build_specs': build_specs,
        'command_line': eb_cmd_line,
        'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata),
        'pr_path': pr_path,
        'robot_path': robot_path,
        'silent': testing,
        'try_to_generate': try_to_generate,
        'valid_stops': [x[0] for x in EasyBlock.get_steps()],
    }
    # initialise the EasyBuild configuration & build options
    config.init(options, config_options_dict)
    config.init_build_options(build_options=build_options, cmdline_options=options)

    if modtool is None:
        modtool = modules_tool(testing=testing)

    if options.last_log:
        # print location to last log file, and exit
        last_log = find_last_log(logfile) or '(none)'
        print_msg(last_log, log=_log, prefix=False)

    # check whether packaging is supported when it's being used
    if options.package:
        check_pkg_support()
    else:
        _log.debug("Packaging not enabled, so not checking for packaging support.")

    # search for easyconfigs, if a query is specified
    if search_query:
        search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
                           terse=options.terse)

    # GitHub options that warrant a silent cleanup & exit
    if options.check_github:
        check_github()

    elif options.install_github_token:
        install_github_token(options.github_user, silent=build_option('silent'))

    elif options.review_pr:
        print review_pr(options.review_pr, colored=use_color(options.color))

    elif options.list_installed_software:
        detailed = options.list_installed_software == 'detailed'
        print list_software(output_format=options.output_format, detailed=detailed, only_installed=True)

    elif options.list_software:
        print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')

    # non-verbose cleanup after handling GitHub integration stuff or printing terse info
    early_stop_options = [
        options.check_github,
        options.install_github_token,
        options.list_installed_software,
        options.list_software,
        options.review_pr,
        options.terse,
        search_query,
    ]
    if any(early_stop_options):
        cleanup(logfile, eb_tmpdir, testing, silent=True)
        sys.exit(0)

    # update session state
    eb_config = eb_go.generate_cmd_line(add_default=True)
    modlist = modtool.list()  # build options must be initialized first before 'module list' works
    init_session_state.update({'easybuild_configuration': eb_config})
    init_session_state.update({'module_list': modlist})
    _log.debug("Initial session state: %s" % init_session_state)

    # determine easybuild-easyconfigs package install path
    easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
    if not easyconfigs_pkg_paths:
        _log.warning("Failed to determine install path for easybuild-easyconfigs package.")

    if options.install_latest_eb_release:
        if orig_paths:
            raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
                                 "other easyconfigs")
        else:
            eb_file = find_easybuild_easyconfig()
            orig_paths.append(eb_file)

    categorized_paths = categorize_files_by_type(orig_paths)

    # command line options that do not require any easyconfigs to be specified
    no_ec_opts = [options.aggregate_regtest, options.new_pr, options.regtest, options.update_pr, search_query]

    # determine paths to easyconfigs
    paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
    if paths:
        # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
        paths = [(p, False) for p in paths]
    else:
        if 'name' in build_specs:
            # try to obtain or generate an easyconfig file via build specifications if a software name is provided
            paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
        elif not any(no_ec_opts):
            print_error(("Please provide one or multiple easyconfig files, or use software build "
                         "options to make EasyBuild search for easyconfigs"),
                        log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
    _log.debug("Paths: %s" % paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        _log.info("Running regression test")
        # fallback: easybuild-easyconfigs install path
        regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool)
        if not regtest_ok:
            _log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    if options.check_style:
        _log.debug("Running style check...")
        if cmdline_easyconfigs_style_check([path[0] for path in paths]):
            print_msg("All style checks passed!", prefix=False)
            cleanup(logfile, eb_tmpdir, testing)
            sys.exit(0)
        else:
            raise EasyBuildError("One or more style checks FAILED!")

    # read easyconfig files
    easyconfigs, generated_ecs = parse_easyconfigs(paths)

    # tweak obtained easyconfig files, if requested
    # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
    # if easyconfig files for the dependencies are not available
    if try_to_generate and build_specs and not generated_ecs:
        easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)

    dry_run_mode = options.dry_run or options.dry_run_short
    new_update_pr = options.new_pr or options.update_pr

    # skip modules that are already installed unless forced
    if not (options.force or options.rebuild or dry_run_mode or options.extended_dry_run or new_update_pr):
        retained_ecs = skip_available(easyconfigs, modtool)
        if not testing:
            for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
                print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
        easyconfigs = retained_ecs

    # determine an order that will allow all specs in the set to build
    if len(easyconfigs) > 0:
        # resolve dependencies if robot is enabled, except in dry run mode
        # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
        if options.robot and (not dry_run_mode or new_update_pr):
            print_msg("resolving dependencies ...", log=_log, silent=testing)
            ordered_ecs = resolve_dependencies(easyconfigs, modtool)
        else:
            ordered_ecs = easyconfigs
    elif new_update_pr:
        ordered_ecs = None
    else:
        print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
        ordered_ecs = []

    # creating/updating PRs
    if new_update_pr:
        if options.new_pr:
            new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr,
                   commit_msg=options.pr_commit_msg)
        else:
            update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg)

        cleanup(logfile, eb_tmpdir, testing, silent=True)
        sys.exit(0)

    # dry_run: print all easyconfigs and dependencies, and whether they are already built
    elif dry_run_mode:
        txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
        print_msg(txt, log=_log, silent=testing, prefix=False)

    elif options.check_conflicts:
        if check_conflicts(easyconfigs, modtool):
            print_error("One or more conflicts detected!")
            sys.exit(1)
        else:
            print_msg("\nNo conflicts detected!\n", prefix=False)

    # dump source script to set up build environment
    elif options.dump_env_script:
        dump_env_script(easyconfigs)

    # cleanup and exit after dry run, searching easyconfigs or submitting regression test
    if any(no_ec_opts + [options.check_conflicts, dry_run_mode, options.dump_env_script]):
        cleanup(logfile, eb_tmpdir, testing)
        sys.exit(0)

    # create dependency graph and exit
    if options.dep_graph:
        _log.info("Creating dependency graph %s" % options.dep_graph)
        dep_graph(options.dep_graph, ordered_ecs)
        cleanup(logfile, eb_tmpdir, testing, silent=True)
        sys.exit(0)

    # submit build as job(s), clean up and exit
    if options.job:
        submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
        if not testing:
            print_msg("Submitted parallel build jobs, exiting now")
            cleanup(logfile, eb_tmpdir, testing)
            sys.exit(0)

    # build software, will exit when errors occurs (except when testing)
    exit_on_failure = not options.dump_test_report and not options.upload_test_report
    if not testing or (testing and do_build):
        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
    else:
        ecs_with_res = [(ec, {}) for ec in ordered_ecs]

    correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
    overall_success = correct_builds_cnt == len(ordered_ecs)
    success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # dump/upload overall test report
    test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
    if test_report_msg is not None:
        print_msg(test_report_msg)

    print_msg(success_msg, log=_log, silent=testing)

    # cleanup and spec files
    for ec in easyconfigs:
        if 'original_spec' in ec and os.path.isfile(ec['spec']):
            os.remove(ec['spec'])

    # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
    stop_logging(logfile, logtostdout=options.logtostdout)
    if overall_success:
        cleanup(logfile, eb_tmpdir, testing)
Example #16
0
    def test_legacy_env_vars(self):
        """Test legacy environment variables."""

        # build path
        test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
        os.environ['EASYBUILDBUILDPATH'] = test_buildpath
        self.configure(args=[])
        self.assertEqual(build_path(), test_buildpath)
        del os.environ['EASYBUILDBUILDPATH']

        # source path
        config.variables = ConfigurationVariables()
        test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')
        os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
        self.configure(args=[])
        self.assertEqual(build_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
                                                    DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_path(), test_sourcepath)
        del os.environ['EASYBUILDSOURCEPATH']

        # install path
        config.variables = ConfigurationVariables()
        test_installpath = os.path.join(self.tmpdir, 'install', 'path')
        os.environ['EASYBUILDINSTALLPATH'] = test_installpath
        self.configure(args=[])
        self.assertEqual(source_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
                                                     DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
                                                                 DEFAULT_PATH_SUBDIRS['subdir_modules']))
        del os.environ['EASYBUILDINSTALLPATH']

        # prefix: should change build/install/source/repo paths
        config.variables = ConfigurationVariables()
        test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
        os.environ['EASYBUILDPREFIX'] = test_prefixpath
        self.configure(args=[])
        self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
                                                               DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = get_repository()
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))

        # build/source/install path overrides prefix
        config.variables = ConfigurationVariables()
        os.environ['EASYBUILDBUILDPATH'] = test_buildpath
        self.configure(args=[])
        self.assertEqual(build_path(), test_buildpath)
        self.assertEqual(source_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
                                                               DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = get_repository()
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
        # also check old style vs new style
        self.assertEqual(config.variables['build_path'], config.variables['buildpath'])
        self.assertEqual(config.variables['install_path'], config.variables['installpath'])
        del os.environ['EASYBUILDBUILDPATH']

        config.variables = ConfigurationVariables()
        os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
        self.configure(args=[])
        self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_path(), test_sourcepath)
        self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
                                                               DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = get_repository()
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
        del os.environ['EASYBUILDSOURCEPATH']

        config.variables = ConfigurationVariables()
        os.environ['EASYBUILDINSTALLPATH'] = test_installpath
        self.configure(args=[])
        self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
                                                               DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = get_repository()
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
        del os.environ['EASYBUILDINSTALLPATH']

        del os.environ['EASYBUILDPREFIX']
Example #17
0
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
    """
    Main function: parse command line options, and act accordingly.
    :param args: command line arguments to use
    :param logfile: log file to use
    :param do_build: whether or not to actually perform the build
    :param testing: enable testing mode
    """
    # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
    init_session_state = session_state()

    eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
    options, orig_paths = eb_go.options, eb_go.args

    global _log
    (build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings

    # load hook implementations (if any)
    hooks = load_hooks(options.hooks)

    run_hook(START, hooks)

    if modtool is None:
        modtool = modules_tool(testing=testing)

    # check whether any (EasyBuild-generated) modules are loaded already in the current session
    modtool.check_loaded_modules()

    if options.last_log:
        # print location to last log file, and exit
        last_log = find_last_log(logfile) or '(none)'
        print_msg(last_log, log=_log, prefix=False)

    # check whether packaging is supported when it's being used
    if options.package:
        check_pkg_support()
    else:
        _log.debug("Packaging not enabled, so not checking for packaging support.")

    # search for easyconfigs, if a query is specified
    if search_query:
        search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
                           terse=options.terse)

    # GitHub options that warrant a silent cleanup & exit
    if options.check_github:
        check_github()

    elif options.install_github_token:
        install_github_token(options.github_user, silent=build_option('silent'))

    elif options.close_pr:
        close_pr(options.close_pr, motivation_msg=options.close_pr_msg)

    elif options.list_prs:
        print(list_prs(options.list_prs))

    elif options.merge_pr:
        merge_pr(options.merge_pr)

    elif options.review_pr:
        print(review_pr(pr=options.review_pr, colored=use_color(options.color)))

    elif options.list_installed_software:
        detailed = options.list_installed_software == 'detailed'
        print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True))

    elif options.list_software:
        print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed'))

    # non-verbose cleanup after handling GitHub integration stuff or printing terse info
    early_stop_options = [
        options.check_github,
        options.install_github_token,
        options.list_installed_software,
        options.list_software,
        options.close_pr,
        options.list_prs,
        options.merge_pr,
        options.review_pr,
        options.terse,
        search_query,
    ]
    if any(early_stop_options):
        clean_exit(logfile, eb_tmpdir, testing, silent=True)

    # update session state
    eb_config = eb_go.generate_cmd_line(add_default=True)
    modlist = modtool.list()  # build options must be initialized first before 'module list' works
    init_session_state.update({'easybuild_configuration': eb_config})
    init_session_state.update({'module_list': modlist})
    _log.debug("Initial session state: %s" % init_session_state)

    # determine easybuild-easyconfigs package install path
    easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
    if not easyconfigs_pkg_paths:
        _log.warning("Failed to determine install path for easybuild-easyconfigs package.")

    if options.install_latest_eb_release:
        if orig_paths:
            raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
                                 "other easyconfigs")
        else:
            eb_file = find_easybuild_easyconfig()
            orig_paths.append(eb_file)

    categorized_paths = categorize_files_by_type(orig_paths)

    # command line options that do not require any easyconfigs to be specified
    new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr
    no_ec_opts = [options.aggregate_regtest, options.regtest, search_query, new_update_preview_pr]

    # determine paths to easyconfigs
    determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
    if determined_paths:
        # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
        paths = [(p, False) for p in determined_paths]
    else:
        if 'name' in build_specs:
            # try to obtain or generate an easyconfig file via build specifications if a software name is provided
            paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
        elif any(no_ec_opts):
            paths = determined_paths
        else:
            print_error(("Please provide one or multiple easyconfig files, or use software build "
                         "options to make EasyBuild search for easyconfigs"),
                        log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
    _log.debug("Paths: %s" % paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        _log.info("Running regression test")
        # fallback: easybuild-easyconfigs install path
        regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool)
        if not regtest_ok:
            _log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    # read easyconfig files
    easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums)

    # handle --check-contrib & --check-style options
    if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style):
        clean_exit(logfile, eb_tmpdir, testing)

    # verify easyconfig filenames, if desired
    if options.verify_easyconfig_filenames:
        _log.info("Verifying easyconfig filenames...")
        for easyconfig in easyconfigs:
            verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec'])

    # tweak obtained easyconfig files, if requested
    # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
    # if easyconfig files for the dependencies are not available
    if try_to_generate and build_specs and not generated_ecs:
        easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)

    if options.containerize:
        # if --containerize/-C create a container recipe (and optionally container image), and stop
        containerize(easyconfigs)
        clean_exit(logfile, eb_tmpdir, testing)

    forced = options.force or options.rebuild
    dry_run_mode = options.dry_run or options.dry_run_short

    # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
    if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums):
        retained_ecs = skip_available(easyconfigs, modtool)
        if not testing:
            for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
                print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
        easyconfigs = retained_ecs

    # determine an order that will allow all specs in the set to build
    if len(easyconfigs) > 0:
        # resolve dependencies if robot is enabled, except in dry run mode
        # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
        if options.robot and (not dry_run_mode or new_update_preview_pr):
            print_msg("resolving dependencies ...", log=_log, silent=testing)
            ordered_ecs = resolve_dependencies(easyconfigs, modtool)
        else:
            ordered_ecs = easyconfigs
    elif new_update_preview_pr:
        ordered_ecs = None
    else:
        print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
        ordered_ecs = []

    # creating/updating PRs
    if new_update_preview_pr:
        if options.new_pr:
            new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr,
                   commit_msg=options.pr_commit_msg)
        elif options.preview_pr:
            print(review_pr(paths=determined_paths, colored=use_color(options.color)))
        else:
            update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg)

    # dry_run: print all easyconfigs and dependencies, and whether they are already built
    elif dry_run_mode:
        txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
        print_msg(txt, log=_log, silent=testing, prefix=False)

    elif options.check_conflicts:
        if check_conflicts(easyconfigs, modtool):
            print_error("One or more conflicts detected!")
            sys.exit(1)
        else:
            print_msg("\nNo conflicts detected!\n", prefix=False)

    # dump source script to set up build environment
    elif options.dump_env_script:
        dump_env_script(easyconfigs)

    elif options.inject_checksums:
        inject_checksums(ordered_ecs, options.inject_checksums)

    # cleanup and exit after dry run, searching easyconfigs or submitting regression test
    stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]
    if any(no_ec_opts) or any(stop_options):
        clean_exit(logfile, eb_tmpdir, testing)

    # create dependency graph and exit
    if options.dep_graph:
        _log.info("Creating dependency graph %s" % options.dep_graph)
        dep_graph(options.dep_graph, ordered_ecs)
        clean_exit(logfile, eb_tmpdir, testing, silent=True)

    # submit build as job(s), clean up and exit
    if options.job:
        submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
        if not testing:
            print_msg("Submitted parallel build jobs, exiting now")
            clean_exit(logfile, eb_tmpdir, testing)

    # build software, will exit when errors occurs (except when testing)
    if not testing or (testing and do_build):
        exit_on_failure = not (options.dump_test_report or options.upload_test_report)

        ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
    else:
        ecs_with_res = [(ec, {}) for ec in ordered_ecs]

    correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
    overall_success = correct_builds_cnt == len(ordered_ecs)
    success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # dump/upload overall test report
    test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
    if test_report_msg is not None:
        print_msg(test_report_msg)

    print_msg(success_msg, log=_log, silent=testing)

    # cleanup and spec files
    for ec in easyconfigs:
        if 'original_spec' in ec and os.path.isfile(ec['spec']):
            os.remove(ec['spec'])

    run_hook(END, hooks)

    # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
    stop_logging(logfile, logtostdout=options.logtostdout)
    if overall_success:
        cleanup(logfile, eb_tmpdir, testing)
Example #18
0
    def test_legacy_env_vars(self):
        """Test legacy environment variables."""
        self.purge_environment()

        # build path
        test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
        os.environ['EASYBUILDBUILDPATH'] = test_buildpath
        self.configure(args=[])
        self.assertEqual(build_path(), test_buildpath)
        del os.environ['EASYBUILDBUILDPATH']

        # source path(s)
        test_sourcepaths = [
            os.path.join(self.tmpdir, 'source', 'path'),
            ':'.join([
                os.path.join(self.tmpdir, 'source', 'path1'),
                os.path.join(self.tmpdir, 'source', 'path2'),
            ]),
            ':'.join([
                os.path.join(self.tmpdir, 'source', 'path1'),
                os.path.join(self.tmpdir, 'source', 'path2'),
                os.path.join(self.tmpdir, 'source', 'path3'),
            ]),
        ]
        for test_sourcepath in test_sourcepaths:
            init_config()
            os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
            self.configure(args=[])
            self.assertEqual(
                build_path(),
                os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
                             DEFAULT_PATH_SUBDIRS['buildpath']))
            self.assertEqual(source_paths(), test_sourcepath.split(':'))
            del os.environ['EASYBUILDSOURCEPATH']

        test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')

        # install path
        init_config()
        test_installpath = os.path.join(self.tmpdir, 'install', 'path')
        os.environ['EASYBUILDINSTALLPATH'] = test_installpath
        self.configure(args=[])
        self.assertEqual(
            source_paths()[0],
            os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
                         DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(
            install_path(),
            os.path.join(test_installpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_installpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        del os.environ['EASYBUILDINSTALLPATH']

        # prefix: should change build/install/source/repo paths
        init_config()
        test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
        os.environ['EASYBUILDPREFIX'] = test_prefixpath
        self.configure(args=[])
        self.assertEqual(
            build_path(),
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(
            source_paths()[0],
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(
            install_path(),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(
            repo.repo,
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['repositorypath']))

        # build/source/install path overrides prefix
        init_config()
        os.environ['EASYBUILDBUILDPATH'] = test_buildpath
        self.configure(args=[])
        self.assertEqual(build_path(), test_buildpath)
        self.assertEqual(
            source_paths()[0],
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(
            install_path(),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(
            repo.repo,
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['repositorypath']))
        del os.environ['EASYBUILDBUILDPATH']

        init_config()
        os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
        self.configure(args=[])
        self.assertEqual(
            build_path(),
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_paths()[0], test_sourcepath)
        self.assertEqual(
            install_path(),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(
            repo.repo,
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['repositorypath']))
        del os.environ['EASYBUILDSOURCEPATH']

        init_config()
        os.environ['EASYBUILDINSTALLPATH'] = test_installpath
        self.configure(args=[])
        self.assertEqual(
            build_path(),
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(
            source_paths()[0],
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(
            install_path(),
            os.path.join(test_installpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_installpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(
            repo.repo,
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['repositorypath']))
Example #19
0
    def test_legacy_config_file(self):
        """Test finding/using legacy configuration files."""
        self.purge_environment()

        cfg_fn = self.configure(args=[])
        self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py'))

        configtxt = """
build_path = '%(buildpath)s'
source_path = '%(sourcepath)s'
install_path = '%(installpath)s'
repository_path = '%(repopath)s'
repository = FileRepository(repository_path)
log_format = ('%(logdir)s', '%(logtmpl)s')
log_dir = '%(tmplogdir)s'
software_install_suffix = '%(softsuffix)s'
modules_install_suffix = '%(modsuffix)s'
"""

        buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path')
        sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path')
        installpath = os.path.join(self.tmpdir, 'my', 'test', 'install',
                                   'path')
        repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path')
        logdir = 'somedir'
        logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log'
        tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir')
        softsuffix = 'myfavoritesoftware'
        modsuffix = 'modulesgohere'

        configdict = {
            'buildpath': buildpath,
            'sourcepath': sourcepath,
            'installpath': installpath,
            'repopath': repopath,
            'logdir': logdir,
            'logtmpl': logtmpl,
            'tmplogdir': tmplogdir,
            'softsuffix': softsuffix,
            'modsuffix': modsuffix
        }

        # create user config file on default location
        myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py')
        if not os.path.exists(os.path.dirname(myconfigfile)):
            os.makedirs(os.path.dirname(myconfigfile))
        write_file(myconfigfile, configtxt % configdict)

        # redefine home so we can test user config file on default location
        home = os.environ.get('HOME', None)
        os.environ['HOME'] = self.tmpdir
        init_config()
        cfg_fn = self.configure(args=[])
        if home is not None:
            os.environ['HOME'] = home

        # check finding and use of config file
        self.assertEqual(cfg_fn, myconfigfile)
        self.assertEqual(build_path(), buildpath)
        self.assertEqual(source_paths()[0], sourcepath)
        self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
        self.assertEqual(install_path(typ='mod'),
                         os.path.join(installpath, modsuffix))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, repopath)
        self.assertEqual(log_file_format(return_directory=True), logdir)
        self.assertEqual(log_file_format(), logtmpl)
        self.assertEqual(get_build_log_path(), tmplogdir)

        # redefine config file entries for proper testing below
        buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build',
                                 'path')
        sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test',
                                  'source', 'path')
        installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test',
                                   'install', 'path')
        repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo',
                                'path')
        logdir = 'somedir_custom'
        logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log'
        tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test',
                                 'tmplogdir')
        softsuffix = 'myfavoritesoftware_custom'
        modsuffix = 'modulesgohere_custom'

        configdict = {
            'buildpath': buildpath,
            'sourcepath': sourcepath,
            'installpath': installpath,
            'repopath': repopath,
            'logdir': logdir,
            'logtmpl': logtmpl,
            'tmplogdir': tmplogdir,
            'softsuffix': softsuffix,
            'modsuffix': modsuffix
        }

        # create custom config file, and point to it
        mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py')
        if not os.path.exists(os.path.dirname(mycustomconfigfile)):
            os.makedirs(os.path.dirname(mycustomconfigfile))
        write_file(mycustomconfigfile, configtxt % configdict)
        os.environ['EASYBUILDCONFIG'] = mycustomconfigfile

        # reconfigure
        init_config()
        cfg_fn = self.configure(args=[])

        # verify configuration
        self.assertEqual(cfg_fn, mycustomconfigfile)
        self.assertEqual(build_path(), buildpath)
        self.assertEqual(source_paths()[0], sourcepath)
        self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
        self.assertEqual(install_path(typ='mod'),
                         os.path.join(installpath, modsuffix))
        repo = init_repository(get_repository(), get_repositorypath())
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(repo.repo, repopath)
        self.assertEqual(log_file_format(return_directory=True), logdir)
        self.assertEqual(log_file_format(), logtmpl)
        self.assertEqual(get_build_log_path(), tmplogdir)
Example #20
0
    def test_legacy_env_vars(self):
        """Test legacy environment variables."""

        # build path
        test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
        os.environ['EASYBUILDBUILDPATH'] = test_buildpath
        self.configure(args=[])
        self.assertEqual(build_path(), test_buildpath)
        del os.environ['EASYBUILDBUILDPATH']

        # source path
        config.variables = ConfigurationVariables()
        test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')
        os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
        self.configure(args=[])
        self.assertEqual(
            build_path(),
            os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
                         DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_path(), test_sourcepath)
        del os.environ['EASYBUILDSOURCEPATH']

        # install path
        config.variables = ConfigurationVariables()
        test_installpath = os.path.join(self.tmpdir, 'install', 'path')
        os.environ['EASYBUILDINSTALLPATH'] = test_installpath
        self.configure(args=[])
        self.assertEqual(
            source_path(),
            os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
                         DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(
            install_path(),
            os.path.join(test_installpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_installpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        del os.environ['EASYBUILDINSTALLPATH']

        # prefix: should change build/install/source/repo paths
        config.variables = ConfigurationVariables()
        test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
        os.environ['EASYBUILDPREFIX'] = test_prefixpath
        self.configure(args=[])
        self.assertEqual(
            build_path(),
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(
            source_path(),
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(
            install_path(),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = get_repository()
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(
            repo.repo,
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['repositorypath']))

        # build/source/install path overrides prefix
        config.variables = ConfigurationVariables()
        os.environ['EASYBUILDBUILDPATH'] = test_buildpath
        self.configure(args=[])
        self.assertEqual(build_path(), test_buildpath)
        self.assertEqual(
            source_path(),
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(
            install_path(),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = get_repository()
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(
            repo.repo,
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['repositorypath']))
        # also check old style vs new style
        self.assertEqual(config.variables['build_path'],
                         config.variables['buildpath'])
        self.assertEqual(config.variables['install_path'],
                         config.variables['installpath'])
        del os.environ['EASYBUILDBUILDPATH']

        config.variables = ConfigurationVariables()
        os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
        self.configure(args=[])
        self.assertEqual(
            build_path(),
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(source_path(), test_sourcepath)
        self.assertEqual(
            install_path(),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = get_repository()
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(
            repo.repo,
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['repositorypath']))
        del os.environ['EASYBUILDSOURCEPATH']

        config.variables = ConfigurationVariables()
        os.environ['EASYBUILDINSTALLPATH'] = test_installpath
        self.configure(args=[])
        self.assertEqual(
            build_path(),
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
        self.assertEqual(
            source_path(),
            os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
        self.assertEqual(
            install_path(),
            os.path.join(test_installpath,
                         DEFAULT_PATH_SUBDIRS['subdir_software']))
        self.assertEqual(
            install_path(typ='mod'),
            os.path.join(test_installpath,
                         DEFAULT_PATH_SUBDIRS['subdir_modules']))
        repo = get_repository()
        self.assertTrue(isinstance(repo, FileRepository))
        self.assertEqual(
            repo.repo,
            os.path.join(test_prefixpath,
                         DEFAULT_PATH_SUBDIRS['repositorypath']))
        del os.environ['EASYBUILDINSTALLPATH']

        del os.environ['EASYBUILDPREFIX']
Example #21
0
            sys.exit(0)

    # build software, will exit when errors occurs (except when regtesting)
    correct_built_cnt = 0
    all_built_cnt = 0
    if not testing:
        for spec in orderedSpecs:
            (success, _) = build_and_install_software(spec, options, origEnviron, silent=testing)
            if success:
                correct_built_cnt += 1
            all_built_cnt += 1

    print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)

    get_repository().cleanup()

    # cleanup and spec files
    for ec in easyconfigs:
        if 'originalSpec' in ec and os.path.isfile(ec['spec']):
            os.remove(ec['spec'])

    # cleanup tmp log file (all is well, all modules have their own log file)
    if options.logtostdout:
        fancylogger.logToScreen(enable=False, stdout=True)
    else:
        fancylogger.logToFile(logfile, enable=False)
        cleanup_logfile_and_exit(logfile, testing, False)
        logfile = None

    return logfile
Example #22
0
    # build software, will exit when errors occurs (except when testing)
    exit_on_failure = not options.dump_test_report and not options.upload_test_report
    if not testing or (testing and do_build):
        ecs_with_res = build_and_install_software(
            ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
    else:
        ecs_with_res = [(ec, {}) for ec in ordered_ecs]

    correct_builds_cnt = len([
        ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)
    ])
    overall_success = correct_builds_cnt == len(ordered_ecs)
    success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt,
                                                        len(ordered_ecs))

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # report back in PR in case of testing
    if options.upload_test_report:
        msg = success_msg + " (%d easyconfigs in this PR)" % len(paths)
        test_report = create_test_report(msg,
                                         ecs_with_res,
                                         init_session_state,
                                         pr_nr=options.from_pr,
                                         gist_log=True)
        if options.from_pr:
            # upload test report to gist and issue a comment in the PR to notify
            msg = post_easyconfigs_pr_test_report(options.from_pr, test_report,
                                                  success_msg,
                                                  init_session_state,
Example #23
0
def main(testing_data=(None, None, None)):
    """
    Main function: parse command line options, and act accordingly.
    @param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build
    """
    # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
    init_session_state = session_state()

    # steer behavior when testing main
    testing = testing_data[0] is not None
    args, logfile, do_build = testing_data

    # initialise options
    eb_go = eboptions.parse_options(args=args)
    options = eb_go.options
    orig_paths = eb_go.args

    # set umask (as early as possible)
    if options.umask is not None:
        new_umask = int(options.umask, 8)
        old_umask = os.umask(new_umask)

    # set temporary directory to use
    eb_tmpdir = set_tmpdir(options.tmpdir)

    # initialise logging for main
    global _log
    _log, logfile = init_logging(logfile,
                                 logtostdout=options.logtostdout,
                                 testing=testing)

    # disallow running EasyBuild as root
    if os.getuid() == 0:
        raise EasyBuildError(
            "You seem to be running EasyBuild with root privileges which is not wise, "
            "so let's end this here.")

    # log startup info
    eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
    log_start(eb_cmd_line, eb_tmpdir)

    if options.umask is not None:
        _log.info("umask set to '%s' (used to be '%s')" %
                  (oct(new_umask), oct(old_umask)))

    # process software build specifications (if any), i.e.
    # software name/version, toolchain name/version, extra patches, ...
    (try_to_generate, build_specs) = process_software_build_specs(options)

    # determine robot path
    # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
    tweaked_ecs = try_to_generate and build_specs
    tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir,
                                                     tweaked_ecs=tweaked_ecs,
                                                     from_pr=options.from_pr)
    auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short
    robot_path = det_robot_path(options.robot_paths,
                                tweaked_ecs_path,
                                pr_path,
                                auto_robot=auto_robot)
    _log.debug("Full robot path: %s" % robot_path)

    # configure & initialize build options
    config_options_dict = eb_go.get_options_by_section('config')
    build_options = {
        'build_specs': build_specs,
        'command_line': eb_cmd_line,
        'pr_path': pr_path,
        'robot_path': robot_path,
        'silent': testing,
        'try_to_generate': try_to_generate,
        'valid_stops': [x[0] for x in EasyBlock.get_steps()],
    }
    # initialise the EasyBuild configuration & build options
    config.init(options, config_options_dict)
    config.init_build_options(build_options=build_options,
                              cmdline_options=options)

    # update session state
    eb_config = eb_go.generate_cmd_line(add_default=True)
    modlist = session_module_list(
        testing=testing
    )  # build options must be initialized first before 'module list' works
    init_session_state.update({'easybuild_configuration': eb_config})
    init_session_state.update({'module_list': modlist})
    _log.debug("Initial session state: %s" % init_session_state)

    # search for easyconfigs, if a query is specified
    query = options.search or options.search_short
    if query:
        search_easyconfigs(query, short=not options.search)

    # determine easybuild-easyconfigs package install path
    easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
    if not easyconfigs_pkg_paths:
        _log.warning(
            "Failed to determine install path for easybuild-easyconfigs package."
        )

    # determine paths to easyconfigs
    paths = det_easyconfig_paths(orig_paths)
    if paths:
        # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
        paths = [(p, False) for p in paths]
    else:
        if 'name' in build_specs:
            # try to obtain or generate an easyconfig file via build specifications if a software name is provided
            paths = find_easyconfigs_by_specs(build_specs,
                                              robot_path,
                                              try_to_generate,
                                              testing=testing)
        elif not any([
                options.aggregate_regtest, options.search,
                options.search_short, options.regtest
        ]):
            print_error((
                "Please provide one or multiple easyconfig files, or use software build "
                "options to make EasyBuild search for easyconfigs"),
                        log=_log,
                        opt_parser=eb_go.parser,
                        exit_on_error=not testing)
    _log.debug("Paths: %s" % paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        _log.info("Running regression test")
        # fallback: easybuild-easyconfigs install path
        regtest_ok = regtest([path[0] for path in paths]
                             or easyconfigs_pkg_paths)
        if not regtest_ok:
            _log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    # read easyconfig files
    easyconfigs, generated_ecs = parse_easyconfigs(paths)

    # tweak obtained easyconfig files, if requested
    # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
    # if easyconfig files for the dependencies are not available
    if try_to_generate and build_specs and not generated_ecs:
        easyconfigs = tweak(easyconfigs,
                            build_specs,
                            targetdir=tweaked_ecs_path)

    # dry_run: print all easyconfigs and dependencies, and whether they are already built
    if options.dry_run or options.dry_run_short:
        txt = dry_run(easyconfigs,
                      short=not options.dry_run,
                      build_specs=build_specs)
        print_msg(txt, log=_log, silent=testing, prefix=False)

    # cleanup and exit after dry run, searching easyconfigs or submitting regression test
    if any([
            options.dry_run, options.dry_run_short, options.regtest,
            options.search, options.search_short
    ]):
        cleanup(logfile, eb_tmpdir, testing)
        sys.exit(0)

    # skip modules that are already installed unless forced
    if not options.force:
        retained_ecs = skip_available(easyconfigs)
        if not testing:
            for skipped_ec in [
                    ec for ec in easyconfigs if ec not in retained_ecs
            ]:
                print_msg("%s is already installed (module found), skipping" %
                          skipped_ec['full_mod_name'])
        easyconfigs = retained_ecs

    # determine an order that will allow all specs in the set to build
    if len(easyconfigs) > 0:
        if options.robot:
            print_msg("resolving dependencies ...", log=_log, silent=testing)
            ordered_ecs = resolve_dependencies(easyconfigs,
                                               build_specs=build_specs)
        else:
            ordered_ecs = easyconfigs
    else:
        print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
        ordered_ecs = []

    # create dependency graph and exit
    if options.dep_graph:
        _log.info("Creating dependency graph %s" % options.dep_graph)
        dep_graph(options.dep_graph, ordered_ecs)
        sys.exit(0)

    # submit build as job(s), clean up and exit
    if options.job:
        job_info_txt = submit_jobs(ordered_ecs,
                                   eb_go.generate_cmd_line(),
                                   testing=testing)
        if not testing:
            print_msg("Submitted parallel build jobs, exiting now: %s" %
                      job_info_txt)
            cleanup(logfile, eb_tmpdir, testing)
            sys.exit(0)

    # build software, will exit when errors occurs (except when testing)
    exit_on_failure = not options.dump_test_report and not options.upload_test_report
    if not testing or (testing and do_build):
        ecs_with_res = build_and_install_software(
            ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
    else:
        ecs_with_res = [(ec, {}) for ec in ordered_ecs]

    correct_builds_cnt = len([
        ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)
    ])
    overall_success = correct_builds_cnt == len(ordered_ecs)
    success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt,
                                                        len(ordered_ecs))

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # dump/upload overall test report
    test_report_msg = overall_test_report(ecs_with_res, len(paths),
                                          overall_success, success_msg,
                                          init_session_state)
    if test_report_msg is not None:
        print_msg(test_report_msg)

    print_msg(success_msg, log=_log, silent=testing)

    # cleanup and spec files
    for ec in easyconfigs:
        if 'original_spec' in ec and os.path.isfile(ec['spec']):
            os.remove(ec['spec'])

    # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path)
    stop_logging(logfile, logtostdout=options.logtostdout)
    if overall_success:
        cleanup(logfile, eb_tmpdir, testing)
Example #24
0
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
    """
    Main function: parse command line options, and act accordingly.
    :param args: command line arguments to use
    :param logfile: log file to use
    :param do_build: whether or not to actually perform the build
    :param testing: enable testing mode
    """
    # purposely session state very early, to avoid modules loaded by EasyBuild meddling in
    init_session_state = session_state()

    # initialise options
    eb_go = eboptions.parse_options(args=args)
    options = eb_go.options
    orig_paths = eb_go.args

    # set umask (as early as possible)
    if options.umask is not None:
        new_umask = int(options.umask, 8)
        old_umask = os.umask(new_umask)

    # set by option parsers via set_tmpdir
    eb_tmpdir = tempfile.gettempdir()

    search_query = options.search or options.search_filename or options.search_short

    # initialise logging for main
    global _log
    _log, logfile = init_logging(logfile,
                                 logtostdout=options.logtostdout,
                                 silent=(testing or options.terse
                                         or search_query),
                                 colorize=options.color)

    # disallow running EasyBuild as root (by default)
    check_root_usage(
        allow_use_as_root=options.allow_use_as_root_and_accept_consequences)

    # log startup info
    eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
    log_start(eb_cmd_line, eb_tmpdir)

    if options.umask is not None:
        _log.info("umask set to '%s' (used to be '%s')" %
                  (oct(new_umask), oct(old_umask)))

    # process software build specifications (if any), i.e.
    # software name/version, toolchain name/version, extra patches, ...
    (try_to_generate, build_specs) = process_software_build_specs(options)

    # determine robot path
    # --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
    tweaked_ecs = try_to_generate and build_specs
    tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir,
                                                      tweaked_ecs=tweaked_ecs,
                                                      from_pr=options.from_pr)
    auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
    robot_path = det_robot_path(options.robot_paths,
                                tweaked_ecs_paths,
                                pr_path,
                                auto_robot=auto_robot)
    _log.debug("Full robot path: %s" % robot_path)

    # configure & initialize build options
    config_options_dict = eb_go.get_options_by_section('config')
    build_options = {
        'build_specs':
        build_specs,
        'command_line':
        eb_cmd_line,
        'external_modules_metadata':
        parse_external_modules_metadata(options.external_modules_metadata),
        'pr_path':
        pr_path,
        'robot_path':
        robot_path,
        'silent':
        testing,
        'try_to_generate':
        try_to_generate,
        'valid_stops': [x[0] for x in EasyBlock.get_steps()],
    }
    # initialise the EasyBuild configuration & build options
    config.init(options, config_options_dict)
    config.init_build_options(build_options=build_options,
                              cmdline_options=options)

    if modtool is None:
        modtool = modules_tool(testing=testing)

    # check whether any (EasyBuild-generated) modules are loaded already in the current session
    modtool.check_loaded_modules()

    if options.last_log:
        # print location to last log file, and exit
        last_log = find_last_log(logfile) or '(none)'
        print_msg(last_log, log=_log, prefix=False)

    # check whether packaging is supported when it's being used
    if options.package:
        check_pkg_support()
    else:
        _log.debug(
            "Packaging not enabled, so not checking for packaging support.")

    # search for easyconfigs, if a query is specified
    if search_query:
        search_easyconfigs(search_query,
                           short=options.search_short,
                           filename_only=options.search_filename,
                           terse=options.terse)

    # GitHub options that warrant a silent cleanup & exit
    if options.check_github:
        check_github()

    elif options.install_github_token:
        install_github_token(options.github_user,
                             silent=build_option('silent'))

    elif options.merge_pr:
        merge_pr(options.merge_pr)

    elif options.review_pr:
        print review_pr(pr=options.review_pr, colored=use_color(options.color))

    elif options.list_installed_software:
        detailed = options.list_installed_software == 'detailed'
        print list_software(output_format=options.output_format,
                            detailed=detailed,
                            only_installed=True)

    elif options.list_software:
        print list_software(output_format=options.output_format,
                            detailed=options.list_software == 'detailed')

    # non-verbose cleanup after handling GitHub integration stuff or printing terse info
    early_stop_options = [
        options.check_github,
        options.install_github_token,
        options.list_installed_software,
        options.list_software,
        options.merge_pr,
        options.review_pr,
        options.terse,
        search_query,
    ]
    if any(early_stop_options):
        cleanup(logfile, eb_tmpdir, testing, silent=True)
        sys.exit(0)

    # update session state
    eb_config = eb_go.generate_cmd_line(add_default=True)
    modlist = modtool.list(
    )  # build options must be initialized first before 'module list' works
    init_session_state.update({'easybuild_configuration': eb_config})
    init_session_state.update({'module_list': modlist})
    _log.debug("Initial session state: %s" % init_session_state)

    # determine easybuild-easyconfigs package install path
    easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
    if not easyconfigs_pkg_paths:
        _log.warning(
            "Failed to determine install path for easybuild-easyconfigs package."
        )

    if options.install_latest_eb_release:
        if orig_paths:
            raise EasyBuildError(
                "Installing the latest EasyBuild release can not be combined with installing "
                "other easyconfigs")
        else:
            eb_file = find_easybuild_easyconfig()
            orig_paths.append(eb_file)

    categorized_paths = categorize_files_by_type(orig_paths)

    # command line options that do not require any easyconfigs to be specified
    new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr
    no_ec_opts = [
        options.aggregate_regtest, options.regtest, search_query,
        new_update_preview_pr
    ]

    # determine paths to easyconfigs
    determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
    if determined_paths:
        # transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
        paths = [(p, False) for p in determined_paths]
    else:
        if 'name' in build_specs:
            # try to obtain or generate an easyconfig file via build specifications if a software name is provided
            paths = find_easyconfigs_by_specs(build_specs,
                                              robot_path,
                                              try_to_generate,
                                              testing=testing)
        elif any(no_ec_opts):
            paths = determined_paths
        else:
            print_error((
                "Please provide one or multiple easyconfig files, or use software build "
                "options to make EasyBuild search for easyconfigs"),
                        log=_log,
                        opt_parser=eb_go.parser,
                        exit_on_error=not testing)
    _log.debug("Paths: %s" % paths)

    # run regtest
    if options.regtest or options.aggregate_regtest:
        _log.info("Running regression test")
        # fallback: easybuild-easyconfigs install path
        regtest_ok = regtest([path[0] for path in paths]
                             or easyconfigs_pkg_paths, modtool)
        if not regtest_ok:
            _log.info("Regression test failed (partially)!")
            sys.exit(31)  # exit -> 3x1t -> 31

    if options.check_style:
        _log.debug("Running style check...")
        if cmdline_easyconfigs_style_check([path[0] for path in paths]):
            print_msg("All style checks passed!", prefix=False)
            cleanup(logfile, eb_tmpdir, testing)
            sys.exit(0)
        else:
            raise EasyBuildError("One or more style checks FAILED!")

    # read easyconfig files
    easyconfigs, generated_ecs = parse_easyconfigs(
        paths, validate=not options.inject_checksums)

    # verify easyconfig filenames, if desired
    if options.verify_easyconfig_filenames:
        _log.info("Verifying easyconfig filenames...")
        for easyconfig in easyconfigs:
            verify_easyconfig_filename(easyconfig['spec'],
                                       easyconfig['ec'],
                                       parsed_ec=easyconfig['ec'])

    # tweak obtained easyconfig files, if requested
    # don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
    # if easyconfig files for the dependencies are not available
    if try_to_generate and build_specs and not generated_ecs:
        easyconfigs = tweak(easyconfigs,
                            build_specs,
                            modtool,
                            targetdirs=tweaked_ecs_paths)

    if options.containerize:
        containerize(easyconfigs)
        cleanup(logfile, eb_tmpdir, testing)
        sys.exit(0)

    forced = options.force or options.rebuild
    dry_run_mode = options.dry_run or options.dry_run_short

    # skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
    if not (forced or dry_run_mode or options.extended_dry_run
            or new_update_preview_pr or options.inject_checksums):
        retained_ecs = skip_available(easyconfigs, modtool)
        if not testing:
            for skipped_ec in [
                    ec for ec in easyconfigs if ec not in retained_ecs
            ]:
                print_msg("%s is already installed (module found), skipping" %
                          skipped_ec['full_mod_name'])
        easyconfigs = retained_ecs

    # determine an order that will allow all specs in the set to build
    if len(easyconfigs) > 0:
        # resolve dependencies if robot is enabled, except in dry run mode
        # one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
        if options.robot and (not dry_run_mode or new_update_preview_pr):
            print_msg("resolving dependencies ...", log=_log, silent=testing)
            ordered_ecs = resolve_dependencies(easyconfigs, modtool)
        else:
            ordered_ecs = easyconfigs
    elif new_update_preview_pr:
        ordered_ecs = None
    else:
        print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
        ordered_ecs = []

    # creating/updating PRs
    if new_update_preview_pr:
        if options.new_pr:
            new_pr(categorized_paths,
                   ordered_ecs,
                   title=options.pr_title,
                   descr=options.pr_descr,
                   commit_msg=options.pr_commit_msg)
        elif options.preview_pr:
            print review_pr(paths=determined_paths,
                            colored=use_color(options.color))
        else:
            update_pr(options.update_pr,
                      categorized_paths,
                      ordered_ecs,
                      commit_msg=options.pr_commit_msg)

    # dry_run: print all easyconfigs and dependencies, and whether they are already built
    elif dry_run_mode:
        txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
        print_msg(txt, log=_log, silent=testing, prefix=False)

    elif options.check_conflicts:
        if check_conflicts(easyconfigs, modtool):
            print_error("One or more conflicts detected!")
            sys.exit(1)
        else:
            print_msg("\nNo conflicts detected!\n", prefix=False)

    # dump source script to set up build environment
    elif options.dump_env_script:
        dump_env_script(easyconfigs)

    elif options.inject_checksums:
        inject_checksums(ordered_ecs, options.inject_checksums)

    # cleanup and exit after dry run, searching easyconfigs or submitting regression test
    stop_options = [
        options.check_conflicts, dry_run_mode, options.dump_env_script,
        options.inject_checksums
    ]
    if any(no_ec_opts) or any(stop_options):
        cleanup(logfile, eb_tmpdir, testing)
        sys.exit(0)

    # create dependency graph and exit
    if options.dep_graph:
        _log.info("Creating dependency graph %s" % options.dep_graph)
        dep_graph(options.dep_graph, ordered_ecs)
        cleanup(logfile, eb_tmpdir, testing, silent=True)
        sys.exit(0)

    # submit build as job(s), clean up and exit
    if options.job:
        submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
        if not testing:
            print_msg("Submitted parallel build jobs, exiting now")
            cleanup(logfile, eb_tmpdir, testing)
            sys.exit(0)

    # build software, will exit when errors occurs (except when testing)
    if not testing or (testing and do_build):
        exit_on_failure = not (options.dump_test_report
                               or options.upload_test_report)
        hooks = load_hooks(options.hooks)

        ecs_with_res = build_and_install_software(
            ordered_ecs,
            init_session_state,
            exit_on_failure=exit_on_failure,
            hooks=hooks)
    else:
        ecs_with_res = [(ec, {}) for ec in ordered_ecs]

    correct_builds_cnt = len([
        ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)
    ])
    overall_success = correct_builds_cnt == len(ordered_ecs)
    success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt,
                                                        len(ordered_ecs))

    repo = init_repository(get_repository(), get_repositorypath())
    repo.cleanup()

    # dump/upload overall test report
    test_report_msg = overall_test_report(ecs_with_res, len(paths),
                                          overall_success, success_msg,
                                          init_session_state)
    if test_report_msg is not None:
        print_msg(test_report_msg)

    print_msg(success_msg, log=_log, silent=testing)

    # cleanup and spec files
    for ec in easyconfigs:
        if 'original_spec' in ec and os.path.isfile(ec['spec']):
            os.remove(ec['spec'])

    # stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
    stop_logging(logfile, logtostdout=options.logtostdout)
    if overall_success:
        cleanup(logfile, eb_tmpdir, testing)