def test_generaloption_overrides_legacy(self):
"""Test whether generaloption overrides legacy configuration."""
self.purge_environment()
# if both legacy and generaloption style configuration is mixed, generaloption wins
legacy_prefix = os.path.join(self.tmpdir, 'legacy')
go_prefix = os.path.join(self.tmpdir, 'generaloption')
# legacy env vars
os.environ['EASYBUILDPREFIX'] = legacy_prefix
os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'build')
# generaloption env vars
os.environ['EASYBUILD_INSTALLPATH'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(legacy_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(legacy_prefix, 'ebfiles_repo'))
del os.environ['EASYBUILDPREFIX']
# legacy env vars
os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'buildagain')
# generaloption env vars
os.environ['EASYBUILD_PREFIX'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(go_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(go_prefix, 'ebfiles_repo'))
del os.environ['EASYBUILDBUILDPATH']
def test_generaloption_overrides_legacy(self):
"""Test whether generaloption overrides legacy configuration."""
self.purge_environment()
# if both legacy and generaloption style configuration is mixed, generaloption wins
legacy_prefix = os.path.join(self.tmpdir, 'legacy')
go_prefix = os.path.join(self.tmpdir, 'generaloption')
# legacy env vars
os.environ['EASYBUILDPREFIX'] = legacy_prefix
os.environ['EASYBUILDBUILDPATH'] = os.path.join(legacy_prefix, 'build')
# generaloption env vars
os.environ['EASYBUILD_INSTALLPATH'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(legacy_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(legacy_prefix,
'ebfiles_repo'))
del os.environ['EASYBUILDPREFIX']
# legacy env vars
os.environ['EASYBUILDBUILDPATH'] = os.path.join(
legacy_prefix, 'buildagain')
# generaloption env vars
os.environ['EASYBUILD_PREFIX'] = go_prefix
init_config()
self.assertEqual(build_path(), os.path.join(go_prefix, 'build'))
self.assertEqual(install_path(), os.path.join(go_prefix, 'software'))
repo = init_repository(get_repository(), get_repositorypath())
self.assertEqual(repo.repo, os.path.join(go_prefix, 'ebfiles_repo'))
del os.environ['EASYBUILDBUILDPATH']
def create_job(build_command,
easyconfig,
output_dir=None,
conn=None,
ppn=None):
"""
Creates a job, to build a *single* easyconfig
@param build_command: format string for command, full path to an easyconfig file will be substituted in it
@param easyconfig: easyconfig as processed by process_easyconfig
@param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
@param conn: open connection to PBS server
@param ppn: ppn setting to use (# 'processors' (cores) per node to use)
returns the job
"""
if output_dir is None:
output_dir = 'easybuild-build'
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" %
easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'],
det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
# create command based on build_command template
command = build_command % {
'spec': easyconfig['spec'],
'output_dir': os.path.join(os.path.abspath(output_dir), name),
}
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command,
name,
easybuild_vars,
resources=resources,
conn=conn,
ppn=ppn)
job.module = easyconfig['ec'].full_mod_name
return job
def create_job(job_backend,
build_command,
easyconfig,
output_dir='easybuild-build'):
"""
Creates a job to build a *single* easyconfig.
:param job_backend: A factory object for querying server parameters and creating actual job objects
:param build_command: format string for command, full path to an easyconfig file will be substituted in it
:param easyconfig: easyconfig as processed by process_easyconfig
:param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
returns the job
"""
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
for env_var in ["PYTHONPATH", "MODULEPATH"]:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" %
easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'],
det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
# determine whether additional options need to be passed to the 'eb' command
add_opts = ''
if easyconfig['hidden']:
add_opts += ' --hidden'
# create command based on build_command template
command = build_command % {
'add_opts': add_opts,
'output_dir': os.path.join(os.path.abspath(output_dir), name),
'spec': easyconfig['spec'],
}
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
extra = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
extra['hours'] = int(math.ceil(previous_time * 2 / 60))
if build_option('job_cores'):
extra['cores'] = build_option('job_cores')
job = job_backend.make_job(command, name, easybuild_vars, **extra)
job.module = easyconfig['ec'].full_mod_name
return job
def create_job(job_backend, build_command, easyconfig, output_dir='easybuild-build'):
"""
Creates a job to build a *single* easyconfig.
:param job_backend: A factory object for querying server parameters and creating actual job objects
:param build_command: format string for command, full path to an easyconfig file will be substituted in it
:param easyconfig: easyconfig as processed by process_easyconfig
:param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
returns the job
"""
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
for env_var in ["PYTHONPATH", "MODULEPATH"]:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
# determine whether additional options need to be passed to the 'eb' command
add_opts = ''
if easyconfig['hidden']:
add_opts += ' --hidden'
# create command based on build_command template
command = build_command % {
'add_opts': add_opts,
'output_dir': os.path.join(os.path.abspath(output_dir), name),
'spec': easyconfig['spec'],
}
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
extra = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
extra['hours'] = int(math.ceil(previous_time * 2 / 60))
if build_option('job_cores'):
extra['cores'] = build_option('job_cores')
job = job_backend.make_job(command, name, easybuild_vars, **extra)
job.module = easyconfig['ec'].full_mod_name
return job
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
build_command is a format string in which a full path to an eb file will be substituted
easyconfig should be in the format as processEasyConfig returns them
output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
returns the job
"""
# create command based on build_command template
command = build_command % {'spec': easyconfig['spec']}
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" %
easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'],
det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
var = config.oldstyle_environment_variables['test_output_path']
easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command,
name,
easybuild_vars,
resources=resources,
conn=conn,
ppn=ppn)
job.module = det_full_module_name(easyconfig['ec'])
return job
def create_job(build_command, easyconfig, output_dir=None, conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
@param build_command: format string for command, full path to an easyconfig file will be substituted in it
@param easyconfig: easyconfig as processed by process_easyconfig
@param output_dir: optional output path; --regtest-output-dir will be used inside the job with this variable
@param conn: open connection to PBS server
@param ppn: ppn setting to use (# 'processors' (cores) per node to use)
returns the job
"""
if output_dir is None:
output_dir = 'easybuild-build'
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
# create command based on build_command template
command = build_command % {
'spec': easyconfig['spec'],
'output_dir': os.path.join(os.path.abspath(output_dir), name),
}
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
job.module = easyconfig['ec'].full_mod_name
return job
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
build_command is a format string in which a full path to an eb file will be substituted
easyconfig should be in the format as processEasyConfig returns them
output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
returns the job
"""
# create command based on build_command template
command = build_command % {'spec': easyconfig['spec']}
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# obtain unique name based on name/easyconfig version tuple
ec_tuple = (easyconfig['ec']['name'], det_full_ec_version(easyconfig['ec']))
name = '-'.join(ec_tuple)
var = config.oldstyle_environment_variables['test_output_path']
easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)
# just use latest build stats
repo = init_repository(get_repository(), get_repositorypath())
buildstats = repo.get_buildstats(*ec_tuple)
resources = {}
if buildstats:
previous_time = buildstats[-1]['build_time']
resources['hours'] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
job.module = det_full_module_name(easyconfig['ec'])
return job
def create_job(build_command, easyconfig, output_dir="", conn=None, ppn=None):
"""
Creates a job, to build a *single* easyconfig
build_command is a format string in which a full path to an eb file will be substituted
easyconfig should be in the format as processEasyConfig returns them
output_dir is an optional path. EASYBUILDTESTOUTPUT will be set inside the job with this variable
returns the job
"""
# create command based on build_command template
command = build_command % {"spec": easyconfig["spec"]}
# capture PYTHONPATH, MODULEPATH and all variables starting with EASYBUILD
easybuild_vars = {}
for name in os.environ:
if name.startswith("EASYBUILD"):
easybuild_vars[name] = os.environ[name]
others = ["PYTHONPATH", "MODULEPATH"]
for env_var in others:
if env_var in os.environ:
easybuild_vars[env_var] = os.environ[env_var]
_log.info("Dictionary of environment variables passed to job: %s" % easybuild_vars)
# create unique name based on module name
name = "%s-%s" % easyconfig["module"]
var = config.oldstyle_environment_variables["test_output_path"]
easybuild_vars[var] = os.path.join(os.path.abspath(output_dir), name)
# just use latest build stats
buildstats = get_repository().get_buildstats(*easyconfig["module"])
resources = {}
if buildstats:
previous_time = buildstats[-1]["build_time"]
resources["hours"] = int(math.ceil(previous_time * 2 / 60))
job = PbsJob(command, name, easybuild_vars, resources=resources, conn=conn, ppn=ppn)
job.module = easyconfig["module"]
return job
def main(args=None, logfile=None, do_build=None, testing=False):
"""
Main function: parse command line options, and act accordingly.
@param args: command line arguments to use
@param logfile: log file to use
@param do_build: whether or not to actually perform the build
@param testing: enable testing mode
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set by option parsers via set_tmpdir
eb_tmpdir = tempfile.gettempdir()
# initialise logging for main
global _log
_log, logfile = init_logging(logfile, logtostdout=options.logtostdout, silent=testing or options.last_log)
# disallow running EasyBuild as root
if os.getuid() == 0:
raise EasyBuildError(
"You seem to be running EasyBuild with root privileges which is not wise, " "so let's end this here."
)
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short
robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section("config")
build_options = {
"build_specs": build_specs,
"command_line": eb_cmd_line,
"pr_path": pr_path,
"robot_path": robot_path,
"silent": testing,
"try_to_generate": try_to_generate,
"valid_stops": [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options, cmdline_options=options)
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or "(none)"
print_msg(last_log, log=_log, prefix=False)
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works
init_session_state.update({"easybuild_configuration": eb_config})
init_session_state.update({"module_list": modlist})
_log.debug("Initial session state: %s" % init_session_state)
# GitHub integration
if options.review_pr or options.new_pr or options.update_pr:
if options.review_pr:
print review_pr(options.review_pr, colored=options.color)
elif options.new_pr:
new_pr(orig_paths, title=options.pr_title, descr=options.pr_descr, commit_msg=options.pr_commit_msg)
elif options.update_pr:
update_pr(options.update_pr, orig_paths, commit_msg=options.pr_commit_msg)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# search for easyconfigs, if a query is specified
query = options.search or options.search_short
if query:
search_easyconfigs(query, short=not options.search)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# command line options that do not require any easyconfigs to be specified
no_ec_opts = [
options.aggregate_regtest,
options.new_pr,
options.review_pr,
options.search,
options.search_short,
options.regtest,
options.update_pr,
]
# determine paths to easyconfigs
paths = det_easyconfig_paths(orig_paths)
if paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in paths]
else:
if "name" in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif not any(no_ec_opts):
print_error(
(
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"
),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing,
)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths)
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
txt = dry_run(easyconfigs, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
if any(no_ec_opts + [options.dry_run, options.dry_run_short]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not (options.force or options.rebuild or options.extended_dry_run):
retained_ecs = skip_available(easyconfigs)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec["full_mod_name"])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
if options.robot:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(
easyconfigs,
minimal_toolchains=build_option("minimal_toolchains"),
use_existing_modules=build_option("use_existing_modules"),
)
else:
ordered_ecs = easyconfigs
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get("success", False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if "original_spec" in ec and os.path.isfile(ec["spec"]):
os.remove(ec["spec"])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when regtesting)
correct_built_cnt = 0
all_built_cnt = 0
if not testing or (testing and do_build):
for ec in ordered_ecs:
(success, _) = build_and_install_software(ec, orig_environ)
if success:
correct_built_cnt += 1
all_built_cnt += 1
print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# cleanup tmp log file (all is well, all modules have their own log file)
if options.logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
else:
fancylogger.logToFile(logfile, enable=False)
cleanup(logfile, eb_tmpdir, testing)
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
"""
Main function: parse command line options, and act accordingly.
:param args: command line arguments to use
:param logfile: log file to use
:param do_build: whether or not to actually perform the build
:param testing: enable testing mode
"""
register_lock_cleanup_signal_handlers()
# if $CDPATH is set, unset it, it'll only cause trouble...
# see https://github.com/easybuilders/easybuild-framework/issues/2944
if 'CDPATH' in os.environ:
del os.environ['CDPATH']
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
options, orig_paths = eb_go.options, eb_go.args
global _log
(build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings
# load hook implementations (if any)
hooks = load_hooks(options.hooks)
run_hook(START, hooks)
if modtool is None:
modtool = modules_tool(testing=testing)
# check whether any (EasyBuild-generated) modules are loaded already in the current session
modtool.check_loaded_modules()
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")
# search for easyconfigs, if a query is specified
if search_query:
search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
terse=options.terse)
# GitHub options that warrant a silent cleanup & exit
if options.check_github:
check_github()
elif options.install_github_token:
install_github_token(options.github_user, silent=build_option('silent'))
elif options.close_pr:
close_pr(options.close_pr, motivation_msg=options.close_pr_msg)
elif options.list_prs:
print(list_prs(options.list_prs))
elif options.merge_pr:
merge_pr(options.merge_pr)
elif options.review_pr:
print(review_pr(pr=options.review_pr, colored=use_color(options.color)))
elif options.list_installed_software:
detailed = options.list_installed_software == 'detailed'
print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True))
elif options.list_software:
print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed'))
elif options.create_index:
print_msg("Creating index for %s..." % options.create_index, prefix=False)
index_fp = dump_index(options.create_index, max_age_sec=options.index_max_age)
index = load_index(options.create_index)
print_msg("Index created at %s (%d files)" % (index_fp, len(index)), prefix=False)
# non-verbose cleanup after handling GitHub integration stuff or printing terse info
early_stop_options = [
options.check_github,
options.create_index,
options.install_github_token,
options.list_installed_software,
options.list_software,
options.close_pr,
options.list_prs,
options.merge_pr,
options.review_pr,
options.terse,
search_query,
]
if any(early_stop_options):
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = modtool.list() # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
if options.install_latest_eb_release:
if orig_paths:
raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
"other easyconfigs")
else:
eb_file = find_easybuild_easyconfig()
orig_paths.append(eb_file)
if len(orig_paths) == 1:
# if only one easyconfig file is specified, use current directory as target directory
target_path = os.getcwd()
elif orig_paths:
# last path is target when --copy-ec is used, so remove that from the list
target_path = orig_paths.pop() if options.copy_ec else None
categorized_paths = categorize_files_by_type(orig_paths)
# command line options that do not require any easyconfigs to be specified
pr_options = options.new_branch_github or options.new_pr or options.new_pr_from_branch or options.preview_pr
pr_options = pr_options or options.sync_branch_with_develop or options.sync_pr_with_develop
pr_options = pr_options or options.update_branch_github or options.update_pr
no_ec_opts = [options.aggregate_regtest, options.regtest, pr_options, search_query]
# determine paths to easyconfigs
determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
if (options.copy_ec and not tweaked_ecs_paths) or options.fix_deprecated_easyconfigs or options.show_ec:
if options.copy_ec:
if len(determined_paths) == 1:
copy_file(determined_paths[0], target_path)
print_msg("%s copied to %s" % (os.path.basename(determined_paths[0]), target_path), prefix=False)
elif len(determined_paths) > 1:
copy_files(determined_paths, target_path)
print_msg("%d file(s) copied to %s" % (len(determined_paths), target_path), prefix=False)
else:
raise EasyBuildError("One of more files to copy should be specified!")
elif options.fix_deprecated_easyconfigs:
fix_deprecated_easyconfigs(determined_paths)
elif options.show_ec:
for path in determined_paths:
print_msg("Contents of %s:" % path)
print_msg(read_file(path), prefix=False)
clean_exit(logfile, eb_tmpdir, testing)
if determined_paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in determined_paths]
elif 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif any(no_ec_opts):
paths = determined_paths
else:
print_error("Please provide one or multiple easyconfig files, or use software build " +
"options to make EasyBuild search for easyconfigs",
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
_log.debug("Paths: %s", paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums)
# handle --check-contrib & --check-style options
if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style):
clean_exit(logfile, eb_tmpdir, testing)
# verify easyconfig filenames, if desired
if options.verify_easyconfig_filenames:
_log.info("Verifying easyconfig filenames...")
for easyconfig in easyconfigs:
verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec'])
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
if options.containerize:
# if --containerize/-C create a container recipe (and optionally container image), and stop
containerize(easyconfigs)
clean_exit(logfile, eb_tmpdir, testing)
forced = options.force or options.rebuild
dry_run_mode = options.dry_run or options.dry_run_short or options.missing_modules
# skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
if not (forced or dry_run_mode or options.extended_dry_run or pr_options or options.inject_checksums):
retained_ecs = skip_available(easyconfigs, modtool)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# keep track for which easyconfigs we should set the corresponding module as default
if options.set_default_module:
for easyconfig in easyconfigs:
easyconfig['ec'].set_default_module = True
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
# resolve dependencies if robot is enabled, except in dry run mode
# one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
if options.robot and (not dry_run_mode or pr_options):
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, modtool)
else:
ordered_ecs = easyconfigs
elif pr_options:
ordered_ecs = None
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
if options.copy_ec and tweaked_ecs_paths:
all_specs = [spec['spec'] for spec in
resolve_dependencies(easyconfigs, modtool, retain_all_deps=True, raise_error_missing_ecs=False)]
tweaked_ecs_in_all_ecs = [path for path in all_specs if
any(tweaked_ecs_path in path for tweaked_ecs_path in tweaked_ecs_paths)]
if tweaked_ecs_in_all_ecs:
# Clean them, then copy them
clean_up_easyconfigs(tweaked_ecs_in_all_ecs)
copy_files(tweaked_ecs_in_all_ecs, target_path)
print_msg("%d file(s) copied to %s" % (len(tweaked_ecs_in_all_ecs), target_path), prefix=False)
# creating/updating PRs
if pr_options:
if options.new_pr:
new_pr(categorized_paths, ordered_ecs)
elif options.new_branch_github:
new_branch_github(categorized_paths, ordered_ecs)
elif options.new_pr_from_branch:
new_pr_from_branch(options.new_pr_from_branch)
elif options.preview_pr:
print(review_pr(paths=determined_paths, colored=use_color(options.color)))
elif options.sync_branch_with_develop:
sync_branch_with_develop(options.sync_branch_with_develop)
elif options.sync_pr_with_develop:
sync_pr_with_develop(options.sync_pr_with_develop)
elif options.update_branch_github:
update_branch(options.update_branch_github, categorized_paths, ordered_ecs)
elif options.update_pr:
update_pr(options.update_pr, categorized_paths, ordered_ecs)
else:
raise EasyBuildError("Unknown PR option!")
# dry_run: print all easyconfigs and dependencies, and whether they are already built
elif dry_run_mode:
if options.missing_modules:
txt = missing_deps(easyconfigs, modtool)
else:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
elif options.check_conflicts:
if check_conflicts(easyconfigs, modtool):
print_error("One or more conflicts detected!")
sys.exit(1)
else:
print_msg("\nNo conflicts detected!\n", prefix=False)
# dump source script to set up build environment
elif options.dump_env_script:
dump_env_script(easyconfigs)
elif options.inject_checksums:
inject_checksums(ordered_ecs, options.inject_checksums)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]
if any(no_ec_opts) or any(stop_options):
clean_exit(logfile, eb_tmpdir, testing)
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
clean_exit(logfile, eb_tmpdir, testing)
# build software, will exit when errors occurs (except when testing)
if not testing or (testing and do_build):
exit_on_failure = not (options.dump_test_report or options.upload_test_report)
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
run_hook(END, hooks)
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def test_legacy_config_file(self):
"""Test finding/using legacy configuration files."""
self.purge_environment()
cfg_fn = self.configure(args=[])
self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py'))
configtxt = """
build_path = '%(buildpath)s'
source_path = '%(sourcepath)s'
install_path = '%(installpath)s'
repository_path = '%(repopath)s'
repository = FileRepository(repository_path)
log_format = ('%(logdir)s', '%(logtmpl)s')
log_dir = '%(tmplogdir)s'
software_install_suffix = '%(softsuffix)s'
modules_install_suffix = '%(modsuffix)s'
"""
buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'test', 'install', 'path')
repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path')
logdir = 'somedir'
logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir')
softsuffix = 'myfavoritesoftware'
modsuffix = 'modulesgohere'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix
}
# create user config file on default location
myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py')
if not os.path.exists(os.path.dirname(myconfigfile)):
os.makedirs(os.path.dirname(myconfigfile))
write_file(myconfigfile, configtxt % configdict)
# redefine home so we can test user config file on default location
home = os.environ.get('HOME', None)
os.environ['HOME'] = self.tmpdir
init_config()
cfg_fn = self.configure(args=[])
if home is not None:
os.environ['HOME'] = home
# check finding and use of config file
self.assertEqual(cfg_fn, myconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
# redefine config file entries for proper testing below
buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build', 'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'install', 'path')
repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo', 'path')
logdir = 'somedir_custom'
logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'tmplogdir')
softsuffix = 'myfavoritesoftware_custom'
modsuffix = 'modulesgohere_custom'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix }
# create custom config file, and point to it
mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py')
if not os.path.exists(os.path.dirname(mycustomconfigfile)):
os.makedirs(os.path.dirname(mycustomconfigfile))
write_file(mycustomconfigfile, configtxt % configdict)
os.environ['EASYBUILDCONFIG'] = mycustomconfigfile
# reconfigure
init_config()
cfg_fn = self.configure(args=[])
# verify configuration
self.assertEqual(cfg_fn, mycustomconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'), os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
def test_legacy_env_vars(self):
"""Test legacy environment variables."""
self.purge_environment()
# build path
test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
del os.environ['EASYBUILDBUILDPATH']
# source path(s)
test_sourcepaths = [
os.path.join(self.tmpdir, 'source', 'path'),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
]),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
os.path.join(self.tmpdir, 'source', 'path3'),
]),
]
for test_sourcepath in test_sourcepaths:
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths(), test_sourcepath.split(':'))
del os.environ['EASYBUILDSOURCEPATH']
test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')
# install path
init_config()
test_installpath = os.path.join(self.tmpdir, 'install', 'path')
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(source_paths()[0], os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
del os.environ['EASYBUILDINSTALLPATH']
# prefix: should change build/install/source/repo paths
init_config()
test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
os.environ['EASYBUILDPREFIX'] = test_prefixpath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
# build/source/install path overrides prefix
init_config()
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDBUILDPATH']
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], test_sourcepath)
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDSOURCEPATH']
init_config()
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
"""
Main function: parse command line options, and act accordingly.
:param args: command line arguments to use
:param logfile: log file to use
:param do_build: whether or not to actually perform the build
:param testing: enable testing mode
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set by option parsers via set_tmpdir
eb_tmpdir = tempfile.gettempdir()
search_query = options.search or options.search_filename or options.search_short
# initialise logging for main
global _log
_log, logfile = init_logging(logfile, logtostdout=options.logtostdout,
silent=(testing or options.terse or search_query), colorize=options.color)
# disallow running EasyBuild as root
if os.getuid() == 0:
raise EasyBuildError("You seem to be running EasyBuild with root privileges which is not wise, "
"so let's end this here.")
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
robot_path = det_robot_path(options.robot_paths, tweaked_ecs_paths, pr_path, auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section('config')
build_options = {
'build_specs': build_specs,
'command_line': eb_cmd_line,
'external_modules_metadata': parse_external_modules_metadata(options.external_modules_metadata),
'pr_path': pr_path,
'robot_path': robot_path,
'silent': testing,
'try_to_generate': try_to_generate,
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options, cmdline_options=options)
if modtool is None:
modtool = modules_tool(testing=testing)
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")
# search for easyconfigs, if a query is specified
if search_query:
search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
terse=options.terse)
# GitHub options that warrant a silent cleanup & exit
if options.check_github:
check_github()
elif options.install_github_token:
install_github_token(options.github_user, silent=build_option('silent'))
elif options.review_pr:
print review_pr(options.review_pr, colored=use_color(options.color))
elif options.list_installed_software:
detailed = options.list_installed_software == 'detailed'
print list_software(output_format=options.output_format, detailed=detailed, only_installed=True)
elif options.list_software:
print list_software(output_format=options.output_format, detailed=options.list_software == 'detailed')
# non-verbose cleanup after handling GitHub integration stuff or printing terse info
early_stop_options = [
options.check_github,
options.install_github_token,
options.list_installed_software,
options.list_software,
options.review_pr,
options.terse,
search_query,
]
if any(early_stop_options):
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = modtool.list() # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
if options.install_latest_eb_release:
if orig_paths:
raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
"other easyconfigs")
else:
eb_file = find_easybuild_easyconfig()
orig_paths.append(eb_file)
categorized_paths = categorize_files_by_type(orig_paths)
# command line options that do not require any easyconfigs to be specified
no_ec_opts = [options.aggregate_regtest, options.new_pr, options.regtest, options.update_pr, search_query]
# determine paths to easyconfigs
paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
if paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in paths]
else:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif not any(no_ec_opts):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if options.check_style:
_log.debug("Running style check...")
if cmdline_easyconfigs_style_check([path[0] for path in paths]):
print_msg("All style checks passed!", prefix=False)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
else:
raise EasyBuildError("One or more style checks FAILED!")
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths)
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
dry_run_mode = options.dry_run or options.dry_run_short
new_update_pr = options.new_pr or options.update_pr
# skip modules that are already installed unless forced
if not (options.force or options.rebuild or dry_run_mode or options.extended_dry_run or new_update_pr):
retained_ecs = skip_available(easyconfigs, modtool)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
# resolve dependencies if robot is enabled, except in dry run mode
# one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
if options.robot and (not dry_run_mode or new_update_pr):
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, modtool)
else:
ordered_ecs = easyconfigs
elif new_update_pr:
ordered_ecs = None
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# creating/updating PRs
if new_update_pr:
if options.new_pr:
new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr,
commit_msg=options.pr_commit_msg)
else:
update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg)
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
elif dry_run_mode:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
elif options.check_conflicts:
if check_conflicts(easyconfigs, modtool):
print_error("One or more conflicts detected!")
sys.exit(1)
else:
print_msg("\nNo conflicts detected!\n", prefix=False)
# dump source script to set up build environment
elif options.dump_env_script:
dump_env_script(easyconfigs)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
if any(no_ec_opts + [options.check_conflicts, dry_run_mode, options.dump_env_script]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def test_legacy_env_vars(self):
"""Test legacy environment variables."""
# build path
test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
del os.environ['EASYBUILDBUILDPATH']
# source path
config.variables = ConfigurationVariables()
test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_path(), test_sourcepath)
del os.environ['EASYBUILDSOURCEPATH']
# install path
config.variables = ConfigurationVariables()
test_installpath = os.path.join(self.tmpdir, 'install', 'path')
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(source_path(), os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
del os.environ['EASYBUILDINSTALLPATH']
# prefix: should change build/install/source/repo paths
config.variables = ConfigurationVariables()
test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
os.environ['EASYBUILDPREFIX'] = test_prefixpath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = get_repository()
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
# build/source/install path overrides prefix
config.variables = ConfigurationVariables()
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
self.assertEqual(source_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = get_repository()
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
# also check old style vs new style
self.assertEqual(config.variables['build_path'], config.variables['buildpath'])
self.assertEqual(config.variables['install_path'], config.variables['installpath'])
del os.environ['EASYBUILDBUILDPATH']
config.variables = ConfigurationVariables()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_path(), test_sourcepath)
self.assertEqual(install_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = get_repository()
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDSOURCEPATH']
config.variables = ConfigurationVariables()
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(build_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_path(), os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(install_path(), os.path.join(test_installpath, DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(install_path(typ='mod'), os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = get_repository()
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDINSTALLPATH']
del os.environ['EASYBUILDPREFIX']
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
"""
Main function: parse command line options, and act accordingly.
:param args: command line arguments to use
:param logfile: log file to use
:param do_build: whether or not to actually perform the build
:param testing: enable testing mode
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
eb_go, cfg_settings = set_up_configuration(args=args, logfile=logfile, testing=testing)
options, orig_paths = eb_go.options, eb_go.args
global _log
(build_specs, _log, logfile, robot_path, search_query, eb_tmpdir, try_to_generate, tweaked_ecs_paths) = cfg_settings
# load hook implementations (if any)
hooks = load_hooks(options.hooks)
run_hook(START, hooks)
if modtool is None:
modtool = modules_tool(testing=testing)
# check whether any (EasyBuild-generated) modules are loaded already in the current session
modtool.check_loaded_modules()
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug("Packaging not enabled, so not checking for packaging support.")
# search for easyconfigs, if a query is specified
if search_query:
search_easyconfigs(search_query, short=options.search_short, filename_only=options.search_filename,
terse=options.terse)
# GitHub options that warrant a silent cleanup & exit
if options.check_github:
check_github()
elif options.install_github_token:
install_github_token(options.github_user, silent=build_option('silent'))
elif options.close_pr:
close_pr(options.close_pr, motivation_msg=options.close_pr_msg)
elif options.list_prs:
print(list_prs(options.list_prs))
elif options.merge_pr:
merge_pr(options.merge_pr)
elif options.review_pr:
print(review_pr(pr=options.review_pr, colored=use_color(options.color)))
elif options.list_installed_software:
detailed = options.list_installed_software == 'detailed'
print(list_software(output_format=options.output_format, detailed=detailed, only_installed=True))
elif options.list_software:
print(list_software(output_format=options.output_format, detailed=options.list_software == 'detailed'))
# non-verbose cleanup after handling GitHub integration stuff or printing terse info
early_stop_options = [
options.check_github,
options.install_github_token,
options.list_installed_software,
options.list_software,
options.close_pr,
options.list_prs,
options.merge_pr,
options.review_pr,
options.terse,
search_query,
]
if any(early_stop_options):
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = modtool.list() # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
if options.install_latest_eb_release:
if orig_paths:
raise EasyBuildError("Installing the latest EasyBuild release can not be combined with installing "
"other easyconfigs")
else:
eb_file = find_easybuild_easyconfig()
orig_paths.append(eb_file)
categorized_paths = categorize_files_by_type(orig_paths)
# command line options that do not require any easyconfigs to be specified
new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr
no_ec_opts = [options.aggregate_regtest, options.regtest, search_query, new_update_preview_pr]
# determine paths to easyconfigs
determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
if determined_paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in determined_paths]
else:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif any(no_ec_opts):
paths = determined_paths
else:
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths, modtool)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths, validate=not options.inject_checksums)
# handle --check-contrib & --check-style options
if run_contrib_style_checks([ec['ec'] for ec in easyconfigs], options.check_contrib, options.check_style):
clean_exit(logfile, eb_tmpdir, testing)
# verify easyconfig filenames, if desired
if options.verify_easyconfig_filenames:
_log.info("Verifying easyconfig filenames...")
for easyconfig in easyconfigs:
verify_easyconfig_filename(easyconfig['spec'], easyconfig['ec'], parsed_ec=easyconfig['ec'])
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, modtool, targetdirs=tweaked_ecs_paths)
if options.containerize:
# if --containerize/-C create a container recipe (and optionally container image), and stop
containerize(easyconfigs)
clean_exit(logfile, eb_tmpdir, testing)
forced = options.force or options.rebuild
dry_run_mode = options.dry_run or options.dry_run_short
# skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
if not (forced or dry_run_mode or options.extended_dry_run or new_update_preview_pr or options.inject_checksums):
retained_ecs = skip_available(easyconfigs, modtool)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
# resolve dependencies if robot is enabled, except in dry run mode
# one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
if options.robot and (not dry_run_mode or new_update_preview_pr):
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, modtool)
else:
ordered_ecs = easyconfigs
elif new_update_preview_pr:
ordered_ecs = None
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# creating/updating PRs
if new_update_preview_pr:
if options.new_pr:
new_pr(categorized_paths, ordered_ecs, title=options.pr_title, descr=options.pr_descr,
commit_msg=options.pr_commit_msg)
elif options.preview_pr:
print(review_pr(paths=determined_paths, colored=use_color(options.color)))
else:
update_pr(options.update_pr, categorized_paths, ordered_ecs, commit_msg=options.pr_commit_msg)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
elif dry_run_mode:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
elif options.check_conflicts:
if check_conflicts(easyconfigs, modtool):
print_error("One or more conflicts detected!")
sys.exit(1)
else:
print_msg("\nNo conflicts detected!\n", prefix=False)
# dump source script to set up build environment
elif options.dump_env_script:
dump_env_script(easyconfigs)
elif options.inject_checksums:
inject_checksums(ordered_ecs, options.inject_checksums)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
stop_options = [options.check_conflicts, dry_run_mode, options.dump_env_script, options.inject_checksums]
if any(no_ec_opts) or any(stop_options):
clean_exit(logfile, eb_tmpdir, testing)
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
clean_exit(logfile, eb_tmpdir, testing, silent=True)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
clean_exit(logfile, eb_tmpdir, testing)
# build software, will exit when errors occurs (except when testing)
if not testing or (testing and do_build):
exit_on_failure = not (options.dump_test_report or options.upload_test_report)
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
run_hook(END, hooks)
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def test_legacy_env_vars(self):
"""Test legacy environment variables."""
self.purge_environment()
# build path
test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
del os.environ['EASYBUILDBUILDPATH']
# source path(s)
test_sourcepaths = [
os.path.join(self.tmpdir, 'source', 'path'),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
]),
':'.join([
os.path.join(self.tmpdir, 'source', 'path1'),
os.path.join(self.tmpdir, 'source', 'path2'),
os.path.join(self.tmpdir, 'source', 'path3'),
]),
]
for test_sourcepath in test_sourcepaths:
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths(), test_sourcepath.split(':'))
del os.environ['EASYBUILDSOURCEPATH']
test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')
# install path
init_config()
test_installpath = os.path.join(self.tmpdir, 'install', 'path')
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(
source_paths()[0],
os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
del os.environ['EASYBUILDINSTALLPATH']
# prefix: should change build/install/source/repo paths
init_config()
test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
os.environ['EASYBUILDPREFIX'] = test_prefixpath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(
source_paths()[0],
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
# build/source/install path overrides prefix
init_config()
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
self.assertEqual(
source_paths()[0],
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDBUILDPATH']
init_config()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_paths()[0], test_sourcepath)
self.assertEqual(
install_path(),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDSOURCEPATH']
init_config()
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(
source_paths()[0],
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
def test_legacy_config_file(self):
"""Test finding/using legacy configuration files."""
self.purge_environment()
cfg_fn = self.configure(args=[])
self.assertTrue(cfg_fn.endswith('easybuild/easybuild_config.py'))
configtxt = """
build_path = '%(buildpath)s'
source_path = '%(sourcepath)s'
install_path = '%(installpath)s'
repository_path = '%(repopath)s'
repository = FileRepository(repository_path)
log_format = ('%(logdir)s', '%(logtmpl)s')
log_dir = '%(tmplogdir)s'
software_install_suffix = '%(softsuffix)s'
modules_install_suffix = '%(modsuffix)s'
"""
buildpath = os.path.join(self.tmpdir, 'my', 'test', 'build', 'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'test', 'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'test', 'install',
'path')
repopath = os.path.join(self.tmpdir, 'my', 'test', 'repo', 'path')
logdir = 'somedir'
logtmpl = 'test-eb-%(name)s%(version)s_date-%(date)s__time-%(time)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'test', 'tmplogdir')
softsuffix = 'myfavoritesoftware'
modsuffix = 'modulesgohere'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix
}
# create user config file on default location
myconfigfile = os.path.join(self.tmpdir, '.easybuild', 'config.py')
if not os.path.exists(os.path.dirname(myconfigfile)):
os.makedirs(os.path.dirname(myconfigfile))
write_file(myconfigfile, configtxt % configdict)
# redefine home so we can test user config file on default location
home = os.environ.get('HOME', None)
os.environ['HOME'] = self.tmpdir
init_config()
cfg_fn = self.configure(args=[])
if home is not None:
os.environ['HOME'] = home
# check finding and use of config file
self.assertEqual(cfg_fn, myconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'),
os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
# redefine config file entries for proper testing below
buildpath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'build',
'path')
sourcepath = os.path.join(self.tmpdir, 'my', 'custom', 'test',
'source', 'path')
installpath = os.path.join(self.tmpdir, 'my', 'custom', 'test',
'install', 'path')
repopath = os.path.join(self.tmpdir, 'my', 'custom', 'test', 'repo',
'path')
logdir = 'somedir_custom'
logtmpl = 'test-custom-eb-%(name)_%(date)s%(time)s__%(version)s.log'
tmplogdir = os.path.join(self.tmpdir, 'my', 'custom', 'test',
'tmplogdir')
softsuffix = 'myfavoritesoftware_custom'
modsuffix = 'modulesgohere_custom'
configdict = {
'buildpath': buildpath,
'sourcepath': sourcepath,
'installpath': installpath,
'repopath': repopath,
'logdir': logdir,
'logtmpl': logtmpl,
'tmplogdir': tmplogdir,
'softsuffix': softsuffix,
'modsuffix': modsuffix
}
# create custom config file, and point to it
mycustomconfigfile = os.path.join(self.tmpdir, 'mycustomconfig.py')
if not os.path.exists(os.path.dirname(mycustomconfigfile)):
os.makedirs(os.path.dirname(mycustomconfigfile))
write_file(mycustomconfigfile, configtxt % configdict)
os.environ['EASYBUILDCONFIG'] = mycustomconfigfile
# reconfigure
init_config()
cfg_fn = self.configure(args=[])
# verify configuration
self.assertEqual(cfg_fn, mycustomconfigfile)
self.assertEqual(build_path(), buildpath)
self.assertEqual(source_paths()[0], sourcepath)
self.assertEqual(install_path(), os.path.join(installpath, softsuffix))
self.assertEqual(install_path(typ='mod'),
os.path.join(installpath, modsuffix))
repo = init_repository(get_repository(), get_repositorypath())
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(repo.repo, repopath)
self.assertEqual(log_file_format(return_directory=True), logdir)
self.assertEqual(log_file_format(), logtmpl)
self.assertEqual(get_build_log_path(), tmplogdir)
def test_legacy_env_vars(self):
"""Test legacy environment variables."""
# build path
test_buildpath = os.path.join(self.tmpdir, 'build', 'path')
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
del os.environ['EASYBUILDBUILDPATH']
# source path
config.variables = ConfigurationVariables()
test_sourcepath = os.path.join(self.tmpdir, 'source', 'path')
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_path(), test_sourcepath)
del os.environ['EASYBUILDSOURCEPATH']
# install path
config.variables = ConfigurationVariables()
test_installpath = os.path.join(self.tmpdir, 'install', 'path')
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(
source_path(),
os.path.join(os.path.expanduser('~'), '.local', 'easybuild',
DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
del os.environ['EASYBUILDINSTALLPATH']
# prefix: should change build/install/source/repo paths
config.variables = ConfigurationVariables()
test_prefixpath = os.path.join(self.tmpdir, 'prefix', 'path')
os.environ['EASYBUILDPREFIX'] = test_prefixpath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(
source_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = get_repository()
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
# build/source/install path overrides prefix
config.variables = ConfigurationVariables()
os.environ['EASYBUILDBUILDPATH'] = test_buildpath
self.configure(args=[])
self.assertEqual(build_path(), test_buildpath)
self.assertEqual(
source_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = get_repository()
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
# also check old style vs new style
self.assertEqual(config.variables['build_path'],
config.variables['buildpath'])
self.assertEqual(config.variables['install_path'],
config.variables['installpath'])
del os.environ['EASYBUILDBUILDPATH']
config.variables = ConfigurationVariables()
os.environ['EASYBUILDSOURCEPATH'] = test_sourcepath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(source_path(), test_sourcepath)
self.assertEqual(
install_path(),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = get_repository()
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDSOURCEPATH']
config.variables = ConfigurationVariables()
os.environ['EASYBUILDINSTALLPATH'] = test_installpath
self.configure(args=[])
self.assertEqual(
build_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['buildpath']))
self.assertEqual(
source_path(),
os.path.join(test_prefixpath, DEFAULT_PATH_SUBDIRS['sourcepath']))
self.assertEqual(
install_path(),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_software']))
self.assertEqual(
install_path(typ='mod'),
os.path.join(test_installpath,
DEFAULT_PATH_SUBDIRS['subdir_modules']))
repo = get_repository()
self.assertTrue(isinstance(repo, FileRepository))
self.assertEqual(
repo.repo,
os.path.join(test_prefixpath,
DEFAULT_PATH_SUBDIRS['repositorypath']))
del os.environ['EASYBUILDINSTALLPATH']
del os.environ['EASYBUILDPREFIX']
sys.exit(0)
# build software, will exit when errors occurs (except when regtesting)
correct_built_cnt = 0
all_built_cnt = 0
if not testing:
for spec in orderedSpecs:
(success, _) = build_and_install_software(spec, options, origEnviron, silent=testing)
if success:
correct_built_cnt += 1
all_built_cnt += 1
print_msg("Build succeeded for %s out of %s" % (correct_built_cnt, all_built_cnt), log=_log, silent=testing)
get_repository().cleanup()
# cleanup and spec files
for ec in easyconfigs:
if 'originalSpec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# cleanup tmp log file (all is well, all modules have their own log file)
if options.logtostdout:
fancylogger.logToScreen(enable=False, stdout=True)
else:
fancylogger.logToFile(logfile, enable=False)
cleanup_logfile_and_exit(logfile, testing, False)
logfile = None
return logfile
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(
ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([
ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)
])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt,
len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# report back in PR in case of testing
if options.upload_test_report:
msg = success_msg + " (%d easyconfigs in this PR)" % len(paths)
test_report = create_test_report(msg,
ecs_with_res,
init_session_state,
pr_nr=options.from_pr,
gist_log=True)
if options.from_pr:
# upload test report to gist and issue a comment in the PR to notify
msg = post_easyconfigs_pr_test_report(options.from_pr, test_report,
success_msg,
init_session_state,
def main(testing_data=(None, None, None)):
"""
Main function: parse command line options, and act accordingly.
@param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
global _log
_log, logfile = init_logging(logfile,
logtostdout=options.logtostdout,
testing=testing)
# disallow running EasyBuild as root
if os.getuid() == 0:
raise EasyBuildError(
"You seem to be running EasyBuild with root privileges which is not wise, "
"so let's end this here.")
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir,
tweaked_ecs=tweaked_ecs,
from_pr=options.from_pr)
auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short
robot_path = det_robot_path(options.robot_paths,
tweaked_ecs_path,
pr_path,
auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section('config')
build_options = {
'build_specs': build_specs,
'command_line': eb_cmd_line,
'pr_path': pr_path,
'robot_path': robot_path,
'silent': testing,
'try_to_generate': try_to_generate,
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options,
cmdline_options=options)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = session_module_list(
testing=testing
) # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs, if a query is specified
query = options.search or options.search_short
if query:
search_easyconfigs(query, short=not options.search)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
# determine paths to easyconfigs
paths = det_easyconfig_paths(orig_paths)
if paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in paths]
else:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs,
robot_path,
try_to_generate,
testing=testing)
elif not any([
options.aggregate_regtest, options.search,
options.search_short, options.regtest
]):
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths]
or easyconfigs_pkg_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths)
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs,
build_specs,
targetdir=tweaked_ecs_path)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
txt = dry_run(easyconfigs,
short=not options.dry_run,
build_specs=build_specs)
print_msg(txt, log=_log, silent=testing, prefix=False)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
if any([
options.dry_run, options.dry_run_short, options.regtest,
options.search, options.search_short
]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not options.force:
retained_ecs = skip_available(easyconfigs)
if not testing:
for skipped_ec in [
ec for ec in easyconfigs if ec not in retained_ecs
]:
print_msg("%s is already installed (module found), skipping" %
skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
if options.robot:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs,
build_specs=build_specs)
else:
ordered_ecs = easyconfigs
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
job_info_txt = submit_jobs(ordered_ecs,
eb_go.generate_cmd_line(),
testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now: %s" %
job_info_txt)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(
ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([
ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)
])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt,
len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths),
overall_success, success_msg,
init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def main(args=None, logfile=None, do_build=None, testing=False, modtool=None):
"""
Main function: parse command line options, and act accordingly.
:param args: command line arguments to use
:param logfile: log file to use
:param do_build: whether or not to actually perform the build
:param testing: enable testing mode
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set by option parsers via set_tmpdir
eb_tmpdir = tempfile.gettempdir()
search_query = options.search or options.search_filename or options.search_short
# initialise logging for main
global _log
_log, logfile = init_logging(logfile,
logtostdout=options.logtostdout,
silent=(testing or options.terse
or search_query),
colorize=options.color)
# disallow running EasyBuild as root (by default)
check_root_usage(
allow_use_as_root=options.allow_use_as_root_and_accept_consequences)
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_paths, pr_path = alt_easyconfig_paths(eb_tmpdir,
tweaked_ecs=tweaked_ecs,
from_pr=options.from_pr)
auto_robot = try_to_generate or options.check_conflicts or options.dep_graph or search_query
robot_path = det_robot_path(options.robot_paths,
tweaked_ecs_paths,
pr_path,
auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section('config')
build_options = {
'build_specs':
build_specs,
'command_line':
eb_cmd_line,
'external_modules_metadata':
parse_external_modules_metadata(options.external_modules_metadata),
'pr_path':
pr_path,
'robot_path':
robot_path,
'silent':
testing,
'try_to_generate':
try_to_generate,
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options,
cmdline_options=options)
if modtool is None:
modtool = modules_tool(testing=testing)
# check whether any (EasyBuild-generated) modules are loaded already in the current session
modtool.check_loaded_modules()
if options.last_log:
# print location to last log file, and exit
last_log = find_last_log(logfile) or '(none)'
print_msg(last_log, log=_log, prefix=False)
# check whether packaging is supported when it's being used
if options.package:
check_pkg_support()
else:
_log.debug(
"Packaging not enabled, so not checking for packaging support.")
# search for easyconfigs, if a query is specified
if search_query:
search_easyconfigs(search_query,
short=options.search_short,
filename_only=options.search_filename,
terse=options.terse)
# GitHub options that warrant a silent cleanup & exit
if options.check_github:
check_github()
elif options.install_github_token:
install_github_token(options.github_user,
silent=build_option('silent'))
elif options.merge_pr:
merge_pr(options.merge_pr)
elif options.review_pr:
print review_pr(pr=options.review_pr, colored=use_color(options.color))
elif options.list_installed_software:
detailed = options.list_installed_software == 'detailed'
print list_software(output_format=options.output_format,
detailed=detailed,
only_installed=True)
elif options.list_software:
print list_software(output_format=options.output_format,
detailed=options.list_software == 'detailed')
# non-verbose cleanup after handling GitHub integration stuff or printing terse info
early_stop_options = [
options.check_github,
options.install_github_token,
options.list_installed_software,
options.list_software,
options.merge_pr,
options.review_pr,
options.terse,
search_query,
]
if any(early_stop_options):
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = modtool.list(
) # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
if options.install_latest_eb_release:
if orig_paths:
raise EasyBuildError(
"Installing the latest EasyBuild release can not be combined with installing "
"other easyconfigs")
else:
eb_file = find_easybuild_easyconfig()
orig_paths.append(eb_file)
categorized_paths = categorize_files_by_type(orig_paths)
# command line options that do not require any easyconfigs to be specified
new_update_preview_pr = options.new_pr or options.update_pr or options.preview_pr
no_ec_opts = [
options.aggregate_regtest, options.regtest, search_query,
new_update_preview_pr
]
# determine paths to easyconfigs
determined_paths = det_easyconfig_paths(categorized_paths['easyconfigs'])
if determined_paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in determined_paths]
else:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs,
robot_path,
try_to_generate,
testing=testing)
elif any(no_ec_opts):
paths = determined_paths
else:
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths]
or easyconfigs_pkg_paths, modtool)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
if options.check_style:
_log.debug("Running style check...")
if cmdline_easyconfigs_style_check([path[0] for path in paths]):
print_msg("All style checks passed!", prefix=False)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
else:
raise EasyBuildError("One or more style checks FAILED!")
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(
paths, validate=not options.inject_checksums)
# verify easyconfig filenames, if desired
if options.verify_easyconfig_filenames:
_log.info("Verifying easyconfig filenames...")
for easyconfig in easyconfigs:
verify_easyconfig_filename(easyconfig['spec'],
easyconfig['ec'],
parsed_ec=easyconfig['ec'])
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs,
build_specs,
modtool,
targetdirs=tweaked_ecs_paths)
if options.containerize:
containerize(easyconfigs)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
forced = options.force or options.rebuild
dry_run_mode = options.dry_run or options.dry_run_short
# skip modules that are already installed unless forced, or unless an option is used that warrants not skipping
if not (forced or dry_run_mode or options.extended_dry_run
or new_update_preview_pr or options.inject_checksums):
retained_ecs = skip_available(easyconfigs, modtool)
if not testing:
for skipped_ec in [
ec for ec in easyconfigs if ec not in retained_ecs
]:
print_msg("%s is already installed (module found), skipping" %
skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
# resolve dependencies if robot is enabled, except in dry run mode
# one exception: deps *are* resolved with --new-pr or --update-pr when dry run mode is enabled
if options.robot and (not dry_run_mode or new_update_preview_pr):
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, modtool)
else:
ordered_ecs = easyconfigs
elif new_update_preview_pr:
ordered_ecs = None
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# creating/updating PRs
if new_update_preview_pr:
if options.new_pr:
new_pr(categorized_paths,
ordered_ecs,
title=options.pr_title,
descr=options.pr_descr,
commit_msg=options.pr_commit_msg)
elif options.preview_pr:
print review_pr(paths=determined_paths,
colored=use_color(options.color))
else:
update_pr(options.update_pr,
categorized_paths,
ordered_ecs,
commit_msg=options.pr_commit_msg)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
elif dry_run_mode:
txt = dry_run(easyconfigs, modtool, short=not options.dry_run)
print_msg(txt, log=_log, silent=testing, prefix=False)
elif options.check_conflicts:
if check_conflicts(easyconfigs, modtool):
print_error("One or more conflicts detected!")
sys.exit(1)
else:
print_msg("\nNo conflicts detected!\n", prefix=False)
# dump source script to set up build environment
elif options.dump_env_script:
dump_env_script(easyconfigs)
elif options.inject_checksums:
inject_checksums(ordered_ecs, options.inject_checksums)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
stop_options = [
options.check_conflicts, dry_run_mode, options.dump_env_script,
options.inject_checksums
]
if any(no_ec_opts) or any(stop_options):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
cleanup(logfile, eb_tmpdir, testing, silent=True)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now")
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
if not testing or (testing and do_build):
exit_on_failure = not (options.dump_test_report
or options.upload_test_report)
hooks = load_hooks(options.hooks)
ecs_with_res = build_and_install_software(
ordered_ecs,
init_session_state,
exit_on_failure=exit_on_failure,
hooks=hooks)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([
ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)
])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt,
len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths),
overall_success, success_msg,
init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)