def parse_easyconfigs(paths, validate=True):
"""
Parse easyconfig files
:param paths: paths to easyconfigs
"""
easyconfigs = []
generated_ecs = False
for (path, generated) in paths:
path = os.path.abspath(path)
# keep track of whether any files were generated
generated_ecs |= generated
if not os.path.exists(path):
raise EasyBuildError("Can't find path %s", path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
for ec_file in ec_files:
kwargs = {'validate': validate}
# only pass build specs when not generating easyconfig files
if not build_option('try_to_generate'):
kwargs['build_specs'] = build_option('build_specs')
easyconfigs.extend(process_easyconfig(ec_file, **kwargs))
except IOError, err:
raise EasyBuildError("Processing easyconfigs in path %s failed: %s", path, err)
def parse_easyconfigs(paths):
"""
Parse easyconfig files
@params paths: paths to easyconfigs
"""
easyconfigs = []
generated_ecs = False
for (path, generated) in paths:
path = os.path.abspath(path)
# keep track of whether any files were generated
generated_ecs |= generated
if not os.path.exists(path):
raise EasyBuildError("Can't find path %s", path)
try:
ec_files = find_easyconfigs(
path, ignore_dirs=build_option('ignore_dirs'))
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
kwargs = {}
if not build_option('try_to_generate'):
kwargs['build_specs'] = build_option('build_specs')
ecs = process_easyconfig(ec_file, **kwargs)
easyconfigs.extend(ecs)
except IOError, err:
raise EasyBuildError(
"Processing easyconfigs in path %s failed: %s", path, err)
def regtest(easyconfig_paths, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
@param easyconfig_paths: path of easyconfigs to run regtest on
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(
aggregate_regtest,
"%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" %
(aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place
# all log files and the test output as xml
basename = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
var = config.OLDSTYLE_ENVIRONMENT_VARIABLES['test_output_path']
regtest_output_dir = build_option('regtest_output_dir')
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif var in os.environ:
output_dir = os.path.abspath(os.environ[var])
else:
# default: current dir + easybuild-test-[timestamp]
output_dir = os.path.join(cur_dir, basename)
mkdir(output_dir, parents=True)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(
path, ignore_dirs=build_option('ignore_dirs'))
else:
_log.error("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(
process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError, err:
test_results.append((ecfile, 'parsing_easyconfigs',
'easyconfig file error: %s' % err, _log))
def regtest(easyconfig_paths, build_options=None, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
@param easyconfig_paths: path of easyconfigs to run regtest on
@param build_options: dictionary specifying build options (e.g. robot_path, check_osdeps, ...)
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_options.get('aggregate_regtest', None)
if aggregate_regtest is not None:
output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place
# all log files and the test output as xml
basename = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
var = config.oldstyle_environment_variables['test_output_path']
regtest_output_dir = build_options.get('regtest_output_dir', None)
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif var in os.environ:
output_dir = os.path.abspath(os.environ[var])
else:
# default: current dir + easybuild-test-[timestamp]
output_dir = os.path.join(cur_dir, basename)
if not os.path.isdir(output_dir):
os.makedirs(output_dir)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(path, ignore_dirs=build_options.get('ignore_dirs', []))
else:
_log.error("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_options=build_options, build_specs=build_specs))
except EasyBuildError, err:
test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
def regtest(easyconfig_paths, modtool, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
@param easyconfig_paths: path of easyconfigs to run regtest on
@param modtool: ModulesTool instance to use
@param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place all log files and the test output as xml
regtest_output_dir = build_option('regtest_output_dir')
testoutput = build_option('testoutput')
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif testoutput is not None:
output_dir = os.path.abspath(testoutput)
else:
# default: current dir + easybuild-test-[timestamp]
dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
output_dir = os.path.join(cur_dir, dirname)
mkdir(output_dir, parents=True)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
else:
raise EasyBuildError("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError, err:
test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
def parse_easyconfigs(paths):
"""
Parse easyconfig files
@params paths: paths to easyconfigs
"""
easyconfigs = []
generated_ecs = False
for (path, generated) in paths:
path = os.path.abspath(path)
# keep track of whether any files were generated
generated_ecs |= generated
if not os.path.exists(path):
_log.error("Can't find path %s" % path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
kwargs = {}
if not build_option('try_to_generate'):
kwargs['build_specs'] = build_option('build_specs')
ecs = process_easyconfig(ec_file, **kwargs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# specified robot paths are preferred over installed easyconfig files
if robot_path:
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest': options.aggregate_regtest,
'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch,
'check_osdeps': not options.ignore_osdeps,
'command_line': eb_command_line,
'debug': options.debug,
'dry_run': options.dry_run,
'easyblock': options.easyblock,
'experimental': options.experimental,
'force': options.force,
'group': options.group,
'ignore_dirs': options.ignore_dirs,
'modules_footer': options.modules_footer,
'only_blocks': options.only_blocks,
'recursive_mod_unload': options.recursive_module_unload,
'regtest_online': options.regtest_online,
'regtest_output_dir': options.regtest_output_dir,
'retain_all_deps': retain_all_deps,
'robot_path': robot_path,
'sequential': options.sequential,
'silent': testing,
'set_gid_bit': options.set_gid_bit,
'skip': options.skip,
'skip_test_cases': options.skip_test_cases,
'sticky_bit': options.sticky_bit,
'stop': options.stop,
'umask': options.umask,
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate': not options.force,
})
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if 'name' in build_specs:
paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate,
exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for f in files:
if not generated and try_to_generate and build_specs:
ec_file = tweak(f, None, build_specs)
else:
ec_file = f
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
def regtest(easyconfig_paths, modtool, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
:param easyconfig_paths: path of easyconfigs to run regtest on
:param modtool: ModulesTool instance to use
:param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(
aggregate_regtest,
"%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" %
(aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place all log files and the test output as xml
regtest_output_dir = build_option('regtest_output_dir')
testoutput = build_option('testoutput')
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif testoutput is not None:
output_dir = os.path.abspath(testoutput)
else:
# default: current dir + easybuild-test-[timestamp]
dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
output_dir = os.path.join(cur_dir, dirname)
mkdir(output_dir, parents=True)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(
path, ignore_dirs=build_option('ignore_dirs'))
else:
raise EasyBuildError("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(
process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError as err:
test_results.append((ecfile, 'parsing_easyconfigs',
'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug(
"Skipping easyconfigs from %s that already have a module available..."
% easyconfigs)
easyconfigs = skip_available(easyconfigs, modtool)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, modtool)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {
'cmd': cmd
}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" %
output_dir)
return True # success
go = FixBrokenEasyconfigsOption(options)
log = go.log
fancylogger.logToScreen(enable=True, stdout=True)
fancylogger.setLogLevel('WARNING')
try:
import easybuild.easyblocks.generic.configuremake
except ImportError, err:
raise EasyBuildError(
"easyblocks are not available in Python search path: %s", err)
for path in go.args:
if not os.path.exists(path):
raise EasyBuildError("Non-existing path %s specified", path)
ec_files = [ec for p in go.args for ec in find_easyconfigs(p)]
if not ec_files:
raise EasyBuildError("No easyconfig files specified")
log.info("Processing %d easyconfigs" % len(ec_files))
for ec_file in ec_files:
try:
process_easyconfig_file(ec_file)
except EasyBuildError, err:
log.warning("Ignoring issue when processing %s: %s", ec_file, err)
except EasyBuildError, err:
sys.stderr.write("ERROR: %s\n" % err)
sys.exit(1)
}
go = FixBrokenEasyconfigsOption(options)
log = go.log
fancylogger.logToScreen(enable=True, stdout=True)
fancylogger.setLogLevel('WARNING')
try:
import easybuild.easyblocks.generic.configuremake
except ImportError, err:
raise EasyBuildError("easyblocks are not available in Python search path: %s", err)
for path in go.args:
if not os.path.exists(path):
raise EasyBuildError("Non-existing path %s specified", path)
ec_files = [ec for p in go.args for ec in find_easyconfigs(p)]
if not ec_files:
raise EasyBuildError("No easyconfig files specified")
log.info("Processing %d easyconfigs" % len(ec_files))
for ec_file in ec_files:
try:
process_easyconfig_file(ec_file)
except EasyBuildError, err:
log.warning("Ignoring issue when processing %s: %s", ec_file, err)
except EasyBuildError, err:
sys.stderr.write("ERROR: %s\n" % err)
sys.exit(1)
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
eb_config = eb_go.generate_cmd_line(add_default=True)
init_session_state.update({'easybuild_configuration': eb_config})
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile),
log=None,
silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error(
"No robot paths specified, and unable to determine easybuild-easyconfigs install path."
)
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# specified robot paths are preferred over installed easyconfig files
# --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs
if robot_path or try_to_generate or options.dep_graph:
if robot_path is None:
robot_path = []
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info(
"Extended list of robot paths with paths for installed easyconfigs: %s"
% robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest':
options.aggregate_regtest,
'allow_modules_tool_mismatch':
options.allow_modules_tool_mismatch,
'check_osdeps':
not options.ignore_osdeps,
'cleanup_builddir':
options.cleanup_builddir,
'command_line':
eb_command_line,
'debug':
options.debug,
'dry_run':
options.dry_run,
'easyblock':
options.easyblock,
'experimental':
options.experimental,
'force':
options.force,
'github_user':
options.github_user,
'group':
options.group,
'ignore_dirs':
options.ignore_dirs,
'modules_footer':
options.modules_footer,
'only_blocks':
options.only_blocks,
'recursive_mod_unload':
options.recursive_module_unload,
'regtest_output_dir':
options.regtest_output_dir,
'retain_all_deps':
retain_all_deps,
'robot_path':
robot_path,
'sequential':
options.sequential,
'silent':
testing,
'set_gid_bit':
options.set_gid_bit,
'skip':
options.skip,
'skip_test_cases':
options.skip_test_cases,
'sticky_bit':
options.sticky_bit,
'stop':
options.stop,
'umask':
options.umask,
'valid_module_classes':
module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate':
not options.force,
})
# obtain list of loaded modules, build options must be initialized first
modlist = session_module_list()
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path,
query,
short=not options.search,
ignore_dirs=ignore_dirs,
silent=silent)
paths = []
if len(orig_paths) == 0:
if options.from_pr:
pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr)
pr_files = fetch_easyconfigs_from_pr(
options.from_pr, path=pr_path, github_user=options.github_user)
paths = [(path, False) for path in pr_files
if path.endswith('.eb')]
elif 'name' in build_specs:
paths = [
obtain_path(build_specs,
easyconfigs_paths,
try_to_generate=try_to_generate,
exit_on_error=not testing)
]
elif not any([
options.aggregate_regtest, options.search,
options.search_short, options.regtest
]):
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(
orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug(
"Looking for missing easyconfig files (%d left) in %s..." %
(len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path,
topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" %
(orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [
d for d in dirnames if not d in options.ignore_dirs
]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
if try_to_generate:
ecs = process_easyconfig(ec_file)
else:
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" %
(path, err))
def regtest(easyconfig_paths, modtool, build_specs=None):
"""
Run regression test, using easyconfigs available in given path
:param easyconfig_paths: path of easyconfigs to run regtest on
:param modtool: ModulesTool instance to use
:param build_specs: dictionary specifying build specifications (e.g. version, toolchain, ...)
"""
cur_dir = os.getcwd()
aggregate_regtest = build_option('aggregate_regtest')
if aggregate_regtest is not None:
output_file = os.path.join(aggregate_regtest, "%s-aggregate.xml" % os.path.basename(aggregate_regtest))
aggregate_xml_in_dirs(aggregate_regtest, output_file)
_log.info("aggregated xml files inside %s, output written to: %s" % (aggregate_regtest, output_file))
sys.exit(0)
# create base directory, which is used to place all log files and the test output as xml
regtest_output_dir = build_option('regtest_output_dir')
testoutput = build_option('testoutput')
if regtest_output_dir is not None:
output_dir = regtest_output_dir
elif testoutput is not None:
output_dir = os.path.abspath(testoutput)
else:
# default: current dir + easybuild-test-[timestamp]
dirname = "easybuild-test-%s" % datetime.now().strftime("%Y%m%d%H%M%S")
output_dir = os.path.join(cur_dir, dirname)
mkdir(output_dir, parents=True)
# find all easyconfigs
ecfiles = []
if easyconfig_paths:
for path in easyconfig_paths:
ecfiles += find_easyconfigs(path, ignore_dirs=build_option('ignore_dirs'))
else:
raise EasyBuildError("No easyconfig paths specified.")
test_results = []
# process all the found easyconfig files
easyconfigs = []
for ecfile in ecfiles:
try:
easyconfigs.extend(process_easyconfig(ecfile, build_specs=build_specs))
except EasyBuildError as err:
test_results.append((ecfile, 'parsing_easyconfigs', 'easyconfig file error: %s' % err, _log))
# skip easyconfigs for which a module is already available, unless forced
if not build_option('force'):
_log.debug("Skipping easyconfigs from %s that already have a module available..." % easyconfigs)
easyconfigs = skip_available(easyconfigs, modtool)
_log.debug("Retained easyconfigs after skipping: %s" % easyconfigs)
if build_option('sequential'):
return build_easyconfigs(easyconfigs, output_dir, test_results)
else:
resolved = resolve_dependencies(easyconfigs, modtool)
cmd = "eb %(spec)s --regtest --sequential -ld --testoutput=%(output_dir)s"
command = "unset TMPDIR && cd %s && %s; " % (cur_dir, cmd)
# retry twice in case of failure, to avoid fluke errors
command += "if [ $? -ne 0 ]; then %(cmd)s --force && %(cmd)s --force; fi" % {'cmd': cmd}
build_easyconfigs_in_parallel(command, resolved, output_dir=output_dir)
_log.info("Submitted regression test as jobs, results in %s" % output_dir)
return True # success
