def suite():
"""Return all easyblock --module-only tests."""
# initialize configuration (required for e.g. default modules_tool setting)
cleanup()
eb_go = eboptions.parse_options(args=['--prefix=%s' % TMPDIR])
config.init(eb_go.options, eb_go.get_options_by_section('config'))
build_options = {
# enable --force --module-only
'force': True,
'module_only': True,
'silent': True,
'suffix_modules_path': GENERAL_CLASS,
'valid_module_classes': config.module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
config.init_build_options(build_options=build_options)
config.set_tmpdir()
# dynamically generate a separate test for each of the available easyblocks
easyblocks_path = get_paths_for("easyblocks")[0]
all_pys = glob.glob('%s/*/*.py' % easyblocks_path)
easyblocks = [eb for eb in all_pys if os.path.basename(eb) != '__init__.py' and '/test/' not in eb]
# filter out no longer supported easyblocks
easyblocks = [e for e in easyblocks if os.path.basename(e) not in ['versionindependendpythonpackage.py']]
for easyblock in easyblocks:
# dynamically define new inner functions that can be added as class methods to ModuleOnlyTest
exec("def innertest(self): template_module_only_test(self, '%s')" % easyblock)
innertest.__doc__ = "Test for using --module-only with easyblock %s" % easyblock
innertest.__name__ = "test_easyblock_%s" % '_'.join(easyblock.replace('.py', '').split('/'))
setattr(ModuleOnlyTest, innertest.__name__, innertest)
return TestLoader().loadTestsFromTestCase(ModuleOnlyTest)
def suite():
"""Return all easyblock --module-only tests."""
# initialize configuration (required for e.g. default modules_tool setting)
cleanup()
eb_go = eboptions.parse_options(args=['--prefix=%s' % TMPDIR])
config.init(eb_go.options, eb_go.get_options_by_section('config'))
build_options = {
# enable --force --module-only
'force': True,
'module_only': True,
'silent': True,
'suffix_modules_path': GENERAL_CLASS,
'valid_module_classes': config.module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
config.init_build_options(build_options=build_options)
config.set_tmpdir()
# dynamically generate a separate test for each of the available easyblocks
easyblocks_path = get_paths_for("easyblocks")[0]
all_pys = glob.glob('%s/*/*.py' % easyblocks_path)
easyblocks = [
eb for eb in all_pys
if os.path.basename(eb) != '__init__.py' and '/test/' not in eb
]
# filter out no longer supported easyblocks, or easyblocks that are tested in a different way
excluded_easyblocks = ['versionindependendpythonpackage.py']
easyblocks = [
e for e in easyblocks if os.path.basename(e) not in excluded_easyblocks
]
for easyblock in easyblocks:
# dynamically define new inner functions that can be added as class methods to ModuleOnlyTest
if os.path.basename(easyblock) == 'systemcompiler.py':
# use GCC as name when testing SystemCompiler easyblock
exec(
"def innertest(self): template_module_only_test(self, '%s', name='GCC', version='system')"
% easyblock)
else:
exec("def innertest(self): template_module_only_test(self, '%s')" %
easyblock)
innertest.__doc__ = "Test for using --module-only with easyblock %s" % easyblock
innertest.__name__ = "test_easyblock_%s" % '_'.join(
easyblock.replace('.py', '').split('/'))
setattr(ModuleOnlyTest, innertest.__name__, innertest)
return TestLoader().loadTestsFromTestCase(ModuleOnlyTest)
def test_set_tmpdir(self):
"""Test set_tmpdir config function."""
self.purge_environment()
for tmpdir in [None, os.path.join(tempfile.gettempdir(), 'foo')]:
parent = tmpdir
if parent is None:
parent = tempfile.gettempdir()
mytmpdir = set_tmpdir(tmpdir=tmpdir)
for var in ['TMPDIR', 'TEMP', 'TMP']:
self.assertTrue(os.environ[var].startswith(
os.path.join(parent, 'easybuild-')))
self.assertEqual(os.environ[var], mytmpdir)
self.assertTrue(tempfile.gettempdir().startswith(
os.path.join(parent, 'easybuild-')))
tempfile_tmpdir = tempfile.mkdtemp()
self.assertTrue(
tempfile_tmpdir.startswith(os.path.join(parent, 'easybuild-')))
fd, tempfile_tmpfile = tempfile.mkstemp()
self.assertTrue(
tempfile_tmpfile.startswith(os.path.join(parent,
'easybuild-')))
# cleanup
os.close(fd)
shutil.rmtree(mytmpdir)
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
def test_set_tmpdir(self):
"""Test set_tmpdir config function."""
self.purge_environment()
for tmpdir in [None, os.path.join(tempfile.gettempdir(), 'foo')]:
parent = tmpdir
if parent is None:
parent = tempfile.gettempdir()
mytmpdir = set_tmpdir(tmpdir=tmpdir)
for var in ['TMPDIR', 'TEMP', 'TMP']:
self.assertTrue(os.environ[var].startswith(os.path.join(parent, 'easybuild-')))
self.assertEqual(os.environ[var], mytmpdir)
self.assertTrue(tempfile.gettempdir().startswith(os.path.join(parent, 'easybuild-')))
tempfile_tmpdir = tempfile.mkdtemp()
self.assertTrue(tempfile_tmpdir.startswith(os.path.join(parent, 'easybuild-')))
fd, tempfile_tmpfile = tempfile.mkstemp()
self.assertTrue(tempfile_tmpfile.startswith(os.path.join(parent, 'easybuild-')))
# cleanup
os.close(fd)
shutil.rmtree(mytmpdir)
modify_env(os.environ, self.orig_environ)
tempfile.tempdir = None
class EasyConfigTest(TestCase):
"""Baseclass for easyconfig testcases."""
if LooseVersion(sys.version) >= LooseVersion('2.6'):
os.environ['EASYBUILD_DEPRECATED'] = '2.0'
# initialize configuration (required for e.g. default modules_tool setting)
eb_go = eboptions.parse_options()
config.init(eb_go.options, eb_go.get_options_by_section('config'))
build_options = {
'check_osdeps': False,
'force': True,
'robot_path': get_paths_for("easyconfigs")[0],
'suffix_modules_path': GENERAL_CLASS,
'valid_module_classes': config.module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
config.init_build_options(build_options=build_options)
config.set_tmpdir()
del eb_go
log = fancylogger.getLogger("EasyConfigTest", fname=False)
# make sure a logger is present for main
main._log = log
ordered_specs = None
parsed_easyconfigs = []
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs if they haven't been already
if not self.parsed_easyconfigs:
for spec in specs:
self.parsed_easyconfigs.extend(process_easyconfig(spec))
self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs)
def test_dep_graph(self):
"""Unit test that builds a full dependency graph."""
# pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6
if LooseVersion(
sys.version) >= LooseVersion('2.6') and single_tests_ok:
# temporary file for dep graph
(hn, fn) = tempfile.mkstemp(suffix='.dot')
os.close(hn)
if self.ordered_specs is None:
self.process_all_easyconfigs()
dep_graph(fn, self.ordered_specs, silent=True)
try:
os.remove(fn)
except OSError, err:
log.error("Failed to remove %s: %s" % (fn, err))
else:
def postprocess(self):
"""Do some postprocessing, in particular print stuff"""
build_log.EXPERIMENTAL = self.options.experimental
# set strictness of run module
if self.options.strict:
run.strictness = self.options.strict
# override current version of EasyBuild with version specified to --deprecated
if self.options.deprecated:
build_log.CURRENT_VERSION = LooseVersion(self.options.deprecated)
# log to specified value of --unittest-file
if self.options.unittest_file:
fancylogger.logToFile(self.options.unittest_file)
# set tmpdir
self.tmpdir = set_tmpdir(self.options.tmpdir)
# take --include options into account
self._postprocess_include()
# prepare for --list/--avail
if any([self.options.avail_easyconfig_params, self.options.avail_easyconfig_templates,
self.options.list_easyblocks, self.options.list_toolchains, self.options.avail_cfgfile_constants,
self.options.avail_easyconfig_constants, self.options.avail_easyconfig_licenses,
self.options.avail_repositories, self.options.show_default_moduleclasses,
self.options.avail_modules_tools, self.options.avail_module_naming_schemes,
self.options.show_default_configfiles,
]):
build_easyconfig_constants_dict() # runs the easyconfig constants sanity check
self._postprocess_list_avail()
# fail early if required dependencies for functionality requiring using GitHub API are not available:
if self.options.from_pr or self.options.upload_test_report:
if not HAVE_GITHUB_API:
raise EasyBuildError("Required support for using GitHub API is not available (see warnings).")
if self.options.module_syntax == ModuleGeneratorLua.SYNTAX and self.options.modules_tool != Lmod.__name__:
raise EasyBuildError("Generating Lua module files requires Lmod as modules tool.")
# make sure a GitHub token is available when it's required
if self.options.upload_test_report:
if not HAVE_KEYRING:
raise EasyBuildError("Python 'keyring' module required for obtaining GitHub token is not available.")
if self.options.github_user is None:
raise EasyBuildError("No GitHub user name provided, required for fetching GitHub token.")
token = fetch_github_token(self.options.github_user)
if token is None:
raise EasyBuildError("Failed to obtain required GitHub token for user '%s'", self.options.github_user)
# make sure autopep8 is available when it needs to be
if self.options.dump_autopep8:
if not HAVE_AUTOPEP8:
raise EasyBuildError("Python 'autopep8' module required to reformat dumped easyconfigs as requested")
self._postprocess_external_modules_metadata()
self._postprocess_config()
class EasyConfigTest(TestCase):
"""Baseclass for easyconfig testcases."""
# initialize configuration (required for e.g. default modules_tool setting)
eb_go = eboptions.parse_options()
config.init(eb_go.options, eb_go.get_options_by_section('config'))
config.set_tmpdir()
del eb_go
log = fancylogger.getLogger("EasyConfigTest", fname=False)
name_regex = re.compile("^name\s*=\s*['\"](.*)['\"]$", re.M)
easyblock_regex = re.compile(r"^\s*easyblock\s*=['\"](.*)['\"]$", re.M)
# make sure a logger is present for main
main._log = log
ordered_specs = None
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs
easyconfigs = []
for spec in specs:
easyconfigs.extend(
process_easyconfig(spec, build_options={'validate': False}))
build_options = {
'robot_path': easyconfigs_path,
'force': True,
}
self.ordered_specs = resolve_dependencies(easyconfigs,
build_options=build_options)
def test_dep_graph(self):
"""Unit test that builds a full dependency graph."""
# pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6
if LooseVersion(
sys.version) >= LooseVersion('2.6') and single_tests_ok:
# temporary file for dep graph
(hn, fn) = tempfile.mkstemp(suffix='.dot')
os.close(hn)
if self.ordered_specs is None:
self.process_all_easyconfigs()
dep_graph(fn, self.ordered_specs, silent=True)
try:
os.remove(fn)
except OSError, err:
log.error("Failed to remove %s: %s" % (fn, err))
else:
def suite():
"""Return all easyblock --module-only tests."""
# initialize configuration (required for e.g. default modules_tool setting)
cleanup()
eb_go = eboptions.parse_options(args=["--prefix=%s" % TMPDIR])
config.init(eb_go.options, eb_go.get_options_by_section("config"))
build_options = {
# enable --force --module-only
"force": True,
"module_only": True,
"silent": True,
"suffix_modules_path": GENERAL_CLASS,
"valid_module_classes": config.module_classes(),
"valid_stops": [x[0] for x in EasyBlock.get_steps()],
}
config.init_build_options(build_options=build_options)
config.set_tmpdir()
# dynamically generate a separate test for each of the available easyblocks
easyblocks_path = get_paths_for("easyblocks")[0]
all_pys = glob.glob("%s/*/*.py" % easyblocks_path)
easyblocks = [eb for eb in all_pys if os.path.basename(eb) != "__init__.py" and "/test/" not in eb]
# filter out no longer supported easyblocks, or easyblocks that are tested in a different way
excluded_easyblocks = ["versionindependendpythonpackage.py"]
easyblocks = [e for e in easyblocks if os.path.basename(e) not in excluded_easyblocks]
for easyblock in easyblocks:
# dynamically define new inner functions that can be added as class methods to ModuleOnlyTest
if os.path.basename(easyblock) == "systemcompiler.py":
# use GCC as name when testing SystemCompiler easyblock
exec("def innertest(self): template_module_only_test(self, '%s', name='GCC', version='system')" % easyblock)
else:
exec("def innertest(self): template_module_only_test(self, '%s')" % easyblock)
innertest.__doc__ = "Test for using --module-only with easyblock %s" % easyblock
innertest.__name__ = "test_easyblock_%s" % "_".join(easyblock.replace(".py", "").split("/"))
setattr(ModuleOnlyTest, innertest.__name__, innertest)
return TestLoader().loadTestsFromTestCase(ModuleOnlyTest)
class InitTest(TestCase):
""" Baseclass for easyblock testcases """
# initialize configuration (required for e.g. default modules_tool setting)
eb_go = eboptions.parse_options()
config.init(eb_go.options, eb_go.get_options_by_section('config'))
build_options = {
'suffix_modules_path': GENERAL_CLASS,
'valid_module_classes': config.module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
config.init_build_options(build_options=build_options)
config.set_tmpdir()
del eb_go
def writeEC(self, easyblock, extratxt=''):
""" create temporary easyconfig file """
txt = '\n'.join([
'easyblock = "%s"',
'name = "foo"',
'version = "1.3.2"',
'homepage = "http://example.com"',
'description = "Dummy easyconfig file."',
'toolchain = {"name": "dummy", "version": "dummy"}',
'sources = []',
extratxt,
])
f = open(self.eb_file, "w")
f.write(txt % easyblock)
f.close()
def setUp(self):
"""Setup test."""
self.log = fancylogger.getLogger("EasyblocksInitTest", fname=False)
fd, self.eb_file = tempfile.mkstemp(prefix='easyblocks_init_test_',
suffix='.eb')
os.close(fd)
def tearDown(self):
"""Cleanup."""
try:
os.remove(self.eb_file)
except OSError, err:
self.log.error("Failed to remove %s/%s: %s" % (self.eb_file, err))
import test.framework.parallelbuild as p
import test.framework.repository as r
import test.framework.robot as robot
import test.framework.run as run
import test.framework.scripts as sc
import test.framework.systemtools as s
import test.framework.toolchain as tc
import test.framework.toolchainvariables as tcv
import test.framework.toy_build as t
import test.framework.tweak as tw
import test.framework.variables as v
# make sure temporary files can be created/used
try:
set_tmpdir(raise_error=True)
except EasyBuildError, err:
sys.stderr.write("No execution rights on temporary files, specify another location via $TMPDIR: %s\n" % err)
sys.exit(1)
# initialize logger for all the unit tests
fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log')
os.close(fd)
os.remove(log_fn)
fancylogger.logToFile(log_fn)
log = fancylogger.getLogger()
# call suite() for each module and then run them all
# note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
tests = [gen, o, r, ef, ev, ebco, ep, e, mg, m, mt, f, run, a, robot, b, v, g, tcv, tc, t, c, s, l, f_c, sc, tw, p]
def setUp(self):
"""Set up testcase."""
super(EnhancedTestCase, self).setUp()
# keep track of log handlers
log = fancylogger.getLogger(fname=False)
self.orig_log_handlers = log.handlers[:]
self.orig_tmpdir = tempfile.gettempdir()
# use a subdirectory for this test (which we can clean up easily after the test completes)
self.test_prefix = set_tmpdir()
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
self.cwd = os.getcwd()
# keep track of original environment to restore
self.orig_environ = copy.deepcopy(os.environ)
# keep track of original environment/Python search path to restore
self.orig_sys_path = sys.path[:]
testdir = os.path.dirname(os.path.abspath(__file__))
self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources')
os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath
os.environ['EASYBUILD_PREFIX'] = self.test_prefix
self.test_buildpath = tempfile.mkdtemp()
os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath
self.test_installpath = tempfile.mkdtemp()
os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath
# make sure that the tests only pick up easyconfigs provided with the tests
os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(testdir, 'easyconfigs')
# make sure no deprecated behaviour is being triggered (unless intended by the test)
# trip *all* log.deprecated statements by setting deprecation version ridiculously high
self.orig_current_version = eb_build_log.CURRENT_VERSION
os.environ['EASYBUILD_DEPRECATED'] = '10000000'
init_config()
# remove any entries in Python search path that seem to provide easyblocks
for path in sys.path[:]:
if os.path.exists(os.path.join(path, 'easybuild', 'easyblocks', '__init__.py')):
sys.path.remove(path)
# add test easyblocks to Python search path and (re)import and reload easybuild modules
import easybuild
sys.path.append(os.path.join(testdir, 'sandbox'))
reload(easybuild)
import easybuild.easyblocks
reload(easybuild.easyblocks)
import easybuild.easyblocks.generic
reload(easybuild.easyblocks.generic)
reload(easybuild.tools.module_naming_scheme) # required to run options unit tests stand-alone
modtool = modules_tool()
self.reset_modulepath([os.path.join(testdir, 'modules')])
# purge out any loaded modules with original $MODULEPATH before running each test
modtool.purge()
class EasyConfigTest(TestCase):
"""Baseclass for easyconfig testcases."""
# initialize configuration (required for e.g. default modules_tool setting)
eb_go = eboptions.parse_options()
config.init(eb_go.options, eb_go.get_options_by_section('config'))
build_options = {
'check_osdeps': False,
'external_modules_metadata': {},
'force': True,
'optarch': 'test',
'robot_path': get_paths_for("easyconfigs")[0],
'suffix_modules_path': GENERAL_CLASS,
'valid_module_classes': config.module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
config.init_build_options(build_options=build_options)
config.set_tmpdir()
del eb_go
# put dummy 'craype-test' module in place, which is required for parsing easyconfigs using Cray* toolchains
TMPDIR = tempfile.mkdtemp()
os.environ['MODULEPATH'] = TMPDIR
write_file(os.path.join(TMPDIR, 'craype-test'), '#%Module\n')
log = fancylogger.getLogger("EasyConfigTest", fname=False)
# make sure a logger is present for main
main._log = log
ordered_specs = None
parsed_easyconfigs = []
def process_all_easyconfigs(self):
"""Process all easyconfigs and resolve inter-easyconfig dependencies."""
# all available easyconfig files
easyconfigs_path = get_paths_for("easyconfigs")[0]
specs = glob.glob('%s/*/*/*.eb' % easyconfigs_path)
# parse all easyconfigs if they haven't been already
if not self.parsed_easyconfigs:
for spec in specs:
self.parsed_easyconfigs.extend(process_easyconfig(spec))
# filter out external modules
for ec in self.parsed_easyconfigs:
for dep in ec['dependencies'][:]:
if dep.get('external_module', False):
ec['dependencies'].remove(dep)
for dep in ec['unresolved_deps'][:]:
if dep.get('external_module', False):
ec['unresolved_deps'].remove(dep)
self.ordered_specs = resolve_dependencies(self.parsed_easyconfigs, retain_all_deps=True)
def test_dep_graph(self):
"""Unit test that builds a full dependency graph."""
# pygraph dependencies required for constructing dependency graph are not available prior to Python 2.6
if LooseVersion(sys.version) >= LooseVersion('2.6') and single_tests_ok:
# temporary file for dep graph
(hn, fn) = tempfile.mkstemp(suffix='.dot')
os.close(hn)
if self.ordered_specs is None:
self.process_all_easyconfigs()
dep_graph(fn, self.ordered_specs, silent=True)
try:
os.remove(fn)
except OSError, err:
log.error("Failed to remove %s: %s" % (fn, err))
else:
def main(testing_data=(None, None, None)):
"""
Main function: parse command line options, and act accordingly.
@param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
global _log
_log, logfile = init_logging(logfile,
logtostdout=options.logtostdout,
testing=testing)
# disallow running EasyBuild as root
if os.getuid() == 0:
raise EasyBuildError(
"You seem to be running EasyBuild with root privileges which is not wise, "
"so let's end this here.")
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir,
tweaked_ecs=tweaked_ecs,
from_pr=options.from_pr)
auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short
robot_path = det_robot_path(options.robot_paths,
tweaked_ecs_path,
pr_path,
auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section('config')
build_options = {
'build_specs': build_specs,
'command_line': eb_cmd_line,
'pr_path': pr_path,
'robot_path': robot_path,
'silent': testing,
'try_to_generate': try_to_generate,
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options,
cmdline_options=options)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = session_module_list(
testing=testing
) # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs, if a query is specified
query = options.search or options.search_short
if query:
search_easyconfigs(query, short=not options.search)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
# determine paths to easyconfigs
paths = det_easyconfig_paths(orig_paths)
if paths:
# transform paths into tuples, use 'False' to indicate the corresponding easyconfig files were not generated
paths = [(p, False) for p in paths]
else:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs,
robot_path,
try_to_generate,
testing=testing)
elif not any([
options.aggregate_regtest, options.search,
options.search_short, options.regtest
]):
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths]
or easyconfigs_pkg_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths)
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs,
build_specs,
targetdir=tweaked_ecs_path)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
txt = dry_run(easyconfigs,
short=not options.dry_run,
build_specs=build_specs)
print_msg(txt, log=_log, silent=testing, prefix=False)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
if any([
options.dry_run, options.dry_run_short, options.regtest,
options.search, options.search_short
]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not options.force:
retained_ecs = skip_available(easyconfigs)
if not testing:
for skipped_ec in [
ec for ec in easyconfigs if ec not in retained_ecs
]:
print_msg("%s is already installed (module found), skipping" %
skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
if options.robot:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs,
build_specs=build_specs)
else:
ordered_ecs = easyconfigs
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
job_info_txt = submit_jobs(ordered_ecs,
eb_go.generate_cmd_line(),
testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now: %s" %
job_info_txt)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(
ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([
ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)
])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt,
len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths),
overall_success, success_msg,
init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def setUp(self):
"""Set up testcase."""
super(EnhancedTestCase, self).setUp()
# keep track of log handlers
log = fancylogger.getLogger(fname=False)
self.orig_log_handlers = log.handlers[:]
self.orig_tmpdir = tempfile.gettempdir()
# use a subdirectory for this test (which we can clean up easily after the test completes)
self.test_prefix = set_tmpdir()
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
fd, self.logfile = tempfile.mkstemp(suffix='.log', prefix='eb-test-')
os.close(fd)
self.cwd = os.getcwd()
# keep track of original environment to restore
self.orig_environ = copy.deepcopy(os.environ)
# keep track of original environment/Python search path to restore
self.orig_sys_path = sys.path[:]
testdir = os.path.dirname(os.path.abspath(__file__))
self.test_sourcepath = os.path.join(testdir, 'sandbox', 'sources')
os.environ['EASYBUILD_SOURCEPATH'] = self.test_sourcepath
os.environ['EASYBUILD_PREFIX'] = self.test_prefix
self.test_buildpath = tempfile.mkdtemp()
os.environ['EASYBUILD_BUILDPATH'] = self.test_buildpath
self.test_installpath = tempfile.mkdtemp()
os.environ['EASYBUILD_INSTALLPATH'] = self.test_installpath
# make sure that the tests only pick up easyconfigs provided with the tests
os.environ['EASYBUILD_ROBOT_PATHS'] = os.path.join(
testdir, 'easyconfigs')
# make sure no deprecated behaviour is being triggered (unless intended by the test)
# trip *all* log.deprecated statements by setting deprecation version ridiculously high
self.orig_current_version = eb_build_log.CURRENT_VERSION
os.environ['EASYBUILD_DEPRECATED'] = '10000000'
init_config()
# remove any entries in Python search path that seem to provide easyblocks
for path in sys.path[:]:
if os.path.exists(
os.path.join(path, 'easybuild', 'easyblocks',
'__init__.py')):
sys.path.remove(path)
# add test easyblocks to Python search path and (re)import and reload easybuild modules
import easybuild
sys.path.append(os.path.join(testdir, 'sandbox'))
reload(easybuild)
import easybuild.easyblocks
reload(easybuild.easyblocks)
import easybuild.easyblocks.generic
reload(easybuild.easyblocks.generic)
reload(easybuild.tools.module_naming_scheme
) # required to run options unit tests stand-alone
modtool = modules_tool()
# purge out any loaded modules with original $MODULEPATH before running each test
modtool.purge()
self.reset_modulepath([os.path.join(testdir, 'modules')])
def main(testing_data=(None, None, None)):
"""
Main function: parse command line options, and act accordingly.
@param testing_data: tuple with command line arguments, log file and boolean indicating whether or not to build
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
global _log
_log, logfile = init_logging(logfile, logtostdout=options.logtostdout, testing=testing)
# disallow running EasyBuild as root
if os.getuid() == 0:
_log.error("You seem to be running EasyBuild with root privileges which is not wise, so let's end this here.")
# log startup info
eb_cmd_line = eb_go.generate_cmd_line() + eb_go.args
log_start(eb_cmd_line, eb_tmpdir)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# determine robot path
# --try-X, --dep-graph, --search use robot path for searching, so enable it with path of installed easyconfigs
tweaked_ecs = try_to_generate and build_specs
tweaked_ecs_path, pr_path = alt_easyconfig_paths(eb_tmpdir, tweaked_ecs=tweaked_ecs, from_pr=options.from_pr)
auto_robot = try_to_generate or options.dep_graph or options.search or options.search_short
robot_path = det_robot_path(options.robot_paths, tweaked_ecs_path, pr_path, auto_robot=auto_robot)
_log.debug("Full robot path: %s" % robot_path)
# configure & initialize build options
config_options_dict = eb_go.get_options_by_section('config')
build_options = {
'build_specs': build_specs,
'command_line': eb_cmd_line,
'pr_path': pr_path,
'robot_path': robot_path,
'silent': testing,
'try_to_generate': try_to_generate,
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
}
# initialise the EasyBuild configuration & build options
config.init(options, config_options_dict)
config.init_build_options(build_options=build_options, cmdline_options=options)
# update session state
eb_config = eb_go.generate_cmd_line(add_default=True)
modlist = session_module_list(testing=testing) # build options must be initialized first before 'module list' works
init_session_state.update({'easybuild_configuration': eb_config})
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs, if a query is specified
query = options.search or options.search_short
if query:
search_easyconfigs(query, short=not options.search)
# determine easybuild-easyconfigs package install path
easyconfigs_pkg_paths = get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR)
if not easyconfigs_pkg_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# determine paths to easyconfigs
paths = det_easyconfig_paths(orig_paths, options.from_pr, easyconfigs_pkg_paths)
if not paths:
if 'name' in build_specs:
# try to obtain or generate an easyconfig file via build specifications if a software name is provided
paths = find_easyconfigs_by_specs(build_specs, robot_path, try_to_generate, testing=testing)
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
# fallback: easybuild-easyconfigs install path
regtest_ok = regtest([path[0] for path in paths] or easyconfigs_pkg_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs, generated_ecs = parse_easyconfigs(paths)
# tweak obtained easyconfig files, if requested
# don't try and tweak anything if easyconfigs were generated, since building a full dep graph will fail
# if easyconfig files for the dependencies are not available
if try_to_generate and build_specs and not generated_ecs:
easyconfigs = tweak(easyconfigs, build_specs, targetdir=tweaked_ecs_path)
# dry_run: print all easyconfigs and dependencies, and whether they are already built
if options.dry_run or options.dry_run_short:
txt = dry_run(easyconfigs, short=not options.dry_run, build_specs=build_specs)
print_msg(txt, log=_log, silent=testing, prefix=False)
# cleanup and exit after dry run, searching easyconfigs or submitting regression test
if any([options.dry_run, options.dry_run_short, options.regtest, options.search, options.search_short]):
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# skip modules that are already installed unless forced
if not options.force:
retained_ecs = skip_available(easyconfigs)
if not testing:
for skipped_ec in [ec for ec in easyconfigs if ec not in retained_ecs]:
print_msg("%s is already installed (module found), skipping" % skipped_ec['full_mod_name'])
easyconfigs = retained_ecs
# determine an order that will allow all specs in the set to build
if len(easyconfigs) > 0:
if options.robot:
print_msg("resolving dependencies ...", log=_log, silent=testing)
ordered_ecs = resolve_dependencies(easyconfigs, build_specs=build_specs)
else:
ordered_ecs = easyconfigs
else:
print_msg("No easyconfigs left to be built.", log=_log, silent=testing)
ordered_ecs = []
# create dependency graph and exit
if options.dep_graph:
_log.info("Creating dependency graph %s" % options.dep_graph)
dep_graph(options.dep_graph, ordered_ecs)
sys.exit(0)
# submit build as job(s), clean up and exit
if options.job:
job_info_txt = submit_jobs(ordered_ecs, eb_go.generate_cmd_line(), testing=testing)
if not testing:
print_msg("Submitted parallel build jobs, exiting now: %s" % job_info_txt)
cleanup(logfile, eb_tmpdir, testing)
sys.exit(0)
# build software, will exit when errors occurs (except when testing)
exit_on_failure = not options.dump_test_report and not options.upload_test_report
if not testing or (testing and do_build):
ecs_with_res = build_and_install_software(ordered_ecs, init_session_state, exit_on_failure=exit_on_failure)
else:
ecs_with_res = [(ec, {}) for ec in ordered_ecs]
correct_builds_cnt = len([ec_res for (_, ec_res) in ecs_with_res if ec_res.get('success', False)])
overall_success = correct_builds_cnt == len(ordered_ecs)
success_msg = "Build succeeded for %s out of %s" % (correct_builds_cnt, len(ordered_ecs))
repo = init_repository(get_repository(), get_repositorypath())
repo.cleanup()
# dump/upload overall test report
test_report_msg = overall_test_report(ecs_with_res, len(paths), overall_success, success_msg, init_session_state)
if test_report_msg is not None:
print_msg(test_report_msg)
print_msg(success_msg, log=_log, silent=testing)
# cleanup and spec files
for ec in easyconfigs:
if 'original_spec' in ec and os.path.isfile(ec['spec']):
os.remove(ec['spec'])
# stop logging and cleanup tmp log file, unless one build failed (individual logs are located in eb_tmpdir path)
stop_logging(logfile, logtostdout=options.logtostdout)
if overall_success:
cleanup(logfile, eb_tmpdir, testing)
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write("ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile), log=None, silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" % (oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error("No robot paths specified, and unable to determine easybuild-easyconfigs install path.")
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning("Failed to determine install path for easybuild-easyconfigs package.")
# specified robot paths are preferred over installed easyconfig files
if robot_path:
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info("Extended list of robot paths with paths for installed easyconfigs: %s" % robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest': options.aggregate_regtest,
'allow_modules_tool_mismatch': options.allow_modules_tool_mismatch,
'check_osdeps': not options.ignore_osdeps,
'command_line': eb_command_line,
'debug': options.debug,
'dry_run': options.dry_run,
'easyblock': options.easyblock,
'experimental': options.experimental,
'force': options.force,
'group': options.group,
'ignore_dirs': options.ignore_dirs,
'modules_footer': options.modules_footer,
'only_blocks': options.only_blocks,
'recursive_mod_unload': options.recursive_module_unload,
'regtest_online': options.regtest_online,
'regtest_output_dir': options.regtest_output_dir,
'retain_all_deps': retain_all_deps,
'robot_path': robot_path,
'sequential': options.sequential,
'silent': testing,
'set_gid_bit': options.set_gid_bit,
'skip': options.skip,
'skip_test_cases': options.skip_test_cases,
'sticky_bit': options.sticky_bit,
'stop': options.stop,
'umask': options.umask,
'valid_module_classes': module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate': not options.force,
})
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path, query, short=not options.search, ignore_dirs=ignore_dirs, silent=silent)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
paths = []
if len(orig_paths) == 0:
if 'name' in build_specs:
paths = [obtain_path(build_specs, easyconfigs_paths, try_to_generate=try_to_generate,
exit_on_error=not testing)]
elif not any([options.aggregate_regtest, options.search, options.search_short, options.regtest]):
print_error(("Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log, opt_parser=eb_go.parser, exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug("Looking for missing easyconfig files (%d left) in %s..." % (len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path, topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" % (orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [d for d in dirnames if not d in options.ignore_dirs]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for f in files:
if not generated and try_to_generate and build_specs:
ec_file = tweak(f, None, build_specs)
else:
ec_file = f
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" % (path, err))
import test.framework.options as o
import test.framework.parallelbuild as p
import test.framework.repository as r
import test.framework.robot as robot
import test.framework.run as run
import test.framework.scripts as sc
import test.framework.systemtools as s
import test.framework.toolchain as tc
import test.framework.toolchainvariables as tcv
import test.framework.toy_build as t
import test.framework.tweak as tw
import test.framework.variables as v
# make sure temporary files can be created/used
try:
set_tmpdir(raise_error=True)
except EasyBuildError, err:
sys.stderr.write(
"No execution rights on temporary files, specify another location via $TMPDIR: %s\n"
% err)
sys.exit(1)
# initialize logger for all the unit tests
fd, log_fn = tempfile.mkstemp(prefix='easybuild-tests-', suffix='.log')
os.close(fd)
os.remove(log_fn)
fancylogger.logToFile(log_fn)
log = fancylogger.getLogger()
# call suite() for each module and then run them all
# note: make sure the options unit tests run first, to avoid running some of them with a readily initialized config
def main(testing_data=(None, None, None)):
"""
Main function:
@arg options: a tuple: (options, paths, logger, logfile, hn) as defined in parse_options
This function will:
- read easyconfig
- build software
"""
# purposely session state very early, to avoid modules loaded by EasyBuild meddling in
init_session_state = session_state()
# disallow running EasyBuild as root
if os.getuid() == 0:
sys.stderr.write(
"ERROR: You seem to be running EasyBuild with root privileges.\n"
"That's not wise, so let's end this here.\n"
"Exiting.\n")
sys.exit(1)
# steer behavior when testing main
testing = testing_data[0] is not None
args, logfile, do_build = testing_data
# initialise options
eb_go = eboptions.parse_options(args=args)
options = eb_go.options
orig_paths = eb_go.args
eb_config = eb_go.generate_cmd_line(add_default=True)
init_session_state.update({'easybuild_configuration': eb_config})
# set umask (as early as possible)
if options.umask is not None:
new_umask = int(options.umask, 8)
old_umask = os.umask(new_umask)
# set temporary directory to use
eb_tmpdir = set_tmpdir(options.tmpdir)
# initialise logging for main
if options.logtostdout:
fancylogger.logToScreen(enable=True, stdout=True)
else:
if logfile is None:
# mkstemp returns (fd,filename), fd is from os.open, not regular open!
fd, logfile = tempfile.mkstemp(suffix='.log', prefix='easybuild-')
os.close(fd)
fancylogger.logToFile(logfile)
print_msg('temporary log file in case of crash %s' % (logfile),
log=None,
silent=testing)
global _log
_log = fancylogger.getLogger(fname=False)
if options.umask is not None:
_log.info("umask set to '%s' (used to be '%s')" %
(oct(new_umask), oct(old_umask)))
# hello world!
_log.info(this_is_easybuild())
# how was EB called?
eb_command_line = eb_go.generate_cmd_line() + eb_go.args
_log.info("Command line: %s" % (" ".join(eb_command_line)))
_log.info("Using %s as temporary directory" % eb_tmpdir)
if not options.robot is None:
if options.robot:
_log.info("Using robot path(s): %s" % options.robot)
else:
_log.error(
"No robot paths specified, and unable to determine easybuild-easyconfigs install path."
)
# do not pass options.robot, it's not a list instance (and it shouldn't be modified)
robot_path = None
if options.robot:
robot_path = list(options.robot)
# determine easybuild-easyconfigs package install path
easyconfigs_paths = get_paths_for("easyconfigs", robot_path=robot_path)
# keep track of paths for install easyconfigs, so we can obtain find specified easyconfigs
easyconfigs_pkg_full_paths = easyconfigs_paths[:]
if not easyconfigs_paths:
_log.warning(
"Failed to determine install path for easybuild-easyconfigs package."
)
# process software build specifications (if any), i.e.
# software name/version, toolchain name/version, extra patches, ...
(try_to_generate, build_specs) = process_software_build_specs(options)
# specified robot paths are preferred over installed easyconfig files
# --try-X and --dep-graph both require --robot, so enable it with path of installed easyconfigs
if robot_path or try_to_generate or options.dep_graph:
if robot_path is None:
robot_path = []
robot_path.extend(easyconfigs_paths)
easyconfigs_paths = robot_path[:]
_log.info(
"Extended list of robot paths with paths for installed easyconfigs: %s"
% robot_path)
# initialise the easybuild configuration
config.init(options, eb_go.get_options_by_section('config'))
# building a dependency graph implies force, so that all dependencies are retained
# and also skips validation of easyconfigs (e.g. checking os dependencies)
retain_all_deps = False
if options.dep_graph:
_log.info("Enabling force to generate dependency graph.")
options.force = True
retain_all_deps = True
config.init_build_options({
'aggregate_regtest':
options.aggregate_regtest,
'allow_modules_tool_mismatch':
options.allow_modules_tool_mismatch,
'check_osdeps':
not options.ignore_osdeps,
'cleanup_builddir':
options.cleanup_builddir,
'command_line':
eb_command_line,
'debug':
options.debug,
'dry_run':
options.dry_run,
'easyblock':
options.easyblock,
'experimental':
options.experimental,
'force':
options.force,
'github_user':
options.github_user,
'group':
options.group,
'ignore_dirs':
options.ignore_dirs,
'modules_footer':
options.modules_footer,
'only_blocks':
options.only_blocks,
'recursive_mod_unload':
options.recursive_module_unload,
'regtest_output_dir':
options.regtest_output_dir,
'retain_all_deps':
retain_all_deps,
'robot_path':
robot_path,
'sequential':
options.sequential,
'silent':
testing,
'set_gid_bit':
options.set_gid_bit,
'skip':
options.skip,
'skip_test_cases':
options.skip_test_cases,
'sticky_bit':
options.sticky_bit,
'stop':
options.stop,
'umask':
options.umask,
'valid_module_classes':
module_classes(),
'valid_stops': [x[0] for x in EasyBlock.get_steps()],
'validate':
not options.force,
})
# obtain list of loaded modules, build options must be initialized first
modlist = session_module_list()
init_session_state.update({'module_list': modlist})
_log.debug("Initial session state: %s" % init_session_state)
# search for easyconfigs
if options.search or options.search_short:
search_path = [os.getcwd()]
if easyconfigs_paths:
search_path = easyconfigs_paths
query = options.search or options.search_short
ignore_dirs = config.build_option('ignore_dirs')
silent = config.build_option('silent')
search_file(search_path,
query,
short=not options.search,
ignore_dirs=ignore_dirs,
silent=silent)
paths = []
if len(orig_paths) == 0:
if options.from_pr:
pr_path = os.path.join(eb_tmpdir, "files_pr%s" % options.from_pr)
pr_files = fetch_easyconfigs_from_pr(
options.from_pr, path=pr_path, github_user=options.github_user)
paths = [(path, False) for path in pr_files
if path.endswith('.eb')]
elif 'name' in build_specs:
paths = [
obtain_path(build_specs,
easyconfigs_paths,
try_to_generate=try_to_generate,
exit_on_error=not testing)
]
elif not any([
options.aggregate_regtest, options.search,
options.search_short, options.regtest
]):
print_error((
"Please provide one or multiple easyconfig files, or use software build "
"options to make EasyBuild search for easyconfigs"),
log=_log,
opt_parser=eb_go.parser,
exit_on_error=not testing)
else:
# look for easyconfigs with relative paths in easybuild-easyconfigs package,
# unless they were found at the given relative paths
if easyconfigs_pkg_full_paths:
# determine which easyconfigs files need to be found, if any
ecs_to_find = []
for idx, orig_path in enumerate(orig_paths):
if orig_path == os.path.basename(
orig_path) and not os.path.exists(orig_path):
ecs_to_find.append((idx, orig_path))
_log.debug("List of easyconfig files to find: %s" % ecs_to_find)
# find missing easyconfigs by walking paths with installed easyconfig files
for path in easyconfigs_pkg_full_paths:
_log.debug(
"Looking for missing easyconfig files (%d left) in %s..." %
(len(ecs_to_find), path))
for (subpath, dirnames, filenames) in os.walk(path,
topdown=True):
for idx, orig_path in ecs_to_find[:]:
if orig_path in filenames:
full_path = os.path.join(subpath, orig_path)
_log.info("Found %s in %s: %s" %
(orig_path, path, full_path))
orig_paths[idx] = full_path
# if file was found, stop looking for it (first hit wins)
ecs_to_find.remove((idx, orig_path))
# stop os.walk insanity as soon as we have all we need (os.walk loop)
if len(ecs_to_find) == 0:
break
# ignore subdirs specified to be ignored by replacing items in dirnames list used by os.walk
dirnames[:] = [
d for d in dirnames if not d in options.ignore_dirs
]
# stop os.walk insanity as soon as we have all we need (paths loop)
if len(ecs_to_find) == 0:
break
# indicate that specified paths do not contain generated easyconfig files
paths = [(path, False) for path in orig_paths]
_log.debug("Paths: %s" % paths)
# run regtest
if options.regtest or options.aggregate_regtest:
_log.info("Running regression test")
if paths:
ec_paths = [path[0] for path in paths]
else: # fallback: easybuild-easyconfigs install path
ec_paths = easyconfigs_pkg_full_paths
regtest_ok = regtest(ec_paths)
if not regtest_ok:
_log.info("Regression test failed (partially)!")
sys.exit(31) # exit -> 3x1t -> 31
# read easyconfig files
easyconfigs = []
for (path, generated) in paths:
path = os.path.abspath(path)
if not os.path.exists(path):
print_error("Can't find path %s" % path)
try:
ec_files = find_easyconfigs(path, ignore_dirs=options.ignore_dirs)
for ec_file in ec_files:
# only pass build specs when not generating easyconfig files
if try_to_generate:
ecs = process_easyconfig(ec_file)
else:
ecs = process_easyconfig(ec_file, build_specs=build_specs)
easyconfigs.extend(ecs)
except IOError, err:
_log.error("Processing easyconfigs in path %s failed: %s" %
(path, err))
