def check_os_dependency(dep):
"""
Check if dependency is available from OS.
"""
# - uses rpm -q and dpkg -s --> can be run as non-root!!
# - fallback on which
# - should be extended to files later?
found = None
cmd = None
if which('rpm'):
cmd = "rpm -q %s" % dep
found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True)
if not found and which('dpkg'):
cmd = "dpkg -s %s" % dep
found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True)
if cmd is None:
# fallback for when os-dependency is a binary/library
found = which(dep)
# try locate if it's available
if not found and which('locate'):
cmd = 'locate --regexp "/%s$"' % dep
found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True)
return found
def test_which(self):
"""Test which function for locating commands."""
python = ft.which('python')
self.assertTrue(python and os.path.exists(python) and os.path.isabs(python))
path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available')
self.assertTrue(path is None)
def check_os_dependency(dep):
"""
Check if dependency is available from OS.
"""
# - uses rpm -q and dpkg -s --> can be run as non-root!!
# - fallback on which
# - should be extended to files later?
cmd = None
if get_os_name() in ['debian', 'ubuntu']:
if which('dpkg'):
cmd = "dpkg -s %s" % dep
else:
# OK for get_os_name() == redhat, fedora, RHEL, SL, centos
if which('rpm'):
cmd = "rpm -q %s" % dep
found = None
if cmd is not None:
found = run_cmd(cmd, simple=True, log_all=False, log_ok=False)
if found is None:
# fallback for when os-dependency is a binary/library
found = which(dep)
# try locate if it's available
if found is None and which('locate'):
cmd = 'locate --regexp "/%s$"' % dep
found = run_cmd(cmd, simple=True, log_all=False, log_ok=False)
return found
def __init__(self, mod_paths=None, testing=False):
"""
Create a ModulesTool object
@param mod_paths: A list of paths where the modules can be located
@type mod_paths: list
"""
# this can/should be set to True during testing
self.testing = testing
self.log = fancylogger.getLogger(self.__class__.__name__, fname=False)
self.mod_paths = None
if mod_paths is not None:
self.set_mod_paths(mod_paths)
# DEPRECATED!
self._modules = []
# actual module command (i.e., not the 'module' wrapper function, but the binary)
self.cmd = self.COMMAND
env_cmd_path = os.environ.get(self.COMMAND_ENVIRONMENT)
# only use command path in environment variable if command in not available in $PATH
if which(self.cmd) is None and env_cmd_path is not None:
self.log.debug("Set command via environment variable %s: %s", self.COMMAND_ENVIRONMENT, self.cmd)
self.cmd = env_cmd_path
# check whether paths obtained via $PATH and $LMOD_CMD are different
elif which(self.cmd) != env_cmd_path:
self.log.debug(
"Different paths found for module command '%s' via which/$PATH and $%s: %s vs %s",
self.COMMAND,
self.COMMAND_ENVIRONMENT,
self.cmd,
env_cmd_path,
)
# make sure the module command was found
if self.cmd is None:
raise EasyBuildError("No command set.")
else:
self.log.debug("Using command %s" % self.cmd)
# version of modules tool
self.version = None
# some initialisation/verification
self.check_cmd_avail()
self.check_module_path()
self.check_module_function(allow_mismatch=build_option("allow_modules_tool_mismatch"))
self.set_and_check_version()
def configure_step(self):
"""Custom configuration procedure for RPMs: rebuild RPMs for relocation if required."""
# make sure that rpm is available
if not which('rpm'):
raise EasyBuildError("Command 'rpm' is required but not available.")
# determine whether RPMs need to be rebuilt to make relocation work
cmd = "rpm --version"
(out, _) = run_cmd(cmd, log_all=True, simple=False)
rpmver_re = re.compile("^RPM\s+version\s+(?P<version>[0-9.]+).*")
res = rpmver_re.match(out)
self.log.debug("RPM version found: %s" % res.group())
if res:
ver = res.groupdict()['version']
# rebuilding is required on SL6, which implies rpm v4.8 (works fine without rebuilding on SL5)
if LooseVersion(ver) >= LooseVersion('4.8.0'):
self.rebuild_rpm = True
self.log.debug("Enabling rebuild of RPMs to make relocation work...")
else:
raise EasyBuildError("Checking RPM version failed, so just carrying on with the default behaviour...")
if self.rebuild_rpm:
self.rebuild_rpms()
def test_which(self):
"""Test which function for locating commands."""
python = ft.which('python')
self.assertTrue(python and os.path.exists(python) and os.path.isabs(python))
path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available')
self.assertTrue(path is None)
os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.environ['PATH'])
foo, bar = os.path.join(self.test_prefix, 'foo'), os.path.join(self.test_prefix, 'bar')
ft.mkdir(foo)
ft.adjust_permissions(foo, stat.S_IRUSR|stat.S_IXUSR)
ft.write_file(bar, '#!/bin/bash')
ft.adjust_permissions(bar, stat.S_IRUSR|stat.S_IXUSR)
self.assertEqual(ft.which('foo'), None)
self.assertTrue(os.path.samefile(ft.which('bar'), bar))
def configure_step(self, cmd_prefix=''):
"""
Configure with Meson.
"""
# make sure both Meson and Ninja are included as build dependencies
build_dep_names = [d['name'] for d in self.cfg.builddependencies()]
for tool in ['Ninja', 'Meson']:
if tool not in build_dep_names:
raise EasyBuildError("%s not included as build dependency", tool)
cmd = tool.lower()
if not which(cmd):
raise EasyBuildError("'%s' command not found", cmd)
if self.cfg.get('separate_build_dir', True):
builddir = os.path.join(self.builddir, 'easybuild_obj')
mkdir(builddir)
change_dir(builddir)
# Make sure libdir doesn't get set to lib/x86_64-linux-gnu or something
# on Debian/Ubuntu multiarch systems and others.
no_Dlibdir = '-Dlibdir' not in self.cfg['configopts']
no_libdir = '--libdir' not in self.cfg['configopts']
if no_Dlibdir and no_libdir:
self.cfg.update('configopts', '-Dlibdir=lib')
cmd = "%(preconfigopts)s meson --prefix %(installdir)s %(configopts)s %(sourcedir)s" % {
'configopts': self.cfg['configopts'],
'installdir': self.installdir,
'preconfigopts': self.cfg['preconfigopts'],
'sourcedir': self.start_dir,
}
(out, _) = run_cmd(cmd, log_all=True, simple=False)
return out
def symlink_commands(self, paths):
"""
Create a symlink for each command to binary/script at specified path.
:param paths: dictionary containing one or mappings, each one specified as a tuple:
(<path/to/script>, <list of commands to symlink to the script>)
"""
symlink_dir = tempfile.mkdtemp()
# prepend location to symlinks to $PATH
setvar('PATH', '%s:%s' % (symlink_dir, os.getenv('PATH')))
for (path, cmds) in paths.values():
for cmd in cmds:
cmd_s = os.path.join(symlink_dir, cmd)
if not os.path.exists(cmd_s):
try:
os.symlink(path, cmd_s)
except OSError as err:
raise EasyBuildError("Failed to symlink %s to %s: %s", path, cmd_s, err)
cmd_path = which(cmd)
self.log.debug("which(%s): %s -> %s", cmd, cmd_path, os.path.realpath(cmd_path))
self.log.info("Commands symlinked to %s via %s: %s", path, symlink_dir, ', '.join(cmds))
def prepare_python(self):
"""Python-specific preperations."""
# pick 'python' command to use
python = None
python_root = get_software_root('Python')
# keep in mind that Python may be listed as an allowed system dependency,
# so just checking Python root is not sufficient
if python_root:
bin_python = os.path.join(python_root, 'bin', 'python')
if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python):
# if Python is listed as a (build) dependency, use 'python' command provided that way
python = os.path.join(python_root, 'bin', 'python')
self.log.debug("Retaining 'python' command for Python dependency: %s", python)
if python is None:
# if using system Python, go hunting for a 'python' command that satisfies the requirements
python = pick_python_cmd(req_maj_ver=self.cfg['req_py_majver'], req_min_ver=self.cfg['req_py_minver'])
if python:
self.python_cmd = python
self.log.info("Python command being used: %s", self.python_cmd)
else:
raise EasyBuildError("Failed to pick Python command to use")
# set Python lib directories
self.set_pylibdirs()
def check_tool(tool_name, min_tool_version=None):
"""
This function is a predicate check for the existence of tool_name on the system PATH.
If min_tool_version is not None, it will check that the version has an equal or higher value.
"""
tool_path = which(tool_name)
if not tool_path:
return False
print_msg("{0} tool found at {1}".format(tool_name, tool_path))
if not min_tool_version:
return True
version_cmd = "{0} --version".format(tool_name)
out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True)
if ec:
raise EasyBuildError("Error running '{0}' for tool {1} with output: {2}".format(version_cmd, tool_name, out))
res = re.search("\d+\.\d+(\.\d+)?", out.strip())
if not res:
raise EasyBuildError("Error parsing version for tool {0}".format(tool_name))
tool_version = res.group(0)
version_ok = LooseVersion(str(min_tool_version)) <= LooseVersion(tool_version)
if version_ok:
print_msg("{0} version '{1}' is {2} or higher ... OK".format(tool_name, tool_version, min_tool_version))
return version_ok
def configure_step(self, cmd_prefix=''):
"""
Configure with Meson.
"""
# make sure both Meson and Ninja are included as build dependencies
build_dep_names = [d['name'] for d in self.cfg.builddependencies()]
for tool in ['Ninja', 'Meson']:
if tool not in build_dep_names:
raise EasyBuildError("%s not included as build dependency", tool)
cmd = tool.lower()
if not which(cmd):
raise EasyBuildError("'%s' command not found", cmd)
if self.cfg.get('separate_build_dir', True):
builddir = os.path.join(self.builddir, 'easybuild_obj')
mkdir(builddir)
change_dir(builddir)
cmd = "%(preconfigopts)s meson --prefix %(installdir)s %(configopts)s %(sourcedir)s" % {
'configopts': self.cfg['configopts'],
'installdir': self.installdir,
'preconfigopts': self.cfg['preconfigopts'],
'sourcedir': self.start_dir,
}
(out, _) = run_cmd(cmd, log_all=True, simple=False)
return out
def write_wrapper(self, wrapper_dir, compiler, i_mpi_root):
"""Helper function to write a compiler wrapper."""
wrapper_txt = INTEL_COMPILER_WRAPPER % {
'compiler_path': which(compiler),
'intel_mpi_root': i_mpi_root,
'cpath': os.getenv('CPATH'),
'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')),
'wrapper_dir': wrapper_dir,
}
wrapper = os.path.join(wrapper_dir, compiler)
write_file(wrapper, wrapper_txt)
if self.dry_run:
self.dry_run_msg("Wrapper for '%s' was put in place: %s", compiler, wrapper)
else:
adjust_permissions(wrapper, stat.S_IXUSR)
self.log.info("Using wrapper script for '%s': %s", compiler, which(compiler))
def test_lmod_specific(self):
"""Lmod-specific test (skipped unless Lmod is used as modules tool)."""
lmod_abspath = which(Lmod.COMMAND)
# only run this test if 'lmod' is available in $PATH
if lmod_abspath is not None:
build_options = {
'allow_modules_tool_mismatch': True,
}
init_config(build_options=build_options)
# drop any location where 'lmod' or 'spider' can be found from $PATH
paths = os.environ.get('PATH', '').split(os.pathsep)
new_paths = []
for path in paths:
lmod_cand_path = os.path.join(path, Lmod.COMMAND)
spider_cand_path = os.path.join(path, 'spider')
if not os.path.isfile(lmod_cand_path) and not os.path.isfile(spider_cand_path):
new_paths.append(path)
os.environ['PATH'] = os.pathsep.join(new_paths)
# make sure $MODULEPATH contains path that provides some modules
os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))
# initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD
os.environ['LMOD_CMD'] = lmod_abspath
lmod = Lmod(testing=True)
# obtain list of availabe modules, should be non-empty
self.assertTrue(lmod.available(), "List of available modules obtained using Lmod is non-empty")
# test updating local spider cache (but don't actually update the local cache file!)
self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty")
def check_pkg_support():
"""Check whether packaging is possible, if required dependencies are available."""
pkgtool = build_option('package_tool')
pkgtool_path = which(pkgtool)
if pkgtool_path:
_log.info("Selected packaging tool '%s' found at %s", pkgtool, pkgtool_path)
# rpmbuild is required for generating RPMs with FPM
if pkgtool == PKG_TOOL_FPM and build_option('package_type') == PKG_TYPE_RPM:
rpmbuild_path = which('rpmbuild')
if rpmbuild_path:
_log.info("Required tool 'rpmbuild' found at %s", rpmbuild_path)
else:
raise EasyBuildError("rpmbuild is required when generating RPM packages but was not found")
else:
raise EasyBuildError("Selected packaging tool '%s' not found", pkgtool)
def check_cmd_avail(self):
"""Check whether modules tool command is available."""
cmd_path = which(self.cmd)
if cmd_path is not None:
self.cmd = cmd_path
self.log.info("Full path for module command is %s, so using it" % self.cmd)
else:
mod_tool = self.__class__.__name__
self.log.error("%s modules tool can not be used, '%s' command is not available." % (mod_tool, self.cmd))
def check_os_dependency(dep):
"""
Check if dependency is available from OS.
"""
# - uses rpm -q and dpkg -s --> can be run as non-root!!
# - fallback on which
# - should be extended to files later?
found = False
cmd = None
os_to_pkg_cmd_map = {
'centos': RPM,
'debian': DPKG,
'redhat': RPM,
'ubuntu': DPKG,
}
pkg_cmd_flag = {
DPKG: '-s',
RPM: '-q',
}
os_name = get_os_name()
if os_name in os_to_pkg_cmd_map:
pkg_cmds = [os_to_pkg_cmd_map[os_name]]
else:
pkg_cmds = [RPM, DPKG]
for pkg_cmd in pkg_cmds:
if which(pkg_cmd):
cmd = ' '.join([pkg_cmd, pkg_cmd_flag.get(pkg_cmd), dep])
found = run_cmd(cmd, simple=True, log_all=False, log_ok=False,
force_in_dry_run=True, trace=False, stream_output=False)
if found:
break
if not found:
# fallback for when os-dependency is a binary/library
found = which(dep)
# try locate if it's available
if not found and which('locate'):
cmd = 'locate --regexp "/%s$"' % dep
found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True, trace=False,
stream_output=False)
return found
def get_system_info():
"""Return a dictionary with system information."""
python_version = '; '.join(sys.version.split('\n'))
return {
'core_count': get_avail_core_count(),
'cpu_model': get_cpu_model(),
'cpu_speed': get_cpu_speed(),
'cpu_vendor': get_cpu_vendor(),
'gcc_version': get_tool_version('gcc', version_option='-v'),
'hostname': gethostname(),
'glibc_version': get_glibc_version(),
'os_name': get_os_name(),
'os_type': get_os_type(),
'os_version': get_os_version(),
'platform_name': get_platform_name(),
'python_version': python_version,
'system_python_path': which('python'),
'system_gcc_path': which('gcc'),
}
def configure_step(self):
"""Custom configuration procedure for Bazel."""
binutils_root = get_software_root('binutils')
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
gcc_ver = get_software_version('GCCcore') or get_software_version('GCC')
# only patch Bazel scripts if binutils & GCC installation prefix could be determined
if binutils_root and gcc_root:
res = glob.glob(os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError("Failed to pinpoint location of GCC include files: %s", res)
gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed')
if not os.path.exists(gcc_lib_inc_fixed):
raise EasyBuildError("Derived directory %s does not exist", gcc_lib_inc_fixed)
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc)
# replace hardcoded paths in CROSSTOOL
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'(cxx_builtin_include_directory:.*)/usr/lib/gcc', r'\1%s' % gcc_lib_inc),
(r'(cxx_builtin_include_directory:.*)/usr/local/include', r'\1%s' % gcc_lib_inc_fixed),
(r'(cxx_builtin_include_directory:.*)/usr/include', r'\1%s' % gcc_cplusplus_inc),
]
for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin', tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'", tool)
apply_regex_substitutions(os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
# replace hardcoded paths in (unix_)cc_configure.bzl
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
]
for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
filepath = os.path.join('tools', 'cpp', conf_bzl)
if os.path.exists(filepath):
apply_regex_substitutions(filepath, regex_subs)
else:
self.log.info("Not patching Bazel build scripts, installation prefix for binutils/GCC not found")
# enable building in parallel
env.setvar('EXTRA_BAZEL_ARGS', '--jobs=%d' % self.cfg['parallel'])
def check_pkg_support():
"""Check whether packaging is supported, i.e. whether the required dependencies are available."""
# packaging support is considered experimental for now (requires using --experimental)
_log.experimental("Support for packaging installed software.")
pkgtool = build_option('package_tool')
pkgtool_path = which(pkgtool)
if pkgtool_path:
_log.info("Selected packaging tool '%s' found at %s", pkgtool, pkgtool_path)
# rpmbuild is required for generating RPMs with FPM
if pkgtool == PKG_TOOL_FPM and build_option('package_type') == PKG_TYPE_RPM:
rpmbuild_path = which('rpmbuild')
if rpmbuild_path:
_log.info("Required tool 'rpmbuild' found at %s", rpmbuild_path)
else:
raise EasyBuildError("rpmbuild is required when generating RPM packages with FPM, but was not found")
else:
raise EasyBuildError("Selected packaging tool '%s' not found", pkgtool)
def get_system_info():
"""Return a dictionary with system information."""
python_version = "; ".join(sys.version.split("\n"))
return {
"core_count": get_avail_core_count(),
"total_memory": get_total_memory(),
"cpu_model": get_cpu_model(),
"cpu_speed": get_cpu_speed(),
"cpu_vendor": get_cpu_vendor(),
"gcc_version": get_tool_version("gcc", version_option="-v"),
"hostname": gethostname(),
"glibc_version": get_glibc_version(),
"os_name": get_os_name(),
"os_type": get_os_type(),
"os_version": get_os_version(),
"platform_name": get_platform_name(),
"python_version": python_version,
"system_python_path": which("python"),
"system_gcc_path": which("gcc"),
}
def test_mock(self):
"""Test the mock module"""
# ue empty mod_path list, otherwise the install_path is called
mmt = MockModulesTool(mod_paths=[])
# the version of the MMT is the commandline option
self.assertEqual(mmt.version, StrictVersion(MockModulesTool.VERSION_OPTION))
cmd_abspath = which(MockModulesTool.COMMAND)
# make sure absolute path of module command is being used
self.assertEqual(mmt.cmd, cmd_abspath)
def test_which(self):
"""Test which function for locating commands."""
python = ft.which('python')
self.assertTrue(python and os.path.exists(python) and os.path.isabs(python))
path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available')
self.assertTrue(path is None)
os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.environ['PATH'])
# put a directory 'foo' in place (should be ignored by 'which')
foo = os.path.join(self.test_prefix, 'foo')
ft.mkdir(foo)
ft.adjust_permissions(foo, stat.S_IRUSR|stat.S_IXUSR)
# put executable file 'bar' in place
bar = os.path.join(self.test_prefix, 'bar')
ft.write_file(bar, '#!/bin/bash')
ft.adjust_permissions(bar, stat.S_IRUSR|stat.S_IXUSR)
self.assertEqual(ft.which('foo'), None)
self.assertTrue(os.path.samefile(ft.which('bar'), bar))
# add another location to 'bar', which should only return the first location by default
barbis = os.path.join(self.test_prefix, 'more', 'bar')
ft.write_file(barbis, '#!/bin/bash')
ft.adjust_permissions(barbis, stat.S_IRUSR|stat.S_IXUSR)
os.environ['PATH'] = '%s:%s' % (os.environ['PATH'], os.path.dirname(barbis))
self.assertTrue(os.path.samefile(ft.which('bar'), bar))
# test getting *all* locations to specified command
res = ft.which('bar', retain_all=True)
self.assertEqual(len(res), 2)
self.assertTrue(os.path.samefile(res[0], bar))
self.assertTrue(os.path.samefile(res[1], barbis))
def build_singularity_image(def_path):
"""Build Singularity container image by calling out to 'singularity' (requires admin privileges!)."""
cont_path = container_path()
def_file = os.path.basename(def_path)
# use --imagename if specified, otherwise derive based on filename of recipe
img_name = build_option('container_image_name')
if img_name is None:
# definition file Singularity.<app>-<version, container name <app>-<version>.<img|simg>
img_name = def_file.split('.', 1)[1]
cmd_opts = ''
image_format = build_option('container_image_format')
# squashfs image format (default for Singularity)
if image_format in [None, CONT_IMAGE_FORMAT_SQUASHFS]:
img_path = os.path.join(cont_path, img_name + '.simg')
# ext3 image format, creating as writable container
elif image_format == CONT_IMAGE_FORMAT_EXT3:
img_path = os.path.join(cont_path, img_name + '.img')
cmd_opts = '--writable'
# sandbox image format, creates as a directory but acts like a container
elif image_format == CONT_IMAGE_FORMAT_SANDBOX:
img_path = os.path.join(cont_path, img_name)
cmd_opts = '--sandbox'
else:
raise EasyBuildError("Unknown container image format specified for Singularity: %s" % image_format)
if os.path.exists(img_path):
if build_option('force'):
print_msg("WARNING: overwriting existing container image at %s due to --force" % img_path)
remove_file(img_path)
else:
raise EasyBuildError("Container image already exists at %s, not overwriting it without --force", img_path)
# resolve full path to 'singularity' binary, since it may not be available via $PATH under sudo...
singularity = which('singularity')
cmd_env = ''
singularity_tmpdir = build_option('container_tmpdir')
if singularity_tmpdir:
cmd_env += 'SINGULARITY_TMPDIR=%s' % singularity_tmpdir
cmd = ' '.join(['sudo', cmd_env, singularity, 'build', cmd_opts, img_path, def_path])
print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd)
run_cmd(cmd, stream_output=True)
print_msg("Singularity image created at %s" % img_path, log=_log)
def test_lmod_specific(self):
"""Lmod-specific test (skipped unless Lmod is used as modules tool)."""
lmod_abspath = which(Lmod.COMMAND)
# only run this test if 'lmod' is available in $PATH
if lmod_abspath is not None:
build_options = {
'allow_modules_tool_mismatch': True,
'update_modules_tool_cache': True,
}
init_config(build_options=build_options)
lmod = Lmod(testing=True)
self.assertEqual(lmod.cmd, os.path.realpath(lmod_abspath))
# drop any location where 'lmod' or 'spider' can be found from $PATH
paths = os.environ.get('PATH', '').split(os.pathsep)
new_paths = []
for path in paths:
lmod_cand_path = os.path.join(path, Lmod.COMMAND)
spider_cand_path = os.path.join(path, 'spider')
if not os.path.isfile(lmod_cand_path) and not os.path.isfile(spider_cand_path):
new_paths.append(path)
os.environ['PATH'] = os.pathsep.join(new_paths)
# make sure $MODULEPATH contains path that provides some modules
os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules'))
# initialize Lmod modules tool, pass (fake) full path to 'lmod' via $LMOD_CMD
fake_path = os.path.join(self.test_installpath, 'lmod')
fake_lmod_txt = '\n'.join([
'#!/bin/bash',
'echo "Modules based on Lua: Version %s " >&2' % Lmod.REQ_VERSION,
'echo "os.environ[\'FOO\'] = \'foo\'"',
])
write_file(fake_path, fake_lmod_txt)
os.chmod(fake_path, stat.S_IRUSR|stat.S_IXUSR)
os.environ['LMOD_CMD'] = fake_path
init_config(build_options=build_options)
lmod = Lmod(testing=True)
self.assertEqual(lmod.cmd, os.path.realpath(fake_path))
# use correct full path for 'lmod' via $LMOD_CMD
os.environ['LMOD_CMD'] = lmod_abspath
init_config(build_options=build_options)
lmod = Lmod(testing=True)
# obtain list of availabe modules, should be non-empty
self.assertTrue(lmod.available(), "List of available modules obtained using Lmod is non-empty")
# test updating local spider cache (but don't actually update the local cache file!)
self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty")
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
import gc3libs # noqa (ignore unused import)
except ImportError:
print "GC3Pie not available, skipping test"
return
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % {
'resourcedir': resourcedir,
'time': which('time'),
}
write_file(gc3pie_cfgfile, gc3pie_cfgtxt)
output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir')
# purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir)
mkdir(output_dir, parents=True)
write_file(os.path.join(output_dir, 'foo'), 'bar')
# remove write permissions on parent dir of specified output dir,
# to check that GC3Pie does not try to rename the (already existing) output directory...
adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False, recursive=False)
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'job_backend_config': gc3pie_cfgfile,
'job_max_walltime': 24,
'job_output_dir': output_dir,
'job_polling_interval': 0.2, # quick polling
'job_target_resource': 'ebtestlocalhost',
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'silent': True,
'valid_module_classes': config.module_classes(),
'validate': False,
}
init_config(args=['--job-backend=GC3Pie'], build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path)
build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0'))
self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def test_end2end_docker_image(self):
topdir = os.path.dirname(os.path.abspath(__file__))
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb')
containerpath = os.path.join(self.test_prefix, 'containers')
os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
# --containerpath must be an existing directory (this is done to avoid misconfiguration)
mkdir(containerpath)
args = [
toy_ec,
'-C', # equivalent with --containerize
'--experimental',
'--container-type=docker',
'--container-base=ubuntu:16.04',
'--container-build-image',
]
if not which('docker'):
error_pattern = "docker not found on your system."
self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True)
# install mocked versions of 'sudo' and 'docker' commands
docker = os.path.join(self.test_prefix, 'bin', 'docker')
write_file(docker, MOCKED_DOCKER)
adjust_permissions(docker, stat.S_IXUSR, add=True)
sudo = os.path.join(self.test_prefix, 'bin', 'sudo')
write_file(sudo, '#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n')
adjust_permissions(sudo, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')])
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs = [
"^== docker tool found at %s/bin/docker" % self.test_prefix,
"^== Dockerfile definition file created at %s/containers/Dockerfile\.toy-0.0" % self.test_prefix,
"^== Running 'sudo docker build -f .* -t .* \.', you may need to enter your 'sudo' password...",
"^== Docker image created at toy-0.0:latest",
]
self.check_regexs(regexs, stdout)
args.extend(['--force', '--extended-dry-run'])
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
self.check_regexs(regexs, stdout)
def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):
"""
Return a list of absolute paths where the specified subdir can be found, determined by the PYTHONPATH
"""
paths = []
# primary search path is robot path
path_list = []
if isinstance(robot_path, list):
path_list = robot_path[:]
elif robot_path is not None:
path_list = [robot_path]
# consider Python search path, e.g. setuptools install path for easyconfigs
path_list.extend(sys.path)
# figure out installation prefix, e.g. distutils install path for easyconfigs
eb_path = which('eb')
if eb_path is None:
_log.warning("'eb' not found in $PATH, failed to determine installation prefix")
else:
# real location to 'eb' should be <install_prefix>/bin/eb
eb_path = resolve_path(eb_path)
install_prefix = os.path.dirname(os.path.dirname(eb_path))
path_list.append(install_prefix)
_log.debug("Also considering installation prefix %s..." % install_prefix)
# look for desired subdirs
for path in path_list:
path = os.path.join(path, "easybuild", subdir)
_log.debug("Checking for easybuild/%s at %s" % (subdir, path))
try:
if os.path.exists(path):
paths.append(os.path.abspath(path))
_log.debug("Added %s to list of paths for easybuild/%s" % (path, subdir))
except OSError as err:
raise EasyBuildError(str(err))
return paths
def check_singularity():
"""Check whether Singularity can be used (if it's needed)."""
# if we're going to build a container image, we'll need a sufficiently recent version of Singularity available
# (and otherwise we don't really care if Singularity is not available)
if build_option('container_build_image'):
path_to_singularity_cmd = which('singularity')
if path_to_singularity_cmd:
print_msg("Singularity tool found at %s" % path_to_singularity_cmd)
out, ec = run_cmd("singularity --version", simple=False, trace=False, force_in_dry_run=True)
if ec:
raise EasyBuildError("Failed to determine Singularity version: %s" % out)
else:
# singularity version format for 2.3.1 and higher is x.y-dist
singularity_version = out.strip().split('-')[0]
if LooseVersion(singularity_version) < LooseVersion('2.4'):
raise EasyBuildError("Please upgrade singularity instance to version 2.4 or higher")
else:
print_msg("Singularity version '%s' is 2.4 or higher ... OK" % singularity_version)
else:
raise EasyBuildError("Singularity not found in your system")
def prepare_compiler_cache(self, cache_tool):
"""
Prepare for using specified compiler caching tool (e.g., ccache, f90cache)
:param cache_tool: name of compiler caching tool to prepare for
"""
compilers = self.comp_cache_compilers(cache_tool)
self.log.debug("Using compiler cache tool '%s' for compilers: %s", cache_tool, compilers)
# set paths that should be used by compiler caching tool
comp_cache_path = build_option('use_%s' % cache_tool)
setvar('%s_DIR' % cache_tool.upper(), comp_cache_path)
setvar('%s_TEMPDIR' % cache_tool.upper(), tempfile.mkdtemp())
cache_path = which(cache_tool)
if cache_path is None:
raise EasyBuildError("%s binary not found in $PATH, required by --use-compiler-cache", cache)
else:
self.symlink_commands({cache_tool: (cache_path, compilers)})
self.cached_compilers.update(compilers)
self.log.debug("Cached compilers (after preparing for %s): %s", cache_tool, self.cached_compilers)
def check_python_cmd(python_cmd):
"""Check whether specified Python command satisfies requirements."""
# check whether specified Python command is available
if os.path.isabs(python_cmd):
if not os.path.isfile(python_cmd):
log.debug("Python command '%s' does not exist", python_cmd)
return False
else:
python_cmd_path = which(python_cmd)
if python_cmd_path is None:
log.debug("Python command '%s' not available through $PATH", python_cmd)
return False
if req_maj_ver is not None:
if req_min_ver is None:
req_majmin_ver = '%s.0' % req_maj_ver
else:
req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver)
pycode = 'import sys; print("%s.%s" % sys.version_info[:2])'
out, _ = run_cmd("%s -c '%s'" % (python_cmd, pycode), simple=False)
out = out.strip()
# (strict) check for major version
maj_ver = out.split('.')[0]
if maj_ver != str(req_maj_ver):
log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver)
return False
# check for minimal minor version
if LooseVersion(out) < LooseVersion(req_majmin_ver):
log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", out, req_majmin_ver)
return False
# all check passed
log.debug("All check passed for Python command '%s'!", python_cmd)
return True
def configure_step(self):
"""Custom configuration procedure for TensorFlow."""
tmpdir = tempfile.mkdtemp(suffix='-bazel-configure')
# filter out paths from CPATH and LIBRARY_PATH
package_filter = self.cfg['package_filter']
for var in ['CPATH', 'LIBRARY_PATH']:
path = os.getenv(var).split(':')
filtered_path = [p for fil in package_filter for p in path if fil not in p]
os.environ[var] = ':'.join(filtered_path)
# put wrapper for Intel C compiler in place (required to make sure license server is found)
# cfr. https://github.com/bazelbuild/bazel/issues/663
if self.toolchain.comp_family() == toolchain.INTELCOMP:
wrapper_dir = os.path.join(tmpdir, 'bin')
icc_wrapper_txt = INTEL_COMPILER_WRAPPER % {
'compiler_path': which('icc'),
'cpath': os.getenv('CPATH'),
'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')),
'wrapper_dir': wrapper_dir,
}
icc_wrapper = os.path.join(wrapper_dir, 'icc')
write_file(icc_wrapper, icc_wrapper_txt)
env.setvar('PATH', ':'.join([os.path.dirname(icc_wrapper), os.getenv('PATH')]))
if self.dry_run:
self.dry_run_msg("Wrapper for 'icc' was put in place: %s", icc_wrapper)
else:
adjust_permissions(icc_wrapper, stat.S_IXUSR)
self.log.info("Using wrapper script for 'icc': %s", which('icc'))
self.prepare_python()
self.handle_jemalloc()
cuda_root = get_software_root('CUDA')
cudnn_root = get_software_root('cuDNN')
opencl_root = get_software_root('OpenCL')
use_mpi = self.toolchain.options.get('usempi', False)
config_env_vars = {
'CC_OPT_FLAGS': os.getenv('CXXFLAGS'),
'MPI_HOME': '',
'PYTHON_BIN_PATH': self.python_cmd,
'PYTHON_LIB_PATH': os.path.join(self.installdir, self.pylibdir),
'TF_CUDA_CLANG': '0',
'TF_ENABLE_XLA': '0', # XLA JIT support
'TF_NEED_CUDA': ('0', '1')[bool(cuda_root)],
'TF_NEED_GCP': '0', # Google Cloud Platform
'TF_NEED_GDR': '0',
'TF_NEED_HDFS': '0', # Hadoop File System
'TF_NEED_JEMALLOC': ('0', '1')[self.cfg['with_jemalloc']],
'TF_NEED_MPI': ('0', '1')[bool(use_mpi)],
'TF_NEED_OPENCL': ('0', '1')[bool(opencl_root)],
'TF_NEED_S3': '0', # Amazon S3 File System
'TF_NEED_VERBS': '0',
}
if cuda_root:
config_env_vars.update({
'CUDA_TOOLKIT_PATH': cuda_root,
'GCC_HOST_COMPILER_PATH': which(os.getenv('CC')),
'TF_CUDA_COMPUTE_CAPABILITIES': ','.join(self.cfg['cuda_compute_capabilities']),
'TF_CUDA_VERSION': get_software_version('CUDA'),
})
if cudnn_root:
config_env_vars.update({
'CUDNN_INSTALL_PATH': cudnn_root,
'TF_CUDNN_VERSION': get_software_version('cuDNN'),
})
for (key, val) in sorted(config_env_vars.items()):
env.setvar(key, val)
# patch configure.py (called by configure script) to avoid that Bazel abuses $HOME/.cache/bazel
regex_subs = [(r"(run_shell\(\['bazel')", r"\1, '--output_base=%s'" % tmpdir)]
apply_regex_substitutions('configure.py', regex_subs)
cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg['configopts']
run_cmd(cmd, log_all=True, simple=True)
def test_make_module_pythonpackage(self):
"""Test make_module_step of PythonPackage easyblock."""
app_class = get_easyblock_class('PythonPackage')
self.writeEC('PythonPackage', name='testpypkg', version='3.14')
app = app_class(EasyConfig(self.eb_file))
# install dir should not be there yet
self.assertFalse(os.path.exists(app.installdir),
"%s should not exist" % app.installdir)
# create install dir and populate it with subdirs/files
mkdir(app.installdir, parents=True)
# $PATH, $LD_LIBRARY_PATH, $LIBRARY_PATH, $CPATH, $PKG_CONFIG_PATH
write_file(os.path.join(app.installdir, 'bin', 'foo'), 'echo foo!')
write_file(os.path.join(app.installdir, 'include', 'foo.h'), 'bar')
write_file(os.path.join(app.installdir, 'lib', 'libfoo.a'), 'libfoo')
pyver = '.'.join(map(str, sys.version_info[:2]))
write_file(
os.path.join(app.installdir, 'lib', 'python%s' % pyver,
'site-packages', 'foo.egg'), 'foo egg')
write_file(
os.path.join(app.installdir, 'lib64', 'pkgconfig', 'foo.pc'),
'libfoo: foo')
# PythonPackage relies on the fact that 'python' points to the right Python version
tmpdir = tempfile.mkdtemp()
python = os.path.join(tmpdir, 'python')
write_file(python, '#!/bin/bash\necho $0 $@\n%s "$@"' % sys.executable)
adjust_permissions(python, stat.S_IXUSR)
os.environ['PATH'] = '%s:%s' % (tmpdir, os.getenv('PATH', ''))
from easybuild.tools.filetools import which
print(which('python'))
# create module file
app.make_module_step()
remove_file(python)
self.assertTrue(TMPDIR in app.installdir)
self.assertTrue(TMPDIR in app.installdir_mod)
modtxt = None
for cand_mod_filename in ['3.14', '3.14.lua']:
full_modpath = os.path.join(app.installdir_mod, 'testpypkg',
cand_mod_filename)
if os.path.exists(full_modpath):
modtxt = read_file(full_modpath)
break
self.assertFalse(modtxt is None)
regexs = [
(r'^prepend.path.*\WCPATH\W.*include"?\W*$', True),
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib"?\W*$', True),
(r'^prepend.path.*\WPATH\W.*bin"?\W*$', True),
(r'^prepend.path.*\WPKG_CONFIG_PATH\W.*lib64/pkgconfig"?\W*$',
True),
(r'^prepend.path.*\WPYTHONPATH\W.*lib/python[23]\.[0-9]/site-packages"?\W*$',
True),
# lib64 doesn't contain any library files, so these are *not* included in $LD_LIBRARY_PATH or $LIBRARY_PATH
(r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib64', False),
(r'^prepend.path.*\WLIBRARY_PATH\W.*lib64', False),
]
for (pattern, found) in regexs:
regex = re.compile(pattern, re.M)
if found:
assert_msg = "Pattern '%s' found in: %s" % (regex.pattern,
modtxt)
else:
assert_msg = "Pattern '%s' not found in: %s" % (regex.pattern,
modtxt)
self.assertEqual(bool(regex.search(modtxt)), found, assert_msg)
def prepare_step(self, *args, **kwargs):
"""Do compiler appropriate prepare step, determine system compiler version and prefix."""
if self.cfg['generate_standalone_module']:
if self.cfg['name'] in ['GCC', 'GCCcore']:
EB_GCC.prepare_step(self, *args, **kwargs)
elif self.cfg['name'] in ['icc']:
EB_icc.prepare_step(self, *args, **kwargs)
elif self.cfg['name'] in ['ifort']:
EB_ifort.prepare_step(self, *args, **kwargs)
else:
raise EasyBuildError("I don't know how to do the prepare_step for %s", self.cfg['name'])
else:
Bundle.prepare_step(self, *args, **kwargs)
# Determine compiler path (real path, with resolved symlinks)
compiler_name = self.cfg['name'].lower()
if compiler_name == 'gcccore':
compiler_name = 'gcc'
path_to_compiler = which(compiler_name)
if path_to_compiler:
path_to_compiler = resolve_path(path_to_compiler)
self.log.info("Found path to compiler '%s' (with symlinks resolved): %s", compiler_name, path_to_compiler)
else:
raise EasyBuildError("%s not found in $PATH", compiler_name)
# Determine compiler version
self.compiler_version = extract_compiler_version(compiler_name)
# Determine installation prefix
if compiler_name == 'gcc':
# strip off 'bin/gcc'
self.compiler_prefix = os.path.dirname(os.path.dirname(path_to_compiler))
elif compiler_name in ['icc', 'ifort']:
intelvars_fn = path_to_compiler + 'vars.sh'
if os.path.isfile(intelvars_fn):
self.log.debug("Trying to determine compiler install prefix from %s", intelvars_fn)
intelvars_txt = read_file(intelvars_fn)
prod_dir_regex = re.compile(r'^PROD_DIR=(.*)$', re.M)
res = prod_dir_regex.search(intelvars_txt)
if res:
self.compiler_prefix = res.group(1)
else:
raise EasyBuildError("Failed to determine %s installation prefix from %s",
compiler_name, intelvars_fn)
else:
# strip off 'bin/intel*/icc'
self.compiler_prefix = os.path.dirname(os.path.dirname(os.path.dirname(path_to_compiler)))
# For versions 2016+ of Intel compilers they changed the installation path so must shave off 2 more
# directories from result of the above
if LooseVersion(self.compiler_version) >= LooseVersion('2016'):
self.compiler_prefix = os.path.dirname(os.path.dirname(self.compiler_prefix))
else:
raise EasyBuildError("Unknown system compiler %s" % self.cfg['name'])
if not os.path.exists(self.compiler_prefix):
raise EasyBuildError("Path derived for system compiler (%s) does not exist: %s!",
compiler_name, self.compiler_prefix)
self.log.debug("Derived version/install prefix for system compiler %s: %s, %s",
compiler_name, self.compiler_version, self.compiler_prefix)
# If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it
if self.cfg['version'] == 'system':
self.log.info("Found specified version '%s', going with derived compiler version '%s'",
self.cfg['version'], self.compiler_version)
elif self.cfg['version'] != self.compiler_version:
raise EasyBuildError("Specified version (%s) does not match version reported by compiler (%s)" %
(self.cfg['version'], self.compiler_version))
def prepare_rpath_wrappers(self,
rpath_filter_dirs=None,
rpath_include_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
self.log.experimental(
"Using wrapper scripts for compiler/linker commands that enforce RPATH linking"
)
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError(
"RPATH linking is currently only supported on Linux")
wrapper_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# prepend location to wrappers to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug(
"No general RPATH filter specified, falling back to default: %s",
rpath_filter)
rpath_filter = ','.join(rpath_filter +
['%s.*' % d for d in rpath_filter_dirs or []])
self.log.debug("Combined RPATH filter: '%s'", rpath_filter)
rpath_include = ','.join(rpath_include_dirs or [])
self.log.debug("Combined RPATH include paths: '%s'", rpath_include)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info(
"%s already seems to be an RPATH wrapper script, not wrapping it again!",
orig_cmd)
continue
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(
orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError(
"Refusing the create a fork bomb, which(%s) == %s",
cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(
tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_include': rpath_include,
'rpath_wrapper_log': rpath_wrapper_log,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd,
which(cmd), rpath_wrapper_log)
else:
self.log.debug(
"Not installing RPATH wrapper for non-existing command '%s'",
cmd)
def test_end2end_singularity_image(self):
"""End-to-end test for --containerize (recipe + image)."""
topdir = os.path.dirname(os.path.abspath(__file__))
toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
containerpath = os.path.join(self.test_prefix, 'containers')
os.environ['EASYBUILD_CONTAINERPATH'] = containerpath
# --containerpath must be an existing directory (this is done to avoid misconfiguration)
mkdir(containerpath)
test_img = os.path.join(self.test_prefix, 'test123.img')
write_file(test_img, '')
args = [
toy_ec,
'-C', # equivalent with --containerize
'--experimental',
'--container-config=bootstrap=localimage,from=%s' % test_img,
'--container-build-image',
]
if which('singularity') is None:
error_pattern = "singularity with version 2.4 or higher not found on your system."
self.assertErrorRegex(EasyBuildError,
error_pattern,
self.eb_main,
args,
raise_error=True)
# install mocked versions of 'sudo' and 'singularity' commands
singularity = os.path.join(self.test_prefix, 'bin', 'singularity')
write_file(singularity, '') # placeholder
adjust_permissions(singularity, stat.S_IXUSR, add=True)
sudo = os.path.join(self.test_prefix, 'bin', 'sudo')
write_file(
sudo,
'#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n'
)
adjust_permissions(sudo, stat.S_IXUSR, add=True)
os.environ['PATH'] = os.path.pathsep.join(
[os.path.join(self.test_prefix, 'bin'),
os.getenv('PATH')])
for (version, ext) in [('2.4.0', 'simg'), ('3.1.0', 'sif')]:
write_file(singularity, MOCKED_SINGULARITY % {'version': version})
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs = [
r"^== singularity tool found at %s/bin/singularity" %
self.test_prefix,
r"^== singularity version '%s' is 2.4 or higher ... OK" %
version,
r"^== Singularity definition file created at %s/containers/Singularity\.toy-0.0"
% self.test_prefix,
r"^== Running 'sudo\s*\S*/singularity build\s*/.* /.*', you may need to enter your 'sudo' password...",
r"^== Singularity image created at %s/containers/toy-0.0\.%s" %
(self.test_prefix, ext),
]
self.check_regexs(regexs, stdout)
self.assertTrue(
os.path.exists(os.path.join(containerpath,
'toy-0.0.%s' % ext)))
remove_file(os.path.join(containerpath, 'Singularity.toy-0.0'))
# check use of --container-image-format & --container-image-name
write_file(singularity, MOCKED_SINGULARITY % {'version': '2.4.0'})
args.extend([
"--container-image-format=ext3",
"--container-image-name=foo-bar",
])
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs = [
r"^== singularity tool found at %s/bin/singularity" %
self.test_prefix,
r"^== singularity version '2.4.0' is 2.4 or higher ... OK",
r"^== Singularity definition file created at %s/containers/Singularity\.foo-bar"
% self.test_prefix,
r"^== Running 'sudo\s*\S*/singularity build --writable /.* /.*', you may need to enter .*",
r"^== Singularity image created at %s/containers/foo-bar\.img$" %
self.test_prefix,
]
self.check_regexs(regexs, stdout)
cont_img = os.path.join(containerpath, 'foo-bar.img')
self.assertTrue(os.path.exists(cont_img))
remove_file(os.path.join(containerpath, 'Singularity.foo-bar'))
# test again with container image already existing
error_pattern = "Container image already exists at %s, not overwriting it without --force" % cont_img
self.mock_stdout(True)
self.assertErrorRegex(EasyBuildError,
error_pattern,
self.run_main,
args,
raise_error=True)
self.mock_stdout(False)
args.append('--force')
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs.extend([
"WARNING: overwriting existing container image at %s due to --force"
% cont_img,
])
self.check_regexs(regexs, stdout)
self.assertTrue(os.path.exists(cont_img))
# also check behaviour under --extended-dry-run
args.append('--extended-dry-run')
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
self.check_regexs(regexs, stdout)
# test use of --container-tmpdir
args.append('--container-tmpdir=%s' % self.test_prefix)
stdout, stderr = self.run_main(args)
self.assertFalse(stderr)
regexs[
-3] = r"^== Running 'sudo\s*SINGULARITY_TMPDIR=%s \S*/singularity build .*" % self.test_prefix
self.check_regexs(regexs, stdout)
def configure_step(self):
"""Custom configuration procedure for GROMACS: set configure options for configure or cmake."""
if LooseVersion(self.version) >= LooseVersion('4.6'):
cuda = get_software_root('CUDA')
if cuda:
# CUDA with double precision is currently not supported in GROMACS yet
# If easyconfig explicitly have double_precision=True error out,
# otherwise warn about it and skip the double precision build
if self.cfg.get('double_precision'):
raise EasyBuildError("Double precision is not available for GPU build. " +
"Please explicitly set \"double_precision = False\" " +
"or remove it in the easyconfig file.")
if self.double_prec_pattern in self.cfg['configopts']:
if self.cfg.get('double_precision') is None:
# Only print warning once when trying double precision
# build the first time
self.cfg['double_precision'] = False
self.log.info("Double precision is not available for " +
"GPU build. Skipping the double precision build.")
self.log.info("skipping configure step")
return
self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda)
else:
# explicitly disable GPU support if CUDA is not available,
# to avoid that GROMACS find and uses a system-wide CUDA compiler
self.cfg.update('configopts', "-DGMX_GPU=OFF")
# check whether PLUMED is loaded as a dependency
plumed_root = get_software_root('PLUMED')
if plumed_root:
# Need to check if PLUMED has an engine for this version
engine = 'gromacs-%s' % self.version
(out, _) = run_cmd("plumed-patch -l", log_all=True, simple=False)
if not re.search(engine, out):
raise EasyBuildError("There is no support in PLUMED version %s for GROMACS %s: %s",
get_software_version('PLUMED'), self.version, out)
# PLUMED patching must be done at different stages depending on
# version of GROMACS. Just prepare first part of cmd here
plumed_cmd = "plumed-patch -p -e %s" % engine
if LooseVersion(self.version) < LooseVersion('4.6'):
self.log.info("Using configure script for configuring GROMACS build.")
if self.cfg['build_shared_libs']:
self.cfg.update('configopts', "--enable-shared --disable-static")
else:
self.cfg.update('configopts', "--enable-static")
# Use external BLAS and LAPACK
self.cfg.update('configopts', "--with-external-blas --with-external-lapack")
env.setvar('LIBS', "%s %s" % (os.environ['LIBLAPACK'], os.environ['LIBS']))
# Don't use the X window system
self.cfg.update('configopts', "--without-x")
# OpenMP is not supported for versions older than 4.5.
if LooseVersion(self.version) >= LooseVersion('4.5'):
# enable OpenMP support if desired
if self.toolchain.options.get('openmp', None):
self.cfg.update('configopts', "--enable-threads")
else:
self.cfg.update('configopts', "--disable-threads")
elif self.toolchain.options.get('openmp', None):
raise EasyBuildError("GROMACS version %s does not support OpenMP" % self.version)
# GSL support
if get_software_root('GSL'):
self.cfg.update('configopts', "--with-gsl")
else:
self.cfg.update('configopts', "--without-gsl")
# actually run configure via ancestor (not direct parent)
self.cfg['configure_cmd'] = "./configure"
ConfigureMake.configure_step(self)
# Now patch GROMACS for PLUMED between configure and build
if plumed_root:
run_cmd(plumed_cmd, log_all=True, simple=True)
else:
if '-DGMX_MPI=ON' in self.cfg['configopts']:
mpi_numprocs = self.cfg.get('mpi_numprocs', 0)
if mpi_numprocs == 0:
self.log.info("No number of test MPI tasks specified -- using default: %s",
self.cfg['parallel'])
mpi_numprocs = self.cfg['parallel']
elif mpi_numprocs > self.cfg['parallel']:
self.log.warning("Number of test MPI tasks (%s) is greater than value for 'parallel': %s",
mpi_numprocs, self.cfg['parallel'])
mpiexec = self.cfg.get('mpiexec')
if mpiexec:
mpiexec_path = which(mpiexec)
if mpiexec_path:
self.cfg.update('configopts', "-DMPIEXEC=%s" % mpiexec_path)
self.cfg.update('configopts', "-DMPIEXEC_NUMPROC_FLAG=%s" %
self.cfg.get('mpiexec_numproc_flag'))
self.cfg.update('configopts', "-DNUMPROC=%s" % mpi_numprocs)
elif self.cfg['runtest']:
raise EasyBuildError("'%s' not found in $PATH", mpiexec)
else:
raise EasyBuildError("No value found for 'mpiexec'")
self.log.info("Using %s as MPI executable when testing, with numprocs flag '%s' and %s tasks",
mpiexec_path, self.cfg.get('mpiexec_numproc_flag'),
mpi_numprocs)
if LooseVersion(self.version) >= LooseVersion('2019'):
# Building the gmxapi interface requires shared libraries
self.cfg['build_shared_libs'] = True
self.cfg.update('configopts', "-DGMXAPI=ON")
if LooseVersion(self.version) >= LooseVersion('2020'):
# build Python bindings if Python is loaded as a dependency
python_root = get_software_root('Python')
if python_root:
bin_python = os.path.join(python_root, 'bin', 'python')
self.cfg.update('configopts', "-DPYTHON_EXECUTABLE=%s" % bin_python)
self.cfg.update('configopts', "-DGMX_PYTHON_PACKAGE=ON")
# Now patch GROMACS for PLUMED before cmake
if plumed_root:
if LooseVersion(self.version) >= LooseVersion('5.1'):
# Use shared or static patch depending on
# setting of self.cfg['build_shared_libs']
# and adapt cmake flags accordingly as per instructions
# from "plumed patch -i"
if self.cfg['build_shared_libs']:
mode = 'shared'
else:
mode = 'static'
plumed_cmd = plumed_cmd + ' -m %s' % mode
run_cmd(plumed_cmd, log_all=True, simple=True)
# prefer static libraries, if available
if self.cfg['build_shared_libs']:
self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=OFF")
else:
self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON")
# always specify to use external BLAS/LAPACK
self.cfg.update('configopts', "-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON")
# disable GUI tools
self.cfg.update('configopts', "-DGMX_X11=OFF")
# convince to build for an older architecture than present on the build node by setting GMX_SIMD CMake flag
# it does not make sense for Cray, because OPTARCH is defined by the Cray Toolchain
if self.toolchain.toolchain_family() != toolchain.CRAYPE:
gmx_simd = self.get_gromacs_arch()
if gmx_simd:
if LooseVersion(self.version) < LooseVersion('5.0'):
self.cfg.update('configopts', "-DGMX_CPU_ACCELERATION=%s" % gmx_simd)
else:
self.cfg.update('configopts', "-DGMX_SIMD=%s" % gmx_simd)
# set regression test path
prefix = 'regressiontests'
if any([src['name'].startswith(prefix) for src in self.src]):
self.cfg.update('configopts', "-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " % prefix)
# enable OpenMP support if desired
if self.toolchain.options.get('openmp', None):
self.cfg.update('configopts', "-DGMX_OPENMP=ON")
else:
self.cfg.update('configopts', "-DGMX_OPENMP=OFF")
imkl_root = get_software_root('imkl')
if imkl_root:
# using MKL for FFT, so it will also be used for BLAS/LAPACK
imkl_include = os.path.join(imkl_root, 'mkl', 'include')
self.cfg.update('configopts', '-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="%s" ' % imkl_include)
libs = os.getenv('LAPACK_STATIC_LIBS').split(',')
mkl_libs = [os.path.join(os.getenv('LAPACK_LIB_DIR'), lib) for lib in libs if lib != 'libgfortran.a']
mkl_libs = ['-Wl,--start-group'] + mkl_libs + ['-Wl,--end-group -lpthread -lm -ldl']
self.cfg.update('configopts', '-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs))
else:
for libname in ['BLAS', 'LAPACK']:
libdir = os.getenv('%s_LIB_DIR' % libname)
if self.toolchain.toolchain_family() == toolchain.CRAYPE:
libsci_mpi_mp_lib = glob.glob(os.path.join(libdir, 'libsci_*_mpi_mp.a'))
if libsci_mpi_mp_lib:
self.cfg.update('configopts', '-DGMX_%s_USER=%s' % (libname, libsci_mpi_mp_lib[0]))
else:
raise EasyBuildError("Failed to find libsci library to link with for %s", libname)
else:
# -DGMX_BLAS_USER & -DGMX_LAPACK_USER require full path to library
libs = os.getenv('%s_STATIC_LIBS' % libname).split(',')
libpaths = [os.path.join(libdir, lib) for lib in libs if lib != 'libgfortran.a']
self.cfg.update('configopts', '-DGMX_%s_USER="******"' % (libname, ';'.join(libpaths)))
# if libgfortran.a is listed, make sure it gets linked in too to avoiding linking issues
if 'libgfortran.a' in libs:
env.setvar('LDFLAGS', "%s -lgfortran -lm" % os.environ.get('LDFLAGS', ''))
# no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472
if LooseVersion(self.version) < LooseVersion('5.0'):
# enable GSL when it's provided
if get_software_root('GSL'):
self.cfg.update('configopts', "-DGMX_GSL=ON")
else:
self.cfg.update('configopts', "-DGMX_GSL=OFF")
# include flags for linking to zlib/XZ in $LDFLAGS if they're listed as a dep;
# this is important for the tests, to correctly link against libxml2
for dep, link_flag in [('XZ', '-llzma'), ('zlib', '-lz')]:
root = get_software_root(dep)
if root:
libdir = get_software_libdir(dep)
ldflags = os.environ.get('LDFLAGS', '')
env.setvar('LDFLAGS', "%s -L%s %s" % (ldflags, os.path.join(root, libdir), link_flag))
# complete configuration with configure_method of parent
out = super(EB_GROMACS, self).configure_step()
# for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used
if LooseVersion(self.version) >= LooseVersion('4.6.5'):
patterns = [
r"Using external FFT library - \S*",
r"Looking for dgemm_ - found",
r"Looking for cheev_ - found",
]
for pattern in patterns:
regex = re.compile(pattern, re.M)
if not regex.search(out):
raise EasyBuildError("Pattern '%s' not found in GROMACS configuration output.", pattern)
def fixup_hardcoded_paths(self):
"""Patch out hard coded paths to /tmp, compiler and binutils tools"""
# replace hardcoded /tmp in java build scripts
regex_subs = [
(r'`mktemp -d /tmp/tmp.XXXXXXXXXX`',
'$$(mktemp -d $${TMPDIR:-/tmp}/tmp.XXXXXXXXXX)'),
]
filepath = os.path.join('src', 'main', 'java', 'com', 'google',
'devtools', 'build', 'lib', 'BUILD')
if os.path.exists(filepath):
apply_regex_substitutions(filepath, regex_subs)
binutils_root = get_software_root('binutils')
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
gcc_ver = get_software_version('GCCcore') or get_software_version(
'GCC')
# only patch Bazel scripts if binutils & GCC installation prefix could be determined
if not binutils_root or not gcc_root:
self.log.info(
"Not patching Bazel build scripts, installation prefix for binutils/GCC not found"
)
return
# replace hardcoded paths in (unix_)cc_configure.bzl
# hard-coded paths in (unix_)cc_configure.bzl were removed in 0.19.0
if LooseVersion(self.version) < LooseVersion('0.19.0'):
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
]
for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
filepath = os.path.join('tools', 'cpp', conf_bzl)
if os.path.exists(filepath):
apply_regex_substitutions(filepath, regex_subs)
# replace hardcoded paths in CROSSTOOL
# CROSSTOOL script is no longer there in Bazel 0.24.0
if LooseVersion(self.version) < LooseVersion('0.24.0'):
res = glob.glob(
os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError(
"Failed to pinpoint location of GCC include files: %s",
res)
gcc_lib_inc_bis = os.path.join(os.path.dirname(gcc_lib_inc),
'include-fixed')
if not os.path.exists(gcc_lib_inc_bis):
self.log.info(
"Derived directory %s does not exist, falling back to %s",
gcc_lib_inc_bis, gcc_lib_inc)
gcc_lib_inc_bis = gcc_lib_inc
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++',
gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist",
gcc_cplusplus_inc)
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'(cxx_builtin_include_directory:.*)/usr/lib/gcc',
r'\1%s' % gcc_lib_inc),
(r'(cxx_builtin_include_directory:.*)/usr/local/include',
r'\1%s' % gcc_lib_inc_bis),
(r'(cxx_builtin_include_directory:.*)/usr/include',
r'\1%s' % gcc_cplusplus_inc),
]
for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin',
tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'",
tool)
apply_regex_substitutions(
os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
def pick_python_cmd(req_maj_ver=None, req_min_ver=None):
"""
Pick 'python' command to use, based on specified version requirements.
If the major version is specified, it must be an exact match (==).
If the minor version is specified, it is considered a minimal minor version (>=).
List of considered 'python' commands (in order)
* 'python' available through $PATH
* 'python<major_ver>' available through $PATH
* 'python<major_ver>.<minor_ver>' available through $PATH
* Python executable used in current session (sys.executable)
"""
log = fancylogger.getLogger('pick_python_cmd', fname=False)
def check_python_cmd(python_cmd):
"""Check whether specified Python command satisfies requirements."""
# check whether specified Python command is available
if os.path.isabs(python_cmd):
if not os.path.isfile(python_cmd):
log.debug("Python command '%s' does not exist", python_cmd)
return False
else:
python_cmd_path = which(python_cmd)
if python_cmd_path is None:
log.debug("Python command '%s' not available through $PATH", python_cmd)
return False
if req_maj_ver is not None:
if req_min_ver is None:
req_majmin_ver = '%s.0' % req_maj_ver
else:
req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver)
pycode = 'import sys; print("%s.%s" % sys.version_info[:2])'
out, _ = run_cmd("%s -c '%s'" % (python_cmd, pycode), simple=False)
out = out.strip()
# (strict) check for major version
maj_ver = out.split('.')[0]
if maj_ver != str(req_maj_ver):
log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver)
return False
# check for minimal minor version
if LooseVersion(out) < LooseVersion(req_majmin_ver):
log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", out, req_majmin_ver)
return False
# all check passed
log.debug("All check passed for Python command '%s'!", python_cmd)
return True
# compose list of 'python' commands to consider
python_cmds = ['python']
if req_maj_ver:
python_cmds.append('python%s' % req_maj_ver)
if req_min_ver:
python_cmds.append('python%s.%s' % (req_maj_ver, req_min_ver))
python_cmds.append(sys.executable)
log.debug("Considering Python commands: %s", ', '.join(python_cmds))
# try and find a 'python' command that satisfies the requirements
res = None
for python_cmd in python_cmds:
if check_python_cmd(python_cmd):
log.debug("Python command '%s' satisfies version requirements!", python_cmd)
if os.path.isabs(python_cmd):
res = python_cmd
else:
res = which(python_cmd)
log.debug("Absolute path to retained Python command: %s", res)
break
else:
log.debug("Python command '%s' does not satisfy version requirements (maj: %s, min: %s), moving on",
req_maj_ver, req_min_ver, python_cmd)
return res
def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None):
"""
Put RPATH wrapper script in place for compiler and linker commands
:param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...)
"""
if get_os_type() == LINUX:
self.log.info("Putting RPATH wrappers in place...")
else:
raise EasyBuildError("RPATH linking is currently only supported on Linux")
if rpath_filter_dirs is None:
rpath_filter_dirs = []
# always include filter for 'stubs' library directory,
# cfr. https://github.com/easybuilders/easybuild-framework/issues/2683
rpath_filter_dirs.append('.*/lib(64)?/stubs/?')
# directory where all wrappers will be placed
wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR)
# must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)?
c_comps, fortran_comps = self.compilers()
rpath_args_py = find_eb_script('rpath_args.py')
rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in')
# figure out list of patterns to use in rpath filter
rpath_filter = build_option('rpath_filter')
if rpath_filter is None:
rpath_filter = ['/lib.*', '/usr.*']
self.log.debug("No general RPATH filter specified, falling back to default: %s", rpath_filter)
rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs])
self.log.debug("Combined RPATH filter: '%s'", rpath_filter)
rpath_include = ','.join(rpath_include_dirs or [])
self.log.debug("Combined RPATH include paths: '%s'", rpath_include)
# create wrappers
for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']):
orig_cmd = which(cmd)
if orig_cmd:
# bail out early if command already is a wrapped;
# this may occur when building extensions
if self.is_rpath_wrapper(orig_cmd):
self.log.info("%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd)
continue
# determine location for this wrapper
# each wrapper is placed in its own subdirectory to enable $PATH filtering per wrapper separately
# avoid '+' character in directory name (for example with 'g++' command), which can cause trouble
# (see https://github.com/easybuilders/easybuild-easyconfigs/issues/7339)
wrapper_dir_name = '%s_wrapper' % cmd.replace('+', 'x')
wrapper_dir = os.path.join(wrappers_dir, wrapper_dir_name)
cmd_wrapper = os.path.join(wrapper_dir, cmd)
# make *very* sure we don't wrap around ourselves and create a fork bomb...
if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper):
raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd)
# enable debug mode in wrapper script by specifying location for log file
if build_option('debug'):
rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd)
else:
rpath_wrapper_log = '/dev/null'
# complete template script and put it in place
cmd_wrapper_txt = read_file(rpath_wrapper_template) % {
'orig_cmd': orig_cmd,
'python': sys.executable,
'rpath_args_py': rpath_args_py,
'rpath_filter': rpath_filter,
'rpath_include': rpath_include,
'rpath_wrapper_log': rpath_wrapper_log,
'wrapper_dir': wrapper_dir,
}
write_file(cmd_wrapper, cmd_wrapper_txt)
adjust_permissions(cmd_wrapper, stat.S_IXUSR)
self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log)
# prepend location to this wrapper to $PATH
setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH')))
else:
self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd)
def configure_step(self, srcdir=None, builddir=None):
"""Configure build using cmake"""
# Set the search paths for CMake
tc_ipaths = self.toolchain.get_variable("CPPFLAGS", list)
tc_lpaths = self.toolchain.get_variable("LDFLAGS", list)
cpaths = os.getenv('CPATH', '').split(os.pathsep)
lpaths = os.getenv('LD_LIBRARY_PATH', '').split(os.pathsep)
include_paths = os.pathsep.join(nub(tc_ipaths + cpaths))
library_paths = os.pathsep.join(nub(tc_lpaths + lpaths))
setvar("CMAKE_INCLUDE_PATH", include_paths)
setvar("CMAKE_LIBRARY_PATH", library_paths)
if builddir is None and self.cfg.get('separate_build_dir', False):
builddir = os.path.join(self.builddir, 'easybuild_obj')
if builddir:
mkdir(builddir, parents=True)
change_dir(builddir)
default_srcdir = self.cfg['start_dir']
else:
default_srcdir = '.'
if srcdir is None:
if self.cfg.get('srcdir', None) is not None:
srcdir = self.cfg['srcdir']
else:
srcdir = default_srcdir
options = ['-DCMAKE_INSTALL_PREFIX=%s' % self.installdir]
env_to_options = {
'CC': 'CMAKE_C_COMPILER',
'CFLAGS': 'CMAKE_C_FLAGS',
'CXX': 'CMAKE_CXX_COMPILER',
'CXXFLAGS': 'CMAKE_CXX_FLAGS',
'F90': 'CMAKE_Fortran_COMPILER',
'FFLAGS': 'CMAKE_Fortran_FLAGS',
}
for env_name, option in env_to_options.items():
value = os.getenv(env_name)
if value is not None:
if option.endswith('_COMPILER') and self.cfg.get(
'abs_path_compilers', False):
value = which(value)
self.log.info(
"Using absolute path to compiler command: %s", value)
options.append("-D%s='%s'" % (option, value))
if build_option('rpath'):
# instruct CMake not to fiddle with RPATH when --rpath is used, since it will undo stuff on install...
# https://github.com/LLNL/spack/blob/0f6a5cd38538e8969d11bd2167f11060b1f53b43/lib/spack/spack/build_environment.py#L416
options.append('-DCMAKE_SKIP_RPATH=ON')
# show what CMake is doing by default
options.append('-DCMAKE_VERBOSE_MAKEFILE=ON')
if not self.cfg.get('allow_system_boost', False):
# don't pick up on system Boost if Boost is included as dependency
# - specify Boost location via -DBOOST_ROOT
# - instruct CMake to not search for Boost headers/libraries in other places
# - disable search for Boost CMake package configuration file
boost_root = get_software_root('Boost')
if boost_root:
options.extend([
'-DBOOST_ROOT=%s' % boost_root,
'-DBoost_NO_SYSTEM_PATHS=ON',
'-DBoost_NO_BOOST_CMAKE=ON',
])
options_string = ' '.join(options)
if self.cfg.get('configure_cmd') == DEFAULT_CONFIGURE_CMD:
command = ' '.join([
self.cfg['preconfigopts'], DEFAULT_CONFIGURE_CMD,
options_string, self.cfg['configopts'], srcdir
])
else:
command = ' '.join([
self.cfg['preconfigopts'],
self.cfg.get('configure_cmd'), self.cfg['configopts']
])
(out, _) = run_cmd(command, log_all=True, simple=False)
return out
def configure_step(self):
"""Custom configuration procedure for Bazel."""
binutils_root = get_software_root('binutils')
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
gcc_ver = get_software_version('GCCcore') or get_software_version(
'GCC')
# only patch Bazel scripts if binutils & GCC installation prefix could be determined
if binutils_root and gcc_root:
res = glob.glob(
os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError(
"Failed to pinpoint location of GCC include files: %s",
res)
gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc),
'include-fixed')
if not os.path.exists(gcc_lib_inc_fixed):
raise EasyBuildError("Derived directory %s does not exist",
gcc_lib_inc_fixed)
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++',
gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist",
gcc_cplusplus_inc)
# replace hardcoded paths in CROSSTOOL
# CROSSTOOL script is no longer there in Bazel 0.24.0
if LooseVersion(self.version) < LooseVersion('0.24.0'):
regex_subs = [
(r'-B/usr/bin',
'-B%s' % os.path.join(binutils_root, 'bin')),
(r'(cxx_builtin_include_directory:.*)/usr/lib/gcc',
r'\1%s' % gcc_lib_inc),
(r'(cxx_builtin_include_directory:.*)/usr/local/include',
r'\1%s' % gcc_lib_inc_fixed),
(r'(cxx_builtin_include_directory:.*)/usr/include',
r'\1%s' % gcc_cplusplus_inc),
]
for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin',
tool), path))
else:
raise EasyBuildError(
"Failed to determine path to '%s'", tool)
apply_regex_substitutions(
os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
# replace hardcoded paths in (unix_)cc_configure.bzl
regex_subs = [
(r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')),
(r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')),
]
for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']:
filepath = os.path.join('tools', 'cpp', conf_bzl)
if os.path.exists(filepath):
apply_regex_substitutions(filepath, regex_subs)
else:
self.log.info(
"Not patching Bazel build scripts, installation prefix for binutils/GCC not found"
)
# enable building in parallel
bazel_args = '--jobs=%d' % self.cfg['parallel']
# Bazel provides a JDK by itself for some architectures
# We want to enforce it using the JDK we provided via modules
# This is required for Power where Bazel does not have a JDK, but requires it for building itself
# See https://github.com/bazelbuild/bazel/issues/10377
bazel_args += ' --host_javabase=@local_jdk//:jdk'
env.setvar('EXTRA_BAZEL_ARGS', bazel_args)
def test_build_easyconfigs_in_parallel_gc3pie(self):
"""Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job."""
try:
import gc3libs # noqa (ignore unused import)
except ImportError:
print "GC3Pie not available, skipping test"
return
# put GC3Pie config in place to use local host and fork/exec
resourcedir = os.path.join(self.test_prefix, 'gc3pie')
gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini')
gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % {
'resourcedir': resourcedir,
'time': which('time'),
}
write_file(gc3pie_cfgfile, gc3pie_cfgtxt)
output_dir = os.path.join(self.test_prefix, 'subdir',
'gc3pie_output_dir')
# purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir)
mkdir(output_dir, parents=True)
write_file(os.path.join(output_dir, 'foo'), 'bar')
# remove write permissions on parent dir of specified output dir,
# to check that GC3Pie does not try to rename the (already existing) output directory...
adjust_permissions(os.path.dirname(output_dir),
stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH,
add=False,
recursive=False)
topdir = os.path.dirname(os.path.abspath(__file__))
build_options = {
'job_backend_config': gc3pie_cfgfile,
'job_max_walltime': 24,
'job_output_dir': output_dir,
'job_polling_interval': 0.2, # quick polling
'job_target_resource': 'ebtestlocalhost',
'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'),
'silent': True,
'valid_module_classes': config.module_classes(),
'validate': False,
}
init_config(args=['--job-backend=GC3Pie'], build_options=build_options)
ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy',
'toy-0.0.eb')
easyconfigs = process_easyconfig(ec_file)
ordered_ecs = resolve_dependencies(easyconfigs, self.modtool)
topdir = os.path.dirname(
os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
test_easyblocks_path = os.path.join(
os.path.dirname(os.path.abspath(__file__)), 'sandbox')
cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (
topdir, test_easyblocks_path)
build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False)
self.assertTrue(
os.path.join(self.test_installpath, 'modules', 'all', 'toy',
'0.0'))
self.assertTrue(
os.path.join(self.test_installpath, 'software', 'toy', '0.0',
'bin', 'toy'))
def build_step(self):
"""Custom build procedure for TensorFlow."""
# pre-create target installation directory
mkdir(os.path.join(self.installdir, self.pylibdir), parents=True)
binutils_root = get_software_root('binutils')
if binutils_root:
binutils_bin = os.path.join(binutils_root, 'bin')
else:
raise EasyBuildError("Failed to determine installation prefix for binutils")
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
if gcc_root:
gcc_lib64 = os.path.join(gcc_root, 'lib64')
gcc_ver = get_software_version('GCCcore') or get_software_version('GCC')
# figure out location of GCC include files
res = glob.glob(os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError("Failed to pinpoint location of GCC include files: %s", res)
# make sure include-fixed directory is where we expect it to be
gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed')
if not os.path.exists(gcc_lib_inc_fixed):
raise EasyBuildError("Derived directory %s does not exist", gcc_lib_inc_fixed)
# also check on location of include/c++/<gcc version> directory
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc)
else:
raise EasyBuildError("Failed to determine installation prefix for GCC")
inc_paths = [gcc_lib_inc, gcc_lib_inc_fixed, gcc_cplusplus_inc]
lib_paths = [gcc_lib64]
cuda_root = get_software_root('CUDA')
if cuda_root:
inc_paths.append(os.path.join(cuda_root, 'include'))
lib_paths.append(os.path.join(cuda_root, 'lib64'))
# fix hardcoded locations of compilers & tools
cxx_inc_dir_lines = '\n'.join(r'cxx_builtin_include_directory: "%s"' % resolve_path(p) for p in inc_paths)
cxx_inc_dir_lines_no_resolv_path = '\n'.join(r'cxx_builtin_include_directory: "%s"' % p for p in inc_paths)
regex_subs = [
(r'-B/usr/bin/', '-B%s/ %s' % (binutils_bin, ' '.join('-L%s/' % p for p in lib_paths))),
(r'(cxx_builtin_include_directory:).*', ''),
(r'^toolchain {', 'toolchain {\n' + cxx_inc_dir_lines + '\n' + cxx_inc_dir_lines_no_resolv_path),
]
for tool in ['ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy', 'objdump', 'strip']:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin', tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'", tool)
# -fPIE/-pie and -fPIC are not compatible, so patch out hardcoded occurences of -fPIE/-pie if -fPIC is used
if self.toolchain.options.get('pic', None):
regex_subs.extend([('-fPIE', '-fPIC'), ('"-pie"', '"-fPIC"')])
# patch all CROSSTOOL* scripts to fix hardcoding of locations of binutils/GCC binaries
for path, dirnames, filenames in os.walk(self.start_dir):
for filename in filenames:
if filename.startswith('CROSSTOOL'):
full_path = os.path.join(path, filename)
self.log.info("Patching %s", full_path)
apply_regex_substitutions(full_path, regex_subs)
tmpdir = tempfile.mkdtemp(suffix='-bazel-build')
# compose "bazel build" command with all its options...
cmd = [self.cfg['prebuildopts'], 'bazel', '--output_base=%s' % tmpdir, 'build']
# build with optimization enabled
# cfr. https://docs.bazel.build/versions/master/user-manual.html#flag--compilation_mode
cmd.append('--compilation_mode=opt')
# select 'opt' config section (this is *not* the same as --compilation_mode=opt!)
# https://docs.bazel.build/versions/master/user-manual.html#flag--config
cmd.append('--config=opt')
# make Bazel print full command line + make it verbose on failures
# https://docs.bazel.build/versions/master/user-manual.html#flag--subcommands
# https://docs.bazel.build/versions/master/user-manual.html#flag--verbose_failures
cmd.extend(['--subcommands', '--verbose_failures'])
# limit the number of parallel jobs running simultaneously (useful on KNL)...
cmd.append('--jobs=%s' % self.cfg['parallel'])
if self.toolchain.options.get('pic', None):
cmd.append('--copt="-fPIC"')
cmd.append(self.cfg['buildopts'])
if cuda_root:
cmd.append('--config=cuda')
# enable mkl-dnn by default, but only if cuDNN is not listed as dependency
if self.cfg['with_mkl_dnn'] is None and get_software_root('cuDNN') is None:
self.log.info("Enabling use of mkl-dnn since cuDNN is not listed as dependency")
self.cfg['with_mkl_dnn'] = True
# if mkl-dnn is listed as a dependency it is used. Otherwise downloaded if with_mkl_dnn is true
mkl_root = get_software_root('mkl-dnn')
if mkl_root:
cmd.extend(['--config=mkl'])
cmd.insert(0, "export TF_MKL_DOWNLOAD=0 &&")
cmd.insert(0, "export TF_MKL_ROOT=%s &&" % mkl_root)
elif self.cfg['with_mkl_dnn']:
# this makes TensorFlow use mkl-dnn (cfr. https://github.com/01org/mkl-dnn)
cmd.extend(['--config=mkl'])
cmd.insert(0, "export TF_MKL_DOWNLOAD=1 && ")
# specify target of the build command as last argument
cmd.append('//tensorflow/tools/pip_package:build_pip_package')
run_cmd(' '.join(cmd), log_all=True, simple=True, log_ok=True)
# run generated 'build_pip_package' script to build the .whl
cmd = "bazel-bin/tensorflow/tools/pip_package/build_pip_package %s" % self.builddir
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
def configure_step(self, srcdir=None, builddir=None):
"""Configure build using cmake"""
setup_cmake_env(self.toolchain)
if builddir is None and self.cfg.get('separate_build_dir', True):
builddir = create_unused_dir(self.builddir, 'easybuild_obj')
if builddir:
mkdir(builddir, parents=True)
change_dir(builddir)
default_srcdir = self.cfg['start_dir']
else:
default_srcdir = '.'
if srcdir is None:
if self.cfg.get('srcdir', None) is not None:
# Note that the join returns srcdir if it is absolute
srcdir = os.path.join(default_srcdir, self.cfg['srcdir'])
else:
srcdir = default_srcdir
options = ['-DCMAKE_INSTALL_PREFIX=%s' % self.installdir]
if self.installdir.startswith('/opt') or self.installdir.startswith(
'/usr'):
# https://cmake.org/cmake/help/latest/module/GNUInstallDirs.html
localstatedir = os.path.join(self.installdir, 'var')
runstatedir = os.path.join(localstatedir, 'run')
sysconfdir = os.path.join(self.installdir, 'etc')
options.append("-DCMAKE_INSTALL_LOCALSTATEDIR=%s" % localstatedir)
options.append("-DCMAKE_INSTALL_RUNSTATEDIR=%s" % runstatedir)
options.append("-DCMAKE_INSTALL_SYSCONFDIR=%s" % sysconfdir)
if '-DCMAKE_BUILD_TYPE=' in self.cfg['configopts']:
if self.cfg.get('build_type') is not None:
self.log.warning(
'CMAKE_BUILD_TYPE is set in configopts. Ignoring build_type'
)
else:
options.append('-DCMAKE_BUILD_TYPE=%s' % self.build_type)
# Add -fPIC flag if necessary
if self.toolchain.options['pic']:
options.append('-DCMAKE_POSITION_INDEPENDENT_CODE=ON')
if self.cfg['generator']:
options.append('-G "%s"' % self.cfg['generator'])
# pass --sysroot value down to CMake,
# and enable using absolute paths to compiler commands to avoid
# that CMake picks up compiler from sysroot rather than toolchain compiler...
sysroot = build_option('sysroot')
if sysroot:
options.append('-DCMAKE_SYSROOT=%s' % sysroot)
self.log.info(
"Using absolute path to compiler commands because of alterate sysroot %s",
sysroot)
self.cfg['abs_path_compilers'] = True
# Set flag for shared libs if requested
# Not adding one allows the project to choose a default
build_shared_libs = self.cfg.get('build_shared_libs')
if build_shared_libs is not None:
# Contrary to other options build_shared_libs takes precedence over configopts which may be unexpected.
# This is to allow self.lib_ext to be determined correctly.
# Usually you want to remove -DBUILD_SHARED_LIBS from configopts and set build_shared_libs to True or False
# If you need it in configopts don't set build_shared_libs (or explicitely set it to `None` (Default))
if '-DBUILD_SHARED_LIBS=' in self.cfg['configopts']:
print_warning(
'Ignoring BUILD_SHARED_LIBS is set in configopts because build_shared_libs is set'
)
self.cfg.update(
'configopts', '-DBUILD_SHARED_LIBS=%s' %
('ON' if build_shared_libs else 'OFF'))
env_to_options = {
'CC': 'CMAKE_C_COMPILER',
'CFLAGS': 'CMAKE_C_FLAGS',
'CXX': 'CMAKE_CXX_COMPILER',
'CXXFLAGS': 'CMAKE_CXX_FLAGS',
'F90': 'CMAKE_Fortran_COMPILER',
'FFLAGS': 'CMAKE_Fortran_FLAGS',
}
for env_name, option in env_to_options.items():
value = os.getenv(env_name)
if value is not None:
if option.endswith('_COMPILER') and self.cfg.get(
'abs_path_compilers', False):
value = which(value)
self.log.info(
"Using absolute path to compiler command: %s", value)
options.append("-D%s='%s'" % (option, value))
if build_option('rpath'):
# instruct CMake not to fiddle with RPATH when --rpath is used, since it will undo stuff on install...
# https://github.com/LLNL/spack/blob/0f6a5cd38538e8969d11bd2167f11060b1f53b43/lib/spack/spack/build_environment.py#L416
options.append('-DCMAKE_SKIP_RPATH=ON')
# show what CMake is doing by default
options.append('-DCMAKE_VERBOSE_MAKEFILE=ON')
# disable CMake user package repository
options.append('-DCMAKE_FIND_USE_PACKAGE_REGISTRY=FALSE')
if not self.cfg.get('allow_system_boost', False):
# don't pick up on system Boost if Boost is included as dependency
# - specify Boost location via -DBOOST_ROOT
# - instruct CMake to not search for Boost headers/libraries in other places
# - disable search for Boost CMake package configuration file
boost_root = get_software_root('Boost')
if boost_root:
options.extend([
'-DBOOST_ROOT=%s' % boost_root,
'-DBoost_NO_SYSTEM_PATHS=ON',
'-DBoost_NO_BOOST_CMAKE=ON',
])
options_string = ' '.join(options)
if self.cfg.get('configure_cmd') == DEFAULT_CONFIGURE_CMD:
command = ' '.join([
self.cfg['preconfigopts'], DEFAULT_CONFIGURE_CMD,
options_string, self.cfg['configopts'], srcdir
])
else:
command = ' '.join([
self.cfg['preconfigopts'],
self.cfg.get('configure_cmd'), self.cfg['configopts']
])
(out, _) = run_cmd(command, log_all=True, simple=False)
return out
def build_image(self, recipe_path):
"""Build container image by calling out to 'sudo singularity build'."""
cont_path = container_path()
def_file = os.path.basename(recipe_path)
# use --imagename if specified, otherwise derive based on filename of recipe
img_name = self.img_name
if img_name is None:
# definition file Singularity.<app>-<version, container name <app>-<version>.<img|simg>
img_name = def_file.split('.', 1)[1]
cmd_opts = ''
image_format = self.image_format
singularity_version = self.singularity_version()
# squashfs image format (default for Singularity)
if image_format in [
None, CONT_IMAGE_FORMAT_SQUASHFS, CONT_IMAGE_FORMAT_SIF
]:
if LooseVersion(singularity_version) > LooseVersion('3.0'):
ext = '.sif'
else:
ext = '.simg'
img_path = os.path.join(cont_path, img_name + ext)
# ext3 image format, creating as writable container
elif image_format == CONT_IMAGE_FORMAT_EXT3:
if LooseVersion(singularity_version) > LooseVersion('3.0'):
raise EasyBuildError(
"ext3 image format is only supported with Singularity 2.x (found Singularity %s)",
singularity_version)
else:
img_path = os.path.join(cont_path, img_name + '.img')
cmd_opts = '--writable'
# sandbox image format, creates as a directory but acts like a container
elif image_format == CONT_IMAGE_FORMAT_SANDBOX:
img_path = os.path.join(cont_path, img_name)
cmd_opts = '--sandbox'
else:
raise EasyBuildError(
"Unknown container image format specified for Singularity: %s"
% image_format)
if os.path.exists(img_path):
if build_option('force'):
print_msg(
"WARNING: overwriting existing container image at %s due to --force"
% img_path)
remove_file(img_path)
else:
raise EasyBuildError(
"Container image already exists at %s, not overwriting it without --force",
img_path)
# resolve full path to 'singularity' binary, since it may not be available via $PATH under sudo...
singularity = which('singularity')
cmd_env = ''
singularity_tmpdir = self.tmpdir
if singularity_tmpdir:
cmd_env += 'SINGULARITY_TMPDIR=%s' % singularity_tmpdir
cmd = ' '.join([
'sudo', cmd_env, singularity, 'build', cmd_opts, img_path,
recipe_path
])
print_msg(
"Running '%s', you may need to enter your 'sudo' password..." %
cmd)
run_cmd(cmd, stream_output=True)
print_msg("Singularity image created at %s" % img_path, log=self.log)
def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None):
"""
Return a list of absolute paths where the specified subdir can be found, determined by the PYTHONPATH
"""
paths = []
# primary search path is robot path
path_list = []
if isinstance(robot_path, list):
path_list = robot_path[:]
elif robot_path is not None:
path_list = [robot_path]
# consider Python search path, e.g. setuptools install path for easyconfigs
path_list.extend(sys.path)
# figure out installation prefix, e.g. distutils install path for easyconfigs
# prefer using path specified in $EB_SCRIPT_PATH (if defined), which is set by 'eb' wrapper script
eb_path = os.getenv('EB_SCRIPT_PATH')
if eb_path is None:
# try to determine location of 'eb' script via $PATH, as fallback mechanism
eb_path = which('eb')
_log.info("Location to 'eb' script (found via $PATH): %s", eb_path)
else:
_log.info("Found location to 'eb' script via $EB_SCRIPT_PATH: %s",
eb_path)
if eb_path is None:
warning_msg = "'eb' not found in $PATH, failed to determine installation prefix!"
_log.warning(warning_msg)
print_warning(warning_msg)
else:
# eb_path is location to 'eb' wrapper script, e.g. <install_prefix>/bin/eb
# so installation prefix is usually two levels up
install_prefix = os.path.dirname(os.path.dirname(eb_path))
# only consider resolved path to 'eb' script if desired subdir is not found relative to 'eb' script location
if os.path.exists(os.path.join(install_prefix, 'easybuild', subdir)):
path_list.append(install_prefix)
_log.info(
"Also considering installation prefix %s (determined via path to 'eb' script)...",
install_prefix)
else:
_log.info("Not considering %s (no easybuild/%s subdir found)",
install_prefix, subdir)
# also consider fully resolved location to 'eb' wrapper
# see https://github.com/easybuilders/easybuild-framework/pull/2248
resolved_eb_path = resolve_path(eb_path)
if eb_path != resolved_eb_path:
install_prefix = os.path.dirname(
os.path.dirname(resolved_eb_path))
path_list.append(install_prefix)
_log.info(
"Also considering installation prefix %s (via resolved path to 'eb')...",
install_prefix)
# look for desired subdirs
for path in path_list:
path = os.path.join(path, "easybuild", subdir)
_log.debug("Checking for easybuild/%s at %s" % (subdir, path))
try:
if os.path.exists(path):
paths.append(os.path.abspath(path))
_log.debug("Added %s to list of paths for easybuild/%s" %
(path, subdir))
except OSError as err:
raise EasyBuildError(str(err))
return paths
def prepare_step(self, *args, **kwargs):
"""Load all dependencies, determine system MPI version, prefix and any associated envvars."""
# Do the bundle prepare step to ensure any deps are loaded (no need to worry about licences for Intel MPI)
Bundle.prepare_step(self, *args, **kwargs)
# Prepare additional parameters: determine system MPI version, prefix and any associated envvars.
mpi_name = self.cfg['name'].lower()
# Determine MPI wrapper path (real path, with resolved symlinks) to ensure it exists
if mpi_name == 'impi':
# For impi the version information is only found in *some* of the wrappers it ships, in particular it is
# not in mpicc
mpi_c_wrapper = 'mpiicc'
path_to_mpi_c_wrapper = which(mpi_c_wrapper)
if not path_to_mpi_c_wrapper:
mpi_c_wrapper = 'mpigcc'
path_to_mpi_c_wrapper = which(mpi_c_wrapper)
if not path_to_mpi_c_wrapper:
raise EasyBuildError("Could not find suitable MPI wrapper to extract version for impi")
else:
mpi_c_wrapper = 'mpicc'
path_to_mpi_c_wrapper = which(mpi_c_wrapper)
if path_to_mpi_c_wrapper:
path_to_mpi_c_wrapper = resolve_path(path_to_mpi_c_wrapper)
self.log.info("Found path to MPI implementation '%s' %s compiler (with symlinks resolved): %s",
mpi_name, mpi_c_wrapper, path_to_mpi_c_wrapper)
else:
raise EasyBuildError("%s not found in $PATH", mpi_c_wrapper)
# Determine MPI version, installation prefix and underlying compiler
if mpi_name in ('openmpi', 'spectrummpi'):
# Spectrum MPI is based on Open MPI so is also covered by this logic
output_of_ompi_info, _ = run_cmd("ompi_info", simple=False)
# Extract the version of the MPI implementation
if mpi_name == 'spectrummpi':
mpi_version_string = 'Spectrum MPI'
else:
mpi_version_string = 'Open MPI'
self.mpi_version = self.extract_ompi_setting(mpi_version_string, output_of_ompi_info)
# Extract the installation prefix
self.mpi_prefix = self.extract_ompi_setting("Prefix", output_of_ompi_info)
# Extract any OpenMPI environment variables in the current environment and ensure they are added to the
# final module
self.mpi_env_vars = dict((key, value) for key, value in os.environ.items() if key.startswith('OMPI_'))
# Extract the C compiler used underneath the MPI implementation, check for the definition of OMPI_MPICC
self.mpi_c_compiler = self.extract_ompi_setting("C compiler", output_of_ompi_info)
elif mpi_name == 'impi':
# Extract the version of IntelMPI
# The prefix in the the mpiicc (or mpigcc) script can be used to extract the explicit version
contents_of_mpixcc = read_file(path_to_mpi_c_wrapper)
prefix_regex = re.compile(r'(?<=compilers_and_libraries_)(.*)(?=/linux/mpi)', re.M)
self.mpi_version = None
res = prefix_regex.search(contents_of_mpixcc)
if res:
self.mpi_version = res.group(1)
else:
# old iimpi version
prefix_regex = re.compile(r'^prefix=(.*)$', re.M)
res = prefix_regex.search(contents_of_mpixcc)
if res:
self.mpi_version = res.group(1).split('/')[-1]
if self.mpi_version is None:
raise EasyBuildError("No version found for system Intel MPI")
else:
self.log.info("Found Intel MPI version %s for system MPI" % self.mpi_version)
# Extract the installation prefix, if I_MPI_ROOT is defined, let's use that
i_mpi_root = os.environ.get('I_MPI_ROOT')
if i_mpi_root:
self.mpi_prefix = i_mpi_root
else:
# Else just go up three directories from where mpiicc is found
# (it's 3 because bin64 is a symlink to intel64/bin and we are assuming 64 bit)
self.mpi_prefix = os.path.dirname(os.path.dirname(os.path.dirname(path_to_mpi_c_wrapper)))
# Extract any IntelMPI environment variables in the current environment and ensure they are added to the
# final module
self.mpi_env_vars = {}
for key, value in os.environ.items():
i_mpi_key = key.startswith('I_MPI_') or key.startswith('MPICH_')
mpi_profile_key = key.startswith('MPI') and key.endswith('PROFILE')
if i_mpi_key or mpi_profile_key:
self.mpi_env_vars[key] = value
# Extract the C compiler used underneath Intel MPI
compile_info, exit_code = run_cmd("%s -compile-info" % mpi_c_wrapper, simple=False)
if exit_code == 0:
self.mpi_c_compiler = compile_info.split(' ', 1)[0]
else:
raise EasyBuildError("Could not determine C compiler underneath Intel MPI, '%s -compiler-info' "
"returned %s", mpi_c_wrapper, compile_info)
else:
raise EasyBuildError("Unrecognised system MPI implementation %s", mpi_name)
# Ensure install path of system MPI actually exists
if not os.path.exists(self.mpi_prefix):
raise EasyBuildError("Path derived for system MPI (%s) does not exist: %s!", mpi_name, self.mpi_prefix)
self.log.debug("Derived version/install prefix for system MPI %s: %s, %s",
mpi_name, self.mpi_version, self.mpi_prefix)
# For the version of the underlying C compiler need to explicitly extract (to be certain)
self.c_compiler_version = extract_compiler_version(self.mpi_c_compiler)
self.log.debug("Derived compiler/version for C compiler underneath system MPI %s: %s, %s",
mpi_name, self.mpi_c_compiler, self.c_compiler_version)
# If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it
if self.cfg['version'] == 'system':
self.log.info("Found specified version '%s', going with derived MPI version '%s'",
self.cfg['version'], self.mpi_version)
elif self.cfg['version'] == self.mpi_version:
self.log.info("Specified MPI version %s matches found version" % self.mpi_version)
else:
raise EasyBuildError("Specified version (%s) does not match version reported by MPI (%s)",
self.cfg['version'], self.mpi_version)
def build_step(self):
"""Custom build procedure for TensorFlow."""
# pre-create target installation directory
mkdir(os.path.join(self.installdir, self.pylibdir), parents=True)
binutils_root = get_software_root('binutils')
if binutils_root:
binutils_bin = os.path.join(binutils_root, 'bin')
else:
raise EasyBuildError(
"Failed to determine installation prefix for binutils")
gcc_root = get_software_root('GCCcore') or get_software_root('GCC')
if gcc_root:
gcc_lib64 = os.path.join(gcc_root, 'lib64')
gcc_ver = get_software_version('GCCcore') or get_software_version(
'GCC')
# figure out location of GCC include files
res = glob.glob(
os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include'))
if res and len(res) == 1:
gcc_lib_inc = res[0]
else:
raise EasyBuildError(
"Failed to pinpoint location of GCC include files: %s",
res)
# make sure include-fixed directory is where we expect it to be
gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc),
'include-fixed')
if not os.path.exists(gcc_lib_inc_fixed):
raise EasyBuildError("Derived directory %s does not exist",
gcc_lib_inc_fixed)
# also check on location of include/c++/<gcc version> directory
gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++',
gcc_ver)
if not os.path.exists(gcc_cplusplus_inc):
raise EasyBuildError("Derived directory %s does not exist",
gcc_cplusplus_inc)
else:
raise EasyBuildError(
"Failed to determine installation prefix for GCC")
inc_paths = [gcc_lib_inc, gcc_lib_inc_fixed, gcc_cplusplus_inc]
lib_paths = [gcc_lib64]
cuda_root = get_software_root('CUDA')
if cuda_root:
inc_paths.append(os.path.join(cuda_root, 'include'))
lib_paths.append(os.path.join(cuda_root, 'lib64'))
# fix hardcoded locations of compilers & tools
cxx_inc_dir_lines = '\n'.join(r'cxx_builtin_include_directory: "%s"' %
resolve_path(p) for p in inc_paths)
cxx_inc_dir_lines_no_resolv_path = '\n'.join(
r'cxx_builtin_include_directory: "%s"' % p for p in inc_paths)
regex_subs = [
(r'-B/usr/bin/',
'-B%s/ %s' % (binutils_bin, ' '.join('-L%s/' % p
for p in lib_paths))),
(r'(cxx_builtin_include_directory:).*', ''),
(r'^toolchain {', 'toolchain {\n' + cxx_inc_dir_lines + '\n' +
cxx_inc_dir_lines_no_resolv_path),
]
for tool in [
'ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy',
'objdump', 'strip'
]:
path = which(tool)
if path:
regex_subs.append((os.path.join('/usr', 'bin', tool), path))
else:
raise EasyBuildError("Failed to determine path to '%s'", tool)
# -fPIE/-pie and -fPIC are not compatible, so patch out hardcoded occurences of -fPIE/-pie if -fPIC is used
if self.toolchain.options.get('pic', None):
regex_subs.extend([('-fPIE', '-fPIC'), ('"-pie"', '"-fPIC"')])
# patch all CROSSTOOL* scripts to fix hardcoding of locations of binutils/GCC binaries
for path, dirnames, filenames in os.walk(os.getcwd()):
for filename in filenames:
if filename.startswith('CROSSTOOL'):
full_path = os.path.join(path, filename)
self.log.info("Patching %s", full_path)
apply_regex_substitutions(full_path, regex_subs)
tmpdir = tempfile.mkdtemp(suffix='-bazel-build')
user_root_tmpdir = tempfile.mkdtemp(suffix='-user_root')
# compose "bazel build" command with all its options...
cmd = [
self.cfg['prebuildopts'], 'bazel',
'--output_base=%s' % tmpdir,
'--install_base=%s' % os.path.join(tmpdir, 'inst_base'),
'--output_user_root=%s' % user_root_tmpdir, 'build'
]
# build with optimization enabled
# cfr. https://docs.bazel.build/versions/master/user-manual.html#flag--compilation_mode
cmd.append('--compilation_mode=opt')
# select 'opt' config section (this is *not* the same as --compilation_mode=opt!)
# https://docs.bazel.build/versions/master/user-manual.html#flag--config
cmd.append('--config=opt')
# make Bazel print full command line + make it verbose on failures
# https://docs.bazel.build/versions/master/user-manual.html#flag--subcommands
# https://docs.bazel.build/versions/master/user-manual.html#flag--verbose_failures
cmd.extend(['--subcommands', '--verbose_failures'])
# limit the number of parallel jobs running simultaneously (useful on KNL)...
cmd.append('--jobs=%s' % self.cfg['parallel'])
if self.toolchain.options.get('pic', None):
cmd.append('--copt="-fPIC"')
# include install location of Python packages in $PYTHONPATH,
# and specify that value of $PYTHONPATH should be passed down into Bazel build environment;
# this is required to make sure that Python packages included as extensions are found at build time;
# see also https://github.com/tensorflow/tensorflow/issues/22395
pythonpath = os.getenv('PYTHONPATH', '')
env.setvar(
'PYTHONPATH',
os.pathsep.join(
[os.path.join(self.installdir, self.pylibdir), pythonpath]))
cmd.append('--action_env=PYTHONPATH')
# Also export $EBPYTHONPREFIXES to handle the multi-deps python setup
# See https://github.com/easybuilders/easybuild-easyblocks/pull/1664
if 'EBPYTHONPREFIXES' in os.environ:
cmd.append('--action_env=EBPYTHONPREFIXES')
# use same configuration for both host and target programs, which can speed up the build
# only done when optarch is enabled, since this implicitely assumes that host and target platform are the same
# see https://docs.bazel.build/versions/master/guide.html#configurations
if self.toolchain.options.get('optarch'):
cmd.append('--distinct_host_configuration=false')
cmd.append(self.cfg['buildopts'])
# building TensorFlow v2.0 requires passing --config=v2 to "bazel build" command...
if LooseVersion(self.version) >= LooseVersion('2.0'):
cmd.append('--config=v2')
if cuda_root:
cmd.append('--config=cuda')
# if mkl-dnn is listed as a dependency it is used. Otherwise downloaded if with_mkl_dnn is true
mkl_root = get_software_root('mkl-dnn')
if mkl_root:
cmd.extend(['--config=mkl'])
cmd.insert(0, "export TF_MKL_DOWNLOAD=0 &&")
cmd.insert(0, "export TF_MKL_ROOT=%s &&" % mkl_root)
elif self.cfg['with_mkl_dnn']:
# this makes TensorFlow use mkl-dnn (cfr. https://github.com/01org/mkl-dnn)
cmd.extend(['--config=mkl'])
cmd.insert(0, "export TF_MKL_DOWNLOAD=1 && ")
# specify target of the build command as last argument
cmd.append('//tensorflow/tools/pip_package:build_pip_package')
run_cmd(' '.join(cmd), log_all=True, simple=True, log_ok=True)
# run generated 'build_pip_package' script to build the .whl
cmd = "bazel-bin/tensorflow/tools/pip_package/build_pip_package %s" % self.builddir
run_cmd(cmd, log_all=True, simple=True, log_ok=True)
def __init__(self, *args, **kwargs):
"""Extra initialization: determine system compiler version and prefix."""
super(SystemCompiler, self).__init__(*args, **kwargs)
# Determine compiler path (real path, with resolved symlinks)
compiler_name = self.cfg['name'].lower()
path_to_compiler = which(compiler_name)
if path_to_compiler:
path_to_compiler = os.path.realpath(path_to_compiler)
self.log.info(
"Found path to compiler '%s' (with symlinks resolved): %s",
compiler_name, path_to_compiler)
else:
raise EasyBuildError("%s not found in $PATH", compiler_name)
# Determine compiler version and installation prefix
if compiler_name == 'gcc':
out, _ = run_cmd("gcc --version", simple=False)
self.extract_compiler_version(out)
# strip off 'bin/gcc'
self.compiler_prefix = os.path.dirname(
os.path.dirname(path_to_compiler))
elif compiler_name in ['icc', 'ifort']:
out, _ = run_cmd("%s -V" % compiler_name, simple=False)
self.extract_compiler_version(out)
intelvars_fn = path_to_compiler + 'vars.sh'
if os.path.isfile(intelvars_fn):
self.log.debug(
"Trying to determine compiler install prefix from %s",
intelvars_fn)
intelvars_txt = read_file(intelvars_fn)
prod_dir_regex = re.compile(r'^PROD_DIR=(.*)$', re.M)
res = prod_dir_regex.search(intelvars_txt)
if res:
self.compiler_prefix = res.group(1)
else:
raise EasyBuildError(
"Failed to determine %s installation prefix from %s",
compiler_name, intelvars_fn)
else:
# strip off 'bin/intel*/icc'
self.compiler_prefix = os.path.dirname(
os.path.dirname(os.path.dirname(path_to_compiler)))
else:
raise EasyBuildError("Unknown system compiler %s" %
self.cfg['name'])
self.log.debug(
"Derived version/install prefix for system compiler %s: %s, %s",
compiler_name, self.compiler_version, self.compiler_prefix)
# If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it
if self.cfg['version'] == 'system':
self.log.info(
"Found specified version '%s', going with derived compiler version '%s'",
self.cfg['version'], self.compiler_version)
elif self.cfg['version'] != self.compiler_version:
raise EasyBuildError(
"Specified version (%s) does not match version reported by compiler (%s)"
% (self.cfg['version'], self.compiler_version))
# fix installdir and module names (may differ because of changes to version)
mns = ActiveMNS()
self.cfg.full_mod_name = mns.det_full_module_name(self.cfg)
self.cfg.short_mod_name = mns.det_short_module_name(self.cfg)
self.cfg.mod_subdir = mns.det_module_subdir(self.cfg)
# keep track of original values, for restoring later
self.orig_version = self.cfg['version']
self.orig_installdir = self.installdir
def configure_step(self):
"""Custom configuration procedure for TensorFlow."""
tmpdir = tempfile.mkdtemp(suffix='-bazel-configure')
# filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that
# might conflict with dependencies on the system and/or installed with EB. For example: protobuf
path_filter = self.cfg['path_filter']
if path_filter:
self.log.info(
"Filtering $CPATH and $LIBRARY_PATH with path filter %s",
path_filter)
for var in ['CPATH', 'LIBRARY_PATH']:
path = os.getenv(var).split(os.pathsep)
self.log.info("$%s old value was %s" % (var, path))
filtered_path = os.pathsep.join(
[p for fil in path_filter for p in path if fil not in p])
env.setvar(var, filtered_path)
wrapper_dir = os.path.join(tmpdir, 'bin')
use_wrapper = False
if self.toolchain.comp_family() == toolchain.INTELCOMP:
# put wrappers for Intel C/C++ compilers in place (required to make sure license server is found)
# cfr. https://github.com/bazelbuild/bazel/issues/663
for compiler in ('icc', 'icpc'):
self.write_wrapper(wrapper_dir, compiler, 'NOT-USED-WITH-ICC')
use_wrapper = True
use_mpi = self.toolchain.options.get('usempi', False)
mpi_home = ''
if use_mpi:
impi_root = get_software_root('impi')
if impi_root:
# put wrappers for Intel MPI compiler wrappers in place
# (required to make sure license server and I_MPI_ROOT are found)
for compiler in (os.getenv('MPICC'), os.getenv('MPICXX')):
self.write_wrapper(wrapper_dir, compiler,
os.getenv('I_MPI_ROOT'))
use_wrapper = True
# set correct value for MPI_HOME
mpi_home = os.path.join(impi_root, 'intel64')
else:
self.log.debug("MPI module name: %s",
self.toolchain.MPI_MODULE_NAME[0])
mpi_home = get_software_root(self.toolchain.MPI_MODULE_NAME[0])
self.log.debug("Derived value for MPI_HOME: %s", mpi_home)
if use_wrapper:
env.setvar('PATH',
os.pathsep.join([wrapper_dir,
os.getenv('PATH')]))
self.prepare_python()
self.handle_jemalloc()
cuda_root = get_software_root('CUDA')
cudnn_root = get_software_root('cuDNN')
opencl_root = get_software_root('OpenCL')
tensorrt_root = get_software_root('TensorRT')
nccl_root = get_software_root('NCCL')
config_env_vars = {
'CC_OPT_FLAGS': os.getenv('CXXFLAGS'),
'MPI_HOME': mpi_home,
'PYTHON_BIN_PATH': self.python_cmd,
'PYTHON_LIB_PATH': os.path.join(self.installdir, self.pylibdir),
'TF_CUDA_CLANG': '0',
'TF_ENABLE_XLA': '0', # XLA JIT support
'TF_NEED_CUDA': ('0', '1')[bool(cuda_root)],
'TF_NEED_GCP': '0', # Google Cloud Platform
'TF_NEED_GDR': '0',
'TF_NEED_HDFS': '0', # Hadoop File System
'TF_NEED_JEMALLOC': ('0', '1')[self.cfg['with_jemalloc']],
'TF_NEED_MPI': ('0', '1')[bool(use_mpi)],
'TF_NEED_OPENCL': ('0', '1')[bool(opencl_root)],
'TF_NEED_OPENCL_SYCL': '0',
'TF_NEED_S3': '0', # Amazon S3 File System
'TF_NEED_TENSORRT': '0',
'TF_NEED_VERBS': '0',
'TF_NEED_AWS': '0', # Amazon AWS Platform
'TF_NEED_KAFKA': '0', # Amazon Kafka Platform
}
if cuda_root:
cuda_version = get_software_version('CUDA')
cuda_maj_min_ver = '.'.join(cuda_version.split('.')[:2])
# $GCC_HOST_COMPILER_PATH should be set to path of the actual compiler (not the MPI compiler wrapper)
if use_mpi:
compiler_path = which(os.getenv('CC_SEQ'))
else:
compiler_path = which(os.getenv('CC'))
config_env_vars.update({
'CUDA_TOOLKIT_PATH':
cuda_root,
'GCC_HOST_COMPILER_PATH':
compiler_path,
'TF_CUDA_COMPUTE_CAPABILITIES':
','.join(self.cfg['cuda_compute_capabilities']),
'TF_CUDA_VERSION':
cuda_maj_min_ver,
})
# for recent TensorFlow versions, $TF_CUDA_PATHS and $TF_CUBLAS_VERSION must also be set
if LooseVersion(self.version) >= LooseVersion('1.14'):
# figure out correct major/minor version for CUBLAS from cublas_api.h
cublas_api_header_glob_pattern = os.path.join(
cuda_root, 'targets', '*', 'include', 'cublas_api.h')
matches = glob.glob(cublas_api_header_glob_pattern)
if len(matches) == 1:
cublas_api_header_path = matches[0]
cublas_api_header_txt = read_file(cublas_api_header_path)
else:
raise EasyBuildError(
"Failed to isolate path to cublas_api.h: %s", matches)
cublas_ver_parts = []
for key in [
'CUBLAS_VER_MAJOR', 'CUBLAS_VER_MINOR',
'CUBLAS_VER_PATCH'
]:
regex = re.compile("^#define %s ([0-9]+)" % key, re.M)
res = regex.search(cublas_api_header_txt)
if res:
cublas_ver_parts.append(res.group(1))
else:
raise EasyBuildError(
"Failed to find pattern '%s' in %s", regex.pattern,
cublas_api_header_path)
config_env_vars.update({
'TF_CUDA_PATHS':
cuda_root,
'TF_CUBLAS_VERSION':
'.'.join(cublas_ver_parts),
})
if cudnn_root:
cudnn_version = get_software_version('cuDNN')
cudnn_maj_min_patch_ver = '.'.join(
cudnn_version.split('.')[:3])
config_env_vars.update({
'CUDNN_INSTALL_PATH':
cudnn_root,
'TF_CUDNN_VERSION':
cudnn_maj_min_patch_ver,
})
else:
raise EasyBuildError(
"TensorFlow has a strict dependency on cuDNN if CUDA is enabled"
)
if nccl_root:
nccl_version = get_software_version('NCCL')
config_env_vars.update({
'NCCL_INSTALL_PATH': nccl_root,
})
else:
nccl_version = '1.3' # Use simple downloadable version
config_env_vars.update({
'TF_NCCL_VERSION': nccl_version,
})
if tensorrt_root:
tensorrt_version = get_software_version('TensorRT')
config_env_vars.update({
'TF_NEED_TENSORRT': '1',
'TENSORRT_INSTALL_PATH': tensorrt_root,
'TF_TENSORRT_VERSION': tensorrt_version,
})
for (key, val) in sorted(config_env_vars.items()):
env.setvar(key, val)
# patch configure.py (called by configure script) to avoid that Bazel abuses $HOME/.cache/bazel
regex_subs = [(r"(run_shell\(\['bazel')",
r"\1, '--output_base=%s', '--install_base=%s'" %
(tmpdir, os.path.join(tmpdir, 'inst_base')))]
apply_regex_substitutions('configure.py', regex_subs)
# Tell Bazel to not use $HOME/.cache/bazel at all
# See https://docs.bazel.build/versions/master/output_directories.html
env.setvar('TEST_TMPDIR', os.path.join(tmpdir, 'output_root'))
cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg[
'configopts']
run_cmd(cmd, log_all=True, simple=True)
def __init__(self, *args, **kwargs):
"""Locate the installation files of OpenSSL in the host system"""
super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs)
# Libraries packaged in OpenSSL
openssl_libs = ['libssl', 'libcrypto']
# list of relevant library extensions per system and version of OpenSSL
# the first item should be the extension of an installation from source,
# it will be used in the sanity checks of the component
openssl_libext = {
'1.0': {
LINUX: ('so.1.0.0', 'so.10'),
DARWIN: ('1.0.dylib', ),
},
'1.1': {
LINUX: ('so.1.1', ),
DARWIN: ('1.1.dylib', ),
},
}
os_type = get_os_type()
if self.version in openssl_libext and os_type in openssl_libext[
self.version]:
# generate matrix of versioned .so filenames
system_versioned_libs = [[
'%s.%s' % (lib, ext)
for ext in openssl_libext[self.version][os_type]
] for lib in openssl_libs]
self.log.info("Matrix of version library names: %s",
system_versioned_libs)
else:
raise EasyBuildError(
"Don't know name of OpenSSL system library for version %s and OS type %s",
self.version, os_type)
# by default target the first option of each OpenSSL library,
# which corresponds to installation from source
self.target_ssl_libs = [
lib_name[0] for lib_name in system_versioned_libs
]
self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs)
# folders containing engines libraries
openssl_engines = {
'1.0': 'engines',
'1.1': 'engines-1.1',
}
self.target_ssl_engine = openssl_engines[self.version]
# Paths to system libraries and headers of OpenSSL
self.system_ssl = {
'bin': None,
'engines': None,
'include': None,
'lib': None,
}
# early return when we're not wrapping the system OpenSSL installation
if not self.cfg.get('wrap_system_openssl'):
self.log.info("Not wrapping system OpenSSL installation!")
return
# Check the system libraries of OpenSSL
# (only needs first library in openssl_libs to find path to libraries)
for idx, libssl in enumerate(system_versioned_libs[0]):
self.system_ssl['lib'] = find_library_path(libssl)
if self.system_ssl['lib']:
# change target libraries to the ones found in the system
self.target_ssl_libs = [
lib_name[idx] for lib_name in system_versioned_libs
]
self.log.info("Target system OpenSSL libraries: %s",
self.target_ssl_libs)
break
if self.system_ssl['lib']:
self.log.info("Found library '%s' in: %s", openssl_libs[0],
self.system_ssl['lib'])
else:
self.log.info(
"OpenSSL library '%s' not found, falling back to OpenSSL in EasyBuild",
openssl_libs[0])
# Directory with engine libraries
if self.system_ssl['lib']:
lib_dir = os.path.dirname(self.system_ssl['lib'])
lib_engines_dir = [
os.path.join(lib_dir, 'openssl', self.target_ssl_engine),
os.path.join(lib_dir, self.target_ssl_engine),
]
for engines_path in lib_engines_dir:
if os.path.isdir(engines_path):
self.system_ssl['engines'] = engines_path
self.log.debug("Found OpenSSL engines in: %s",
self.system_ssl['engines'])
break
if not self.system_ssl['engines']:
self.system_ssl['lib'] = None
self.log.info(
"OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild"
)
# Check system include paths for OpenSSL headers
cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null"
(out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False)
sys_include_dirs = []
for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE):
sys_include_dirs.extend(match.groups())
self.log.debug(
"Found the following include directories in host system: %s",
', '.join(sys_include_dirs))
# headers are located in 'include/openssl' by default
ssl_include_subdirs = [self.name.lower()]
if self.version == '1.1':
# but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7
# prefer 'include/openssl' as long as the version of headers matches
ssl_include_subdirs.append(
os.path.join('openssl11', self.name.lower()))
ssl_include_dirs = [
os.path.join(incd, subd) for incd in sys_include_dirs
for subd in ssl_include_subdirs
]
ssl_include_dirs = [
include for include in ssl_include_dirs if os.path.isdir(include)
]
# find location of header files, verify that the headers match our OpenSSL version
openssl_version_regex = re.compile(
r"SHLIB_VERSION_NUMBER\s\"([0-9]+\.[0-9]+)", re.M)
for include_dir in ssl_include_dirs:
opensslv_path = os.path.join(include_dir, 'opensslv.h')
self.log.debug("Checking OpenSSL version in %s...", opensslv_path)
if os.path.exists(opensslv_path):
opensslv = read_file(opensslv_path)
header_majmin_version = openssl_version_regex.search(opensslv)
if header_majmin_version:
header_majmin_version = header_majmin_version.group(1)
if re.match('^' + header_majmin_version, self.version):
self.system_ssl['include'] = include_dir
self.log.info(
"Found OpenSSL headers in host system: %s",
self.system_ssl['include'])
break
else:
self.log.debug(
"Header major/minor version '%s' doesn't match with %s",
header_majmin_version, self.version)
else:
self.log.debug("Pattern '%s' not found in %s",
openssl_version_regex.pattern,
opensslv_path)
else:
self.log.info("OpenSSL header file %s not found",
opensslv_path)
if not self.system_ssl['include']:
self.log.info(
"OpenSSL headers not found in host system, falling back to OpenSSL in EasyBuild"
)
# Check system OpenSSL binary
if self.version == '1.1':
# prefer 'openssl11' over 'openssl' with v1.1
self.system_ssl['bin'] = which('openssl11')
if not self.system_ssl['bin']:
self.system_ssl['bin'] = which(self.name.lower())
if self.system_ssl['bin']:
self.log.info("System OpenSSL binary found: %s",
self.system_ssl['bin'])
else:
self.log.info("System OpenSSL binary not found!")
def configure_step(self):
"""Custom configuration procedure for TensorFlow."""
tmpdir = tempfile.mkdtemp(suffix='-bazel-configure')
# filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that
# might conflict with dependencies on the system and/or installed with EB. For example: protobuf
path_filter = self.cfg['path_filter']
if path_filter:
self.log.info("Filtering $CPATH and $LIBRARY_PATH with path filter %s", path_filter)
for var in ['CPATH', 'LIBRARY_PATH']:
path = os.getenv(var).split(os.pathsep)
self.log.info("$%s old value was %s" % (var, path))
filtered_path = os.pathsep.join([p for fil in path_filter for p in path if fil not in p])
env.setvar(var, filtered_path)
wrapper_dir = os.path.join(tmpdir, 'bin')
use_wrapper = False
if self.toolchain.comp_family() == toolchain.INTELCOMP:
# put wrappers for Intel C/C++ compilers in place (required to make sure license server is found)
# cfr. https://github.com/bazelbuild/bazel/issues/663
for compiler in ('icc', 'icpc'):
self.write_wrapper(wrapper_dir, compiler, 'NOT-USED-WITH-ICC')
use_wrapper = True
use_mpi = self.toolchain.options.get('usempi', False)
mpi_home = ''
if use_mpi:
impi_root = get_software_root('impi')
if impi_root:
# put wrappers for Intel MPI compiler wrappers in place
# (required to make sure license server and I_MPI_ROOT are found)
for compiler in (os.getenv('MPICC'), os.getenv('MPICXX')):
self.write_wrapper(wrapper_dir, compiler, os.getenv('I_MPI_ROOT'))
use_wrapper = True
# set correct value for MPI_HOME
mpi_home = os.path.join(impi_root, 'intel64')
else:
self.log.debug("MPI module name: %s", self.toolchain.MPI_MODULE_NAME[0])
mpi_home = get_software_root(self.toolchain.MPI_MODULE_NAME[0])
self.log.debug("Derived value for MPI_HOME: %s", mpi_home)
if use_wrapper:
env.setvar('PATH', os.pathsep.join([wrapper_dir, os.getenv('PATH')]))
self.prepare_python()
self.handle_jemalloc()
cuda_root = get_software_root('CUDA')
cudnn_root = get_software_root('cuDNN')
opencl_root = get_software_root('OpenCL')
tensorrt_root = get_software_root('TensorRT')
nccl_root = get_software_root('NCCL')
config_env_vars = {
'CC_OPT_FLAGS': os.getenv('CXXFLAGS'),
'MPI_HOME': mpi_home,
'PYTHON_BIN_PATH': self.python_cmd,
'PYTHON_LIB_PATH': os.path.join(self.installdir, self.pylibdir),
'TF_CUDA_CLANG': '0',
'TF_ENABLE_XLA': '0', # XLA JIT support
'TF_NEED_CUDA': ('0', '1')[bool(cuda_root)],
'TF_NEED_GCP': '0', # Google Cloud Platform
'TF_NEED_GDR': '0',
'TF_NEED_HDFS': '0', # Hadoop File System
'TF_NEED_JEMALLOC': ('0', '1')[self.cfg['with_jemalloc']],
'TF_NEED_MPI': ('0', '1')[bool(use_mpi)],
'TF_NEED_OPENCL': ('0', '1')[bool(opencl_root)],
'TF_NEED_OPENCL_SYCL': '0',
'TF_NEED_S3': '0', # Amazon S3 File System
'TF_NEED_TENSORRT': '0',
'TF_NEED_VERBS': '0',
'TF_NEED_AWS': '0', # Amazon AWS Platform
'TF_NEED_KAFKA': '0', # Amazon Kafka Platform
}
if cuda_root:
config_env_vars.update({
'CUDA_TOOLKIT_PATH': cuda_root,
'GCC_HOST_COMPILER_PATH': which(os.getenv('CC')),
'TF_CUDA_COMPUTE_CAPABILITIES': ','.join(self.cfg['cuda_compute_capabilities']),
'TF_CUDA_VERSION': get_software_version('CUDA'),
})
if cudnn_root:
config_env_vars.update({
'CUDNN_INSTALL_PATH': cudnn_root,
'TF_CUDNN_VERSION': get_software_version('cuDNN'),
})
else:
raise EasyBuildError("TensorFlow has a strict dependency on cuDNN if CUDA is enabled")
if nccl_root:
nccl_version = get_software_version('NCCL')
config_env_vars.update({
'NCCL_INSTALL_PATH': nccl_root,
})
else:
nccl_version = '1.3' # Use simple downloadable version
config_env_vars.update({
'TF_NCCL_VERSION': nccl_version,
})
if tensorrt_root:
tensorrt_version = get_software_version('TensorRT')
config_env_vars.update({
'TF_NEED_TENSORRT': '1',
'TENSORRT_INSTALL_PATH': tensorrt_root,
'TF_TENSORRT_VERSION': tensorrt_version,
})
for (key, val) in sorted(config_env_vars.items()):
env.setvar(key, val)
# patch configure.py (called by configure script) to avoid that Bazel abuses $HOME/.cache/bazel
regex_subs = [(r"(run_shell\(\['bazel')",
r"\1, '--output_base=%s', '--install_base=%s'" % (tmpdir, os.path.join(tmpdir, 'inst_base')))]
apply_regex_substitutions('configure.py', regex_subs)
cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg['configopts']
run_cmd(cmd, log_all=True, simple=True)
def install_step(self):
"""
Custom install step for GROMACS; figure out where libraries were installed to.
Also, install the MPI version of the executable in a separate step.
"""
# run 'make install' in parallel since it involves more compilation
self.cfg.update('installopts', "-j %s" % self.cfg['parallel'])
super(EB_GROMACS, self).install_step()
# the GROMACS libraries get installed in different locations (deeper subdirectory), depending on the platform;
# this is determined by the GNUInstallDirs CMake module;
# rather than trying to replicate the logic, we just figure out where the library was placed
if self.toolchain.options.get('dynamic', False):
self.libext = get_shared_lib_ext()
else:
self.libext = 'a'
if LooseVersion(self.version) < LooseVersion('5.0'):
libname = 'libgmx*.%s' % self.libext
else:
libname = 'libgromacs*.%s' % self.libext
for libdir in ['lib', 'lib64']:
if os.path.exists(os.path.join(self.installdir, libdir)):
for subdir in [libdir, os.path.join(libdir, '*')]:
libpaths = glob.glob(
os.path.join(self.installdir, subdir, libname))
if libpaths:
self.lib_subdir = os.path.dirname(
libpaths[0])[len(self.installdir) + 1:]
self.log.info(
"Found lib subdirectory that contains %s: %s",
libname, self.lib_subdir)
break
if not self.lib_subdir:
raise EasyBuildError("Failed to determine lib subdirectory in %s",
self.installdir)
# Install a version with the MPI suffix
if self.toolchain.options.get('usempi', None):
if LooseVersion(self.version) < LooseVersion('4.6'):
cmd = "make distclean"
(out, _) = run_cmd(cmd, log_all=True, simple=False)
self.cfg.update(
'configopts', "--enable-mpi --program-suffix={0}".format(
self.cfg['mpisuffix']))
ConfigureMake.configure_step(self)
super(EB_GROMACS, self).build_step()
super(EB_GROMACS, self).install_step()
else:
self.cfg['configopts'] = re.sub(r'-DGMX_MPI=OFF', r'',
self.cfg['configopts'])
if self.cfg['mpi_numprocs'] == 0:
self.log.info(
"No number of test MPI tasks specified -- using default: %s"
% self.cfg['parallel'])
self.cfg['mpi_numprocs'] = self.cfg['parallel']
elif self.cfg['mpi_numprocs'] > self.cfg['parallel']:
self.log.warning(
"Number of test MPI tasks (%s) is greater than value for 'parallel': %s",
self.cfg['mpi_numprocs'], self.cfg['parallel'])
self.cfg.update('configopts',
"-DGMX_MPI=ON -DGMX_THREAD_MPI=OFF")
mpiexec = which(self.cfg['mpiexec'])
if mpiexec:
self.cfg.update('configopts', "-DMPIEXEC=%s" % mpiexec)
self.cfg.update(
'configopts', "-DMPIEXEC_NUMPROC_FLAG=%s" %
self.cfg['mpiexec_numproc_flag'])
self.cfg.update('configopts',
"-DNUMPROC=%s" % self.cfg['mpi_numprocs'])
elif self.cfg['runtest']:
raise EasyBuildError("'%s' not found in $PATH",
self.cfg['mpiexec'])
self.log.info(
"Using %s as MPI executable when testing, with numprocs flag '%s' and %s tasks",
self.cfg['mpiexec'], self.cfg['mpiexec_numproc_flag'],
self.cfg['mpi_numprocs'])
# clean up obj dir before reconfiguring
shutil.rmtree(os.path.join(self.builddir, 'easybuild_obj'))
# rebuild/test/install with MPI options
super(EB_GROMACS, self).configure_step()
super(EB_GROMACS, self).build_step()
super(EB_GROMACS, self).test_step()
super(EB_GROMACS, self).install_step()
self.log.info(
"A full regression test suite is available from the GROMACS web site"
)
def configure_step(self):
"""Custom configure step for jaxlib."""
super(EB_jaxlib, self).configure_step()
binutils_root = get_software_root('binutils')
if not binutils_root:
raise EasyBuildError(
"Failed to determine installation prefix for binutils")
config_env_vars = {
# This is the binutils bin folder: https://github.com/tensorflow/tensorflow/issues/39263
'GCC_HOST_COMPILER_PREFIX': os.path.join(binutils_root, 'bin'),
}
# Collect options for the build script
# Used only by the build script
options = [
'--target_cpu_features=default', # Using copt for optimizations
]
# Passed directly to bazel
bazel_startup_options = [
'--output_user_root=%s' %
tempfile.mkdtemp(suffix='-bazel', dir=self.builddir),
]
# Passed to the build command of bazel
bazel_options = [
'--jobs=%s' % self.cfg['parallel'],
'--subcommands',
'--action_env=PYTHONPATH',
'--action_env=EBPYTHONPREFIXES',
]
if self.toolchain.options.get('debug', None):
bazel_options.extend(['--strip=never', '--copt="-Og"'])
# Add optimization flags set by EasyBuild each as a separate option
bazel_options.extend(
['--copt=%s' % i for i in os.environ['CXXFLAGS'].split(' ')])
cuda_root = get_software_root('CUDA')
if cuda_root:
cudnn_root = get_software_root('cuDNN')
if not cudnn_root:
raise EasyBuildError(
'For CUDA-enabled builds cuDNN is also required')
cuda_version = '.'.join(
get_software_version('CUDA').split('.')[:2]) # maj.minor
cudnn_version = '.'.join(
get_software_version('cuDNN').split('.')
[:3]) # maj.minor.patch
options.extend([
'--enable_cuda',
'--cuda_path=' + cuda_root,
'--cuda_compute_capabilities=' +
self.cfg.get_cuda_cc_template_value(
'cuda_compute_capabilities'),
'--cuda_version=' + cuda_version,
'--cudnn_path=' + cudnn_root,
'--cudnn_version=' + cudnn_version,
])
nccl_root = get_software_root('NCCL')
if nccl_root:
options.append('--enable_nccl')
else:
options.append('--noenable_nccl')
config_env_vars['GCC_HOST_COMPILER_PATH'] = which(os.getenv('CC'))
else:
options.append('--noenable_cuda')
# Prepend to buildopts so users can overwrite this
self.cfg['buildopts'] = ' '.join(options + [
'--bazel_startup_options="%s"' % i for i in bazel_startup_options
] + ['--bazel_options="%s"' % i
for i in bazel_options] + [self.cfg['buildopts']])
for key, val in sorted(config_env_vars.items()):
env.setvar(key, val)
# Print output of build at the end
apply_regex_substitutions(
'build/build.py',
[(r' shell\(command\)', ' print(shell(command))')])
