def check_os_dependency(dep): """ Check if dependency is available from OS. """ # - uses rpm -q and dpkg -s --> can be run as non-root!! # - fallback on which # - should be extended to files later? found = None cmd = None if which('rpm'): cmd = "rpm -q %s" % dep found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True) if not found and which('dpkg'): cmd = "dpkg -s %s" % dep found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True) if cmd is None: # fallback for when os-dependency is a binary/library found = which(dep) # try locate if it's available if not found and which('locate'): cmd = 'locate --regexp "/%s$"' % dep found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True) return found
def test_which(self): """Test which function for locating commands.""" python = ft.which('python') self.assertTrue(python and os.path.exists(python) and os.path.isabs(python)) path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available') self.assertTrue(path is None)
def check_os_dependency(dep): """ Check if dependency is available from OS. """ # - uses rpm -q and dpkg -s --> can be run as non-root!! # - fallback on which # - should be extended to files later? cmd = None if get_os_name() in ['debian', 'ubuntu']: if which('dpkg'): cmd = "dpkg -s %s" % dep else: # OK for get_os_name() == redhat, fedora, RHEL, SL, centos if which('rpm'): cmd = "rpm -q %s" % dep found = None if cmd is not None: found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) if found is None: # fallback for when os-dependency is a binary/library found = which(dep) # try locate if it's available if found is None and which('locate'): cmd = 'locate --regexp "/%s$"' % dep found = run_cmd(cmd, simple=True, log_all=False, log_ok=False) return found
def __init__(self, mod_paths=None, testing=False): """ Create a ModulesTool object @param mod_paths: A list of paths where the modules can be located @type mod_paths: list """ # this can/should be set to True during testing self.testing = testing self.log = fancylogger.getLogger(self.__class__.__name__, fname=False) self.mod_paths = None if mod_paths is not None: self.set_mod_paths(mod_paths) # DEPRECATED! self._modules = [] # actual module command (i.e., not the 'module' wrapper function, but the binary) self.cmd = self.COMMAND env_cmd_path = os.environ.get(self.COMMAND_ENVIRONMENT) # only use command path in environment variable if command in not available in $PATH if which(self.cmd) is None and env_cmd_path is not None: self.log.debug("Set command via environment variable %s: %s", self.COMMAND_ENVIRONMENT, self.cmd) self.cmd = env_cmd_path # check whether paths obtained via $PATH and $LMOD_CMD are different elif which(self.cmd) != env_cmd_path: self.log.debug( "Different paths found for module command '%s' via which/$PATH and $%s: %s vs %s", self.COMMAND, self.COMMAND_ENVIRONMENT, self.cmd, env_cmd_path, ) # make sure the module command was found if self.cmd is None: raise EasyBuildError("No command set.") else: self.log.debug("Using command %s" % self.cmd) # version of modules tool self.version = None # some initialisation/verification self.check_cmd_avail() self.check_module_path() self.check_module_function(allow_mismatch=build_option("allow_modules_tool_mismatch")) self.set_and_check_version()
def configure_step(self): """Custom configuration procedure for RPMs: rebuild RPMs for relocation if required.""" # make sure that rpm is available if not which('rpm'): raise EasyBuildError("Command 'rpm' is required but not available.") # determine whether RPMs need to be rebuilt to make relocation work cmd = "rpm --version" (out, _) = run_cmd(cmd, log_all=True, simple=False) rpmver_re = re.compile("^RPM\s+version\s+(?P<version>[0-9.]+).*") res = rpmver_re.match(out) self.log.debug("RPM version found: %s" % res.group()) if res: ver = res.groupdict()['version'] # rebuilding is required on SL6, which implies rpm v4.8 (works fine without rebuilding on SL5) if LooseVersion(ver) >= LooseVersion('4.8.0'): self.rebuild_rpm = True self.log.debug("Enabling rebuild of RPMs to make relocation work...") else: raise EasyBuildError("Checking RPM version failed, so just carrying on with the default behaviour...") if self.rebuild_rpm: self.rebuild_rpms()
def test_which(self): """Test which function for locating commands.""" python = ft.which('python') self.assertTrue(python and os.path.exists(python) and os.path.isabs(python)) path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available') self.assertTrue(path is None) os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.environ['PATH']) foo, bar = os.path.join(self.test_prefix, 'foo'), os.path.join(self.test_prefix, 'bar') ft.mkdir(foo) ft.adjust_permissions(foo, stat.S_IRUSR|stat.S_IXUSR) ft.write_file(bar, '#!/bin/bash') ft.adjust_permissions(bar, stat.S_IRUSR|stat.S_IXUSR) self.assertEqual(ft.which('foo'), None) self.assertTrue(os.path.samefile(ft.which('bar'), bar))
def configure_step(self, cmd_prefix=''): """ Configure with Meson. """ # make sure both Meson and Ninja are included as build dependencies build_dep_names = [d['name'] for d in self.cfg.builddependencies()] for tool in ['Ninja', 'Meson']: if tool not in build_dep_names: raise EasyBuildError("%s not included as build dependency", tool) cmd = tool.lower() if not which(cmd): raise EasyBuildError("'%s' command not found", cmd) if self.cfg.get('separate_build_dir', True): builddir = os.path.join(self.builddir, 'easybuild_obj') mkdir(builddir) change_dir(builddir) # Make sure libdir doesn't get set to lib/x86_64-linux-gnu or something # on Debian/Ubuntu multiarch systems and others. no_Dlibdir = '-Dlibdir' not in self.cfg['configopts'] no_libdir = '--libdir' not in self.cfg['configopts'] if no_Dlibdir and no_libdir: self.cfg.update('configopts', '-Dlibdir=lib') cmd = "%(preconfigopts)s meson --prefix %(installdir)s %(configopts)s %(sourcedir)s" % { 'configopts': self.cfg['configopts'], 'installdir': self.installdir, 'preconfigopts': self.cfg['preconfigopts'], 'sourcedir': self.start_dir, } (out, _) = run_cmd(cmd, log_all=True, simple=False) return out
def symlink_commands(self, paths): """ Create a symlink for each command to binary/script at specified path. :param paths: dictionary containing one or mappings, each one specified as a tuple: (<path/to/script>, <list of commands to symlink to the script>) """ symlink_dir = tempfile.mkdtemp() # prepend location to symlinks to $PATH setvar('PATH', '%s:%s' % (symlink_dir, os.getenv('PATH'))) for (path, cmds) in paths.values(): for cmd in cmds: cmd_s = os.path.join(symlink_dir, cmd) if not os.path.exists(cmd_s): try: os.symlink(path, cmd_s) except OSError as err: raise EasyBuildError("Failed to symlink %s to %s: %s", path, cmd_s, err) cmd_path = which(cmd) self.log.debug("which(%s): %s -> %s", cmd, cmd_path, os.path.realpath(cmd_path)) self.log.info("Commands symlinked to %s via %s: %s", path, symlink_dir, ', '.join(cmds))
def prepare_python(self): """Python-specific preperations.""" # pick 'python' command to use python = None python_root = get_software_root('Python') # keep in mind that Python may be listed as an allowed system dependency, # so just checking Python root is not sufficient if python_root: bin_python = os.path.join(python_root, 'bin', 'python') if os.path.exists(bin_python) and os.path.samefile(which('python'), bin_python): # if Python is listed as a (build) dependency, use 'python' command provided that way python = os.path.join(python_root, 'bin', 'python') self.log.debug("Retaining 'python' command for Python dependency: %s", python) if python is None: # if using system Python, go hunting for a 'python' command that satisfies the requirements python = pick_python_cmd(req_maj_ver=self.cfg['req_py_majver'], req_min_ver=self.cfg['req_py_minver']) if python: self.python_cmd = python self.log.info("Python command being used: %s", self.python_cmd) else: raise EasyBuildError("Failed to pick Python command to use") # set Python lib directories self.set_pylibdirs()
def check_tool(tool_name, min_tool_version=None): """ This function is a predicate check for the existence of tool_name on the system PATH. If min_tool_version is not None, it will check that the version has an equal or higher value. """ tool_path = which(tool_name) if not tool_path: return False print_msg("{0} tool found at {1}".format(tool_name, tool_path)) if not min_tool_version: return True version_cmd = "{0} --version".format(tool_name) out, ec = run_cmd(version_cmd, simple=False, trace=False, force_in_dry_run=True) if ec: raise EasyBuildError("Error running '{0}' for tool {1} with output: {2}".format(version_cmd, tool_name, out)) res = re.search("\d+\.\d+(\.\d+)?", out.strip()) if not res: raise EasyBuildError("Error parsing version for tool {0}".format(tool_name)) tool_version = res.group(0) version_ok = LooseVersion(str(min_tool_version)) <= LooseVersion(tool_version) if version_ok: print_msg("{0} version '{1}' is {2} or higher ... OK".format(tool_name, tool_version, min_tool_version)) return version_ok
def configure_step(self, cmd_prefix=''): """ Configure with Meson. """ # make sure both Meson and Ninja are included as build dependencies build_dep_names = [d['name'] for d in self.cfg.builddependencies()] for tool in ['Ninja', 'Meson']: if tool not in build_dep_names: raise EasyBuildError("%s not included as build dependency", tool) cmd = tool.lower() if not which(cmd): raise EasyBuildError("'%s' command not found", cmd) if self.cfg.get('separate_build_dir', True): builddir = os.path.join(self.builddir, 'easybuild_obj') mkdir(builddir) change_dir(builddir) cmd = "%(preconfigopts)s meson --prefix %(installdir)s %(configopts)s %(sourcedir)s" % { 'configopts': self.cfg['configopts'], 'installdir': self.installdir, 'preconfigopts': self.cfg['preconfigopts'], 'sourcedir': self.start_dir, } (out, _) = run_cmd(cmd, log_all=True, simple=False) return out
def write_wrapper(self, wrapper_dir, compiler, i_mpi_root): """Helper function to write a compiler wrapper.""" wrapper_txt = INTEL_COMPILER_WRAPPER % { 'compiler_path': which(compiler), 'intel_mpi_root': i_mpi_root, 'cpath': os.getenv('CPATH'), 'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')), 'wrapper_dir': wrapper_dir, } wrapper = os.path.join(wrapper_dir, compiler) write_file(wrapper, wrapper_txt) if self.dry_run: self.dry_run_msg("Wrapper for '%s' was put in place: %s", compiler, wrapper) else: adjust_permissions(wrapper, stat.S_IXUSR) self.log.info("Using wrapper script for '%s': %s", compiler, which(compiler))
def test_lmod_specific(self): """Lmod-specific test (skipped unless Lmod is used as modules tool).""" lmod_abspath = which(Lmod.COMMAND) # only run this test if 'lmod' is available in $PATH if lmod_abspath is not None: build_options = { 'allow_modules_tool_mismatch': True, } init_config(build_options=build_options) # drop any location where 'lmod' or 'spider' can be found from $PATH paths = os.environ.get('PATH', '').split(os.pathsep) new_paths = [] for path in paths: lmod_cand_path = os.path.join(path, Lmod.COMMAND) spider_cand_path = os.path.join(path, 'spider') if not os.path.isfile(lmod_cand_path) and not os.path.isfile(spider_cand_path): new_paths.append(path) os.environ['PATH'] = os.pathsep.join(new_paths) # make sure $MODULEPATH contains path that provides some modules os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) # initialize Lmod modules tool, pass full path to 'lmod' via $LMOD_CMD os.environ['LMOD_CMD'] = lmod_abspath lmod = Lmod(testing=True) # obtain list of availabe modules, should be non-empty self.assertTrue(lmod.available(), "List of available modules obtained using Lmod is non-empty") # test updating local spider cache (but don't actually update the local cache file!) self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty")
def check_pkg_support(): """Check whether packaging is possible, if required dependencies are available.""" pkgtool = build_option('package_tool') pkgtool_path = which(pkgtool) if pkgtool_path: _log.info("Selected packaging tool '%s' found at %s", pkgtool, pkgtool_path) # rpmbuild is required for generating RPMs with FPM if pkgtool == PKG_TOOL_FPM and build_option('package_type') == PKG_TYPE_RPM: rpmbuild_path = which('rpmbuild') if rpmbuild_path: _log.info("Required tool 'rpmbuild' found at %s", rpmbuild_path) else: raise EasyBuildError("rpmbuild is required when generating RPM packages but was not found") else: raise EasyBuildError("Selected packaging tool '%s' not found", pkgtool)
def check_cmd_avail(self): """Check whether modules tool command is available.""" cmd_path = which(self.cmd) if cmd_path is not None: self.cmd = cmd_path self.log.info("Full path for module command is %s, so using it" % self.cmd) else: mod_tool = self.__class__.__name__ self.log.error("%s modules tool can not be used, '%s' command is not available." % (mod_tool, self.cmd))
def check_os_dependency(dep): """ Check if dependency is available from OS. """ # - uses rpm -q and dpkg -s --> can be run as non-root!! # - fallback on which # - should be extended to files later? found = False cmd = None os_to_pkg_cmd_map = { 'centos': RPM, 'debian': DPKG, 'redhat': RPM, 'ubuntu': DPKG, } pkg_cmd_flag = { DPKG: '-s', RPM: '-q', } os_name = get_os_name() if os_name in os_to_pkg_cmd_map: pkg_cmds = [os_to_pkg_cmd_map[os_name]] else: pkg_cmds = [RPM, DPKG] for pkg_cmd in pkg_cmds: if which(pkg_cmd): cmd = ' '.join([pkg_cmd, pkg_cmd_flag.get(pkg_cmd), dep]) found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True, trace=False, stream_output=False) if found: break if not found: # fallback for when os-dependency is a binary/library found = which(dep) # try locate if it's available if not found and which('locate'): cmd = 'locate --regexp "/%s$"' % dep found = run_cmd(cmd, simple=True, log_all=False, log_ok=False, force_in_dry_run=True, trace=False, stream_output=False) return found
def get_system_info(): """Return a dictionary with system information.""" python_version = '; '.join(sys.version.split('\n')) return { 'core_count': get_avail_core_count(), 'cpu_model': get_cpu_model(), 'cpu_speed': get_cpu_speed(), 'cpu_vendor': get_cpu_vendor(), 'gcc_version': get_tool_version('gcc', version_option='-v'), 'hostname': gethostname(), 'glibc_version': get_glibc_version(), 'os_name': get_os_name(), 'os_type': get_os_type(), 'os_version': get_os_version(), 'platform_name': get_platform_name(), 'python_version': python_version, 'system_python_path': which('python'), 'system_gcc_path': which('gcc'), }
def configure_step(self): """Custom configuration procedure for Bazel.""" binutils_root = get_software_root('binutils') gcc_root = get_software_root('GCCcore') or get_software_root('GCC') gcc_ver = get_software_version('GCCcore') or get_software_version('GCC') # only patch Bazel scripts if binutils & GCC installation prefix could be determined if binutils_root and gcc_root: res = glob.glob(os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include')) if res and len(res) == 1: gcc_lib_inc = res[0] else: raise EasyBuildError("Failed to pinpoint location of GCC include files: %s", res) gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed') if not os.path.exists(gcc_lib_inc_fixed): raise EasyBuildError("Derived directory %s does not exist", gcc_lib_inc_fixed) gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver) if not os.path.exists(gcc_cplusplus_inc): raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc) # replace hardcoded paths in CROSSTOOL regex_subs = [ (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')), (r'(cxx_builtin_include_directory:.*)/usr/lib/gcc', r'\1%s' % gcc_lib_inc), (r'(cxx_builtin_include_directory:.*)/usr/local/include', r'\1%s' % gcc_lib_inc_fixed), (r'(cxx_builtin_include_directory:.*)/usr/include', r'\1%s' % gcc_cplusplus_inc), ] for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']: path = which(tool) if path: regex_subs.append((os.path.join('/usr', 'bin', tool), path)) else: raise EasyBuildError("Failed to determine path to '%s'", tool) apply_regex_substitutions(os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs) # replace hardcoded paths in (unix_)cc_configure.bzl regex_subs = [ (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')), (r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')), ] for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']: filepath = os.path.join('tools', 'cpp', conf_bzl) if os.path.exists(filepath): apply_regex_substitutions(filepath, regex_subs) else: self.log.info("Not patching Bazel build scripts, installation prefix for binutils/GCC not found") # enable building in parallel env.setvar('EXTRA_BAZEL_ARGS', '--jobs=%d' % self.cfg['parallel'])
def check_pkg_support(): """Check whether packaging is supported, i.e. whether the required dependencies are available.""" # packaging support is considered experimental for now (requires using --experimental) _log.experimental("Support for packaging installed software.") pkgtool = build_option('package_tool') pkgtool_path = which(pkgtool) if pkgtool_path: _log.info("Selected packaging tool '%s' found at %s", pkgtool, pkgtool_path) # rpmbuild is required for generating RPMs with FPM if pkgtool == PKG_TOOL_FPM and build_option('package_type') == PKG_TYPE_RPM: rpmbuild_path = which('rpmbuild') if rpmbuild_path: _log.info("Required tool 'rpmbuild' found at %s", rpmbuild_path) else: raise EasyBuildError("rpmbuild is required when generating RPM packages with FPM, but was not found") else: raise EasyBuildError("Selected packaging tool '%s' not found", pkgtool)
def get_system_info(): """Return a dictionary with system information.""" python_version = "; ".join(sys.version.split("\n")) return { "core_count": get_avail_core_count(), "total_memory": get_total_memory(), "cpu_model": get_cpu_model(), "cpu_speed": get_cpu_speed(), "cpu_vendor": get_cpu_vendor(), "gcc_version": get_tool_version("gcc", version_option="-v"), "hostname": gethostname(), "glibc_version": get_glibc_version(), "os_name": get_os_name(), "os_type": get_os_type(), "os_version": get_os_version(), "platform_name": get_platform_name(), "python_version": python_version, "system_python_path": which("python"), "system_gcc_path": which("gcc"), }
def test_mock(self): """Test the mock module""" # ue empty mod_path list, otherwise the install_path is called mmt = MockModulesTool(mod_paths=[]) # the version of the MMT is the commandline option self.assertEqual(mmt.version, StrictVersion(MockModulesTool.VERSION_OPTION)) cmd_abspath = which(MockModulesTool.COMMAND) # make sure absolute path of module command is being used self.assertEqual(mmt.cmd, cmd_abspath)
def test_which(self): """Test which function for locating commands.""" python = ft.which('python') self.assertTrue(python and os.path.exists(python) and os.path.isabs(python)) path = ft.which('i_really_do_not_expect_a_command_with_a_name_like_this_to_be_available') self.assertTrue(path is None) os.environ['PATH'] = '%s:%s' % (self.test_prefix, os.environ['PATH']) # put a directory 'foo' in place (should be ignored by 'which') foo = os.path.join(self.test_prefix, 'foo') ft.mkdir(foo) ft.adjust_permissions(foo, stat.S_IRUSR|stat.S_IXUSR) # put executable file 'bar' in place bar = os.path.join(self.test_prefix, 'bar') ft.write_file(bar, '#!/bin/bash') ft.adjust_permissions(bar, stat.S_IRUSR|stat.S_IXUSR) self.assertEqual(ft.which('foo'), None) self.assertTrue(os.path.samefile(ft.which('bar'), bar)) # add another location to 'bar', which should only return the first location by default barbis = os.path.join(self.test_prefix, 'more', 'bar') ft.write_file(barbis, '#!/bin/bash') ft.adjust_permissions(barbis, stat.S_IRUSR|stat.S_IXUSR) os.environ['PATH'] = '%s:%s' % (os.environ['PATH'], os.path.dirname(barbis)) self.assertTrue(os.path.samefile(ft.which('bar'), bar)) # test getting *all* locations to specified command res = ft.which('bar', retain_all=True) self.assertEqual(len(res), 2) self.assertTrue(os.path.samefile(res[0], bar)) self.assertTrue(os.path.samefile(res[1], barbis))
def build_singularity_image(def_path): """Build Singularity container image by calling out to 'singularity' (requires admin privileges!).""" cont_path = container_path() def_file = os.path.basename(def_path) # use --imagename if specified, otherwise derive based on filename of recipe img_name = build_option('container_image_name') if img_name is None: # definition file Singularity.<app>-<version, container name <app>-<version>.<img|simg> img_name = def_file.split('.', 1)[1] cmd_opts = '' image_format = build_option('container_image_format') # squashfs image format (default for Singularity) if image_format in [None, CONT_IMAGE_FORMAT_SQUASHFS]: img_path = os.path.join(cont_path, img_name + '.simg') # ext3 image format, creating as writable container elif image_format == CONT_IMAGE_FORMAT_EXT3: img_path = os.path.join(cont_path, img_name + '.img') cmd_opts = '--writable' # sandbox image format, creates as a directory but acts like a container elif image_format == CONT_IMAGE_FORMAT_SANDBOX: img_path = os.path.join(cont_path, img_name) cmd_opts = '--sandbox' else: raise EasyBuildError("Unknown container image format specified for Singularity: %s" % image_format) if os.path.exists(img_path): if build_option('force'): print_msg("WARNING: overwriting existing container image at %s due to --force" % img_path) remove_file(img_path) else: raise EasyBuildError("Container image already exists at %s, not overwriting it without --force", img_path) # resolve full path to 'singularity' binary, since it may not be available via $PATH under sudo... singularity = which('singularity') cmd_env = '' singularity_tmpdir = build_option('container_tmpdir') if singularity_tmpdir: cmd_env += 'SINGULARITY_TMPDIR=%s' % singularity_tmpdir cmd = ' '.join(['sudo', cmd_env, singularity, 'build', cmd_opts, img_path, def_path]) print_msg("Running '%s', you may need to enter your 'sudo' password..." % cmd) run_cmd(cmd, stream_output=True) print_msg("Singularity image created at %s" % img_path, log=_log)
def test_lmod_specific(self): """Lmod-specific test (skipped unless Lmod is used as modules tool).""" lmod_abspath = which(Lmod.COMMAND) # only run this test if 'lmod' is available in $PATH if lmod_abspath is not None: build_options = { 'allow_modules_tool_mismatch': True, 'update_modules_tool_cache': True, } init_config(build_options=build_options) lmod = Lmod(testing=True) self.assertEqual(lmod.cmd, os.path.realpath(lmod_abspath)) # drop any location where 'lmod' or 'spider' can be found from $PATH paths = os.environ.get('PATH', '').split(os.pathsep) new_paths = [] for path in paths: lmod_cand_path = os.path.join(path, Lmod.COMMAND) spider_cand_path = os.path.join(path, 'spider') if not os.path.isfile(lmod_cand_path) and not os.path.isfile(spider_cand_path): new_paths.append(path) os.environ['PATH'] = os.pathsep.join(new_paths) # make sure $MODULEPATH contains path that provides some modules os.environ['MODULEPATH'] = os.path.abspath(os.path.join(os.path.dirname(__file__), 'modules')) # initialize Lmod modules tool, pass (fake) full path to 'lmod' via $LMOD_CMD fake_path = os.path.join(self.test_installpath, 'lmod') fake_lmod_txt = '\n'.join([ '#!/bin/bash', 'echo "Modules based on Lua: Version %s " >&2' % Lmod.REQ_VERSION, 'echo "os.environ[\'FOO\'] = \'foo\'"', ]) write_file(fake_path, fake_lmod_txt) os.chmod(fake_path, stat.S_IRUSR|stat.S_IXUSR) os.environ['LMOD_CMD'] = fake_path init_config(build_options=build_options) lmod = Lmod(testing=True) self.assertEqual(lmod.cmd, os.path.realpath(fake_path)) # use correct full path for 'lmod' via $LMOD_CMD os.environ['LMOD_CMD'] = lmod_abspath init_config(build_options=build_options) lmod = Lmod(testing=True) # obtain list of availabe modules, should be non-empty self.assertTrue(lmod.available(), "List of available modules obtained using Lmod is non-empty") # test updating local spider cache (but don't actually update the local cache file!) self.assertTrue(lmod.update(), "Updated local Lmod spider cache is non-empty")
def test_build_easyconfigs_in_parallel_gc3pie(self): """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" try: import gc3libs # noqa (ignore unused import) except ImportError: print "GC3Pie not available, skipping test" return # put GC3Pie config in place to use local host and fork/exec resourcedir = os.path.join(self.test_prefix, 'gc3pie') gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini') gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % { 'resourcedir': resourcedir, 'time': which('time'), } write_file(gc3pie_cfgfile, gc3pie_cfgtxt) output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir') # purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir) mkdir(output_dir, parents=True) write_file(os.path.join(output_dir, 'foo'), 'bar') # remove write permissions on parent dir of specified output dir, # to check that GC3Pie does not try to rename the (already existing) output directory... adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=GC3Pie'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) topdir = os.path.dirname(os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_easyblocks_path = os.path.join(os.path.dirname(os.path.abspath(__file__)), 'sandbox') cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % (topdir, test_easyblocks_path) build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) self.assertTrue(os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')) self.assertTrue(os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def test_end2end_docker_image(self): topdir = os.path.dirname(os.path.abspath(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) mkdir(containerpath) args = [ toy_ec, '-C', # equivalent with --containerize '--experimental', '--container-type=docker', '--container-base=ubuntu:16.04', '--container-build-image', ] if not which('docker'): error_pattern = "docker not found on your system." self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True) # install mocked versions of 'sudo' and 'docker' commands docker = os.path.join(self.test_prefix, 'bin', 'docker') write_file(docker, MOCKED_DOCKER) adjust_permissions(docker, stat.S_IXUSR, add=True) sudo = os.path.join(self.test_prefix, 'bin', 'sudo') write_file(sudo, '#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n') adjust_permissions(sudo, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join([os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs = [ "^== docker tool found at %s/bin/docker" % self.test_prefix, "^== Dockerfile definition file created at %s/containers/Dockerfile\.toy-0.0" % self.test_prefix, "^== Running 'sudo docker build -f .* -t .* \.', you may need to enter your 'sudo' password...", "^== Docker image created at toy-0.0:latest", ] self.check_regexs(regexs, stdout) args.extend(['--force', '--extended-dry-run']) stdout, stderr = self.run_main(args) self.assertFalse(stderr) self.check_regexs(regexs, stdout)
def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): """ Return a list of absolute paths where the specified subdir can be found, determined by the PYTHONPATH """ paths = [] # primary search path is robot path path_list = [] if isinstance(robot_path, list): path_list = robot_path[:] elif robot_path is not None: path_list = [robot_path] # consider Python search path, e.g. setuptools install path for easyconfigs path_list.extend(sys.path) # figure out installation prefix, e.g. distutils install path for easyconfigs eb_path = which('eb') if eb_path is None: _log.warning("'eb' not found in $PATH, failed to determine installation prefix") else: # real location to 'eb' should be <install_prefix>/bin/eb eb_path = resolve_path(eb_path) install_prefix = os.path.dirname(os.path.dirname(eb_path)) path_list.append(install_prefix) _log.debug("Also considering installation prefix %s..." % install_prefix) # look for desired subdirs for path in path_list: path = os.path.join(path, "easybuild", subdir) _log.debug("Checking for easybuild/%s at %s" % (subdir, path)) try: if os.path.exists(path): paths.append(os.path.abspath(path)) _log.debug("Added %s to list of paths for easybuild/%s" % (path, subdir)) except OSError as err: raise EasyBuildError(str(err)) return paths
def check_singularity(): """Check whether Singularity can be used (if it's needed).""" # if we're going to build a container image, we'll need a sufficiently recent version of Singularity available # (and otherwise we don't really care if Singularity is not available) if build_option('container_build_image'): path_to_singularity_cmd = which('singularity') if path_to_singularity_cmd: print_msg("Singularity tool found at %s" % path_to_singularity_cmd) out, ec = run_cmd("singularity --version", simple=False, trace=False, force_in_dry_run=True) if ec: raise EasyBuildError("Failed to determine Singularity version: %s" % out) else: # singularity version format for 2.3.1 and higher is x.y-dist singularity_version = out.strip().split('-')[0] if LooseVersion(singularity_version) < LooseVersion('2.4'): raise EasyBuildError("Please upgrade singularity instance to version 2.4 or higher") else: print_msg("Singularity version '%s' is 2.4 or higher ... OK" % singularity_version) else: raise EasyBuildError("Singularity not found in your system")
def prepare_compiler_cache(self, cache_tool): """ Prepare for using specified compiler caching tool (e.g., ccache, f90cache) :param cache_tool: name of compiler caching tool to prepare for """ compilers = self.comp_cache_compilers(cache_tool) self.log.debug("Using compiler cache tool '%s' for compilers: %s", cache_tool, compilers) # set paths that should be used by compiler caching tool comp_cache_path = build_option('use_%s' % cache_tool) setvar('%s_DIR' % cache_tool.upper(), comp_cache_path) setvar('%s_TEMPDIR' % cache_tool.upper(), tempfile.mkdtemp()) cache_path = which(cache_tool) if cache_path is None: raise EasyBuildError("%s binary not found in $PATH, required by --use-compiler-cache", cache) else: self.symlink_commands({cache_tool: (cache_path, compilers)}) self.cached_compilers.update(compilers) self.log.debug("Cached compilers (after preparing for %s): %s", cache_tool, self.cached_compilers)
def check_python_cmd(python_cmd): """Check whether specified Python command satisfies requirements.""" # check whether specified Python command is available if os.path.isabs(python_cmd): if not os.path.isfile(python_cmd): log.debug("Python command '%s' does not exist", python_cmd) return False else: python_cmd_path = which(python_cmd) if python_cmd_path is None: log.debug("Python command '%s' not available through $PATH", python_cmd) return False if req_maj_ver is not None: if req_min_ver is None: req_majmin_ver = '%s.0' % req_maj_ver else: req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver) pycode = 'import sys; print("%s.%s" % sys.version_info[:2])' out, _ = run_cmd("%s -c '%s'" % (python_cmd, pycode), simple=False) out = out.strip() # (strict) check for major version maj_ver = out.split('.')[0] if maj_ver != str(req_maj_ver): log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver) return False # check for minimal minor version if LooseVersion(out) < LooseVersion(req_majmin_ver): log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", out, req_majmin_ver) return False # all check passed log.debug("All check passed for Python command '%s'!", python_cmd) return True
def configure_step(self): """Custom configuration procedure for TensorFlow.""" tmpdir = tempfile.mkdtemp(suffix='-bazel-configure') # filter out paths from CPATH and LIBRARY_PATH package_filter = self.cfg['package_filter'] for var in ['CPATH', 'LIBRARY_PATH']: path = os.getenv(var).split(':') filtered_path = [p for fil in package_filter for p in path if fil not in p] os.environ[var] = ':'.join(filtered_path) # put wrapper for Intel C compiler in place (required to make sure license server is found) # cfr. https://github.com/bazelbuild/bazel/issues/663 if self.toolchain.comp_family() == toolchain.INTELCOMP: wrapper_dir = os.path.join(tmpdir, 'bin') icc_wrapper_txt = INTEL_COMPILER_WRAPPER % { 'compiler_path': which('icc'), 'cpath': os.getenv('CPATH'), 'intel_license_file': os.getenv('INTEL_LICENSE_FILE', os.getenv('LM_LICENSE_FILE')), 'wrapper_dir': wrapper_dir, } icc_wrapper = os.path.join(wrapper_dir, 'icc') write_file(icc_wrapper, icc_wrapper_txt) env.setvar('PATH', ':'.join([os.path.dirname(icc_wrapper), os.getenv('PATH')])) if self.dry_run: self.dry_run_msg("Wrapper for 'icc' was put in place: %s", icc_wrapper) else: adjust_permissions(icc_wrapper, stat.S_IXUSR) self.log.info("Using wrapper script for 'icc': %s", which('icc')) self.prepare_python() self.handle_jemalloc() cuda_root = get_software_root('CUDA') cudnn_root = get_software_root('cuDNN') opencl_root = get_software_root('OpenCL') use_mpi = self.toolchain.options.get('usempi', False) config_env_vars = { 'CC_OPT_FLAGS': os.getenv('CXXFLAGS'), 'MPI_HOME': '', 'PYTHON_BIN_PATH': self.python_cmd, 'PYTHON_LIB_PATH': os.path.join(self.installdir, self.pylibdir), 'TF_CUDA_CLANG': '0', 'TF_ENABLE_XLA': '0', # XLA JIT support 'TF_NEED_CUDA': ('0', '1')[bool(cuda_root)], 'TF_NEED_GCP': '0', # Google Cloud Platform 'TF_NEED_GDR': '0', 'TF_NEED_HDFS': '0', # Hadoop File System 'TF_NEED_JEMALLOC': ('0', '1')[self.cfg['with_jemalloc']], 'TF_NEED_MPI': ('0', '1')[bool(use_mpi)], 'TF_NEED_OPENCL': ('0', '1')[bool(opencl_root)], 'TF_NEED_S3': '0', # Amazon S3 File System 'TF_NEED_VERBS': '0', } if cuda_root: config_env_vars.update({ 'CUDA_TOOLKIT_PATH': cuda_root, 'GCC_HOST_COMPILER_PATH': which(os.getenv('CC')), 'TF_CUDA_COMPUTE_CAPABILITIES': ','.join(self.cfg['cuda_compute_capabilities']), 'TF_CUDA_VERSION': get_software_version('CUDA'), }) if cudnn_root: config_env_vars.update({ 'CUDNN_INSTALL_PATH': cudnn_root, 'TF_CUDNN_VERSION': get_software_version('cuDNN'), }) for (key, val) in sorted(config_env_vars.items()): env.setvar(key, val) # patch configure.py (called by configure script) to avoid that Bazel abuses $HOME/.cache/bazel regex_subs = [(r"(run_shell\(\['bazel')", r"\1, '--output_base=%s'" % tmpdir)] apply_regex_substitutions('configure.py', regex_subs) cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg['configopts'] run_cmd(cmd, log_all=True, simple=True)
def test_make_module_pythonpackage(self): """Test make_module_step of PythonPackage easyblock.""" app_class = get_easyblock_class('PythonPackage') self.writeEC('PythonPackage', name='testpypkg', version='3.14') app = app_class(EasyConfig(self.eb_file)) # install dir should not be there yet self.assertFalse(os.path.exists(app.installdir), "%s should not exist" % app.installdir) # create install dir and populate it with subdirs/files mkdir(app.installdir, parents=True) # $PATH, $LD_LIBRARY_PATH, $LIBRARY_PATH, $CPATH, $PKG_CONFIG_PATH write_file(os.path.join(app.installdir, 'bin', 'foo'), 'echo foo!') write_file(os.path.join(app.installdir, 'include', 'foo.h'), 'bar') write_file(os.path.join(app.installdir, 'lib', 'libfoo.a'), 'libfoo') pyver = '.'.join(map(str, sys.version_info[:2])) write_file( os.path.join(app.installdir, 'lib', 'python%s' % pyver, 'site-packages', 'foo.egg'), 'foo egg') write_file( os.path.join(app.installdir, 'lib64', 'pkgconfig', 'foo.pc'), 'libfoo: foo') # PythonPackage relies on the fact that 'python' points to the right Python version tmpdir = tempfile.mkdtemp() python = os.path.join(tmpdir, 'python') write_file(python, '#!/bin/bash\necho $0 $@\n%s "$@"' % sys.executable) adjust_permissions(python, stat.S_IXUSR) os.environ['PATH'] = '%s:%s' % (tmpdir, os.getenv('PATH', '')) from easybuild.tools.filetools import which print(which('python')) # create module file app.make_module_step() remove_file(python) self.assertTrue(TMPDIR in app.installdir) self.assertTrue(TMPDIR in app.installdir_mod) modtxt = None for cand_mod_filename in ['3.14', '3.14.lua']: full_modpath = os.path.join(app.installdir_mod, 'testpypkg', cand_mod_filename) if os.path.exists(full_modpath): modtxt = read_file(full_modpath) break self.assertFalse(modtxt is None) regexs = [ (r'^prepend.path.*\WCPATH\W.*include"?\W*$', True), (r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib"?\W*$', True), (r'^prepend.path.*\WLIBRARY_PATH\W.*lib"?\W*$', True), (r'^prepend.path.*\WPATH\W.*bin"?\W*$', True), (r'^prepend.path.*\WPKG_CONFIG_PATH\W.*lib64/pkgconfig"?\W*$', True), (r'^prepend.path.*\WPYTHONPATH\W.*lib/python[23]\.[0-9]/site-packages"?\W*$', True), # lib64 doesn't contain any library files, so these are *not* included in $LD_LIBRARY_PATH or $LIBRARY_PATH (r'^prepend.path.*\WLD_LIBRARY_PATH\W.*lib64', False), (r'^prepend.path.*\WLIBRARY_PATH\W.*lib64', False), ] for (pattern, found) in regexs: regex = re.compile(pattern, re.M) if found: assert_msg = "Pattern '%s' found in: %s" % (regex.pattern, modtxt) else: assert_msg = "Pattern '%s' not found in: %s" % (regex.pattern, modtxt) self.assertEqual(bool(regex.search(modtxt)), found, assert_msg)
def prepare_step(self, *args, **kwargs): """Do compiler appropriate prepare step, determine system compiler version and prefix.""" if self.cfg['generate_standalone_module']: if self.cfg['name'] in ['GCC', 'GCCcore']: EB_GCC.prepare_step(self, *args, **kwargs) elif self.cfg['name'] in ['icc']: EB_icc.prepare_step(self, *args, **kwargs) elif self.cfg['name'] in ['ifort']: EB_ifort.prepare_step(self, *args, **kwargs) else: raise EasyBuildError("I don't know how to do the prepare_step for %s", self.cfg['name']) else: Bundle.prepare_step(self, *args, **kwargs) # Determine compiler path (real path, with resolved symlinks) compiler_name = self.cfg['name'].lower() if compiler_name == 'gcccore': compiler_name = 'gcc' path_to_compiler = which(compiler_name) if path_to_compiler: path_to_compiler = resolve_path(path_to_compiler) self.log.info("Found path to compiler '%s' (with symlinks resolved): %s", compiler_name, path_to_compiler) else: raise EasyBuildError("%s not found in $PATH", compiler_name) # Determine compiler version self.compiler_version = extract_compiler_version(compiler_name) # Determine installation prefix if compiler_name == 'gcc': # strip off 'bin/gcc' self.compiler_prefix = os.path.dirname(os.path.dirname(path_to_compiler)) elif compiler_name in ['icc', 'ifort']: intelvars_fn = path_to_compiler + 'vars.sh' if os.path.isfile(intelvars_fn): self.log.debug("Trying to determine compiler install prefix from %s", intelvars_fn) intelvars_txt = read_file(intelvars_fn) prod_dir_regex = re.compile(r'^PROD_DIR=(.*)$', re.M) res = prod_dir_regex.search(intelvars_txt) if res: self.compiler_prefix = res.group(1) else: raise EasyBuildError("Failed to determine %s installation prefix from %s", compiler_name, intelvars_fn) else: # strip off 'bin/intel*/icc' self.compiler_prefix = os.path.dirname(os.path.dirname(os.path.dirname(path_to_compiler))) # For versions 2016+ of Intel compilers they changed the installation path so must shave off 2 more # directories from result of the above if LooseVersion(self.compiler_version) >= LooseVersion('2016'): self.compiler_prefix = os.path.dirname(os.path.dirname(self.compiler_prefix)) else: raise EasyBuildError("Unknown system compiler %s" % self.cfg['name']) if not os.path.exists(self.compiler_prefix): raise EasyBuildError("Path derived for system compiler (%s) does not exist: %s!", compiler_name, self.compiler_prefix) self.log.debug("Derived version/install prefix for system compiler %s: %s, %s", compiler_name, self.compiler_version, self.compiler_prefix) # If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it if self.cfg['version'] == 'system': self.log.info("Found specified version '%s', going with derived compiler version '%s'", self.cfg['version'], self.compiler_version) elif self.cfg['version'] != self.compiler_version: raise EasyBuildError("Specified version (%s) does not match version reported by compiler (%s)" % (self.cfg['version'], self.compiler_version))
def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None): """ Put RPATH wrapper script in place for compiler and linker commands :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) """ self.log.experimental( "Using wrapper scripts for compiler/linker commands that enforce RPATH linking" ) if get_os_type() == LINUX: self.log.info("Putting RPATH wrappers in place...") else: raise EasyBuildError( "RPATH linking is currently only supported on Linux") wrapper_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)? c_comps, fortran_comps = self.compilers() rpath_args_py = find_eb_script('rpath_args.py') rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in') # prepend location to wrappers to $PATH setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH'))) # figure out list of patterns to use in rpath filter rpath_filter = build_option('rpath_filter') if rpath_filter is None: rpath_filter = ['/lib.*', '/usr.*'] self.log.debug( "No general RPATH filter specified, falling back to default: %s", rpath_filter) rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs or []]) self.log.debug("Combined RPATH filter: '%s'", rpath_filter) rpath_include = ','.join(rpath_include_dirs or []) self.log.debug("Combined RPATH include paths: '%s'", rpath_include) # create wrappers for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']): orig_cmd = which(cmd) if orig_cmd: # bail out early if command already is a wrapped; # this may occur when building extensions if self.is_rpath_wrapper(orig_cmd): self.log.info( "%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd) continue cmd_wrapper = os.path.join(wrapper_dir, cmd) # make *very* sure we don't wrap around ourselves and create a fork bomb... if os.path.exists(cmd_wrapper) and os.path.exists( orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper): raise EasyBuildError( "Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd) # enable debug mode in wrapper script by specifying location for log file if build_option('debug'): rpath_wrapper_log = os.path.join( tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd) else: rpath_wrapper_log = '/dev/null' # complete template script and put it in place cmd_wrapper_txt = read_file(rpath_wrapper_template) % { 'orig_cmd': orig_cmd, 'python': sys.executable, 'rpath_args_py': rpath_args_py, 'rpath_filter': rpath_filter, 'rpath_include': rpath_include, 'rpath_wrapper_log': rpath_wrapper_log, } write_file(cmd_wrapper, cmd_wrapper_txt) adjust_permissions(cmd_wrapper, stat.S_IXUSR) self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log) else: self.log.debug( "Not installing RPATH wrapper for non-existing command '%s'", cmd)
def test_end2end_singularity_image(self): """End-to-end test for --containerize (recipe + image).""" topdir = os.path.dirname(os.path.abspath(__file__)) toy_ec = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') containerpath = os.path.join(self.test_prefix, 'containers') os.environ['EASYBUILD_CONTAINERPATH'] = containerpath # --containerpath must be an existing directory (this is done to avoid misconfiguration) mkdir(containerpath) test_img = os.path.join(self.test_prefix, 'test123.img') write_file(test_img, '') args = [ toy_ec, '-C', # equivalent with --containerize '--experimental', '--container-config=bootstrap=localimage,from=%s' % test_img, '--container-build-image', ] if which('singularity') is None: error_pattern = "singularity with version 2.4 or higher not found on your system." self.assertErrorRegex(EasyBuildError, error_pattern, self.eb_main, args, raise_error=True) # install mocked versions of 'sudo' and 'singularity' commands singularity = os.path.join(self.test_prefix, 'bin', 'singularity') write_file(singularity, '') # placeholder adjust_permissions(singularity, stat.S_IXUSR, add=True) sudo = os.path.join(self.test_prefix, 'bin', 'sudo') write_file( sudo, '#!/bin/bash\necho "running command \'$@\' with sudo..."\neval "$@"\n' ) adjust_permissions(sudo, stat.S_IXUSR, add=True) os.environ['PATH'] = os.path.pathsep.join( [os.path.join(self.test_prefix, 'bin'), os.getenv('PATH')]) for (version, ext) in [('2.4.0', 'simg'), ('3.1.0', 'sif')]: write_file(singularity, MOCKED_SINGULARITY % {'version': version}) stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs = [ r"^== singularity tool found at %s/bin/singularity" % self.test_prefix, r"^== singularity version '%s' is 2.4 or higher ... OK" % version, r"^== Singularity definition file created at %s/containers/Singularity\.toy-0.0" % self.test_prefix, r"^== Running 'sudo\s*\S*/singularity build\s*/.* /.*', you may need to enter your 'sudo' password...", r"^== Singularity image created at %s/containers/toy-0.0\.%s" % (self.test_prefix, ext), ] self.check_regexs(regexs, stdout) self.assertTrue( os.path.exists(os.path.join(containerpath, 'toy-0.0.%s' % ext))) remove_file(os.path.join(containerpath, 'Singularity.toy-0.0')) # check use of --container-image-format & --container-image-name write_file(singularity, MOCKED_SINGULARITY % {'version': '2.4.0'}) args.extend([ "--container-image-format=ext3", "--container-image-name=foo-bar", ]) stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs = [ r"^== singularity tool found at %s/bin/singularity" % self.test_prefix, r"^== singularity version '2.4.0' is 2.4 or higher ... OK", r"^== Singularity definition file created at %s/containers/Singularity\.foo-bar" % self.test_prefix, r"^== Running 'sudo\s*\S*/singularity build --writable /.* /.*', you may need to enter .*", r"^== Singularity image created at %s/containers/foo-bar\.img$" % self.test_prefix, ] self.check_regexs(regexs, stdout) cont_img = os.path.join(containerpath, 'foo-bar.img') self.assertTrue(os.path.exists(cont_img)) remove_file(os.path.join(containerpath, 'Singularity.foo-bar')) # test again with container image already existing error_pattern = "Container image already exists at %s, not overwriting it without --force" % cont_img self.mock_stdout(True) self.assertErrorRegex(EasyBuildError, error_pattern, self.run_main, args, raise_error=True) self.mock_stdout(False) args.append('--force') stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs.extend([ "WARNING: overwriting existing container image at %s due to --force" % cont_img, ]) self.check_regexs(regexs, stdout) self.assertTrue(os.path.exists(cont_img)) # also check behaviour under --extended-dry-run args.append('--extended-dry-run') stdout, stderr = self.run_main(args) self.assertFalse(stderr) self.check_regexs(regexs, stdout) # test use of --container-tmpdir args.append('--container-tmpdir=%s' % self.test_prefix) stdout, stderr = self.run_main(args) self.assertFalse(stderr) regexs[ -3] = r"^== Running 'sudo\s*SINGULARITY_TMPDIR=%s \S*/singularity build .*" % self.test_prefix self.check_regexs(regexs, stdout)
def configure_step(self): """Custom configuration procedure for GROMACS: set configure options for configure or cmake.""" if LooseVersion(self.version) >= LooseVersion('4.6'): cuda = get_software_root('CUDA') if cuda: # CUDA with double precision is currently not supported in GROMACS yet # If easyconfig explicitly have double_precision=True error out, # otherwise warn about it and skip the double precision build if self.cfg.get('double_precision'): raise EasyBuildError("Double precision is not available for GPU build. " + "Please explicitly set \"double_precision = False\" " + "or remove it in the easyconfig file.") if self.double_prec_pattern in self.cfg['configopts']: if self.cfg.get('double_precision') is None: # Only print warning once when trying double precision # build the first time self.cfg['double_precision'] = False self.log.info("Double precision is not available for " + "GPU build. Skipping the double precision build.") self.log.info("skipping configure step") return self.cfg.update('configopts', "-DGMX_GPU=ON -DCUDA_TOOLKIT_ROOT_DIR=%s" % cuda) else: # explicitly disable GPU support if CUDA is not available, # to avoid that GROMACS find and uses a system-wide CUDA compiler self.cfg.update('configopts', "-DGMX_GPU=OFF") # check whether PLUMED is loaded as a dependency plumed_root = get_software_root('PLUMED') if plumed_root: # Need to check if PLUMED has an engine for this version engine = 'gromacs-%s' % self.version (out, _) = run_cmd("plumed-patch -l", log_all=True, simple=False) if not re.search(engine, out): raise EasyBuildError("There is no support in PLUMED version %s for GROMACS %s: %s", get_software_version('PLUMED'), self.version, out) # PLUMED patching must be done at different stages depending on # version of GROMACS. Just prepare first part of cmd here plumed_cmd = "plumed-patch -p -e %s" % engine if LooseVersion(self.version) < LooseVersion('4.6'): self.log.info("Using configure script for configuring GROMACS build.") if self.cfg['build_shared_libs']: self.cfg.update('configopts', "--enable-shared --disable-static") else: self.cfg.update('configopts', "--enable-static") # Use external BLAS and LAPACK self.cfg.update('configopts', "--with-external-blas --with-external-lapack") env.setvar('LIBS', "%s %s" % (os.environ['LIBLAPACK'], os.environ['LIBS'])) # Don't use the X window system self.cfg.update('configopts', "--without-x") # OpenMP is not supported for versions older than 4.5. if LooseVersion(self.version) >= LooseVersion('4.5'): # enable OpenMP support if desired if self.toolchain.options.get('openmp', None): self.cfg.update('configopts', "--enable-threads") else: self.cfg.update('configopts', "--disable-threads") elif self.toolchain.options.get('openmp', None): raise EasyBuildError("GROMACS version %s does not support OpenMP" % self.version) # GSL support if get_software_root('GSL'): self.cfg.update('configopts', "--with-gsl") else: self.cfg.update('configopts', "--without-gsl") # actually run configure via ancestor (not direct parent) self.cfg['configure_cmd'] = "./configure" ConfigureMake.configure_step(self) # Now patch GROMACS for PLUMED between configure and build if plumed_root: run_cmd(plumed_cmd, log_all=True, simple=True) else: if '-DGMX_MPI=ON' in self.cfg['configopts']: mpi_numprocs = self.cfg.get('mpi_numprocs', 0) if mpi_numprocs == 0: self.log.info("No number of test MPI tasks specified -- using default: %s", self.cfg['parallel']) mpi_numprocs = self.cfg['parallel'] elif mpi_numprocs > self.cfg['parallel']: self.log.warning("Number of test MPI tasks (%s) is greater than value for 'parallel': %s", mpi_numprocs, self.cfg['parallel']) mpiexec = self.cfg.get('mpiexec') if mpiexec: mpiexec_path = which(mpiexec) if mpiexec_path: self.cfg.update('configopts', "-DMPIEXEC=%s" % mpiexec_path) self.cfg.update('configopts', "-DMPIEXEC_NUMPROC_FLAG=%s" % self.cfg.get('mpiexec_numproc_flag')) self.cfg.update('configopts', "-DNUMPROC=%s" % mpi_numprocs) elif self.cfg['runtest']: raise EasyBuildError("'%s' not found in $PATH", mpiexec) else: raise EasyBuildError("No value found for 'mpiexec'") self.log.info("Using %s as MPI executable when testing, with numprocs flag '%s' and %s tasks", mpiexec_path, self.cfg.get('mpiexec_numproc_flag'), mpi_numprocs) if LooseVersion(self.version) >= LooseVersion('2019'): # Building the gmxapi interface requires shared libraries self.cfg['build_shared_libs'] = True self.cfg.update('configopts', "-DGMXAPI=ON") if LooseVersion(self.version) >= LooseVersion('2020'): # build Python bindings if Python is loaded as a dependency python_root = get_software_root('Python') if python_root: bin_python = os.path.join(python_root, 'bin', 'python') self.cfg.update('configopts', "-DPYTHON_EXECUTABLE=%s" % bin_python) self.cfg.update('configopts', "-DGMX_PYTHON_PACKAGE=ON") # Now patch GROMACS for PLUMED before cmake if plumed_root: if LooseVersion(self.version) >= LooseVersion('5.1'): # Use shared or static patch depending on # setting of self.cfg['build_shared_libs'] # and adapt cmake flags accordingly as per instructions # from "plumed patch -i" if self.cfg['build_shared_libs']: mode = 'shared' else: mode = 'static' plumed_cmd = plumed_cmd + ' -m %s' % mode run_cmd(plumed_cmd, log_all=True, simple=True) # prefer static libraries, if available if self.cfg['build_shared_libs']: self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=OFF") else: self.cfg.update('configopts', "-DGMX_PREFER_STATIC_LIBS=ON") # always specify to use external BLAS/LAPACK self.cfg.update('configopts', "-DGMX_EXTERNAL_BLAS=ON -DGMX_EXTERNAL_LAPACK=ON") # disable GUI tools self.cfg.update('configopts', "-DGMX_X11=OFF") # convince to build for an older architecture than present on the build node by setting GMX_SIMD CMake flag # it does not make sense for Cray, because OPTARCH is defined by the Cray Toolchain if self.toolchain.toolchain_family() != toolchain.CRAYPE: gmx_simd = self.get_gromacs_arch() if gmx_simd: if LooseVersion(self.version) < LooseVersion('5.0'): self.cfg.update('configopts', "-DGMX_CPU_ACCELERATION=%s" % gmx_simd) else: self.cfg.update('configopts', "-DGMX_SIMD=%s" % gmx_simd) # set regression test path prefix = 'regressiontests' if any([src['name'].startswith(prefix) for src in self.src]): self.cfg.update('configopts', "-DREGRESSIONTEST_PATH='%%(builddir)s/%s-%%(version)s' " % prefix) # enable OpenMP support if desired if self.toolchain.options.get('openmp', None): self.cfg.update('configopts', "-DGMX_OPENMP=ON") else: self.cfg.update('configopts', "-DGMX_OPENMP=OFF") imkl_root = get_software_root('imkl') if imkl_root: # using MKL for FFT, so it will also be used for BLAS/LAPACK imkl_include = os.path.join(imkl_root, 'mkl', 'include') self.cfg.update('configopts', '-DGMX_FFT_LIBRARY=mkl -DMKL_INCLUDE_DIR="%s" ' % imkl_include) libs = os.getenv('LAPACK_STATIC_LIBS').split(',') mkl_libs = [os.path.join(os.getenv('LAPACK_LIB_DIR'), lib) for lib in libs if lib != 'libgfortran.a'] mkl_libs = ['-Wl,--start-group'] + mkl_libs + ['-Wl,--end-group -lpthread -lm -ldl'] self.cfg.update('configopts', '-DMKL_LIBRARIES="%s" ' % ';'.join(mkl_libs)) else: for libname in ['BLAS', 'LAPACK']: libdir = os.getenv('%s_LIB_DIR' % libname) if self.toolchain.toolchain_family() == toolchain.CRAYPE: libsci_mpi_mp_lib = glob.glob(os.path.join(libdir, 'libsci_*_mpi_mp.a')) if libsci_mpi_mp_lib: self.cfg.update('configopts', '-DGMX_%s_USER=%s' % (libname, libsci_mpi_mp_lib[0])) else: raise EasyBuildError("Failed to find libsci library to link with for %s", libname) else: # -DGMX_BLAS_USER & -DGMX_LAPACK_USER require full path to library libs = os.getenv('%s_STATIC_LIBS' % libname).split(',') libpaths = [os.path.join(libdir, lib) for lib in libs if lib != 'libgfortran.a'] self.cfg.update('configopts', '-DGMX_%s_USER="******"' % (libname, ';'.join(libpaths))) # if libgfortran.a is listed, make sure it gets linked in too to avoiding linking issues if 'libgfortran.a' in libs: env.setvar('LDFLAGS', "%s -lgfortran -lm" % os.environ.get('LDFLAGS', '')) # no more GSL support in GROMACS 5.x, see http://redmine.gromacs.org/issues/1472 if LooseVersion(self.version) < LooseVersion('5.0'): # enable GSL when it's provided if get_software_root('GSL'): self.cfg.update('configopts', "-DGMX_GSL=ON") else: self.cfg.update('configopts', "-DGMX_GSL=OFF") # include flags for linking to zlib/XZ in $LDFLAGS if they're listed as a dep; # this is important for the tests, to correctly link against libxml2 for dep, link_flag in [('XZ', '-llzma'), ('zlib', '-lz')]: root = get_software_root(dep) if root: libdir = get_software_libdir(dep) ldflags = os.environ.get('LDFLAGS', '') env.setvar('LDFLAGS', "%s -L%s %s" % (ldflags, os.path.join(root, libdir), link_flag)) # complete configuration with configure_method of parent out = super(EB_GROMACS, self).configure_step() # for recent GROMACS versions, make very sure that a decent BLAS, LAPACK and FFT is found and used if LooseVersion(self.version) >= LooseVersion('4.6.5'): patterns = [ r"Using external FFT library - \S*", r"Looking for dgemm_ - found", r"Looking for cheev_ - found", ] for pattern in patterns: regex = re.compile(pattern, re.M) if not regex.search(out): raise EasyBuildError("Pattern '%s' not found in GROMACS configuration output.", pattern)
def fixup_hardcoded_paths(self): """Patch out hard coded paths to /tmp, compiler and binutils tools""" # replace hardcoded /tmp in java build scripts regex_subs = [ (r'`mktemp -d /tmp/tmp.XXXXXXXXXX`', '$$(mktemp -d $${TMPDIR:-/tmp}/tmp.XXXXXXXXXX)'), ] filepath = os.path.join('src', 'main', 'java', 'com', 'google', 'devtools', 'build', 'lib', 'BUILD') if os.path.exists(filepath): apply_regex_substitutions(filepath, regex_subs) binutils_root = get_software_root('binutils') gcc_root = get_software_root('GCCcore') or get_software_root('GCC') gcc_ver = get_software_version('GCCcore') or get_software_version( 'GCC') # only patch Bazel scripts if binutils & GCC installation prefix could be determined if not binutils_root or not gcc_root: self.log.info( "Not patching Bazel build scripts, installation prefix for binutils/GCC not found" ) return # replace hardcoded paths in (unix_)cc_configure.bzl # hard-coded paths in (unix_)cc_configure.bzl were removed in 0.19.0 if LooseVersion(self.version) < LooseVersion('0.19.0'): regex_subs = [ (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')), (r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')), ] for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']: filepath = os.path.join('tools', 'cpp', conf_bzl) if os.path.exists(filepath): apply_regex_substitutions(filepath, regex_subs) # replace hardcoded paths in CROSSTOOL # CROSSTOOL script is no longer there in Bazel 0.24.0 if LooseVersion(self.version) < LooseVersion('0.24.0'): res = glob.glob( os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include')) if res and len(res) == 1: gcc_lib_inc = res[0] else: raise EasyBuildError( "Failed to pinpoint location of GCC include files: %s", res) gcc_lib_inc_bis = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed') if not os.path.exists(gcc_lib_inc_bis): self.log.info( "Derived directory %s does not exist, falling back to %s", gcc_lib_inc_bis, gcc_lib_inc) gcc_lib_inc_bis = gcc_lib_inc gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver) if not os.path.exists(gcc_cplusplus_inc): raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc) regex_subs = [ (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')), (r'(cxx_builtin_include_directory:.*)/usr/lib/gcc', r'\1%s' % gcc_lib_inc), (r'(cxx_builtin_include_directory:.*)/usr/local/include', r'\1%s' % gcc_lib_inc_bis), (r'(cxx_builtin_include_directory:.*)/usr/include', r'\1%s' % gcc_cplusplus_inc), ] for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']: path = which(tool) if path: regex_subs.append((os.path.join('/usr', 'bin', tool), path)) else: raise EasyBuildError("Failed to determine path to '%s'", tool) apply_regex_substitutions( os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs)
def pick_python_cmd(req_maj_ver=None, req_min_ver=None): """ Pick 'python' command to use, based on specified version requirements. If the major version is specified, it must be an exact match (==). If the minor version is specified, it is considered a minimal minor version (>=). List of considered 'python' commands (in order) * 'python' available through $PATH * 'python<major_ver>' available through $PATH * 'python<major_ver>.<minor_ver>' available through $PATH * Python executable used in current session (sys.executable) """ log = fancylogger.getLogger('pick_python_cmd', fname=False) def check_python_cmd(python_cmd): """Check whether specified Python command satisfies requirements.""" # check whether specified Python command is available if os.path.isabs(python_cmd): if not os.path.isfile(python_cmd): log.debug("Python command '%s' does not exist", python_cmd) return False else: python_cmd_path = which(python_cmd) if python_cmd_path is None: log.debug("Python command '%s' not available through $PATH", python_cmd) return False if req_maj_ver is not None: if req_min_ver is None: req_majmin_ver = '%s.0' % req_maj_ver else: req_majmin_ver = '%s.%s' % (req_maj_ver, req_min_ver) pycode = 'import sys; print("%s.%s" % sys.version_info[:2])' out, _ = run_cmd("%s -c '%s'" % (python_cmd, pycode), simple=False) out = out.strip() # (strict) check for major version maj_ver = out.split('.')[0] if maj_ver != str(req_maj_ver): log.debug("Major Python version does not match: %s vs %s", maj_ver, req_maj_ver) return False # check for minimal minor version if LooseVersion(out) < LooseVersion(req_majmin_ver): log.debug("Minimal requirement for minor Python version not satisfied: %s vs %s", out, req_majmin_ver) return False # all check passed log.debug("All check passed for Python command '%s'!", python_cmd) return True # compose list of 'python' commands to consider python_cmds = ['python'] if req_maj_ver: python_cmds.append('python%s' % req_maj_ver) if req_min_ver: python_cmds.append('python%s.%s' % (req_maj_ver, req_min_ver)) python_cmds.append(sys.executable) log.debug("Considering Python commands: %s", ', '.join(python_cmds)) # try and find a 'python' command that satisfies the requirements res = None for python_cmd in python_cmds: if check_python_cmd(python_cmd): log.debug("Python command '%s' satisfies version requirements!", python_cmd) if os.path.isabs(python_cmd): res = python_cmd else: res = which(python_cmd) log.debug("Absolute path to retained Python command: %s", res) break else: log.debug("Python command '%s' does not satisfy version requirements (maj: %s, min: %s), moving on", req_maj_ver, req_min_ver, python_cmd) return res
def prepare_rpath_wrappers(self, rpath_filter_dirs=None, rpath_include_dirs=None): """ Put RPATH wrapper script in place for compiler and linker commands :param rpath_filter_dirs: extra directories to include in RPATH filter (e.g. build dir, tmpdir, ...) """ if get_os_type() == LINUX: self.log.info("Putting RPATH wrappers in place...") else: raise EasyBuildError("RPATH linking is currently only supported on Linux") if rpath_filter_dirs is None: rpath_filter_dirs = [] # always include filter for 'stubs' library directory, # cfr. https://github.com/easybuilders/easybuild-framework/issues/2683 rpath_filter_dirs.append('.*/lib(64)?/stubs/?') # directory where all wrappers will be placed wrappers_dir = os.path.join(tempfile.mkdtemp(), RPATH_WRAPPERS_SUBDIR) # must also wrap compilers commands, required e.g. for Clang ('gcc' on OS X)? c_comps, fortran_comps = self.compilers() rpath_args_py = find_eb_script('rpath_args.py') rpath_wrapper_template = find_eb_script('rpath_wrapper_template.sh.in') # figure out list of patterns to use in rpath filter rpath_filter = build_option('rpath_filter') if rpath_filter is None: rpath_filter = ['/lib.*', '/usr.*'] self.log.debug("No general RPATH filter specified, falling back to default: %s", rpath_filter) rpath_filter = ','.join(rpath_filter + ['%s.*' % d for d in rpath_filter_dirs]) self.log.debug("Combined RPATH filter: '%s'", rpath_filter) rpath_include = ','.join(rpath_include_dirs or []) self.log.debug("Combined RPATH include paths: '%s'", rpath_include) # create wrappers for cmd in nub(c_comps + fortran_comps + ['ld', 'ld.gold', 'ld.bfd']): orig_cmd = which(cmd) if orig_cmd: # bail out early if command already is a wrapped; # this may occur when building extensions if self.is_rpath_wrapper(orig_cmd): self.log.info("%s already seems to be an RPATH wrapper script, not wrapping it again!", orig_cmd) continue # determine location for this wrapper # each wrapper is placed in its own subdirectory to enable $PATH filtering per wrapper separately # avoid '+' character in directory name (for example with 'g++' command), which can cause trouble # (see https://github.com/easybuilders/easybuild-easyconfigs/issues/7339) wrapper_dir_name = '%s_wrapper' % cmd.replace('+', 'x') wrapper_dir = os.path.join(wrappers_dir, wrapper_dir_name) cmd_wrapper = os.path.join(wrapper_dir, cmd) # make *very* sure we don't wrap around ourselves and create a fork bomb... if os.path.exists(cmd_wrapper) and os.path.exists(orig_cmd) and os.path.samefile(orig_cmd, cmd_wrapper): raise EasyBuildError("Refusing the create a fork bomb, which(%s) == %s", cmd, orig_cmd) # enable debug mode in wrapper script by specifying location for log file if build_option('debug'): rpath_wrapper_log = os.path.join(tempfile.gettempdir(), 'rpath_wrapper_%s.log' % cmd) else: rpath_wrapper_log = '/dev/null' # complete template script and put it in place cmd_wrapper_txt = read_file(rpath_wrapper_template) % { 'orig_cmd': orig_cmd, 'python': sys.executable, 'rpath_args_py': rpath_args_py, 'rpath_filter': rpath_filter, 'rpath_include': rpath_include, 'rpath_wrapper_log': rpath_wrapper_log, 'wrapper_dir': wrapper_dir, } write_file(cmd_wrapper, cmd_wrapper_txt) adjust_permissions(cmd_wrapper, stat.S_IXUSR) self.log.info("Wrapper script for %s: %s (log: %s)", orig_cmd, which(cmd), rpath_wrapper_log) # prepend location to this wrapper to $PATH setvar('PATH', '%s:%s' % (wrapper_dir, os.getenv('PATH'))) else: self.log.debug("Not installing RPATH wrapper for non-existing command '%s'", cmd)
def configure_step(self, srcdir=None, builddir=None): """Configure build using cmake""" # Set the search paths for CMake tc_ipaths = self.toolchain.get_variable("CPPFLAGS", list) tc_lpaths = self.toolchain.get_variable("LDFLAGS", list) cpaths = os.getenv('CPATH', '').split(os.pathsep) lpaths = os.getenv('LD_LIBRARY_PATH', '').split(os.pathsep) include_paths = os.pathsep.join(nub(tc_ipaths + cpaths)) library_paths = os.pathsep.join(nub(tc_lpaths + lpaths)) setvar("CMAKE_INCLUDE_PATH", include_paths) setvar("CMAKE_LIBRARY_PATH", library_paths) if builddir is None and self.cfg.get('separate_build_dir', False): builddir = os.path.join(self.builddir, 'easybuild_obj') if builddir: mkdir(builddir, parents=True) change_dir(builddir) default_srcdir = self.cfg['start_dir'] else: default_srcdir = '.' if srcdir is None: if self.cfg.get('srcdir', None) is not None: srcdir = self.cfg['srcdir'] else: srcdir = default_srcdir options = ['-DCMAKE_INSTALL_PREFIX=%s' % self.installdir] env_to_options = { 'CC': 'CMAKE_C_COMPILER', 'CFLAGS': 'CMAKE_C_FLAGS', 'CXX': 'CMAKE_CXX_COMPILER', 'CXXFLAGS': 'CMAKE_CXX_FLAGS', 'F90': 'CMAKE_Fortran_COMPILER', 'FFLAGS': 'CMAKE_Fortran_FLAGS', } for env_name, option in env_to_options.items(): value = os.getenv(env_name) if value is not None: if option.endswith('_COMPILER') and self.cfg.get( 'abs_path_compilers', False): value = which(value) self.log.info( "Using absolute path to compiler command: %s", value) options.append("-D%s='%s'" % (option, value)) if build_option('rpath'): # instruct CMake not to fiddle with RPATH when --rpath is used, since it will undo stuff on install... # https://github.com/LLNL/spack/blob/0f6a5cd38538e8969d11bd2167f11060b1f53b43/lib/spack/spack/build_environment.py#L416 options.append('-DCMAKE_SKIP_RPATH=ON') # show what CMake is doing by default options.append('-DCMAKE_VERBOSE_MAKEFILE=ON') if not self.cfg.get('allow_system_boost', False): # don't pick up on system Boost if Boost is included as dependency # - specify Boost location via -DBOOST_ROOT # - instruct CMake to not search for Boost headers/libraries in other places # - disable search for Boost CMake package configuration file boost_root = get_software_root('Boost') if boost_root: options.extend([ '-DBOOST_ROOT=%s' % boost_root, '-DBoost_NO_SYSTEM_PATHS=ON', '-DBoost_NO_BOOST_CMAKE=ON', ]) options_string = ' '.join(options) if self.cfg.get('configure_cmd') == DEFAULT_CONFIGURE_CMD: command = ' '.join([ self.cfg['preconfigopts'], DEFAULT_CONFIGURE_CMD, options_string, self.cfg['configopts'], srcdir ]) else: command = ' '.join([ self.cfg['preconfigopts'], self.cfg.get('configure_cmd'), self.cfg['configopts'] ]) (out, _) = run_cmd(command, log_all=True, simple=False) return out
def configure_step(self): """Custom configuration procedure for Bazel.""" binutils_root = get_software_root('binutils') gcc_root = get_software_root('GCCcore') or get_software_root('GCC') gcc_ver = get_software_version('GCCcore') or get_software_version( 'GCC') # only patch Bazel scripts if binutils & GCC installation prefix could be determined if binutils_root and gcc_root: res = glob.glob( os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include')) if res and len(res) == 1: gcc_lib_inc = res[0] else: raise EasyBuildError( "Failed to pinpoint location of GCC include files: %s", res) gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed') if not os.path.exists(gcc_lib_inc_fixed): raise EasyBuildError("Derived directory %s does not exist", gcc_lib_inc_fixed) gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver) if not os.path.exists(gcc_cplusplus_inc): raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc) # replace hardcoded paths in CROSSTOOL # CROSSTOOL script is no longer there in Bazel 0.24.0 if LooseVersion(self.version) < LooseVersion('0.24.0'): regex_subs = [ (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')), (r'(cxx_builtin_include_directory:.*)/usr/lib/gcc', r'\1%s' % gcc_lib_inc), (r'(cxx_builtin_include_directory:.*)/usr/local/include', r'\1%s' % gcc_lib_inc_fixed), (r'(cxx_builtin_include_directory:.*)/usr/include', r'\1%s' % gcc_cplusplus_inc), ] for tool in ['ar', 'cpp', 'dwp', 'gcc', 'ld']: path = which(tool) if path: regex_subs.append((os.path.join('/usr', 'bin', tool), path)) else: raise EasyBuildError( "Failed to determine path to '%s'", tool) apply_regex_substitutions( os.path.join('tools', 'cpp', 'CROSSTOOL'), regex_subs) # replace hardcoded paths in (unix_)cc_configure.bzl regex_subs = [ (r'-B/usr/bin', '-B%s' % os.path.join(binutils_root, 'bin')), (r'"/usr/bin', '"' + os.path.join(binutils_root, 'bin')), ] for conf_bzl in ['cc_configure.bzl', 'unix_cc_configure.bzl']: filepath = os.path.join('tools', 'cpp', conf_bzl) if os.path.exists(filepath): apply_regex_substitutions(filepath, regex_subs) else: self.log.info( "Not patching Bazel build scripts, installation prefix for binutils/GCC not found" ) # enable building in parallel bazel_args = '--jobs=%d' % self.cfg['parallel'] # Bazel provides a JDK by itself for some architectures # We want to enforce it using the JDK we provided via modules # This is required for Power where Bazel does not have a JDK, but requires it for building itself # See https://github.com/bazelbuild/bazel/issues/10377 bazel_args += ' --host_javabase=@local_jdk//:jdk' env.setvar('EXTRA_BAZEL_ARGS', bazel_args)
def test_build_easyconfigs_in_parallel_gc3pie(self): """Test build_easyconfigs_in_parallel(), using GC3Pie with local config as backend for --job.""" try: import gc3libs # noqa (ignore unused import) except ImportError: print "GC3Pie not available, skipping test" return # put GC3Pie config in place to use local host and fork/exec resourcedir = os.path.join(self.test_prefix, 'gc3pie') gc3pie_cfgfile = os.path.join(self.test_prefix, 'gc3pie_local.ini') gc3pie_cfgtxt = GC3PIE_LOCAL_CONFIGURATION % { 'resourcedir': resourcedir, 'time': which('time'), } write_file(gc3pie_cfgfile, gc3pie_cfgtxt) output_dir = os.path.join(self.test_prefix, 'subdir', 'gc3pie_output_dir') # purposely pre-create output dir, and put a file in it (to check whether GC3Pie tries to rename the output dir) mkdir(output_dir, parents=True) write_file(os.path.join(output_dir, 'foo'), 'bar') # remove write permissions on parent dir of specified output dir, # to check that GC3Pie does not try to rename the (already existing) output directory... adjust_permissions(os.path.dirname(output_dir), stat.S_IWUSR | stat.S_IWGRP | stat.S_IWOTH, add=False, recursive=False) topdir = os.path.dirname(os.path.abspath(__file__)) build_options = { 'job_backend_config': gc3pie_cfgfile, 'job_max_walltime': 24, 'job_output_dir': output_dir, 'job_polling_interval': 0.2, # quick polling 'job_target_resource': 'ebtestlocalhost', 'robot_path': os.path.join(topdir, 'easyconfigs', 'test_ecs'), 'silent': True, 'valid_module_classes': config.module_classes(), 'validate': False, } init_config(args=['--job-backend=GC3Pie'], build_options=build_options) ec_file = os.path.join(topdir, 'easyconfigs', 'test_ecs', 't', 'toy', 'toy-0.0.eb') easyconfigs = process_easyconfig(ec_file) ordered_ecs = resolve_dependencies(easyconfigs, self.modtool) topdir = os.path.dirname( os.path.dirname(os.path.dirname(os.path.abspath(__file__)))) test_easyblocks_path = os.path.join( os.path.dirname(os.path.abspath(__file__)), 'sandbox') cmd = "PYTHONPATH=%s:%s:$PYTHONPATH eb %%(spec)s -df" % ( topdir, test_easyblocks_path) build_easyconfigs_in_parallel(cmd, ordered_ecs, prepare_first=False) self.assertTrue( os.path.join(self.test_installpath, 'modules', 'all', 'toy', '0.0')) self.assertTrue( os.path.join(self.test_installpath, 'software', 'toy', '0.0', 'bin', 'toy'))
def build_step(self): """Custom build procedure for TensorFlow.""" # pre-create target installation directory mkdir(os.path.join(self.installdir, self.pylibdir), parents=True) binutils_root = get_software_root('binutils') if binutils_root: binutils_bin = os.path.join(binutils_root, 'bin') else: raise EasyBuildError("Failed to determine installation prefix for binutils") gcc_root = get_software_root('GCCcore') or get_software_root('GCC') if gcc_root: gcc_lib64 = os.path.join(gcc_root, 'lib64') gcc_ver = get_software_version('GCCcore') or get_software_version('GCC') # figure out location of GCC include files res = glob.glob(os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include')) if res and len(res) == 1: gcc_lib_inc = res[0] else: raise EasyBuildError("Failed to pinpoint location of GCC include files: %s", res) # make sure include-fixed directory is where we expect it to be gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed') if not os.path.exists(gcc_lib_inc_fixed): raise EasyBuildError("Derived directory %s does not exist", gcc_lib_inc_fixed) # also check on location of include/c++/<gcc version> directory gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver) if not os.path.exists(gcc_cplusplus_inc): raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc) else: raise EasyBuildError("Failed to determine installation prefix for GCC") inc_paths = [gcc_lib_inc, gcc_lib_inc_fixed, gcc_cplusplus_inc] lib_paths = [gcc_lib64] cuda_root = get_software_root('CUDA') if cuda_root: inc_paths.append(os.path.join(cuda_root, 'include')) lib_paths.append(os.path.join(cuda_root, 'lib64')) # fix hardcoded locations of compilers & tools cxx_inc_dir_lines = '\n'.join(r'cxx_builtin_include_directory: "%s"' % resolve_path(p) for p in inc_paths) cxx_inc_dir_lines_no_resolv_path = '\n'.join(r'cxx_builtin_include_directory: "%s"' % p for p in inc_paths) regex_subs = [ (r'-B/usr/bin/', '-B%s/ %s' % (binutils_bin, ' '.join('-L%s/' % p for p in lib_paths))), (r'(cxx_builtin_include_directory:).*', ''), (r'^toolchain {', 'toolchain {\n' + cxx_inc_dir_lines + '\n' + cxx_inc_dir_lines_no_resolv_path), ] for tool in ['ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy', 'objdump', 'strip']: path = which(tool) if path: regex_subs.append((os.path.join('/usr', 'bin', tool), path)) else: raise EasyBuildError("Failed to determine path to '%s'", tool) # -fPIE/-pie and -fPIC are not compatible, so patch out hardcoded occurences of -fPIE/-pie if -fPIC is used if self.toolchain.options.get('pic', None): regex_subs.extend([('-fPIE', '-fPIC'), ('"-pie"', '"-fPIC"')]) # patch all CROSSTOOL* scripts to fix hardcoding of locations of binutils/GCC binaries for path, dirnames, filenames in os.walk(self.start_dir): for filename in filenames: if filename.startswith('CROSSTOOL'): full_path = os.path.join(path, filename) self.log.info("Patching %s", full_path) apply_regex_substitutions(full_path, regex_subs) tmpdir = tempfile.mkdtemp(suffix='-bazel-build') # compose "bazel build" command with all its options... cmd = [self.cfg['prebuildopts'], 'bazel', '--output_base=%s' % tmpdir, 'build'] # build with optimization enabled # cfr. https://docs.bazel.build/versions/master/user-manual.html#flag--compilation_mode cmd.append('--compilation_mode=opt') # select 'opt' config section (this is *not* the same as --compilation_mode=opt!) # https://docs.bazel.build/versions/master/user-manual.html#flag--config cmd.append('--config=opt') # make Bazel print full command line + make it verbose on failures # https://docs.bazel.build/versions/master/user-manual.html#flag--subcommands # https://docs.bazel.build/versions/master/user-manual.html#flag--verbose_failures cmd.extend(['--subcommands', '--verbose_failures']) # limit the number of parallel jobs running simultaneously (useful on KNL)... cmd.append('--jobs=%s' % self.cfg['parallel']) if self.toolchain.options.get('pic', None): cmd.append('--copt="-fPIC"') cmd.append(self.cfg['buildopts']) if cuda_root: cmd.append('--config=cuda') # enable mkl-dnn by default, but only if cuDNN is not listed as dependency if self.cfg['with_mkl_dnn'] is None and get_software_root('cuDNN') is None: self.log.info("Enabling use of mkl-dnn since cuDNN is not listed as dependency") self.cfg['with_mkl_dnn'] = True # if mkl-dnn is listed as a dependency it is used. Otherwise downloaded if with_mkl_dnn is true mkl_root = get_software_root('mkl-dnn') if mkl_root: cmd.extend(['--config=mkl']) cmd.insert(0, "export TF_MKL_DOWNLOAD=0 &&") cmd.insert(0, "export TF_MKL_ROOT=%s &&" % mkl_root) elif self.cfg['with_mkl_dnn']: # this makes TensorFlow use mkl-dnn (cfr. https://github.com/01org/mkl-dnn) cmd.extend(['--config=mkl']) cmd.insert(0, "export TF_MKL_DOWNLOAD=1 && ") # specify target of the build command as last argument cmd.append('//tensorflow/tools/pip_package:build_pip_package') run_cmd(' '.join(cmd), log_all=True, simple=True, log_ok=True) # run generated 'build_pip_package' script to build the .whl cmd = "bazel-bin/tensorflow/tools/pip_package/build_pip_package %s" % self.builddir run_cmd(cmd, log_all=True, simple=True, log_ok=True)
def configure_step(self, srcdir=None, builddir=None): """Configure build using cmake""" setup_cmake_env(self.toolchain) if builddir is None and self.cfg.get('separate_build_dir', True): builddir = create_unused_dir(self.builddir, 'easybuild_obj') if builddir: mkdir(builddir, parents=True) change_dir(builddir) default_srcdir = self.cfg['start_dir'] else: default_srcdir = '.' if srcdir is None: if self.cfg.get('srcdir', None) is not None: # Note that the join returns srcdir if it is absolute srcdir = os.path.join(default_srcdir, self.cfg['srcdir']) else: srcdir = default_srcdir options = ['-DCMAKE_INSTALL_PREFIX=%s' % self.installdir] if self.installdir.startswith('/opt') or self.installdir.startswith( '/usr'): # https://cmake.org/cmake/help/latest/module/GNUInstallDirs.html localstatedir = os.path.join(self.installdir, 'var') runstatedir = os.path.join(localstatedir, 'run') sysconfdir = os.path.join(self.installdir, 'etc') options.append("-DCMAKE_INSTALL_LOCALSTATEDIR=%s" % localstatedir) options.append("-DCMAKE_INSTALL_RUNSTATEDIR=%s" % runstatedir) options.append("-DCMAKE_INSTALL_SYSCONFDIR=%s" % sysconfdir) if '-DCMAKE_BUILD_TYPE=' in self.cfg['configopts']: if self.cfg.get('build_type') is not None: self.log.warning( 'CMAKE_BUILD_TYPE is set in configopts. Ignoring build_type' ) else: options.append('-DCMAKE_BUILD_TYPE=%s' % self.build_type) # Add -fPIC flag if necessary if self.toolchain.options['pic']: options.append('-DCMAKE_POSITION_INDEPENDENT_CODE=ON') if self.cfg['generator']: options.append('-G "%s"' % self.cfg['generator']) # pass --sysroot value down to CMake, # and enable using absolute paths to compiler commands to avoid # that CMake picks up compiler from sysroot rather than toolchain compiler... sysroot = build_option('sysroot') if sysroot: options.append('-DCMAKE_SYSROOT=%s' % sysroot) self.log.info( "Using absolute path to compiler commands because of alterate sysroot %s", sysroot) self.cfg['abs_path_compilers'] = True # Set flag for shared libs if requested # Not adding one allows the project to choose a default build_shared_libs = self.cfg.get('build_shared_libs') if build_shared_libs is not None: # Contrary to other options build_shared_libs takes precedence over configopts which may be unexpected. # This is to allow self.lib_ext to be determined correctly. # Usually you want to remove -DBUILD_SHARED_LIBS from configopts and set build_shared_libs to True or False # If you need it in configopts don't set build_shared_libs (or explicitely set it to `None` (Default)) if '-DBUILD_SHARED_LIBS=' in self.cfg['configopts']: print_warning( 'Ignoring BUILD_SHARED_LIBS is set in configopts because build_shared_libs is set' ) self.cfg.update( 'configopts', '-DBUILD_SHARED_LIBS=%s' % ('ON' if build_shared_libs else 'OFF')) env_to_options = { 'CC': 'CMAKE_C_COMPILER', 'CFLAGS': 'CMAKE_C_FLAGS', 'CXX': 'CMAKE_CXX_COMPILER', 'CXXFLAGS': 'CMAKE_CXX_FLAGS', 'F90': 'CMAKE_Fortran_COMPILER', 'FFLAGS': 'CMAKE_Fortran_FLAGS', } for env_name, option in env_to_options.items(): value = os.getenv(env_name) if value is not None: if option.endswith('_COMPILER') and self.cfg.get( 'abs_path_compilers', False): value = which(value) self.log.info( "Using absolute path to compiler command: %s", value) options.append("-D%s='%s'" % (option, value)) if build_option('rpath'): # instruct CMake not to fiddle with RPATH when --rpath is used, since it will undo stuff on install... # https://github.com/LLNL/spack/blob/0f6a5cd38538e8969d11bd2167f11060b1f53b43/lib/spack/spack/build_environment.py#L416 options.append('-DCMAKE_SKIP_RPATH=ON') # show what CMake is doing by default options.append('-DCMAKE_VERBOSE_MAKEFILE=ON') # disable CMake user package repository options.append('-DCMAKE_FIND_USE_PACKAGE_REGISTRY=FALSE') if not self.cfg.get('allow_system_boost', False): # don't pick up on system Boost if Boost is included as dependency # - specify Boost location via -DBOOST_ROOT # - instruct CMake to not search for Boost headers/libraries in other places # - disable search for Boost CMake package configuration file boost_root = get_software_root('Boost') if boost_root: options.extend([ '-DBOOST_ROOT=%s' % boost_root, '-DBoost_NO_SYSTEM_PATHS=ON', '-DBoost_NO_BOOST_CMAKE=ON', ]) options_string = ' '.join(options) if self.cfg.get('configure_cmd') == DEFAULT_CONFIGURE_CMD: command = ' '.join([ self.cfg['preconfigopts'], DEFAULT_CONFIGURE_CMD, options_string, self.cfg['configopts'], srcdir ]) else: command = ' '.join([ self.cfg['preconfigopts'], self.cfg.get('configure_cmd'), self.cfg['configopts'] ]) (out, _) = run_cmd(command, log_all=True, simple=False) return out
def build_image(self, recipe_path): """Build container image by calling out to 'sudo singularity build'.""" cont_path = container_path() def_file = os.path.basename(recipe_path) # use --imagename if specified, otherwise derive based on filename of recipe img_name = self.img_name if img_name is None: # definition file Singularity.<app>-<version, container name <app>-<version>.<img|simg> img_name = def_file.split('.', 1)[1] cmd_opts = '' image_format = self.image_format singularity_version = self.singularity_version() # squashfs image format (default for Singularity) if image_format in [ None, CONT_IMAGE_FORMAT_SQUASHFS, CONT_IMAGE_FORMAT_SIF ]: if LooseVersion(singularity_version) > LooseVersion('3.0'): ext = '.sif' else: ext = '.simg' img_path = os.path.join(cont_path, img_name + ext) # ext3 image format, creating as writable container elif image_format == CONT_IMAGE_FORMAT_EXT3: if LooseVersion(singularity_version) > LooseVersion('3.0'): raise EasyBuildError( "ext3 image format is only supported with Singularity 2.x (found Singularity %s)", singularity_version) else: img_path = os.path.join(cont_path, img_name + '.img') cmd_opts = '--writable' # sandbox image format, creates as a directory but acts like a container elif image_format == CONT_IMAGE_FORMAT_SANDBOX: img_path = os.path.join(cont_path, img_name) cmd_opts = '--sandbox' else: raise EasyBuildError( "Unknown container image format specified for Singularity: %s" % image_format) if os.path.exists(img_path): if build_option('force'): print_msg( "WARNING: overwriting existing container image at %s due to --force" % img_path) remove_file(img_path) else: raise EasyBuildError( "Container image already exists at %s, not overwriting it without --force", img_path) # resolve full path to 'singularity' binary, since it may not be available via $PATH under sudo... singularity = which('singularity') cmd_env = '' singularity_tmpdir = self.tmpdir if singularity_tmpdir: cmd_env += 'SINGULARITY_TMPDIR=%s' % singularity_tmpdir cmd = ' '.join([ 'sudo', cmd_env, singularity, 'build', cmd_opts, img_path, recipe_path ]) print_msg( "Running '%s', you may need to enter your 'sudo' password..." % cmd) run_cmd(cmd, stream_output=True) print_msg("Singularity image created at %s" % img_path, log=self.log)
def get_paths_for(subdir=EASYCONFIGS_PKG_SUBDIR, robot_path=None): """ Return a list of absolute paths where the specified subdir can be found, determined by the PYTHONPATH """ paths = [] # primary search path is robot path path_list = [] if isinstance(robot_path, list): path_list = robot_path[:] elif robot_path is not None: path_list = [robot_path] # consider Python search path, e.g. setuptools install path for easyconfigs path_list.extend(sys.path) # figure out installation prefix, e.g. distutils install path for easyconfigs # prefer using path specified in $EB_SCRIPT_PATH (if defined), which is set by 'eb' wrapper script eb_path = os.getenv('EB_SCRIPT_PATH') if eb_path is None: # try to determine location of 'eb' script via $PATH, as fallback mechanism eb_path = which('eb') _log.info("Location to 'eb' script (found via $PATH): %s", eb_path) else: _log.info("Found location to 'eb' script via $EB_SCRIPT_PATH: %s", eb_path) if eb_path is None: warning_msg = "'eb' not found in $PATH, failed to determine installation prefix!" _log.warning(warning_msg) print_warning(warning_msg) else: # eb_path is location to 'eb' wrapper script, e.g. <install_prefix>/bin/eb # so installation prefix is usually two levels up install_prefix = os.path.dirname(os.path.dirname(eb_path)) # only consider resolved path to 'eb' script if desired subdir is not found relative to 'eb' script location if os.path.exists(os.path.join(install_prefix, 'easybuild', subdir)): path_list.append(install_prefix) _log.info( "Also considering installation prefix %s (determined via path to 'eb' script)...", install_prefix) else: _log.info("Not considering %s (no easybuild/%s subdir found)", install_prefix, subdir) # also consider fully resolved location to 'eb' wrapper # see https://github.com/easybuilders/easybuild-framework/pull/2248 resolved_eb_path = resolve_path(eb_path) if eb_path != resolved_eb_path: install_prefix = os.path.dirname( os.path.dirname(resolved_eb_path)) path_list.append(install_prefix) _log.info( "Also considering installation prefix %s (via resolved path to 'eb')...", install_prefix) # look for desired subdirs for path in path_list: path = os.path.join(path, "easybuild", subdir) _log.debug("Checking for easybuild/%s at %s" % (subdir, path)) try: if os.path.exists(path): paths.append(os.path.abspath(path)) _log.debug("Added %s to list of paths for easybuild/%s" % (path, subdir)) except OSError as err: raise EasyBuildError(str(err)) return paths
def prepare_step(self, *args, **kwargs): """Load all dependencies, determine system MPI version, prefix and any associated envvars.""" # Do the bundle prepare step to ensure any deps are loaded (no need to worry about licences for Intel MPI) Bundle.prepare_step(self, *args, **kwargs) # Prepare additional parameters: determine system MPI version, prefix and any associated envvars. mpi_name = self.cfg['name'].lower() # Determine MPI wrapper path (real path, with resolved symlinks) to ensure it exists if mpi_name == 'impi': # For impi the version information is only found in *some* of the wrappers it ships, in particular it is # not in mpicc mpi_c_wrapper = 'mpiicc' path_to_mpi_c_wrapper = which(mpi_c_wrapper) if not path_to_mpi_c_wrapper: mpi_c_wrapper = 'mpigcc' path_to_mpi_c_wrapper = which(mpi_c_wrapper) if not path_to_mpi_c_wrapper: raise EasyBuildError("Could not find suitable MPI wrapper to extract version for impi") else: mpi_c_wrapper = 'mpicc' path_to_mpi_c_wrapper = which(mpi_c_wrapper) if path_to_mpi_c_wrapper: path_to_mpi_c_wrapper = resolve_path(path_to_mpi_c_wrapper) self.log.info("Found path to MPI implementation '%s' %s compiler (with symlinks resolved): %s", mpi_name, mpi_c_wrapper, path_to_mpi_c_wrapper) else: raise EasyBuildError("%s not found in $PATH", mpi_c_wrapper) # Determine MPI version, installation prefix and underlying compiler if mpi_name in ('openmpi', 'spectrummpi'): # Spectrum MPI is based on Open MPI so is also covered by this logic output_of_ompi_info, _ = run_cmd("ompi_info", simple=False) # Extract the version of the MPI implementation if mpi_name == 'spectrummpi': mpi_version_string = 'Spectrum MPI' else: mpi_version_string = 'Open MPI' self.mpi_version = self.extract_ompi_setting(mpi_version_string, output_of_ompi_info) # Extract the installation prefix self.mpi_prefix = self.extract_ompi_setting("Prefix", output_of_ompi_info) # Extract any OpenMPI environment variables in the current environment and ensure they are added to the # final module self.mpi_env_vars = dict((key, value) for key, value in os.environ.items() if key.startswith('OMPI_')) # Extract the C compiler used underneath the MPI implementation, check for the definition of OMPI_MPICC self.mpi_c_compiler = self.extract_ompi_setting("C compiler", output_of_ompi_info) elif mpi_name == 'impi': # Extract the version of IntelMPI # The prefix in the the mpiicc (or mpigcc) script can be used to extract the explicit version contents_of_mpixcc = read_file(path_to_mpi_c_wrapper) prefix_regex = re.compile(r'(?<=compilers_and_libraries_)(.*)(?=/linux/mpi)', re.M) self.mpi_version = None res = prefix_regex.search(contents_of_mpixcc) if res: self.mpi_version = res.group(1) else: # old iimpi version prefix_regex = re.compile(r'^prefix=(.*)$', re.M) res = prefix_regex.search(contents_of_mpixcc) if res: self.mpi_version = res.group(1).split('/')[-1] if self.mpi_version is None: raise EasyBuildError("No version found for system Intel MPI") else: self.log.info("Found Intel MPI version %s for system MPI" % self.mpi_version) # Extract the installation prefix, if I_MPI_ROOT is defined, let's use that i_mpi_root = os.environ.get('I_MPI_ROOT') if i_mpi_root: self.mpi_prefix = i_mpi_root else: # Else just go up three directories from where mpiicc is found # (it's 3 because bin64 is a symlink to intel64/bin and we are assuming 64 bit) self.mpi_prefix = os.path.dirname(os.path.dirname(os.path.dirname(path_to_mpi_c_wrapper))) # Extract any IntelMPI environment variables in the current environment and ensure they are added to the # final module self.mpi_env_vars = {} for key, value in os.environ.items(): i_mpi_key = key.startswith('I_MPI_') or key.startswith('MPICH_') mpi_profile_key = key.startswith('MPI') and key.endswith('PROFILE') if i_mpi_key or mpi_profile_key: self.mpi_env_vars[key] = value # Extract the C compiler used underneath Intel MPI compile_info, exit_code = run_cmd("%s -compile-info" % mpi_c_wrapper, simple=False) if exit_code == 0: self.mpi_c_compiler = compile_info.split(' ', 1)[0] else: raise EasyBuildError("Could not determine C compiler underneath Intel MPI, '%s -compiler-info' " "returned %s", mpi_c_wrapper, compile_info) else: raise EasyBuildError("Unrecognised system MPI implementation %s", mpi_name) # Ensure install path of system MPI actually exists if not os.path.exists(self.mpi_prefix): raise EasyBuildError("Path derived for system MPI (%s) does not exist: %s!", mpi_name, self.mpi_prefix) self.log.debug("Derived version/install prefix for system MPI %s: %s, %s", mpi_name, self.mpi_version, self.mpi_prefix) # For the version of the underlying C compiler need to explicitly extract (to be certain) self.c_compiler_version = extract_compiler_version(self.mpi_c_compiler) self.log.debug("Derived compiler/version for C compiler underneath system MPI %s: %s, %s", mpi_name, self.mpi_c_compiler, self.c_compiler_version) # If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it if self.cfg['version'] == 'system': self.log.info("Found specified version '%s', going with derived MPI version '%s'", self.cfg['version'], self.mpi_version) elif self.cfg['version'] == self.mpi_version: self.log.info("Specified MPI version %s matches found version" % self.mpi_version) else: raise EasyBuildError("Specified version (%s) does not match version reported by MPI (%s)", self.cfg['version'], self.mpi_version)
def build_step(self): """Custom build procedure for TensorFlow.""" # pre-create target installation directory mkdir(os.path.join(self.installdir, self.pylibdir), parents=True) binutils_root = get_software_root('binutils') if binutils_root: binutils_bin = os.path.join(binutils_root, 'bin') else: raise EasyBuildError( "Failed to determine installation prefix for binutils") gcc_root = get_software_root('GCCcore') or get_software_root('GCC') if gcc_root: gcc_lib64 = os.path.join(gcc_root, 'lib64') gcc_ver = get_software_version('GCCcore') or get_software_version( 'GCC') # figure out location of GCC include files res = glob.glob( os.path.join(gcc_root, 'lib', 'gcc', '*', gcc_ver, 'include')) if res and len(res) == 1: gcc_lib_inc = res[0] else: raise EasyBuildError( "Failed to pinpoint location of GCC include files: %s", res) # make sure include-fixed directory is where we expect it to be gcc_lib_inc_fixed = os.path.join(os.path.dirname(gcc_lib_inc), 'include-fixed') if not os.path.exists(gcc_lib_inc_fixed): raise EasyBuildError("Derived directory %s does not exist", gcc_lib_inc_fixed) # also check on location of include/c++/<gcc version> directory gcc_cplusplus_inc = os.path.join(gcc_root, 'include', 'c++', gcc_ver) if not os.path.exists(gcc_cplusplus_inc): raise EasyBuildError("Derived directory %s does not exist", gcc_cplusplus_inc) else: raise EasyBuildError( "Failed to determine installation prefix for GCC") inc_paths = [gcc_lib_inc, gcc_lib_inc_fixed, gcc_cplusplus_inc] lib_paths = [gcc_lib64] cuda_root = get_software_root('CUDA') if cuda_root: inc_paths.append(os.path.join(cuda_root, 'include')) lib_paths.append(os.path.join(cuda_root, 'lib64')) # fix hardcoded locations of compilers & tools cxx_inc_dir_lines = '\n'.join(r'cxx_builtin_include_directory: "%s"' % resolve_path(p) for p in inc_paths) cxx_inc_dir_lines_no_resolv_path = '\n'.join( r'cxx_builtin_include_directory: "%s"' % p for p in inc_paths) regex_subs = [ (r'-B/usr/bin/', '-B%s/ %s' % (binutils_bin, ' '.join('-L%s/' % p for p in lib_paths))), (r'(cxx_builtin_include_directory:).*', ''), (r'^toolchain {', 'toolchain {\n' + cxx_inc_dir_lines + '\n' + cxx_inc_dir_lines_no_resolv_path), ] for tool in [ 'ar', 'cpp', 'dwp', 'gcc', 'gcov', 'ld', 'nm', 'objcopy', 'objdump', 'strip' ]: path = which(tool) if path: regex_subs.append((os.path.join('/usr', 'bin', tool), path)) else: raise EasyBuildError("Failed to determine path to '%s'", tool) # -fPIE/-pie and -fPIC are not compatible, so patch out hardcoded occurences of -fPIE/-pie if -fPIC is used if self.toolchain.options.get('pic', None): regex_subs.extend([('-fPIE', '-fPIC'), ('"-pie"', '"-fPIC"')]) # patch all CROSSTOOL* scripts to fix hardcoding of locations of binutils/GCC binaries for path, dirnames, filenames in os.walk(os.getcwd()): for filename in filenames: if filename.startswith('CROSSTOOL'): full_path = os.path.join(path, filename) self.log.info("Patching %s", full_path) apply_regex_substitutions(full_path, regex_subs) tmpdir = tempfile.mkdtemp(suffix='-bazel-build') user_root_tmpdir = tempfile.mkdtemp(suffix='-user_root') # compose "bazel build" command with all its options... cmd = [ self.cfg['prebuildopts'], 'bazel', '--output_base=%s' % tmpdir, '--install_base=%s' % os.path.join(tmpdir, 'inst_base'), '--output_user_root=%s' % user_root_tmpdir, 'build' ] # build with optimization enabled # cfr. https://docs.bazel.build/versions/master/user-manual.html#flag--compilation_mode cmd.append('--compilation_mode=opt') # select 'opt' config section (this is *not* the same as --compilation_mode=opt!) # https://docs.bazel.build/versions/master/user-manual.html#flag--config cmd.append('--config=opt') # make Bazel print full command line + make it verbose on failures # https://docs.bazel.build/versions/master/user-manual.html#flag--subcommands # https://docs.bazel.build/versions/master/user-manual.html#flag--verbose_failures cmd.extend(['--subcommands', '--verbose_failures']) # limit the number of parallel jobs running simultaneously (useful on KNL)... cmd.append('--jobs=%s' % self.cfg['parallel']) if self.toolchain.options.get('pic', None): cmd.append('--copt="-fPIC"') # include install location of Python packages in $PYTHONPATH, # and specify that value of $PYTHONPATH should be passed down into Bazel build environment; # this is required to make sure that Python packages included as extensions are found at build time; # see also https://github.com/tensorflow/tensorflow/issues/22395 pythonpath = os.getenv('PYTHONPATH', '') env.setvar( 'PYTHONPATH', os.pathsep.join( [os.path.join(self.installdir, self.pylibdir), pythonpath])) cmd.append('--action_env=PYTHONPATH') # Also export $EBPYTHONPREFIXES to handle the multi-deps python setup # See https://github.com/easybuilders/easybuild-easyblocks/pull/1664 if 'EBPYTHONPREFIXES' in os.environ: cmd.append('--action_env=EBPYTHONPREFIXES') # use same configuration for both host and target programs, which can speed up the build # only done when optarch is enabled, since this implicitely assumes that host and target platform are the same # see https://docs.bazel.build/versions/master/guide.html#configurations if self.toolchain.options.get('optarch'): cmd.append('--distinct_host_configuration=false') cmd.append(self.cfg['buildopts']) # building TensorFlow v2.0 requires passing --config=v2 to "bazel build" command... if LooseVersion(self.version) >= LooseVersion('2.0'): cmd.append('--config=v2') if cuda_root: cmd.append('--config=cuda') # if mkl-dnn is listed as a dependency it is used. Otherwise downloaded if with_mkl_dnn is true mkl_root = get_software_root('mkl-dnn') if mkl_root: cmd.extend(['--config=mkl']) cmd.insert(0, "export TF_MKL_DOWNLOAD=0 &&") cmd.insert(0, "export TF_MKL_ROOT=%s &&" % mkl_root) elif self.cfg['with_mkl_dnn']: # this makes TensorFlow use mkl-dnn (cfr. https://github.com/01org/mkl-dnn) cmd.extend(['--config=mkl']) cmd.insert(0, "export TF_MKL_DOWNLOAD=1 && ") # specify target of the build command as last argument cmd.append('//tensorflow/tools/pip_package:build_pip_package') run_cmd(' '.join(cmd), log_all=True, simple=True, log_ok=True) # run generated 'build_pip_package' script to build the .whl cmd = "bazel-bin/tensorflow/tools/pip_package/build_pip_package %s" % self.builddir run_cmd(cmd, log_all=True, simple=True, log_ok=True)
def __init__(self, *args, **kwargs): """Extra initialization: determine system compiler version and prefix.""" super(SystemCompiler, self).__init__(*args, **kwargs) # Determine compiler path (real path, with resolved symlinks) compiler_name = self.cfg['name'].lower() path_to_compiler = which(compiler_name) if path_to_compiler: path_to_compiler = os.path.realpath(path_to_compiler) self.log.info( "Found path to compiler '%s' (with symlinks resolved): %s", compiler_name, path_to_compiler) else: raise EasyBuildError("%s not found in $PATH", compiler_name) # Determine compiler version and installation prefix if compiler_name == 'gcc': out, _ = run_cmd("gcc --version", simple=False) self.extract_compiler_version(out) # strip off 'bin/gcc' self.compiler_prefix = os.path.dirname( os.path.dirname(path_to_compiler)) elif compiler_name in ['icc', 'ifort']: out, _ = run_cmd("%s -V" % compiler_name, simple=False) self.extract_compiler_version(out) intelvars_fn = path_to_compiler + 'vars.sh' if os.path.isfile(intelvars_fn): self.log.debug( "Trying to determine compiler install prefix from %s", intelvars_fn) intelvars_txt = read_file(intelvars_fn) prod_dir_regex = re.compile(r'^PROD_DIR=(.*)$', re.M) res = prod_dir_regex.search(intelvars_txt) if res: self.compiler_prefix = res.group(1) else: raise EasyBuildError( "Failed to determine %s installation prefix from %s", compiler_name, intelvars_fn) else: # strip off 'bin/intel*/icc' self.compiler_prefix = os.path.dirname( os.path.dirname(os.path.dirname(path_to_compiler))) else: raise EasyBuildError("Unknown system compiler %s" % self.cfg['name']) self.log.debug( "Derived version/install prefix for system compiler %s: %s, %s", compiler_name, self.compiler_version, self.compiler_prefix) # If EasyConfig specified "real" version (not 'system' which means 'derive automatically'), check it if self.cfg['version'] == 'system': self.log.info( "Found specified version '%s', going with derived compiler version '%s'", self.cfg['version'], self.compiler_version) elif self.cfg['version'] != self.compiler_version: raise EasyBuildError( "Specified version (%s) does not match version reported by compiler (%s)" % (self.cfg['version'], self.compiler_version)) # fix installdir and module names (may differ because of changes to version) mns = ActiveMNS() self.cfg.full_mod_name = mns.det_full_module_name(self.cfg) self.cfg.short_mod_name = mns.det_short_module_name(self.cfg) self.cfg.mod_subdir = mns.det_module_subdir(self.cfg) # keep track of original values, for restoring later self.orig_version = self.cfg['version'] self.orig_installdir = self.installdir
def configure_step(self): """Custom configuration procedure for TensorFlow.""" tmpdir = tempfile.mkdtemp(suffix='-bazel-configure') # filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that # might conflict with dependencies on the system and/or installed with EB. For example: protobuf path_filter = self.cfg['path_filter'] if path_filter: self.log.info( "Filtering $CPATH and $LIBRARY_PATH with path filter %s", path_filter) for var in ['CPATH', 'LIBRARY_PATH']: path = os.getenv(var).split(os.pathsep) self.log.info("$%s old value was %s" % (var, path)) filtered_path = os.pathsep.join( [p for fil in path_filter for p in path if fil not in p]) env.setvar(var, filtered_path) wrapper_dir = os.path.join(tmpdir, 'bin') use_wrapper = False if self.toolchain.comp_family() == toolchain.INTELCOMP: # put wrappers for Intel C/C++ compilers in place (required to make sure license server is found) # cfr. https://github.com/bazelbuild/bazel/issues/663 for compiler in ('icc', 'icpc'): self.write_wrapper(wrapper_dir, compiler, 'NOT-USED-WITH-ICC') use_wrapper = True use_mpi = self.toolchain.options.get('usempi', False) mpi_home = '' if use_mpi: impi_root = get_software_root('impi') if impi_root: # put wrappers for Intel MPI compiler wrappers in place # (required to make sure license server and I_MPI_ROOT are found) for compiler in (os.getenv('MPICC'), os.getenv('MPICXX')): self.write_wrapper(wrapper_dir, compiler, os.getenv('I_MPI_ROOT')) use_wrapper = True # set correct value for MPI_HOME mpi_home = os.path.join(impi_root, 'intel64') else: self.log.debug("MPI module name: %s", self.toolchain.MPI_MODULE_NAME[0]) mpi_home = get_software_root(self.toolchain.MPI_MODULE_NAME[0]) self.log.debug("Derived value for MPI_HOME: %s", mpi_home) if use_wrapper: env.setvar('PATH', os.pathsep.join([wrapper_dir, os.getenv('PATH')])) self.prepare_python() self.handle_jemalloc() cuda_root = get_software_root('CUDA') cudnn_root = get_software_root('cuDNN') opencl_root = get_software_root('OpenCL') tensorrt_root = get_software_root('TensorRT') nccl_root = get_software_root('NCCL') config_env_vars = { 'CC_OPT_FLAGS': os.getenv('CXXFLAGS'), 'MPI_HOME': mpi_home, 'PYTHON_BIN_PATH': self.python_cmd, 'PYTHON_LIB_PATH': os.path.join(self.installdir, self.pylibdir), 'TF_CUDA_CLANG': '0', 'TF_ENABLE_XLA': '0', # XLA JIT support 'TF_NEED_CUDA': ('0', '1')[bool(cuda_root)], 'TF_NEED_GCP': '0', # Google Cloud Platform 'TF_NEED_GDR': '0', 'TF_NEED_HDFS': '0', # Hadoop File System 'TF_NEED_JEMALLOC': ('0', '1')[self.cfg['with_jemalloc']], 'TF_NEED_MPI': ('0', '1')[bool(use_mpi)], 'TF_NEED_OPENCL': ('0', '1')[bool(opencl_root)], 'TF_NEED_OPENCL_SYCL': '0', 'TF_NEED_S3': '0', # Amazon S3 File System 'TF_NEED_TENSORRT': '0', 'TF_NEED_VERBS': '0', 'TF_NEED_AWS': '0', # Amazon AWS Platform 'TF_NEED_KAFKA': '0', # Amazon Kafka Platform } if cuda_root: cuda_version = get_software_version('CUDA') cuda_maj_min_ver = '.'.join(cuda_version.split('.')[:2]) # $GCC_HOST_COMPILER_PATH should be set to path of the actual compiler (not the MPI compiler wrapper) if use_mpi: compiler_path = which(os.getenv('CC_SEQ')) else: compiler_path = which(os.getenv('CC')) config_env_vars.update({ 'CUDA_TOOLKIT_PATH': cuda_root, 'GCC_HOST_COMPILER_PATH': compiler_path, 'TF_CUDA_COMPUTE_CAPABILITIES': ','.join(self.cfg['cuda_compute_capabilities']), 'TF_CUDA_VERSION': cuda_maj_min_ver, }) # for recent TensorFlow versions, $TF_CUDA_PATHS and $TF_CUBLAS_VERSION must also be set if LooseVersion(self.version) >= LooseVersion('1.14'): # figure out correct major/minor version for CUBLAS from cublas_api.h cublas_api_header_glob_pattern = os.path.join( cuda_root, 'targets', '*', 'include', 'cublas_api.h') matches = glob.glob(cublas_api_header_glob_pattern) if len(matches) == 1: cublas_api_header_path = matches[0] cublas_api_header_txt = read_file(cublas_api_header_path) else: raise EasyBuildError( "Failed to isolate path to cublas_api.h: %s", matches) cublas_ver_parts = [] for key in [ 'CUBLAS_VER_MAJOR', 'CUBLAS_VER_MINOR', 'CUBLAS_VER_PATCH' ]: regex = re.compile("^#define %s ([0-9]+)" % key, re.M) res = regex.search(cublas_api_header_txt) if res: cublas_ver_parts.append(res.group(1)) else: raise EasyBuildError( "Failed to find pattern '%s' in %s", regex.pattern, cublas_api_header_path) config_env_vars.update({ 'TF_CUDA_PATHS': cuda_root, 'TF_CUBLAS_VERSION': '.'.join(cublas_ver_parts), }) if cudnn_root: cudnn_version = get_software_version('cuDNN') cudnn_maj_min_patch_ver = '.'.join( cudnn_version.split('.')[:3]) config_env_vars.update({ 'CUDNN_INSTALL_PATH': cudnn_root, 'TF_CUDNN_VERSION': cudnn_maj_min_patch_ver, }) else: raise EasyBuildError( "TensorFlow has a strict dependency on cuDNN if CUDA is enabled" ) if nccl_root: nccl_version = get_software_version('NCCL') config_env_vars.update({ 'NCCL_INSTALL_PATH': nccl_root, }) else: nccl_version = '1.3' # Use simple downloadable version config_env_vars.update({ 'TF_NCCL_VERSION': nccl_version, }) if tensorrt_root: tensorrt_version = get_software_version('TensorRT') config_env_vars.update({ 'TF_NEED_TENSORRT': '1', 'TENSORRT_INSTALL_PATH': tensorrt_root, 'TF_TENSORRT_VERSION': tensorrt_version, }) for (key, val) in sorted(config_env_vars.items()): env.setvar(key, val) # patch configure.py (called by configure script) to avoid that Bazel abuses $HOME/.cache/bazel regex_subs = [(r"(run_shell\(\['bazel')", r"\1, '--output_base=%s', '--install_base=%s'" % (tmpdir, os.path.join(tmpdir, 'inst_base')))] apply_regex_substitutions('configure.py', regex_subs) # Tell Bazel to not use $HOME/.cache/bazel at all # See https://docs.bazel.build/versions/master/output_directories.html env.setvar('TEST_TMPDIR', os.path.join(tmpdir, 'output_root')) cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg[ 'configopts'] run_cmd(cmd, log_all=True, simple=True)
def __init__(self, *args, **kwargs): """Locate the installation files of OpenSSL in the host system""" super(EB_OpenSSL_wrapper, self).__init__(*args, **kwargs) # Libraries packaged in OpenSSL openssl_libs = ['libssl', 'libcrypto'] # list of relevant library extensions per system and version of OpenSSL # the first item should be the extension of an installation from source, # it will be used in the sanity checks of the component openssl_libext = { '1.0': { LINUX: ('so.1.0.0', 'so.10'), DARWIN: ('1.0.dylib', ), }, '1.1': { LINUX: ('so.1.1', ), DARWIN: ('1.1.dylib', ), }, } os_type = get_os_type() if self.version in openssl_libext and os_type in openssl_libext[ self.version]: # generate matrix of versioned .so filenames system_versioned_libs = [[ '%s.%s' % (lib, ext) for ext in openssl_libext[self.version][os_type] ] for lib in openssl_libs] self.log.info("Matrix of version library names: %s", system_versioned_libs) else: raise EasyBuildError( "Don't know name of OpenSSL system library for version %s and OS type %s", self.version, os_type) # by default target the first option of each OpenSSL library, # which corresponds to installation from source self.target_ssl_libs = [ lib_name[0] for lib_name in system_versioned_libs ] self.log.info("Target OpenSSL libraries: %s", self.target_ssl_libs) # folders containing engines libraries openssl_engines = { '1.0': 'engines', '1.1': 'engines-1.1', } self.target_ssl_engine = openssl_engines[self.version] # Paths to system libraries and headers of OpenSSL self.system_ssl = { 'bin': None, 'engines': None, 'include': None, 'lib': None, } # early return when we're not wrapping the system OpenSSL installation if not self.cfg.get('wrap_system_openssl'): self.log.info("Not wrapping system OpenSSL installation!") return # Check the system libraries of OpenSSL # (only needs first library in openssl_libs to find path to libraries) for idx, libssl in enumerate(system_versioned_libs[0]): self.system_ssl['lib'] = find_library_path(libssl) if self.system_ssl['lib']: # change target libraries to the ones found in the system self.target_ssl_libs = [ lib_name[idx] for lib_name in system_versioned_libs ] self.log.info("Target system OpenSSL libraries: %s", self.target_ssl_libs) break if self.system_ssl['lib']: self.log.info("Found library '%s' in: %s", openssl_libs[0], self.system_ssl['lib']) else: self.log.info( "OpenSSL library '%s' not found, falling back to OpenSSL in EasyBuild", openssl_libs[0]) # Directory with engine libraries if self.system_ssl['lib']: lib_dir = os.path.dirname(self.system_ssl['lib']) lib_engines_dir = [ os.path.join(lib_dir, 'openssl', self.target_ssl_engine), os.path.join(lib_dir, self.target_ssl_engine), ] for engines_path in lib_engines_dir: if os.path.isdir(engines_path): self.system_ssl['engines'] = engines_path self.log.debug("Found OpenSSL engines in: %s", self.system_ssl['engines']) break if not self.system_ssl['engines']: self.system_ssl['lib'] = None self.log.info( "OpenSSL engines not found in host system, falling back to OpenSSL in EasyBuild" ) # Check system include paths for OpenSSL headers cmd = "LC_ALL=C gcc -E -Wp,-v -xc /dev/null" (out, ec) = run_cmd(cmd, log_all=True, simple=False, trace=False) sys_include_dirs = [] for match in re.finditer(r'^\s(/[^\0\n]*)+', out, re.MULTILINE): sys_include_dirs.extend(match.groups()) self.log.debug( "Found the following include directories in host system: %s", ', '.join(sys_include_dirs)) # headers are located in 'include/openssl' by default ssl_include_subdirs = [self.name.lower()] if self.version == '1.1': # but version 1.1 can be installed in 'include/openssl11/openssl' as well, for example in CentOS 7 # prefer 'include/openssl' as long as the version of headers matches ssl_include_subdirs.append( os.path.join('openssl11', self.name.lower())) ssl_include_dirs = [ os.path.join(incd, subd) for incd in sys_include_dirs for subd in ssl_include_subdirs ] ssl_include_dirs = [ include for include in ssl_include_dirs if os.path.isdir(include) ] # find location of header files, verify that the headers match our OpenSSL version openssl_version_regex = re.compile( r"SHLIB_VERSION_NUMBER\s\"([0-9]+\.[0-9]+)", re.M) for include_dir in ssl_include_dirs: opensslv_path = os.path.join(include_dir, 'opensslv.h') self.log.debug("Checking OpenSSL version in %s...", opensslv_path) if os.path.exists(opensslv_path): opensslv = read_file(opensslv_path) header_majmin_version = openssl_version_regex.search(opensslv) if header_majmin_version: header_majmin_version = header_majmin_version.group(1) if re.match('^' + header_majmin_version, self.version): self.system_ssl['include'] = include_dir self.log.info( "Found OpenSSL headers in host system: %s", self.system_ssl['include']) break else: self.log.debug( "Header major/minor version '%s' doesn't match with %s", header_majmin_version, self.version) else: self.log.debug("Pattern '%s' not found in %s", openssl_version_regex.pattern, opensslv_path) else: self.log.info("OpenSSL header file %s not found", opensslv_path) if not self.system_ssl['include']: self.log.info( "OpenSSL headers not found in host system, falling back to OpenSSL in EasyBuild" ) # Check system OpenSSL binary if self.version == '1.1': # prefer 'openssl11' over 'openssl' with v1.1 self.system_ssl['bin'] = which('openssl11') if not self.system_ssl['bin']: self.system_ssl['bin'] = which(self.name.lower()) if self.system_ssl['bin']: self.log.info("System OpenSSL binary found: %s", self.system_ssl['bin']) else: self.log.info("System OpenSSL binary not found!")
def configure_step(self): """Custom configuration procedure for TensorFlow.""" tmpdir = tempfile.mkdtemp(suffix='-bazel-configure') # filter out paths from CPATH and LIBRARY_PATH. This is needed since bazel will pull some dependencies that # might conflict with dependencies on the system and/or installed with EB. For example: protobuf path_filter = self.cfg['path_filter'] if path_filter: self.log.info("Filtering $CPATH and $LIBRARY_PATH with path filter %s", path_filter) for var in ['CPATH', 'LIBRARY_PATH']: path = os.getenv(var).split(os.pathsep) self.log.info("$%s old value was %s" % (var, path)) filtered_path = os.pathsep.join([p for fil in path_filter for p in path if fil not in p]) env.setvar(var, filtered_path) wrapper_dir = os.path.join(tmpdir, 'bin') use_wrapper = False if self.toolchain.comp_family() == toolchain.INTELCOMP: # put wrappers for Intel C/C++ compilers in place (required to make sure license server is found) # cfr. https://github.com/bazelbuild/bazel/issues/663 for compiler in ('icc', 'icpc'): self.write_wrapper(wrapper_dir, compiler, 'NOT-USED-WITH-ICC') use_wrapper = True use_mpi = self.toolchain.options.get('usempi', False) mpi_home = '' if use_mpi: impi_root = get_software_root('impi') if impi_root: # put wrappers for Intel MPI compiler wrappers in place # (required to make sure license server and I_MPI_ROOT are found) for compiler in (os.getenv('MPICC'), os.getenv('MPICXX')): self.write_wrapper(wrapper_dir, compiler, os.getenv('I_MPI_ROOT')) use_wrapper = True # set correct value for MPI_HOME mpi_home = os.path.join(impi_root, 'intel64') else: self.log.debug("MPI module name: %s", self.toolchain.MPI_MODULE_NAME[0]) mpi_home = get_software_root(self.toolchain.MPI_MODULE_NAME[0]) self.log.debug("Derived value for MPI_HOME: %s", mpi_home) if use_wrapper: env.setvar('PATH', os.pathsep.join([wrapper_dir, os.getenv('PATH')])) self.prepare_python() self.handle_jemalloc() cuda_root = get_software_root('CUDA') cudnn_root = get_software_root('cuDNN') opencl_root = get_software_root('OpenCL') tensorrt_root = get_software_root('TensorRT') nccl_root = get_software_root('NCCL') config_env_vars = { 'CC_OPT_FLAGS': os.getenv('CXXFLAGS'), 'MPI_HOME': mpi_home, 'PYTHON_BIN_PATH': self.python_cmd, 'PYTHON_LIB_PATH': os.path.join(self.installdir, self.pylibdir), 'TF_CUDA_CLANG': '0', 'TF_ENABLE_XLA': '0', # XLA JIT support 'TF_NEED_CUDA': ('0', '1')[bool(cuda_root)], 'TF_NEED_GCP': '0', # Google Cloud Platform 'TF_NEED_GDR': '0', 'TF_NEED_HDFS': '0', # Hadoop File System 'TF_NEED_JEMALLOC': ('0', '1')[self.cfg['with_jemalloc']], 'TF_NEED_MPI': ('0', '1')[bool(use_mpi)], 'TF_NEED_OPENCL': ('0', '1')[bool(opencl_root)], 'TF_NEED_OPENCL_SYCL': '0', 'TF_NEED_S3': '0', # Amazon S3 File System 'TF_NEED_TENSORRT': '0', 'TF_NEED_VERBS': '0', 'TF_NEED_AWS': '0', # Amazon AWS Platform 'TF_NEED_KAFKA': '0', # Amazon Kafka Platform } if cuda_root: config_env_vars.update({ 'CUDA_TOOLKIT_PATH': cuda_root, 'GCC_HOST_COMPILER_PATH': which(os.getenv('CC')), 'TF_CUDA_COMPUTE_CAPABILITIES': ','.join(self.cfg['cuda_compute_capabilities']), 'TF_CUDA_VERSION': get_software_version('CUDA'), }) if cudnn_root: config_env_vars.update({ 'CUDNN_INSTALL_PATH': cudnn_root, 'TF_CUDNN_VERSION': get_software_version('cuDNN'), }) else: raise EasyBuildError("TensorFlow has a strict dependency on cuDNN if CUDA is enabled") if nccl_root: nccl_version = get_software_version('NCCL') config_env_vars.update({ 'NCCL_INSTALL_PATH': nccl_root, }) else: nccl_version = '1.3' # Use simple downloadable version config_env_vars.update({ 'TF_NCCL_VERSION': nccl_version, }) if tensorrt_root: tensorrt_version = get_software_version('TensorRT') config_env_vars.update({ 'TF_NEED_TENSORRT': '1', 'TENSORRT_INSTALL_PATH': tensorrt_root, 'TF_TENSORRT_VERSION': tensorrt_version, }) for (key, val) in sorted(config_env_vars.items()): env.setvar(key, val) # patch configure.py (called by configure script) to avoid that Bazel abuses $HOME/.cache/bazel regex_subs = [(r"(run_shell\(\['bazel')", r"\1, '--output_base=%s', '--install_base=%s'" % (tmpdir, os.path.join(tmpdir, 'inst_base')))] apply_regex_substitutions('configure.py', regex_subs) cmd = self.cfg['preconfigopts'] + './configure ' + self.cfg['configopts'] run_cmd(cmd, log_all=True, simple=True)
def install_step(self): """ Custom install step for GROMACS; figure out where libraries were installed to. Also, install the MPI version of the executable in a separate step. """ # run 'make install' in parallel since it involves more compilation self.cfg.update('installopts', "-j %s" % self.cfg['parallel']) super(EB_GROMACS, self).install_step() # the GROMACS libraries get installed in different locations (deeper subdirectory), depending on the platform; # this is determined by the GNUInstallDirs CMake module; # rather than trying to replicate the logic, we just figure out where the library was placed if self.toolchain.options.get('dynamic', False): self.libext = get_shared_lib_ext() else: self.libext = 'a' if LooseVersion(self.version) < LooseVersion('5.0'): libname = 'libgmx*.%s' % self.libext else: libname = 'libgromacs*.%s' % self.libext for libdir in ['lib', 'lib64']: if os.path.exists(os.path.join(self.installdir, libdir)): for subdir in [libdir, os.path.join(libdir, '*')]: libpaths = glob.glob( os.path.join(self.installdir, subdir, libname)) if libpaths: self.lib_subdir = os.path.dirname( libpaths[0])[len(self.installdir) + 1:] self.log.info( "Found lib subdirectory that contains %s: %s", libname, self.lib_subdir) break if not self.lib_subdir: raise EasyBuildError("Failed to determine lib subdirectory in %s", self.installdir) # Install a version with the MPI suffix if self.toolchain.options.get('usempi', None): if LooseVersion(self.version) < LooseVersion('4.6'): cmd = "make distclean" (out, _) = run_cmd(cmd, log_all=True, simple=False) self.cfg.update( 'configopts', "--enable-mpi --program-suffix={0}".format( self.cfg['mpisuffix'])) ConfigureMake.configure_step(self) super(EB_GROMACS, self).build_step() super(EB_GROMACS, self).install_step() else: self.cfg['configopts'] = re.sub(r'-DGMX_MPI=OFF', r'', self.cfg['configopts']) if self.cfg['mpi_numprocs'] == 0: self.log.info( "No number of test MPI tasks specified -- using default: %s" % self.cfg['parallel']) self.cfg['mpi_numprocs'] = self.cfg['parallel'] elif self.cfg['mpi_numprocs'] > self.cfg['parallel']: self.log.warning( "Number of test MPI tasks (%s) is greater than value for 'parallel': %s", self.cfg['mpi_numprocs'], self.cfg['parallel']) self.cfg.update('configopts', "-DGMX_MPI=ON -DGMX_THREAD_MPI=OFF") mpiexec = which(self.cfg['mpiexec']) if mpiexec: self.cfg.update('configopts', "-DMPIEXEC=%s" % mpiexec) self.cfg.update( 'configopts', "-DMPIEXEC_NUMPROC_FLAG=%s" % self.cfg['mpiexec_numproc_flag']) self.cfg.update('configopts', "-DNUMPROC=%s" % self.cfg['mpi_numprocs']) elif self.cfg['runtest']: raise EasyBuildError("'%s' not found in $PATH", self.cfg['mpiexec']) self.log.info( "Using %s as MPI executable when testing, with numprocs flag '%s' and %s tasks", self.cfg['mpiexec'], self.cfg['mpiexec_numproc_flag'], self.cfg['mpi_numprocs']) # clean up obj dir before reconfiguring shutil.rmtree(os.path.join(self.builddir, 'easybuild_obj')) # rebuild/test/install with MPI options super(EB_GROMACS, self).configure_step() super(EB_GROMACS, self).build_step() super(EB_GROMACS, self).test_step() super(EB_GROMACS, self).install_step() self.log.info( "A full regression test suite is available from the GROMACS web site" )
def configure_step(self): """Custom configure step for jaxlib.""" super(EB_jaxlib, self).configure_step() binutils_root = get_software_root('binutils') if not binutils_root: raise EasyBuildError( "Failed to determine installation prefix for binutils") config_env_vars = { # This is the binutils bin folder: https://github.com/tensorflow/tensorflow/issues/39263 'GCC_HOST_COMPILER_PREFIX': os.path.join(binutils_root, 'bin'), } # Collect options for the build script # Used only by the build script options = [ '--target_cpu_features=default', # Using copt for optimizations ] # Passed directly to bazel bazel_startup_options = [ '--output_user_root=%s' % tempfile.mkdtemp(suffix='-bazel', dir=self.builddir), ] # Passed to the build command of bazel bazel_options = [ '--jobs=%s' % self.cfg['parallel'], '--subcommands', '--action_env=PYTHONPATH', '--action_env=EBPYTHONPREFIXES', ] if self.toolchain.options.get('debug', None): bazel_options.extend(['--strip=never', '--copt="-Og"']) # Add optimization flags set by EasyBuild each as a separate option bazel_options.extend( ['--copt=%s' % i for i in os.environ['CXXFLAGS'].split(' ')]) cuda_root = get_software_root('CUDA') if cuda_root: cudnn_root = get_software_root('cuDNN') if not cudnn_root: raise EasyBuildError( 'For CUDA-enabled builds cuDNN is also required') cuda_version = '.'.join( get_software_version('CUDA').split('.')[:2]) # maj.minor cudnn_version = '.'.join( get_software_version('cuDNN').split('.') [:3]) # maj.minor.patch options.extend([ '--enable_cuda', '--cuda_path=' + cuda_root, '--cuda_compute_capabilities=' + self.cfg.get_cuda_cc_template_value( 'cuda_compute_capabilities'), '--cuda_version=' + cuda_version, '--cudnn_path=' + cudnn_root, '--cudnn_version=' + cudnn_version, ]) nccl_root = get_software_root('NCCL') if nccl_root: options.append('--enable_nccl') else: options.append('--noenable_nccl') config_env_vars['GCC_HOST_COMPILER_PATH'] = which(os.getenv('CC')) else: options.append('--noenable_cuda') # Prepend to buildopts so users can overwrite this self.cfg['buildopts'] = ' '.join(options + [ '--bazel_startup_options="%s"' % i for i in bazel_startup_options ] + ['--bazel_options="%s"' % i for i in bazel_options] + [self.cfg['buildopts']]) for key, val in sorted(config_env_vars.items()): env.setvar(key, val) # Print output of build at the end apply_regex_substitutions( 'build/build.py', [(r' shell\(command\)', ' print(shell(command))')])