def mime_type(file): """Returns the mime type and subtype of a file. Args: file: file to be analyzed Returns: Tuple containing the MIME type and subtype """ file_cmd = Executable('file') output = file_cmd('-b', '-h', '--mime-type', file, output=str, error=str) tty.debug('[MIME_TYPE] {0} -> {1}'.format(file, output.strip())) return tuple(output.strip().split('/'))
def macos_sdk_version(): """Return the version of the active macOS SDK. The SDK version usually corresponds to the installed Xcode version and can affect how some packages (especially those that use the GUI) can fail. This information should somehow be embedded into the future "compilers are dependencies" feature. The macOS deployment target cannot be greater than the SDK version, but usually it can be at least a few versions less. """ xcrun = Executable('xcrun') return Version(xcrun('--show-sdk-version', output=str).rstrip())
def compile_c_and_execute(source_file, include_flags, link_flags): """Compile C @p source_file with @p include_flags and @p link_flags, run and return the output. """ cc = which('cc') flags = include_flags flags.extend([source_file]) cc('-c', *flags) name = os.path.splitext(os.path.basename(source_file))[0] cc('-o', "check", "%s.o" % name, *link_flags) check = Executable('./check') return check(output=str)
def build_environment(): cc = Executable(os.path.join(build_env_path, "cc")) cxx = Executable(os.path.join(build_env_path, "c++")) fc = Executable(os.path.join(build_env_path, "fc")) realcc = "/bin/mycc" prefix = "/spack-test-prefix" os.environ['SPACK_CC'] = realcc os.environ['SPACK_CXX'] = realcc os.environ['SPACK_FC'] = realcc os.environ['SPACK_PREFIX'] = prefix os.environ['SPACK_ENV_PATH'] = "test" os.environ['SPACK_DEBUG_LOG_DIR'] = "." os.environ['SPACK_DEBUG_LOG_ID'] = "foo-hashabc" os.environ['SPACK_COMPILER_SPEC'] = "[email protected]" os.environ['SPACK_SHORT_SPEC'] = ( "[email protected] arch=linux-rhel6-x86_64 /hashabc") os.environ['SPACK_CC_RPATH_ARG'] = "-Wl,-rpath," os.environ['SPACK_CXX_RPATH_ARG'] = "-Wl,-rpath," os.environ['SPACK_F77_RPATH_ARG'] = "-Wl,-rpath," os.environ['SPACK_FC_RPATH_ARG'] = "-Wl,-rpath," os.environ['SPACK_SYSTEM_DIRS'] = '/usr/include /usr/lib' if 'SPACK_DEPENDENCIES' in os.environ: del os.environ['SPACK_DEPENDENCIES'] yield {'cc': cc, 'cxx': cxx, 'fc': fc} for name in ('SPACK_CC', 'SPACK_CXX', 'SPACK_FC', 'SPACK_PREFIX', 'SPACK_ENV_PATH', 'SPACK_DEBUG_LOG_DIR', 'SPACK_COMPILER_SPEC', 'SPACK_SHORT_SPEC', 'SPACK_CC_RPATH_ARG', 'SPACK_CXX_RPATH_ARG', 'SPACK_F77_RPATH_ARG', 'SPACK_FC_RPATH_ARG', 'SPACK_SYSTEM_DIRS'): del os.environ[name]
def install(self, spec, prefix): bash = Executable('bash') # Installer writes files in ~/intel set HOME so it goes to prefix bash.add_default_env('HOME', prefix) version = spec.versions.lowest() release = self._release(version) bash('./%s' % self._oneapi_file(version, release), '-s', '-a', '-s', '--action', 'install', '--eula', 'accept', '--components', self._components, '--install-dir', prefix)
def default_version(cls, comp): if comp not in _version_cache: compiler = Executable(comp) output = compiler('--version', output=str, error=str) ver = 'unknown' match = re.search(r'Arm C/C++/Fortran Compiler version ([^ )]+)', output) if match: ver = match.group(1) _version_cache[comp] = ver return _version_cache[comp]
def get_existing_elf_rpaths(path_name): """ Return the RPATHS returned by patchelf --print-rpath path_name as a list of strings. """ if platform.system() == 'Linux': command = Executable(get_patchelf()) output = command('--print-rpath', '%s' % path_name, output=str, err=str) return output.rstrip('\n').split(':') else: tty.die('relocation not supported for this platform') return
def modify_elf_object(path_name, orig_rpath, new_rpath): """ Replace orig_rpath with new_rpath in RPATH of elf object path_name """ if platform.system() == 'Linux': new_joined = ':'.join(new_rpath) patchelf = Executable(get_patchelf()) patchelf('--force-rpath', '--set-rpath', '%s' % new_joined, '%s' % path_name, output=str, cmd=str) else: tty.die('relocation not supported for this platform')
def _gcc_get_libstdcxx_version(self, version): """Returns gcc ABI compatibility info by getting the library version of a compiler's libstdc++.so or libgcc_s.so""" spec = CompilerSpec("gcc", version) compilers = spack.compilers.compilers_for_spec(spec) if not compilers: return None compiler = compilers[0] rungcc = None libname = None output = None if compiler.cxx: rungcc = Executable(compiler.cxx) libname = "libstdc++.so" elif compiler.cc: rungcc = Executable(compiler.cc) libname = "libgcc_s.so" else: return None try: output = rungcc("--print-file-name=%s" % libname, return_output=True) except ProcessError, e: return None
def patch_makefile(self, action, targets): """Patch predefined flags in Makefile. MPAS Makefile uses strings rather Makefile variables for its compiler flags. This patch substitutes the strings with flags set in `target:`.""" # Target `all:` does not contain any strings with compiler flags if action == "all": return sed = Executable('sed') for target in targets: key = target.split('=')[0] sed( "-i", "-e", "/^" + action + ":.*$/,/^$/s?" + key + ".*\\\" \\\\?" + target + "\\\" \\\\?1", "Makefile")
def _gcc_get_libstdcxx_version(self, version): """Returns gcc ABI compatibility info by getting the library version of a compiler's libstdc++ or libgcc_s""" spec = CompilerSpec("gcc", version) compilers = spack.compilers.compilers_for_spec(spec) if not compilers: return None compiler = compilers[0] rungcc = None libname = None output = None if compiler.cxx: rungcc = Executable(compiler.cxx) libname = "libstdc++." + dso_suffix elif compiler.cc: rungcc = Executable(compiler.cc) libname = "libgcc_s." + dso_suffix else: return None try: # Some gcc's are actually clang and don't respond properly to # --print-file-name (they just print the filename, not the # full path). Ignore these and expect them to be handled as clang. if Clang.default_version(rungcc.exe[0]) != 'unknown': return None output = rungcc("--print-file-name=%s" % libname, return_output=True) except ProcessError: return None if not output: return None libpath = os.readlink(output.strip()) if not libpath: return None return os.path.basename(libpath)
def set_configure_or_die(self): """Checks the presence of a ``configure`` file after the autoreconf phase. If it is found sets a module attribute appropriately, otherwise raises an error. :raises RuntimeError: if a configure script is not found in :py:meth:`~AutotoolsPackage.configure_directory` """ # Check if a configure script is there. If not raise a RuntimeError. if not os.path.exists(self.configure_abs_path): msg = 'configure script not found in {0}' raise RuntimeError(msg.format(self.configure_directory)) # Monkey-patch the configure script in the corresponding module inspect.getmodule(self).configure = Executable(self.configure_abs_path)
def get_existing_elf_rpaths(path_name): """ Return the RPATHS returned by patchelf --print-rpath path_name as a list of strings. """ # if we're relocating patchelf itself, use it if path_name[-13:] == "/bin/patchelf": patchelf = Executable(path_name) else: patchelf = Executable(get_patchelf()) try: output = patchelf('--print-rpath', '%s' % path_name, output=str, error=str) return output.rstrip('\n').split(':') except ProcessError as e: tty.debug('patchelf --print-rpath produced an error on %s' % path_name, e) return [] return
def modify_elf_object(path_name, new_rpaths): """ Replace orig_rpath with new_rpath in RPATH of elf object path_name """ if platform.system() == 'Linux': new_joined = ':'.join(new_rpaths) patchelf = Executable(get_patchelf()) try: patchelf('--force-rpath', '--set-rpath', '%s' % new_joined, '%s' % path_name, output=str, error=str) except ProcessError as e: tty.die('patchelf --set-rpath %s failed' % path_name, e) pass else: tty.die('relocation not supported for this platform')
def install(self, spec, prefix): """Install everything from build directory.""" raco = Executable("raco") with working_dir(self.build_directory): allow_parallel = self.parallel and (not env_flag(SPACK_NO_PARALLEL_MAKE)) args = ['pkg', 'install', '-t', 'dir', '-n', self.name, '--deps', 'fail', '--ignore-implies', '--copy', '-i', '-j', str(determine_number_of_jobs(allow_parallel)), '--', os.getcwd()] try: raco(*args) except ProcessError: args.insert(-2, "--skip-installed") raco(*args) tty.warn(("Racket package {0} was already installed, uninstalling via " "Spack may make someone unhappy!").format(self.name))
def mime_type(file): """Returns the mime type and subtype of a file. Args: file: file to be analyzed Returns: Tuple containing the MIME type and subtype """ file_cmd = Executable('file') output = file_cmd('-b', '-h', '--mime-type', file, output=str, error=str) tty.debug('[MIME_TYPE] {0} -> {1}'.format(file, output.strip())) if '/' not in output: output += '/' split_by_slash = output.strip().split('/') return (split_by_slash[0], "/".join(split_by_slash[1:]))
def modify_elf_object(path_name, new_rpaths): """ Replace orig_rpath with new_rpath in RPATH of elf object path_name """ new_joined = ':'.join(new_rpaths) patchelf = Executable(get_patchelf()) try: patchelf('--force-rpath', '--set-rpath', '%s' % new_joined, '%s' % path_name, output=str, error=str) except ProcessError as e: tty.die('patchelf --set-rpath %s failed' % path_name, e) pass
def set_module_variables_for_package(pkg): """Populate the module scope of install() with some useful functions. This makes things easier for package writers. """ m = pkg.module m.make = MakeExecutable('make', pkg.parallel) m.gmake = MakeExecutable('gmake', pkg.parallel) # easy shortcut to os.environ m.env = os.environ # number of jobs spack prefers to build with. m.make_jobs = multiprocessing.cpu_count() # Find the configure script in the archive path # Don't use which for this; we want to find it in the current dir. m.configure = Executable('./configure') # TODO: shouldn't really use "which" here. Consider adding notion # TODO: of build dependencies, as opposed to link dependencies. # TODO: Currently, everything is a link dependency, but tools like # TODO: this shouldn't be. m.cmake = which("cmake") # standard CMake arguments m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % pkg.prefix, '-DCMAKE_BUILD_TYPE=RelWithDebInfo'] if platform.mac_ver()[0]: m.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST') # Emulate some shell commands for convenience m.pwd = os.getcwd m.cd = os.chdir m.mkdir = os.mkdir m.makedirs = os.makedirs m.remove = os.remove m.removedirs = os.removedirs m.mkdirp = mkdirp m.install = install m.rmtree = shutil.rmtree m.move = shutil.move # Useful directories within the prefix are encapsulated in # a Prefix object. m.prefix = pkg.prefix
def default_version(cls, comp): """The ``--version`` option works for clang compilers. On most platforms, output looks like this:: clang version 3.1 (trunk 149096) Target: x86_64-unknown-linux-gnu Thread model: posix On macOS, it looks like this:: Apple LLVM version 7.0.2 (clang-700.1.81) Target: x86_64-apple-darwin15.2.0 Thread model: posix """ compiler = Executable(comp) output = compiler('--version', output=str, error=str) return cls.extract_version_from_output(output)
def get_existing_elf_rpaths(path_name): """ Return the RPATHS returned by patchelf --print-rpath path_name as a list of strings. """ patchelf = Executable(get_patchelf()) try: output = patchelf('--print-rpath', '%s' % path_name, output=str, error=str) return output.rstrip('\n').split(':') except ProcessError as e: tty.debug('patchelf --print-rpath produced an error on %s' % path_name, e) return [] return
def tcl2py(module, mode): tcl2py_exe = os.path.join(pymod.paths.bin_path, "tcl2py.tcl") tcl2py = Executable(tcl2py_exe) env = pymod.environ.filtered(include_os=True) mode = pymod.modes.as_string(mode) mode = {"show": "display"}.get(mode, mode) args = [] # loaded modules loaded_modules = pymod.mc.get_loaded_modules() lm_names = list( set([x for m in loaded_modules for x in [m.name, m.fullname]])) args.extend(("-l", ":".join(lm_names))) args.extend(("-f", module.fullname)) args.extend(("-m", mode)) args.extend(("-u", module.name)) args.extend(("-s", "bash")) ldlib = env.get(pymod.names.platform_ld_library_path) if ldlib: args.extend(("-L", ldlib)) ld_preload = env.get(pymod.names.ld_preload) if ld_preload: args.extend(("-P", ld_preload)) args.append(module.filename) kwargs = {"env": env, "output": str} output = tcl2py(*args, **kwargs) # name = module.name # family = None # if name.endswith('python'): # family = 'python' # elif name.startswith(('gcc', 'intel', 'pgi')): # family = 'compiler' # elif name.startswith(('openmpi', 'mpich', )): # family = 'mpi' # else: # family = None # # if family is not None: # output = 'family("{0}")\n'.format(family) + output return output
def get_existing_elf_rpaths(path_name): """ Return the RPATHS returned by patchelf --print-rpath path_name as a list of strings. """ if platform.system() == 'Linux': patchelf = Executable(get_patchelf()) try: output = patchelf('--print-rpath', '%s' % path_name, output=str, error=str) return output.rstrip('\n').split(':') except ProcessError as e: tty.debug('patchelf --print-rpath produced an error on %s' % path_name, e) return [] else: tty.die('relocation not supported for this platform') return
def mime_type(file): """Returns the mime type and subtype of a file. Args: file: file to be analyzed Returns: Tuple containing the MIME type and subtype """ file_cmd = Executable('file') output = file_cmd('-b', '-h', '--mime-type', file, output=str, error=str) tty.debug('[MIME_TYPE] {0} -> {1}'.format(file, output.strip())) # In corner cases the output does not contain a subtype prefixed with a / # In those cases add the / so the tuple can be formed. if '/' not in output: output += '/' split_by_slash = output.strip().split('/') return (split_by_slash[0], "/".join(split_by_slash[1:]))
def macos_version(): """Get the current macOS version as a version object. This has three mechanisms for determining the macOS version, which is used for spack identification (the ``os`` in the spec's ``arch``) and indirectly for setting the value of ``MACOSX_DEPLOYMENT_TARGET``, which affects the ``minos`` value of the ``LC_BUILD_VERSION`` macho header. Mixing ``minos`` values can lead to lots of linker warnings, and using a consistent version (pinned to the major OS version) allows distribution across clients that might be slightly behind. The version determination is made with three mechanisms in decreasing priority: 1. The ``MACOSX_DEPLOYMENT_TARGET`` variable overrides the actual operating system version, just like the value can be used to build for older macOS targets on newer systems. Spack currently will truncate this value when building packages, but at least the major version will be the same. 2. The system ``sw_vers`` command reports the actual operating system version. 3. The Python ``platform.mac_ver`` function is a fallback if the operating system identification fails, because some Python versions and/or installations report the OS on which Python was *built* rather than the one on which it is running. """ env_ver = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None) if env_ver: return Version(env_ver) try: output = Executable('sw_vers')(output=str, fail_on_error=False) except Exception: # FileNotFoundError, or spack.util.executable.ProcessError pass else: match = re.search(r'ProductVersion:\s*([0-9.]+)', output) if match: return Version(match.group(1)) # Fall back to python-reported version, which can be inaccurate around # macOS 11 (e.g. showing 10.16 for macOS 12) return Version(py_platform.mac_ver()[0])
def install(self, spec, prefix, installer_path=None): """Shared install method for all oneapi packages.""" # intel-oneapi-compilers overrides the installer_path when # installing fortran, which comes from a spack resource if installer_path is None: installer_path = basename(self.url_for_version(spec.version)) if platform.system() == 'Linux': # Intel installer assumes and enforces that all components # are installed into a single prefix. Spack wants to # install each component in a separate prefix. The # installer mechanism is implemented by saving install # information in a directory called installercache for # future runs. The location of the installercache depends # on the userid. For root it is always in /var/intel. For # non-root it is in $HOME/intel. # # The method for preventing this install from interfering # with other install depends on the userid. For root, we # delete the installercache before and after install. For # non root we redefine the HOME environment variable. if getpass.getuser() == 'root': shutil.rmtree('/var/intel/installercache', ignore_errors=True) bash = Executable('bash') # Installer writes files in ~/intel set HOME so it goes to prefix bash.add_default_env('HOME', prefix) # Installer checks $XDG_RUNTIME_DIR/.bootstrapper_lock_file as well bash.add_default_env('XDG_RUNTIME_DIR', join_path(self.stage.path, 'runtime')) bash(installer_path, '-s', '-a', '-s', '--action', 'install', '--eula', 'accept', '--install-dir', prefix) if getpass.getuser() == 'root': shutil.rmtree('/var/intel/installercache', ignore_errors=True) # Some installers have a bug and do not return an error code when failing if not isdir(join_path(prefix, self.component_dir)): raise RuntimeError('install failed')
def install(self, spec, prefix): # reference to other package managers # https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/a/atlas.py # https://github.com/macports/macports-ports/blob/master/math/atlas/Portfile # https://github.com/Homebrew/homebrew-science/pull/3571 options = [] if '+shared' in spec: options.extend([ '--shared' ]) # TODO: for non GNU add '-Fa', 'alg', '-fPIC' ? # configure for 64-bit build options.extend([ '-b', '64' ]) # set compilers: options.extend([ '-C', 'ic', spack_cc, '-C', 'if', spack_f77 ]) # Lapack resource to provide full lapack build. Note that # ATLAS only provides a few LAPACK routines natively. options.append('--with-netlib-lapack-tarfile=%s' % self.stage[1].archive_file) with working_dir('spack-build', create=True): configure = Executable('../configure') configure('--prefix=%s' % prefix, *options) make() make('check') make('ptcheck') make('time') if '+shared' in spec: with working_dir('lib'): make('shared_all') make("install") if self.run_tests: self.install_test()
def install(self, spec, prefix): options = [] if '+shared' in spec: options.append('--shared') # Lapack resource lapack_stage = self.stage[1] lapack_tarfile = os.path.basename(lapack_stage.fetcher.url) lapack_tarfile_path = join_path(lapack_stage.path, lapack_tarfile) options.append('--with-netlib-lapack-tarfile=%s' % lapack_tarfile_path) with working_dir('spack-build', create=True): configure = Executable('../configure') configure('--prefix=%s' % prefix, *options) make() make('check') make('ptcheck') make('time') make("install")
def _default_target_from_env(self): '''Set and return the default CrayPE target loaded in a clean login session. A bash subshell is launched with a wiped environment and the list of loaded modules is parsed for the first acceptable CrayPE target. ''' # env -i /bin/bash -lc echo $CRAY_CPU_TARGET 2> /dev/null if getattr(self, 'default', None) is None: bash = Executable('/bin/bash') output = bash('-lc', 'echo $CRAY_CPU_TARGET', env={'TERM': os.environ.get('TERM', '')}, output=str, error=os.devnull) output = ''.join(output.split()) # remove all whitespace if output: self.default = output tty.debug("Found default module:%s" % self.default) return self.default
def install(self, spec, prefix, installer_path=None): """Shared install method for all oneapi packages.""" # intel-oneapi-compilers overrides the installer_path when # installing fortran, which comes from a spack resource if installer_path is None: installer_path = basename(self.url_for_version(spec.version)) if platform == 'linux': bash = Executable('bash') # Installer writes files in ~/intel set HOME so it goes to prefix bash.add_default_env('HOME', prefix) bash(installer_path, '-s', '-a', '-s', '--action', 'install', '--eula', 'accept', '--install-dir', prefix) # Some installers have a bug and do not return an error code when failing if not isdir(join_path(prefix, self.component_dir)): raise RuntimeError('install failed')
def macos_cltools_version(): """Find the last installed version of the CommandLineTools. The CLT version might only affect the build if it's selected as the macOS SDK path. """ pkgutil = Executable('pkgutil') output = pkgutil('--pkg-info=com.apple.pkg.cltools_executables', output=str, fail_on_error=False) match = re.search(r'version:\s*([0-9.]+)', output) if match: return Version(match.group(1)) # No CLTools installed by package manager: try Xcode output = pkgutil('--pkg-info=com.apple.pkg.Xcode', output=str, fail_on_error=False) match = re.search(r'version:\s*([0-9.]+)', output) if match: return Version(match.group(1)) return None