def mime_type(file):
"""Returns the mime type and subtype of a file.
Args:
file: file to be analyzed
Returns:
Tuple containing the MIME type and subtype
"""
file_cmd = Executable('file')
output = file_cmd('-b', '-h', '--mime-type', file, output=str, error=str)
tty.debug('[MIME_TYPE] {0} -> {1}'.format(file, output.strip()))
return tuple(output.strip().split('/'))
def macos_sdk_version():
"""Return the version of the active macOS SDK.
The SDK version usually corresponds to the installed Xcode version and can
affect how some packages (especially those that use the GUI) can fail. This
information should somehow be embedded into the future "compilers are
dependencies" feature.
The macOS deployment target cannot be greater than the SDK version, but
usually it can be at least a few versions less.
"""
xcrun = Executable('xcrun')
return Version(xcrun('--show-sdk-version', output=str).rstrip())
def compile_c_and_execute(source_file, include_flags, link_flags):
"""Compile C @p source_file with @p include_flags and @p link_flags,
run and return the output.
"""
cc = which('cc')
flags = include_flags
flags.extend([source_file])
cc('-c', *flags)
name = os.path.splitext(os.path.basename(source_file))[0]
cc('-o', "check", "%s.o" % name, *link_flags)
check = Executable('./check')
return check(output=str)
def build_environment():
cc = Executable(os.path.join(build_env_path, "cc"))
cxx = Executable(os.path.join(build_env_path, "c++"))
fc = Executable(os.path.join(build_env_path, "fc"))
realcc = "/bin/mycc"
prefix = "/spack-test-prefix"
os.environ['SPACK_CC'] = realcc
os.environ['SPACK_CXX'] = realcc
os.environ['SPACK_FC'] = realcc
os.environ['SPACK_PREFIX'] = prefix
os.environ['SPACK_ENV_PATH'] = "test"
os.environ['SPACK_DEBUG_LOG_DIR'] = "."
os.environ['SPACK_DEBUG_LOG_ID'] = "foo-hashabc"
os.environ['SPACK_COMPILER_SPEC'] = "[email protected]"
os.environ['SPACK_SHORT_SPEC'] = (
"[email protected] arch=linux-rhel6-x86_64 /hashabc")
os.environ['SPACK_CC_RPATH_ARG'] = "-Wl,-rpath,"
os.environ['SPACK_CXX_RPATH_ARG'] = "-Wl,-rpath,"
os.environ['SPACK_F77_RPATH_ARG'] = "-Wl,-rpath,"
os.environ['SPACK_FC_RPATH_ARG'] = "-Wl,-rpath,"
os.environ['SPACK_SYSTEM_DIRS'] = '/usr/include /usr/lib'
if 'SPACK_DEPENDENCIES' in os.environ:
del os.environ['SPACK_DEPENDENCIES']
yield {'cc': cc, 'cxx': cxx, 'fc': fc}
for name in ('SPACK_CC', 'SPACK_CXX', 'SPACK_FC', 'SPACK_PREFIX',
'SPACK_ENV_PATH', 'SPACK_DEBUG_LOG_DIR',
'SPACK_COMPILER_SPEC', 'SPACK_SHORT_SPEC',
'SPACK_CC_RPATH_ARG', 'SPACK_CXX_RPATH_ARG',
'SPACK_F77_RPATH_ARG', 'SPACK_FC_RPATH_ARG',
'SPACK_SYSTEM_DIRS'):
del os.environ[name]
def install(self, spec, prefix):
bash = Executable('bash')
# Installer writes files in ~/intel set HOME so it goes to prefix
bash.add_default_env('HOME', prefix)
version = spec.versions.lowest()
release = self._release(version)
bash('./%s' % self._oneapi_file(version, release),
'-s', '-a', '-s', '--action', 'install',
'--eula', 'accept',
'--components',
self._components,
'--install-dir', prefix)
def default_version(cls, comp):
if comp not in _version_cache:
compiler = Executable(comp)
output = compiler('--version', output=str, error=str)
ver = 'unknown'
match = re.search(r'Arm C/C++/Fortran Compiler version ([^ )]+)',
output)
if match:
ver = match.group(1)
_version_cache[comp] = ver
return _version_cache[comp]
def get_existing_elf_rpaths(path_name):
"""
Return the RPATHS returned by patchelf --print-rpath path_name
as a list of strings.
"""
if platform.system() == 'Linux':
command = Executable(get_patchelf())
output = command('--print-rpath',
'%s' % path_name,
output=str,
err=str)
return output.rstrip('\n').split(':')
else:
tty.die('relocation not supported for this platform')
return
def modify_elf_object(path_name, orig_rpath, new_rpath):
"""
Replace orig_rpath with new_rpath in RPATH of elf object path_name
"""
if platform.system() == 'Linux':
new_joined = ':'.join(new_rpath)
patchelf = Executable(get_patchelf())
patchelf('--force-rpath',
'--set-rpath',
'%s' % new_joined,
'%s' % path_name,
output=str,
cmd=str)
else:
tty.die('relocation not supported for this platform')
def _gcc_get_libstdcxx_version(self, version):
"""Returns gcc ABI compatibility info by getting the library version of
a compiler's libstdc++.so or libgcc_s.so"""
spec = CompilerSpec("gcc", version)
compilers = spack.compilers.compilers_for_spec(spec)
if not compilers:
return None
compiler = compilers[0]
rungcc = None
libname = None
output = None
if compiler.cxx:
rungcc = Executable(compiler.cxx)
libname = "libstdc++.so"
elif compiler.cc:
rungcc = Executable(compiler.cc)
libname = "libgcc_s.so"
else:
return None
try:
output = rungcc("--print-file-name=%s" % libname,
return_output=True)
except ProcessError, e:
return None
def patch_makefile(self, action, targets):
"""Patch predefined flags in Makefile.
MPAS Makefile uses strings rather Makefile variables for its compiler flags.
This patch substitutes the strings with flags set in `target:`."""
# Target `all:` does not contain any strings with compiler flags
if action == "all":
return
sed = Executable('sed')
for target in targets:
key = target.split('=')[0]
sed(
"-i", "-e", "/^" + action + ":.*$/,/^$/s?" + key +
".*\\\" \\\\?" + target + "\\\" \\\\?1", "Makefile")
def _gcc_get_libstdcxx_version(self, version):
"""Returns gcc ABI compatibility info by getting the library version of
a compiler's libstdc++ or libgcc_s"""
spec = CompilerSpec("gcc", version)
compilers = spack.compilers.compilers_for_spec(spec)
if not compilers:
return None
compiler = compilers[0]
rungcc = None
libname = None
output = None
if compiler.cxx:
rungcc = Executable(compiler.cxx)
libname = "libstdc++." + dso_suffix
elif compiler.cc:
rungcc = Executable(compiler.cc)
libname = "libgcc_s." + dso_suffix
else:
return None
try:
# Some gcc's are actually clang and don't respond properly to
# --print-file-name (they just print the filename, not the
# full path). Ignore these and expect them to be handled as clang.
if Clang.default_version(rungcc.exe[0]) != 'unknown':
return None
output = rungcc("--print-file-name=%s" % libname,
return_output=True)
except ProcessError:
return None
if not output:
return None
libpath = os.readlink(output.strip())
if not libpath:
return None
return os.path.basename(libpath)
def set_configure_or_die(self):
"""Checks the presence of a ``configure`` file after the
autoreconf phase. If it is found sets a module attribute
appropriately, otherwise raises an error.
:raises RuntimeError: if a configure script is not found in
:py:meth:`~AutotoolsPackage.configure_directory`
"""
# Check if a configure script is there. If not raise a RuntimeError.
if not os.path.exists(self.configure_abs_path):
msg = 'configure script not found in {0}'
raise RuntimeError(msg.format(self.configure_directory))
# Monkey-patch the configure script in the corresponding module
inspect.getmodule(self).configure = Executable(self.configure_abs_path)
def get_existing_elf_rpaths(path_name):
"""
Return the RPATHS returned by patchelf --print-rpath path_name
as a list of strings.
"""
# if we're relocating patchelf itself, use it
if path_name[-13:] == "/bin/patchelf":
patchelf = Executable(path_name)
else:
patchelf = Executable(get_patchelf())
try:
output = patchelf('--print-rpath',
'%s' % path_name,
output=str,
error=str)
return output.rstrip('\n').split(':')
except ProcessError as e:
tty.debug('patchelf --print-rpath produced an error on %s' % path_name,
e)
return []
return
def modify_elf_object(path_name, new_rpaths):
"""
Replace orig_rpath with new_rpath in RPATH of elf object path_name
"""
if platform.system() == 'Linux':
new_joined = ':'.join(new_rpaths)
patchelf = Executable(get_patchelf())
try:
patchelf('--force-rpath', '--set-rpath', '%s' % new_joined,
'%s' % path_name, output=str, error=str)
except ProcessError as e:
tty.die('patchelf --set-rpath %s failed' %
path_name, e)
pass
else:
tty.die('relocation not supported for this platform')
def install(self, spec, prefix):
"""Install everything from build directory."""
raco = Executable("raco")
with working_dir(self.build_directory):
allow_parallel = self.parallel and (not env_flag(SPACK_NO_PARALLEL_MAKE))
args = ['pkg', 'install', '-t', 'dir', '-n', self.name, '--deps', 'fail',
'--ignore-implies', '--copy', '-i', '-j',
str(determine_number_of_jobs(allow_parallel)),
'--', os.getcwd()]
try:
raco(*args)
except ProcessError:
args.insert(-2, "--skip-installed")
raco(*args)
tty.warn(("Racket package {0} was already installed, uninstalling via "
"Spack may make someone unhappy!").format(self.name))
def mime_type(file):
"""Returns the mime type and subtype of a file.
Args:
file: file to be analyzed
Returns:
Tuple containing the MIME type and subtype
"""
file_cmd = Executable('file')
output = file_cmd('-b', '-h', '--mime-type', file, output=str, error=str)
tty.debug('[MIME_TYPE] {0} -> {1}'.format(file, output.strip()))
if '/' not in output:
output += '/'
split_by_slash = output.strip().split('/')
return (split_by_slash[0], "/".join(split_by_slash[1:]))
def modify_elf_object(path_name, new_rpaths):
"""
Replace orig_rpath with new_rpath in RPATH of elf object path_name
"""
new_joined = ':'.join(new_rpaths)
patchelf = Executable(get_patchelf())
try:
patchelf('--force-rpath',
'--set-rpath',
'%s' % new_joined,
'%s' % path_name,
output=str,
error=str)
except ProcessError as e:
tty.die('patchelf --set-rpath %s failed' % path_name, e)
pass
def set_module_variables_for_package(pkg):
"""Populate the module scope of install() with some useful functions.
This makes things easier for package writers.
"""
m = pkg.module
m.make = MakeExecutable('make', pkg.parallel)
m.gmake = MakeExecutable('gmake', pkg.parallel)
# easy shortcut to os.environ
m.env = os.environ
# number of jobs spack prefers to build with.
m.make_jobs = multiprocessing.cpu_count()
# Find the configure script in the archive path
# Don't use which for this; we want to find it in the current dir.
m.configure = Executable('./configure')
# TODO: shouldn't really use "which" here. Consider adding notion
# TODO: of build dependencies, as opposed to link dependencies.
# TODO: Currently, everything is a link dependency, but tools like
# TODO: this shouldn't be.
m.cmake = which("cmake")
# standard CMake arguments
m.std_cmake_args = ['-DCMAKE_INSTALL_PREFIX=%s' % pkg.prefix,
'-DCMAKE_BUILD_TYPE=RelWithDebInfo']
if platform.mac_ver()[0]:
m.std_cmake_args.append('-DCMAKE_FIND_FRAMEWORK=LAST')
# Emulate some shell commands for convenience
m.pwd = os.getcwd
m.cd = os.chdir
m.mkdir = os.mkdir
m.makedirs = os.makedirs
m.remove = os.remove
m.removedirs = os.removedirs
m.mkdirp = mkdirp
m.install = install
m.rmtree = shutil.rmtree
m.move = shutil.move
# Useful directories within the prefix are encapsulated in
# a Prefix object.
m.prefix = pkg.prefix
def default_version(cls, comp):
"""The ``--version`` option works for clang compilers.
On most platforms, output looks like this::
clang version 3.1 (trunk 149096)
Target: x86_64-unknown-linux-gnu
Thread model: posix
On macOS, it looks like this::
Apple LLVM version 7.0.2 (clang-700.1.81)
Target: x86_64-apple-darwin15.2.0
Thread model: posix
"""
compiler = Executable(comp)
output = compiler('--version', output=str, error=str)
return cls.extract_version_from_output(output)
def get_existing_elf_rpaths(path_name):
"""
Return the RPATHS returned by patchelf --print-rpath path_name
as a list of strings.
"""
patchelf = Executable(get_patchelf())
try:
output = patchelf('--print-rpath',
'%s' % path_name,
output=str,
error=str)
return output.rstrip('\n').split(':')
except ProcessError as e:
tty.debug('patchelf --print-rpath produced an error on %s' % path_name,
e)
return []
return
def tcl2py(module, mode):
tcl2py_exe = os.path.join(pymod.paths.bin_path, "tcl2py.tcl")
tcl2py = Executable(tcl2py_exe)
env = pymod.environ.filtered(include_os=True)
mode = pymod.modes.as_string(mode)
mode = {"show": "display"}.get(mode, mode)
args = []
# loaded modules
loaded_modules = pymod.mc.get_loaded_modules()
lm_names = list(
set([x for m in loaded_modules for x in [m.name, m.fullname]]))
args.extend(("-l", ":".join(lm_names)))
args.extend(("-f", module.fullname))
args.extend(("-m", mode))
args.extend(("-u", module.name))
args.extend(("-s", "bash"))
ldlib = env.get(pymod.names.platform_ld_library_path)
if ldlib:
args.extend(("-L", ldlib))
ld_preload = env.get(pymod.names.ld_preload)
if ld_preload:
args.extend(("-P", ld_preload))
args.append(module.filename)
kwargs = {"env": env, "output": str}
output = tcl2py(*args, **kwargs)
# name = module.name
# family = None
# if name.endswith('python'):
# family = 'python'
# elif name.startswith(('gcc', 'intel', 'pgi')):
# family = 'compiler'
# elif name.startswith(('openmpi', 'mpich', )):
# family = 'mpi'
# else:
# family = None
#
# if family is not None:
# output = 'family("{0}")\n'.format(family) + output
return output
def get_existing_elf_rpaths(path_name):
"""
Return the RPATHS returned by patchelf --print-rpath path_name
as a list of strings.
"""
if platform.system() == 'Linux':
patchelf = Executable(get_patchelf())
try:
output = patchelf('--print-rpath', '%s' %
path_name, output=str, error=str)
return output.rstrip('\n').split(':')
except ProcessError as e:
tty.debug('patchelf --print-rpath produced an error on %s' %
path_name, e)
return []
else:
tty.die('relocation not supported for this platform')
return
def mime_type(file):
"""Returns the mime type and subtype of a file.
Args:
file: file to be analyzed
Returns:
Tuple containing the MIME type and subtype
"""
file_cmd = Executable('file')
output = file_cmd('-b', '-h', '--mime-type', file, output=str, error=str)
tty.debug('[MIME_TYPE] {0} -> {1}'.format(file, output.strip()))
# In corner cases the output does not contain a subtype prefixed with a /
# In those cases add the / so the tuple can be formed.
if '/' not in output:
output += '/'
split_by_slash = output.strip().split('/')
return (split_by_slash[0], "/".join(split_by_slash[1:]))
def macos_version():
"""Get the current macOS version as a version object.
This has three mechanisms for determining the macOS version, which is used
for spack identification (the ``os`` in the spec's ``arch``) and indirectly
for setting the value of ``MACOSX_DEPLOYMENT_TARGET``, which affects the
``minos`` value of the ``LC_BUILD_VERSION`` macho header. Mixing ``minos``
values can lead to lots of linker warnings, and using a consistent version
(pinned to the major OS version) allows distribution across clients that
might be slightly behind.
The version determination is made with three mechanisms in decreasing
priority:
1. The ``MACOSX_DEPLOYMENT_TARGET`` variable overrides the actual operating
system version, just like the value can be used to build for older macOS
targets on newer systems. Spack currently will truncate this value when
building packages, but at least the major version will be the same.
2. The system ``sw_vers`` command reports the actual operating system
version.
3. The Python ``platform.mac_ver`` function is a fallback if the operating
system identification fails, because some Python versions and/or
installations report the OS
on which Python was *built* rather than the one on which it is running.
"""
env_ver = os.environ.get('MACOSX_DEPLOYMENT_TARGET', None)
if env_ver:
return Version(env_ver)
try:
output = Executable('sw_vers')(output=str, fail_on_error=False)
except Exception:
# FileNotFoundError, or spack.util.executable.ProcessError
pass
else:
match = re.search(r'ProductVersion:\s*([0-9.]+)', output)
if match:
return Version(match.group(1))
# Fall back to python-reported version, which can be inaccurate around
# macOS 11 (e.g. showing 10.16 for macOS 12)
return Version(py_platform.mac_ver()[0])
def install(self, spec, prefix, installer_path=None):
"""Shared install method for all oneapi packages."""
# intel-oneapi-compilers overrides the installer_path when
# installing fortran, which comes from a spack resource
if installer_path is None:
installer_path = basename(self.url_for_version(spec.version))
if platform.system() == 'Linux':
# Intel installer assumes and enforces that all components
# are installed into a single prefix. Spack wants to
# install each component in a separate prefix. The
# installer mechanism is implemented by saving install
# information in a directory called installercache for
# future runs. The location of the installercache depends
# on the userid. For root it is always in /var/intel. For
# non-root it is in $HOME/intel.
#
# The method for preventing this install from interfering
# with other install depends on the userid. For root, we
# delete the installercache before and after install. For
# non root we redefine the HOME environment variable.
if getpass.getuser() == 'root':
shutil.rmtree('/var/intel/installercache', ignore_errors=True)
bash = Executable('bash')
# Installer writes files in ~/intel set HOME so it goes to prefix
bash.add_default_env('HOME', prefix)
# Installer checks $XDG_RUNTIME_DIR/.bootstrapper_lock_file as well
bash.add_default_env('XDG_RUNTIME_DIR',
join_path(self.stage.path, 'runtime'))
bash(installer_path, '-s', '-a', '-s', '--action', 'install',
'--eula', 'accept', '--install-dir', prefix)
if getpass.getuser() == 'root':
shutil.rmtree('/var/intel/installercache', ignore_errors=True)
# Some installers have a bug and do not return an error code when failing
if not isdir(join_path(prefix, self.component_dir)):
raise RuntimeError('install failed')
def install(self, spec, prefix):
# reference to other package managers
# https://github.com/hpcugent/easybuild-easyblocks/blob/master/easybuild/easyblocks/a/atlas.py
# https://github.com/macports/macports-ports/blob/master/math/atlas/Portfile
# https://github.com/Homebrew/homebrew-science/pull/3571
options = []
if '+shared' in spec:
options.extend([
'--shared'
])
# TODO: for non GNU add '-Fa', 'alg', '-fPIC' ?
# configure for 64-bit build
options.extend([
'-b', '64'
])
# set compilers:
options.extend([
'-C', 'ic', spack_cc,
'-C', 'if', spack_f77
])
# Lapack resource to provide full lapack build. Note that
# ATLAS only provides a few LAPACK routines natively.
options.append('--with-netlib-lapack-tarfile=%s' %
self.stage[1].archive_file)
with working_dir('spack-build', create=True):
configure = Executable('../configure')
configure('--prefix=%s' % prefix, *options)
make()
make('check')
make('ptcheck')
make('time')
if '+shared' in spec:
with working_dir('lib'):
make('shared_all')
make("install")
if self.run_tests:
self.install_test()
def install(self, spec, prefix):
options = []
if '+shared' in spec:
options.append('--shared')
# Lapack resource
lapack_stage = self.stage[1]
lapack_tarfile = os.path.basename(lapack_stage.fetcher.url)
lapack_tarfile_path = join_path(lapack_stage.path, lapack_tarfile)
options.append('--with-netlib-lapack-tarfile=%s' % lapack_tarfile_path)
with working_dir('spack-build', create=True):
configure = Executable('../configure')
configure('--prefix=%s' % prefix, *options)
make()
make('check')
make('ptcheck')
make('time')
make("install")
def _default_target_from_env(self):
'''Set and return the default CrayPE target loaded in a clean login
session.
A bash subshell is launched with a wiped environment and the list of
loaded modules is parsed for the first acceptable CrayPE target.
'''
# env -i /bin/bash -lc echo $CRAY_CPU_TARGET 2> /dev/null
if getattr(self, 'default', None) is None:
bash = Executable('/bin/bash')
output = bash('-lc',
'echo $CRAY_CPU_TARGET',
env={'TERM': os.environ.get('TERM', '')},
output=str,
error=os.devnull)
output = ''.join(output.split()) # remove all whitespace
if output:
self.default = output
tty.debug("Found default module:%s" % self.default)
return self.default
def install(self, spec, prefix, installer_path=None):
"""Shared install method for all oneapi packages."""
# intel-oneapi-compilers overrides the installer_path when
# installing fortran, which comes from a spack resource
if installer_path is None:
installer_path = basename(self.url_for_version(spec.version))
if platform == 'linux':
bash = Executable('bash')
# Installer writes files in ~/intel set HOME so it goes to prefix
bash.add_default_env('HOME', prefix)
bash(installer_path, '-s', '-a', '-s', '--action', 'install',
'--eula', 'accept', '--install-dir', prefix)
# Some installers have a bug and do not return an error code when failing
if not isdir(join_path(prefix, self.component_dir)):
raise RuntimeError('install failed')
def macos_cltools_version():
"""Find the last installed version of the CommandLineTools.
The CLT version might only affect the build if it's selected as the macOS
SDK path.
"""
pkgutil = Executable('pkgutil')
output = pkgutil('--pkg-info=com.apple.pkg.cltools_executables',
output=str, fail_on_error=False)
match = re.search(r'version:\s*([0-9.]+)', output)
if match:
return Version(match.group(1))
# No CLTools installed by package manager: try Xcode
output = pkgutil('--pkg-info=com.apple.pkg.Xcode',
output=str, fail_on_error=False)
match = re.search(r'version:\s*([0-9.]+)', output)
if match:
return Version(match.group(1))
return None
