def test_relocate_links(tmpdir):
with tmpdir.as_cwd():
old_layout_root = os.path.join(
'%s' % tmpdir, 'home', 'spack', 'opt', 'spack')
old_install_prefix = os.path.join(
'%s' % old_layout_root, 'debian6', 'test')
old_binname = os.path.join(old_install_prefix, 'binfile')
placeholder = _placeholder(old_layout_root)
re.sub(old_layout_root, placeholder, old_binname)
filenames = ['link.ln', 'outsideprefix.ln']
new_layout_root = os.path.join(
'%s' % tmpdir, 'opt', 'rh', 'devtoolset')
new_install_prefix = os.path.join(
'%s' % new_layout_root, 'test', 'debian6')
new_linkname = os.path.join(new_install_prefix, 'link.ln')
new_linkname2 = os.path.join(new_install_prefix, 'outsideprefix.ln')
new_binname = os.path.join(new_install_prefix, 'binfile')
mkdirp(new_install_prefix)
with open(new_binname, 'w') as f:
f.write('\n')
os.utime(new_binname, None)
symlink(old_binname, new_linkname)
symlink('/usr/lib/libc.so', new_linkname2)
relocate_links(filenames, old_layout_root,
old_install_prefix, new_install_prefix)
assert os.readlink(new_linkname) == new_binname
assert os.readlink(new_linkname2) == '/usr/lib/libc.so'
def _relocate_spliced_links(links, orig_prefix, new_prefix):
"""Re-linking function which differs from `relocate.relocate_links` by
reading the old link rather than the new link, since the latter wasn't moved
in our case. This still needs to be called after the copy to destination
because it expects the new directory structure to be in place."""
for link in links:
link_target = os.readlink(os.path.join(orig_prefix, link))
link_target = re.sub('^' + orig_prefix, new_prefix, link_target)
new_link_path = os.path.join(new_prefix, link)
os.unlink(new_link_path)
symlink(link_target, new_link_path)
def test_check_prefix_manifest(tmpdir):
# Test the verification of an entire prefix and its contents
prefix_path = tmpdir.join('prefix')
prefix = str(prefix_path)
spec = spack.spec.Spec('libelf')
spec._mark_concrete()
spec.prefix = prefix
results = spack.verify.check_spec_manifest(spec)
assert results.has_errors()
assert prefix in results.errors
assert results.errors[prefix] == ['manifest missing']
metadata_dir = str(prefix_path.join('.spack'))
bin_dir = str(prefix_path.join('bin'))
other_dir = str(prefix_path.join('other'))
for d in (metadata_dir, bin_dir, other_dir):
fs.mkdirp(d)
file = os.path.join(other_dir, 'file')
with open(file, 'w') as f:
f.write("I'm a little file short and stout")
link = os.path.join(bin_dir, 'run')
symlink(file, link)
spack.verify.write_manifest(spec)
results = spack.verify.check_spec_manifest(spec)
assert not results.has_errors()
os.remove(link)
malware = os.path.join(metadata_dir, 'hiddenmalware')
with open(malware, 'w') as f:
f.write("Foul evil deeds")
results = spack.verify.check_spec_manifest(spec)
assert results.has_errors()
assert all(x in results.errors for x in (malware, link))
assert len(results.errors) == 2
assert results.errors[link] == ['deleted']
assert results.errors[malware] == ['added']
manifest_file = os.path.join(spec.prefix, spack.store.layout.metadata_dir,
spack.store.layout.manifest_file_name)
with open(manifest_file, 'w') as f:
f.write("{This) string is not proper json")
results = spack.verify.check_spec_manifest(spec)
assert results.has_errors()
assert results.errors[spec.prefix] == ['manifest corrupted']
def make_link_relative(new_links, orig_links):
"""Compute the relative target from the original link and
make the new link relative.
Args:
new_links (list): new links to be made relative
orig_links (list): original links
"""
for new_link, orig_link in zip(new_links, orig_links):
target = os.readlink(orig_link)
relative_target = os.path.relpath(target, os.path.dirname(orig_link))
os.unlink(new_link)
symlink(relative_target, new_link)
def symlink_windows(self):
if not is_windows:
return
win_install_path = os.path.join(self.prefix.bin, "MSWin32")
if self.is_64bit():
win_install_path += "-x64"
else:
win_install_path += "-x86"
if self.spec.satisfies("+threads"):
win_install_path += "-multi-thread"
else:
win_install_path += "-perlio"
for f in os.listdir(os.path.join(self.prefix.bin, win_install_path)):
lnk_path = os.path.join(self.prefix.bin, f)
src_path = os.path.join(win_install_path, f)
if not os.path.exists(lnk_path):
symlink(src_path, lnk_path)
def symlink(self, mirror_ref):
"""Symlink a human readible path in our mirror to the actual
storage location."""
cosmetic_path = os.path.join(self.root, mirror_ref.cosmetic_path)
storage_path = os.path.join(self.root, mirror_ref.storage_path)
relative_dst = os.path.relpath(
storage_path,
start=os.path.dirname(cosmetic_path))
if not os.path.exists(cosmetic_path):
if os.path.lexists(cosmetic_path):
# In this case the link itself exists but it is broken: remove
# it and recreate it (in order to fix any symlinks broken prior
# to https://github.com/spack/spack/pull/13908)
os.unlink(cosmetic_path)
mkdirp(os.path.dirname(cosmetic_path))
symlink(relative_dst, cosmetic_path)
def symlink_license(pkg):
"""Create local symlinks that point to the global license file."""
target = pkg.global_license_file
for filename in pkg.license_files:
link_name = os.path.join(pkg.prefix, filename)
link_name = os.path.abspath(link_name)
license_dir = os.path.dirname(link_name)
if not os.path.exists(license_dir):
mkdirp(license_dir)
# If example file already exists, overwrite it with a symlink
if os.path.lexists(link_name):
os.remove(link_name)
if os.path.exists(target):
symlink(target, link_name)
tty.msg("Added local symlink %s to global license file" %
link_name)
def stage(tmpdir_factory):
"""Creates a stage with the directory structure for the tests."""
s = tmpdir_factory.mktemp('filesystem_test')
with s.as_cwd():
# Create source file hierarchy
fs.touchp('source/1')
fs.touchp('source/a/b/2')
fs.touchp('source/a/b/3')
fs.touchp('source/c/4')
fs.touchp('source/c/d/5')
fs.touchp('source/c/d/6')
fs.touchp('source/c/d/e/7')
fs.touchp('source/g/h/i/8')
fs.touchp('source/g/h/i/9')
fs.touchp('source/g/i/j/10')
# Create symlinks
symlink(os.path.abspath('source/1'), 'source/2')
symlink('b/2', 'source/a/b2')
symlink('a/b', 'source/f')
# Create destination directory
fs.mkdirp('dest')
yield s
def relocate_links(links, orig_layout_root,
orig_install_prefix, new_install_prefix):
"""Relocate links to a new install prefix.
The symbolic links are relative to the original installation prefix.
The old link target is read and the placeholder is replaced by the old
layout root. If the old link target is in the old install prefix, the new
link target is create by replacing the old install prefix with the new
install prefix.
Args:
links (list): list of links to be relocated
orig_layout_root (str): original layout root
orig_install_prefix (str): install prefix of the original installation
new_install_prefix (str): install prefix where we want to relocate
"""
placeholder = _placeholder(orig_layout_root)
abs_links = [os.path.join(new_install_prefix, link) for link in links]
for abs_link in abs_links:
link_target = os.readlink(abs_link)
link_target = re.sub(placeholder, orig_layout_root, link_target)
# If the link points to a file in the original install prefix,
# compute the corresponding target in the new prefix and relink
if link_target.startswith(orig_install_prefix):
link_target = re.sub(
orig_install_prefix, new_install_prefix, link_target
)
os.unlink(abs_link)
symlink(link_target, abs_link)
# If the link is absolute and has not been relocated then
# warn the user about that
if (os.path.isabs(link_target) and
not link_target.startswith(new_install_prefix)):
msg = ('Link target "{0}" for symbolic link "{1}" is outside'
' of the new install prefix {2}')
tty.warn(msg.format(link_target, abs_link, new_install_prefix))
def view_symlink(src, dst, **kwargs):
# keyword arguments are irrelevant
# here to fit required call signature
symlink(src, dst)
def test_buildcache(mock_archive, tmpdir):
# tweak patchelf to only do a download
pspec = Spec("patchelf").concretized()
pkg = pspec.package
fake_fetchify(pkg.fetcher, pkg)
mkdirp(os.path.join(pkg.prefix, "bin"))
patchelfscr = os.path.join(pkg.prefix, "bin", "patchelf")
f = open(patchelfscr, 'w')
body = """#!/bin/bash
echo $PATH"""
f.write(body)
f.close()
st = os.stat(patchelfscr)
os.chmod(patchelfscr, st.st_mode | stat.S_IEXEC)
# Install the test package
spec = Spec('trivial-install-test-package')
spec.concretize()
assert spec.concrete
pkg = spec.package
fake_fetchify(mock_archive.url, pkg)
pkg.do_install()
pkghash = '/' + str(spec.dag_hash(7))
# Put some non-relocatable file in there
filename = os.path.join(spec.prefix, "dummy.txt")
with open(filename, "w") as script:
script.write(spec.prefix)
# Create an absolute symlink
linkname = os.path.join(spec.prefix, "link_to_dummy.txt")
symlink(filename, linkname)
# Create the build cache and
# put it directly into the mirror
mirror_path = os.path.join(str(tmpdir), 'test-mirror')
spack.mirror.create(mirror_path, specs=[])
# register mirror with spack config
mirrors = {'spack-mirror-test': 'file://' + mirror_path}
spack.config.set('mirrors', mirrors)
stage = spack.stage.Stage(
mirrors['spack-mirror-test'], name="build_cache", keep=True)
stage.create()
# setup argument parser
parser = argparse.ArgumentParser()
buildcache.setup_parser(parser)
create_args = ['create', '-a', '-f', '-d', mirror_path, pkghash]
# Create a private key to sign package with if gpg2 available
spack.util.gpg.create(name='test key 1', expires='0',
email='*****@*****.**',
comment='Spack test key')
create_args.insert(create_args.index('-a'), '--rebuild-index')
args = parser.parse_args(create_args)
buildcache.buildcache(parser, args)
# trigger overwrite warning
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
install_args = ['install', '-a', '-f', pkghash]
args = parser.parse_args(install_args)
# Test install
buildcache.buildcache(parser, args)
files = os.listdir(spec.prefix)
assert 'link_to_dummy.txt' in files
assert 'dummy.txt' in files
# Validate the relocation information
buildinfo = bindist.read_buildinfo_file(spec.prefix)
assert(buildinfo['relocate_textfiles'] == ['dummy.txt'])
assert(buildinfo['relocate_links'] == ['link_to_dummy.txt'])
# create build cache with relative path
create_args.insert(create_args.index('-a'), '-f')
create_args.insert(create_args.index('-a'), '-r')
args = parser.parse_args(create_args)
buildcache.buildcache(parser, args)
# Uninstall the package
pkg.do_uninstall(force=True)
args = parser.parse_args(install_args)
buildcache.buildcache(parser, args)
# test overwrite install
install_args.insert(install_args.index('-a'), '-f')
args = parser.parse_args(install_args)
buildcache.buildcache(parser, args)
files = os.listdir(spec.prefix)
assert 'link_to_dummy.txt' in files
assert 'dummy.txt' in files
# assert os.path.realpath(
# os.path.join(spec.prefix, 'link_to_dummy.txt')
# ) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt'))
args = parser.parse_args(['keys'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['list', 'trivial'])
buildcache.buildcache(parser, args)
# Copy a key to the mirror to have something to download
shutil.copyfile(mock_gpg_keys_path + '/external.key',
mirror_path + '/external.key')
args = parser.parse_args(['keys'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['keys', '-f'])
buildcache.buildcache(parser, args)
args = parser.parse_args(['keys', '-i', '-t'])
buildcache.buildcache(parser, args)
# unregister mirror with spack config
mirrors = {}
spack.config.set('mirrors', mirrors)
shutil.rmtree(mirror_path)
stage.destroy()
# Remove cached binary specs since we deleted the mirror
bindist._cached_specs = set()
def setup_custom_environment(self, pkg, env):
"""Set the DEVELOPER_DIR environment for the Xcode toolchain.
On macOS, not all buildsystems support querying CC and CXX for the
compilers to use and instead query the Xcode toolchain for what
compiler to run. This side-steps the spack wrappers. In order to inject
spack into this setup, we need to copy (a subset of) Xcode.app and
replace the compiler executables with symlinks to the spack wrapper.
Currently, the stage is used to store the Xcode.app copies. We then set
the 'DEVELOPER_DIR' environment variables to cause the xcrun and
related tools to use this Xcode.app.
"""
super(AppleClang, self).setup_custom_environment(pkg, env)
if not pkg.use_xcode:
# if we do it for all packages, we get into big troubles with MPI:
# filter_compilers(self) will use mockup XCode compilers on macOS
# with Clang. Those point to Spack's compiler wrappers and
# consequently render MPI non-functional outside of Spack.
return
# Use special XCode versions of compiler wrappers when using XCode
# Overwrites build_environment's setting of SPACK_CC and SPACK_CXX
xcrun = spack.util.executable.Executable('xcrun')
xcode_clang = xcrun('-f', 'clang', output=str).strip()
xcode_clangpp = xcrun('-f', 'clang++', output=str).strip()
env.set('SPACK_CC', xcode_clang, force=True)
env.set('SPACK_CXX', xcode_clangpp, force=True)
xcode_select = spack.util.executable.Executable('xcode-select')
# Get the path of the active developer directory
real_root = xcode_select('--print-path', output=str).strip()
# The path name can be used to determine whether the full Xcode suite
# or just the command-line tools are installed
if real_root.endswith('Developer'):
# The full Xcode suite is installed
pass
else:
if real_root.endswith('CommandLineTools'):
# Only the command-line tools are installed
msg = 'It appears that you have the Xcode command-line tools '
msg += 'but not the full Xcode suite installed.\n'
else:
# Xcode is not installed
msg = 'It appears that you do not have Xcode installed.\n'
msg += 'In order to use Spack to build the requested application, '
msg += 'you need the full Xcode suite. It can be installed '
msg += 'through the App Store. Make sure you launch the '
msg += 'application and accept the license agreement.\n'
raise OSError(msg)
real_root = os.path.dirname(os.path.dirname(real_root))
developer_root = os.path.join(spack.stage.get_stage_root(),
'xcode-select', self.name,
str(self.version))
xcode_link = os.path.join(developer_root, 'Xcode.app')
if not os.path.exists(developer_root):
tty.warn('Copying Xcode from %s to %s in order to add spack '
'wrappers to it. Please do not interrupt.' %
(real_root, developer_root))
# We need to make a new Xcode.app instance, but with symlinks to
# the spack wrappers for the compilers it ships. This is necessary
# because some projects insist on just asking xcrun and related
# tools where the compiler runs. These tools are very hard to trick
# as they do realpath and end up ignoring the symlinks in a
# "softer" tree of nothing but symlinks in the right places.
shutil.copytree(real_root,
developer_root,
symlinks=True,
ignore=shutil.ignore_patterns(
'AppleTV*.platform', 'Watch*.platform',
'iPhone*.platform', 'Documentation', 'swift*'))
real_dirs = [
'Toolchains/XcodeDefault.xctoolchain/usr/bin',
'usr/bin',
]
bins = ['c++', 'c89', 'c99', 'cc', 'clang', 'clang++', 'cpp']
for real_dir in real_dirs:
dev_dir = os.path.join(developer_root, 'Contents', 'Developer',
real_dir)
for fname in os.listdir(dev_dir):
if fname in bins:
os.unlink(os.path.join(dev_dir, fname))
symlink(os.path.join(spack.paths.build_env_path, 'cc'),
os.path.join(dev_dir, fname))
symlink(developer_root, xcode_link)
env.set('DEVELOPER_DIR', xcode_link)
