def test_relocate_links(tmpdir): with tmpdir.as_cwd(): old_layout_root = os.path.join( '%s' % tmpdir, 'home', 'spack', 'opt', 'spack') old_install_prefix = os.path.join( '%s' % old_layout_root, 'debian6', 'test') old_binname = os.path.join(old_install_prefix, 'binfile') placeholder = _placeholder(old_layout_root) re.sub(old_layout_root, placeholder, old_binname) filenames = ['link.ln', 'outsideprefix.ln'] new_layout_root = os.path.join( '%s' % tmpdir, 'opt', 'rh', 'devtoolset') new_install_prefix = os.path.join( '%s' % new_layout_root, 'test', 'debian6') new_linkname = os.path.join(new_install_prefix, 'link.ln') new_linkname2 = os.path.join(new_install_prefix, 'outsideprefix.ln') new_binname = os.path.join(new_install_prefix, 'binfile') mkdirp(new_install_prefix) with open(new_binname, 'w') as f: f.write('\n') os.utime(new_binname, None) symlink(old_binname, new_linkname) symlink('/usr/lib/libc.so', new_linkname2) relocate_links(filenames, old_layout_root, old_install_prefix, new_install_prefix) assert os.readlink(new_linkname) == new_binname assert os.readlink(new_linkname2) == '/usr/lib/libc.so'
def _relocate_spliced_links(links, orig_prefix, new_prefix): """Re-linking function which differs from `relocate.relocate_links` by reading the old link rather than the new link, since the latter wasn't moved in our case. This still needs to be called after the copy to destination because it expects the new directory structure to be in place.""" for link in links: link_target = os.readlink(os.path.join(orig_prefix, link)) link_target = re.sub('^' + orig_prefix, new_prefix, link_target) new_link_path = os.path.join(new_prefix, link) os.unlink(new_link_path) symlink(link_target, new_link_path)
def test_check_prefix_manifest(tmpdir): # Test the verification of an entire prefix and its contents prefix_path = tmpdir.join('prefix') prefix = str(prefix_path) spec = spack.spec.Spec('libelf') spec._mark_concrete() spec.prefix = prefix results = spack.verify.check_spec_manifest(spec) assert results.has_errors() assert prefix in results.errors assert results.errors[prefix] == ['manifest missing'] metadata_dir = str(prefix_path.join('.spack')) bin_dir = str(prefix_path.join('bin')) other_dir = str(prefix_path.join('other')) for d in (metadata_dir, bin_dir, other_dir): fs.mkdirp(d) file = os.path.join(other_dir, 'file') with open(file, 'w') as f: f.write("I'm a little file short and stout") link = os.path.join(bin_dir, 'run') symlink(file, link) spack.verify.write_manifest(spec) results = spack.verify.check_spec_manifest(spec) assert not results.has_errors() os.remove(link) malware = os.path.join(metadata_dir, 'hiddenmalware') with open(malware, 'w') as f: f.write("Foul evil deeds") results = spack.verify.check_spec_manifest(spec) assert results.has_errors() assert all(x in results.errors for x in (malware, link)) assert len(results.errors) == 2 assert results.errors[link] == ['deleted'] assert results.errors[malware] == ['added'] manifest_file = os.path.join(spec.prefix, spack.store.layout.metadata_dir, spack.store.layout.manifest_file_name) with open(manifest_file, 'w') as f: f.write("{This) string is not proper json") results = spack.verify.check_spec_manifest(spec) assert results.has_errors() assert results.errors[spec.prefix] == ['manifest corrupted']
def make_link_relative(new_links, orig_links): """Compute the relative target from the original link and make the new link relative. Args: new_links (list): new links to be made relative orig_links (list): original links """ for new_link, orig_link in zip(new_links, orig_links): target = os.readlink(orig_link) relative_target = os.path.relpath(target, os.path.dirname(orig_link)) os.unlink(new_link) symlink(relative_target, new_link)
def symlink_windows(self): if not is_windows: return win_install_path = os.path.join(self.prefix.bin, "MSWin32") if self.is_64bit(): win_install_path += "-x64" else: win_install_path += "-x86" if self.spec.satisfies("+threads"): win_install_path += "-multi-thread" else: win_install_path += "-perlio" for f in os.listdir(os.path.join(self.prefix.bin, win_install_path)): lnk_path = os.path.join(self.prefix.bin, f) src_path = os.path.join(win_install_path, f) if not os.path.exists(lnk_path): symlink(src_path, lnk_path)
def symlink(self, mirror_ref): """Symlink a human readible path in our mirror to the actual storage location.""" cosmetic_path = os.path.join(self.root, mirror_ref.cosmetic_path) storage_path = os.path.join(self.root, mirror_ref.storage_path) relative_dst = os.path.relpath( storage_path, start=os.path.dirname(cosmetic_path)) if not os.path.exists(cosmetic_path): if os.path.lexists(cosmetic_path): # In this case the link itself exists but it is broken: remove # it and recreate it (in order to fix any symlinks broken prior # to https://github.com/spack/spack/pull/13908) os.unlink(cosmetic_path) mkdirp(os.path.dirname(cosmetic_path)) symlink(relative_dst, cosmetic_path)
def symlink_license(pkg): """Create local symlinks that point to the global license file.""" target = pkg.global_license_file for filename in pkg.license_files: link_name = os.path.join(pkg.prefix, filename) link_name = os.path.abspath(link_name) license_dir = os.path.dirname(link_name) if not os.path.exists(license_dir): mkdirp(license_dir) # If example file already exists, overwrite it with a symlink if os.path.lexists(link_name): os.remove(link_name) if os.path.exists(target): symlink(target, link_name) tty.msg("Added local symlink %s to global license file" % link_name)
def stage(tmpdir_factory): """Creates a stage with the directory structure for the tests.""" s = tmpdir_factory.mktemp('filesystem_test') with s.as_cwd(): # Create source file hierarchy fs.touchp('source/1') fs.touchp('source/a/b/2') fs.touchp('source/a/b/3') fs.touchp('source/c/4') fs.touchp('source/c/d/5') fs.touchp('source/c/d/6') fs.touchp('source/c/d/e/7') fs.touchp('source/g/h/i/8') fs.touchp('source/g/h/i/9') fs.touchp('source/g/i/j/10') # Create symlinks symlink(os.path.abspath('source/1'), 'source/2') symlink('b/2', 'source/a/b2') symlink('a/b', 'source/f') # Create destination directory fs.mkdirp('dest') yield s
def relocate_links(links, orig_layout_root, orig_install_prefix, new_install_prefix): """Relocate links to a new install prefix. The symbolic links are relative to the original installation prefix. The old link target is read and the placeholder is replaced by the old layout root. If the old link target is in the old install prefix, the new link target is create by replacing the old install prefix with the new install prefix. Args: links (list): list of links to be relocated orig_layout_root (str): original layout root orig_install_prefix (str): install prefix of the original installation new_install_prefix (str): install prefix where we want to relocate """ placeholder = _placeholder(orig_layout_root) abs_links = [os.path.join(new_install_prefix, link) for link in links] for abs_link in abs_links: link_target = os.readlink(abs_link) link_target = re.sub(placeholder, orig_layout_root, link_target) # If the link points to a file in the original install prefix, # compute the corresponding target in the new prefix and relink if link_target.startswith(orig_install_prefix): link_target = re.sub( orig_install_prefix, new_install_prefix, link_target ) os.unlink(abs_link) symlink(link_target, abs_link) # If the link is absolute and has not been relocated then # warn the user about that if (os.path.isabs(link_target) and not link_target.startswith(new_install_prefix)): msg = ('Link target "{0}" for symbolic link "{1}" is outside' ' of the new install prefix {2}') tty.warn(msg.format(link_target, abs_link, new_install_prefix))
def view_symlink(src, dst, **kwargs): # keyword arguments are irrelevant # here to fit required call signature symlink(src, dst)
def test_buildcache(mock_archive, tmpdir): # tweak patchelf to only do a download pspec = Spec("patchelf").concretized() pkg = pspec.package fake_fetchify(pkg.fetcher, pkg) mkdirp(os.path.join(pkg.prefix, "bin")) patchelfscr = os.path.join(pkg.prefix, "bin", "patchelf") f = open(patchelfscr, 'w') body = """#!/bin/bash echo $PATH""" f.write(body) f.close() st = os.stat(patchelfscr) os.chmod(patchelfscr, st.st_mode | stat.S_IEXEC) # Install the test package spec = Spec('trivial-install-test-package') spec.concretize() assert spec.concrete pkg = spec.package fake_fetchify(mock_archive.url, pkg) pkg.do_install() pkghash = '/' + str(spec.dag_hash(7)) # Put some non-relocatable file in there filename = os.path.join(spec.prefix, "dummy.txt") with open(filename, "w") as script: script.write(spec.prefix) # Create an absolute symlink linkname = os.path.join(spec.prefix, "link_to_dummy.txt") symlink(filename, linkname) # Create the build cache and # put it directly into the mirror mirror_path = os.path.join(str(tmpdir), 'test-mirror') spack.mirror.create(mirror_path, specs=[]) # register mirror with spack config mirrors = {'spack-mirror-test': 'file://' + mirror_path} spack.config.set('mirrors', mirrors) stage = spack.stage.Stage( mirrors['spack-mirror-test'], name="build_cache", keep=True) stage.create() # setup argument parser parser = argparse.ArgumentParser() buildcache.setup_parser(parser) create_args = ['create', '-a', '-f', '-d', mirror_path, pkghash] # Create a private key to sign package with if gpg2 available spack.util.gpg.create(name='test key 1', expires='0', email='*****@*****.**', comment='Spack test key') create_args.insert(create_args.index('-a'), '--rebuild-index') args = parser.parse_args(create_args) buildcache.buildcache(parser, args) # trigger overwrite warning buildcache.buildcache(parser, args) # Uninstall the package pkg.do_uninstall(force=True) install_args = ['install', '-a', '-f', pkghash] args = parser.parse_args(install_args) # Test install buildcache.buildcache(parser, args) files = os.listdir(spec.prefix) assert 'link_to_dummy.txt' in files assert 'dummy.txt' in files # Validate the relocation information buildinfo = bindist.read_buildinfo_file(spec.prefix) assert(buildinfo['relocate_textfiles'] == ['dummy.txt']) assert(buildinfo['relocate_links'] == ['link_to_dummy.txt']) # create build cache with relative path create_args.insert(create_args.index('-a'), '-f') create_args.insert(create_args.index('-a'), '-r') args = parser.parse_args(create_args) buildcache.buildcache(parser, args) # Uninstall the package pkg.do_uninstall(force=True) args = parser.parse_args(install_args) buildcache.buildcache(parser, args) # test overwrite install install_args.insert(install_args.index('-a'), '-f') args = parser.parse_args(install_args) buildcache.buildcache(parser, args) files = os.listdir(spec.prefix) assert 'link_to_dummy.txt' in files assert 'dummy.txt' in files # assert os.path.realpath( # os.path.join(spec.prefix, 'link_to_dummy.txt') # ) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt')) args = parser.parse_args(['keys']) buildcache.buildcache(parser, args) args = parser.parse_args(['list']) buildcache.buildcache(parser, args) args = parser.parse_args(['list']) buildcache.buildcache(parser, args) args = parser.parse_args(['list', 'trivial']) buildcache.buildcache(parser, args) # Copy a key to the mirror to have something to download shutil.copyfile(mock_gpg_keys_path + '/external.key', mirror_path + '/external.key') args = parser.parse_args(['keys']) buildcache.buildcache(parser, args) args = parser.parse_args(['keys', '-f']) buildcache.buildcache(parser, args) args = parser.parse_args(['keys', '-i', '-t']) buildcache.buildcache(parser, args) # unregister mirror with spack config mirrors = {} spack.config.set('mirrors', mirrors) shutil.rmtree(mirror_path) stage.destroy() # Remove cached binary specs since we deleted the mirror bindist._cached_specs = set()
def setup_custom_environment(self, pkg, env): """Set the DEVELOPER_DIR environment for the Xcode toolchain. On macOS, not all buildsystems support querying CC and CXX for the compilers to use and instead query the Xcode toolchain for what compiler to run. This side-steps the spack wrappers. In order to inject spack into this setup, we need to copy (a subset of) Xcode.app and replace the compiler executables with symlinks to the spack wrapper. Currently, the stage is used to store the Xcode.app copies. We then set the 'DEVELOPER_DIR' environment variables to cause the xcrun and related tools to use this Xcode.app. """ super(AppleClang, self).setup_custom_environment(pkg, env) if not pkg.use_xcode: # if we do it for all packages, we get into big troubles with MPI: # filter_compilers(self) will use mockup XCode compilers on macOS # with Clang. Those point to Spack's compiler wrappers and # consequently render MPI non-functional outside of Spack. return # Use special XCode versions of compiler wrappers when using XCode # Overwrites build_environment's setting of SPACK_CC and SPACK_CXX xcrun = spack.util.executable.Executable('xcrun') xcode_clang = xcrun('-f', 'clang', output=str).strip() xcode_clangpp = xcrun('-f', 'clang++', output=str).strip() env.set('SPACK_CC', xcode_clang, force=True) env.set('SPACK_CXX', xcode_clangpp, force=True) xcode_select = spack.util.executable.Executable('xcode-select') # Get the path of the active developer directory real_root = xcode_select('--print-path', output=str).strip() # The path name can be used to determine whether the full Xcode suite # or just the command-line tools are installed if real_root.endswith('Developer'): # The full Xcode suite is installed pass else: if real_root.endswith('CommandLineTools'): # Only the command-line tools are installed msg = 'It appears that you have the Xcode command-line tools ' msg += 'but not the full Xcode suite installed.\n' else: # Xcode is not installed msg = 'It appears that you do not have Xcode installed.\n' msg += 'In order to use Spack to build the requested application, ' msg += 'you need the full Xcode suite. It can be installed ' msg += 'through the App Store. Make sure you launch the ' msg += 'application and accept the license agreement.\n' raise OSError(msg) real_root = os.path.dirname(os.path.dirname(real_root)) developer_root = os.path.join(spack.stage.get_stage_root(), 'xcode-select', self.name, str(self.version)) xcode_link = os.path.join(developer_root, 'Xcode.app') if not os.path.exists(developer_root): tty.warn('Copying Xcode from %s to %s in order to add spack ' 'wrappers to it. Please do not interrupt.' % (real_root, developer_root)) # We need to make a new Xcode.app instance, but with symlinks to # the spack wrappers for the compilers it ships. This is necessary # because some projects insist on just asking xcrun and related # tools where the compiler runs. These tools are very hard to trick # as they do realpath and end up ignoring the symlinks in a # "softer" tree of nothing but symlinks in the right places. shutil.copytree(real_root, developer_root, symlinks=True, ignore=shutil.ignore_patterns( 'AppleTV*.platform', 'Watch*.platform', 'iPhone*.platform', 'Documentation', 'swift*')) real_dirs = [ 'Toolchains/XcodeDefault.xctoolchain/usr/bin', 'usr/bin', ] bins = ['c++', 'c89', 'c99', 'cc', 'clang', 'clang++', 'cpp'] for real_dir in real_dirs: dev_dir = os.path.join(developer_root, 'Contents', 'Developer', real_dir) for fname in os.listdir(dev_dir): if fname in bins: os.unlink(os.path.join(dev_dir, fname)) symlink(os.path.join(spack.paths.build_env_path, 'cc'), os.path.join(dev_dir, fname)) symlink(developer_root, xcode_link) env.set('DEVELOPER_DIR', xcode_link)