def test_yaml_directory_layout_build_path(tmpdir, config):
"""This tests build path method."""
spec = Spec('python')
spec.concretize()
layout = YamlDirectoryLayout(str(tmpdir))
rel_path = os.path.join(layout.metadata_dir, layout.packages_dir)
assert layout.build_packages_path(spec) == os.path.join(
spec.prefix, rel_path)
def layout_and_dir(tmpdir):
"""Returns a directory layout and the corresponding directory."""
layout = YamlDirectoryLayout(str(tmpdir))
old_layout = spack.store.layout
spack.store.layout = layout
yield layout, str(tmpdir)
spack.store.layout = old_layout
def test_python_namespace_conflict(tmpdir, namespace_extensions,
builtin_and_mock_packages):
"""Test the view update logic in PythonPackage reports an error when two
python extensions with different namespaces have a conflicting __init__
file.
"""
ext1_prefix, ext2_prefix, py_namespace = namespace_extensions
other_namespace = py_namespace + 'other'
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
py_namespace)
ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
other_namespace)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext1_pkg, view)
view.extensions_layout.add_extension(python_spec, ext1_pkg.spec)
with pytest.raises(MergeConflictError):
python_pkg.activate(ext2_pkg, view)
def test_python_ignore_namespace_init_conflict(tmpdir, namespace_extensions,
builtin_and_mock_packages):
"""Test the view update logic in PythonPackage ignores conflicting
instances of __init__ for packages which are in the same namespace.
"""
ext1_prefix, ext2_prefix, py_namespace = namespace_extensions
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
py_namespace)
ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
py_namespace)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext1_pkg, view)
# Normally handled by Package.do_activate, but here we activate directly
view.extensions_layout.add_extension(python_spec, ext1_pkg.spec)
python_pkg.activate(ext2_pkg, view)
f1 = 'lib/python2.7/site-packages/examplenamespace/ext1_sample.py'
f2 = 'lib/python2.7/site-packages/examplenamespace/ext2_sample.py'
init_file = 'lib/python2.7/site-packages/examplenamespace/__init__.py'
assert os.path.exists(os.path.join(view_dir, f1))
assert os.path.exists(os.path.join(view_dir, f2))
assert os.path.exists(os.path.join(view_dir, init_file))
def test_yaml_directory_layout_parameters(
tmpdir, config
):
"""This tests the various parameters that can be used to configure
the install location """
spec = Spec('python')
spec.concretize()
# Ensure default layout matches expected spec format
layout_default = YamlDirectoryLayout(str(tmpdir))
path_default = layout_default.relative_path_for_spec(spec)
assert(path_default == spec.format(
"${ARCHITECTURE}/"
"${COMPILERNAME}-${COMPILERVER}/"
"${PACKAGE}-${VERSION}-${HASH}"))
# Test hash_length parameter works correctly
layout_10 = YamlDirectoryLayout(str(tmpdir), hash_len=10)
path_10 = layout_10.relative_path_for_spec(spec)
layout_7 = YamlDirectoryLayout(str(tmpdir), hash_len=7)
path_7 = layout_7.relative_path_for_spec(spec)
assert(len(path_default) - len(path_10) == 22)
assert(len(path_default) - len(path_7) == 25)
# Test path_scheme
arch, compiler, package7 = path_7.split('/')
scheme_package7 = "${PACKAGE}-${VERSION}-${HASH:7}"
layout_package7 = YamlDirectoryLayout(str(tmpdir),
path_scheme=scheme_package7)
path_package7 = layout_package7.relative_path_for_spec(spec)
assert(package7 == path_package7)
# Test separation of architecture
arch_scheme_package = "${PLATFORM}/${TARGET}/${OS}/${PACKAGE}/${VERSION}/${HASH:7}" # NOQA: ignore=E501
layout_arch_package = YamlDirectoryLayout(str(tmpdir),
path_scheme=arch_scheme_package)
arch_path_package = layout_arch_package.relative_path_for_spec(spec)
assert(arch_path_package == spec.format(arch_scheme_package))
# Ensure conflicting parameters caught
with pytest.raises(InvalidDirectoryLayoutParametersError):
YamlDirectoryLayout(str(tmpdir),
hash_len=20,
path_scheme=scheme_package7)
def install_dir_non_default_layout(tmpdir):
"""Hooks a fake install directory with a non-default layout"""
real_store = spack.store.store
real_layout = spack.store.layout
spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')),
path_scheme=ndef_install_path_scheme) # noqa: E501
yield spack.store
spack.store.store = real_store
spack.store.layout = real_layout
def test_remove_extensions_ordered(install_mockery, mock_fetch, tmpdir):
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
e2 = Spec('extension2').concretized()
e2.package.do_install()
view.add_specs(e2)
e1 = e2['extension1']
view.remove_specs(e1, e2)
def install_dir_non_default_layout(tmpdir):
"""Hooks a fake install directory with a non-default layout"""
scheme = os.path.join(
'${name}', '${version}',
'${architecture}-${compiler.name}-${compiler.version}-${hash}')
real_store, real_layout = spack.store.store, spack.store.layout
opt_dir = tmpdir.join('opt')
spack.store.store = spack.store.Store(str(opt_dir))
spack.store.layout = YamlDirectoryLayout(str(opt_dir), path_scheme=scheme)
try:
yield spack.store
finally:
spack.store.store = real_store
spack.store.layout = real_layout
def setUp(self):
super(InstallTest, self).setUp()
# create a simple installable package directory and tarball
self.repo = MockArchive()
# We use a fake package, so skip the checksum.
spack.do_checksum = False
# Use a fake install directory to avoid conflicts bt/w
# installed pkgs and mock packages.
self.tmpdir = tempfile.mkdtemp()
self.orig_layout = spack.install_layout
spack.install_layout = YamlDirectoryLayout(self.tmpdir)
def setUp(self):
super(MockDatabase, self).setUp()
#
# TODO: make the mockup below easier.
#
# Make a fake install directory
self.install_path = tempfile.mkdtemp()
self.spack_install_path = spack.install_path
spack.install_path = self.install_path
self.install_layout = YamlDirectoryLayout(self.install_path)
self.spack_install_layout = spack.install_layout
spack.install_layout = self.install_layout
# Make fake database and fake install directory.
self.installed_db = Database(self.install_path)
self.spack_installed_db = spack.installed_db
spack.installed_db = self.installed_db
# make a mock database with some packages installed note that
# the ref count for dyninst here will be 3, as it's recycled
# across each install.
#
# Here is what the mock DB looks like:
#
# o mpileaks o mpileaks' o mpileaks''
# |\ |\ |\
# | o callpath | o callpath' | o callpath''
# |/| |/| |/|
# o | mpich o | mpich2 o | zmpi
# | | o | fake
# | | |
# | |______________/
# | .____________/
# |/
# o dyninst
# |\
# | o libdwarf
# |/
# o libelf
#
# Transaction used to avoid repeated writes.
with spack.installed_db.write_transaction():
self._mock_install('mpileaks ^mpich')
self._mock_install('mpileaks ^mpich2')
self._mock_install('mpileaks ^zmpi')
def install_mockery(tmpdir, config, builtin_mock):
"""Hooks a fake install directory and a fake db into Spack."""
layout = spack.store.layout
db = spack.store.db
# Use a fake install directory to avoid conflicts bt/w
# installed pkgs and mock packages.
spack.store.layout = YamlDirectoryLayout(str(tmpdir))
spack.store.db = Database(str(tmpdir))
# We use a fake package, so skip the checksum.
spack.do_checksum = False
yield
# Turn checksumming back on
spack.do_checksum = True
# Restore Spack's layout.
spack.store.layout = layout
spack.store.db = db
def test_perl_activation_view(tmpdir, perl_and_extension_dirs):
perl_prefix, ext_prefix = perl_and_extension_dirs
perl_spec = spack.spec.Spec('[email protected]')
perl_spec._concrete = True
perl_spec.package.spec.prefix = perl_prefix
ext_pkg = FakeExtensionPackage('perl-extension', ext_prefix)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
perl_pkg = perl_spec.package
perl_pkg.activate(ext_pkg, extensions_layout=view.extensions_layout)
assert not os.path.exists(join_path(perl_prefix, 'bin/perl-ext-tool'))
assert os.path.exists(join_path(view_dir, 'bin/perl-ext-tool'))
def test_python_activation_view(tmpdir, python_and_extension_dirs):
python_prefix, ext_prefix = python_and_extension_dirs
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
python_spec.package.spec._set_test_prefix(python_prefix)
ext_pkg = FakeExtensionPackage('py-extension', ext_prefix)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext_pkg, extensions_layout=view.extensions_layout)
assert not os.path.exists(join_path(python_prefix, 'bin/py-ext-tool'))
assert os.path.exists(join_path(view_dir, 'bin/py-ext-tool'))
def test_perl_activation_view(tmpdir, perl_and_extension_dirs,
builtin_and_mock_packages):
perl_prefix, ext_prefix = perl_and_extension_dirs
perl_spec = spack.spec.Spec('[email protected]')
perl_spec._concrete = True
perl_spec.package.spec.prefix = perl_prefix
ext_pkg = create_ext_pkg('perl-extension', ext_prefix, perl_spec)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
perl_pkg = perl_spec.package
perl_pkg.activate(ext_pkg, view)
assert not os.path.exists(os.path.join(perl_prefix, 'bin/perl-ext-tool'))
assert os.path.exists(os.path.join(view_dir, 'bin/perl-ext-tool'))
def test_python_activation_view(tmpdir, python_and_extension_dirs,
builtin_and_mock_packages, monkeypatch):
python_prefix, ext_prefix = python_and_extension_dirs
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
python_spec.package.spec.prefix = python_prefix
ext_pkg = create_python_ext_pkg('py-extension1', ext_prefix, python_spec,
monkeypatch)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext_pkg, view)
assert not os.path.exists(os.path.join(python_prefix, 'bin/py-ext-tool'))
assert os.path.exists(os.path.join(view_dir, 'bin/py-ext-tool'))
def test_python_keep_namespace_init(tmpdir, namespace_extensions,
builtin_and_mock_packages):
"""Test the view update logic in PythonPackage keeps the namespace
__init__ file as long as one package in the namespace still
exists.
"""
ext1_prefix, ext2_prefix, py_namespace = namespace_extensions
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
py_namespace)
ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
py_namespace)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext1_pkg, view)
# Normally handled by Package.do_activate, but here we activate directly
view.extensions_layout.add_extension(python_spec, ext1_pkg.spec)
python_pkg.activate(ext2_pkg, view)
view.extensions_layout.add_extension(python_spec, ext2_pkg.spec)
f1 = 'lib/python2.7/site-packages/examplenamespace/ext1_sample.py'
init_file = 'lib/python2.7/site-packages/examplenamespace/__init__.py'
python_pkg.deactivate(ext1_pkg, view)
view.extensions_layout.remove_extension(python_spec, ext1_pkg.spec)
assert not os.path.exists(os.path.join(view_dir, f1))
assert os.path.exists(os.path.join(view_dir, init_file))
python_pkg.deactivate(ext2_pkg, view)
view.extensions_layout.remove_extension(python_spec, ext2_pkg.spec)
assert not os.path.exists(os.path.join(view_dir, init_file))
def layout_and_dir(tmpdir):
"""Returns a directory layout and the corresponding directory."""
yield YamlDirectoryLayout(str(tmpdir)), str(tmpdir)
def test_yaml_directory_layout_parameters(
tmpdir, config
):
"""This tests the various parameters that can be used to configure
the install location """
spec = Spec('python')
spec.concretize()
# Ensure default layout matches expected spec format
layout_default = YamlDirectoryLayout(str(tmpdir))
path_default = layout_default.relative_path_for_spec(spec)
assert(path_default == spec.format(
"${ARCHITECTURE}/"
"${COMPILERNAME}-${COMPILERVER}/"
"${PACKAGE}-${VERSION}-${HASH}"))
# Test hash_length parameter works correctly
layout_10 = YamlDirectoryLayout(str(tmpdir), hash_len=10)
path_10 = layout_10.relative_path_for_spec(spec)
layout_7 = YamlDirectoryLayout(str(tmpdir), hash_len=7)
path_7 = layout_7.relative_path_for_spec(spec)
assert(len(path_default) - len(path_10) == 22)
assert(len(path_default) - len(path_7) == 25)
# Test path_scheme
arch, compiler, package7 = path_7.split('/')
scheme_package7 = "${PACKAGE}-${VERSION}-${HASH:7}"
layout_package7 = YamlDirectoryLayout(str(tmpdir),
path_scheme=scheme_package7)
path_package7 = layout_package7.relative_path_for_spec(spec)
assert(package7 == path_package7)
# Ensure conflicting parameters caught
with pytest.raises(InvalidDirectoryLayoutParametersError):
YamlDirectoryLayout(str(tmpdir),
hash_len=20,
path_scheme=scheme_package7)
def setUp(self):
self.tmpdir = tempfile.mkdtemp()
self.layout = YamlDirectoryLayout(self.tmpdir)
def test_yaml_directory_layout_parameters(tmpdir, config):
"""This tests the various parameters that can be used to configure
the install location """
spec = Spec('python')
spec.concretize()
# Ensure default layout matches expected spec format
layout_default = YamlDirectoryLayout(str(tmpdir))
path_default = layout_default.relative_path_for_spec(spec)
assert (path_default == spec.format("{architecture}/"
"{compiler.name}-{compiler.version}/"
"{name}-{version}-{hash}"))
# Test hash_length parameter works correctly
layout_10 = YamlDirectoryLayout(str(tmpdir), hash_length=10)
path_10 = layout_10.relative_path_for_spec(spec)
layout_7 = YamlDirectoryLayout(str(tmpdir), hash_length=7)
path_7 = layout_7.relative_path_for_spec(spec)
assert (len(path_default) - len(path_10) == 22)
assert (len(path_default) - len(path_7) == 25)
# Test path_scheme
arch, compiler, package7 = path_7.split('/')
projections_package7 = {'all': "{name}-{version}-{hash:7}"}
layout_package7 = YamlDirectoryLayout(str(tmpdir),
projections=projections_package7)
path_package7 = layout_package7.relative_path_for_spec(spec)
assert (package7 == path_package7)
# Test separation of architecture or namespace
spec2 = Spec('libelf').concretized()
arch_scheme = "{architecture.platform}/{architecture.target}/{architecture.os}/{name}/{version}/{hash:7}" # NOQA: ignore=E501
ns_scheme = "${ARCHITECTURE}/${NAMESPACE}/${PACKAGE}-${VERSION}-${HASH:7}" # NOQA: ignore=E501
arch_ns_scheme_projections = {'all': arch_scheme, 'python': ns_scheme}
layout_arch_ns = YamlDirectoryLayout(
str(tmpdir), projections=arch_ns_scheme_projections)
arch_path_spec2 = layout_arch_ns.relative_path_for_spec(spec2)
assert (arch_path_spec2 == spec2.format(arch_scheme))
ns_path_spec = layout_arch_ns.relative_path_for_spec(spec)
assert (ns_path_spec == spec.format(ns_scheme))
# Ensure conflicting parameters caught
with pytest.raises(InvalidDirectoryLayoutParametersError):
YamlDirectoryLayout(str(tmpdir),
hash_length=20,
projections=projections_package7)
def test_update_sbang(tmpdir, test_mirror):
"""Test the creation and installation of buildcaches with default rpaths
into the non-default directory layout scheme, triggering an update of the
sbang.
"""
scheme = os.path.join(
'${name}', '${version}',
'${architecture}-${compiler.name}-${compiler.version}-${hash}')
spec_str = 'old-sbang'
# Concretize a package with some old-fashioned sbang lines.
old_spec = Spec(spec_str).concretized()
old_spec_hash_str = '/{0}'.format(old_spec.dag_hash())
# Need a fake mirror with *function* scope.
mirror_dir = test_mirror
mirror_url = 'file://{0}'.format(mirror_dir)
# Assume all commands will concretize old_spec the same way.
install_cmd('--no-cache', old_spec.name)
# Create a buildcache with the installed spec.
buildcache_cmd('create', '-u', '-a', '-d', mirror_dir, old_spec_hash_str)
# Need to force an update of the buildcache index
buildcache_cmd('update-index', '-d', mirror_url)
# Uninstall the original package.
uninstall_cmd('-y', old_spec_hash_str)
# Switch the store to the new install tree locations
newtree_dir = tmpdir.join('newtree')
s = spack.store.Store(str(newtree_dir))
s.layout = YamlDirectoryLayout(str(newtree_dir), path_scheme=scheme)
with spack.store.use_store(s):
new_spec = Spec('old-sbang')
new_spec.concretize()
assert new_spec.dag_hash() == old_spec.dag_hash()
# Install package from buildcache
buildcache_cmd('install', '-a', '-u', '-f', new_spec.name)
# Continue blowing away caches
bindist.clear_spec_cache()
spack.stage.purge()
# test that the sbang was updated by the move
sbang_style_1_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin)
sbang_style_2_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin)
installed_script_style_1_path = new_spec.prefix.bin.join(
'sbang-style-1.sh')
assert sbang_style_1_expected == \
open(str(installed_script_style_1_path)).read()
installed_script_style_2_path = new_spec.prefix.bin.join(
'sbang-style-2.sh')
assert sbang_style_2_expected == \
open(str(installed_script_style_2_path)).read()
uninstall_cmd('-y', '/%s' % new_spec.dag_hash())
class DirectoryLayoutTest(MockPackagesTest):
"""Tests that a directory layout works correctly and produces a
consistent install path."""
def setUp(self):
super(DirectoryLayoutTest, self).setUp()
self.tmpdir = tempfile.mkdtemp()
self.layout = YamlDirectoryLayout(self.tmpdir)
def tearDown(self):
super(DirectoryLayoutTest, self).tearDown()
shutil.rmtree(self.tmpdir, ignore_errors=True)
self.layout = None
def test_read_and_write_spec(self):
"""This goes through each package in spack and creates a directory for
it. It then ensures that the spec for the directory's
installed package can be read back in consistently, and
finally that the directory can be removed by the directory
layout.
"""
packages = list(spack.repo.all_packages())[:max_packages]
for pkg in packages:
if pkg.name.startswith('external'):
#External package tests cannot be installed
continue
spec = pkg.spec
# If a spec fails to concretize, just skip it. If it is a
# real error, it will be caught by concretization tests.
try:
spec.concretize()
except:
continue
self.layout.create_install_directory(spec)
install_dir = self.layout.path_for_spec(spec)
spec_path = self.layout.spec_file_path(spec)
# Ensure directory has been created in right place.
self.assertTrue(os.path.isdir(install_dir))
self.assertTrue(install_dir.startswith(self.tmpdir))
# Ensure spec file exists when directory is created
self.assertTrue(os.path.isfile(spec_path))
self.assertTrue(spec_path.startswith(install_dir))
# Make sure spec file can be read back in to get the original spec
spec_from_file = self.layout.read_spec(spec_path)
self.assertEqual(spec, spec_from_file)
self.assertTrue(spec.eq_dag, spec_from_file)
self.assertTrue(spec_from_file.concrete)
# Ensure that specs that come out "normal" are really normal.
with open(spec_path) as spec_file:
read_separately = Spec.from_yaml(spec_file.read())
read_separately.normalize()
self.assertEqual(read_separately, spec_from_file)
read_separately.concretize()
self.assertEqual(read_separately, spec_from_file)
# Make sure the hash of the read-in spec is the same
self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash())
# Ensure directories are properly removed
self.layout.remove_install_directory(spec)
self.assertFalse(os.path.isdir(install_dir))
self.assertFalse(os.path.exists(install_dir))
def test_handle_unknown_package(self):
"""This test ensures that spack can at least do *some*
operations with packages that are installed but that it
does not know about. This is actually not such an uncommon
scenario with spack; it can happen when you switch from a
git branch where you're working on a new package.
This test ensures that the directory layout stores enough
information about installed packages' specs to uninstall
or query them again if the package goes away.
"""
mock_db = RepoPath(spack.mock_packages_path)
not_in_mock = set.difference(
set(spack.repo.all_package_names()),
set(mock_db.all_package_names()))
packages = list(not_in_mock)[:max_packages]
# Create all the packages that are not in mock.
installed_specs = {}
for pkg_name in packages:
spec = spack.repo.get(pkg_name).spec
# If a spec fails to concretize, just skip it. If it is a
# real error, it will be caught by concretization tests.
try:
spec.concretize()
except:
continue
self.layout.create_install_directory(spec)
installed_specs[spec] = self.layout.path_for_spec(spec)
spack.repo.swap(mock_db)
# Now check that even without the package files, we know
# enough to read a spec from the spec file.
for spec, path in installed_specs.items():
spec_from_file = self.layout.read_spec(
join_path(path, '.spack', 'spec.yaml'))
# To satisfy these conditions, directory layouts need to
# read in concrete specs from their install dirs somehow.
self.assertEqual(path, self.layout.path_for_spec(spec_from_file))
self.assertEqual(spec, spec_from_file)
self.assertTrue(spec.eq_dag(spec_from_file))
self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash())
spack.repo.swap(mock_db)
def test_find(self):
"""Test that finding specs within an install layout works."""
packages = list(spack.repo.all_packages())[:max_packages]
# Create install prefixes for all packages in the list
installed_specs = {}
for pkg in packages:
if pkg.name.startswith('external'):
#External package tests cannot be installed
continue
spec = pkg.spec.concretized()
installed_specs[spec.name] = spec
self.layout.create_install_directory(spec)
# Make sure all the installed specs appear in DirectoryLayout.all_specs()
found_specs = dict((s.name, s) for s in self.layout.all_specs())
for name, spec in found_specs.items():
self.assertTrue(name in found_specs)
self.assertTrue(found_specs[name].eq_dag(spec))
def setUp(self):
super(DirectoryLayoutTest, self).setUp()
self.tmpdir = tempfile.mkdtemp()
self.layout = YamlDirectoryLayout(self.tmpdir)
def test_update_sbang(tmpdir, install_mockery, function_mirror):
"""
Test the creation and installation of buildcaches with default rpaths
into the non-default directory layout scheme, triggering an update of the
sbang.
"""
# Save the original store and layout before we touch ANYTHING.
real_store = spack.store.store
real_layout = spack.store.layout
# Concretize a package with some old-fashioned sbang lines.
sspec = Spec('old-sbang')
sspec.concretize()
# Need a fake mirror with *function* scope.
mirror_dir = function_mirror
# Assumes all commands will concretize sspec the same way.
install_cmd('--no-cache', sspec.name)
# Create a buildcache with the installed spec.
buildcache_cmd('create', '-u', '-a', '-d', mirror_dir,
'/%s' % sspec.dag_hash())
# Need to force an update of the buildcache index
buildcache_cmd('update-index', '-d', 'file://%s' % mirror_dir)
# Uninstall the original package.
uninstall_cmd('-y', '/%s' % sspec.dag_hash())
try:
# New install tree locations...
# Too fine-grained to do be done in a fixture
spack.store.store = spack.store.Store(str(tmpdir.join('newtree')))
spack.store.layout = YamlDirectoryLayout(
str(tmpdir.join('newtree')),
path_scheme=ndef_install_path_scheme) # noqa: E501
# Install package from buildcache
buildcache_cmd('install', '-a', '-u', '-f', sspec.name)
# Continue blowing away caches
bindist.clear_spec_cache()
spack.stage.purge()
# test that the sbang was updated by the move
sbang_style_1_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), sspec.prefix.bin)
sbang_style_2_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), sspec.prefix.bin)
installed_script_style_1_path = \
sspec.prefix.bin.join('sbang-style-1.sh')
assert sbang_style_1_expected == \
open(str(installed_script_style_1_path)).read()
installed_script_style_2_path = \
sspec.prefix.bin.join('sbang-style-2.sh')
assert sbang_style_2_expected == \
open(str(installed_script_style_2_path)).read()
uninstall_cmd('-y', '/%s' % sspec.dag_hash())
finally:
spack.store.store = real_store
spack.store.layout = real_layout
class DirectoryLayoutTest(MockPackagesTest):
"""Tests that a directory layout works correctly and produces a
consistent install path."""
def setUp(self):
super(DirectoryLayoutTest, self).setUp()
self.tmpdir = tempfile.mkdtemp()
self.layout = YamlDirectoryLayout(self.tmpdir)
def tearDown(self):
super(DirectoryLayoutTest, self).tearDown()
shutil.rmtree(self.tmpdir, ignore_errors=True)
self.layout = None
def test_read_and_write_spec(self):
"""This goes through each package in spack and creates a directory for
it. It then ensures that the spec for the directory's
installed package can be read back in consistently, and
finally that the directory can be removed by the directory
layout.
"""
packages = list(spack.repo.all_packages())[:max_packages]
for pkg in packages:
if pkg.name.startswith('external'):
# External package tests cannot be installed
continue
spec = pkg.spec
# If a spec fails to concretize, just skip it. If it is a
# real error, it will be caught by concretization tests.
try:
spec.concretize()
except:
continue
self.layout.create_install_directory(spec)
install_dir = self.layout.path_for_spec(spec)
spec_path = self.layout.spec_file_path(spec)
# Ensure directory has been created in right place.
self.assertTrue(os.path.isdir(install_dir))
self.assertTrue(install_dir.startswith(self.tmpdir))
# Ensure spec file exists when directory is created
self.assertTrue(os.path.isfile(spec_path))
self.assertTrue(spec_path.startswith(install_dir))
# Make sure spec file can be read back in to get the original spec
spec_from_file = self.layout.read_spec(spec_path)
# currently we don't store build dependency information when
# we write out specs to the filesystem.
# TODO: fix this when we can concretize more loosely based on
# TODO: what is installed. We currently omit these to
# TODO: increase reuse of build dependencies.
stored_deptypes = ('link', 'run')
expected = spec.copy(deps=stored_deptypes)
self.assertEqual(expected, spec_from_file)
self.assertTrue(expected.eq_dag, spec_from_file)
self.assertTrue(spec_from_file.concrete)
# Ensure that specs that come out "normal" are really normal.
with open(spec_path) as spec_file:
read_separately = Spec.from_yaml(spec_file.read())
# TODO: revise this when build deps are in dag_hash
norm = read_separately.normalized().copy(deps=stored_deptypes)
self.assertEqual(norm, spec_from_file)
# TODO: revise this when build deps are in dag_hash
conc = read_separately.concretized().copy(deps=stored_deptypes)
self.assertEqual(conc, spec_from_file)
# Make sure the hash of the read-in spec is the same
self.assertEqual(expected.dag_hash(), spec_from_file.dag_hash())
# Ensure directories are properly removed
self.layout.remove_install_directory(spec)
self.assertFalse(os.path.isdir(install_dir))
self.assertFalse(os.path.exists(install_dir))
def test_handle_unknown_package(self):
"""This test ensures that spack can at least do *some*
operations with packages that are installed but that it
does not know about. This is actually not such an uncommon
scenario with spack; it can happen when you switch from a
git branch where you're working on a new package.
This test ensures that the directory layout stores enough
information about installed packages' specs to uninstall
or query them again if the package goes away.
"""
mock_db = RepoPath(spack.mock_packages_path)
not_in_mock = set.difference(set(spack.repo.all_package_names()),
set(mock_db.all_package_names()))
packages = list(not_in_mock)[:max_packages]
# Create all the packages that are not in mock.
installed_specs = {}
for pkg_name in packages:
spec = spack.repo.get(pkg_name).spec
# If a spec fails to concretize, just skip it. If it is a
# real error, it will be caught by concretization tests.
try:
spec.concretize()
except:
continue
self.layout.create_install_directory(spec)
installed_specs[spec] = self.layout.path_for_spec(spec)
spack.repo.swap(mock_db)
# Now check that even without the package files, we know
# enough to read a spec from the spec file.
for spec, path in installed_specs.items():
spec_from_file = self.layout.read_spec(
join_path(path, '.spack', 'spec.yaml'))
# To satisfy these conditions, directory layouts need to
# read in concrete specs from their install dirs somehow.
self.assertEqual(path, self.layout.path_for_spec(spec_from_file))
self.assertEqual(spec, spec_from_file)
self.assertTrue(spec.eq_dag(spec_from_file))
self.assertEqual(spec.dag_hash(), spec_from_file.dag_hash())
spack.repo.swap(mock_db)
def test_find(self):
"""Test that finding specs within an install layout works."""
packages = list(spack.repo.all_packages())[:max_packages]
# Create install prefixes for all packages in the list
installed_specs = {}
for pkg in packages:
if pkg.name.startswith('external'):
# External package tests cannot be installed
continue
spec = pkg.spec.concretized()
installed_specs[spec.name] = spec
self.layout.create_install_directory(spec)
# Make sure all the installed specs appear in
# DirectoryLayout.all_specs()
found_specs = dict((s.name, s) for s in self.layout.all_specs())
for name, spec in found_specs.items():
self.assertTrue(name in found_specs)
self.assertTrue(found_specs[name].eq_dag(spec))
def setUp(self):
super(DirectoryLayoutTest, self).setUp()
self.tmpdir = tempfile.mkdtemp()
self.layout = YamlDirectoryLayout(self.tmpdir)
# from spack.database import Database installed_db = Database(install_path) # # Paths to built-in Spack repositories. # packages_path = join_path(repos_path, "builtin") mock_packages_path = join_path(repos_path, "builtin.mock") # # This controls how spack lays out install prefixes and # stage directories. # from spack.directory_layout import YamlDirectoryLayout install_layout = YamlDirectoryLayout(install_path) # # This controls how packages are sorted when trying to choose # the most preferred package. More preferred packages are sorted # first. # from spack.preferred_packages import PreferredPackages pkgsort = PreferredPackages() # # This tests ABI compatibility between packages # from spack.abi import ABI abi = ABI()
from spack.directory_layout import YamlExtensionsLayout
__author__ = "Benedikt Hegner (CERN)"
__all__ = ['db', 'extensions', 'layout', 'root']
#
# Read in the config
#
config = spack.config.get_config("config")
#
# Set up the install path
#
root = canonicalize_path(
config.get('install_tree', os.path.join(spack.opt_path, 'spack')))
#
# Set up the installed packages database
#
db = Database(root)
#
# This controls how spack lays out install prefixes and
# stage directories.
#
layout = YamlDirectoryLayout(root,
hash_len=config.get('install_hash_length'),
path_scheme=config.get('install_path_scheme'))
extensions = YamlExtensionsLayout(root, layout)
import spack
import spack.config
from spack.util.path import canonicalize_path
from spack.database import Database
from spack.directory_layout import YamlDirectoryLayout
__author__ = "Benedikt Hegner (CERN)"
__all__ = ['db', 'layout', 'root']
#
# Read in the config
#
config = spack.config.get_config("config")
#
# Set up the install path
#
root = canonicalize_path(
config.get('install_tree', os.path.join(spack.opt_path, 'spack')))
#
# Set up the installed packages database
#
db = Database(root)
#
# This controls how spack lays out install prefixes and
# stage directories.
#
layout = YamlDirectoryLayout(root)
