def layout_and_dir(tmpdir):
"""Returns a directory layout and the corresponding directory."""
layout = YamlDirectoryLayout(str(tmpdir))
old_layout = spack.store.layout
spack.store.layout = layout
yield layout, str(tmpdir)
spack.store.layout = old_layout
def test_python_ignore_namespace_init_conflict(tmpdir, namespace_extensions,
builtin_and_mock_packages):
"""Test the view update logic in PythonPackage ignores conflicting
instances of __init__ for packages which are in the same namespace.
"""
ext1_prefix, ext2_prefix, py_namespace = namespace_extensions
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
py_namespace)
ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
py_namespace)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext1_pkg, view)
# Normally handled by Package.do_activate, but here we activate directly
view.extensions_layout.add_extension(python_spec, ext1_pkg.spec)
python_pkg.activate(ext2_pkg, view)
f1 = 'lib/python2.7/site-packages/examplenamespace/ext1_sample.py'
f2 = 'lib/python2.7/site-packages/examplenamespace/ext2_sample.py'
init_file = 'lib/python2.7/site-packages/examplenamespace/__init__.py'
assert os.path.exists(os.path.join(view_dir, f1))
assert os.path.exists(os.path.join(view_dir, f2))
assert os.path.exists(os.path.join(view_dir, init_file))
def test_python_namespace_conflict(tmpdir, namespace_extensions,
builtin_and_mock_packages):
"""Test the view update logic in PythonPackage reports an error when two
python extensions with different namespaces have a conflicting __init__
file.
"""
ext1_prefix, ext2_prefix, py_namespace = namespace_extensions
other_namespace = py_namespace + 'other'
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
py_namespace)
ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
other_namespace)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext1_pkg, view)
view.extensions_layout.add_extension(python_spec, ext1_pkg.spec)
with pytest.raises(MergeConflictError):
python_pkg.activate(ext2_pkg, view)
def test_yaml_directory_layout_build_path(tmpdir, config):
"""This tests build path method."""
spec = Spec('python')
spec.concretize()
layout = YamlDirectoryLayout(str(tmpdir))
rel_path = os.path.join(layout.metadata_dir, layout.packages_dir)
assert layout.build_packages_path(spec) == os.path.join(
spec.prefix, rel_path)
def install_dir_non_default_layout(tmpdir):
"""Hooks a fake install directory with a non-default layout"""
real_store = spack.store.store
real_layout = spack.store.layout
spack.store.store = spack.store.Store(str(tmpdir.join('opt')))
spack.store.layout = YamlDirectoryLayout(str(tmpdir.join('opt')),
path_scheme=ndef_install_path_scheme) # noqa: E501
yield spack.store
spack.store.store = real_store
spack.store.layout = real_layout
def test_remove_extensions_ordered(install_mockery, mock_fetch, tmpdir):
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
e2 = Spec('extension2').concretized()
e2.package.do_install()
view.add_specs(e2)
e1 = e2['extension1']
view.remove_specs(e1, e2)
def test_yaml_directory_layout_parameters(
tmpdir, config
):
"""This tests the various parameters that can be used to configure
the install location """
spec = Spec('python')
spec.concretize()
# Ensure default layout matches expected spec format
layout_default = YamlDirectoryLayout(str(tmpdir))
path_default = layout_default.relative_path_for_spec(spec)
assert(path_default == spec.format(
"${ARCHITECTURE}/"
"${COMPILERNAME}-${COMPILERVER}/"
"${PACKAGE}-${VERSION}-${HASH}"))
# Test hash_length parameter works correctly
layout_10 = YamlDirectoryLayout(str(tmpdir), hash_len=10)
path_10 = layout_10.relative_path_for_spec(spec)
layout_7 = YamlDirectoryLayout(str(tmpdir), hash_len=7)
path_7 = layout_7.relative_path_for_spec(spec)
assert(len(path_default) - len(path_10) == 22)
assert(len(path_default) - len(path_7) == 25)
# Test path_scheme
arch, compiler, package7 = path_7.split('/')
scheme_package7 = "${PACKAGE}-${VERSION}-${HASH:7}"
layout_package7 = YamlDirectoryLayout(str(tmpdir),
path_scheme=scheme_package7)
path_package7 = layout_package7.relative_path_for_spec(spec)
assert(package7 == path_package7)
# Ensure conflicting parameters caught
with pytest.raises(InvalidDirectoryLayoutParametersError):
YamlDirectoryLayout(str(tmpdir),
hash_len=20,
path_scheme=scheme_package7)
def install_dir_non_default_layout(tmpdir):
"""Hooks a fake install directory with a non-default layout"""
scheme = os.path.join(
'${name}', '${version}',
'${architecture}-${compiler.name}-${compiler.version}-${hash}')
real_store, real_layout = spack.store.store, spack.store.layout
opt_dir = tmpdir.join('opt')
spack.store.store = spack.store.Store(str(opt_dir))
spack.store.layout = YamlDirectoryLayout(str(opt_dir), path_scheme=scheme)
try:
yield spack.store
finally:
spack.store.store = real_store
spack.store.layout = real_layout
def setUp(self):
super(InstallTest, self).setUp()
# create a simple installable package directory and tarball
self.repo = MockArchive()
# We use a fake package, so skip the checksum.
spack.do_checksum = False
# Use a fake install directory to avoid conflicts bt/w
# installed pkgs and mock packages.
self.tmpdir = tempfile.mkdtemp()
self.orig_layout = spack.install_layout
spack.install_layout = YamlDirectoryLayout(self.tmpdir)
def test_yaml_directory_layout_parameters(tmpdir, config):
"""This tests the various parameters that can be used to configure
the install location """
spec = Spec('python')
spec.concretize()
# Ensure default layout matches expected spec format
layout_default = YamlDirectoryLayout(str(tmpdir))
path_default = layout_default.relative_path_for_spec(spec)
assert(path_default == spec.format(
"{architecture}/"
"{compiler.name}-{compiler.version}/"
"{name}-{version}-{hash}"))
# Test hash_length parameter works correctly
layout_10 = YamlDirectoryLayout(str(tmpdir), hash_len=10)
path_10 = layout_10.relative_path_for_spec(spec)
layout_7 = YamlDirectoryLayout(str(tmpdir), hash_len=7)
path_7 = layout_7.relative_path_for_spec(spec)
assert(len(path_default) - len(path_10) == 22)
assert(len(path_default) - len(path_7) == 25)
# Test path_scheme
arch, compiler, package7 = path_7.split('/')
scheme_package7 = "{name}-{version}-{hash:7}"
layout_package7 = YamlDirectoryLayout(str(tmpdir),
path_scheme=scheme_package7)
path_package7 = layout_package7.relative_path_for_spec(spec)
assert(package7 == path_package7)
# Test separation of architecture
arch_scheme_package = "{architecture.platform}/{architecture.target}/{architecture.os}/{name}/{version}/{hash:7}" # NOQA: ignore=E501
layout_arch_package = YamlDirectoryLayout(str(tmpdir),
path_scheme=arch_scheme_package)
arch_path_package = layout_arch_package.relative_path_for_spec(spec)
assert(arch_path_package == spec.format(arch_scheme_package))
# Test separation of namespace
ns_scheme_package = "${ARCHITECTURE}/${NAMESPACE}/${PACKAGE}-${VERSION}-${HASH:7}" # NOQA: ignore=E501
layout_ns_package = YamlDirectoryLayout(str(tmpdir),
path_scheme=ns_scheme_package)
ns_path_package = layout_ns_package.relative_path_for_spec(spec)
assert(ns_path_package == spec.format(ns_scheme_package))
# Ensure conflicting parameters caught
with pytest.raises(InvalidDirectoryLayoutParametersError):
YamlDirectoryLayout(str(tmpdir),
hash_len=20,
path_scheme=scheme_package7)
def setUp(self):
super(MockDatabase, self).setUp()
#
# TODO: make the mockup below easier.
#
# Make a fake install directory
self.install_path = tempfile.mkdtemp()
self.spack_install_path = spack.install_path
spack.install_path = self.install_path
self.install_layout = YamlDirectoryLayout(self.install_path)
self.spack_install_layout = spack.install_layout
spack.install_layout = self.install_layout
# Make fake database and fake install directory.
self.installed_db = Database(self.install_path)
self.spack_installed_db = spack.installed_db
spack.installed_db = self.installed_db
# make a mock database with some packages installed note that
# the ref count for dyninst here will be 3, as it's recycled
# across each install.
#
# Here is what the mock DB looks like:
#
# o mpileaks o mpileaks' o mpileaks''
# |\ |\ |\
# | o callpath | o callpath' | o callpath''
# |/| |/| |/|
# o | mpich o | mpich2 o | zmpi
# | | o | fake
# | | |
# | |______________/
# | .____________/
# |/
# o dyninst
# |\
# | o libdwarf
# |/
# o libelf
#
# Transaction used to avoid repeated writes.
with spack.installed_db.write_transaction():
self._mock_install('mpileaks ^mpich')
self._mock_install('mpileaks ^mpich2')
self._mock_install('mpileaks ^zmpi')
def install_mockery(tmpdir, config, builtin_mock):
"""Hooks a fake install directory and a fake db into Spack."""
layout = spack.store.layout
db = spack.store.db
# Use a fake install directory to avoid conflicts bt/w
# installed pkgs and mock packages.
spack.store.layout = YamlDirectoryLayout(str(tmpdir))
spack.store.db = Database(str(tmpdir))
# We use a fake package, so skip the checksum.
spack.do_checksum = False
yield
# Turn checksumming back on
spack.do_checksum = True
# Restore Spack's layout.
spack.store.layout = layout
spack.store.db = db
def test_perl_activation_view(tmpdir, perl_and_extension_dirs):
perl_prefix, ext_prefix = perl_and_extension_dirs
perl_spec = spack.spec.Spec('[email protected]')
perl_spec._concrete = True
perl_spec.package.spec.prefix = perl_prefix
ext_pkg = FakeExtensionPackage('perl-extension', ext_prefix)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
perl_pkg = perl_spec.package
perl_pkg.activate(ext_pkg, extensions_layout=view.extensions_layout)
assert not os.path.exists(join_path(perl_prefix, 'bin/perl-ext-tool'))
assert os.path.exists(join_path(view_dir, 'bin/perl-ext-tool'))
def test_python_activation_view(tmpdir, python_and_extension_dirs):
python_prefix, ext_prefix = python_and_extension_dirs
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
python_spec.package.spec._set_test_prefix(python_prefix)
ext_pkg = FakeExtensionPackage('py-extension', ext_prefix)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext_pkg, extensions_layout=view.extensions_layout)
assert not os.path.exists(join_path(python_prefix, 'bin/py-ext-tool'))
assert os.path.exists(join_path(view_dir, 'bin/py-ext-tool'))
def test_perl_activation_view(tmpdir, perl_and_extension_dirs,
builtin_and_mock_packages):
perl_prefix, ext_prefix = perl_and_extension_dirs
perl_spec = spack.spec.Spec('[email protected]')
perl_spec._concrete = True
perl_spec.package.spec.prefix = perl_prefix
ext_pkg = create_ext_pkg('perl-extension', ext_prefix, perl_spec)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
perl_pkg = perl_spec.package
perl_pkg.activate(ext_pkg, view)
assert not os.path.exists(os.path.join(perl_prefix, 'bin/perl-ext-tool'))
assert os.path.exists(os.path.join(view_dir, 'bin/perl-ext-tool'))
def test_python_activation_view(tmpdir, python_and_extension_dirs,
builtin_and_mock_packages, monkeypatch):
python_prefix, ext_prefix = python_and_extension_dirs
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
python_spec.package.spec.prefix = python_prefix
ext_pkg = create_python_ext_pkg('py-extension1', ext_prefix, python_spec,
monkeypatch)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext_pkg, view)
assert not os.path.exists(os.path.join(python_prefix, 'bin/py-ext-tool'))
assert os.path.exists(os.path.join(view_dir, 'bin/py-ext-tool'))
def test_python_keep_namespace_init(tmpdir, namespace_extensions,
builtin_and_mock_packages):
"""Test the view update logic in PythonPackage keeps the namespace
__init__ file as long as one package in the namespace still
exists.
"""
ext1_prefix, ext2_prefix, py_namespace = namespace_extensions
python_spec = spack.spec.Spec('[email protected]')
python_spec._concrete = True
ext1_pkg = create_python_ext_pkg('py-extension1', ext1_prefix, python_spec,
py_namespace)
ext2_pkg = create_python_ext_pkg('py-extension2', ext2_prefix, python_spec,
py_namespace)
view_dir = str(tmpdir.join('view'))
layout = YamlDirectoryLayout(view_dir)
view = YamlFilesystemView(view_dir, layout)
python_pkg = python_spec.package
python_pkg.activate(ext1_pkg, view)
# Normally handled by Package.do_activate, but here we activate directly
view.extensions_layout.add_extension(python_spec, ext1_pkg.spec)
python_pkg.activate(ext2_pkg, view)
view.extensions_layout.add_extension(python_spec, ext2_pkg.spec)
f1 = 'lib/python2.7/site-packages/examplenamespace/ext1_sample.py'
init_file = 'lib/python2.7/site-packages/examplenamespace/__init__.py'
python_pkg.deactivate(ext1_pkg, view)
view.extensions_layout.remove_extension(python_spec, ext1_pkg.spec)
assert not os.path.exists(os.path.join(view_dir, f1))
assert os.path.exists(os.path.join(view_dir, init_file))
python_pkg.deactivate(ext2_pkg, view)
view.extensions_layout.remove_extension(python_spec, ext2_pkg.spec)
assert not os.path.exists(os.path.join(view_dir, init_file))
def layout_and_dir(tmpdir):
"""Returns a directory layout and the corresponding directory."""
yield YamlDirectoryLayout(str(tmpdir)), str(tmpdir)
def test_update_sbang(tmpdir, test_mirror):
"""Test the creation and installation of buildcaches with default rpaths
into the non-default directory layout scheme, triggering an update of the
sbang.
"""
scheme = os.path.join(
'${name}', '${version}',
'${architecture}-${compiler.name}-${compiler.version}-${hash}')
spec_str = 'old-sbang'
# Concretize a package with some old-fashioned sbang lines.
old_spec = Spec(spec_str).concretized()
old_spec_hash_str = '/{0}'.format(old_spec.dag_hash())
# Need a fake mirror with *function* scope.
mirror_dir = test_mirror
mirror_url = 'file://{0}'.format(mirror_dir)
# Assume all commands will concretize old_spec the same way.
install_cmd('--no-cache', old_spec.name)
# Create a buildcache with the installed spec.
buildcache_cmd('create', '-u', '-a', '-d', mirror_dir, old_spec_hash_str)
# Need to force an update of the buildcache index
buildcache_cmd('update-index', '-d', mirror_url)
# Uninstall the original package.
uninstall_cmd('-y', old_spec_hash_str)
# Switch the store to the new install tree locations
newtree_dir = tmpdir.join('newtree')
s = spack.store.Store(str(newtree_dir))
s.layout = YamlDirectoryLayout(str(newtree_dir), path_scheme=scheme)
with spack.store.use_store(s):
new_spec = Spec('old-sbang')
new_spec.concretize()
assert new_spec.dag_hash() == old_spec.dag_hash()
# Install package from buildcache
buildcache_cmd('install', '-a', '-u', '-f', new_spec.name)
# Continue blowing away caches
bindist.clear_spec_cache()
spack.stage.purge()
# test that the sbang was updated by the move
sbang_style_1_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin)
sbang_style_2_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin)
installed_script_style_1_path = new_spec.prefix.bin.join(
'sbang-style-1.sh')
assert sbang_style_1_expected == \
open(str(installed_script_style_1_path)).read()
installed_script_style_2_path = new_spec.prefix.bin.join(
'sbang-style-2.sh')
assert sbang_style_2_expected == \
open(str(installed_script_style_2_path)).read()
uninstall_cmd('-y', '/%s' % new_spec.dag_hash())
def test_update_sbang(tmpdir, install_mockery, function_mirror):
"""
Test the creation and installation of buildcaches with default rpaths
into the non-default directory layout scheme, triggering an update of the
sbang.
"""
# Save the original store and layout before we touch ANYTHING.
real_store = spack.store.store
real_layout = spack.store.layout
# Concretize a package with some old-fashioned sbang lines.
sspec = Spec('old-sbang')
sspec.concretize()
# Need a fake mirror with *function* scope.
mirror_dir = function_mirror
# Assumes all commands will concretize sspec the same way.
install_cmd('--no-cache', sspec.name)
# Create a buildcache with the installed spec.
buildcache_cmd('create', '-u', '-a', '-d', mirror_dir,
'/%s' % sspec.dag_hash())
# Need to force an update of the buildcache index
buildcache_cmd('update-index', '-d', 'file://%s' % mirror_dir)
# Uninstall the original package.
uninstall_cmd('-y', '/%s' % sspec.dag_hash())
try:
# New install tree locations...
# Too fine-grained to do be done in a fixture
spack.store.store = spack.store.Store(str(tmpdir.join('newtree')))
spack.store.layout = YamlDirectoryLayout(
str(tmpdir.join('newtree')),
path_scheme=ndef_install_path_scheme) # noqa: E501
# Install package from buildcache
buildcache_cmd('install', '-a', '-u', '-f', sspec.name)
# Continue blowing away caches
bindist.clear_spec_cache()
spack.stage.purge()
# test that the sbang was updated by the move
sbang_style_1_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), sspec.prefix.bin)
sbang_style_2_expected = '''{0}
#!/usr/bin/env python
{1}
'''.format(sbang.sbang_shebang_line(), sspec.prefix.bin)
installed_script_style_1_path = \
sspec.prefix.bin.join('sbang-style-1.sh')
assert sbang_style_1_expected == \
open(str(installed_script_style_1_path)).read()
installed_script_style_2_path = \
sspec.prefix.bin.join('sbang-style-2.sh')
assert sbang_style_2_expected == \
open(str(installed_script_style_2_path)).read()
uninstall_cmd('-y', '/%s' % sspec.dag_hash())
finally:
spack.store.store = real_store
spack.store.layout = real_layout
# from spack.database import Database installed_db = Database(install_path) # # Paths to built-in Spack repositories. # packages_path = join_path(repos_path, "builtin") mock_packages_path = join_path(repos_path, "builtin.mock") # # This controls how spack lays out install prefixes and # stage directories. # from spack.directory_layout import YamlDirectoryLayout install_layout = YamlDirectoryLayout(install_path) # # This controls how packages are sorted when trying to choose # the most preferred package. More preferred packages are sorted # first. # from spack.preferred_packages import PreferredPackages pkgsort = PreferredPackages() # # This tests ABI compatibility between packages # from spack.abi import ABI abi = ABI()
def setUp(self):
super(DirectoryLayoutTest, self).setUp()
self.tmpdir = tempfile.mkdtemp()
self.layout = YamlDirectoryLayout(self.tmpdir)
from spack.directory_layout import YamlExtensionsLayout
__author__ = "Benedikt Hegner (CERN)"
__all__ = ['db', 'extensions', 'layout', 'root']
#
# Read in the config
#
config = spack.config.get_config("config")
#
# Set up the install path
#
root = canonicalize_path(
config.get('install_tree', os.path.join(spack.opt_path, 'spack')))
#
# Set up the installed packages database
#
db = Database(root)
#
# This controls how spack lays out install prefixes and
# stage directories.
#
layout = YamlDirectoryLayout(root,
hash_len=config.get('install_hash_length'),
path_scheme=config.get('install_path_scheme'))
extensions = YamlExtensionsLayout(root, layout)
import spack
import spack.config
from spack.util.path import canonicalize_path
from spack.database import Database
from spack.directory_layout import YamlDirectoryLayout
__author__ = "Benedikt Hegner (CERN)"
__all__ = ['db', 'layout', 'root']
#
# Read in the config
#
config = spack.config.get_config("config")
#
# Set up the install path
#
root = canonicalize_path(
config.get('install_tree', os.path.join(spack.opt_path, 'spack')))
#
# Set up the installed packages database
#
db = Database(root)
#
# This controls how spack lays out install prefixes and
# stage directories.
#
layout = YamlDirectoryLayout(root)
