def test_conditional_dep_with_user_constraints():
"""This sets up packages X->Y such that X depends on Y conditionally. It
then constructs a Spec with X but with no constraints on X, so that the
initial normalization pass cannot determine whether the constraints are
met to add the dependency; this checks whether a user-specified constraint
on Y is applied properly.
"""
default = ('build', 'link')
y = MockPackage('y', [], [])
x_on_y_conditions = {y.name: {'x@2:': 'y'}}
x = MockPackage('x', [y], [default], conditions=x_on_y_conditions)
mock_repo = MockPackageMultiRepo([x, y])
with spack.repo.swap(mock_repo):
spec = Spec('x ^y@2')
spec.concretize()
assert ('y@2' in spec)
with spack.repo.swap(mock_repo):
spec = Spec('x@1')
spec.concretize()
assert ('y' not in spec)
with spack.repo.swap(mock_repo):
spec = Spec('x')
spec.concretize()
assert ('y@3' in spec)
def test_installed_upstream(upstream_and_downstream_db):
upstream_db, upstream_layout, downstream_db, downstream_layout = (
upstream_and_downstream_db)
default = ('build', 'link')
x = MockPackage('x', [], [])
z = MockPackage('z', [], [])
y = MockPackage('y', [z], [default])
w = MockPackage('w', [x, y], [default, default])
mock_repo = MockPackageMultiRepo([w, x, y, z])
with spack.repo.swap(mock_repo):
spec = spack.spec.Spec('w')
spec.concretize()
for dep in spec.traverse(root=False):
upstream_db.add(dep, upstream_layout)
new_spec = spack.spec.Spec('w')
new_spec.concretize()
downstream_db.add(new_spec, downstream_layout)
for dep in new_spec.traverse(root=False):
upstream, record = downstream_db.query_by_spec_hash(
dep.dag_hash())
assert upstream
assert record.path == upstream_layout.path_for_spec(dep)
upstream, record = downstream_db.query_by_spec_hash(
new_spec.dag_hash())
assert not upstream
assert record.installed
upstream_db._check_ref_counts()
downstream_db._check_ref_counts()
def test_installed_deps():
"""Preinstall a package P with a constrained build dependency D, then
concretize a dependent package which also depends on P and D, specifying
that the installed instance of P should be used. In this case, D should
not be constrained by P since P is already built.
"""
default = ('build', 'link')
build_only = ('build', )
e = MockPackage('e', [], [])
d = MockPackage('d', [], [])
c_conditions = {d.name: {'c': 'd@2'}, e.name: {'c': 'e@2'}}
c = MockPackage('c', [d, e], [build_only, default],
conditions=c_conditions)
b = MockPackage('b', [d, e], [default, default])
a = MockPackage('a', [b, c], [default, default])
mock_repo = MockPackageMultiRepo([a, b, c, d, e])
with spack.repo.swap(mock_repo):
c_spec = Spec('c')
c_spec.concretize()
assert c_spec['d'].version == spack.version.Version('2')
c_installed = spack.spec.Spec.from_dict(c_spec.to_dict())
for spec in c_installed.traverse():
setattr(spec.package, 'installed', True)
a_spec = Spec('a')
a_spec._add_dependency(c_installed, default)
a_spec.concretize()
assert a_spec['d'].version == spack.version.Version('3')
assert a_spec['e'].version == spack.version.Version('2')
def test_indirect_build_dep():
"""Simple case of X->Y->Z where Y is a build/link dep and Z is a
build-only dep. Make sure this concrete DAG is preserved when writing the
environment out and reading it back.
"""
default = ('build', 'link')
build_only = ('build', )
z = MockPackage('z', [], [])
y = MockPackage('y', [z], [build_only])
x = MockPackage('x', [y], [default])
mock_repo = MockPackageMultiRepo([x, y, z])
def noop(*args):
pass
setattr(mock_repo, 'dump_provenance', noop)
with spack.repo.swap(mock_repo):
x_spec = Spec('x')
x_concretized = x_spec.concretized()
_env_create('test', with_view=False)
e = ev.read('test')
e.add(x_spec)
e.concretize()
e.write()
e_read = ev.read('test')
x_env_hash, = e_read.concretized_order
x_env_spec = e_read.specs_by_hash[x_env_hash]
assert x_env_spec == x_concretized
def test_architecture_deep_inheritance(self):
"""Make sure that indirect dependencies receive architecture
information from the root even when partial architecture information
is provided by an intermediate dependency.
"""
saved_repo = spack.repo
default_dep = ('link', 'build')
bazpkg = MockPackage('bazpkg', [], [])
barpkg = MockPackage('barpkg', [bazpkg], [default_dep])
foopkg = MockPackage('foopkg', [barpkg], [default_dep])
mock_repo = MockPackageMultiRepo([foopkg, barpkg, bazpkg])
spack.repo = mock_repo
try:
spec = Spec('foopkg %[email protected] os=CNL target=footar' +
' ^barpkg os=SuSE11 ^bazpkg os=be')
spec.concretize()
for s in spec.traverse(root=False):
assert s.architecture.target == spec.architecture.target
finally:
spack.repo = saved_repo
def test_test_deptype():
"""Ensure that test-only dependencies are only included for specified
packages in the following spec DAG::
w
/|
x y
|
z
w->y deptypes are (link, build), w->x and y->z deptypes are (test)
"""
default = ('build', 'link')
test_only = ('test', )
x = MockPackage('x', [], [])
z = MockPackage('z', [], [])
y = MockPackage('y', [z], [test_only])
w = MockPackage('w', [x, y], [test_only, default])
mock_repo = MockPackageMultiRepo([w, x, y, z])
with spack.repo.swap(mock_repo):
spec = Spec('w')
spec.concretize(tests=(w.name, ))
assert ('x' in spec)
assert ('z' not in spec)
def test_removed_upstream_dep(upstream_and_downstream_db):
upstream_db, upstream_layout, downstream_db, downstream_layout = (
upstream_and_downstream_db)
default = ('build', 'link')
z = MockPackage('z', [], [])
y = MockPackage('y', [z], [default])
mock_repo = MockPackageMultiRepo([y, z])
with spack.repo.swap(mock_repo):
spec = spack.spec.Spec('y')
spec.concretize()
upstream_db.add(spec['z'], upstream_layout)
new_spec = spack.spec.Spec('y')
new_spec.concretize()
downstream_db.add(new_spec, downstream_layout)
upstream_db.remove(new_spec['z'])
new_downstream = spack.database.Database(
downstream_db.root, upstream_dbs=[upstream_db])
new_downstream._fail_when_missing_deps = True
with pytest.raises(spack.database.MissingDependenciesError):
new_downstream._read()
def test_read_old_lock_and_write_new(tmpdir):
build_only = ('build', )
y = MockPackage('y', [], [])
x = MockPackage('x', [y], [build_only])
mock_repo = MockPackageMultiRepo([x, y])
with spack.repo.swap(mock_repo):
x = Spec('x')
x.concretize()
y = x['y']
test_lockfile_dict = create_v1_lockfile_dict([x], [x, y])
test_lockfile_path = str(tmpdir.join('test.lock'))
with open(test_lockfile_path, 'w') as f:
sjson.dump(test_lockfile_dict, stream=f)
_env_create('test', test_lockfile_path, with_view=False)
e = ev.read('test')
hashes = set(e._to_lockfile_dict()['concrete_specs'])
# When the lockfile is rewritten, it should adopt the new hash scheme
# which accounts for all dependencies, including build dependencies
assert hashes == set([x.build_hash(), y.build_hash()])
def test_store_different_build_deps():
r"""Ensure that an environment can store two instances of a build-only
dependency::
x y
/| (l) | (b)
(b) | y z2
\| (b)
z1
"""
default = ('build', 'link')
build_only = ('build', )
z = MockPackage('z', [], [])
y = MockPackage('y', [z], [build_only])
x = MockPackage('x', [y, z], [default, build_only])
mock_repo = MockPackageMultiRepo([x, y, z])
def noop(*args):
pass
setattr(mock_repo, 'dump_provenance', noop)
with spack.repo.swap(mock_repo):
y_spec = Spec('y ^z@3')
y_concretized = y_spec.concretized()
x_spec = Spec('x ^z@2')
x_concretized = x_spec.concretized()
# Even though x chose a different 'z', it should choose the same y
# according to the DAG hash (since build deps are excluded from
# comparison by default). Although the dag hashes are equal, the specs
# are not considered equal because they compare build deps.
assert x_concretized['y'].dag_hash() == y_concretized.dag_hash()
_env_create('test', with_view=False)
e = ev.read('test')
e.add(y_spec)
e.add(x_spec)
e.concretize()
e.write()
e_read = ev.read('test')
y_env_hash, x_env_hash = e_read.concretized_order
y_read = e_read.specs_by_hash[y_env_hash]
x_read = e_read.specs_by_hash[x_env_hash]
assert x_read['z'] != y_read['z']
def test_add_to_upstream_after_downstream(upstream_and_downstream_db):
"""An upstream DB can add a package after it is installed in the downstream
DB. When a package is recorded as installed in both, the results should
refer to the downstream DB.
"""
upstream_db, upstream_layout, downstream_db, downstream_layout = (
upstream_and_downstream_db)
x = MockPackage('x', [], [])
mock_repo = MockPackageMultiRepo([x])
with spack.repo.swap(mock_repo):
spec = spack.spec.Spec('x')
spec.concretize()
downstream_db.add(spec, downstream_layout)
upstream_db.add(spec, upstream_layout)
upstream, record = downstream_db.query_by_spec_hash(spec.dag_hash())
# Even though the package is recorded as installed in the upstream DB,
# we prefer the locally-installed instance
assert not upstream
qresults = downstream_db.query('x')
assert len(qresults) == 1
queried_spec, = qresults
try:
orig_db = spack.store.db
spack.store.db = downstream_db
assert queried_spec.prefix == downstream_layout.path_for_spec(spec)
finally:
spack.store.db = orig_db
def test_save_dependency_spec_yamls_subset(tmpdir, config):
output_path = str(tmpdir.mkdir('spec_yamls'))
default = ('build', 'link')
g = MockPackage('g', [], [])
f = MockPackage('f', [], [])
e = MockPackage('e', [], [])
d = MockPackage('d', [f, g], [default, default])
c = MockPackage('c', [], [])
b = MockPackage('b', [d, e], [default, default])
a = MockPackage('a', [b, c], [default, default])
mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g])
with repo.swap(mock_repo):
spec_a = Spec('a')
spec_a.concretize()
b_spec = spec_a['b']
c_spec = spec_a['c']
spec_a_yaml = spec_a.to_yaml(hash=ht.build_hash)
save_dependency_spec_yamls(spec_a_yaml, output_path, ['b', 'c'])
assert check_specs_equal(b_spec, os.path.join(output_path, 'b.yaml'))
assert check_specs_equal(c_spec, os.path.join(output_path, 'c.yaml'))
def test_recursive_upstream_dbs(tmpdir_factory, test_store, gen_mock_layout):
roots = [str(tmpdir_factory.mktemp(x)) for x in ['a', 'b', 'c']]
layouts = [gen_mock_layout(x) for x in ['/ra/', '/rb/', '/rc/']]
default = ('build', 'link')
z = MockPackage('z', [], [])
y = MockPackage('y', [z], [default])
x = MockPackage('x', [y], [default])
mock_repo = MockPackageMultiRepo([x, y, z])
with spack.repo.swap(mock_repo):
spec = spack.spec.Spec('x')
spec.concretize()
db_c = spack.database.Database(roots[2])
db_c.add(spec['z'], layouts[2])
db_b = spack.database.Database(roots[1], upstream_dbs=[db_c])
db_b.add(spec['y'], layouts[1])
db_a = spack.database.Database(roots[0], upstream_dbs=[db_b, db_c])
db_a.add(spec['x'], layouts[0])
upstream_dbs_from_scratch = (
spack.store._construct_upstream_dbs_from_install_roots(
[roots[1], roots[2]], _test=True))
db_a_from_scratch = spack.database.Database(
roots[0], upstream_dbs=upstream_dbs_from_scratch)
assert db_a_from_scratch.db_for_spec_hash(
spec.dag_hash()) == (db_a_from_scratch)
assert db_a_from_scratch.db_for_spec_hash(
spec['y'].dag_hash()) == (upstream_dbs_from_scratch[0])
assert db_a_from_scratch.db_for_spec_hash(
spec['z'].dag_hash()) == (upstream_dbs_from_scratch[1])
db_a_from_scratch._check_ref_counts()
upstream_dbs_from_scratch[0]._check_ref_counts()
upstream_dbs_from_scratch[1]._check_ref_counts()
assert (db_a_from_scratch.installed_relatives(spec) == set(
spec.traverse(root=False)))
assert (db_a_from_scratch.installed_relatives(
spec['z'], direction='parents') == set([spec, spec['y']]))
def test_specs_staging(config):
"""Make sure we achieve the best possible staging for the following
spec DAG::
a
/|
c b
|\
e d
|\
f g
In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage,
and then 'd', 'b', and 'a' to be put in the next three stages, respectively.
"""
default = ('build', 'link')
g = MockPackage('g', [], [])
f = MockPackage('f', [], [])
e = MockPackage('e', [], [])
d = MockPackage('d', [f, g], [default, default])
c = MockPackage('c', [], [])
b = MockPackage('b', [d, e], [default, default])
a = MockPackage('a', [b, c], [default, default])
mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g])
with repo.swap(mock_repo):
spec_a = Spec('a')
spec_a.concretize()
spec_a_label = spec_deps_key_label(spec_a)[1]
spec_b_label = spec_deps_key_label(spec_a['b'])[1]
spec_c_label = spec_deps_key_label(spec_a['c'])[1]
spec_d_label = spec_deps_key_label(spec_a['d'])[1]
spec_e_label = spec_deps_key_label(spec_a['e'])[1]
spec_f_label = spec_deps_key_label(spec_a['f'])[1]
spec_g_label = spec_deps_key_label(spec_a['g'])[1]
spec_labels, dependencies, stages = stage_spec_jobs([spec_a])
assert (len(stages) == 4)
assert (len(stages[0]) == 4)
assert (spec_c_label in stages[0])
assert (spec_e_label in stages[0])
assert (spec_f_label in stages[0])
assert (spec_g_label in stages[0])
assert (len(stages[1]) == 1)
assert (spec_d_label in stages[1])
assert (len(stages[2]) == 1)
assert (spec_b_label in stages[2])
assert (len(stages[3]) == 1)
assert (spec_a_label in stages[3])
def test_specify_preinstalled_dep():
"""Specify the use of a preinstalled package during concretization with a
transitive dependency that is only supplied by the preinstalled package.
"""
default = ('build', 'link')
c = MockPackage('c', [], [])
b = MockPackage('b', [c], [default])
a = MockPackage('a', [b], [default])
mock_repo = MockPackageMultiRepo([a, b, c])
with spack.repo.swap(mock_repo):
b_spec = Spec('b')
b_spec.concretize()
for spec in b_spec.traverse():
setattr(spec.package, 'installed', True)
a_spec = Spec('a')
a_spec._add_dependency(b_spec, default)
a_spec.concretize()
assert set(x.name for x in a_spec.traverse()) == set(['a', 'b', 'c'])
def test_cannot_write_upstream(tmpdir_factory, test_store, gen_mock_layout):
roots = [str(tmpdir_factory.mktemp(x)) for x in ['a', 'b']]
layouts = [gen_mock_layout(x) for x in ['/ra/', '/rb/']]
x = MockPackage('x', [], [])
mock_repo = MockPackageMultiRepo([x])
# Instantiate the database that will be used as the upstream DB and make
# sure it has an index file
upstream_db_independent = spack.database.Database(roots[1])
with upstream_db_independent.write_transaction():
pass
upstream_dbs = spack.store._construct_upstream_dbs_from_install_roots(
[roots[1]], _test=True)
with spack.repo.swap(mock_repo):
spec = spack.spec.Spec('x')
spec.concretize()
with pytest.raises(spack.database.ForbiddenLockError):
upstream_dbs[0].add(spec, layouts[1])
def test_read_old_lock_creates_backup(tmpdir):
"""When reading a version-1 lockfile, make sure that a backup of that file
is created.
"""
y = MockPackage('y', [], [])
mock_repo = MockPackageMultiRepo([y])
with spack.repo.swap(mock_repo):
y = Spec('y')
y.concretize()
test_lockfile_dict = create_v1_lockfile_dict([y], [y])
env_root = tmpdir.mkdir('test-root')
test_lockfile_path = str(env_root.join(ev.lockfile_name))
with open(test_lockfile_path, 'w') as f:
sjson.dump(test_lockfile_dict, stream=f)
e = ev.Environment(str(env_root))
assert os.path.exists(e._lock_backup_v1_path)
with open(e._lock_backup_v1_path, 'r') as backup_v1_file:
lockfile_dict_v1 = sjson.load(backup_v1_file)
# Make sure that the backup file follows the v1 hash scheme
assert y.dag_hash() in lockfile_dict_v1['concrete_specs']
def test_specs_staging(config):
"""Make sure we achieve the best possible staging for the following
spec DAG::
a
/|
c b
|\
e d
|\
f g
In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage,
and then 'd', 'b', and 'a' to be put in the next three stages, respectively.
"""
current_system = sys_type()
config_compilers = config.get_config('compilers')
first_compiler = config_compilers[0]
compiler_spec = first_compiler['compiler']['spec']
# Whatever that first compiler in the configuration was, let's make sure
# we mock up an entry like we'd find in os-container-mapping.yaml which
# has that compiler.
mock_containers = {}
mock_containers[current_system] = {
"image": "dontcare",
"compilers": [{
"name": compiler_spec,
}],
}
default = ('build', 'link')
g = MockPackage('g', [], [])
f = MockPackage('f', [], [])
e = MockPackage('e', [], [])
d = MockPackage('d', [f, g], [default, default])
c = MockPackage('c', [], [])
b = MockPackage('b', [d, e], [default, default])
a = MockPackage('a', [b, c], [default, default])
mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g])
with repo.swap(mock_repo):
# Now we'll ask for the root package to be compiled with whatever that
# first compiler in the configuration was.
spec_a = Spec('a%{0}'.format(compiler_spec))
spec_a.concretize()
spec_a_label = spec_deps_key_label(spec_a)[1]
spec_b_label = spec_deps_key_label(spec_a['b'])[1]
spec_c_label = spec_deps_key_label(spec_a['c'])[1]
spec_d_label = spec_deps_key_label(spec_a['d'])[1]
spec_e_label = spec_deps_key_label(spec_a['e'])[1]
spec_f_label = spec_deps_key_label(spec_a['f'])[1]
spec_g_label = spec_deps_key_label(spec_a['g'])[1]
spec_labels, dependencies, stages = stage_spec_jobs([spec_a],
mock_containers,
current_system)
assert (len(stages) == 4)
assert (len(stages[0]) == 4)
assert (spec_c_label in stages[0])
assert (spec_e_label in stages[0])
assert (spec_f_label in stages[0])
assert (spec_g_label in stages[0])
assert (len(stages[1]) == 1)
assert (spec_d_label in stages[1])
assert (len(stages[2]) == 1)
assert (spec_b_label in stages[2])
assert (len(stages[3]) == 1)
assert (spec_a_label in stages[3])
