def test_installed_upstream(upstream_and_downstream_db): upstream_db, upstream_layout, downstream_db, downstream_layout = ( upstream_and_downstream_db) default = ('build', 'link') x = MockPackage('x', [], []) z = MockPackage('z', [], []) y = MockPackage('y', [z], [default]) w = MockPackage('w', [x, y], [default, default]) mock_repo = MockPackageMultiRepo([w, x, y, z]) with spack.repo.swap(mock_repo): spec = spack.spec.Spec('w') spec.concretize() for dep in spec.traverse(root=False): upstream_db.add(dep, upstream_layout) new_spec = spack.spec.Spec('w') new_spec.concretize() downstream_db.add(new_spec, downstream_layout) for dep in new_spec.traverse(root=False): upstream, record = downstream_db.query_by_spec_hash( dep.dag_hash()) assert upstream assert record.path == upstream_layout.path_for_spec(dep) upstream, record = downstream_db.query_by_spec_hash( new_spec.dag_hash()) assert not upstream assert record.installed upstream_db._check_ref_counts() downstream_db._check_ref_counts()
def test_save_dependency_spec_yamls_subset(tmpdir, config): output_path = str(tmpdir.mkdir('spec_yamls')) default = ('build', 'link') g = MockPackage('g', [], []) f = MockPackage('f', [], []) e = MockPackage('e', [], []) d = MockPackage('d', [f, g], [default, default]) c = MockPackage('c', [], []) b = MockPackage('b', [d, e], [default, default]) a = MockPackage('a', [b, c], [default, default]) mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g]) with repo.swap(mock_repo): spec_a = Spec('a') spec_a.concretize() b_spec = spec_a['b'] c_spec = spec_a['c'] spec_a_yaml = spec_a.to_yaml(hash=ht.build_hash) save_dependency_spec_yamls(spec_a_yaml, output_path, ['b', 'c']) assert check_specs_equal(b_spec, os.path.join(output_path, 'b.yaml')) assert check_specs_equal(c_spec, os.path.join(output_path, 'c.yaml'))
def test_conditional_dep_with_user_constraints(): """This sets up packages X->Y such that X depends on Y conditionally. It then constructs a Spec with X but with no constraints on X, so that the initial normalization pass cannot determine whether the constraints are met to add the dependency; this checks whether a user-specified constraint on Y is applied properly. """ default = ('build', 'link') y = MockPackage('y', [], []) x_on_y_conditions = {y.name: {'x@2:': 'y'}} x = MockPackage('x', [y], [default], conditions=x_on_y_conditions) mock_repo = MockPackageMultiRepo([x, y]) with spack.repo.swap(mock_repo): spec = Spec('x ^y@2') spec.concretize() assert ('y@2' in spec) with spack.repo.swap(mock_repo): spec = Spec('x@1') spec.concretize() assert ('y' not in spec) with spack.repo.swap(mock_repo): spec = Spec('x') spec.concretize() assert ('y@3' in spec)
def test_read_old_lock_and_write_new(tmpdir): build_only = ('build', ) y = MockPackage('y', [], []) x = MockPackage('x', [y], [build_only]) mock_repo = MockPackageMultiRepo([x, y]) with spack.repo.swap(mock_repo): x = Spec('x') x.concretize() y = x['y'] test_lockfile_dict = create_v1_lockfile_dict([x], [x, y]) test_lockfile_path = str(tmpdir.join('test.lock')) with open(test_lockfile_path, 'w') as f: sjson.dump(test_lockfile_dict, stream=f) _env_create('test', test_lockfile_path, with_view=False) e = ev.read('test') hashes = set(e._to_lockfile_dict()['concrete_specs']) # When the lockfile is rewritten, it should adopt the new hash scheme # which accounts for all dependencies, including build dependencies assert hashes == set([x.build_hash(), y.build_hash()])
def test_installed_deps(): """Preinstall a package P with a constrained build dependency D, then concretize a dependent package which also depends on P and D, specifying that the installed instance of P should be used. In this case, D should not be constrained by P since P is already built. """ default = ('build', 'link') build_only = ('build', ) e = MockPackage('e', [], []) d = MockPackage('d', [], []) c_conditions = {d.name: {'c': 'd@2'}, e.name: {'c': 'e@2'}} c = MockPackage('c', [d, e], [build_only, default], conditions=c_conditions) b = MockPackage('b', [d, e], [default, default]) a = MockPackage('a', [b, c], [default, default]) mock_repo = MockPackageMultiRepo([a, b, c, d, e]) with spack.repo.swap(mock_repo): c_spec = Spec('c') c_spec.concretize() assert c_spec['d'].version == spack.version.Version('2') c_installed = spack.spec.Spec.from_dict(c_spec.to_dict()) for spec in c_installed.traverse(): setattr(spec.package, 'installed', True) a_spec = Spec('a') a_spec._add_dependency(c_installed, default) a_spec.concretize() assert a_spec['d'].version == spack.version.Version('3') assert a_spec['e'].version == spack.version.Version('2')
def test_test_deptype(): """Ensure that test-only dependencies are only included for specified packages in the following spec DAG:: w /| x y | z w->y deptypes are (link, build), w->x and y->z deptypes are (test) """ default = ('build', 'link') test_only = ('test', ) x = MockPackage('x', [], []) z = MockPackage('z', [], []) y = MockPackage('y', [z], [test_only]) w = MockPackage('w', [x, y], [test_only, default]) mock_repo = MockPackageMultiRepo([w, x, y, z]) with spack.repo.swap(mock_repo): spec = Spec('w') spec.concretize(tests=(w.name, )) assert ('x' in spec) assert ('z' not in spec)
def test_add_to_upstream_after_downstream(upstream_and_downstream_db): """An upstream DB can add a package after it is installed in the downstream DB. When a package is recorded as installed in both, the results should refer to the downstream DB. """ upstream_db, upstream_layout, downstream_db, downstream_layout = ( upstream_and_downstream_db) x = MockPackage('x', [], []) mock_repo = MockPackageMultiRepo([x]) with spack.repo.swap(mock_repo): spec = spack.spec.Spec('x') spec.concretize() downstream_db.add(spec, downstream_layout) upstream_db.add(spec, upstream_layout) upstream, record = downstream_db.query_by_spec_hash(spec.dag_hash()) # Even though the package is recorded as installed in the upstream DB, # we prefer the locally-installed instance assert not upstream qresults = downstream_db.query('x') assert len(qresults) == 1 queried_spec, = qresults try: orig_db = spack.store.db spack.store.db = downstream_db assert queried_spec.prefix == downstream_layout.path_for_spec(spec) finally: spack.store.db = orig_db
def test_115_reindex_with_packages_not_in_repo(mutable_database): # Dont add any package definitions to this repository, the idea is that # packages should not have to be defined in the repository once they # are installed with spack.repo.swap(MockPackageMultiRepo([])): spack.store.store.reindex() _check_db_sanity(mutable_database)
def test_architecture_deep_inheritance(self): """Make sure that indirect dependencies receive architecture information from the root even when partial architecture information is provided by an intermediate dependency. """ saved_repo = spack.repo default_dep = ('link', 'build') bazpkg = MockPackage('bazpkg', [], []) barpkg = MockPackage('barpkg', [bazpkg], [default_dep]) foopkg = MockPackage('foopkg', [barpkg], [default_dep]) mock_repo = MockPackageMultiRepo([foopkg, barpkg, bazpkg]) spack.repo = mock_repo try: spec = Spec('foopkg %[email protected] os=CNL target=footar' + ' ^barpkg os=SuSE11 ^bazpkg os=be') spec.concretize() for s in spec.traverse(root=False): assert s.architecture.target == spec.architecture.target finally: spack.repo = saved_repo
def test_indirect_build_dep(): """Simple case of X->Y->Z where Y is a build/link dep and Z is a build-only dep. Make sure this concrete DAG is preserved when writing the environment out and reading it back. """ default = ('build', 'link') build_only = ('build', ) z = MockPackage('z', [], []) y = MockPackage('y', [z], [build_only]) x = MockPackage('x', [y], [default]) mock_repo = MockPackageMultiRepo([x, y, z]) def noop(*args): pass setattr(mock_repo, 'dump_provenance', noop) with spack.repo.swap(mock_repo): x_spec = Spec('x') x_concretized = x_spec.concretized() _env_create('test', with_view=False) e = ev.read('test') e.add(x_spec) e.concretize() e.write() e_read = ev.read('test') x_env_hash, = e_read.concretized_order x_env_spec = e_read.specs_by_hash[x_env_hash] assert x_env_spec == x_concretized
def test_removed_upstream_dep(upstream_and_downstream_db): upstream_db, upstream_layout, downstream_db, downstream_layout = ( upstream_and_downstream_db) default = ('build', 'link') z = MockPackage('z', [], []) y = MockPackage('y', [z], [default]) mock_repo = MockPackageMultiRepo([y, z]) with spack.repo.swap(mock_repo): spec = spack.spec.Spec('y') spec.concretize() upstream_db.add(spec['z'], upstream_layout) new_spec = spack.spec.Spec('y') new_spec.concretize() downstream_db.add(new_spec, downstream_layout) upstream_db.remove(new_spec['z']) new_downstream = spack.database.Database( downstream_db.root, upstream_dbs=[upstream_db]) new_downstream._fail_when_missing_deps = True with pytest.raises(spack.database.MissingDependenciesError): new_downstream._read()
def test_specs_staging(config): """Make sure we achieve the best possible staging for the following spec DAG:: a /| c b |\ e d |\ f g In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage, and then 'd', 'b', and 'a' to be put in the next three stages, respectively. """ default = ('build', 'link') g = MockPackage('g', [], []) f = MockPackage('f', [], []) e = MockPackage('e', [], []) d = MockPackage('d', [f, g], [default, default]) c = MockPackage('c', [], []) b = MockPackage('b', [d, e], [default, default]) a = MockPackage('a', [b, c], [default, default]) mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g]) with repo.swap(mock_repo): spec_a = Spec('a') spec_a.concretize() spec_a_label = spec_deps_key_label(spec_a)[1] spec_b_label = spec_deps_key_label(spec_a['b'])[1] spec_c_label = spec_deps_key_label(spec_a['c'])[1] spec_d_label = spec_deps_key_label(spec_a['d'])[1] spec_e_label = spec_deps_key_label(spec_a['e'])[1] spec_f_label = spec_deps_key_label(spec_a['f'])[1] spec_g_label = spec_deps_key_label(spec_a['g'])[1] spec_labels, dependencies, stages = stage_spec_jobs([spec_a]) assert (len(stages) == 4) assert (len(stages[0]) == 4) assert (spec_c_label in stages[0]) assert (spec_e_label in stages[0]) assert (spec_f_label in stages[0]) assert (spec_g_label in stages[0]) assert (len(stages[1]) == 1) assert (spec_d_label in stages[1]) assert (len(stages[2]) == 1) assert (spec_b_label in stages[2]) assert (len(stages[3]) == 1) assert (spec_a_label in stages[3])
def test_store_different_build_deps(): r"""Ensure that an environment can store two instances of a build-only dependency:: x y /| (l) | (b) (b) | y z2 \| (b) z1 """ default = ('build', 'link') build_only = ('build', ) z = MockPackage('z', [], []) y = MockPackage('y', [z], [build_only]) x = MockPackage('x', [y, z], [default, build_only]) mock_repo = MockPackageMultiRepo([x, y, z]) def noop(*args): pass setattr(mock_repo, 'dump_provenance', noop) with spack.repo.swap(mock_repo): y_spec = Spec('y ^z@3') y_concretized = y_spec.concretized() x_spec = Spec('x ^z@2') x_concretized = x_spec.concretized() # Even though x chose a different 'z', it should choose the same y # according to the DAG hash (since build deps are excluded from # comparison by default). Although the dag hashes are equal, the specs # are not considered equal because they compare build deps. assert x_concretized['y'].dag_hash() == y_concretized.dag_hash() _env_create('test', with_view=False) e = ev.read('test') e.add(y_spec) e.add(x_spec) e.concretize() e.write() e_read = ev.read('test') y_env_hash, x_env_hash = e_read.concretized_order y_read = e_read.specs_by_hash[y_env_hash] x_read = e_read.specs_by_hash[x_env_hash] assert x_read['z'] != y_read['z']
def test_115_reindex_with_packages_not_in_repo(database, refresh_db_on_exit): install_db = database.mock.db saved_repo = spack.repo # Dont add any package definitions to this repository, the idea is that # packages should not have to be defined in the repository once they # are installed mock_repo = MockPackageMultiRepo([]) try: spack.repo = mock_repo spack.store.db.reindex(spack.store.layout) _check_db_sanity(install_db) finally: spack.repo = saved_repo
def test_recursive_upstream_dbs(tmpdir_factory, test_store, gen_mock_layout): roots = [str(tmpdir_factory.mktemp(x)) for x in ['a', 'b', 'c']] layouts = [gen_mock_layout(x) for x in ['/ra/', '/rb/', '/rc/']] default = ('build', 'link') z = MockPackage('z', [], []) y = MockPackage('y', [z], [default]) x = MockPackage('x', [y], [default]) mock_repo = MockPackageMultiRepo([x, y, z]) with spack.repo.swap(mock_repo): spec = spack.spec.Spec('x') spec.concretize() db_c = spack.database.Database(roots[2]) db_c.add(spec['z'], layouts[2]) db_b = spack.database.Database(roots[1], upstream_dbs=[db_c]) db_b.add(spec['y'], layouts[1]) db_a = spack.database.Database(roots[0], upstream_dbs=[db_b, db_c]) db_a.add(spec['x'], layouts[0]) upstream_dbs_from_scratch = ( spack.store._construct_upstream_dbs_from_install_roots( [roots[1], roots[2]], _test=True)) db_a_from_scratch = spack.database.Database( roots[0], upstream_dbs=upstream_dbs_from_scratch) assert db_a_from_scratch.db_for_spec_hash( spec.dag_hash()) == (db_a_from_scratch) assert db_a_from_scratch.db_for_spec_hash( spec['y'].dag_hash()) == (upstream_dbs_from_scratch[0]) assert db_a_from_scratch.db_for_spec_hash( spec['z'].dag_hash()) == (upstream_dbs_from_scratch[1]) db_a_from_scratch._check_ref_counts() upstream_dbs_from_scratch[0]._check_ref_counts() upstream_dbs_from_scratch[1]._check_ref_counts() assert (db_a_from_scratch.installed_relatives(spec) == set( spec.traverse(root=False))) assert (db_a_from_scratch.installed_relatives( spec['z'], direction='parents') == set([spec, spec['y']]))
def test_cannot_write_upstream(tmpdir_factory, test_store, gen_mock_layout): roots = [str(tmpdir_factory.mktemp(x)) for x in ['a', 'b']] layouts = [gen_mock_layout(x) for x in ['/ra/', '/rb/']] x = MockPackage('x', [], []) mock_repo = MockPackageMultiRepo([x]) # Instantiate the database that will be used as the upstream DB and make # sure it has an index file upstream_db_independent = spack.database.Database(roots[1]) with upstream_db_independent.write_transaction(): pass upstream_dbs = spack.store._construct_upstream_dbs_from_install_roots( [roots[1]], _test=True) with spack.repo.swap(mock_repo): spec = spack.spec.Spec('x') spec.concretize() with pytest.raises(spack.database.ForbiddenLockError): upstream_dbs[0].add(spec, layouts[1])
def test_specify_preinstalled_dep(): """Specify the use of a preinstalled package during concretization with a transitive dependency that is only supplied by the preinstalled package. """ default = ('build', 'link') c = MockPackage('c', [], []) b = MockPackage('b', [c], [default]) a = MockPackage('a', [b], [default]) mock_repo = MockPackageMultiRepo([a, b, c]) with spack.repo.swap(mock_repo): b_spec = Spec('b') b_spec.concretize() for spec in b_spec.traverse(): setattr(spec.package, 'installed', True) a_spec = Spec('a') a_spec._add_dependency(b_spec, default) a_spec.concretize() assert set(x.name for x in a_spec.traverse()) == set(['a', 'b', 'c'])
def test_read_old_lock_creates_backup(tmpdir): """When reading a version-1 lockfile, make sure that a backup of that file is created. """ y = MockPackage('y', [], []) mock_repo = MockPackageMultiRepo([y]) with spack.repo.swap(mock_repo): y = Spec('y') y.concretize() test_lockfile_dict = create_v1_lockfile_dict([y], [y]) env_root = tmpdir.mkdir('test-root') test_lockfile_path = str(env_root.join(ev.lockfile_name)) with open(test_lockfile_path, 'w') as f: sjson.dump(test_lockfile_dict, stream=f) e = ev.Environment(str(env_root)) assert os.path.exists(e._lock_backup_v1_path) with open(e._lock_backup_v1_path, 'r') as backup_v1_file: lockfile_dict_v1 = sjson.load(backup_v1_file) # Make sure that the backup file follows the v1 hash scheme assert y.dag_hash() in lockfile_dict_v1['concrete_specs']
def test_specs_staging(config): """Make sure we achieve the best possible staging for the following spec DAG:: a /| c b |\ e d |\ f g In this case, we would expect 'c', 'e', 'f', and 'g' to be in the first stage, and then 'd', 'b', and 'a' to be put in the next three stages, respectively. """ current_system = sys_type() config_compilers = config.get_config('compilers') first_compiler = config_compilers[0] compiler_spec = first_compiler['compiler']['spec'] # Whatever that first compiler in the configuration was, let's make sure # we mock up an entry like we'd find in os-container-mapping.yaml which # has that compiler. mock_containers = {} mock_containers[current_system] = { "image": "dontcare", "compilers": [{ "name": compiler_spec, }], } default = ('build', 'link') g = MockPackage('g', [], []) f = MockPackage('f', [], []) e = MockPackage('e', [], []) d = MockPackage('d', [f, g], [default, default]) c = MockPackage('c', [], []) b = MockPackage('b', [d, e], [default, default]) a = MockPackage('a', [b, c], [default, default]) mock_repo = MockPackageMultiRepo([a, b, c, d, e, f, g]) with repo.swap(mock_repo): # Now we'll ask for the root package to be compiled with whatever that # first compiler in the configuration was. spec_a = Spec('a%{0}'.format(compiler_spec)) spec_a.concretize() spec_a_label = spec_deps_key_label(spec_a)[1] spec_b_label = spec_deps_key_label(spec_a['b'])[1] spec_c_label = spec_deps_key_label(spec_a['c'])[1] spec_d_label = spec_deps_key_label(spec_a['d'])[1] spec_e_label = spec_deps_key_label(spec_a['e'])[1] spec_f_label = spec_deps_key_label(spec_a['f'])[1] spec_g_label = spec_deps_key_label(spec_a['g'])[1] spec_labels, dependencies, stages = stage_spec_jobs([spec_a], mock_containers, current_system) assert (len(stages) == 4) assert (len(stages[0]) == 4) assert (spec_c_label in stages[0]) assert (spec_e_label in stages[0]) assert (spec_f_label in stages[0]) assert (spec_g_label in stages[0]) assert (len(stages[1]) == 1) assert (spec_d_label in stages[1]) assert (len(stages[2]) == 1) assert (spec_b_label in stages[2]) assert (len(stages[3]) == 1) assert (spec_a_label in stages[3])