Ejemplo n.º 1
0
    def test_splice_dict_roundtrip(self, transitive):
        spec = Spec('splice-t')
        dep = Spec('splice-h+foo')
        spec.concretize()
        dep.concretize()
        out = spec.splice(dep, transitive)

        # Sanity check all hashes are unique...
        assert spec.dag_hash() != dep.dag_hash()
        assert out.dag_hash() != dep.dag_hash()
        assert out.dag_hash() != spec.dag_hash()
        out_rt_spec = Spec.from_dict(out.to_dict())  # rt is "round trip"
        assert out_rt_spec.dag_hash() == out.dag_hash()
        out_rt_spec_bld_hash = out_rt_spec.build_spec.dag_hash()
        out_rt_spec_h_bld_hash = out_rt_spec['splice-h'].build_spec.dag_hash()
        out_rt_spec_z_bld_hash = out_rt_spec['splice-z'].build_spec.dag_hash()

        # In any case, the build spec for splice-t (root) should point to the
        # original spec, preserving build provenance.
        assert spec.dag_hash() == out_rt_spec_bld_hash
        assert out_rt_spec.dag_hash() != out_rt_spec_bld_hash

        # The build spec for splice-h should always point to the introduced
        # spec, since that is the spec spliced in.
        assert dep['splice-h'].dag_hash() == out_rt_spec_h_bld_hash

        # The build spec for splice-z will depend on whether or not the splice
        # was transitive.
        expected_z_bld_hash = (dep['splice-z'].dag_hash()
                               if transitive else spec['splice-z'].dag_hash())
        assert expected_z_bld_hash == out_rt_spec_z_bld_hash
Ejemplo n.º 2
0
 def test_splice_input_unchanged(self, transitive):
     spec = Spec('splice-t').concretized()
     dep = Spec('splice-h+foo').concretized()
     orig_spec_hash = spec.dag_hash()
     orig_dep_hash = dep.dag_hash()
     spec.splice(dep, transitive)
     # Post-splice, dag hash should still be different; no changes should be
     # made to these specs.
     assert spec.dag_hash() == orig_spec_hash
     assert dep.dag_hash() == orig_dep_hash
Ejemplo n.º 3
0
    def test_regression_issue_7941(self):
        # The string representation of a spec containing
        # an explicit multi-valued variant and a dependency
        # might be parsed differently than the originating spec
        s = Spec('a foobar=bar ^b')
        t = Spec(str(s))

        s.concretize()
        t.concretize()

        assert s.dag_hash() == t.dag_hash()
Ejemplo n.º 4
0
    def test_regression_issue_7941(self):
        # The string representation of a spec containing
        # an explicit multi-valued variant and a dependency
        # might be parsed differently than the originating spec
        s = Spec('a foobar=bar ^b')
        t = Spec(str(s))

        s.concretize()
        t.concretize()

        assert s.dag_hash() == t.dag_hash()
Ejemplo n.º 5
0
    def test_reuse_installed_packages(self, context, mutable_database,
                                      repo_with_changing_recipe):
        # Install a spec
        root = Spec('root').concretized()
        dependency = root['changing'].copy()
        root.package.do_install(fake=True, explicit=True)

        # Modify package.py
        repo_with_changing_recipe.change(context)

        # Try to concretize with the spec installed previously
        new_root = Spec('root ^/{0}'.format(
            dependency.dag_hash())).concretized()

        assert root.dag_hash() == new_root.dag_hash()
Ejemplo n.º 6
0
def test_dynamic_dot_graph_mpileaks(mock_packages, config):
    """Test dynamically graphing the mpileaks package."""
    s = Spec('mpileaks').concretized()

    stream = StringIO()
    graph_dot([s], static=False, out=stream)

    dot = stream.getvalue()
    print(dot)

    mpileaks_hash, mpileaks_lbl = s.dag_hash(), s.format('{name}')
    mpi_hash, mpi_lbl = s['mpi'].dag_hash(), s['mpi'].format('{name}')
    callpath_hash, callpath_lbl = (s['callpath'].dag_hash(),
                                   s['callpath'].format('{name}'))
    dyninst_hash, dyninst_lbl = (s['dyninst'].dag_hash(),
                                 s['dyninst'].format('{name}'))
    libdwarf_hash, libdwarf_lbl = (s['libdwarf'].dag_hash(),
                                   s['libdwarf'].format('{name}'))
    libelf_hash, libelf_lbl = (s['libelf'].dag_hash(),
                               s['libelf'].format('{name}'))

    assert '  "%s" [label="%s"]\n' % (mpileaks_hash, mpileaks_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (callpath_hash, callpath_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (mpi_hash, mpi_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (dyninst_hash, dyninst_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (libdwarf_hash, libdwarf_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (libelf_hash, libelf_lbl) in dot

    assert '  "%s" -> "%s"\n' % (dyninst_hash, libdwarf_hash) in dot
    assert '  "%s" -> "%s"\n' % (callpath_hash, dyninst_hash) in dot
    assert '  "%s" -> "%s"\n' % (mpileaks_hash, mpi_hash) in dot
    assert '  "%s" -> "%s"\n' % (libdwarf_hash, libelf_hash) in dot
    assert '  "%s" -> "%s"\n' % (callpath_hash, mpi_hash) in dot
    assert '  "%s" -> "%s"\n' % (mpileaks_hash, callpath_hash) in dot
    assert '  "%s" -> "%s"\n' % (dyninst_hash, libelf_hash) in dot
Ejemplo n.º 7
0
 def test_splice(self, transitive):
     # Tests the new splice function in Spec using a somewhat simple case
     # with a variant with a conditional dependency.
     # TODO: Test being able to splice in different provider for a virtual.
     # Example: mvapich for mpich.
     spec = Spec('splice-t')
     dep = Spec('splice-h+foo')
     spec.concretize()
     dep.concretize()
     # Sanity checking that these are not the same thing.
     assert dep.dag_hash() != spec['splice-h'].dag_hash()
     assert dep.build_hash() != spec['splice-h'].build_hash()
     # Do the splice.
     out = spec.splice(dep, transitive)
     # Returned spec should still be concrete.
     assert out.concrete
     # Traverse the spec and assert that all dependencies are accounted for.
     for node in spec.traverse():
         assert node.name in out
     # If the splice worked, then the full hash of the spliced dep should
     # now match the full hash of the build spec of the dependency from the
     # returned spec.
     out_h_build = out['splice-h'].build_spec
     assert out_h_build.full_hash() == dep.full_hash()
     # Transitivity should determine whether the transitive dependency was
     # changed.
     expected_z = dep['splice-z'] if transitive else spec['splice-z']
     assert out['splice-z'].full_hash() == expected_z.full_hash()
     # Sanity check build spec of out should be the original spec.
     assert (out['splice-t'].build_spec.full_hash() ==
             spec['splice-t'].full_hash())
     # Finally, the spec should know it's been spliced:
     assert out.spliced
Ejemplo n.º 8
0
def test_rewire(mock_fetch, install_mockery, transitive):
    spec = Spec('splice-t^splice-h~foo').concretized()
    dep = Spec('splice-h+foo').concretized()
    spec.package.do_install()
    dep.package.do_install()
    spliced_spec = spec.splice(dep, transitive=transitive)
    assert spec.dag_hash() != spliced_spec.dag_hash()

    spack.rewiring.rewire(spliced_spec)

    # check that the prefix exists
    assert os.path.exists(spliced_spec.prefix)

    # test that it made it into the database
    rec = spack.store.db.get_record(spliced_spec)
    installed_in_db = rec.installed if rec else False
    assert installed_in_db

    # check the file in the prefix has the correct paths
    for node in spliced_spec.traverse(root=True):
        text_file_path = os.path.join(node.prefix, node.name)
        with open(text_file_path, 'r') as f:
            text = f.read()
            for modded_spec in node.traverse(root=True):
                assert modded_spec.prefix in text
Ejemplo n.º 9
0
def test_rewire_bin(mock_fetch, install_mockery, transitive):
    spec = Spec('quux').concretized()
    dep = Spec('garply cflags=-g').concretized()
    spec.package.do_install()
    dep.package.do_install()
    spliced_spec = spec.splice(dep, transitive=transitive)
    assert spec.dag_hash() != spliced_spec.dag_hash()

    spack.rewiring.rewire(spliced_spec)

    # check that the prefix exists
    assert os.path.exists(spliced_spec.prefix)

    # test that it made it into the database
    rec = spack.store.db.get_record(spliced_spec)
    installed_in_db = rec.installed if rec else False
    assert installed_in_db

    # check the file in the prefix has the correct paths
    bin_names = {
        'garply': 'garplinator',
        'corge': 'corgegator',
        'quux': 'quuxifier'
    }
    for node in spliced_spec.traverse(root=True):
        for dep in node.traverse(root=True):
            bin_file_path = os.path.join(dep.prefix.bin, bin_names[dep.name])
            assert text_in_bin(dep.prefix, bin_file_path)
Ejemplo n.º 10
0
    def remove(self, query_spec, force=False):
        """Remove specs from an environment that match a query_spec"""
        query_spec = Spec(query_spec)

        # try abstract specs first
        matches = []
        if not query_spec.concrete:
            matches = [s for s in self.user_specs if s.satisfies(query_spec)]

        if not matches:
            # concrete specs match against concrete specs in the env
            specs_hashes = zip(self.concretized_user_specs,
                               self.concretized_order)
            matches = [
                s for s, h in specs_hashes if query_spec.dag_hash() == h
            ]

        if not matches:
            raise SpackEnvironmentError("Not found: {0}".format(query_spec))

        for spec in matches:
            if spec in self.user_specs:
                self.user_specs.remove(spec)

            if force and spec in self.concretized_user_specs:
                i = self.concretized_user_specs.index(spec)
                del self.concretized_user_specs[i]

                dag_hash = self.concretized_order[i]
                del self.concretized_order[i]
                del self.specs_by_hash[dag_hash]
Ejemplo n.º 11
0
    def remove(self, query_spec, force=False):
        """Remove specs from an environment that match a query_spec"""
        query_spec = Spec(query_spec)

        # try abstract specs first
        matches = []
        if not query_spec.concrete:
            matches = [s for s in self.user_specs if s.satisfies(query_spec)]

        if not matches:
            # concrete specs match against concrete specs in the env
            specs_hashes = zip(
                self.concretized_user_specs, self.concretized_order)
            matches = [
                s for s, h in specs_hashes if query_spec.dag_hash() == h]

        if not matches:
            raise SpackEnvironmentError("Not found: {0}".format(query_spec))

        for spec in matches:
            if spec in self.user_specs:
                self.user_specs.remove(spec)

            if force and spec in self.concretized_user_specs:
                i = self.concretized_user_specs.index(spec)
                del self.concretized_user_specs[i]

                dag_hash = self.concretized_order[i]
                del self.concretized_order[i]
                del self.specs_by_hash[dag_hash]
Ejemplo n.º 12
0
Archivo: graph.py Proyecto: LLNL/spack
def test_dynamic_dot_graph_mpileaks(mock_packages):
    """Test dynamically graphing the mpileaks package."""
    s = Spec('mpileaks').normalized()

    stream = StringIO()
    graph_dot([s], static=False, out=stream)

    dot = stream.getvalue()

    mpileaks_hash, mpileaks_lbl = s.dag_hash(), s.format('$_$/')
    mpi_hash, mpi_lbl = s['mpi'].dag_hash(), s['mpi'].format('$_$/')
    callpath_hash, callpath_lbl = (
        s['callpath'].dag_hash(), s['callpath'].format('$_$/'))
    dyninst_hash, dyninst_lbl = (
        s['dyninst'].dag_hash(), s['dyninst'].format('$_$/'))
    libdwarf_hash, libdwarf_lbl = (
        s['libdwarf'].dag_hash(), s['libdwarf'].format('$_$/'))
    libelf_hash, libelf_lbl = (
        s['libelf'].dag_hash(), s['libelf'].format('$_$/'))

    assert '  "%s" [label="%s"]\n' % (mpileaks_hash, mpileaks_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (callpath_hash, callpath_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (mpi_hash,      mpi_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (dyninst_hash,  dyninst_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (libdwarf_hash, libdwarf_lbl) in dot
    assert '  "%s" [label="%s"]\n' % (libelf_hash, libelf_lbl) in dot

    assert '  "%s" -> "%s"\n' % (dyninst_hash, libdwarf_hash)  in dot
    assert '  "%s" -> "%s"\n' % (callpath_hash, dyninst_hash)  in dot
    assert '  "%s" -> "%s"\n' % (mpileaks_hash, mpi_hash)  in dot
    assert '  "%s" -> "%s"\n' % (libdwarf_hash, libelf_hash)  in dot
    assert '  "%s" -> "%s"\n' % (callpath_hash, mpi_hash)  in dot
    assert '  "%s" -> "%s"\n' % (mpileaks_hash, callpath_hash)  in dot
    assert '  "%s" -> "%s"\n' % (dyninst_hash, libelf_hash)  in dot
Ejemplo n.º 13
0
    def test_splice_with_cached_hashes(self, transitive):
        spec = Spec('splice-t')
        dep = Spec('splice-h+foo')
        spec.concretize()
        dep.concretize()

        # monkeypatch hashes so we can test that they are cached
        spec._hash = 'aaaaaa'
        dep._hash = 'bbbbbb'
        spec['splice-h']._hash = 'cccccc'
        spec['splice-z']._hash = 'dddddd'
        dep['splice-z']._hash = 'eeeeee'

        out = spec.splice(dep, transitive=transitive)
        out_z_expected = (dep if transitive else spec)['splice-z']

        assert out.dag_hash() != spec.dag_hash()
        assert (out['splice-h'].dag_hash() == dep.dag_hash()) == transitive
        assert out['splice-z'].dag_hash() == out_z_expected.dag_hash()
Ejemplo n.º 14
0
    def test_external_packages_have_consistent_hash(self):
        if spack.config.get('config:concretizer') == 'original':
            pytest.skip('This tests needs the ASP-based concretizer')

        s, t = Spec('externaltool'), Spec('externaltool')
        s._old_concretize(), t._new_concretize()

        assert s.dag_hash() == t.dag_hash()
        assert s.build_hash() == t.build_hash()
        assert s.full_hash() == t.full_hash()
Ejemplo n.º 15
0
def test_installed_dependency_request_conflicts(install_mockery, mock_fetch,
                                                mutable_mock_repo):
    dependency = Spec('dependency-install')
    dependency.concretize()
    dependency.package.do_install()

    dependency_hash = dependency.dag_hash()
    dependent = Spec('conflicting-dependent ^/' + dependency_hash)
    with pytest.raises(spack.error.UnsatisfiableSpecError):
        dependent.concretize()
Ejemplo n.º 16
0
def test_installed_dependency_request_conflicts(
        install_mockery, mock_fetch, mutable_mock_packages):
    dependency = Spec('dependency-install')
    dependency.concretize()
    dependency.package.do_install()

    dependency_hash = dependency.dag_hash()
    dependent = Spec(
        'conflicting-dependent ^/' + dependency_hash)
    with pytest.raises(spack.spec.UnsatisfiableSpecError):
        dependent.concretize()
Ejemplo n.º 17
0
    def test_satisfies_same_spec_with_different_hash(self):
        """Ensure that concrete specs are matched *exactly* by hash."""
        s1 = Spec('mpileaks').concretized()
        s2 = s1.copy()

        assert s1.satisfies(s2)
        assert s2.satisfies(s1)

        # Simulate specs that were installed before and after a change to
        # Spack's hashing algorithm.  This just reverses s2's hash.
        s2._hash = s1.dag_hash()[-1::-1]

        assert not s1.satisfies(s2)
        assert not s2.satisfies(s1)
Ejemplo n.º 18
0
def test_dont_add_patches_to_installed_package(install_mockery, mock_fetch):
    dependency = Spec('dependency-install')
    dependency.concretize()
    dependency.package.do_install()

    dependency_hash = dependency.dag_hash()
    dependent = Spec('dependent-install ^/' + dependency_hash)
    dependent.concretize()

    dependency.package.patches['dependency-install'] = [
        spack.patch.UrlPatch(
            dependent.package, 'file://fake.patch', sha256='unused-hash')]

    assert dependent['dependency-install'] == dependency
Ejemplo n.º 19
0
    def test_satisfies_same_spec_with_different_hash(self):
        """Ensure that concrete specs are matched *exactly* by hash."""
        s1 = Spec('mpileaks').concretized()
        s2 = s1.copy()

        assert s1.satisfies(s2)
        assert s2.satisfies(s1)

        # Simulate specs that were installed before and after a change to
        # Spack's hashing algorithm.  This just reverses s2's hash.
        s2._hash = s1.dag_hash()[-1::-1]

        assert not s1.satisfies(s2)
        assert not s2.satisfies(s1)
Ejemplo n.º 20
0
def test_dont_add_patches_to_installed_package(install_mockery, mock_fetch):
    import sys
    dependency = Spec('dependency-install')
    dependency.concretize()
    dependency.package.do_install()

    dependency.package.patches['dependency-install'] = [
        sys.modules['spack.patch'].Patch.create(
            None, 'file://fake.patch', sha256='unused-hash')]

    dependency_hash = dependency.dag_hash()
    dependent = Spec('dependent-install ^/' + dependency_hash)
    dependent.concretize()

    assert dependent['dependency-install'] == dependency
Ejemplo n.º 21
0
    def remove(self, query_spec, list_name=user_speclist_name, force=False):
        """Remove specs from an environment that match a query_spec"""
        query_spec = Spec(query_spec)

        list_to_change = self.spec_lists[list_name]
        matches = []

        if not query_spec.concrete:
            matches = [s for s in list_to_change if s.satisfies(query_spec)]

        if not matches:
            # concrete specs match against concrete specs in the env
            specs_hashes = zip(
                self.concretized_user_specs, self.concretized_order)
            matches = [
                s for s, h in specs_hashes
                if query_spec.dag_hash() == h
            ]

        if not matches:
            raise SpackEnvironmentError(
                "Not found: {0}".format(query_spec))

        old_specs = set(self.user_specs)
        for spec in matches:
            if spec in list_to_change:
                list_to_change.remove(spec)

        self.update_stale_references(list_name)

        # If force, update stale concretized specs
        # Only check specs removed by this operation
        new_specs = set(self.user_specs)
        for spec in old_specs - new_specs:
            if force and spec in self.concretized_user_specs:
                i = self.concretized_user_specs.index(spec)
                del self.concretized_user_specs[i]

                dag_hash = self.concretized_order[i]
                del self.concretized_order[i]
                del self.specs_by_hash[dag_hash]
Ejemplo n.º 22
0
def test_failing_overwrite_install_should_keep_previous_installation(
    mock_fetch, install_mockery
):
    """
    Make sure that whenever `spack install --overwrite` fails, spack restores
    the original install prefix instead of cleaning it.
    """
    # Do a successful install
    s = Spec('canfail').concretized()
    s.package.succeed = True

    # Do a failing overwrite install
    s.package.do_install()
    s.package.succeed = False
    kwargs = {'overwrite': [s.dag_hash()]}

    with pytest.raises(Exception):
        s.package.do_install(**kwargs)

    assert s.package.spec.installed
    assert os.path.exists(s.prefix)
Ejemplo n.º 23
0
    def test_splice_dict(self, transitive):
        spec = Spec('splice-t')
        dep = Spec('splice-h+foo')
        spec.concretize()
        dep.concretize()
        out = spec.splice(dep, transitive)

        # Sanity check all hashes are unique...
        assert spec.dag_hash() != dep.dag_hash()
        assert out.dag_hash() != dep.dag_hash()
        assert out.dag_hash() != spec.dag_hash()
        node_list = out.to_dict()['spec']['nodes']
        root_nodes = [n for n in node_list if n['hash'] == out.dag_hash()]
        build_spec_nodes = [n for n in node_list if n['hash'] == spec.dag_hash()]
        assert spec.dag_hash() == out.build_spec.dag_hash()
        assert len(root_nodes) == 1
        assert len(build_spec_nodes) == 1
Ejemplo n.º 24
0
def test_update_sbang(tmpdir, install_mockery, function_mirror):
    """
    Test the creation and installation of buildcaches with default rpaths
    into the non-default directory layout scheme, triggering an update of the
    sbang.
    """

    # Save the original store and layout before we touch ANYTHING.
    real_store = spack.store.store
    real_layout = spack.store.layout

    # Concretize a package with some old-fashioned sbang lines.
    sspec = Spec('old-sbang')
    sspec.concretize()

    # Need a fake mirror with *function* scope.
    mirror_dir = function_mirror

    # Assumes all commands will concretize sspec the same way.
    install_cmd('--no-cache', sspec.name)

    # Create a buildcache with the installed spec.
    buildcache_cmd('create', '-u', '-a', '-d', mirror_dir,
                   '/%s' % sspec.dag_hash())

    # Need to force an update of the buildcache index
    buildcache_cmd('update-index', '-d', 'file://%s' % mirror_dir)

    # Uninstall the original package.
    uninstall_cmd('-y', '/%s' % sspec.dag_hash())

    try:
        # New install tree locations...
        # Too fine-grained to do be done in a fixture
        spack.store.store = spack.store.Store(str(tmpdir.join('newtree')))
        spack.store.layout = YamlDirectoryLayout(
            str(tmpdir.join('newtree')),
            path_scheme=ndef_install_path_scheme)  # noqa: E501

        # Install package from buildcache
        buildcache_cmd('install', '-a', '-u', '-f', sspec.name)

        # Continue blowing away caches
        bindist.clear_spec_cache()
        spack.stage.purge()

        # test that the sbang was updated by the move
        sbang_style_1_expected = '''{0}
#!/usr/bin/env python

{1}
        '''.format(sbang.sbang_shebang_line(), sspec.prefix.bin)
        sbang_style_2_expected = '''{0}
#!/usr/bin/env python

{1}
        '''.format(sbang.sbang_shebang_line(), sspec.prefix.bin)

        installed_script_style_1_path = \
            sspec.prefix.bin.join('sbang-style-1.sh')
        assert sbang_style_1_expected == \
            open(str(installed_script_style_1_path)).read()

        installed_script_style_2_path = \
            sspec.prefix.bin.join('sbang-style-2.sh')
        assert sbang_style_2_expected == \
            open(str(installed_script_style_2_path)).read()

        uninstall_cmd('-y', '/%s' % sspec.dag_hash())

    finally:
        spack.store.store = real_store
        spack.store.layout = real_layout
Ejemplo n.º 25
0
def test_buildcache(mock_archive, tmpdir):
    # tweak patchelf to only do a download
    spec = Spec("patchelf")
    spec.concretize()
    pkg = spack.repo.get(spec)
    fake_fetchify(pkg.fetcher, pkg)
    mkdirp(os.path.join(pkg.prefix, "bin"))
    patchelfscr = os.path.join(pkg.prefix, "bin", "patchelf")
    f = open(patchelfscr, 'w')
    body = """#!/bin/bash
echo $PATH"""
    f.write(body)
    f.close()
    st = os.stat(patchelfscr)
    os.chmod(patchelfscr, st.st_mode | stat.S_IEXEC)

    # Install the test package
    spec = Spec('trivial-install-test-package')
    spec.concretize()
    assert spec.concrete
    pkg = spec.package
    fake_fetchify(mock_archive.url, pkg)
    pkg.do_install()
    pkghash = '/' + spec.dag_hash(7)

    # Put some non-relocatable file in there
    filename = os.path.join(spec.prefix, "dummy.txt")
    with open(filename, "w") as script:
        script.write(spec.prefix)

    # Create the build cache  and
    # put it directly into the mirror

    mirror_path = os.path.join(str(tmpdir), 'test-mirror')
    spack.mirror.create(
        mirror_path, specs=[], no_checksum=True
    )

    # register mirror with spack config
    mirrors = {'spack-mirror-test': 'file://' + mirror_path}
    spack.config.set('mirrors', mirrors)

    stage = spack.stage.Stage(
        mirrors['spack-mirror-test'], name="build_cache", keep=True)
    stage.create()
    # setup argument parser
    parser = argparse.ArgumentParser()
    buildcache.setup_parser(parser)

    # Create a private key to sign package with if gpg2 available
    if has_gnupg2():
        spack.util.gpg.Gpg.create(name='test key 1', expires='0',
                                  email='*****@*****.**',
                                  comment='Spack test key')
        # Create build cache with signing
        args = parser.parse_args(['create', '-d', mirror_path, str(spec)])
        buildcache.buildcache(parser, args)

        # Uninstall the package
        pkg.do_uninstall(force=True)

        # test overwrite install
        args = parser.parse_args(['install', '-f', str(pkghash)])
        buildcache.buildcache(parser, args)

        # create build cache with relative path and signing
        args = parser.parse_args(
            ['create', '-d', mirror_path, '-f', '-r', str(spec)])
        buildcache.buildcache(parser, args)

        # Uninstall the package
        pkg.do_uninstall(force=True)

        # install build cache with verification
        args = parser.parse_args(['install', str(spec)])
        buildcache.install_tarball(spec, args)

        # test overwrite install
        args = parser.parse_args(['install', '-f', str(pkghash)])
        buildcache.buildcache(parser, args)

    else:
        # create build cache without signing
        args = parser.parse_args(
            ['create', '-d', mirror_path, '-u', str(spec)])
        buildcache.buildcache(parser, args)

        # Uninstall the package
        pkg.do_uninstall(force=True)

        # install build cache without verification
        args = parser.parse_args(['install', '-u', str(spec)])
        buildcache.install_tarball(spec, args)

        # test overwrite install without verification
        args = parser.parse_args(['install', '-f', '-u', str(pkghash)])
        buildcache.buildcache(parser, args)

        # create build cache with relative path
        args = parser.parse_args(
            ['create', '-d', mirror_path, '-f', '-r', '-u', str(pkghash)])
        buildcache.buildcache(parser, args)

        # Uninstall the package
        pkg.do_uninstall(force=True)

        # install build cache
        args = parser.parse_args(['install', '-u', str(spec)])
        buildcache.install_tarball(spec, args)

        # test overwrite install
        args = parser.parse_args(['install', '-f', '-u', str(pkghash)])
        buildcache.buildcache(parser, args)

    # Validate the relocation information
    buildinfo = bindist.read_buildinfo_file(spec.prefix)
    assert(buildinfo['relocate_textfiles'] == ['dummy.txt'])

    args = parser.parse_args(['list'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['list', '-f'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['list', 'trivial'])
    buildcache.buildcache(parser, args)

    # Copy a key to the mirror to have something to download
    shutil.copyfile(mock_gpg_keys_path + '/external.key',
                    mirror_path + '/external.key')

    args = parser.parse_args(['keys'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['keys', '-f'])
    buildcache.buildcache(parser, args)

    # unregister mirror with spack config
    mirrors = {}
    spack.config.set('mirrors', mirrors)
    shutil.rmtree(mirror_path)
    stage.destroy()
Ejemplo n.º 26
0
def test_packaging(mock_archive, tmpdir):
    # tweak patchelf to only do a download
    spec = Spec("patchelf")
    spec.concretize()
    pkg = spack.repo.get(spec)
    fake_fetchify(pkg.fetcher, pkg)
    mkdirp(os.path.join(pkg.prefix, "bin"))
    patchelfscr = os.path.join(pkg.prefix, "bin", "patchelf")
    f = open(patchelfscr, 'w')
    body = """#!/bin/bash
echo $PATH"""
    f.write(body)
    f.close()
    st = os.stat(patchelfscr)
    os.chmod(patchelfscr, st.st_mode | stat.S_IEXEC)

    # Install the test package
    spec = Spec('trivial-install-test-package')
    spec.concretize()
    assert spec.concrete
    pkg = spack.repo.get(spec)
    fake_fetchify(mock_archive.url, pkg)
    pkg.do_install()
    pkghash = '/' + spec.dag_hash(7)

    # Put some non-relocatable file in there
    filename = os.path.join(spec.prefix, "dummy.txt")
    with open(filename, "w") as script:
        script.write(spec.prefix)

    # Create the build cache  and
    # put it directly into the mirror

    mirror_path = os.path.join(str(tmpdir), 'test-mirror')
    specs = [spec]
    spack.mirror.create(mirror_path, specs, no_checksum=True)

    # register mirror with spack config
    mirrors = {'spack-mirror-test': 'file://' + mirror_path}
    spack.config.update_config('mirrors', mirrors)

    stage = spack.stage.Stage(mirrors['spack-mirror-test'],
                              name="build_cache",
                              keep=True)
    stage.create()
    # setup argument parser
    parser = argparse.ArgumentParser()
    buildcache.setup_parser(parser)

    # Create a private key to sign package with if gpg2 available
    if has_gnupg2():
        spack.util.gpg.Gpg.create(name='test key 1',
                                  expires='0',
                                  email='*****@*****.**',
                                  comment='Spack test key')
        # Create build cache with signing
        args = parser.parse_args(['create', '-d', mirror_path, str(spec)])
        buildcache.buildcache(parser, args)

        # Uninstall the package
        pkg.do_uninstall(force=True)

        # test overwrite install
        args = parser.parse_args(['install', '-f', str(pkghash)])
        buildcache.buildcache(parser, args)

        # create build cache with relative path and signing
        args = parser.parse_args(
            ['create', '-d', mirror_path, '-f', '-r',
             str(spec)])
        buildcache.buildcache(parser, args)

        # Uninstall the package
        pkg.do_uninstall(force=True)

        # install build cache with verification
        args = parser.parse_args(['install', str(spec)])
        buildcache.install_tarball(spec, args)

        # test overwrite install
        args = parser.parse_args(['install', '-f', str(pkghash)])
        buildcache.buildcache(parser, args)

    else:
        # create build cache without signing
        args = parser.parse_args(
            ['create', '-d', mirror_path, '-y',
             str(spec)])
        buildcache.buildcache(parser, args)

        # Uninstall the package
        pkg.do_uninstall(force=True)

        # install build cache without verification
        args = parser.parse_args(['install', '-y', str(spec)])
        buildcache.install_tarball(spec, args)

        # test overwrite install without verification
        args = parser.parse_args(['install', '-f', '-y', str(pkghash)])
        buildcache.buildcache(parser, args)

        # create build cache with relative path
        args = parser.parse_args(
            ['create', '-d', mirror_path, '-f', '-r', '-y',
             str(pkghash)])
        buildcache.buildcache(parser, args)

        # Uninstall the package
        pkg.do_uninstall(force=True)

        # install build cache
        args = parser.parse_args(['install', '-y', str(spec)])
        buildcache.install_tarball(spec, args)

        # test overwrite install
        args = parser.parse_args(['install', '-f', '-y', str(pkghash)])
        buildcache.buildcache(parser, args)

    # Validate the relocation information
    buildinfo = bindist.read_buildinfo_file(spec.prefix)
    assert (buildinfo['relocate_textfiles'] == ['dummy.txt'])

    args = parser.parse_args(['list'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['list', '-f'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['list', 'trivial'])
    buildcache.buildcache(parser, args)

    # Copy a key to the mirror to have something to download
    shutil.copyfile(spack.mock_gpg_keys_path + '/external.key',
                    mirror_path + '/external.key')

    args = parser.parse_args(['keys'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['keys', '-f'])
    buildcache.buildcache(parser, args)

    # unregister mirror with spack config
    mirrors = {}
    spack.config.update_config('mirrors', mirrors)
    shutil.rmtree(mirror_path)
    stage.destroy()
Ejemplo n.º 27
0
def test_update_sbang(tmpdir, test_mirror):
    """Test the creation and installation of buildcaches with default rpaths
    into the non-default directory layout scheme, triggering an update of the
    sbang.
    """
    scheme = os.path.join(
        '${name}', '${version}',
        '${architecture}-${compiler.name}-${compiler.version}-${hash}')
    spec_str = 'old-sbang'
    # Concretize a package with some old-fashioned sbang lines.
    old_spec = Spec(spec_str).concretized()
    old_spec_hash_str = '/{0}'.format(old_spec.dag_hash())

    # Need a fake mirror with *function* scope.
    mirror_dir = test_mirror
    mirror_url = 'file://{0}'.format(mirror_dir)

    # Assume all commands will concretize old_spec the same way.
    install_cmd('--no-cache', old_spec.name)

    # Create a buildcache with the installed spec.
    buildcache_cmd('create', '-u', '-a', '-d', mirror_dir, old_spec_hash_str)

    # Need to force an update of the buildcache index
    buildcache_cmd('update-index', '-d', mirror_url)

    # Uninstall the original package.
    uninstall_cmd('-y', old_spec_hash_str)

    # Switch the store to the new install tree locations
    newtree_dir = tmpdir.join('newtree')
    s = spack.store.Store(str(newtree_dir))
    s.layout = DirectoryLayout(str(newtree_dir), path_scheme=scheme)

    with spack.store.use_store(s):
        new_spec = Spec('old-sbang')
        new_spec.concretize()
        assert new_spec.dag_hash() == old_spec.dag_hash()

        # Install package from buildcache
        buildcache_cmd('install', '-a', '-u', '-f', new_spec.name)

        # Continue blowing away caches
        bindist.clear_spec_cache()
        spack.stage.purge()

        # test that the sbang was updated by the move
        sbang_style_1_expected = '''{0}
#!/usr/bin/env python

{1}
'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin)
        sbang_style_2_expected = '''{0}
#!/usr/bin/env python

{1}
'''.format(sbang.sbang_shebang_line(), new_spec.prefix.bin)

        installed_script_style_1_path = new_spec.prefix.bin.join(
            'sbang-style-1.sh')
        assert sbang_style_1_expected == \
            open(str(installed_script_style_1_path)).read()

        installed_script_style_2_path = new_spec.prefix.bin.join(
            'sbang-style-2.sh')
        assert sbang_style_2_expected == \
            open(str(installed_script_style_2_path)).read()

        uninstall_cmd('-y', '/%s' % new_spec.dag_hash())
Ejemplo n.º 28
0
def test_buildcache(mock_archive, tmpdir):
    # tweak patchelf to only do a download
    pspec = Spec("patchelf")
    pspec.concretize()
    pkg = spack.repo.get(pspec)
    fake_fetchify(pkg.fetcher, pkg)
    mkdirp(os.path.join(pkg.prefix, "bin"))
    patchelfscr = os.path.join(pkg.prefix, "bin", "patchelf")
    f = open(patchelfscr, 'w')
    body = """#!/bin/bash
echo $PATH"""
    f.write(body)
    f.close()
    st = os.stat(patchelfscr)
    os.chmod(patchelfscr, st.st_mode | stat.S_IEXEC)

    # Install the test package
    spec = Spec('trivial-install-test-package')
    spec.concretize()
    assert spec.concrete
    pkg = spec.package
    fake_fetchify(mock_archive.url, pkg)
    pkg.do_install()
    pkghash = '/' + str(spec.dag_hash(7))

    # Put some non-relocatable file in there
    filename = os.path.join(spec.prefix, "dummy.txt")
    with open(filename, "w") as script:
        script.write(spec.prefix)

    # Create an absolute symlink
    linkname = os.path.join(spec.prefix, "link_to_dummy.txt")
    os.symlink(filename, linkname)

    # Create the build cache  and
    # put it directly into the mirror
    mirror_path = os.path.join(str(tmpdir), 'test-mirror')
    spack.mirror.create(mirror_path, specs=[])

    # register mirror with spack config
    mirrors = {'spack-mirror-test': 'file://' + mirror_path}
    spack.config.set('mirrors', mirrors)

    stage = spack.stage.Stage(
        mirrors['spack-mirror-test'], name="build_cache", keep=True)
    stage.create()

    # setup argument parser
    parser = argparse.ArgumentParser()
    buildcache.setup_parser(parser)

    create_args = ['create', '-a', '-f', '-d', mirror_path, pkghash]
    # Create a private key to sign package with if gpg2 available
    spack.util.gpg.create(name='test key 1', expires='0',
                          email='*****@*****.**',
                          comment='Spack test key')

    create_args.insert(create_args.index('-a'), '--rebuild-index')

    args = parser.parse_args(create_args)
    buildcache.buildcache(parser, args)
    # trigger overwrite warning
    buildcache.buildcache(parser, args)

    # Uninstall the package
    pkg.do_uninstall(force=True)

    install_args = ['install', '-a', '-f', pkghash]
    args = parser.parse_args(install_args)
    # Test install
    buildcache.buildcache(parser, args)

    files = os.listdir(spec.prefix)

    assert 'link_to_dummy.txt' in files
    assert 'dummy.txt' in files

    # Validate the relocation information
    buildinfo = bindist.read_buildinfo_file(spec.prefix)
    assert(buildinfo['relocate_textfiles'] == ['dummy.txt'])
    assert(buildinfo['relocate_links'] == ['link_to_dummy.txt'])

    # create build cache with relative path
    create_args.insert(create_args.index('-a'), '-f')
    create_args.insert(create_args.index('-a'), '-r')
    args = parser.parse_args(create_args)
    buildcache.buildcache(parser, args)

    # Uninstall the package
    pkg.do_uninstall(force=True)

    args = parser.parse_args(install_args)
    buildcache.buildcache(parser, args)

    # test overwrite install
    install_args.insert(install_args.index('-a'), '-f')
    args = parser.parse_args(install_args)
    buildcache.buildcache(parser, args)

    files = os.listdir(spec.prefix)
    assert 'link_to_dummy.txt' in files
    assert 'dummy.txt' in files
#    assert os.path.realpath(
#        os.path.join(spec.prefix, 'link_to_dummy.txt')
#    ) == os.path.realpath(os.path.join(spec.prefix, 'dummy.txt'))

    args = parser.parse_args(['keys'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['list'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['list'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['list', 'trivial'])
    buildcache.buildcache(parser, args)

    # Copy a key to the mirror to have something to download
    shutil.copyfile(mock_gpg_keys_path + '/external.key',
                    mirror_path + '/external.key')

    args = parser.parse_args(['keys'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['keys', '-f'])
    buildcache.buildcache(parser, args)

    args = parser.parse_args(['keys', '-i', '-t'])
    buildcache.buildcache(parser, args)

    # unregister mirror with spack config
    mirrors = {}
    spack.config.set('mirrors', mirrors)
    shutil.rmtree(mirror_path)
    stage.destroy()

    # Remove cached binary specs since we deleted the mirror
    bindist._cached_specs = set()
Ejemplo n.º 29
0
def test_ordered_read_not_required_for_consistent_dag_hash(
        config, mock_packages):
    """Make sure ordered serialization isn't required to preserve hashes.

    For consistent hashes, we require that YAML and json documents
    have their keys serialized in a deterministic order. However, we
    don't want to require them to be serialized in order. This
    ensures that is not required.
    """
    specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
    for spec in specs:
        spec = Spec(spec)
        spec.concretize()

        #
        # Dict & corresponding YAML & JSON from the original spec.
        #
        spec_dict = spec.to_dict()
        spec_yaml = spec.to_yaml()
        spec_json = spec.to_json()

        #
        # Make a spec with reversed OrderedDicts for every
        # OrderedDict in the original.
        #
        reversed_spec_dict = reverse_all_dicts(spec.to_dict())

        #
        # Dump to YAML and JSON
        #
        yaml_string = syaml.dump(spec_dict, default_flow_style=False)
        reversed_yaml_string = syaml.dump(reversed_spec_dict,
                                          default_flow_style=False)
        json_string = sjson.dump(spec_dict)
        reversed_json_string = sjson.dump(reversed_spec_dict)

        #
        # Do many consistency checks
        #

        # spec yaml is ordered like the spec dict
        assert yaml_string == spec_yaml
        assert json_string == spec_json

        # reversed string is different from the original, so it
        # *would* generate a different hash
        assert yaml_string != reversed_yaml_string
        assert json_string != reversed_json_string

        # build specs from the "wrongly" ordered data
        round_trip_yaml_spec = Spec.from_yaml(yaml_string)
        round_trip_json_spec = Spec.from_json(json_string)
        round_trip_reversed_yaml_spec = Spec.from_yaml(reversed_yaml_string)
        round_trip_reversed_json_spec = Spec.from_yaml(reversed_json_string)

        # TODO: remove this when build deps are in provenance.
        spec = spec.copy(deps=('link', 'run'))
        # specs are equal to the original
        assert spec == round_trip_yaml_spec
        assert spec == round_trip_json_spec
        assert spec == round_trip_reversed_yaml_spec
        assert spec == round_trip_reversed_json_spec
        assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
        assert round_trip_json_spec == round_trip_reversed_json_spec
        # dag_hashes are equal
        assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
        assert spec.dag_hash() == round_trip_json_spec.dag_hash()
        assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
        assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()
        # full_hashes are equal
        spec.concretize()
        round_trip_yaml_spec.concretize()
        round_trip_json_spec.concretize()
        round_trip_reversed_yaml_spec.concretize()
        round_trip_reversed_json_spec.concretize()
        assert spec.full_hash() == round_trip_yaml_spec.full_hash()
        assert spec.full_hash() == round_trip_json_spec.full_hash()
        assert spec.full_hash() == round_trip_reversed_yaml_spec.full_hash()
        assert spec.full_hash() == round_trip_reversed_json_spec.full_hash()
Ejemplo n.º 30
0
    def test_spec_formatting(self):
        spec = Spec("multivalue-variant cflags=-O2")
        spec.concretize()

        # Since the default is the full spec see if the string rep of
        # spec is the same as the output of spec.format()
        # ignoring whitespace (though should we?) and ignoring dependencies
        spec_string = str(spec)
        idx = spec_string.index(' ^')
        assert spec_string[:idx] == spec.format().strip()

        # Testing named strings ie {string} and whether we get
        # the correct component
        # Mixed case intentional to test both
        package_segments = [("{NAME}", "name"), ("{VERSION}", "versions"),
                            ("{compiler}", "compiler"),
                            ("{compiler_flags}", "compiler_flags"),
                            ("{variants}", "variants"),
                            ("{architecture}", "architecture")]

        sigil_package_segments = [("{@VERSIONS}", '@' + str(spec.version)),
                                  ("{%compiler}", '%' + str(spec.compiler)),
                                  ("{arch=architecture}",
                                   'arch=' + str(spec.architecture))]

        compiler_segments = [("{compiler.name}", "name"),
                             ("{compiler.version}", "versions")]

        sigil_compiler_segments = [
            ("{%compiler.name}", '%' + spec.compiler.name),
            ("{@compiler.version}", '@' + str(spec.compiler.version))
        ]

        architecture_segments = [("{architecture.platform}", "platform"),
                                 ("{architecture.os}", "os"),
                                 ("{architecture.target}", "target")]

        other_segments = [('{spack_root}', spack.paths.spack_root),
                          ('{spack_install}', spack.store.layout.root),
                          ('{hash:7}', spec.dag_hash(7)),
                          ('{/hash}', '/' + spec.dag_hash())]

        for named_str, prop in package_segments:
            expected = getattr(spec, prop, "")
            actual = spec.format(named_str)
            assert str(expected).strip() == actual

        for named_str, expected in sigil_package_segments:
            actual = spec.format(named_str)
            assert expected == actual

        compiler = spec.compiler
        for named_str, prop in compiler_segments:
            expected = getattr(compiler, prop, "")
            actual = spec.format(named_str)
            assert str(expected) == actual

        for named_str, expected in sigil_compiler_segments:
            actual = spec.format(named_str)
            assert expected == actual

        arch = spec.architecture
        for named_str, prop in architecture_segments:
            expected = getattr(arch, prop, "")
            actual = spec.format(named_str)
            assert str(expected) == actual

        for named_str, expected in other_segments:
            actual = spec.format(named_str)
            assert expected == actual
Ejemplo n.º 31
0
def test_ordered_read_not_required_for_consistent_dag_hash(
        config, builtin_mock
):
    """Make sure ordered serialization isn't required to preserve hashes.

    For consistent hashes, we require that YAML and json documents
    have their keys serialized in a deterministic order. However, we
    don't want to require them to be serialized in order. This
    ensures that is not required.
    """
    specs = ['mpileaks ^zmpi', 'dttop', 'dtuse']
    for spec in specs:
        spec = Spec(spec)
        spec.concretize()

        #
        # Dict & corresponding YAML & JSON from the original spec.
        #
        spec_dict = spec.to_dict()
        spec_yaml = spec.to_yaml()
        spec_json = spec.to_json()

        #
        # Make a spec with reversed OrderedDicts for every
        # OrderedDict in the original.
        #
        reversed_spec_dict = reverse_all_dicts(spec.to_dict())

        #
        # Dump to YAML and JSON
        #
        yaml_string = syaml.dump(spec_dict, default_flow_style=False)
        reversed_yaml_string = syaml.dump(reversed_spec_dict,
                                          default_flow_style=False)
        json_string = sjson.dump(spec_dict)
        reversed_json_string = sjson.dump(reversed_spec_dict)

        #
        # Do many consistency checks
        #

        # spec yaml is ordered like the spec dict
        assert yaml_string == spec_yaml
        assert json_string == spec_json

        # reversed string is different from the original, so it
        # *would* generate a different hash
        assert yaml_string != reversed_yaml_string
        assert json_string != reversed_json_string

        # build specs from the "wrongly" ordered data
        round_trip_yaml_spec = Spec.from_yaml(yaml_string)
        round_trip_json_spec = Spec.from_json(json_string)
        round_trip_reversed_yaml_spec = Spec.from_yaml(
            reversed_yaml_string
        )
        round_trip_reversed_json_spec = Spec.from_yaml(
            reversed_json_string
        )

        # TODO: remove this when build deps are in provenance.
        spec = spec.copy(deps=('link', 'run'))
        # specs are equal to the original
        assert spec == round_trip_yaml_spec
        assert spec == round_trip_json_spec
        assert spec == round_trip_reversed_yaml_spec
        assert spec == round_trip_reversed_json_spec
        assert round_trip_yaml_spec == round_trip_reversed_yaml_spec
        assert round_trip_json_spec == round_trip_reversed_json_spec
        # dag_hashes are equal
        assert spec.dag_hash() == round_trip_yaml_spec.dag_hash()
        assert spec.dag_hash() == round_trip_json_spec.dag_hash()
        assert spec.dag_hash() == round_trip_reversed_yaml_spec.dag_hash()
        assert spec.dag_hash() == round_trip_reversed_json_spec.dag_hash()