Esempio n. 1
0
File: create.py Progetto: d-tk/spack
def fetch_tarballs(url, name, version):
    """Try to find versions of the supplied archive by scraping the web.

    Prompts the user to select how many to download if many are found.


    """
    versions = spack.util.web.find_versions_of_archive(url)
    rkeys = sorted(versions.keys(), reverse=True)
    versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))

    archives_to_fetch = 1
    if not versions:
        # If the fetch failed for some reason, revert to what the user provided
        versions = { version : url }
    elif len(versions) > 1:
        tty.msg("Found %s versions of %s:" % (len(versions), name),
                *spack.cmd.elide_list(
                    ["%-10s%s" % (v,u) for v, u in versions.iteritems()]))
        print
        archives_to_fetch = tty.get_number(
            "Include how many checksums in the package file?",
            default=5, abort='q')

        if not archives_to_fetch:
            tty.die("Aborted.")

    sorted_versions = sorted(versions.keys(), reverse=True)
    sorted_urls = [versions[v] for v in sorted_versions]
    return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch]
Esempio n. 2
0
def fetch_tarballs(url, name, version):
    """Try to find versions of the supplied archive by scraping the web.
    Prompts the user to select how many to download if many are found."""
    versions = spack.util.web.find_versions_of_archive(url)
    rkeys = sorted(versions.keys(), reverse=True)
    versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))

    archives_to_fetch = 1
    if not versions:
        # If the fetch failed for some reason, revert to what the user provided
        versions = {version: url}
    elif len(versions) > 1:
        tty.msg("Found %s versions of %s:" % (len(versions), name),
                *spack.cmd.elide_list(
                    ["%-10s%s" % (v, u) for v, u in versions.iteritems()]))
        print('')
        archives_to_fetch = tty.get_number(
            "Include how many checksums in the package file?",
            default=5, abort='q')

        if not archives_to_fetch:
            tty.die("Aborted.")

    sorted_versions = sorted(versions.keys(), reverse=True)
    sorted_urls = [versions[v] for v in sorted_versions]
    return sorted_versions[:archives_to_fetch], sorted_urls[:archives_to_fetch]
Esempio n. 3
0
    def _read_manifest(self, f):
        """Read manifest file and set up user specs."""
        self.yaml = _read_yaml(f)

        self.spec_lists = OrderedDict()

        for item in config_dict(self.yaml).get('definitions', []):
            entry = copy.deepcopy(item)
            when = _eval_conditional(entry.pop('when', 'True'))
            assert len(entry) == 1
            if when:
                name, spec_list = next(iter(entry.items()))
                user_specs = SpecList(name, spec_list, self.spec_lists.copy())
                if name in self.spec_lists:
                    self.spec_lists[name].extend(user_specs)
                else:
                    self.spec_lists[name] = user_specs

        spec_list = config_dict(self.yaml).get(user_speclist_name)
        user_specs = SpecList(user_speclist_name, [s for s in spec_list if s],
                              self.spec_lists.copy())
        self.spec_lists[user_speclist_name] = user_specs

        enable_view = config_dict(self.yaml).get('view')
        # enable_view can be boolean, string, or None
        if enable_view is True or enable_view is None:
            self.views = {
                default_view_name: ViewDescriptor(self.view_path_default)}
        elif isinstance(enable_view, six.string_types):
            self.views = {default_view_name: ViewDescriptor(enable_view)}
        elif enable_view:
            self.views = dict((name, ViewDescriptor.from_dict(values))
                              for name, values in enable_view.items())
        else:
            self.views = {}
Esempio n. 4
0
 def save(self, name, modules):
     collection = OrderedDict()
     for module in modules:
         ar = pymod.mc.archive_module(module)
         ar["refcount"] = 0
         collection.setdefault(module.modulepath, []).append(ar)
     collection = list(collection.items())
     self.data.update({name: collection})
     self.write(self.data, self.filename)
     return None
Esempio n. 5
0
    def __init__(self, *scopes):
        """Initialize a configuration with an initial list of scopes.

        Args:
            scopes (list of ConfigScope): list of scopes to add to this
                Configuration, ordered from lowest to highest precedence

        """
        self.scopes = OrderedDict()
        for scope in scopes:
            self.push_scope(scope)
Esempio n. 6
0
    def initmock(self):
        # Use the mock packages database for these tests.  This allows
        # us to set up contrived packages that don't interfere with
        # real ones.
        self.db = RepoPath(spack.mock_packages_path)
        spack.repo.swap(self.db)

        # Mock up temporary configuration directories
        self.temp_config = tempfile.mkdtemp()
        self.mock_site_config = os.path.join(self.temp_config, 'site')
        self.mock_user_config = os.path.join(self.temp_config, 'user')
        mkdirp(self.mock_site_config)
        mkdirp(self.mock_user_config)
        for filename, data in mock_configs.items():
            conf_yaml = os.path.join(self.mock_site_config, filename)
            with open(conf_yaml, 'w') as f:
                f.write(data)

        # TODO: Mocking this up is kind of brittle b/c ConfigScope
        # TODO: constructor modifies config_scopes.  Make it cleaner.
        spack.config.clear_config_caches()
        self.real_scopes = spack.config.config_scopes

        spack.config.config_scopes = OrderedDict()
        spack.config.ConfigScope('site', self.mock_site_config)
        spack.config.ConfigScope('user', self.mock_user_config)

        # Keep tests from interfering with the actual module path.
        self.real_share_path = spack.share_path
        spack.share_path = tempfile.mkdtemp()

        # Store changes to the package's dependencies so we can
        # restore later.
        self.saved_deps = {}
Esempio n. 7
0
    def initmock(self):
        # Use the mock packages database for these tests.  This allows
        # us to set up contrived packages that don't interfere with
        # real ones.
        self.db = RepoPath(spack.mock_packages_path)
        spack.repo.swap(self.db)

        spack.config.clear_config_caches()
        self.real_scopes = spack.config.config_scopes

        # Mock up temporary configuration directories
        self.temp_config = tempfile.mkdtemp()
        self.mock_site_config = os.path.join(self.temp_config, 'site')
        self.mock_user_config = os.path.join(self.temp_config, 'user')
        mkdirp(self.mock_site_config)
        mkdirp(self.mock_user_config)
        for confs in [('compilers.yaml', mock_compiler_config), ('packages.yaml', mock_packages_config)]:
            conf_yaml = os.path.join(self.mock_site_config, confs[0])
            with open(conf_yaml, 'w') as f:
                f.write(confs[1])

        # TODO: Mocking this up is kind of brittle b/c ConfigScope
        # TODO: constructor modifies config_scopes.  Make it cleaner.
        spack.config.config_scopes = OrderedDict()
        spack.config.ConfigScope('site', self.mock_site_config)
        spack.config.ConfigScope('user', self.mock_user_config)

        # Store changes to the package's dependencies so we can
        # restore later.
        self.saved_deps = {}
Esempio n. 8
0
def slice_items(items, ignore, scoped_argkeys_cache):
    # we pick the first item which uses a fixture instance in the
    # requested scope and which we haven't seen yet.  We slice the input
    # items list into a list of items_nomatch, items_same and
    # items_other
    if scoped_argkeys_cache:  # do we need to do work at all?
        it = iter(items)
        # first find a slicing key
        for i, item in enumerate(it):
            argkeys = scoped_argkeys_cache.get(item)
            if argkeys is not None:
                newargkeys = OrderedDict.fromkeys(k for k in argkeys
                                                  if k not in ignore)
                if newargkeys:  # found a slicing key
                    slicing_argkey, _ = newargkeys.popitem()
                    items_before = items[:i]
                    items_same = [item]
                    items_other = []
                    # now slice the remainder of the list
                    for item in it:
                        argkeys = scoped_argkeys_cache.get(item)
                        if argkeys and slicing_argkey in argkeys and \
                                slicing_argkey not in ignore:
                            items_same.append(item)
                        else:
                            items_other.append(item)
                    newignore = ignore.copy()
                    newignore.add(slicing_argkey)
                    return (items_before, items_same, items_other, newignore)
    return items, None, None, None
Esempio n. 9
0
def slice_items(items, ignore, scoped_argkeys_cache):
    # we pick the first item which uses a fixture instance in the
    # requested scope and which we haven't seen yet.  We slice the input
    # items list into a list of items_nomatch, items_same and
    # items_other
    if scoped_argkeys_cache:  # do we need to do work at all?
        it = iter(items)
        # first find a slicing key
        for i, item in enumerate(it):
            argkeys = scoped_argkeys_cache.get(item)
            if argkeys is not None:
                newargkeys = OrderedDict.fromkeys(k for k in argkeys if k not in ignore)
                if newargkeys:  # found a slicing key
                    slicing_argkey, _ = newargkeys.popitem()
                    items_before = items[:i]
                    items_same = [item]
                    items_other = []
                    # now slice the remainder of the list
                    for item in it:
                        argkeys = scoped_argkeys_cache.get(item)
                        if argkeys and slicing_argkey in argkeys and \
                                slicing_argkey not in ignore:
                            items_same.append(item)
                        else:
                            items_other.append(item)
                    newignore = ignore.copy()
                    newignore.add(slicing_argkey)
                    return (items_before, items_same, items_other, newignore)
    return items, None, None, None
Esempio n. 10
0
    def __init__(self, args):
        Reporter.__init__(self, args)
        self.template_dir = os.path.join('reports', 'cdash')
        self.cdash_upload_url = args.cdash_upload_url

        if self.cdash_upload_url:
            self.buildid_regexp = re.compile("<buildId>([0-9]+)</buildId>")
        self.phase_regexp = re.compile(r"Executing phase: '(.*)'")

        if args.package:
            packages = args.package
        else:
            packages = []
            for file in args.specfiles:
                with open(file, 'r') as f:
                    s = spack.spec.Spec.from_yaml(f)
                    packages.append(s.format())
        self.install_command = ' '.join(packages)
        self.base_buildname = args.cdash_build or self.install_command
        self.site = args.cdash_site or socket.gethostname()
        self.osname = platform.system()
        self.endtime = int(time.time())
        if args.cdash_buildstamp:
            self.buildstamp = args.cdash_buildstamp
        else:
            buildstamp_format = "%Y%m%d-%H%M-{0}".format(args.cdash_track)
            self.buildstamp = time.strftime(buildstamp_format,
                                            time.localtime(self.endtime))
        self.buildIds = OrderedDict()
        self.revision = ''
        git = which('git')
        with working_dir(spack.paths.spack_root):
            self.revision = git('rev-parse', 'HEAD', output=str).strip()
        self.multiple_packages = False
Esempio n. 11
0
 def setUp(self):
     super(ConfigTest, self).setUp()
     self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-')
     spack.config.config_scopes = OrderedDict()
     spack.config.ConfigScope('test_low_priority',
                              os.path.join(self.tmp_dir, 'low'))
     spack.config.ConfigScope('test_high_priority',
                              os.path.join(self.tmp_dir, 'high'))
Esempio n. 12
0
def reorder_items(items):
    argkeys_cache = {}
    for scopenum in range(0, scopenum_function):
        argkeys_cache[scopenum] = d = {}
        for item in items:
            keys = OrderedDict.fromkeys(get_parametrized_fixture_keys(item, scopenum))
            if keys:
                d[item] = keys
    return reorder_items_atscope(items, set(), argkeys_cache, 0)
Esempio n. 13
0
def reorder_items(items):
    argkeys_cache = {}
    for scopenum in range(0, scopenum_function):
        argkeys_cache[scopenum] = d = {}
        for item in items:
            keys = OrderedDict.fromkeys(
                get_parametrized_fixture_keys(item, scopenum))
            if keys:
                d[item] = keys
    return reorder_items_atscope(items, set(), argkeys_cache, 0)
Esempio n. 14
0
File: config.py Progetto: LLNL/spack
    def __init__(self, *scopes):
        """Initialize a configuration with an initial list of scopes.

        Args:
            scopes (list of ConfigScope): list of scopes to add to this
                Configuration, ordered from lowest to highest precedence

        """
        self.scopes = OrderedDict()
        for scope in scopes:
            self.push_scope(scope)
Esempio n. 15
0
    def _valid_virtuals_and_externals(self, spec):
        """Returns a list of candidate virtual dep providers and external
           packages that coiuld be used to concretize a spec.

           Preferred specs come first in the list.
        """
        # First construct a list of concrete candidates to replace spec with.
        candidates = [spec]
        pref_key = lambda spec: 0  # no-op pref key

        if spec.virtual:
            candidates = spack.repo.providers_for(spec)
            if not candidates:
                raise UnsatisfiableProviderSpecError(candidates[0], spec)

            # Find nearest spec in the DAG (up then down) that has prefs.
            spec_w_prefs = find_spec(
                spec, lambda p: PackagePrefs.has_preferred_providers(
                    p.name, spec.name), spec)  # default to spec itself.

            # Create a key to sort candidates by the prefs we found
            pref_key = PackagePrefs(spec_w_prefs.name, 'providers', spec.name)

        # For each candidate package, if it has externals, add those
        # to the usable list.  if it's not buildable, then *only* add
        # the externals.
        #
        # Use an OrderedDict to avoid duplicates (use it like a set)
        usable = OrderedDict()
        for cspec in candidates:
            if is_spec_buildable(cspec):
                usable[cspec] = True

            externals = spec_externals(cspec)
            for ext in externals:
                if ext.satisfies(spec):
                    usable[ext] = True

        # If nothing is in the usable list now, it's because we aren't
        # allowed to build anything.
        if not usable:
            raise NoBuildError(spec)

        # Use a sort key to order the results
        return sorted(
            usable,
            key=lambda spec: (
                not spec.external,  # prefer externals
                pref_key(spec),  # respect prefs
                spec.name,  # group by name
                reverse_order(spec.versions),  # latest version
                spec  # natural order
            ))
Esempio n. 16
0
 def add_to_loaded_collection(self, name):
     """Add a module `name` to the currently loaded collection"""
     collection_name = pymod.environ.get(pymod.names.loaded_collection)
     if collection_name is None:  # pragma: no cover
         tty.die("There is no collection currently loaded")
     data = OrderedDict(self.data.pop(collection_name))
     module = pymod.modulepath.get(name)
     if module is None:
         raise pymod.error.ModuleNotFoundError(name)
     if not module.is_loaded:
         pymod.mc.load_impl(module)
     for (mp, modules) in data.items():
         if mp != module.modulepath:
             continue
         for other in modules:
             if other["fullname"] == module.fullname:  # pragma: no cover
                 tty.warn("{0} is already in collection {1}".format(
                     name, collection_name))
                 return
     ar = pymod.mc.archive_module(module)
     ar["refcount"] = 0
     data.setdefault(module.modulepath, []).append(ar)
     data = list(data.items())
     self.data.update({collection_name: data})
     self.write(self.data, self.filename)
     return None
Esempio n. 17
0
def test_get_header():
    headers = {
        'Content-type': 'text/plain'
    }

    # looking up headers should just work like a plain dict
    # lookup when there is an entry with the right key
    assert(web_util.get_header(headers, 'Content-type') == 'text/plain')

    # looking up headers should still work if there is a fuzzy match
    assert(web_util.get_header(headers, 'contentType') == 'text/plain')

    # ...unless there is an exact match for the "fuzzy" spelling.
    headers['contentType'] = 'text/html'
    assert(web_util.get_header(headers, 'contentType') == 'text/html')

    # If lookup has to fallback to fuzzy matching and there are more than one
    # fuzzy match, the result depends on the internal ordering of the given
    # mapping
    headers = OrderedDict()
    headers['Content-type'] = 'text/plain'
    headers['contentType'] = 'text/html'

    assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/plain')
    del headers['Content-type']
    assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/html')

    # Same as above, but different ordering
    headers = OrderedDict()
    headers['contentType'] = 'text/html'
    headers['Content-type'] = 'text/plain'

    assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/html')
    del headers['contentType']
    assert(web_util.get_header(headers, 'CONTENT_TYPE') == 'text/plain')

    # If there isn't even a fuzzy match, raise KeyError
    with pytest.raises(KeyError):
        web_util.get_header(headers, 'ContentLength')
Esempio n. 18
0
    def setUp(self):
        super(ConfigTest, self).setUp()
        self.tmp_dir = mkdtemp('.tmp', 'spack-config-test-')
        self.a_comp_specs = [
            ac['compiler']['spec'] for ac in a_comps['compilers']
        ]
        self.b_comp_specs = [
            bc['compiler']['spec'] for bc in b_comps['compilers']
        ]

        spack.config.config_scopes = OrderedDict()
        for priority in ['low', 'high']:
            scope_dir = os.path.join(self.tmp_dir, priority)
            spack.config.ConfigScope(priority, scope_dir)
Esempio n. 19
0
 def pop_from_loaded_collection(self, name):
     """Remove a module `name` to the currently loaded collection"""
     collection_name = pymod.environ.get(pymod.names.loaded_collection)
     if collection_name is None:
         tty.die(
             "There is no collection currently loaded")  # pragma: no cover
     data = OrderedDict(self.data.pop(collection_name))
     module = pymod.modulepath.get(name)
     if module is None:  # pragma: no cover
         raise pymod.error.ModuleNotFoundError(name)
     if module.is_loaded:
         pymod.mc.unload_impl(module)
     for (mp, modules) in data.items():
         if mp != module.modulepath:  # pragma: no cover
             continue
         data[mp] = [
             other for other in modules
             if other["fullname"] == module.fullname
         ]
         break
     data = list(data.items())
     self.data.update({collection_name: data})
     self.write(self.data, self.filename)
     return None
Esempio n. 20
0
def view_copy(src, dst, view, spec=None):
    """
    Copy a file from src to dst.

    Use spec and view to generate relocations
    """
    shutil.copy2(src, dst)
    if spec and not spec.external:
        # Not metadata, we have to relocate it

        # Get information on where to relocate from/to

        # This is vestigial code for the *old* location of sbang. Previously,
        # sbang was a bash script, and it lived in the spack prefix. It is
        # now a POSIX script that lives in the install prefix. Old packages
        # will have the old sbang location in their shebangs.
        # TODO: Not sure which one to use...
        import spack.hooks.sbang as sbang

        # Break a package include cycle
        import spack.relocate

        orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(spack.paths.spack_root)
        new_sbang = sbang.sbang_shebang_line()

        prefix_to_projection = OrderedDict(
            {spec.prefix: view.get_projection_for_spec(spec)})

        for dep in spec.traverse():
            if not dep.external:
                prefix_to_projection[dep.prefix] = \
                    view.get_projection_for_spec(dep)

        if spack.relocate.is_binary(dst):
            spack.relocate.relocate_text_bin(binaries=[dst],
                                             prefixes=prefix_to_projection)
        else:
            prefix_to_projection[spack.store.layout.root] = view._root
            prefix_to_projection[orig_sbang] = new_sbang
            spack.relocate.relocate_text(files=[dst],
                                         prefixes=prefix_to_projection)
        try:
            stat = os.stat(src)
            os.chown(dst, stat.st_uid, stat.st_gid)
        except OSError:
            tty.debug('Can\'t change the permissions for %s' % dst)
Esempio n. 21
0
    def __init__(self, args):
        Reporter.__init__(self, args)
        tty.set_verbose(args.verbose)
        self.success = True
        self.template_dir = os.path.join('reports', 'cdash')
        self.cdash_upload_url = args.cdash_upload_url

        if self.cdash_upload_url:
            self.buildid_regexp = re.compile("<buildId>([0-9]+)</buildId>")
        self.phase_regexp = re.compile(r"Executing phase: '(.*)'")

        self.authtoken = None
        if 'SPACK_CDASH_AUTH_TOKEN' in os.environ:
            tty.verbose("Using CDash auth token from environment")
            self.authtoken = os.environ.get('SPACK_CDASH_AUTH_TOKEN')

        if getattr(args, 'spec', ''):
            packages = args.spec
        elif getattr(args, 'specs', ''):
            packages = args.specs
        else:
            packages = []
            for file in args.specfiles:
                with open(file, 'r') as f:
                    s = spack.spec.Spec.from_yaml(f)
                    packages.append(s.format())
        self.install_command = ' '.join(packages)
        self.base_buildname = args.cdash_build or self.install_command
        self.site = args.cdash_site or socket.gethostname()
        self.osname = platform.system()
        self.endtime = int(time.time())
        if args.cdash_buildstamp:
            self.buildstamp = args.cdash_buildstamp
        else:
            buildstamp_format = "%Y%m%d-%H%M-{0}".format(args.cdash_track)
            self.buildstamp = time.strftime(buildstamp_format,
                                            time.localtime(self.endtime))
        self.buildIds = OrderedDict()
        self.revision = ''
        git = which('git')
        with working_dir(spack.paths.spack_root):
            self.revision = git('rev-parse', 'HEAD', output=str).strip()
        self.multiple_packages = False
Esempio n. 22
0
def view_copy(src, dst, view, spec=None):
    """
    Copy a file from src to dst.

    Use spec and view to generate relocations
    """
    shutil.copyfile(src, dst)
    if spec:
        # Not metadata, we have to relocate it

        # Get information on where to relocate from/to

        # This is vestigial code for the *old* location of sbang. Previously,
        # sbang was a bash script, and it lived in the spack prefix. It is
        # now a POSIX script that lives in the install prefix. Old packages
        # will have the old sbang location in their shebangs.
        # TODO: Not sure which one to use...
        import spack.hooks.sbang as sbang
        orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(spack.paths.spack_root)
        new_sbang = sbang.sbang_shebang_line()

        prefix_to_projection = OrderedDict({
            spec.prefix: view.get_projection_for_spec(spec),
            spack.paths.spack_root: view._root})

        for dep in spec.traverse():
            prefix_to_projection[dep.prefix] = \
                view.get_projection_for_spec(dep)

        if spack.relocate.is_binary(dst):
            spack.relocate.relocate_text_bin(
                binaries=[dst],
                prefixes=prefix_to_projection
            )
        else:
            prefix_to_projection[spack.store.layout.root] = view._root
            prefix_to_projection[orig_sbang] = new_sbang
            spack.relocate.relocate_text(
                files=[dst],
                prefixes=prefix_to_projection
            )
Esempio n. 23
0
def relocate_text_bin(binaries, prefixes, concurrency=32):
    """Replace null terminated path strings hard coded into binaries.

    The new install prefix must be shorter than the original one.

    Args:
        binaries (list): binaries to be relocated
        prefixes (OrderedDict): String prefixes which need to be changed.
        concurrency (int): Desired degree of parallelism.

    Raises:
      BinaryTextReplaceError: when the new path is longer than the old path
    """
    byte_prefixes = OrderedDict({})

    for orig_prefix, new_prefix in prefixes.items():
        if orig_prefix != new_prefix:
            if isinstance(orig_prefix, bytes):
                orig_bytes = orig_prefix
            else:
                orig_bytes = orig_prefix.encode('utf-8')
            if isinstance(new_prefix, bytes):
                new_bytes = new_prefix
            else:
                new_bytes = new_prefix.encode('utf-8')
            byte_prefixes[orig_bytes] = new_bytes

    # Do relocations on text in binaries that refers to the install tree
    # multiprocesing.ThreadPool.map requires single argument
    args = []

    for binary in binaries:
        args.append((binary, byte_prefixes))

    tp = multiprocessing.pool.ThreadPool(processes=concurrency)

    try:
        tp.map(llnl.util.lang.star(_replace_prefix_bin), args)
    finally:
        tp.terminate()
        tp.join()
Esempio n. 24
0
def relocate_text(files, prefixes, concurrency=32):
    """Relocate text file from the original installation prefix to the
     new prefix.

     Relocation also affects the the path in Spack's sbang script.

     Args:
         files (list): Text files to be relocated
         prefixes (OrderedDict): String prefixes which need to be changed
         concurrency (int): Preferred degree of parallelism
    """

    # This now needs to be handled by the caller in all cases
    # orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(orig_spack)
    # new_sbang = '#!/bin/bash {0}/bin/sbang'.format(new_spack)

    compiled_prefixes = OrderedDict({})

    for orig_prefix, new_prefix in prefixes.items():
        if orig_prefix != new_prefix:
            orig_bytes = orig_prefix.encode('utf-8')
            orig_prefix_rexp = re.compile(
                b'(?<![\\w\\-_/])([\\w\\-_]*?)%s([\\w\\-_/]*)' % orig_bytes)
            new_bytes = b'\\1%s\\2' % new_prefix.encode('utf-8')
            compiled_prefixes[orig_prefix_rexp] = new_bytes

    # Do relocations on text that refers to the install tree
    # multiprocesing.ThreadPool.map requires single argument

    args = []
    for filename in files:
        args.append((filename, compiled_prefixes))

    tp = multiprocessing.pool.ThreadPool(processes=concurrency)
    try:
        tp.map(llnl.util.lang.star(_replace_prefix_text), args)
    finally:
        tp.terminate()
        tp.join()
Esempio n. 25
0
File: config.py Progetto: LLNL/spack
class Configuration(object):
    """A full Spack configuration, from a hierarchy of config files.

    This class makes it easy to add a new scope on top of an existing one.
    """

    def __init__(self, *scopes):
        """Initialize a configuration with an initial list of scopes.

        Args:
            scopes (list of ConfigScope): list of scopes to add to this
                Configuration, ordered from lowest to highest precedence

        """
        self.scopes = OrderedDict()
        for scope in scopes:
            self.push_scope(scope)

    def push_scope(self, scope):
        """Add a higher precedence scope to the Configuration."""
        cmd_line_scope = None
        if self.scopes:
            highest_precedence_scope = list(self.scopes.values())[-1]
            if highest_precedence_scope.name == 'command_line':
                # If the command-line scope is present, it should always
                # be the scope of highest precedence
                cmd_line_scope = self.pop_scope()

        self.scopes[scope.name] = scope
        if cmd_line_scope:
            self.scopes['command_line'] = cmd_line_scope

    def pop_scope(self):
        """Remove the highest precedence scope and return it."""
        name, scope = self.scopes.popitem(last=True)
        return scope

    def remove_scope(self, scope_name):
        return self.scopes.pop(scope_name)

    @property
    def file_scopes(self):
        """List of writable scopes with an associated file."""
        return [s for s in self.scopes.values() if type(s) == ConfigScope]

    def highest_precedence_scope(self):
        """Non-internal scope with highest precedence."""
        return next(reversed(self.file_scopes), None)

    def _validate_scope(self, scope):
        """Ensure that scope is valid in this configuration.

        This should be used by routines in ``config.py`` to validate
        scope name arguments, and to determine a default scope where no
        scope is specified.

        Raises:
            ValueError: if ``scope`` is not valid

        Returns:
            ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
        """
        if scope is None:
            # default to the scope with highest precedence.
            return self.highest_precedence_scope()

        elif scope in self.scopes:
            return self.scopes[scope]

        else:
            raise ValueError("Invalid config scope: '%s'.  Must be one of %s"
                             % (scope, self.scopes.keys()))

    def get_config_filename(self, scope, section):
        """For some scope and section, get the name of the configuration file.
        """
        scope = self._validate_scope(scope)
        return scope.get_section_filename(section)

    def clear_caches(self):
        """Clears the caches for configuration files,

        This will cause files to be re-read upon the next request."""
        for scope in self.scopes.values():
            scope.clear()

    def update_config(self, section, update_data, scope=None):
        """Update the configuration file for a particular scope.

        Overwrites contents of a section in a scope with update_data,
        then writes out the config file.

        update_data should have the top-level section name stripped off
        (it will be re-added).  Data itself can be a list, dict, or any
        other yaml-ish structure.
        """
        _validate_section_name(section)  # validate section name
        scope = self._validate_scope(scope)  # get ConfigScope object

        # read only the requested section's data.
        scope.sections[section] = {section: update_data}
        scope.write_section(section)

    def get_config(self, section, scope=None):
        """Get configuration settings for a section.

        If ``scope`` is ``None`` or not provided, return the merged contents
        of all of Spack's configuration scopes.  If ``scope`` is provided,
        return only the confiugration as specified in that scope.

        This off the top-level name from the YAML section.  That is, for a
        YAML config file that looks like this::

           config:
             install_tree: $spack/opt/spack
             module_roots:
               lmod:   $spack/share/spack/lmod

        ``get_config('config')`` will return::

           { 'install_tree': '$spack/opt/spack',
             'module_roots: {
                 'lmod': '$spack/share/spack/lmod'
             }
           }

        """
        _validate_section_name(section)

        if scope is None:
            scopes = self.scopes.values()
        else:
            scopes = [self._validate_scope(scope)]

        merged_section = syaml.syaml_dict()
        for scope in scopes:
            # read potentially cached data from the scope.

            data = scope.get_section(section)

            # Skip empty configs
            if not data or not isinstance(data, dict):
                continue

            if section not in data:
                continue

            merged_section = _merge_yaml(merged_section, data)

        # no config files -- empty config.
        if section not in merged_section:
            return {}

        # take the top key off before returning.
        return merged_section[section]

    def get(self, path, default=None, scope=None):
        """Get a config section or a single value from one.

        Accepts a path syntax that allows us to grab nested config map
        entries.  Getting the 'config' section would look like::

            spack.config.get('config')

        and the ``dirty`` section in the ``config`` scope would be::

            spack.config.get('config:dirty')

        We use ``:`` as the separator, like YAML objects.
    """
        # TODO: Currently only handles maps. Think about lists if neded.
        section, _, rest = path.partition(':')

        value = self.get_config(section, scope=scope)
        if not rest:
            return value

        parts = rest.split(':')
        while parts:
            key = parts.pop(0)
            value = value.get(key, default)

        return value

    def set(self, path, value, scope=None):
        """Convenience function for setting single values in config files.

        Accepts the path syntax described in ``get()``.
        """
        section, _, rest = path.partition(':')

        if not rest:
            self.update_config(section, value, scope=scope)
        else:
            section_data = self.get_config(section, scope=scope)

            parts = rest.split(':')
            data = section_data
            while len(parts) > 1:
                key = parts.pop(0)
                data = data[key]
            data[parts[0]] = value

            self.update_config(section, section_data, scope=scope)

    def __iter__(self):
        """Iterate over scopes in this configuration."""
        for scope in self.scopes.values():
            yield scope

    def print_section(self, section, blame=False):
        """Print a configuration to stdout."""
        try:
            data = syaml.syaml_dict()
            data[section] = self.get_config(section)
            syaml.dump(
                data, stream=sys.stdout, default_flow_style=False, blame=blame)
        except (yaml.YAMLError, IOError):
            raise ConfigError("Error reading configuration: %s" % section)
Esempio n. 26
0
                    },
                    'dotkit': {
                        'allOf': [
                            {'$ref': '#/definitions/module_type_configuration'},  # Base configuration
                            {}  # Specific dotkit extensions
                        ]
                    },
                }
            },
        },
    },
}
"""OrderedDict of config scopes keyed by name.
   Later scopes will override earlier scopes.
"""
config_scopes = OrderedDict()


def validate_section_name(section):
    """Exit if the section is not a valid section."""
    if section not in section_schemas:
        tty.die("Invalid config section: '%s'. Options are: %s" %
                (section, " ".join(section_schemas.keys())))


def extend_with_default(validator_class):
    """Add support for the 'default' attribute for properties and patternProperties.

       jsonschema does not handle this out of the box -- it only
       validates.  This allows us to set default values for configs
       where certain fields are `None` b/c they're deleted or
Esempio n. 27
0
class Configuration(object):
    """A full Spack configuration, from a hierarchy of config files.

    This class makes it easy to add a new scope on top of an existing one.
    """

    def __init__(self, *scopes):
        """Initialize a configuration with an initial list of scopes.

        Args:
            scopes (list of ConfigScope): list of scopes to add to this
                Configuration, ordered from lowest to highest precedence

        """
        self.scopes = OrderedDict()
        for scope in scopes:
            self.push_scope(scope)
        self.format_updates = collections.defaultdict(list)

    @_config_mutator
    def push_scope(self, scope):
        """Add a higher precedence scope to the Configuration."""
        cmd_line_scope = None
        if self.scopes:
            highest_precedence_scope = list(self.scopes.values())[-1]
            if highest_precedence_scope.name == 'command_line':
                # If the command-line scope is present, it should always
                # be the scope of highest precedence
                cmd_line_scope = self.pop_scope()

        self.scopes[scope.name] = scope
        if cmd_line_scope:
            self.scopes['command_line'] = cmd_line_scope

    @_config_mutator
    def pop_scope(self):
        """Remove the highest precedence scope and return it."""
        name, scope = self.scopes.popitem(last=True)
        return scope

    @_config_mutator
    def remove_scope(self, scope_name):
        return self.scopes.pop(scope_name)

    @property
    def file_scopes(self):
        """List of writable scopes with an associated file."""
        return [s for s in self.scopes.values()
                if (type(s) == ConfigScope
                    or type(s) == SingleFileScope)]

    def highest_precedence_scope(self):
        """Non-internal scope with highest precedence."""
        return next(reversed(self.file_scopes), None)

    def highest_precedence_non_platform_scope(self):
        """Non-internal non-platform scope with highest precedence

        Platform-specific scopes are of the form scope/platform"""
        generator = reversed(self.file_scopes)
        highest = next(generator, None)
        while highest and highest.is_platform_dependent:
            highest = next(generator, None)
        return highest

    def matching_scopes(self, reg_expr):
        """
        List of all scopes whose names match the provided regular expression.

        For example, matching_scopes(r'^command') will return all scopes
        whose names begin with `command`.
        """
        return [s for s in self.scopes.values() if re.search(reg_expr, s.name)]

    def _validate_scope(self, scope):
        """Ensure that scope is valid in this configuration.

        This should be used by routines in ``config.py`` to validate
        scope name arguments, and to determine a default scope where no
        scope is specified.

        Raises:
            ValueError: if ``scope`` is not valid

        Returns:
            ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
        """
        if scope is None:
            # default to the scope with highest precedence.
            return self.highest_precedence_scope()

        elif scope in self.scopes:
            return self.scopes[scope]

        else:
            raise ValueError("Invalid config scope: '%s'.  Must be one of %s"
                             % (scope, self.scopes.keys()))

    def get_config_filename(self, scope, section):
        """For some scope and section, get the name of the configuration file.
        """
        scope = self._validate_scope(scope)
        return scope.get_section_filename(section)

    @_config_mutator
    def clear_caches(self):
        """Clears the caches for configuration files,

        This will cause files to be re-read upon the next request."""
        for scope in self.scopes.values():
            scope.clear()

    @_config_mutator
    def update_config(self, section, update_data, scope=None, force=False):
        """Update the configuration file for a particular scope.

        Overwrites contents of a section in a scope with update_data,
        then writes out the config file.

        update_data should have the top-level section name stripped off
        (it will be re-added).  Data itself can be a list, dict, or any
        other yaml-ish structure.

        Configuration scopes that are still written in an old schema
        format will fail to update unless ``force`` is True.

        Args:
            section (str): section of the configuration to be updated
            update_data (dict): data to be used for the update
            scope (str): scope to be updated
            force (str): force the update
        """
        if self.format_updates.get(section) and not force:
            msg = ('The "{0}" section of the configuration needs to be written'
                   ' to disk, but is currently using a deprecated format. '
                   'Please update it using:\n\n'
                   '\tspack config [--scope=<scope] update {0}\n\n'
                   'Note that previous versions of Spack will not be able to '
                   'use the updated configuration.')
            msg = msg.format(section)
            raise RuntimeError(msg)

        _validate_section_name(section)  # validate section name
        scope = self._validate_scope(scope)  # get ConfigScope object

        # manually preserve comments
        need_comment_copy = (section in scope.sections and
                             scope.sections[section] is not None)
        if need_comment_copy:
            comments = getattr(scope.sections[section][section],
                               yaml.comments.Comment.attrib,
                               None)

        # read only the requested section's data.
        scope.sections[section] = syaml.syaml_dict({section: update_data})
        if need_comment_copy and comments:
            setattr(scope.sections[section][section],
                    yaml.comments.Comment.attrib,
                    comments)

        scope._write_section(section)

    def get_config(self, section, scope=None):
        """Get configuration settings for a section.

        If ``scope`` is ``None`` or not provided, return the merged contents
        of all of Spack's configuration scopes.  If ``scope`` is provided,
        return only the configuration as specified in that scope.

        This off the top-level name from the YAML section.  That is, for a
        YAML config file that looks like this::

           config:
             install_tree: $spack/opt/spack
             module_roots:
               lmod:   $spack/share/spack/lmod

        ``get_config('config')`` will return::

           { 'install_tree': '$spack/opt/spack',
             'module_roots: {
                 'lmod': '$spack/share/spack/lmod'
             }
           }

        """
        return self._get_config_memoized(section, scope)

    @llnl.util.lang.memoized
    def _get_config_memoized(self, section, scope):
        _validate_section_name(section)

        if scope is None:
            scopes = self.scopes.values()
        else:
            scopes = [self._validate_scope(scope)]

        merged_section = syaml.syaml_dict()
        for scope in scopes:
            # read potentially cached data from the scope.

            data = scope.get_section(section)

            # Skip empty configs
            if not data or not isinstance(data, dict):
                continue

            if section not in data:
                continue

            # We might be reading configuration files in an old format,
            # thus read data and update it in memory if need be.
            changed = _update_in_memory(data, section)
            if changed:
                self.format_updates[section].append(scope)

            merged_section = merge_yaml(merged_section, data)

        # no config files -- empty config.
        if section not in merged_section:
            return syaml.syaml_dict()

        # take the top key off before returning.
        ret = merged_section[section]
        if isinstance(ret, dict):
            ret = syaml.syaml_dict(ret)
        return ret

    def get(self, path, default=None, scope=None):
        """Get a config section or a single value from one.

        Accepts a path syntax that allows us to grab nested config map
        entries.  Getting the 'config' section would look like::

            spack.config.get('config')

        and the ``dirty`` section in the ``config`` scope would be::

            spack.config.get('config:dirty')

        We use ``:`` as the separator, like YAML objects.
    """
        # TODO: Currently only handles maps. Think about lists if needed.
        parts = process_config_path(path)
        section = parts.pop(0)

        value = self.get_config(section, scope=scope)

        while parts:
            key = parts.pop(0)
            value = value.get(key, default)

        return value

    @_config_mutator
    def set(self, path, value, scope=None):
        """Convenience function for setting single values in config files.

        Accepts the path syntax described in ``get()``.
        """
        if ':' not in path:
            # handle bare section name as path
            self.update_config(path, value, scope=scope)
            return

        parts = process_config_path(path)
        section = parts.pop(0)

        section_data = self.get_config(section, scope=scope)

        data = section_data
        while len(parts) > 1:
            key = parts.pop(0)

            if _override(key):
                new = type(data[key])()
                del data[key]
            else:
                new = data[key]

            if isinstance(new, dict):
                # Make it an ordered dict
                new = syaml.syaml_dict(new)
                # reattach to parent object
                data[key] = new
            data = new

        if _override(parts[0]):
            data.pop(parts[0], None)

        # update new value
        data[parts[0]] = value

        self.update_config(section, section_data, scope=scope)

    def __iter__(self):
        """Iterate over scopes in this configuration."""
        for scope in self.scopes.values():
            yield scope

    def print_section(self, section, blame=False):
        """Print a configuration to stdout."""
        try:
            data = syaml.syaml_dict()
            data[section] = self.get_config(section)
            syaml.dump_config(
                data, stream=sys.stdout, default_flow_style=False, blame=blame)
        except (yaml.YAMLError, IOError):
            raise ConfigError("Error reading configuration: %s" % section)
Esempio n. 28
0
File: create.py Progetto: trws/spack
        try:
            name = spack.url.parse_name(url, version)
        except spack.url.UndetectableNameError, e:
            # Use a user-supplied name if one is present
            tty.die("Couldn't guess a name for this package. Try running:", "",
                    "spack create --name <name> <url>")

    if not valid_module_name(name):
        tty.die("Package name can only contain A-Z, a-z, 0-9, '_' and '-'")

    tty.msg("This looks like a URL for %s version %s." % (name, version))
    tty.msg("Creating template for package %s" % name)

    versions = spack.util.web.find_versions_of_archive(url)
    rkeys = sorted(versions.keys(), reverse=True)
    versions = OrderedDict(zip(rkeys, (versions[v] for v in rkeys)))

    archives_to_fetch = 1
    if not versions:
        # If the fetch failed for some reason, revert to what the user provided
        versions = { version : url }
    elif len(versions) > 1:
        tty.msg("Found %s versions of %s:" % (len(versions), name),
                *spack.cmd.elide_list(
                    ["%-10s%s" % (v,u) for v, u in versions.iteritems()]))
        print
        archives_to_fetch = tty.get_number(
            "Include how many checksums in the package file?",
            default=5, abort='q')

        if not archives_to_fetch:
Esempio n. 29
0
import re
import inspect
import textwrap
from six import StringIO
from argparse import ArgumentParser
from ordereddict_backport import OrderedDict

import pymod.paths
import pymod.callback

category_descriptions = OrderedDict(
    environment="Functions for modifying the environment",
    path="Functions for modifying path-like variables",
    alias="Functions for defining shell aliases and functions",
    module="General module functions",
    interaction="Functions for interacting with other modules",
    family="Functions for interacting with module families",
    modulepath="Functions for interacting with the MODULEPATH",
    info="Functions for relaying information",
    utility="General purpose utilities",
)


def fill_with_paragraphs(string, indent=""):
    filled = []
    lines = []
    for line in string.split("\n"):
        if line.split():
            lines.append(line)
        elif lines:
            filled.append(
Esempio n. 30
0
class Environment(object):
    def __init__(self, path, init_file=None, with_view=None):
        """Create a new environment.

        The environment can be optionally initialized with either a
        spack.yaml or spack.lock file.

        Arguments:
            path (str): path to the root directory of this environment
            init_file (str or file object): filename or file object to
                initialize the environment
            with_view (str or bool): whether a view should be maintained for
                the environment. If the value is a string, it specifies the
                path to the view.
        """
        self.path = os.path.abspath(path)
        self.clear()

        if init_file:
            with fs.open_if_filename(init_file) as f:
                if hasattr(f, 'name') and f.name.endswith('.lock'):
                    self._read_manifest(default_manifest_yaml)
                    self._read_lockfile(f)
                    self._set_user_specs_from_lockfile()
                else:
                    self._read_manifest(f)
        else:
            default_manifest = not os.path.exists(self.manifest_path)
            if default_manifest:
                # No manifest, use default yaml
                self._read_manifest(default_manifest_yaml)
            else:
                with open(self.manifest_path) as f:
                    self._read_manifest(f)

            if os.path.exists(self.lock_path):
                with open(self.lock_path) as f:
                    read_lock_version = self._read_lockfile(f)
                if default_manifest:
                    # No manifest, set user specs from lockfile
                    self._set_user_specs_from_lockfile()

                if read_lock_version == 1:
                    tty.debug(
                        "Storing backup of old lockfile {0} at {1}".format(
                            self.lock_path, self._lock_backup_v1_path))
                    shutil.copy(self.lock_path, self._lock_backup_v1_path)

        if with_view is False:
            self.views = {}
        elif with_view is True:
            self.views = {
                default_view_name: ViewDescriptor(self.view_path_default)
            }
        elif isinstance(with_view, six.string_types):
            self.views = {default_view_name: ViewDescriptor(with_view)}
        # If with_view is None, then defer to the view settings determined by
        # the manifest file

    def _read_manifest(self, f):
        """Read manifest file and set up user specs."""
        self.yaml = _read_yaml(f)

        self.spec_lists = OrderedDict()

        for item in config_dict(self.yaml).get('definitions', []):
            entry = copy.deepcopy(item)
            when = _eval_conditional(entry.pop('when', 'True'))
            assert len(entry) == 1
            if when:
                name, spec_list = next(iter(entry.items()))
                user_specs = SpecList(name, spec_list, self.spec_lists.copy())
                if name in self.spec_lists:
                    self.spec_lists[name].extend(user_specs)
                else:
                    self.spec_lists[name] = user_specs

        spec_list = config_dict(self.yaml).get(user_speclist_name)
        user_specs = SpecList(user_speclist_name, [s for s in spec_list if s],
                              self.spec_lists.copy())
        self.spec_lists[user_speclist_name] = user_specs

        enable_view = config_dict(self.yaml).get('view')
        # enable_view can be boolean, string, or None
        if enable_view is True or enable_view is None:
            self.views = {
                default_view_name: ViewDescriptor(self.view_path_default)
            }
        elif isinstance(enable_view, six.string_types):
            self.views = {default_view_name: ViewDescriptor(enable_view)}
        elif enable_view:
            self.views = dict((name, ViewDescriptor.from_dict(values))
                              for name, values in enable_view.items())
        else:
            self.views = {}

    @property
    def user_specs(self):
        return self.spec_lists[user_speclist_name]

    def _set_user_specs_from_lockfile(self):
        """Copy user_specs from a read-in lockfile."""
        self.spec_lists = {
            user_speclist_name:
            SpecList(user_speclist_name,
                     [str(s) for s in self.concretized_user_specs])
        }

    def clear(self):
        self.spec_lists = {user_speclist_name: SpecList()}  # specs from yaml
        self.concretized_user_specs = []  # user specs from last concretize
        self.concretized_order = []  # roots of last concretize, in order
        self.specs_by_hash = {}  # concretized specs by hash
        self.new_specs = []  # write packages for these on write()
        self._repo = None  # RepoPath for this env (memoized)
        self._previous_active = None  # previously active environment

    @property
    def internal(self):
        """Whether this environment is managed by Spack."""
        return self.path.startswith(env_path)

    @property
    def name(self):
        """Human-readable representation of the environment.

        This is the path for directory environments, and just the name
        for named environments.
        """
        if self.internal:
            return os.path.basename(self.path)
        else:
            return self.path

    @property
    def active(self):
        """True if this environment is currently active."""
        return _active_environment and self.path == _active_environment.path

    @property
    def manifest_path(self):
        """Path to spack.yaml file in this environment."""
        return os.path.join(self.path, manifest_name)

    @property
    def lock_path(self):
        """Path to spack.lock file in this environment."""
        return os.path.join(self.path, lockfile_name)

    @property
    def _lock_backup_v1_path(self):
        """Path to backup of v1 lockfile before conversion to v2"""
        return self.lock_path + '.backup.v1'

    @property
    def env_subdir_path(self):
        """Path to directory where the env stores repos, logs, views."""
        return os.path.join(self.path, env_subdir_name)

    @property
    def repos_path(self):
        return os.path.join(self.path, env_subdir_name, 'repos')

    @property
    def log_path(self):
        return os.path.join(self.path, env_subdir_name, 'logs')

    @property
    def view_path_default(self):
        # default path for environment views
        return os.path.join(self.env_subdir_path, 'view')

    @property
    def repo(self):
        if self._repo is None:
            self._repo = make_repo_path(self.repos_path)
        return self._repo

    def included_config_scopes(self):
        """List of included configuration scopes from the environment.

        Scopes are listed in the YAML file in order from highest to
        lowest precedence, so configuration from earlier scope will take
        precedence over later ones.

        This routine returns them in the order they should be pushed onto
        the internal scope stack (so, in reverse, from lowest to highest).
        """
        scopes = []

        # load config scopes added via 'include:', in reverse so that
        # highest-precedence scopes are last.
        includes = config_dict(self.yaml).get('include', [])
        for i, config_path in enumerate(reversed(includes)):
            # allow paths to contain environment variables
            config_path = config_path.format(**os.environ)

            # treat relative paths as relative to the environment
            if not os.path.isabs(config_path):
                config_path = os.path.join(self.path, config_path)
                config_path = os.path.normpath(os.path.realpath(config_path))

            if os.path.isdir(config_path):
                # directories are treated as regular ConfigScopes
                config_name = 'env:%s:%s' % (self.name,
                                             os.path.basename(config_path))
                scope = spack.config.ConfigScope(config_name, config_path)
            else:
                # files are assumed to be SingleFileScopes
                base, ext = os.path.splitext(os.path.basename(config_path))
                config_name = 'env:%s:%s' % (self.name, base)
                scope = spack.config.SingleFileScope(
                    config_name, config_path, spack.schema.merged.schema)

            scopes.append(scope)

        return scopes

    def env_file_config_scope_name(self):
        """Name of the config scope of this environment's manifest file."""
        return 'env:%s' % self.name

    def env_file_config_scope(self):
        """Get the configuration scope for the environment's manifest file."""
        config_name = self.env_file_config_scope_name()
        return spack.config.SingleFileScope(config_name, self.manifest_path,
                                            spack.schema.env.schema,
                                            [env_schema_keys])

    def config_scopes(self):
        """A list of all configuration scopes for this environment."""
        return self.included_config_scopes() + [self.env_file_config_scope()]

    def destroy(self):
        """Remove this environment from Spack entirely."""
        shutil.rmtree(self.path)

    def update_stale_references(self, from_list=None):
        """Iterate over spec lists updating references."""
        if not from_list:
            from_list = next(iter(self.spec_lists.keys()))
        index = list(self.spec_lists.keys()).index(from_list)

        # spec_lists is an OrderedDict, all list entries after the modified
        # list may refer to the modified list. Update stale references
        for i, (name, speclist) in enumerate(
                list(self.spec_lists.items())[index + 1:], index + 1):
            new_reference = dict((n, self.spec_lists[n])
                                 for n in list(self.spec_lists.keys())[:i])
            speclist.update_reference(new_reference)

    def add(self, user_spec, list_name=user_speclist_name):
        """Add a single user_spec (non-concretized) to the Environment

        Returns:
            (bool): True if the spec was added, False if it was already
                present and did not need to be added

        """
        spec = Spec(user_spec)

        if list_name not in self.spec_lists:
            raise SpackEnvironmentError('No list %s exists in environment %s' %
                                        (list_name, self.name))

        if list_name == user_speclist_name:
            if not spec.name:
                raise SpackEnvironmentError(
                    'cannot add anonymous specs to an environment!')
            elif not spack.repo.path.exists(spec.name):
                raise SpackEnvironmentError('no such package: %s' % spec.name)

        list_to_change = self.spec_lists[list_name]
        existing = str(spec) in list_to_change.yaml_list
        if not existing:
            list_to_change.add(str(spec))
            self.update_stale_references(list_name)

        return bool(not existing)

    def remove(self, query_spec, list_name=user_speclist_name, force=False):
        """Remove specs from an environment that match a query_spec"""
        query_spec = Spec(query_spec)

        list_to_change = self.spec_lists[list_name]
        matches = []

        if not query_spec.concrete:
            matches = [s for s in list_to_change if s.satisfies(query_spec)]

        if not matches:
            # concrete specs match against concrete specs in the env
            specs_hashes = zip(self.concretized_user_specs,
                               self.concretized_order)
            matches = [
                s for s, h in specs_hashes if query_spec.dag_hash() == h
            ]

        if not matches:
            raise SpackEnvironmentError("Not found: {0}".format(query_spec))

        old_specs = set(self.user_specs)
        for spec in matches:
            if spec in list_to_change:
                list_to_change.remove(spec)

        self.update_stale_references(list_name)

        # If force, update stale concretized specs
        # Only check specs removed by this operation
        new_specs = set(self.user_specs)
        for spec in old_specs - new_specs:
            if force and spec in self.concretized_user_specs:
                i = self.concretized_user_specs.index(spec)
                del self.concretized_user_specs[i]

                dag_hash = self.concretized_order[i]
                del self.concretized_order[i]
                del self.specs_by_hash[dag_hash]

    def concretize(self, force=False):
        """Concretize user_specs in this environment.

        Only concretizes specs that haven't been concretized yet unless
        force is ``True``.

        This only modifies the environment in memory. ``write()`` will
        write out a lockfile containing concretized specs.

        Arguments:
            force (bool): re-concretize ALL specs, even those that were
               already concretized

        Returns:
            List of specs that have been concretized. Each entry is a tuple of
            the user spec and the corresponding concretized spec.
        """
        if force:
            # Clear previously concretized specs
            self.concretized_user_specs = []
            self.concretized_order = []
            self.specs_by_hash = {}

        # keep any concretized specs whose user specs are still in the manifest
        old_concretized_user_specs = self.concretized_user_specs
        old_concretized_order = self.concretized_order
        old_specs_by_hash = self.specs_by_hash

        self.concretized_user_specs = []
        self.concretized_order = []
        self.specs_by_hash = {}

        for s, h in zip(old_concretized_user_specs, old_concretized_order):
            if s in self.user_specs:
                concrete = old_specs_by_hash[h]
                self._add_concrete_spec(s, concrete, new=False)

        # Concretize any new user specs that we haven't concretized yet
        concretized_specs = []
        for uspec, uspec_constraints in zip(
                self.user_specs, self.user_specs.specs_as_constraints):
            if uspec not in old_concretized_user_specs:
                concrete = _concretize_from_constraints(uspec_constraints)
                self._add_concrete_spec(uspec, concrete)
                concretized_specs.append((uspec, concrete))
        return concretized_specs

    def install(self, user_spec, concrete_spec=None, **install_args):
        """Install a single spec into an environment.

        This will automatically concretize the single spec, but it won't
        affect other as-yet unconcretized specs.
        """
        spec = Spec(user_spec)

        if self.add(spec):
            concrete = concrete_spec if concrete_spec else spec.concretized()
            self._add_concrete_spec(spec, concrete)
        else:
            # spec might be in the user_specs, but not installed.
            # TODO: Redo name-based comparison for old style envs
            spec = next(s for s in self.user_specs if s.satisfies(user_spec))
            concrete = self.specs_by_hash.get(spec.build_hash())
            if not concrete:
                concrete = spec.concretized()
                self._add_concrete_spec(spec, concrete)

        self._install(concrete, **install_args)

    def _install(self, spec, **install_args):
        spec.package.do_install(**install_args)

        # Make sure log directory exists
        log_path = self.log_path
        fs.mkdirp(log_path)

        with fs.working_dir(self.path):
            # Link the resulting log file into logs dir
            build_log_link = os.path.join(
                log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
            if os.path.lexists(build_log_link):
                os.remove(build_log_link)
            os.symlink(spec.package.build_log_path, build_log_link)

    @property
    def default_view(self):
        if not self.views:
            raise SpackEnvironmentError(
                "{0} does not have a view enabled".format(self.name))

        if default_view_name not in self.views:
            raise SpackEnvironmentError(
                "{0} does not have a default view enabled".format(self.name))

        return self.views[default_view_name]

    def update_default_view(self, viewpath):
        name = default_view_name
        if name in self.views and self.default_view.root != viewpath:
            shutil.rmtree(self.default_view.root)

        if viewpath:
            if name in self.views:
                self.default_view.root = viewpath
            else:
                self.views[name] = ViewDescriptor(viewpath)
        else:
            self.views.pop(name, None)

    def regenerate_views(self):
        if not self.views:
            tty.debug("Skip view update, this environment does not"
                      " maintain a view")
            return

        specs = self._get_environment_specs()
        for view in self.views.values():
            view.regenerate(specs, self.roots())

    def _shell_vars(self):
        updates = [
            ('PATH', ['bin']),
            ('MANPATH', ['man', 'share/man']),
            ('ACLOCAL_PATH', ['share/aclocal']),
            ('LD_LIBRARY_PATH', ['lib', 'lib64']),
            ('LIBRARY_PATH', ['lib', 'lib64']),
            ('CPATH', ['include']),
            ('PKG_CONFIG_PATH', ['lib/pkgconfig', 'lib64/pkgconfig']),
            ('CMAKE_PREFIX_PATH', ['']),
        ]
        path_updates = list()
        if default_view_name in self.views:
            for var, subdirs in updates:
                paths = filter(
                    lambda x: os.path.exists(x),
                    list(
                        os.path.join(self.default_view.root, x)
                        for x in subdirs))
                path_updates.append((var, paths))
        return path_updates

    def add_default_view_to_shell(self, shell):
        env_mod = EnvironmentModifications()
        for var, paths in self._shell_vars():
            for path in paths:
                env_mod.prepend_path(var, path)
        return env_mod.shell_modifications(shell)

    def rm_default_view_from_shell(self, shell):
        env_mod = EnvironmentModifications()
        for var, paths in self._shell_vars():
            for path in paths:
                env_mod.remove_path(var, path)
        return env_mod.shell_modifications(shell)

    def _add_concrete_spec(self, spec, concrete, new=True):
        """Called when a new concretized spec is added to the environment.

        This ensures that all internal data structures are kept in sync.

        Arguments:
            spec (Spec): user spec that resulted in the concrete spec
            concrete (Spec): spec concretized within this environment
            new (bool): whether to write this spec's package to the env
                repo on write()
        """
        assert concrete.concrete

        # when a spec is newly concretized, we need to make a note so
        # that we can write its package to the env repo on write()
        if new:
            self.new_specs.append(concrete)

        # update internal lists of specs
        self.concretized_user_specs.append(spec)

        h = concrete.build_hash()
        self.concretized_order.append(h)
        self.specs_by_hash[h] = concrete

    def install_all(self, args=None):
        """Install all concretized specs in an environment."""
        for concretized_hash in self.concretized_order:
            spec = self.specs_by_hash[concretized_hash]

            # Parse cli arguments and construct a dictionary
            # that will be passed to Package.do_install API
            kwargs = dict()
            if args:
                spack.cmd.install.update_kwargs_from_args(args, kwargs)

            self._install(spec, **kwargs)

            if not spec.external:
                # Link the resulting log file into logs dir
                build_log_link = os.path.join(
                    self.log_path, '%s-%s.log' % (spec.name, spec.dag_hash(7)))
                if os.path.lexists(build_log_link):
                    os.remove(build_log_link)
                os.symlink(spec.package.build_log_path, build_log_link)

        self.regenerate_views()

    def all_specs_by_hash(self):
        """Map of hashes to spec for all specs in this environment."""
        # Note this uses dag-hashes calculated without build deps as keys,
        # whereas the environment tracks specs based on dag-hashes calculated
        # with all dependencies. This function should not be used by an
        # Environment object for management of its own data structures
        hashes = {}
        for h in self.concretized_order:
            specs = self.specs_by_hash[h].traverse(deptype=('link', 'run'))
            for spec in specs:
                hashes[spec.dag_hash()] = spec
        return hashes

    def all_specs(self):
        """Return all specs, even those a user spec would shadow."""
        return sorted(self.all_specs_by_hash().values())

    def all_hashes(self):
        """Return all specs, even those a user spec would shadow."""
        return list(self.all_specs_by_hash().keys())

    def roots(self):
        """Specs explicitly requested by the user *in this environment*.

        Yields both added and installed specs that have user specs in
        `spack.yaml`.
        """
        concretized = dict(self.concretized_specs())
        for spec in self.user_specs:
            concrete = concretized.get(spec)
            yield concrete if concrete else spec

    def added_specs(self):
        """Specs that are not yet installed.

        Yields the user spec for non-concretized specs, and the concrete
        spec for already concretized but not yet installed specs.
        """
        concretized = dict(self.concretized_specs())
        for spec in self.user_specs:
            concrete = concretized.get(spec)
            if not concrete:
                yield spec
            elif not concrete.package.installed:
                yield concrete

    def concretized_specs(self):
        """Tuples of (user spec, concrete spec) for all concrete specs."""
        for s, h in zip(self.concretized_user_specs, self.concretized_order):
            yield (s, self.specs_by_hash[h])

    def removed_specs(self):
        """Tuples of (user spec, concrete spec) for all specs that will be
           removed on nexg concretize."""
        needed = set()
        for s, c in self.concretized_specs():
            if s in self.user_specs:
                for d in c.traverse():
                    needed.add(d)

        for s, c in self.concretized_specs():
            for d in c.traverse():
                if d not in needed:
                    yield d

    def _get_environment_specs(self, recurse_dependencies=True):
        """Returns the specs of all the packages in an environment.

        If these specs appear under different user_specs, only one copy
        is added to the list returned.
        """
        spec_list = list()

        for spec_hash in self.concretized_order:
            spec = self.specs_by_hash[spec_hash]

            specs = (spec.traverse(
                deptype=('link', 'run')) if recurse_dependencies else (spec, ))

            spec_list.extend(specs)

        return spec_list

    def _to_lockfile_dict(self):
        """Create a dictionary to store a lockfile for this environment."""
        concrete_specs = {}
        for spec in self.specs_by_hash.values():
            for s in spec.traverse():
                dag_hash_all = s.build_hash()
                if dag_hash_all not in concrete_specs:
                    spec_dict = s.to_node_dict(hash=ht.build_hash)
                    spec_dict[s.name]['hash'] = s.dag_hash()
                    concrete_specs[dag_hash_all] = spec_dict

        hash_spec_list = zip(self.concretized_order,
                             self.concretized_user_specs)

        # this is the lockfile we'll write out
        data = {
            # metadata about the format
            '_meta': {
                'file-type': 'spack-lockfile',
                'lockfile-version': lockfile_format_version,
            },

            # users specs + hashes are the 'roots' of the environment
            'roots': [{
                'hash': h,
                'spec': str(s)
            } for h, s in hash_spec_list],

            # Concrete specs by hash, including dependencies
            'concrete_specs': concrete_specs,
        }

        return data

    def _read_lockfile(self, file_or_json):
        """Read a lockfile from a file or from a raw string."""
        lockfile_dict = sjson.load(file_or_json)
        self._read_lockfile_dict(lockfile_dict)
        return lockfile_dict['_meta']['lockfile-version']

    def _read_lockfile_dict(self, d):
        """Read a lockfile dictionary into this environment."""
        roots = d['roots']
        self.concretized_user_specs = [Spec(r['spec']) for r in roots]
        self.concretized_order = [r['hash'] for r in roots]

        json_specs_by_hash = d['concrete_specs']
        root_hashes = set(self.concretized_order)

        specs_by_hash = {}
        for dag_hash, node_dict in json_specs_by_hash.items():
            specs_by_hash[dag_hash] = Spec.from_node_dict(node_dict)

        for dag_hash, node_dict in json_specs_by_hash.items():
            for dep_name, dep_hash, deptypes in (
                    Spec.dependencies_from_node_dict(node_dict)):
                specs_by_hash[dag_hash]._add_dependency(
                    specs_by_hash[dep_hash], deptypes)

        # If we are reading an older lockfile format (which uses dag hashes
        # that exclude build deps), we use this to convert the old
        # concretized_order to the full hashes (preserving the order)
        old_hash_to_new = {}
        self.specs_by_hash = {}
        for _, spec in specs_by_hash.items():
            dag_hash = spec.dag_hash()
            build_hash = spec.build_hash()
            if dag_hash in root_hashes:
                old_hash_to_new[dag_hash] = build_hash

            if (dag_hash in root_hashes or build_hash in root_hashes):
                self.specs_by_hash[build_hash] = spec

        if old_hash_to_new:
            # Replace any older hashes in concretized_order with hashes
            # that include build deps
            self.concretized_order = [
                old_hash_to_new.get(h, h) for h in self.concretized_order
            ]

    def write(self):
        """Writes an in-memory environment to its location on disk.

        This will also write out package files for each newly concretized spec.
        """
        # ensure path in var/spack/environments
        fs.mkdirp(self.path)

        if self.specs_by_hash:
            # ensure the prefix/.env directory exists
            fs.mkdirp(self.env_subdir_path)

            for spec in self.new_specs:
                for dep in spec.traverse():
                    if not dep.concrete:
                        raise ValueError('specs passed to environment.write() '
                                         'must be concrete!')

                    root = os.path.join(self.repos_path, dep.namespace)
                    repo = spack.repo.create_or_construct(root, dep.namespace)
                    pkg_dir = repo.dirname_for_package_name(dep.name)

                    fs.mkdirp(pkg_dir)
                    spack.repo.path.dump_provenance(dep, pkg_dir)
            self.new_specs = []

            # write the lock file last
            with fs.write_tmp_and_move(self.lock_path) as f:
                sjson.dump(self._to_lockfile_dict(), stream=f)
        else:
            if os.path.exists(self.lock_path):
                os.unlink(self.lock_path)

        # invalidate _repo cache
        self._repo = None

        # put any changes in the definitions in the YAML
        for name, speclist in self.spec_lists.items():
            if name == user_speclist_name:
                # The primary list is handled differently
                continue

            conf = config_dict(self.yaml)
            active_yaml_lists = [
                l for l in conf.get('definitions', [])
                if name in l and _eval_conditional(l.get('when', 'True'))
            ]

            # Remove any specs in yaml that are not in internal representation
            for ayl in active_yaml_lists:
                # If it's not a string, it's a matrix. Those can't have changed
                # If it is a string that starts with '$', it's a reference.
                # Those also can't have changed.
                ayl[name][:] = [
                    s for s in ayl.setdefault(name, [])
                    if (not isinstance(s, six.string_types))
                    or s.startswith('$') or Spec(s) in speclist.specs
                ]

            # Put the new specs into the first active list from the yaml
            new_specs = [
                entry for entry in speclist.yaml_list
                if isinstance(entry, six.string_types) and not any(
                    entry in ayl[name] for ayl in active_yaml_lists)
            ]
            list_for_new_specs = active_yaml_lists[0].setdefault(name, [])
            list_for_new_specs[:] = list_for_new_specs + new_specs

        # put the new user specs in the YAML.
        # This can be done directly because there can't be multiple definitions
        # nor when clauses for `specs` list.
        yaml_spec_list = config_dict(self.yaml).setdefault(
            user_speclist_name, [])
        yaml_spec_list[:] = self.user_specs.yaml_list

        default_name = default_view_name
        if self.views and len(self.views) == 1 and default_name in self.views:
            path = self.default_view.root
            if self.default_view == ViewDescriptor(self.view_path_default):
                view = True
            elif self.default_view == ViewDescriptor(path):
                view = path
            else:
                view = dict((name, view.to_dict())
                            for name, view in self.views.items())
        elif self.views:
            view = dict(
                (name, view.to_dict()) for name, view in self.views.items())
        else:
            view = False

        yaml_dict = config_dict(self.yaml)
        if view is not True:
            # The default case is to keep an active view inside of the
            # Spack environment directory. To avoid cluttering the config,
            # we omit the setting in this case.
            yaml_dict['view'] = view
        elif 'view' in yaml_dict:
            del yaml_dict['view']

        # if all that worked, write out the manifest file at the top level
        with fs.write_tmp_and_move(self.manifest_path) as f:
            _write_yaml(self.yaml, f)

        # TODO: for operations that just add to the env (install etc.) this
        # could just call update_view
        self.regenerate_views()

    def __enter__(self):
        self._previous_active = _active_environment
        activate(self)
        return

    def __exit__(self, exc_type, exc_val, exc_tb):
        deactivate()
        if self._previous_active:
            activate(self._previous_active)
Esempio n. 31
0
class Configuration(object):
    """A full Spack configuration, from a hierarchy of config files.

    This class makes it easy to add a new scope on top of an existing one.
    """

    def __init__(self, *scopes):
        """Initialize a configuration with an initial list of scopes.

        Args:
            scopes (list of ConfigScope): list of scopes to add to this
                Configuration, ordered from lowest to highest precedence

        """
        self.scopes = OrderedDict()
        for scope in scopes:
            self.push_scope(scope)

    def push_scope(self, scope):
        """Add a higher precedence scope to the Configuration."""
        cmd_line_scope = None
        if self.scopes:
            highest_precedence_scope = list(self.scopes.values())[-1]
            if highest_precedence_scope.name == 'command_line':
                # If the command-line scope is present, it should always
                # be the scope of highest precedence
                cmd_line_scope = self.pop_scope()

        self.scopes[scope.name] = scope
        if cmd_line_scope:
            self.scopes['command_line'] = cmd_line_scope

    def pop_scope(self):
        """Remove the highest precedence scope and return it."""
        name, scope = self.scopes.popitem(last=True)
        return scope

    def remove_scope(self, scope_name):
        return self.scopes.pop(scope_name)

    @property
    def file_scopes(self):
        """List of writable scopes with an associated file."""
        return [s for s in self.scopes.values() if type(s) == ConfigScope]

    def highest_precedence_scope(self):
        """Non-internal scope with highest precedence."""
        return next(reversed(self.file_scopes), None)

    def matching_scopes(self, reg_expr):
        """
        List of all scopes whose names match the provided regular expression.

        For example, matching_scopes(r'^command') will return all scopes
        whose names begin with `command`.
        """
        return [s for s in self.scopes.values() if re.search(reg_expr, s.name)]

    def _validate_scope(self, scope):
        """Ensure that scope is valid in this configuration.

        This should be used by routines in ``config.py`` to validate
        scope name arguments, and to determine a default scope where no
        scope is specified.

        Raises:
            ValueError: if ``scope`` is not valid

        Returns:
            ConfigScope: a valid ConfigScope if ``scope`` is ``None`` or valid
        """
        if scope is None:
            # default to the scope with highest precedence.
            return self.highest_precedence_scope()

        elif scope in self.scopes:
            return self.scopes[scope]

        else:
            raise ValueError("Invalid config scope: '%s'.  Must be one of %s"
                             % (scope, self.scopes.keys()))

    def get_config_filename(self, scope, section):
        """For some scope and section, get the name of the configuration file.
        """
        scope = self._validate_scope(scope)
        return scope.get_section_filename(section)

    def clear_caches(self):
        """Clears the caches for configuration files,

        This will cause files to be re-read upon the next request."""
        for scope in self.scopes.values():
            scope.clear()

    def update_config(self, section, update_data, scope=None):
        """Update the configuration file for a particular scope.

        Overwrites contents of a section in a scope with update_data,
        then writes out the config file.

        update_data should have the top-level section name stripped off
        (it will be re-added).  Data itself can be a list, dict, or any
        other yaml-ish structure.
        """
        _validate_section_name(section)  # validate section name
        scope = self._validate_scope(scope)  # get ConfigScope object

        # read only the requested section's data.
        scope.sections[section] = {section: update_data}
        scope.write_section(section)

    def get_config(self, section, scope=None):
        """Get configuration settings for a section.

        If ``scope`` is ``None`` or not provided, return the merged contents
        of all of Spack's configuration scopes.  If ``scope`` is provided,
        return only the confiugration as specified in that scope.

        This off the top-level name from the YAML section.  That is, for a
        YAML config file that looks like this::

           config:
             install_tree: $spack/opt/spack
             module_roots:
               lmod:   $spack/share/spack/lmod

        ``get_config('config')`` will return::

           { 'install_tree': '$spack/opt/spack',
             'module_roots: {
                 'lmod': '$spack/share/spack/lmod'
             }
           }

        """
        _validate_section_name(section)

        if scope is None:
            scopes = self.scopes.values()
        else:
            scopes = [self._validate_scope(scope)]

        merged_section = syaml.syaml_dict()
        for scope in scopes:
            # read potentially cached data from the scope.

            data = scope.get_section(section)

            # Skip empty configs
            if not data or not isinstance(data, dict):
                continue

            if section not in data:
                continue

            merged_section = _merge_yaml(merged_section, data)

        # no config files -- empty config.
        if section not in merged_section:
            return {}

        # take the top key off before returning.
        return merged_section[section]

    def get(self, path, default=None, scope=None):
        """Get a config section or a single value from one.

        Accepts a path syntax that allows us to grab nested config map
        entries.  Getting the 'config' section would look like::

            spack.config.get('config')

        and the ``dirty`` section in the ``config`` scope would be::

            spack.config.get('config:dirty')

        We use ``:`` as the separator, like YAML objects.
    """
        # TODO: Currently only handles maps. Think about lists if neded.
        section, _, rest = path.partition(':')

        value = self.get_config(section, scope=scope)
        if not rest:
            return value

        parts = rest.split(':')
        while parts:
            key = parts.pop(0)
            value = value.get(key, default)

        return value

    def set(self, path, value, scope=None):
        """Convenience function for setting single values in config files.

        Accepts the path syntax described in ``get()``.
        """
        section, _, rest = path.partition(':')

        if not rest:
            self.update_config(section, value, scope=scope)
        else:
            section_data = self.get_config(section, scope=scope)

            parts = rest.split(':')
            data = section_data
            while len(parts) > 1:
                key = parts.pop(0)
                data = data[key]
            data[parts[0]] = value

            self.update_config(section, section_data, scope=scope)

    def __iter__(self):
        """Iterate over scopes in this configuration."""
        for scope in self.scopes.values():
            yield scope

    def print_section(self, section, blame=False):
        """Print a configuration to stdout."""
        try:
            data = syaml.syaml_dict()
            data[section] = self.get_config(section)
            syaml.dump_config(
                data, stream=sys.stdout, default_flow_style=False, blame=blame)
        except (yaml.YAMLError, IOError):
            raise ConfigError("Error reading configuration: %s" % section)
Esempio n. 32
0
def upgrade_None_to_1_0(new, old, depth=[0]):

    depth[0] += 1
    if depth[0] > 1:
        raise ValueError("Recursion!")

    version_string = ".".join(str(_) for _ in new.version)
    tty.info(
        "Converting Modulecmd.py collections version 0.0 to "
        "version {0}".format(version_string)
    )
    new_collections = {}
    for (name, old_collection) in old.items():
        new_collection = OrderedDict()
        mp = pymod.modulepath.Modulepath([])
        for (path, m_descs) in old_collection:
            if new_collection is None:
                break
            if not os.path.isdir(path):
                tty.warn(
                    "Collection {0} contains directory {1} which "
                    "does not exist!  This collection will be skipped".format(
                        name, path
                    )
                )
                new_collection = None
                break
            avail = mp.append_path(path)
            if avail is None:
                tty.warn(
                    "Collection {0} contains directory {1} which "
                    "does not have any available modules!  "
                    "This collection will be skipped".format(name, path)
                )
                new_collection = None
                break
            for (fullname, filename, opts) in m_descs:
                m = mp.get(filename)
                if m is None:
                    tty.warn(
                        "Collection {0} requests module {1} which "
                        "can not be found! This collection will be skipped".format(
                            name, fullname
                        )
                    )
                    new_collection = None
                    break
                m.opts = opts
                m.acquired_as = m.fullname
                ar = pymod.mc.archive_module(m)
                new_collection.setdefault(m.modulepath, []).append(ar)

        if new_collection is None:
            tty.warn(
                "Skipping collection {0} because of previous " "errors".format(name)
            )
            continue

        new_collections[name] = list(new_collection.items())

    bak = new.filename + ".bak"
    with open(bak, "w") as fh:
        json.dump(old, fh, indent=2)

    new.write(list(new_collections.items()), new.filename)
    return new_collections
Esempio n. 33
0
class MirrorCollection(Mapping):
    """A mapping of mirror names to mirrors."""
    def __init__(self, mirrors=None, scope=None):
        self._mirrors = OrderedDict(
            (name, Mirror.from_dict(mirror, name)) for name, mirror in (
                mirrors.items() if mirrors is not None else spack.config.
                get('mirrors', scope=scope).items()))

    def to_json(self, stream=None):
        return sjson.dump(self.to_dict(True), stream)

    def to_yaml(self, stream=None):
        return syaml.dump(self.to_dict(True), stream)

    # TODO: this isn't called anywhere
    @staticmethod
    def from_yaml(stream, name=None):
        try:
            data = syaml.load(stream)
            return MirrorCollection(data)
        except yaml_error.MarkedYAMLError as e:
            raise syaml.SpackYAMLError("error parsing YAML spec:", str(e))

    @staticmethod
    def from_json(stream, name=None):
        d = sjson.load(stream)
        return MirrorCollection(d)

    def to_dict(self, recursive=False):
        return syaml_dict(
            sorted(((k, (v.to_dict() if recursive else v))
                    for (k, v) in self._mirrors.items()),
                   key=operator.itemgetter(0)))

    @staticmethod
    def from_dict(d):
        return MirrorCollection(d)

    def __getitem__(self, item):
        return self._mirrors[item]

    def display(self):
        max_len = max(len(mirror.name) for mirror in self._mirrors.values())
        for mirror in self._mirrors.values():
            mirror.display(max_len)

    def lookup(self, name_or_url):
        """Looks up and returns a Mirror.

        If this MirrorCollection contains a named Mirror under the name
        [name_or_url], then that mirror is returned.  Otherwise, [name_or_url]
        is assumed to be a mirror URL, and an anonymous mirror with the given
        URL is returned.
        """
        result = self.get(name_or_url)

        if result is None:
            result = Mirror(fetch_url=name_or_url)

        return result

    def __iter__(self):
        return iter(self._mirrors)

    def __len__(self):
        return len(self._mirrors)
Esempio n. 34
0
 def __init__(self, mirrors=None, scope=None):
     self._mirrors = OrderedDict(
         (name, Mirror.from_dict(mirror, name)) for name, mirror in (
             mirrors.items() if mirrors is not None else spack.config.
             get('mirrors', scope=scope).items()))