def find_spec(spec, condition): """Searches the dag from spec in an intelligent order and looks for a spec that matches a condition""" # First search parents, then search children deptype = ('build', 'link') dagiter = chain( spec.traverse(direction='parents', deptype=deptype, root=False), spec.traverse(direction='children', deptype=deptype, root=False)) visited = set() for relative in dagiter: if condition(relative): return relative visited.add(id(relative)) # Then search all other relatives in the DAG *except* spec for relative in spec.root.traverse(deptypes=spack.alldeps): if relative is spec: continue if id(relative) in visited: continue if condition(relative): return relative # Finally search spec itself. if condition(spec): return spec return None # Nothing matched the condition.
def _get_environment_specs(self, recurse_dependencies=True): """Returns the specs of all the packages in an environment. If these specs appear under different user_specs, only one copy is added to the list returned. """ package_to_spec = {} spec_list = list() for spec_hash in self.concretized_order: spec = self.specs_by_hash[spec_hash] specs = (spec.traverse(deptype=('link', 'run')) if recurse_dependencies else (spec,)) for dep in specs: prior = package_to_spec.get(dep.name) if prior and prior != dep: tty.debug("{0} takes priority over {1}" .format(package_to_spec[dep.name].format(), dep.format())) else: package_to_spec[dep.name] = dep spec_list.append(dep) return spec_list
def _to_lockfile_dict(self): """Create a dictionary to store a lockfile for this environment.""" concrete_specs = {} for spec in self.specs_by_hash.values(): for s in spec.traverse(): dag_hash = s.dag_hash() if dag_hash not in concrete_specs: concrete_specs[dag_hash] = s.to_node_dict(all_deps=True) hash_spec_list = zip( self.concretized_order, self.concretized_user_specs) # this is the lockfile we'll write out data = { # metadata about the format '_meta': { 'file-type': 'spack-lockfile', 'lockfile-version': lockfile_format_version, }, # users specs + hashes are the 'roots' of the environment 'roots': [{ 'hash': h, 'spec': str(s) } for h, s in hash_spec_list], # Concrete specs by hash, including dependencies 'concrete_specs': concrete_specs, } return data
def installed_relatives(self, spec, direction='children', transitive=True): """Return installed specs related to this one.""" if direction not in ('parents', 'children'): raise ValueError("Invalid direction: %s" % direction) relatives = set() for spec in self.query(spec): if transitive: to_add = spec.traverse(direction=direction, root=False) elif direction == 'parents': to_add = spec.dependents() else: # direction == 'children' to_add = spec.dependencies() for relative in to_add: hash_key = relative.dag_hash() if hash_key not in self._data: reltype = ('Dependent' if direction == 'parents' else 'Dependency') tty.warn("Inconsistent state! %s %s of %s not in DB" % (reltype, hash_key, spec.dag_hash())) continue if not self._data[hash_key].installed: continue relatives.add(relative) return relatives
def installed_dependents(self, spec): """List the installed specs that depend on this one.""" dependents = set() for spec in self.query(spec): for dependent in spec.traverse(direction='parents', root=False): dependents.add(dependent) return dependents
def test_installed_upstream(upstream_and_downstream_db): upstream_db, upstream_layout, downstream_db, downstream_layout = ( upstream_and_downstream_db) default = ('build', 'link') x = MockPackage('x', [], []) z = MockPackage('z', [], []) y = MockPackage('y', [z], [default]) w = MockPackage('w', [x, y], [default, default]) mock_repo = MockPackageMultiRepo([w, x, y, z]) with spack.repo.swap(mock_repo): spec = spack.spec.Spec('w') spec.concretize() for dep in spec.traverse(root=False): upstream_db.add(dep, upstream_layout) new_spec = spack.spec.Spec('w') new_spec.concretize() downstream_db.add(new_spec, downstream_layout) for dep in new_spec.traverse(root=False): upstream, record = downstream_db.query_by_spec_hash( dep.dag_hash()) assert upstream assert record.path == upstream_layout.path_for_spec(dep) upstream, record = downstream_db.query_by_spec_hash( new_spec.dag_hash()) assert not upstream assert record.installed upstream_db._check_ref_counts() downstream_db._check_ref_counts()
def make_concretization_repository(abstract_specs): """Returns the path to a temporary repository created to contain a fake package that depends on all of the abstract specs. """ tmpdir = tempfile.mkdtemp() repo_path, _ = spack.repo.create_repo(tmpdir) debug_msg = '[CONCRETIZATION]: Creating helper repository in {0}' tty.debug(debug_msg.format(repo_path)) pkg_dir = os.path.join(repo_path, 'packages', 'concretizationroot') fs.mkdirp(pkg_dir) environment = spack.tengine.make_environment() template = environment.get_template('misc/coconcretization.pyt') # Split recursive specs, as it seems the concretizer has issue # respecting conditions on dependents expressed like # depends_on('foo ^[email protected]'), see issue #11160 split_specs = [dep.copy(deps=False) for spec in abstract_specs for dep in spec.traverse(root=True)] with open(os.path.join(pkg_dir, 'package.py'), 'w') as f: f.write(template.render(specs=[str(s) for s in split_specs])) return spack.repo.Repo(repo_path)
def installed_relatives(self, spec, direction='children', transitive=True, deptype='all'): """Return installed specs related to this one.""" if direction not in ('parents', 'children'): raise ValueError("Invalid direction: %s" % direction) relatives = set() for spec in self.query(spec): if transitive: to_add = spec.traverse( direction=direction, root=False, deptype=deptype) elif direction == 'parents': to_add = spec.dependents(deptype=deptype) else: # direction == 'children' to_add = spec.dependencies(deptype=deptype) for relative in to_add: hash_key = relative.dag_hash() upstream, record = self.query_by_spec_hash(hash_key) if not record: reltype = ('Dependent' if direction == 'parents' else 'Dependency') msg = ("Inconsistent state! %s %s of %s not in DB" % (reltype, hash_key, spec.dag_hash())) if self._fail_when_missing_deps: raise MissingDependenciesError(msg) tty.warn(msg) continue if not record.installed: continue relatives.add(relative) return relatives
def format_list(specs): """Display a single list of specs, with no groups""" # create the final, formatted versions of all specs formatted = [] for spec in specs: formatted.append((fmt(spec), spec)) if deps: for depth, dep in spec.traverse(root=False, depth=True): formatted.append((fmt(dep, depth), dep)) formatted.append(('', None)) # mark newlines # unless any of these are set, we can just colify and be done. if not any((deps, paths)): colify((f[0] for f in formatted), indent=indent) return # otherwise, we'll print specs one by one max_width = max(len(f[0]) for f in formatted) path_fmt = "%%-%ds%%s" % (max_width + 2) # getting lots of prefixes requires DB lookups. Ensure # all spec.prefix calls are in one transaction. with spack.store.db.read_transaction(): for string, spec in formatted: if not string: print() # print newline from above continue if paths: print(path_fmt % (string, spec.prefix)) else: print(string)
def _to_lockfile_dict(self): """Create a dictionary to store a lockfile for this environment.""" concrete_specs = {} for spec in self.specs_by_hash.values(): for s in spec.traverse(): dag_hash = s.dag_hash() if dag_hash not in concrete_specs: concrete_specs[dag_hash] = s.to_node_dict(all_deps=True) hash_spec_list = zip(self.concretized_order, self.concretized_user_specs) # this is the lockfile we'll write out data = { # metadata about the format '_meta': { 'file-type': 'spack-lockfile', 'lockfile-version': lockfile_format_version, }, # users specs + hashes are the 'roots' of the environment 'roots': [{ 'hash': h, 'spec': str(s) } for h, s in hash_spec_list], # Concrete specs by hash, including dependencies 'concrete_specs': concrete_specs, } return data
def _get_environment_specs(self, recurse_dependencies=True): """Returns the specs of all the packages in an environment. If these specs appear under different user_specs, only one copy is added to the list returned. """ package_to_spec = {} spec_list = list() for spec_hash in self.concretized_order: spec = self.specs_by_hash[spec_hash] specs = (spec.traverse( deptype=('link', 'run')) if recurse_dependencies else (spec, )) for dep in specs: prior = package_to_spec.get(dep.name) if prior and prior != dep: tty.debug("{0} takes priority over {1}".format( package_to_spec[dep.name].format(), dep.format())) else: package_to_spec[dep.name] = dep spec_list.append(dep) return spec_list
def remove_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) with_dependents = kwargs.get("with_dependents", True) with_dependencies = kwargs.get("with_dependencies", False) # caller can pass this in, as get_all_specs() is expensive all_specs = kwargs.get("all_specs", None) or set(self.get_all_specs()) specs = set(specs) if with_dependencies: specs = get_dependencies(specs) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) to_deactivate = specs to_keep = all_specs - to_deactivate dependents = find_dependents(to_keep, to_deactivate) if with_dependents: # remove all packages depending on the ones to remove if len(dependents) > 0: tty.warn(self._croot + "The following dependents will be removed: %s" % ", ".join((s.name for s in dependents))) to_deactivate.update(dependents) elif len(dependents) > 0: tty.warn(self._croot + "The following packages will be unusable: %s" % ", ".join((s.name for s in dependents))) # Determine the order that packages should be removed from the view; # dependents come before their dependencies. to_deactivate_sorted = list() depmap = dict() for spec in to_deactivate: depmap[spec] = set(d for d in spec.traverse(root=False) if d in to_deactivate) while depmap: for spec in [s for s, d in depmap.items() if not d]: to_deactivate_sorted.append(spec) for s in depmap.keys(): depmap[s].discard(spec) depmap.pop(spec) to_deactivate_sorted.reverse() # Ensure that the sorted list contains all the packages assert set(to_deactivate_sorted) == to_deactivate # Remove the packages from the view for spec in to_deactivate_sorted: if spec.package.is_extension: self.remove_extension(spec, with_dependents=with_dependents) else: self.remove_standalone(spec) self._purge_empty_directories()
def test_update_failed_no_dependent_task(install_mockery): """Test _update_failed with missing dependent build tasks.""" spec, installer = create_installer('dependent-install') for dep in spec.traverse(root=False): task = create_build_task(dep.package) installer._update_failed(task, mark=False) assert installer.failed[task.pkg_id] is None
def _mirror(args): mirror_dir = spack.util.path.canonicalize_path( os.path.join(args.root_dir, LOCAL_MIRROR_DIR)) # TODO: Here we are adding gnuconfig manually, but this can be fixed # TODO: as soon as we have an option to add to a mirror all the possible # TODO: dependencies of a spec root_specs = spack.bootstrap.all_root_specs( development=args.dev) + ['gnuconfig'] for spec_str in root_specs: msg = 'Adding "{0}" and dependencies to the mirror at {1}' llnl.util.tty.msg(msg.format(spec_str, mirror_dir)) # Suppress tty from the call below for terser messages llnl.util.tty.set_msg_enabled(False) spec = spack.spec.Spec(spec_str).concretized() for node in spec.traverse(): spack.mirror.create(mirror_dir, [node]) llnl.util.tty.set_msg_enabled(True) if args.binary_packages: msg = 'Adding binary packages from "{0}" to the mirror at {1}' llnl.util.tty.msg(msg.format(BINARY_TARBALL, mirror_dir)) llnl.util.tty.set_msg_enabled(False) stage = spack.stage.Stage(BINARY_TARBALL, path=tempfile.mkdtemp()) stage.create() stage.fetch() stage.expand_archive() build_cache_dir = os.path.join(stage.source_path, 'build_cache') shutil.move(build_cache_dir, mirror_dir) llnl.util.tty.set_msg_enabled(True) def write_metadata(subdir, metadata): metadata_rel_dir = os.path.join('metadata', subdir) metadata_yaml = os.path.join(args.root_dir, metadata_rel_dir, 'metadata.yaml') llnl.util.filesystem.mkdirp(os.path.dirname(metadata_yaml)) with open(metadata_yaml, mode='w') as f: spack.util.spack_yaml.dump(metadata, stream=f) return os.path.dirname(metadata_yaml), metadata_rel_dir instructions = ( '\nTo register the mirror on the platform where it\'s supposed ' 'to be used, move "{0}" to its final location and run the ' 'following command(s):\n\n').format(args.root_dir) cmd = ' % spack bootstrap add --trust {0} <final-path>/{1}\n' _, rel_directory = write_metadata(subdir='sources', metadata=SOURCE_METADATA) instructions += cmd.format('local-sources', rel_directory) if args.binary_packages: abs_directory, rel_directory = write_metadata(subdir='binaries', metadata=BINARY_METADATA) shutil.copy(spack.util.path.canonicalize_path(CLINGO_JSON), abs_directory) shutil.copy(spack.util.path.canonicalize_path(GNUPG_JSON), abs_directory) instructions += cmd.format('local-binaries', rel_directory) print(instructions)
def concretize_compiler_flags(self, spec): """ The compiler flags are updated to match those of the spec whose compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ # Pass on concretizing the compiler flags if the target or operating # system is not set. if not spec.architecture.concrete: # We haven't changed, but other changes need to happen before we # continue. `return True` here to force concretization to keep # running. return True compiler_match = lambda other: ( spec.compiler == other.compiler and spec.architecture == other.architecture) ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): if flag not in spec.compiler_flags: spec.compiler_flags[flag] = list() try: nearest = next(p for p in spec.traverse(direction='parents') if (compiler_match(p) and (p is not spec) and flag in p.compiler_flags)) nearest_flags = nearest.compiler_flags.get(flag, []) flags = spec.compiler_flags.get(flag, []) if set(nearest_flags) - set(flags): spec.compiler_flags[flag] = list( llnl.util.lang.dedupe(nearest_flags + flags) ) ret = True except StopIteration: pass # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. try: compiler = spack.compilers.compiler_for_spec( spec.compiler, spec.architecture) except spack.compilers.NoCompilerForSpecError: if self.check_for_compiler_existence: raise return ret for flag in compiler.flags: config_flags = compiler.flags.get(flag, []) flags = spec.compiler_flags.get(flag, []) spec.compiler_flags[flag] = list( llnl.util.lang.dedupe(config_flags + flags) ) if set(config_flags) - set(flags): ret = True return ret
def concretize_compiler_flags(self, spec): """ The compiler flags are updated to match those of the spec whose compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ # Pass on concretizing the compiler flags if the target or operating # system is not set. if not (spec.architecture.platform_os and spec.architecture.target): # We haven't changed, but other changes need to happen before we # continue. `return True` here to force concretization to keep # running. return True compiler_match = lambda other: ( spec.compiler == other.compiler and spec.architecture == other.architecture) ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): if flag not in spec.compiler_flags: spec.compiler_flags[flag] = list() try: nearest = next(p for p in spec.traverse(direction='parents') if (compiler_match(p) and (p is not spec) and flag in p.compiler_flags)) nearest_flags = set(nearest.compiler_flags.get(flag, [])) flags = set(spec.compiler_flags.get(flag, [])) if (nearest_flags - flags): # TODO: these set operations may reorder the flags, which # for some orders of flags can be invalid. See: # https://github.com/spack/spack/issues/6154#issuecomment-342365573 spec.compiler_flags[flag] = list(nearest_flags | flags) ret = True except StopIteration: pass # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. try: compiler = spack.compilers.compiler_for_spec( spec.compiler, spec.architecture) except spack.compilers.NoCompilerForSpecError: if self.check_for_compiler_existence: raise return ret for flag in compiler.flags: config_flags = set(compiler.flags.get(flag, [])) flags = set(spec.compiler_flags.get(flag, [])) spec.compiler_flags[flag] = list(config_flags | flags) if (config_flags - flags): ret = True return ret
def test_update_failed_no_dependent_task(install_mockery): """Test _update_failed with missing dependent build tasks.""" const_arg = installer_args(['dependent-install'], {}) installer = create_installer(const_arg) spec = installer.build_requests[0].pkg.spec for dep in spec.traverse(root=False): task = create_build_task(dep.package) installer._update_failed(task, mark=False) assert installer.failed[task.pkg_id] is None
def test_installed_upstream(upstream_and_downstream_db): upstream_write_db, upstream_db, upstream_layout,\ downstream_db, downstream_layout = (upstream_and_downstream_db) default = ('build', 'link') mock_repo = MockPackageMultiRepo() x = mock_repo.add_package('x', [], []) z = mock_repo.add_package('z', [], []) y = mock_repo.add_package('y', [z], [default]) mock_repo.add_package('w', [x, y], [default, default]) with spack.repo.use_repositories(mock_repo): spec = spack.spec.Spec('w') spec.concretize() for dep in spec.traverse(root=False): upstream_write_db.add(dep, upstream_layout) upstream_db._read() for dep in spec.traverse(root=False): record = downstream_db.get_by_hash(dep.dag_hash()) assert record is not None with pytest.raises(spack.database.ForbiddenLockError): record = upstream_db.get_by_hash(dep.dag_hash()) new_spec = spack.spec.Spec('w') new_spec.concretize() downstream_db.add(new_spec, downstream_layout) for dep in new_spec.traverse(root=False): upstream, record = downstream_db.query_by_spec_hash( dep.dag_hash()) assert upstream assert record.path == upstream_layout.path_for_spec(dep) upstream, record = downstream_db.query_by_spec_hash( new_spec.dag_hash()) assert not upstream assert record.installed upstream_db._check_ref_counts() downstream_db._check_ref_counts()
def concretize_compiler_flags(self, spec): """ The compiler flags are updated to match those of the spec whose compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ # Pass on concretizing the compiler flags if the target or operating # system is not set. if not (spec.architecture.platform_os and spec.architecture.target): # We haven't changed, but other changes need to happen before we # continue. `return True` here to force concretization to keep # running. return True compiler_match = lambda other: ( spec.compiler == other.compiler and spec.architecture == other.architecture) ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): if flag not in spec.compiler_flags: spec.compiler_flags[flag] = list() try: nearest = next(p for p in spec.traverse(direction='parents') if (compiler_match(p) and (p is not spec) and flag in p.compiler_flags)) nearest_flags = set(nearest.compiler_flags.get(flag, [])) flags = set(spec.compiler_flags.get(flag, [])) if (nearest_flags - flags): spec.compiler_flags[flag] = list(nearest_flags | flags) ret = True except StopIteration: pass # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. compiler = spack.compilers.compiler_for_spec( spec.compiler, spec.architecture) for flag in compiler.flags: config_flags = set(compiler.flags.get(flag, [])) flags = set(spec.compiler_flags.get(flag, [])) spec.compiler_flags[flag] = list(config_flags | flags) if (config_flags - flags): ret = True return ret
def view_copy(src, dst, view, spec=None): """ Copy a file from src to dst. Use spec and view to generate relocations """ shutil.copy2(src, dst) if spec and not spec.external: # Not metadata, we have to relocate it # Get information on where to relocate from/to # This is vestigial code for the *old* location of sbang. Previously, # sbang was a bash script, and it lived in the spack prefix. It is # now a POSIX script that lives in the install prefix. Old packages # will have the old sbang location in their shebangs. # TODO: Not sure which one to use... import spack.hooks.sbang as sbang # Break a package include cycle import spack.relocate orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(spack.paths.spack_root) new_sbang = sbang.sbang_shebang_line() prefix_to_projection = collections.OrderedDict( {spec.prefix: view.get_projection_for_spec(spec)}) for dep in spec.traverse(): if not dep.external: prefix_to_projection[dep.prefix] = \ view.get_projection_for_spec(dep) if spack.relocate.is_binary(dst): spack.relocate.relocate_text_bin(binaries=[dst], prefixes=prefix_to_projection) else: prefix_to_projection[spack.store.layout.root] = view._root prefix_to_projection[orig_sbang] = new_sbang spack.relocate.relocate_text(files=[dst], prefixes=prefix_to_projection) try: stat = os.stat(src) os.chown(dst, stat.st_uid, stat.st_gid) except OSError: tty.debug('Can\'t change the permissions for %s' % dst)
def _get_environment_specs(self, recurse_dependencies=True): """Returns the specs of all the packages in an environment. If these specs appear under different user_specs, only one copy is added to the list returned. """ spec_list = list() for spec_hash in self.concretized_order: spec = self.specs_by_hash[spec_hash] specs = (spec.traverse( deptype=('link', 'run')) if recurse_dependencies else (spec, )) spec_list.extend(specs) return spec_list
def display_specs_as_json(specs, deps=False): """Convert specs to a list of json records.""" seen = set() records = [] for spec in specs: if spec.dag_hash() in seen: continue seen.add(spec.dag_hash()) records.append(spec.to_record_dict()) if deps: for dep in spec.traverse(): if dep.dag_hash() in seen: continue seen.add(dep.dag_hash()) records.append(dep.to_record_dict()) sjson.dump(records, sys.stdout)
def test_recursive_upstream_dbs(tmpdir_factory, test_store, gen_mock_layout): roots = [str(tmpdir_factory.mktemp(x)) for x in ['a', 'b', 'c']] layouts = [gen_mock_layout(x) for x in ['/ra/', '/rb/', '/rc/']] default = ('build', 'link') z = MockPackage('z', [], []) y = MockPackage('y', [z], [default]) x = MockPackage('x', [y], [default]) mock_repo = MockPackageMultiRepo([x, y, z]) with spack.repo.swap(mock_repo): spec = spack.spec.Spec('x') spec.concretize() db_c = spack.database.Database(roots[2]) db_c.add(spec['z'], layouts[2]) db_b = spack.database.Database(roots[1], upstream_dbs=[db_c]) db_b.add(spec['y'], layouts[1]) db_a = spack.database.Database(roots[0], upstream_dbs=[db_b, db_c]) db_a.add(spec['x'], layouts[0]) upstream_dbs_from_scratch = ( spack.store._construct_upstream_dbs_from_install_roots( [roots[1], roots[2]], _test=True)) db_a_from_scratch = spack.database.Database( roots[0], upstream_dbs=upstream_dbs_from_scratch) assert db_a_from_scratch.db_for_spec_hash( spec.dag_hash()) == (db_a_from_scratch) assert db_a_from_scratch.db_for_spec_hash( spec['y'].dag_hash()) == (upstream_dbs_from_scratch[0]) assert db_a_from_scratch.db_for_spec_hash( spec['z'].dag_hash()) == (upstream_dbs_from_scratch[1]) db_a_from_scratch._check_ref_counts() upstream_dbs_from_scratch[0]._check_ref_counts() upstream_dbs_from_scratch[1]._check_ref_counts() assert (db_a_from_scratch.installed_relatives(spec) == set( spec.traverse(root=False))) assert (db_a_from_scratch.installed_relatives( spec['z'], direction='parents') == set([spec, spec['y']]))
def write(self): """Writes an in-memory environment to its location on disk. This will also write out package files for each newly concretized spec. """ # ensure path in var/spack/environments fs.mkdirp(self.path) if self.specs_by_hash: # ensure the prefix/.env directory exists fs.mkdirp(self.env_subdir_path) for spec in self.new_specs: for dep in spec.traverse(): if not dep.concrete: raise ValueError('specs passed to environment.write() ' 'must be concrete!') root = os.path.join(self.repos_path, dep.namespace) repo = spack.repo.create_or_construct(root, dep.namespace) pkg_dir = repo.dirname_for_package_name(dep.name) fs.mkdirp(pkg_dir) spack.repo.path.dump_provenance(dep, pkg_dir) self.new_specs = [] # write the lock file last with fs.write_tmp_and_move(self.lock_path) as f: sjson.dump(self._to_lockfile_dict(), stream=f) else: if os.path.exists(self.lock_path): os.unlink(self.lock_path) # invalidate _repo cache self._repo = None # put the new user specs in the YAML yaml_spec_list = config_dict(self.yaml).setdefault('specs', []) yaml_spec_list[:] = [str(s) for s in self.user_specs] # if all that worked, write out the manifest file at the top level with fs.write_tmp_and_move(self.manifest_path) as f: _write_yaml(self.yaml, f)
def display_specs_as_json(specs, deps=False): """Convert specs to a list of json records.""" seen = set() records = [] for spec in specs: dag_hash = spec.dag_hash() if dag_hash in seen: continue records.append(spec.node_dict_with_hashes()) seen.add(dag_hash) if deps: for dep in spec.traverse(): dep_dag_hash = dep.dag_hash() if dep_dag_hash in seen: continue records.append(dep.node_dict_with_hashes()) seen.add(dep_dag_hash) sjson.dump(records, sys.stdout)
def view_copy(src, dst, view, spec=None): """ Copy a file from src to dst. Use spec and view to generate relocations """ shutil.copyfile(src, dst) if spec: # Not metadata, we have to relocate it # Get information on where to relocate from/to # This is vestigial code for the *old* location of sbang. Previously, # sbang was a bash script, and it lived in the spack prefix. It is # now a POSIX script that lives in the install prefix. Old packages # will have the old sbang location in their shebangs. # TODO: Not sure which one to use... import spack.hooks.sbang as sbang orig_sbang = '#!/bin/bash {0}/bin/sbang'.format(spack.paths.spack_root) new_sbang = sbang.sbang_shebang_line() prefix_to_projection = OrderedDict({ spec.prefix: view.get_projection_for_spec(spec), spack.paths.spack_root: view._root}) for dep in spec.traverse(): prefix_to_projection[dep.prefix] = \ view.get_projection_for_spec(dep) if spack.relocate.is_binary(dst): spack.relocate.relocate_text_bin( binaries=[dst], prefixes=prefix_to_projection ) else: prefix_to_projection[spack.store.layout.root] = view._root prefix_to_projection[orig_sbang] = new_sbang spack.relocate.relocate_text( files=[dst], prefixes=prefix_to_projection )
def concretize_compiler(self, spec): """If the spec already has a compiler, we're done. If not, then take the compiler used for the nearest ancestor with a compiler spec and use that. If the ancestor's compiler is not concrete, then give it a valid version. If there is no ancestor with a compiler, use the system default compiler. Intuition: Use the system default if no package that depends on this one has a strict compiler requirement. Otherwise, try to build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ all_compilers = spack.compilers.all_compilers() if (spec.compiler and spec.compiler.concrete and spec.compiler in all_compilers): return False try: nearest = next(p for p in spec.traverse(direction='parents') if p.compiler is not None).compiler if not nearest in all_compilers: # Take the newest compiler that saisfies the spec matches = sorted(spack.compilers.find(nearest)) if not matches: raise UnavailableCompilerVersionError(nearest) # copy concrete version into nearest spec nearest.versions = matches[-1].versions.copy() assert(nearest.concrete) spec.compiler = nearest.copy() except StopIteration: spec.compiler = spack.compilers.default_compiler().copy() return True # things changed.
def view_copy(src, dst, view, spec=None): """ Copy a file from src to dst. Use spec and view to generate relocations """ shutil.copyfile(src, dst) if spec: # Not metadata, we have to relocate it # Get information on where to relocate from/to prefix_to_projection = dict( (dep.prefix, view.get_projection_for_spec(dep)) for dep in spec.traverse() ) if spack.relocate.is_binary(dst): # relocate binaries spack.relocate.relocate_text_bin( binaries=[dst], orig_install_prefix=spec.prefix, new_install_prefix=view.get_projection_for_spec(spec), orig_spack=spack.paths.spack_root, new_spack=view._root, new_prefixes=prefix_to_projection ) else: # relocate text spack.relocate.relocate_text( files=[dst], orig_layout_root=spack.store.layout.root, new_layout_root=view._root, orig_install_prefix=spec.prefix, new_install_prefix=view.get_projection_for_spec(spec), orig_spack=spack.paths.spack_root, new_spack=view._root, new_prefixes=prefix_to_projection )
def concretize_architecture(self, spec): """If the spec is empty provide the defaults of the platform. If the architecture is not a string type, then check if either the platform, target or operating system are concretized. If any of the fields are changed then return True. If everything is concretized (i.e the architecture attribute is a namedtuple of classes) then return False. If the target is a string type, then convert the string into a concretized architecture. If it has no architecture and the root of the DAG has an architecture, then use the root otherwise use the defaults on the platform. """ try: # Get the nearest architecture with any fields set nearest = next(p for p in spec.traverse(direction='parents') if (p.architecture and p is not spec)) nearest_arch = nearest.architecture except StopIteration: # Default to the system architecture if nothing set nearest_arch = spack.spec.ArchSpec(spack.architecture.sys_type()) spec_changed = False # ensure type safety for the architecture if spec.architecture is None: spec.architecture = spack.spec.ArchSpec() spec_changed = True # replace each of the fields (platform, os, target) separately nearest_dict = nearest_arch.to_cmp_dict() replacement_fields = [ k for k, v in iteritems(nearest_dict) if v and not getattr(spec.architecture, k) ] for field in replacement_fields: setattr(spec.architecture, field, getattr(nearest_arch, field)) spec_changed = True return spec_changed
def concretize_compiler(self, spec): """If the spec already has a compiler, we're done. If not, then take the compiler used for the nearest ancestor with a compiler spec and use that. If the ancestor's compiler is not concrete, then give it a valid version. If there is no ancestor with a compiler, use the system default compiler. Intuition: Use the system default if no package that depends on this one has a strict compiler requirement. Otherwise, try to build with the compiler that will be used by libraries that link to this one, to maximize compatibility. """ all_compilers = spack.compilers.all_compilers() if (spec.compiler and spec.compiler.concrete and spec.compiler in all_compilers): return try: nearest = next(p for p in spec.traverse(direction='parents') if p.compiler is not None).compiler if not nearest in all_compilers: # Take the newest compiler that saisfies the spec matches = sorted(spack.compilers.find(nearest)) if not matches: raise UnavailableCompilerVersionError(nearest) # copy concrete version into nearest spec nearest.versions = matches[-1].versions.copy() assert(nearest.concrete) spec.compiler = nearest.copy() except StopIteration: spec.compiler = spack.compilers.default_compiler().copy()
def concretize_compiler_flags(self, spec): """ The compiler flags are updated to match those of the spec whose compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ # Pass on concretizing the compiler flags if the target or operating # system is not set. if not (spec.architecture.platform_os and spec.architecture.target): # We haven't changed, but other changes need to happen before we # continue. `return True` here to force concretization to keep # running. return True ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): try: nearest = next( p for p in spec.traverse(direction='parents') if ((p.compiler == spec.compiler and p is not spec) and flag in p.compiler_flags)) if flag not in spec.compiler_flags or \ not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): if flag in spec.compiler_flags: spec.compiler_flags[flag] = list( set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: spec.compiler_flags[flag] = nearest.compiler_flags[ flag] ret = True except StopIteration: if (flag in spec.root.compiler_flags and ((flag not in spec.compiler_flags) or sorted(spec.compiler_flags[flag]) != sorted( spec.root.compiler_flags[flag]))): if flag in spec.compiler_flags: spec.compiler_flags[flag] = list( set(spec.compiler_flags[flag]) | set(spec.root.compiler_flags[flag])) else: spec.compiler_flags[flag] = spec.root.compiler_flags[ flag] ret = True else: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = [] # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. compiler = spack.compilers.compiler_for_spec(spec.compiler, spec.architecture) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] if compiler.flags[flag] != []: ret = True else: if ((sorted(spec.compiler_flags[flag]) != sorted( compiler.flags[flag])) and (not set(spec.compiler_flags[flag]) >= set( compiler.flags[flag]))): ret = True spec.compiler_flags[flag] = list( set(spec.compiler_flags[flag]) | set(compiler.flags[flag])) return ret
def get_executable(exe, spec=None, install=False): """Find an executable named exe, either in PATH or in Spack Args: exe (str): needed executable name spec (Spec or str): spec to search for exe in (default exe) install (bool): install spec if not available When ``install`` is True, Spack will use the python used to run Spack as an external. The ``install`` option should only be used with packages that install quickly (when using external python) or are guaranteed by Spack organization to be in a binary mirror (clingo).""" # Search the system first runner = spack.util.executable.which(exe) if runner: return runner # Check whether it's already installed spec = spack.spec.Spec(spec or exe) installed_specs = spack.store.db.query(spec, installed=True) for ispec in installed_specs: # filter out directories of the same name as the executable exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe) if fs.is_exe(exe_p)] if exe_path: ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in ispec.traverse(root=True, order='post'): envmod.extend(uenv.environment_modifications_for_spec(dep)) ret.add_default_envmod(envmod) return ret else: tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix)) def _raise_error(executable, exe_spec): error_msg = 'cannot find the executable "{0}"'.format(executable) if exe_spec: error_msg += ' from spec "{0}'.format(exe_spec) raise RuntimeError(error_msg) # If we're not allowed to install this for ourselves, we can't find it if not install: _raise_error(exe, spec) with spack_python_interpreter(): # We will install for ourselves, using this python if needed # Concretize the spec spec.concretize() spec.package.do_install() # filter out directories of the same name as the executable exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe) if fs.is_exe(exe_p)] if exe_path: ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in spec.traverse(root=True, order='post'): envmod.extend(uenv.environment_modifications_for_spec(dep)) ret.add_default_envmod(envmod) return ret _raise_error(exe, spec)
def concretize_compiler_flags(self, spec): """ The compiler flags are updated to match those of the spec whose compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): try: nearest = next( p for p in spec.traverse(direction='parents') if ((p.compiler == spec.compiler and p is not spec) and flag in p.compiler_flags)) if ((not flag in spec.compiler_flags) or sorted(spec.compiler_flags[flag]) != sorted( nearest.compiler_flags[flag])): if flag in spec.compiler_flags: spec.compiler_flags[flag] = list( set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: spec.compiler_flags[flag] = nearest.compiler_flags[ flag] ret = True except StopIteration: if (flag in spec.root.compiler_flags and ((not flag in spec.compiler_flags) or sorted(spec.compiler_flags[flag]) != sorted( spec.root.compiler_flags[flag]))): if flag in spec.compiler_flags: spec.compiler_flags[flag] = list( set(spec.compiler_flags[flag]) | set(spec.root.compiler_flags[flag])) else: spec.compiler_flags[flag] = spec.root.compiler_flags[ flag] ret = True else: if not flag in spec.compiler_flags: spec.compiler_flags[flag] = [] # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. compiler = spack.compilers.compiler_for_spec(spec.compiler) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] if compiler.flags[flag] != []: ret = True else: if ((sorted(spec.compiler_flags[flag]) != sorted( compiler.flags[flag])) and (not set(spec.compiler_flags[flag]) >= set( compiler.flags[flag]))): ret = True spec.compiler_flags[flag] = list( set(spec.compiler_flags[flag]) | set(compiler.flags[flag])) return ret
def write(self): """Writes an in-memory environment to its location on disk. This will also write out package files for each newly concretized spec. """ # ensure path in var/spack/environments fs.mkdirp(self.path) if self.specs_by_hash: # ensure the prefix/.env directory exists fs.mkdirp(self.env_subdir_path) for spec in self.new_specs: for dep in spec.traverse(): if not dep.concrete: raise ValueError('specs passed to environment.write() ' 'must be concrete!') root = os.path.join(self.repos_path, dep.namespace) repo = spack.repo.create_or_construct(root, dep.namespace) pkg_dir = repo.dirname_for_package_name(dep.name) fs.mkdirp(pkg_dir) spack.repo.path.dump_provenance(dep, pkg_dir) self.new_specs = [] # write the lock file last with fs.write_tmp_and_move(self.lock_path) as f: sjson.dump(self._to_lockfile_dict(), stream=f) else: if os.path.exists(self.lock_path): os.unlink(self.lock_path) # invalidate _repo cache self._repo = None # put any changes in the definitions in the YAML for name, speclist in self.spec_lists.items(): if name == user_speclist_name: # The primary list is handled differently continue conf = config_dict(self.yaml) active_yaml_lists = [ l for l in conf.get('definitions', []) if name in l and _eval_conditional(l.get('when', 'True')) ] # Remove any specs in yaml that are not in internal representation for ayl in active_yaml_lists: # If it's not a string, it's a matrix. Those can't have changed # If it is a string that starts with '$', it's a reference. # Those also can't have changed. ayl[name][:] = [ s for s in ayl.setdefault(name, []) if (not isinstance(s, six.string_types)) or s.startswith('$') or Spec(s) in speclist.specs ] # Put the new specs into the first active list from the yaml new_specs = [ entry for entry in speclist.yaml_list if isinstance(entry, six.string_types) and not any( entry in ayl[name] for ayl in active_yaml_lists) ] list_for_new_specs = active_yaml_lists[0].setdefault(name, []) list_for_new_specs[:] = list_for_new_specs + new_specs # put the new user specs in the YAML. # This can be done directly because there can't be multiple definitions # nor when clauses for `specs` list. yaml_spec_list = config_dict(self.yaml).setdefault( user_speclist_name, []) yaml_spec_list[:] = self.user_specs.yaml_list default_name = default_view_name if self.views and len(self.views) == 1 and default_name in self.views: path = self.default_view.root if self.default_view == ViewDescriptor(self.view_path_default): view = True elif self.default_view == ViewDescriptor(path): view = path else: view = dict((name, view.to_dict()) for name, view in self.views.items()) elif self.views: view = dict( (name, view.to_dict()) for name, view in self.views.items()) else: view = False yaml_dict = config_dict(self.yaml) if view is not True: # The default case is to keep an active view inside of the # Spack environment directory. To avoid cluttering the config, # we omit the setting in this case. yaml_dict['view'] = view elif 'view' in yaml_dict: del yaml_dict['view'] # if all that worked, write out the manifest file at the top level with fs.write_tmp_and_move(self.manifest_path) as f: _write_yaml(self.yaml, f) # TODO: for operations that just add to the env (install etc.) this # could just call update_view self.regenerate_views()
def concretize_compiler_flags(self, spec): """ The compiler flags are updated to match those of the spec whose compiler is used, defaulting to no compiler flags in the spec. Default specs set at the compiler level will still be added later. """ # Pass on concretizing the compiler flags if the target or operating # system is not set. if not (spec.architecture.platform_os and spec.architecture.target): # We haven't changed, but other changes need to happen before we # continue. `return True` here to force concretization to keep # running. return True ret = False for flag in spack.spec.FlagMap.valid_compiler_flags(): try: nearest = next(p for p in spec.traverse(direction='parents') if ((p.compiler == spec.compiler and p is not spec) and flag in p.compiler_flags)) if flag not in spec.compiler_flags or \ not (sorted(spec.compiler_flags[flag]) >= sorted(nearest.compiler_flags[flag])): if flag in spec.compiler_flags: spec.compiler_flags[flag] = list( set(spec.compiler_flags[flag]) | set(nearest.compiler_flags[flag])) else: spec.compiler_flags[ flag] = nearest.compiler_flags[flag] ret = True except StopIteration: if (flag in spec.root.compiler_flags and ((flag not in spec.compiler_flags) or sorted(spec.compiler_flags[flag]) != sorted(spec.root.compiler_flags[flag]))): if flag in spec.compiler_flags: spec.compiler_flags[flag] = list( set(spec.compiler_flags[flag]) | set(spec.root.compiler_flags[flag])) else: spec.compiler_flags[ flag] = spec.root.compiler_flags[flag] ret = True else: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = [] # Include the compiler flag defaults from the config files # This ensures that spack will detect conflicts that stem from a change # in default compiler flags. compiler = spack.compilers.compiler_for_spec( spec.compiler, spec.architecture) for flag in compiler.flags: if flag not in spec.compiler_flags: spec.compiler_flags[flag] = compiler.flags[flag] if compiler.flags[flag] != []: ret = True else: if ((sorted(spec.compiler_flags[flag]) != sorted(compiler.flags[flag])) and (not set(spec.compiler_flags[flag]) >= set(compiler.flags[flag]))): ret = True spec.compiler_flags[flag] = list( set(spec.compiler_flags[flag]) | set(compiler.flags[flag])) return ret