def merge(self, dest_root, **kwargs): """Link all files in src into dest, creating directories if necessary. If ignore_conflicts is True, do not break when the target exists but rather return a list of files that could not be linked. Note that files blocking directories will still cause an error. """ ignore_conflicts = kwargs.get("ignore_conflicts", False) ignore = kwargs.get('ignore', lambda x: False) conflict = self.find_conflict( dest_root, ignore=ignore, ignore_file_conflicts=ignore_conflicts) if conflict: raise MergeConflictError(conflict) self.merge_directories(dest_root, ignore) existing = [] merge_file = kwargs.get('merge_file', merge_link) for src, dst in self.get_file_map(dest_root, ignore).items(): if os.path.exists(dst): existing.append(dst) else: merge_file(src, dst) for c in existing: tty.warn("Could not merge: %s" % c)
def link_one(spec, path, link=os.symlink, verbose=False): 'Link all files in `spec` into directory `path`.' dotspack = transform_path(spec, '.spack', path) if os.path.exists(dotspack): tty.warn('Skipping existing package: "%s"' % spec.name) return if verbose: tty.info('Linking package: "%s"' % spec.name) for dirpath, dirnames, filenames in os.walk(spec.prefix): if not filenames: continue # avoid explicitly making empty dirs targdir = transform_path(spec, dirpath, path) assuredir(targdir) for fname in filenames: src = os.path.join(dirpath, fname) dst = os.path.join(targdir, fname) if os.path.exists(dst): if '.spack' in dst.split(os.path.sep): continue # silence these tty.warn("Skipping existing file: %s" % dst) continue link(src, dst)
def process_environment_command(self, env): for command in env: try: yield self.formats[type(command)].format(**command.args) except KeyError: tty.warn('Cannot handle command of type {command} : skipping request'.format(command=type(command))) tty.warn('{context} at {filename}:{lineno}'.format(**command.args))
def __init__(self, *repo_dirs, **kwargs): # super-namespace for all packages in the RepoPath self.super_namespace = kwargs.get('namespace', repo_namespace) self.repos = [] self.by_namespace = NamespaceTrie() self.by_path = {} self._all_package_names = None self._provider_index = None # If repo_dirs is empty, just use the configuration if not repo_dirs: import spack.config repo_dirs = spack.config.get_config('repos') if not repo_dirs: raise NoRepoConfiguredError( "Spack configuration contains no package repositories.") # Add each repo to this path. for root in repo_dirs: try: repo = Repo(root, self.super_namespace) self.put_last(repo) except RepoError as e: tty.warn("Failed to initialize repository at '%s'." % root, e.message, "To remove the bad repository, run this command:", " spack repo rm %s" % root)
def dump_provenance(self, spec, path): """Dump provenance information for a spec to a particular path. This dumps the package file and any associated patch files. Raises UnknownPackageError if not found. """ # Some preliminary checks. if spec.virtual: raise UnknownPackageError(spec.name) if spec.namespace and spec.namespace != self.namespace: raise UnknownPackageError( "Repository %s does not contain package %s." % (self.namespace, spec.fullname)) # Install any patch files needed by packages. mkdirp(path) for spec, patches in spec.package.patches.items(): for patch in patches: if patch.path: if os.path.exists(patch.path): install(patch.path, path) else: tty.warn("Patch file did not exist: %s" % patch.path) # Install the package.py file itself. install(self.filename_for_package_name(spec), path)
def filter_shebang(path): """Adds a second shebang line, using sbang, at the beginning of a file.""" with open(path, 'r') as original_file: original = original_file.read() # This line will be prepended to file new_sbang_line = '#!/bin/bash %s/bin/sbang\n' % spack.spack_root # Skip files that are already using sbang. if original.startswith(new_sbang_line): return # Use --! instead of #! on second line for lua. if re.search(r'^#!(/[^/]*)*lua\b', original): original = re.sub(r'^#', '--', original) # Change non-writable files to be writable if needed. saved_mode = None if not os.access(path, os.W_OK): st = os.stat(path) saved_mode = st.st_mode os.chmod(path, saved_mode | stat.S_IWRITE) with open(path, 'w') as new_file: new_file.write(new_sbang_line) new_file.write(original) # Restore original permissions. if saved_mode is not None: os.chmod(path, saved_mode) tty.warn("Patched overlong shebang in %s" % path)
def installed_relatives(self, spec, direction='children', transitive=True): """Return installed specs related to this one.""" if direction not in ('parents', 'children'): raise ValueError("Invalid direction: %s" % direction) relatives = set() for spec in self.query(spec): if transitive: to_add = spec.traverse(direction=direction, root=False) elif direction == 'parents': to_add = spec.dependents() else: # direction == 'children' to_add = spec.dependencies() for relative in to_add: hash_key = relative.dag_hash() if hash_key not in self._data: reltype = ('Dependent' if direction == 'parents' else 'Dependency') tty.warn("Inconsistent state! %s %s of %s not in DB" % (reltype, hash_key, spec.dag_hash())) continue if not self._data[hash_key].installed: continue relatives.add(relative) return relatives
def print_status(self, *specs, **kwargs): if kwargs.get("with_dependencies", False): specs = set(get_dependencies(specs)) specs = sorted(specs, key=lambda s: s.name) in_view = list(map(self.get_spec, specs)) for s, v in zip(specs, in_view): if not v: tty.error(self._croot + 'Package not linked: %s' % s.name) elif s != v: self.print_conflict(v, s, level="warn") in_view = list(filter(None, in_view)) if len(specs) > 0: tty.msg("Packages linked in %s:" % self._croot[:-1]) # avoid circular dependency import spack.cmd spack.cmd.display_specs(in_view, flags=True, variants=True, long=self.verbose) else: tty.warn(self._croot + "No packages found.")
def get_matching_versions(specs, **kwargs): """Get a spec for EACH known version matching any spec in the list.""" matching = [] for spec in specs: pkg = spec.package # Skip any package that has no known versions. if not pkg.versions: tty.msg("No safe (checksummed) versions for package %s" % pkg.name) continue num_versions = kwargs.get('num_versions', 0) matching_spec = [] for i, v in enumerate(reversed(sorted(pkg.versions))): # Generate no more than num_versions versions for each spec. if num_versions and i >= num_versions: break # Generate only versions that satisfy the spec. if v.satisfies(spec.versions): s = Spec(pkg.name) s.versions = VersionList([v]) s.variants = spec.variants.copy() # This is needed to avoid hanging references during the # concretization phase s.variants.spec = s matching_spec.append(s) if not matching_spec: tty.warn("No known version matches spec: %s" % spec) matching.extend(matching_spec) return matching
def fetch_from_url(self, url): # Run curl but grab the mime type from the http headers headers = spack.curl('-#', # status bar '-O', # save file to disk '-D', '-', # print out HTML headers '-L', url, return_output=True, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. if self.archive_file: os.remove(self.archive_file) if spack.curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( url, "Curl was unable to fetch due to invalid certificate. " "This is either an attack, or your cluster's SSL configuration " "is bad. If you believe your SSL configuration is bad, you " "can try running spack -k, which will not check SSL certificates." "Use this at your own risk.") # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle # redirects properly. content_types = re.findall(r'Content-Type:[^\r\n]+', headers) if content_types and 'text/html' in content_types[-1]: tty.warn("The contents of " + self.archive_file + " look like HTML.", "The checksum will likely be bad. If it is, you can use", "'spack clean --dist' to remove the bad archive, then fix", "your internet gateway issue and install again.")
def get_build_system(args, guesser): """Determine the build system template. If a template is specified, always use that. Otherwise, if a URL is provided, download the tarball and peek inside to guess what build system it uses. Otherwise, use a generic template by default. Args: args (argparse.Namespace): The arguments given to ``spack create`` guesser (BuildSystemGuesser): The first_stage_function given to ``spack checksum`` which records the build system it detects Returns: str: The name of the build system template to use """ # Default template template = 'generic' if args.template: # Use a user-supplied template if one is present template = args.template tty.msg("Using specified package template: '{0}'".format(template)) elif args.url: # Use whatever build system the guesser detected template = guesser.build_system if template == 'generic': tty.warn("Unable to detect a build system. " "Using a generic package template.") else: msg = "This package looks like it uses the {0} build system" tty.msg(msg.format(template)) return template
def get_config(section, scope=None): """Get configuration settings for a section. Strips off the top-level section name from the YAML dict. """ validate_section_name(section) merged_section = syaml.syaml_dict() if scope is None: scopes = config_scopes.values() else: scopes = [validate_scope(scope)] for scope in scopes: # read potentially cached data from the scope. data = scope.get_section(section) # Skip empty configs if not data or not isinstance(data, dict): continue # Allow complete override of site config with '<section>::' override_key = section + ':' if not (section in data or override_key in data): tty.warn("Skipping bad configuration file: '%s'" % scope.path) continue if override_key in data: merged_section = data[override_key] else: merged_section = _merge_yaml(merged_section, data[section]) return merged_section
def setup_main_options(args): """Configure spack globals based on the basic options.""" # Set up environment based on args. tty.set_verbose(args.verbose) tty.set_debug(args.debug) tty.set_stacktrace(args.stacktrace) # debug must be set first so that it can even affect behvaior of # errors raised by spack.config. if args.debug: spack.error.debug = True spack.util.debug.register_interrupt_handler() spack.config.set('config:debug', True, scope='command_line') # override lock configuration if passed on command line if args.locks is not None: spack.util.lock.check_lock_safety(spack.paths.prefix) spack.config.set('config:locks', False, scope='command_line') if args.mock: rp = spack.repo.RepoPath(spack.paths.mock_packages_path) spack.repo.set_path(rp) # If the user asked for it, don't check ssl certs. if args.insecure: tty.warn("You asked for --insecure. Will NOT check SSL certificates.") spack.config.set('config:verify_ssl', False, scope='command_line') # when to use color (takes always, auto, or never) tty.color.set_color_when(args.color)
def view(parser, args): 'Produce a view of a set of packages.' specs = spack.cmd.parse_specs(args.specs) path = args.path[0] view = YamlFilesystemView( path, spack.store.layout, ignore_conflicts=getattr(args, "ignore_conflicts", False), link=os.link if args.action in ["hardlink", "hard"] else os.symlink, verbose=args.verbose) # Process common args and specs if getattr(args, "all", False): specs = view.get_all_specs() if len(specs) == 0: tty.warn("Found no specs in %s" % path) elif args.action in actions_link: # only link commands need to disambiguate specs specs = [spack.cmd.disambiguate_spec(s) for s in specs] elif args.action in actions_status: # no specs implies all if len(specs) == 0: specs = view.get_all_specs() else: specs = relaxed_disambiguate(specs, view) else: # status and remove can map the name to packages in view specs = relaxed_disambiguate(specs, view) with_dependencies = args.dependencies.lower() in ['true', 'yes'] # Map action to corresponding functionality if args.action in actions_link: try: view.add_specs(*specs, with_dependencies=with_dependencies, exclude=args.exclude) except MergeConflictError: tty.info("Some file blocked the merge, adding the '-i' flag will " "ignore this conflict. For more information see e.g. " "https://github.com/spack/spack/issues/9029") raise elif args.action in actions_remove: view.remove_specs(*specs, with_dependencies=with_dependencies, exclude=args.exclude, with_dependents=not args.no_remove_dependents) elif args.action in actions_status: view.print_status(*specs, with_dependencies=with_dependencies) else: tty.error('Unknown action: "%s"' % args.action)
def ask_for_confirmation(message): while True: tty.msg(message + '[y/n]') choice = raw_input().lower() if choice == 'y': break elif choice == 'n': raise SystemExit('Operation aborted') tty.warn('Please reply either "y" or "n"')
def clean(self): """By default just runs make clean. Override if this isn't good.""" try: # TODO: should we really call make clean, ro just blow away the directory? make = build_env.MakeExecutable('make', self.parallel) make('clean') tty.msg("Successfully cleaned %s" % self.name) except subprocess.CalledProcessError, e: tty.warn("Warning: 'make clean' didn't work. Consider 'spack clean --work'.")
def __call__(self, *args, **kwargs): """Run the executable with subprocess.check_output, return output.""" return_output = kwargs.get("return_output", False) fail_on_error = kwargs.get("fail_on_error", True) ignore_errors = kwargs.get("ignore_errors", ()) output = kwargs.get("output", sys.stdout) error = kwargs.get("error", sys.stderr) input = kwargs.get("input", None) def streamify(arg, mode): if isinstance(arg, basestring): return open(arg, mode), True elif arg is None and mode != 'r': return open(os.devnull, mode), True return arg, False output, ostream = streamify(output, 'w') error, estream = streamify(error, 'w') input, istream = streamify(input, 'r') # if they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): ignore_errors = (ignore_errors,) quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] if quoted_args: tty.warn("Quotes in command arguments can confuse scripts like configure.", "The following arguments may cause problems when executed:", str("\n".join([" "+arg for arg in quoted_args])), "Quotes aren't needed because spack doesn't use a shell.", "Consider removing them") cmd = self.exe + list(args) cmd_line = ' '.join(cmd) tty.debug(cmd_line) try: proc = subprocess.Popen( cmd, stdin=input, stderr=error, stdout=subprocess.PIPE if return_output else output) out, err = proc.communicate() self.returncode = proc.returncode rc = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): raise ProcessError("Command exited with status %d:" % proc.returncode, cmd_line) if return_output: return out except OSError, e: raise ProcessError( "%s: %s" % (self.exe[0], e.strerror), "Command: " + cmd_line)
def process_environment_command(self, env): for command in env: try: yield self.environment_modifications_formats[type( command)].format(**command.args) except KeyError: message = 'Cannot handle command of type {command} : skipping request' # NOQA: ignore=E501 details = '{context} at {filename}:{lineno}' tty.warn(message.format(command=type(command))) tty.warn(details.format(**command.args))
def _for_each_enabled(spec, method_name): """Calls a method for each enabled module""" for name in enabled: generator = spack.modules.module_types[name](spec) try: getattr(generator, method_name)() except RuntimeError as e: msg = 'cannot perform the requested {0} operation on module files' msg += ' [{1}]' tty.warn(msg.format(method_name, str(e)))
def get_specs(force=False): """ Get spec.yaml's for build caches available on mirror """ global _cached_specs if _cached_specs: tty.debug("Using previously-retrieved specs") return _cached_specs mirrors = spack.config.get('mirrors') if len(mirrors) == 0: tty.warn("No Spack mirrors are currently configured") return {} path = str(spack.architecture.sys_type()) urls = set() for key in mirrors: url = mirrors[key] if url.startswith('file'): mirror = url.replace('file://', '') + '/build_cache' tty.msg("Finding buildcaches in %s" % mirror) if os.path.exists(mirror): files = os.listdir(mirror) for file in files: if re.search('spec.yaml', file): link = 'file://' + mirror + '/' + file urls.add(link) else: tty.msg("Finding buildcaches on %s" % url) p, links = spider(url + "/build_cache") for link in links: if re.search("spec.yaml", link) and re.search(path, link): urls.add(link) _cached_specs = set() for link in urls: with Stage(link, name="build_cache", keep=True) as stage: if force and os.path.exists(stage.save_filename): os.remove(stage.save_filename) if not os.path.exists(stage.save_filename): try: stage.fetch() except fs.FetchError: continue with open(stage.save_filename, 'r') as f: # read the spec from the build cache file. All specs # in build caches are concrete (as they are built) so # we need to mark this spec concrete on read-in. spec = spack.spec.Spec.from_yaml(f) spec._mark_concrete() _cached_specs.add(spec) return _cached_specs
def _check_no_ext_conflicts(self, spec): """ Check that there is no extension conflict for specs. """ extendee = spec.package.extendee_spec try: self.extensions_layout.check_extension_conflict(extendee, spec) except ExtensionAlreadyInstalledError: # we print the warning here because later on the order in which # packages get activated is not clear (set-sorting) tty.warn(self._croot + 'Skipping already activated package: %s' % spec.name)
def check(self): """Check the downloaded archive against a checksum digest. No-op if this stage checks code out of a repository.""" if self.fetcher is not self.default_fetcher and self.skip_checksum_for_mirror: tty.warn("Fetching from mirror without a checksum!", "This package is normally checked out from a version " "control system, but it has been archived on a spack " "mirror. This means we cannot know a checksum for the " "tarball in advance. Be sure that your connection to " "this mirror is secure!.") else: self.fetcher.check()
def _add(self, spec, directory_layout=None, explicit=False): """Add an install record for this spec to the database. Assumes spec is installed in ``layout.path_for_spec(spec)``. Also ensures dependencies are present and updated in the DB as either intsalled or missing. """ if not spec.concrete: raise NonConcreteSpecAddError( "Specs added to DB must be concrete.") for dep in spec.dependencies(_tracked_deps): dkey = dep.dag_hash() if dkey not in self._data: self._add(dep, directory_layout, explicit=False) key = spec.dag_hash() if key not in self._data: installed = bool(spec.external) path = None if not spec.external and directory_layout: path = directory_layout.path_for_spec(spec) try: directory_layout.check_installed(spec) installed = True except DirectoryLayoutError as e: tty.warn( 'Dependency missing due to corrupt install directory:', path, str(e)) # Create a new install record with no deps initially. new_spec = spec.copy(deps=False) self._data[key] = InstallRecord( new_spec, path, installed, ref_count=0, explicit=explicit) # Connect dependencies from the DB to the new copy. for name, dep in iteritems(spec.dependencies_dict(_tracked_deps)): dkey = dep.spec.dag_hash() new_spec._add_dependency(self._data[dkey].spec, dep.deptypes) self._data[dkey].ref_count += 1 # Mark concrete once everything is built, and preserve # the original hash of concrete specs. new_spec._mark_concrete() new_spec._hash = key else: # If it is already there, mark it as installed. self._data[key].installed = True self._data[key].explicit = explicit
def add_single_spec(spec, mirror_root, categories, **kwargs): tty.msg("Adding package {pkg} to mirror".format(pkg=spec.format("$_$@"))) spec_exists_in_mirror = True try: with spec.package.stage: # fetcher = stage.fetcher # fetcher.fetch() # ... # fetcher.archive(archive_path) for ii, stage in enumerate(spec.package.stage): fetcher = stage.fetcher if ii == 0: # create a subdirectory for the current package@version archive_path = os.path.abspath(join_path( mirror_root, mirror_archive_path(spec, fetcher))) name = spec.format("$_$@") else: resource = stage.resource archive_path = os.path.abspath(join_path( mirror_root, mirror_archive_path(spec, fetcher, resource.name))) name = "{resource} ({pkg}).".format( resource=resource.name, pkg=spec.format("$_$@")) subdir = os.path.dirname(archive_path) mkdirp(subdir) if os.path.exists(archive_path): tty.msg("{name} : already added".format(name=name)) else: spec_exists_in_mirror = False fetcher.fetch() if not kwargs.get('no_checksum', False): fetcher.check() tty.msg("{name} : checksum passed".format(name=name)) # Fetchers have to know how to archive their files. Use # that to move/copy/create an archive in the mirror. fetcher.archive(archive_path) tty.msg("{name} : added".format(name=name)) if spec_exists_in_mirror: categories['present'].append(spec) else: categories['mirrored'].append(spec) except Exception as e: if spack.debug: sys.excepthook(*sys.exc_info()) else: tty.warn("Error while fetching %s" % spec.format('$_$@'), e.message) categories['error'].append(spec)
def get_config(section, scope=None): """Get configuration settings for a section. If ``scope`` is ``None`` or not provided, return the merged contents of all of Spack's configuration scopes. If ``scope`` is provided, return only the confiugration as specified in that scope. This off the top-level name from the YAML section. That is, for a YAML config file that looks like this:: config: install_tree: $spack/opt/spack module_roots: lmod: $spack/share/spack/lmod ``get_config('config')`` will return:: { 'install_tree': '$spack/opt/spack', 'module_roots: { 'lmod': '$spack/share/spack/lmod' } } """ validate_section_name(section) merged_section = syaml.syaml_dict() if scope is None: scopes = config_scopes.values() else: scopes = [validate_scope(scope)] for scope in scopes: # read potentially cached data from the scope. data = scope.get_section(section) # Skip empty configs if not data or not isinstance(data, dict): continue if section not in data: tty.warn("Skipping bad configuration file: '%s'" % scope.path) continue merged_section = _merge_yaml(merged_section, data) # no config files -- empty config. if section not in merged_section: return {} # take the top key off before returning. return merged_section[section]
def setup_environment(self, spack_env, run_env): spec = self.spec # TODO: The '--no-user-cfg' option for Python installation is only in # Python v2.7 and v3.4+ (see https://bugs.python.org/issue1180) and # adding support for ignoring user configuration will require # significant changes to this package for other Python versions. if not spec.satisfies('@2.7,3.4:'): tty.warn(('Python v{0} may not install properly if Python ' 'user configurations are present.').format(self.version)) # Need this to allow python build to find the Python installation. spack_env.set('MACOSX_DEPLOYMENT_TARGET', platform.mac_ver()[0])
def remove_specs(self, *specs, **kwargs): assert all((s.concrete for s in specs)) with_dependents = kwargs.get("with_dependents", True) with_dependencies = kwargs.get("with_dependencies", False) specs = set(specs) if with_dependencies: specs = get_dependencies(specs) if kwargs.get("exclude", None): specs = set(filter_exclude(specs, kwargs["exclude"])) all_specs = set(self.get_all_specs()) to_deactivate = specs to_keep = all_specs - to_deactivate dependents = find_dependents(to_keep, to_deactivate) if with_dependents: # remove all packages depending on the ones to remove if len(dependents) > 0: tty.warn(self._croot + "The following dependents will be removed: %s" % ", ".join((s.name for s in dependents))) to_deactivate.update(dependents) elif len(dependents) > 0: tty.warn(self._croot + "The following packages will be unusable: %s" % ", ".join((s.name for s in dependents))) extensions = set(filter(lambda s: s.package.is_extension, to_deactivate)) standalones = to_deactivate - extensions # Please note that a traversal of the DAG in post-order and then # forcibly removing each package should remove the need to specify # with_dependents for deactivating extensions/allow removal without # additional checks (force=True). If removal performance becomes # unbearable for whatever reason, this should be the first point of # attack. # # see: https://github.com/spack/spack/pull/3227#discussion_r117147475 remove_extension = ft.partial(self.remove_extension, with_dependents=with_dependents) set(map(remove_extension, extensions)) set(map(self.remove_standalone, standalones)) self.purge_empty_directories()
def autoreconf(self, spec, prefix): """Not needed usually, configure should be already there""" # If configure exists nothing needs to be done if os.path.exists(self.configure_abs_path): return # Else try to regenerate it autotools = ['m4', 'autoconf', 'automake', 'libtool'] missing = [x for x in autotools if x not in spec] if missing: msg = 'Cannot generate configure: missing dependencies {0}' raise RuntimeError(msg.format(missing)) tty.msg('Configure script not found: trying to generate it') tty.warn('*********************************************************') tty.warn('* If the default procedure fails, consider implementing *') tty.warn('* a custom AUTORECONF phase in the package *') tty.warn('*********************************************************') with working_dir(self.configure_directory): m = inspect.getmodule(self) # This part should be redundant in principle, but # won't hurt m.libtoolize() m.aclocal() # This line is what is needed most of the time # --install, --verbose, --force autoreconf_args = ['-ivf'] if 'pkg-config' in spec: autoreconf_args += [ '-I', join_path(spec['pkg-config'].prefix, 'share', 'aclocal'), ] autoreconf_args += self.autoreconf_extra_args m.autoreconf(*autoreconf_args)
def remove_standalone(self, spec): """ Remove (unlink) a standalone package from this view. """ if not self.check_added(spec): tty.warn(self._croot + 'Skipping package not linked in view: %s' % spec.name) return self.unmerge(spec) self.unlink_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Removed package: %s' % colorize_spec(spec))
def _create_new_cache(self): """Create a new cache for packages in a repo. The implementation here should try to minimize filesystem calls. At the moment, it is O(number of packages) and makes about one stat call per package. This is reasonably fast, and avoids actually importing packages in Spack, which is slow. """ # Create a dictionary that will store the mapping between a # package name and its stat info cache = {} for pkg_name in os.listdir(self.packages_path): # Skip non-directories in the package root. pkg_dir = os.path.join(self.packages_path, pkg_name) # Warn about invalid names that look like packages. if not valid_module_name(pkg_name): msg = 'Skipping package at {0}. ' msg += '"{1}" is not a valid Spack module name.' tty.warn(msg.format(pkg_dir, pkg_name)) continue # Construct the file name from the directory pkg_file = os.path.join( self.packages_path, pkg_name, package_file_name ) # Use stat here to avoid lots of calls to the filesystem. try: sinfo = os.stat(pkg_file) except OSError as e: if e.errno == errno.ENOENT: # No package.py file here. continue elif e.errno == errno.EACCES: tty.warn("Can't read package file %s." % pkg_file) continue raise e # If it's not a file, skip it. if stat.S_ISDIR(sinfo.st_mode): continue # If it is a file, then save the stats under the # appropriate key cache[pkg_name] = sinfo return cache
def test(self): config_args = self._generate_make_hdr_for_test() # Write configuration options to make.inc file make_file_inc = join_path(self.test_suite.current_test_cache_dir, self.make_hdr_file) with open(make_file_inc, 'w') as inc: for option in config_args: inc.write('{0}\n'.format(option)) args = [] if self.version < Version('5.2.2'): args.append('HEADER=' + self.prefix.include) args.append('superlu') test_dir = join_path(self.test_suite.current_test_cache_dir, self.examples_src_dir) exe = 'superlu' if not os.path.isfile(join_path(test_dir, '{0}.c'.format(exe))): tty.warn('Skipping superlu test:' 'missing file {0}.c'.format(exe)) return self.run_superlu_test(test_dir, exe, args)
def dump_provenance(self, spec, path): """Dump provenance information for a spec to a particular path. This dumps the package file and any associated patch files. Raises UnknownPackageError if not found. """ if spec.namespace and spec.namespace != self.namespace: raise UnknownPackageError( "Repository %s does not contain package %s." % (self.namespace, spec.fullname)) # Install patch files needed by the package. fs.mkdirp(path) for patch in itertools.chain.from_iterable( spec.package.patches.values()): if patch.path: if os.path.exists(patch.path): fs.install(patch.path, path) else: tty.warn("Patch file did not exist: %s" % patch.path) # Install the package.py file itself. fs.install(self.filename_for_package_name(spec.name), path)
def remove_path(self, dirname): """Remove `dirname` from the modulepath Parameters ---------- dirname : str The directory to remove Returns ------- modules_in_dir : list of Module The modules in the directory that was removed """ dirname = Path.expand_name(dirname) if dirname not in self: # pragma: no cover tty.warn("Modulepath: {0!r} is not in modulepath".format(dirname)) return [] modules_in_dir = self.getby_dirname(dirname) self.path.pop(self.index(dirname)) self.path_modified() return modules_in_dir
def add_standalone(self, spec): if spec.package.is_extension: tty.error(self._croot + 'Package %s is an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if self.check_added(spec): tty.warn(self._croot + 'Skipping already linked package: %s' % colorize_spec(spec)) return True if spec.package.extendable: # Check for globally activated extensions in the extendee that # we're looking at. activated = [p.spec for p in spack.store.db.activated_extensions_for(spec)] if activated: tty.error("Globally activated extensions cannot be used in " "conjunction with filesystem views. " "Please deactivate the following specs: ") spack.cmd.display_specs(activated, flags=True, variants=True, long=False) return False self.merge(spec) self.link_meta_folder(spec) if self.verbose: tty.info(self._croot + 'Linked package: %s' % colorize_spec(spec)) return True
def create_stage_root(path): # type: (str) -> None """Create the stage root directory and ensure appropriate access perms.""" assert os.path.isabs(path) and len(path.strip()) > 1 err_msg = 'Cannot create stage root {0}: Access to {1} is denied' user_uid = getuid() # Obtain lists of ancestor and descendant paths of the $user node, if any. group_paths, user_node, user_paths = partition_path( path, getpass.getuser()) for p in group_paths: if not os.path.exists(p): # Ensure access controls of subdirs created above `$user` inherit # from the parent and share the group. par_stat = os.stat(os.path.dirname(p)) mkdirp(p, group=par_stat.st_gid, mode=par_stat.st_mode) p_stat = os.stat(p) if par_stat.st_gid != p_stat.st_gid: tty.warn( "Expected {0} to have group {1}, but it is {2}".format( p, par_stat.st_gid, p_stat.st_gid)) if par_stat.st_mode & p_stat.st_mode != par_stat.st_mode: tty.warn( "Expected {0} to support mode {1}, but it is {2}".format( p, par_stat.st_mode, p_stat.st_mode)) if not can_access(p): raise OSError(errno.EACCES, err_msg.format(path, p)) # Add the path ending with the $user node to the user paths to ensure paths # from $user (on down) meet the ownership and permission requirements. if user_node: user_paths.insert(0, user_node) for p in user_paths: # Ensure access controls of subdirs from `$user` on down are # restricted to the user. owner_uid = get_owner_uid(p) if user_uid != owner_uid: tty.warn( "Expected user {0} to own {1}, but it is owned by {2}".format( user_uid, p, owner_uid)) spack_src_subdir = os.path.join(path, _source_path_subdir) # When staging into a user-specified directory with `spack stage -p <PATH>`, we need # to ensure the `spack-src` subdirectory exists, as we can't rely on it being # created automatically by spack. It's not clear why this is the case for `spack # stage -p`, but since `mkdirp()` is idempotent, this should not change the behavior # for any other code paths. if not os.path.isdir(spack_src_subdir): mkdirp(spack_src_subdir, mode=stat.S_IRWXU)
def relocate_links(links, orig_layout_root, orig_install_prefix, new_install_prefix): """Relocate links to a new install prefix. The symbolic links are relative to the original installation prefix. The old link target is read and the placeholder is replaced by the old layout root. If the old link target is in the old install prefix, the new link target is create by replacing the old install prefix with the new install prefix. Args: links (list): list of links to be relocated orig_layout_root (str): original layout root orig_install_prefix (str): install prefix of the original installation new_install_prefix (str): install prefix where we want to relocate """ placeholder = _placeholder(orig_layout_root) abs_links = [os.path.join(new_install_prefix, link) for link in links] for abs_link in abs_links: link_target = os.readlink(abs_link) link_target = re.sub(placeholder, orig_layout_root, link_target) # If the link points to a file in the original install prefix, # compute the corresponding target in the new prefix and relink if link_target.startswith(orig_install_prefix): link_target = re.sub(orig_install_prefix, new_install_prefix, link_target) os.unlink(abs_link) os.symlink(link_target, abs_link) # If the link is absolute and has not been relocated then # warn the user about that if (os.path.isabs(link_target) and not link_target.startswith(new_install_prefix)): msg = ('Link target "{0}" for symbolic link "{1}" is outside' ' of the new install prefix {2}') tty.warn(msg.format(link_target, abs_link, new_install_prefix))
def _elf_rpaths_for(path): """Return the RPATHs for an executable or a library. The RPATHs are obtained by ``patchelf --print-rpath PATH``. Args: path (str): full path to the executable or library Return: RPATHs as a list of strings. """ # If we're relocating patchelf itself, use it patchelf_path = path if path.endswith("/bin/patchelf") else _patchelf() patchelf = executable.Executable(patchelf_path) output = '' try: output = patchelf('--print-rpath', path, output=str, error=str) output = output.strip('\n') except executable.ProcessError as e: msg = 'patchelf --print-rpath {0} produced an error [{1}]' tty.warn(msg.format(path, str(e))) return output.split(':') if output else []
def _set_elf_rpaths(target, rpaths): """Replace the original RPATH of the target with the paths passed as arguments. This function uses ``patchelf`` to set RPATHs. Args: target: target executable. Must be an ELF object. rpaths: paths to be set in the RPATH Returns: A string concatenating the stdout and stderr of the call to ``patchelf`` """ # Join the paths using ':' as a separator rpaths_str = ':'.join(rpaths) # If we're relocating patchelf itself, make a copy and use it bak_path = None if target.endswith("/bin/patchelf"): bak_path = target + ".bak" shutil.copy(target, bak_path) patchelf, output = executable.Executable(bak_path or _patchelf()), None try: # TODO: revisit the use of --force-rpath as it might be conditional # TODO: if we want to support setting RUNPATH from binary packages patchelf_args = ['--force-rpath', '--set-rpath', rpaths_str, target] output = patchelf(*patchelf_args, output=str, error=str) except executable.ProcessError as e: msg = 'patchelf --force-rpath --set-rpath {0} failed with error {1}' tty.warn(msg.format(target, e)) finally: if bak_path and os.path.exists(bak_path): os.remove(bak_path) return output
def get_existing_elf_rpaths(path_name): """ Return the RPATHS returned by patchelf --print-rpath path_name as a list of strings. """ # if we're relocating patchelf itself, use it if path_name[-13:] == "/bin/patchelf": patchelf = Executable(path_name) else: patchelf = Executable(get_patchelf()) rpaths = list() try: output = patchelf('--print-rpath', '%s' % path_name, output=str, error=str) rpaths = output.rstrip('\n').split(':') except ProcessError as e: msg = 'patchelf --print-rpath %s produced an error %s' % (path_name, e) tty.warn(msg) return rpaths
def caveats(self): perm_script = 'spack_perms_fix.sh' perm_script_path = join_path(self.spec.prefix.bin, perm_script) with open(perm_script_path, 'w') as f: env = spack.tengine.make_environment(dirs=self.package_dir) t = env.get_template(perm_script + '.j2') f.write(t.render({'prefix': self.spec.prefix})) chmod = which('chmod') chmod('0555', perm_script_path) tty.warn(""" For a working GaussianView installation, all executable files can only be accessible by the owner and the group but not the world. We've installed a script that will make the necessary changes; read through it and then execute it: {0} If you have to give others access, please customize the group membership of the package files as documented here: https://spack.readthedocs.io/en/latest/build_settings.html#package-permissions""" .format(perm_script_path))
def remove_files(self, files): def needs_file(spec, file): # convert the file we want to remove to a source in this spec projection = self.get_projection_for_spec(spec) relative_path = os.path.relpath(file, projection) test_path = os.path.join(spec.prefix, relative_path) # check if this spec owns a file of that name (through the # manifest in the metadata dir, which we have in the view). manifest_file = os.path.join(self.get_path_meta_folder(spec), spack.store.layout.manifest_file_name) try: with open(manifest_file, 'r') as f: manifest = s_json.load(f) except (OSError, IOError): # if we can't load it, assume it doesn't know about the file. manifest = {} return test_path in manifest specs = self.get_all_specs() for file in files: if not os.path.lexists(file): tty.warn("Tried to remove %s which does not exist" % file) continue # remove if file is not owned by any other package in the view # This will only be false if two packages are merged into a prefix # and have a conflicting file # check all specs for whether they own the file. That include the spec # we are currently removing, as we remove files before unlinking the # metadata directory. if len([s for s in specs if needs_file(s, file)]) <= 1: tty.debug("Removing file " + file) os.remove(file)
def setup_main_options(args): """Configure spack globals based on the basic options.""" # Assign a custom function to show warnings warnings.showwarning = send_warning_to_tty # Set up environment based on args. tty.set_verbose(args.verbose) tty.set_debug(args.debug) tty.set_stacktrace(args.stacktrace) # debug must be set first so that it can even affect behvaior of # errors raised by spack.config. if args.debug: spack.error.debug = True spack.util.debug.register_interrupt_handler() spack.config.set('config:debug', True, scope='command_line') if args.timestamp: tty.set_timestamp(True) # override lock configuration if passed on command line if args.locks is not None: spack.util.lock.check_lock_safety(spack.paths.prefix) spack.config.set('config:locks', False, scope='command_line') if args.mock: rp = spack.repo.RepoPath(spack.paths.mock_packages_path) spack.repo.set_path(rp) # If the user asked for it, don't check ssl certs. if args.insecure: tty.warn("You asked for --insecure. Will NOT check SSL certificates.") spack.config.set('config:verify_ssl', False, scope='command_line') # when to use color (takes always, auto, or never) color.set_color_when(args.color)
def _for_each_enabled(spec, method_name): """Calls a method for each enabled module""" set_names = set(spack.config.get('modules', {}).keys()) # If we have old-style modules enabled, we put those in the default set old_default_enabled = spack.config.get('modules:enable') if old_default_enabled: set_names.add('default') for name in set_names: enabled = spack.config.get('modules:%s:enable' % name) if name == 'default': # combine enabled modules from default and old format enabled = spack.config.merge_yaml(old_default_enabled, enabled) if not enabled: tty.debug('NO MODULE WRITTEN: list of enabled module files is empty') continue for type in enabled: generator = spack.modules.module_types[type](spec, name) try: getattr(generator, method_name)() except RuntimeError as e: msg = 'cannot perform the requested {0} operation on module files' msg += ' [{1}]' tty.warn(msg.format(method_name, str(e)))
def configure_args(self): spec = self.spec args = [ '--enable-ndb', '--disable-inhibit-plugin', '--with-crypto={0}'.format(spec.variants['crypto'].value) ] args += self.enable_or_disable('nls') args += self.enable_or_disable('sqlite') args += self.with_or_without('selinux') args += self.with_or_without('python') # OpenMP multithreading support automatically enabled if C compiler has # support for OpenMP version 4.5 or higher args += self.enable_or_disable('openmp') # Option got removed in 4.17 if self.spec.satisfies('@:4.16'): args += self.with_or_without('lua') # Legacy berkely db support if 'berkeley-db=full' in spec: args.extend(['--enable-bdb', '--disable-bdb-ro']) elif 'berkeley-db=readonly' in spec: args.extend(['--disable-bdb', '--enable-bdb-ro']) else: args.extend(['--disable-bdb', '--disable-bdb-ro']) # enable POSIX.1e draft 15 file capabilities support if '+posix' in spec: args.append('--with-cap') if 'crypto=openssl' in spec: tty.warn(openssl_warning) return args
def add_standalone(self, spec): if spec.package.is_extension: tty.error(self._croot + 'Package %s is an extension.' % spec.name) return False if spec.external: tty.warn(self._croot + 'Skipping external package: %s' % colorize_spec(spec)) return True if self.check_added(spec): tty.warn(self._croot + 'Skipping already linked package: %s' % colorize_spec(spec)) return True if spec.package.extendable: # Check for globally activated extensions in the extendee that # we're looking at. activated = [ p.spec for p in spack.store.db.activated_extensions_for(spec) ] if activated: tty.error("Globally activated extensions cannot be used in " "conjunction with filesystem views. " "Please deactivate the following specs: ") spack.cmd.display_specs(activated, flags=True, variants=True, long=False) return False tree = LinkTree(spec.prefix) if not self.ignore_conflicts: conflict = tree.find_conflict(self.root) if conflict is not None: tty.error(self._croot + "Cannot link package %s, file already exists: %s" % (spec.name, conflict)) return False conflicts = tree.merge(self.root, link=self.link, ignore=ignore_metadata_dir, ignore_conflicts=self.ignore_conflicts) self.link_meta_folder(spec) if self.ignore_conflicts: for c in conflicts: tty.warn(self._croot + "Could not link: %s" % c) if self.verbose: tty.info(self._croot + 'Linked package: %s' % colorize_spec(spec)) return True
def _create_stage_root(path): """Create the stage root directory and ensure appropriate access perms.""" assert path.startswith(os.path.sep) and len(path.strip()) > 1 err_msg = 'Cannot create stage root {0}: Access to {1} is denied' user_uid = os.getuid() # Obtain lists of ancestor and descendant paths of the $user node, if any. group_paths, user_node, user_paths = partition_path(path, getpass.getuser()) for p in group_paths: if not os.path.exists(p): # Ensure access controls of subdirs created above `$user` inherit # from the parent and share the group. par_stat = os.stat(os.path.dirname(p)) mkdirp(p, group=par_stat.st_gid, mode=par_stat.st_mode) p_stat = os.stat(p) if par_stat.st_gid != p_stat.st_gid: tty.warn("Expected {0} to have group {1}, but it is {2}" .format(p, par_stat.st_gid, p_stat.st_gid)) if par_stat.st_mode & p_stat.st_mode != par_stat.st_mode: tty.warn("Expected {0} to support mode {1}, but it is {2}" .format(p, par_stat.st_mode, p_stat.st_mode)) if not can_access(p): raise OSError(errno.EACCES, err_msg.format(path, p)) # Add the path ending with the $user node to the user paths to ensure paths # from $user (on down) meet the ownership and permission requirements. if user_node: user_paths.insert(0, user_node) for p in user_paths: # Ensure access controls of subdirs from `$user` on down are # restricted to the user. if not os.path.exists(p): mkdirp(p, mode=stat.S_IRWXU) p_stat = os.stat(p) if p_stat.st_mode & stat.S_IRWXU != stat.S_IRWXU: tty.error("Expected {0} to support mode {1}, but it is {2}" .format(p, stat.S_IRWXU, p_stat.st_mode)) raise OSError(errno.EACCES, err_msg.format(path, p)) else: p_stat = os.stat(p) if user_uid != p_stat.st_uid: tty.warn("Expected user {0} to own {1}, but it is owned by {2}" .format(user_uid, p, p_stat.st_uid))
def md5(parser, args): if not args.files: setup_parser.parser.print_help() return 1 results = [] for url in args.files: try: checksum = compute_md5_checksum(url) results.append((checksum, url)) except FailedDownloadError as e: tty.warn("Failed to fetch %s" % url) tty.warn("%s" % e) except IOError as e: tty.warn("Error when reading %s" % url) tty.warn("%s" % e) # Dump the MD5s at last without interleaving them with downloads tty.msg("%d MD5 checksums:" % len(results)) for checksum, url in results: print "%s %s" % (checksum, url)
def detect_scheduler(): if (os.environ.get('CROSS') is None): srunbin = which('srun') if srunbin is None: aprunbin = which('aprun') if aprunbin is None: tty.warn("CROSS has not been set, however " "cannot detect scheduler.") return 'none' else: tty.warn("CROSS has not been set, however " "aprun has been found, assuming alps scheduler.") return 'alps' else: tty.warn("CROSS has not been set, however " "srun has been found, assuming slurm scheduler.") return 'slurm' else: tty.warn("CROSS has been set to %s by the user." % os.environ.get('CROSS')) return 'user'
def do_checksum(parser, args, algo): if not args.files: setup_parser.parser.print_help() return 1 urls = [x for x in normalized(args.files)] results = [] for url in urls: try: checksum = compute_checksum(url, algo) results.append((checksum, url)) except FailedDownloadError as e: tty.warn("Failed to fetch %s" % url) tty.warn("%s" % e) except IOError as e: tty.warn("Error when reading %s" % url) tty.warn("%s" % e) # Dump the hashes last, without interleaving them with downloads checksum = 'checksum' if len(results) == 1 else 'checksums' tty.msg("%d %s %s:" % (len(results), algo, checksum)) for checksum, url in results: print("{0} {1}".format(checksum, url))
def autoreconf(self, spec, prefix): """Not needed usually, configure should be already there""" # If configure exists nothing needs to be done if os.path.exists(self.configure_abs_path): return # Else try to regenerate it autotools = ['m4', 'autoconf', 'automake', 'libtool'] missing = [x for x in autotools if x not in spec] if missing: msg = 'Cannot generate configure: missing dependencies {0}' raise RuntimeError(msg.format(missing)) tty.msg('Configure script not found: trying to generate it') tty.warn('*********************************************************') tty.warn('* If the default procedure fails, consider implementing *') tty.warn('* a custom AUTORECONF phase in the package *') tty.warn('*********************************************************') with working_dir(self.configure_directory): m = inspect.getmodule(self) # This line is what is needed most of the time # --install, --verbose, --force autoreconf_args = ['-ivf'] autoreconf_args += self.autoreconf_search_path_args autoreconf_args += self.autoreconf_extra_args m.autoreconf(*autoreconf_args)
def set_up_license(pkg): """Prompt the user, letting them know that a license is required. For packages that rely on license files, a global license file is created and opened for editing. For packages that rely on environment variables to point to a license, a warning message is printed. For all other packages, documentation on how to set up a license is printed.""" # If the license can be stored in a file, create one if pkg.license_files: license_path = pkg.global_license_file if not os.path.exists(license_path): # Create a new license file write_license_file(pkg, license_path) # Open up file in user's favorite $EDITOR for editing spack.editor(license_path) tty.msg("Added global license file %s" % license_path) else: # Use already existing license file tty.msg("Found already existing license %s" % license_path) # If not a file, what about an environment variable? elif pkg.license_vars: tty.warn("A license is required to use %s. Please set %s to the " "full pathname to the license file, or port@host if you" " store your license keys on a dedicated license server" % (pkg.name, ' or '.join(pkg.license_vars))) # If not a file or variable, suggest a website for further info elif pkg.license_url: tty.warn("A license is required to use %s. See %s for details" % (pkg.name, pkg.license_url)) # If all else fails, you're on your own else: tty.warn("A license is required to use %s" % pkg.name)
def get_origin_info(remote): git_dir = os.path.join(spack.paths.prefix, '.git') git = which('git', required=True) try: branch = git('symbolic-ref', '--short', 'HEAD', output=str) except ProcessError: branch = 'develop' tty.warn('No branch found; using default branch: %s' % branch) if remote == 'origin' and \ branch not in ('master', 'develop'): branch = 'develop' tty.warn('Unknown branch found; using default branch: %s' % branch) try: origin_url = git( '--git-dir=%s' % git_dir, 'config', '--get', 'remote.%s.url' % remote, output=str) except ProcessError: origin_url = _SPACK_UPSTREAM tty.warn('No git repository found; ' 'using default upstream URL: %s' % origin_url) return (origin_url.strip(), branch.strip())
def install_tarball(spec, args): s = spack.spec.Spec(spec) if s.external or s.virtual: tty.warn("Skipping external or virtual package %s" % spec.format()) return yes_to_all = False if args.yes_to_all: yes_to_all = True force = False if args.force: force = True for d in s.dependencies(deptype=('link', 'run')): tty.msg("Installing buildcache for dependency spec %s" % d) install_tarball(d, args) package = spack.repo.get(spec) if s.concrete and package.installed and not force: tty.warn("Package for spec %s already installed." % spec.format(), " Use -f flag to overwrite.") else: tarball = bindist.download_tarball(spec) if tarball: tty.msg('Installing buildcache for spec %s' % spec.format()) try: bindist.extract_tarball(spec, tarball, yes_to_all, force) except NoOverwriteException as e: tty.warn("%s exists. use -f to force overwrite." % e.args) except NoVerifyException: tty.die("Package spec file failed signature verification," " use -y flag to install build cache") except NoChecksumException: tty.die("Package tarball failed checksum verification," " use -y flag to install build cache") finally: spack.store.db.reindex(spack.store.layout) else: tty.die('Download of binary cache file for spec %s failed.' % spec.format())
def __call__(self, *args, **kwargs): """Run this executable in a subprocess. Parameters: *args (str): Command-line arguments to the executable to run Keyword Arguments: _dump_env (dict): Dict to be set to the environment actually used (envisaged for testing purposes only) env (dict or EnvironmentModifications): The environment with which to run the executable extra_env (dict or EnvironmentModifications): Extra items to add to the environment (neither requires nor precludes env) fail_on_error (bool): Raise an exception if the subprocess returns an error. Default is True. The return code is available as ``exe.returncode`` ignore_errors (int or list): A list of error codes to ignore. If these codes are returned, this process will not raise an exception even if ``fail_on_error`` is set to ``True`` ignore_quotes (bool): If False, warn users that quotes are not needed as Spack does not use a shell. Defaults to False. input: Where to read stdin from output: Where to send stdout error: Where to send stderr Accepted values for input, output, and error: * python streams, e.g. open Python file objects, or ``os.devnull`` * filenames, which will be automatically opened for writing * ``str``, as in the Python string type. If you set these to ``str``, output and error will be written to pipes and returned as a string. If both ``output`` and ``error`` are set to ``str``, then one string is returned containing output concatenated with error. Not valid for ``input`` * ``str.split``, as in the ``split`` method of the Python string type. Behaves the same as ``str``, except that value is also written to ``stdout`` or ``stderr``. By default, the subprocess inherits the parent's file descriptors. """ # Environment env_arg = kwargs.get('env', None) # Setup default environment env = os.environ.copy() if env_arg is None else {} self.default_envmod.apply_modifications(env) env.update(self.default_env) from spack.util.environment import EnvironmentModifications # no cycle # Apply env argument if isinstance(env_arg, EnvironmentModifications): env_arg.apply_modifications(env) elif env_arg: env.update(env_arg) # Apply extra env extra_env = kwargs.get('extra_env', {}) if isinstance(extra_env, EnvironmentModifications): extra_env.apply_modifications(env) else: env.update(extra_env) if '_dump_env' in kwargs: kwargs['_dump_env'].clear() kwargs['_dump_env'].update(env) fail_on_error = kwargs.pop('fail_on_error', True) ignore_errors = kwargs.pop('ignore_errors', ()) ignore_quotes = kwargs.pop('ignore_quotes', False) # If they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): ignore_errors = (ignore_errors, ) input = kwargs.pop('input', None) output = kwargs.pop('output', None) error = kwargs.pop('error', None) if input is str: raise ValueError('Cannot use `str` as input stream.') def streamify(arg, mode): if isinstance(arg, string_types): return open(arg, mode), True elif arg in (str, str.split): return subprocess.PIPE, False else: return arg, False ostream, close_ostream = streamify(output, 'w') estream, close_estream = streamify(error, 'w') istream, close_istream = streamify(input, 'r') if not ignore_quotes: quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] if quoted_args: tty.warn( "Quotes in command arguments can confuse scripts like" " configure.", "The following arguments may cause problems when executed:", str("\n".join([" " + arg for arg in quoted_args])), "Quotes aren't needed because spack doesn't use a shell. " "Consider removing them.", "If multiple levels of quotation are required, use " "`ignore_quotes=True`.") cmd = self.exe + list(args) cmd_line = "'%s'" % "' '".join( map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) tty.debug(cmd_line) try: proc = subprocess.Popen( cmd, stdin=istream, stderr=estream, stdout=ostream, env=env) out, err = proc.communicate() result = None if output in (str, str.split) or error in (str, str.split): result = '' if output in (str, str.split): outstr = text_type(out.decode('utf-8')) result += outstr if output is str.split: sys.stdout.write(outstr) if error in (str, str.split): errstr = text_type(err.decode('utf-8')) result += errstr if error is str.split: sys.stderr.write(errstr) rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): long_msg = cmd_line if result: # If the output is not captured in the result, it will have # been stored either in the specified files (e.g. if # 'output' specifies a file) or written to the parent's # stdout/stderr (e.g. if 'output' is not specified) long_msg += '\n' + result raise ProcessError('Command exited with status %d:' % proc.returncode, long_msg) return result except OSError as e: raise ProcessError( '%s: %s' % (self.exe[0], e.strerror), 'Command: ' + cmd_line) except subprocess.CalledProcessError as e: if fail_on_error: raise ProcessError( str(e), '\nExit status %d when invoking command: %s' % (proc.returncode, cmd_line)) finally: if close_ostream: ostream.close() if close_estream: estream.close() if close_istream: istream.close()
def _save_distutil_vars(self): """ Run before changing automatically generated contents of the _sysconfigdata.py, which is used by distutils to figure out what executables to use while compiling and linking extensions. If we build extensions with spack those executables should be spack's wrappers. Spack partially covers this by setting environment variables that are also accounted for by distutils. Currently there is one more known variable that must be set, which is LDSHARED, so the method saves its autogenerated value to pass it to the dependent package's setup script. """ self._distutil_vars = {} input_filename = self.get_sysconfigdata_name() input_dict = None try: with open(input_filename) as input_file: match = re.search(r'build_time_vars\s*=\s*(?P<dict>{.*})', input_file.read(), flags=re.DOTALL) if match: input_dict = ast.literal_eval(match.group('dict')) except (IOError, SyntaxError): pass if not input_dict: tty.warn("Failed to find 'build_time_vars' dictionary in file " "'%s'. This might cause the extensions that are " "installed with distutils to call compilers directly " "avoiding Spack's wrappers." % input_filename) return for var_name in Python._DISTUTIL_VARS_TO_SAVE: if var_name in input_dict: self._distutil_vars[var_name] = input_dict[var_name] else: tty.warn("Failed to find key '%s' in 'build_time_vars' " "dictionary in file '%s'. This might cause the " "extensions that are installed with distutils to " "call compilers directly avoiding Spack's wrappers." % (var_name, input_filename)) if len(self._distutil_vars) > 0: output_filename = None try: output_filename = join_path( spack.store.layout.metadata_path(self.spec), Python._DISTUTIL_CACHE_FILENAME) with open(output_filename, 'w') as output_file: sjson.dump(self._distutil_vars, output_file) except Exception: tty.warn("Failed to save metadata for distutils. This might " "cause the extensions that are installed with " "distutils to call compilers directly avoiding " "Spack's wrappers.") # We make the cache empty if we failed to save it to file # to provide the same behaviour as in the case when the cache # is initialized by the method load_distutils_data(). self._distutil_vars = {} if output_filename: force_remove(output_filename)
def flake8(parser, args): tty.warn("spack flake8 is deprecated", "please use `spack style` to run style checks") return spack.cmd.style.style(parser, args)
def ci_rebuild(args): """Check a single spec against the remote mirror, and rebuild it from source if the mirror does not contain the full hash match of the spec as computed locally. """ env = ev.get_env(args, 'ci rebuild', required=True) # Make sure the environment is "gitlab-enabled", or else there's nothing # to do. yaml_root = ev.config_dict(env.yaml) gitlab_ci = None if 'gitlab-ci' in yaml_root: gitlab_ci = yaml_root['gitlab-ci'] if not gitlab_ci: tty.die('spack ci rebuild requires an env containing gitlab-ci cfg') # Grab the environment variables we need. These either come from the # pipeline generation step ("spack ci generate"), where they were written # out as variables, or else provided by GitLab itself. pipeline_artifacts_dir = get_env_var('SPACK_ARTIFACTS_ROOT') job_log_dir = get_env_var('SPACK_JOB_LOG_DIR') repro_dir = get_env_var('SPACK_JOB_REPRO_DIR') local_mirror_dir = get_env_var('SPACK_LOCAL_MIRROR_DIR') concrete_env_dir = get_env_var('SPACK_CONCRETE_ENV_DIR') ci_pipeline_id = get_env_var('CI_PIPELINE_ID') ci_job_name = get_env_var('CI_JOB_NAME') signing_key = get_env_var('SPACK_SIGNING_KEY') root_spec = get_env_var('SPACK_ROOT_SPEC') job_spec_pkg_name = get_env_var('SPACK_JOB_SPEC_PKG_NAME') compiler_action = get_env_var('SPACK_COMPILER_ACTION') cdash_build_name = get_env_var('SPACK_CDASH_BUILD_NAME') related_builds = get_env_var('SPACK_RELATED_BUILDS_CDASH') spack_pipeline_type = get_env_var('SPACK_PIPELINE_TYPE') pr_mirror_url = get_env_var('SPACK_PR_MIRROR_URL') remote_mirror_url = get_env_var('SPACK_REMOTE_MIRROR_URL') # Construct absolute paths relative to current $CI_PROJECT_DIR ci_project_dir = get_env_var('CI_PROJECT_DIR') pipeline_artifacts_dir = os.path.join(ci_project_dir, pipeline_artifacts_dir) job_log_dir = os.path.join(ci_project_dir, job_log_dir) repro_dir = os.path.join(ci_project_dir, repro_dir) local_mirror_dir = os.path.join(ci_project_dir, local_mirror_dir) concrete_env_dir = os.path.join(ci_project_dir, concrete_env_dir) # Debug print some of the key environment variables we should have received tty.debug('pipeline_artifacts_dir = {0}'.format(pipeline_artifacts_dir)) tty.debug('root_spec = {0}'.format(root_spec)) tty.debug('remote_mirror_url = {0}'.format(remote_mirror_url)) tty.debug('job_spec_pkg_name = {0}'.format(job_spec_pkg_name)) tty.debug('compiler_action = {0}'.format(compiler_action)) # Query the environment manifest to find out whether we're reporting to a # CDash instance, and if so, gather some information from the manifest to # support that task. enable_cdash = False if 'cdash' in yaml_root: enable_cdash = True ci_cdash = yaml_root['cdash'] job_spec_buildgroup = ci_cdash['build-group'] cdash_base_url = ci_cdash['url'] cdash_project = ci_cdash['project'] proj_enc = urlencode({'project': cdash_project}) eq_idx = proj_enc.find('=') + 1 cdash_project_enc = proj_enc[eq_idx:] cdash_site = ci_cdash['site'] tty.debug('cdash_base_url = {0}'.format(cdash_base_url)) tty.debug('cdash_project = {0}'.format(cdash_project)) tty.debug('cdash_project_enc = {0}'.format(cdash_project_enc)) tty.debug('cdash_build_name = {0}'.format(cdash_build_name)) tty.debug('cdash_site = {0}'.format(cdash_site)) tty.debug('related_builds = {0}'.format(related_builds)) tty.debug('job_spec_buildgroup = {0}'.format(job_spec_buildgroup)) # Is this a pipeline run on a spack PR or a merge to develop? It might # be neither, e.g. a pipeline run on some environment repository. spack_is_pr_pipeline = spack_pipeline_type == 'spack_pull_request' spack_is_develop_pipeline = spack_pipeline_type == 'spack_protected_branch' tty.debug('Pipeline type - PR: {0}, develop: {1}'.format( spack_is_pr_pipeline, spack_is_develop_pipeline)) # Figure out what is our temporary storage mirror: Is it artifacts # buildcache? Or temporary-storage-url-prefix? In some cases we need to # force something or pipelines might not have a way to propagate build # artifacts from upstream to downstream jobs. pipeline_mirror_url = None temp_storage_url_prefix = None if 'temporary-storage-url-prefix' in gitlab_ci: temp_storage_url_prefix = gitlab_ci['temporary-storage-url-prefix'] pipeline_mirror_url = url_util.join(temp_storage_url_prefix, ci_pipeline_id) enable_artifacts_mirror = False if 'enable-artifacts-buildcache' in gitlab_ci: enable_artifacts_mirror = gitlab_ci['enable-artifacts-buildcache'] if (enable_artifacts_mirror or (spack_is_pr_pipeline and not enable_artifacts_mirror and not temp_storage_url_prefix)): # If you explicitly enabled the artifacts buildcache feature, or # if this is a PR pipeline but you did not enable either of the # per-pipeline temporary storage features, we force the use of # artifacts buildcache. Otherwise jobs will not have binary # dependencies from previous stages available since we do not # allow pushing binaries to the remote mirror during PR pipelines. enable_artifacts_mirror = True pipeline_mirror_url = 'file://' + local_mirror_dir mirror_msg = 'artifact buildcache enabled, mirror url: {0}'.format( pipeline_mirror_url) tty.debug(mirror_msg) # Whatever form of root_spec we got, use it to get a map giving us concrete # specs for this job and all of its dependencies. spec_map = spack_ci.get_concrete_specs(env, root_spec, job_spec_pkg_name, related_builds, compiler_action) job_spec = spec_map[job_spec_pkg_name] job_spec_yaml_file = '{0}.yaml'.format(job_spec_pkg_name) job_spec_yaml_path = os.path.join(repro_dir, job_spec_yaml_file) # To provide logs, cdash reports, etc for developer download/perusal, # these things have to be put into artifacts. This means downstream # jobs that "need" this job will get those artifacts too. So here we # need to clean out the artifacts we may have got from upstream jobs. cdash_report_dir = os.path.join(pipeline_artifacts_dir, 'cdash_report') if os.path.exists(cdash_report_dir): shutil.rmtree(cdash_report_dir) if os.path.exists(job_log_dir): shutil.rmtree(job_log_dir) if os.path.exists(repro_dir): shutil.rmtree(repro_dir) # Now that we removed them if they existed, create the directories we # need for storing artifacts. The cdash_report directory will be # created internally if needed. os.makedirs(job_log_dir) os.makedirs(repro_dir) # Copy the concrete environment files to the repro directory so we can # expose them as artifacts and not conflict with the concrete environment # files we got as artifacts from the upstream pipeline generation job. # Try to cast a slightly wider net too, and hopefully get the generated # pipeline yaml. If we miss it, the user will still be able to go to the # pipeline generation job and get it from there. target_dirs = [concrete_env_dir, pipeline_artifacts_dir] for dir_to_list in target_dirs: for file_name in os.listdir(dir_to_list): src_file = os.path.join(dir_to_list, file_name) if os.path.isfile(src_file): dst_file = os.path.join(repro_dir, file_name) shutil.copyfile(src_file, dst_file) # If signing key was provided via "SPACK_SIGNING_KEY", then try to # import it. if signing_key: spack_ci.import_signing_key(signing_key) # Depending on the specifics of this job, we might need to turn on the # "config:install_missing compilers" option (to build this job spec # with a bootstrapped compiler), or possibly run "spack compiler find" # (to build a bootstrap compiler or one of its deps in a # compiler-agnostic way), or maybe do nothing at all (to build a spec # using a compiler already installed on the target system). spack_ci.configure_compilers(compiler_action) # Write this job's spec yaml into the reproduction directory, and it will # also be used in the generated "spack install" command to install the spec tty.debug('job concrete spec path: {0}'.format(job_spec_yaml_path)) with open(job_spec_yaml_path, 'w') as fd: fd.write(job_spec.to_yaml(hash=ht.build_hash)) # Write the concrete root spec yaml into the reproduction directory root_spec_yaml_path = os.path.join(repro_dir, 'root.yaml') with open(root_spec_yaml_path, 'w') as fd: fd.write(spec_map['root'].to_yaml(hash=ht.build_hash)) # Write some other details to aid in reproduction into an artifact repro_file = os.path.join(repro_dir, 'repro.json') repro_details = { 'job_name': ci_job_name, 'job_spec_yaml': job_spec_yaml_file, 'root_spec_yaml': 'root.yaml', 'ci_project_dir': ci_project_dir } with open(repro_file, 'w') as fd: fd.write(json.dumps(repro_details)) # Write information about spack into an artifact in the repro dir spack_info = spack_ci.get_spack_info() spack_info_file = os.path.join(repro_dir, 'spack_info.txt') with open(spack_info_file, 'w') as fd: fd.write('\n{0}\n'.format(spack_info)) # If we decided there should be a temporary storage mechanism, add that # mirror now so it's used when we check for a full hash match already # built for this spec. if pipeline_mirror_url: spack.mirror.add(spack_ci.TEMP_STORAGE_MIRROR_NAME, pipeline_mirror_url, cfg.default_modify_scope()) cdash_build_id = None cdash_build_stamp = None # Check configured mirrors for a built spec with a matching full hash matches = bindist.get_mirrors_for_spec(job_spec, full_hash_match=True, index_only=False) if matches: # Got a full hash match on at least one configured mirror. All # matches represent the fully up-to-date spec, so should all be # equivalent. If artifacts mirror is enabled, we just pick one # of the matches and download the buildcache files from there to # the artifacts, so they're available to be used by dependent # jobs in subsequent stages. tty.msg('No need to rebuild {0}, found full hash match at: '.format( job_spec_pkg_name)) for match in matches: tty.msg(' {0}'.format(match['mirror_url'])) if enable_artifacts_mirror: matching_mirror = matches[0]['mirror_url'] build_cache_dir = os.path.join(local_mirror_dir, 'build_cache') tty.debug('Getting {0} buildcache from {1}'.format( job_spec_pkg_name, matching_mirror)) tty.debug('Downloading to {0}'.format(build_cache_dir)) buildcache.download_buildcache_files(job_spec, build_cache_dir, False, matching_mirror) # Now we are done and successful sys.exit(0) # No full hash match anywhere means we need to rebuild spec # Start with spack arguments install_args = [base_arg for base_arg in CI_REBUILD_INSTALL_BASE_ARGS] config = cfg.get('config') if not config['verify_ssl']: install_args.append('-k') install_args.extend([ 'install', '--keep-stage', '--require-full-hash-match', ]) can_verify = spack_ci.can_verify_binaries() verify_binaries = can_verify and spack_is_pr_pipeline is False if not verify_binaries: install_args.append('--no-check-signature') # If CDash reporting is enabled, we first register this build with # the specified CDash instance, then relate the build to those of # its dependencies. if enable_cdash: tty.debug('CDash: Registering build') (cdash_build_id, cdash_build_stamp) = spack_ci.register_cdash_build( cdash_build_name, cdash_base_url, cdash_project, cdash_site, job_spec_buildgroup) if cdash_build_id is not None: cdash_upload_url = '{0}/submit.php?project={1}'.format( cdash_base_url, cdash_project_enc) install_args.extend([ '--cdash-upload-url', cdash_upload_url, '--cdash-build', cdash_build_name, '--cdash-site', cdash_site, '--cdash-buildstamp', cdash_build_stamp, ]) tty.debug('CDash: Relating build with dependency builds') spack_ci.relate_cdash_builds( spec_map, cdash_base_url, cdash_build_id, cdash_project, [pipeline_mirror_url, pr_mirror_url, remote_mirror_url]) # A compiler action of 'FIND_ANY' means we are building a bootstrap # compiler or one of its deps. # TODO: when compilers are dependencies, we should include --no-add if compiler_action != 'FIND_ANY': install_args.append('--no-add') # TODO: once we have the concrete spec registry, use the DAG hash # to identify the spec to install, rather than the concrete spec # yaml file. install_args.extend(['-f', job_spec_yaml_path]) tty.debug('Installing {0} from source'.format(job_spec.name)) tty.debug('spack install arguments: {0}'.format(install_args)) # Write the install command to a shell script with open('install.sh', 'w') as fd: fd.write('#!/bin/bash\n\n') fd.write('\n# spack install command\n') fd.write(' '.join(['"{0}"'.format(i) for i in install_args])) fd.write('\n') st = os.stat('install.sh') os.chmod('install.sh', st.st_mode | stat.S_IEXEC) install_copy_path = os.path.join(repro_dir, 'install.sh') shutil.copyfile('install.sh', install_copy_path) # Run the generated install.sh shell script as if it were being run in # a login shell. try: install_process = subprocess.Popen(['bash', '-l', './install.sh']) install_process.wait() install_exit_code = install_process.returncode except (ValueError, subprocess.CalledProcessError, OSError) as inst: tty.error('Encountered error running install script') tty.error(inst) # Now do the post-install tasks tty.debug('spack install exited {0}'.format(install_exit_code)) # If a spec fails to build in a spack develop pipeline, we add it to a # list of known broken full hashes. This allows spack PR pipelines to # avoid wasting compute cycles attempting to build those hashes. if install_exit_code == INSTALL_FAIL_CODE and spack_is_develop_pipeline: tty.debug('Install failed on develop') if 'broken-specs-url' in gitlab_ci: broken_specs_url = gitlab_ci['broken-specs-url'] dev_fail_hash = job_spec.full_hash() broken_spec_path = url_util.join(broken_specs_url, dev_fail_hash) tty.msg('Reporting broken develop build as: {0}'.format( broken_spec_path)) tmpdir = tempfile.mkdtemp() empty_file_path = os.path.join(tmpdir, 'empty.txt') broken_spec_details = { 'broken-spec': { 'job-url': get_env_var('CI_JOB_URL'), 'pipeline-url': get_env_var('CI_PIPELINE_URL'), 'concrete-spec-yaml': job_spec.to_dict(hash=ht.full_hash) } } try: with open(empty_file_path, 'w') as efd: efd.write(syaml.dump(broken_spec_details)) web_util.push_to_url(empty_file_path, broken_spec_path, keep_original=False, extra_args={'ContentType': 'text/plain'}) except Exception as err: # If we got some kind of S3 (access denied or other connection # error), the first non boto-specific class in the exception # hierarchy is Exception. Just print a warning and return msg = 'Error writing to broken specs list {0}: {1}'.format( broken_spec_path, err) tty.warn(msg) finally: shutil.rmtree(tmpdir) # We generated the "spack install ..." command to "--keep-stage", copy # any logs from the staging directory to artifacts now spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) # Create buildcache on remote mirror, either on pr-specific mirror or # on the main mirror defined in the gitlab-enabled spack environment if spack_is_pr_pipeline: buildcache_mirror_url = pr_mirror_url else: buildcache_mirror_url = remote_mirror_url # If the install succeeded, create a buildcache entry for this job spec # and push it to one or more mirrors. If the install did not succeed, # print out some instructions on how to reproduce this build failure # outside of the pipeline environment. if install_exit_code == 0: can_sign = spack_ci.can_sign_binaries() sign_binaries = can_sign and spack_is_pr_pipeline is False # Create buildcache in either the main remote mirror, or in the # per-PR mirror, if this is a PR pipeline if buildcache_mirror_url: spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, buildcache_mirror_url, sign_binaries) if cdash_build_id: tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( cdash_build_id, buildcache_mirror_url)) spack_ci.write_cdashid_to_mirror(cdash_build_id, job_spec, buildcache_mirror_url) # Create another copy of that buildcache in the per-pipeline # temporary storage mirror (this is only done if either # artifacts buildcache is enabled or a temporary storage url # prefix is set) if pipeline_mirror_url: spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, pipeline_mirror_url, sign_binaries) if cdash_build_id: tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( cdash_build_id, pipeline_mirror_url)) spack_ci.write_cdashid_to_mirror(cdash_build_id, job_spec, pipeline_mirror_url) # If this is a develop pipeline, check if the spec that we just built is # on the broken-specs list. If so, remove it. if spack_is_develop_pipeline and 'broken-specs-url' in gitlab_ci: broken_specs_url = gitlab_ci['broken-specs-url'] just_built_hash = job_spec.full_hash() broken_spec_path = url_util.join(broken_specs_url, just_built_hash) if web_util.url_exists(broken_spec_path): tty.msg('Removing {0} from the list of broken specs'.format( broken_spec_path)) try: web_util.remove_url(broken_spec_path) except Exception as err: # If we got some kind of S3 (access denied or other connection # error), the first non boto-specific class in the exception # hierarchy is Exception. Just print a warning and return msg = 'Error removing {0} from broken specs list: {1}'.format( broken_spec_path, err) tty.warn(msg) else: tty.debug('spack install exited non-zero, will not create buildcache') api_root_url = get_env_var('CI_API_V4_URL') ci_project_id = get_env_var('CI_PROJECT_ID') ci_job_id = get_env_var('CI_JOB_ID') repro_job_url = '{0}/projects/{1}/jobs/{2}/artifacts'.format( api_root_url, ci_project_id, ci_job_id) # Control characters cause this to be printed in blue so it stands out reproduce_msg = """ \033[34mTo reproduce this build locally, run: spack ci reproduce-build {0} [--working-dir <dir>] If this project does not have public pipelines, you will need to first: export GITLAB_PRIVATE_TOKEN=<generated_token> ... then follow the printed instructions.\033[0;0m """.format(repro_job_url) print(reproduce_msg) # Tie job success/failure to the success/failure of building the spec return install_exit_code
def fetch(self): self.stage.chdir() if self.archive_file: tty.msg("Already downloaded %s" % self.archive_file) return possible_files = self.stage.expected_archive_files save_file = None partial_file = None if possible_files: save_file = self.stage.expected_archive_files[0] partial_file = self.stage.expected_archive_files[0] + '.part' tty.msg("Trying to fetch from %s" % self.url) if partial_file: save_args = [ '-C', '-', # continue partial downloads '-o', partial_file ] # use a .part file else: save_args = ['-O'] curl_args = save_args + [ '-f', # fail on >400 errors '-D', '-', # print out HTML headers '-L', # resolve 3xx redirects self.url, ] if sys.stdout.isatty(): curl_args.append('-#') # status bar when using a tty else: curl_args.append('-sS') # just errors when not. # Run curl but grab the mime type from the http headers headers = spack.curl(*curl_args, output=str, fail_on_error=False) if spack.curl.returncode != 0: # clean up archive on failure. if self.archive_file: os.remove(self.archive_file) if partial_file and os.path.exists(partial_file): os.remove(partial_file) if spack.curl.returncode == 22: # This is a 404. Curl will print the error. raise FailedDownloadError(self.url, "URL %s was not found!" % self.url) elif spack.curl.returncode == 60: # This is a certificate error. Suggest spack -k raise FailedDownloadError( self.url, "Curl was unable to fetch due to invalid certificate. " "This is either an attack, or your cluster's SSL " "configuration is bad. If you believe your SSL " "configuration is bad, you can try running spack -k, " "which will not check SSL certificates." "Use this at your own risk.") else: # This is some other curl error. Curl will print the # error, but print a spack message too raise FailedDownloadError( self.url, "Curl failed with error %d" % spack.curl.returncode) # Check if we somehow got an HTML file rather than the archive we # asked for. We only look at the last content type, to handle # redirects properly. content_types = re.findall(r'Content-Type:[^\r\n]+', headers) if content_types and 'text/html' in content_types[-1]: tty.warn( "The contents of ", (self.archive_file if self.archive_file is not None else "the archive"), " look like HTML.", "The checksum will likely be bad. If it is, you can use", "'spack clean <package>' to remove the bad archive, then", "fix your internet gateway issue and install again.") if save_file: os.rename(partial_file, save_file) if not self.archive_file: raise FailedDownloadError(self.url)
def refresh(module_types, specs, args): """Regenerates the module files for every spec in specs and every module type in module types. """ # Prompt a message to the user about what is going to change if not specs: tty.msg('No package matches your query') return if not args.yes_to_all: msg = 'You are about to regenerate {types} module files for:\n' types = ', '.join(module_types) tty.msg(msg.format(types=types)) spack.cmd.display_specs(specs, long=True) print('') answer = tty.get_yes_or_no('Do you want to proceed?') if not answer: tty.die('Module file regeneration aborted.') # Cycle over the module types and regenerate module files for module_type in module_types: cls = spack.modules.module_types[module_type] writers = [ cls(spec) for spec in specs if spack.repo.exists(spec.name) ] # skip unknown packages. # Filter blacklisted packages early writers = [x for x in writers if not x.conf.blacklisted] # Detect name clashes in module files file2writer = collections.defaultdict(list) for item in writers: file2writer[item.layout.filename].append(item) if len(file2writer) != len(writers): message = 'Name clashes detected in module files:\n' for filename, writer_list in file2writer.items(): if len(writer_list) > 1: message += '\nfile: {0}\n'.format(filename) for x in writer_list: message += 'spec: {0}\n'.format(x.spec.format()) tty.error(message) tty.error('Operation aborted') raise SystemExit(1) if len(writers) == 0: msg = 'Nothing to be done for {0} module files.' tty.msg(msg.format(module_type)) continue # If we arrived here we have at least one writer module_type_root = writers[0].layout.dirname() # Proceed regenerating module files tty.msg('Regenerating {name} module files'.format(name=module_type)) if os.path.isdir(module_type_root) and args.delete_tree: shutil.rmtree(module_type_root, ignore_errors=False) filesystem.mkdirp(module_type_root) for x in writers: try: x.write(overwrite=True) except Exception as e: msg = 'Could not write module file [{0}]' tty.warn(msg.format(x.layout.filename)) tty.warn('\t--> {0} <--'.format(str(e)))
def handle_fetch_error(self, error): tty.warn("Fetching OpenSSL failed. This may indicate that OpenSSL has " "been updated, and the version in your instance of Spack is " "insecure. Consider updating to the latest OpenSSL version.")