def setup_package(pkg, dirty): """Execute all environment setup routines.""" build_env = EnvironmentModifications() if not dirty: clean_environment() set_compiler_environment_variables(pkg, build_env) set_build_environment_variables(pkg, build_env, dirty) pkg.architecture.platform.setup_platform_environment(pkg, build_env) build_env.extend( modifications_from_dependencies(pkg.spec, context='build') ) if (not dirty) and (not build_env.is_unset('CPATH')): tty.debug("A dependency has updated CPATH, this may lead pkg-config" " to assume that the package is part of the system" " includes and omit it when invoked with '--cflags'.") set_module_variables_for_package(pkg) pkg.setup_build_environment(build_env) # Loading modules, in particular if they are meant to be used outside # of Spack, can change environment variables that are relevant to the # build of packages. To avoid a polluted environment, preserve the # value of a few, selected, environment variables # With the current ordering of environment modifications, this is strictly # unnecessary. Modules affecting these variables will be overwritten anyway with preserve_environment('CC', 'CXX', 'FC', 'F77'): # All module loads that otherwise would belong in previous # functions have to occur after the build_env object has its # modifications applied. Otherwise the environment modifications # could undo module changes, such as unsetting LD_LIBRARY_PATH # after a module changes it. for mod in pkg.compiler.modules: # Fixes issue https://github.com/spack/spack/issues/3153 if os.environ.get("CRAY_CPU_TARGET") == "mic-knl": load_module("cce") load_module(mod) # kludge to handle cray libsci being automatically loaded by PrgEnv # modules on cray platform. Module unload does no damage when # unnecessary module('unload', 'cray-libsci') if pkg.architecture.target.module_name: load_module(pkg.architecture.target.module_name) load_external_modules(pkg) implicit_rpaths = pkg.compiler.implicit_rpaths() if implicit_rpaths: build_env.set('SPACK_COMPILER_IMPLICIT_RPATHS', ':'.join(implicit_rpaths)) # Make sure nothing's strange about the Spack environment. validate(build_env, tty.warn) build_env.apply_modifications()
def test_source_files(files_to_be_sourced): """Tests the construction of a list of environment modifications that are the result of sourcing a file. """ env = EnvironmentModifications() for filename in files_to_be_sourced: if filename.endswith('sourceme_parameters.sh'): env.extend( EnvironmentModifications.from_sourcing_file( filename, 'intel64')) else: env.extend(EnvironmentModifications.from_sourcing_file(filename)) modifications = env.group_by_name() # This is sensitive to the user's environment; can include # spurious entries for things like PS1 # # TODO: figure out how to make a bit more robust. assert len(modifications) >= 5 # Set new variables assert len(modifications['NEW_VAR']) == 1 assert isinstance(modifications['NEW_VAR'][0], SetEnv) assert modifications['NEW_VAR'][0].value == 'new' assert len(modifications['FOO']) == 1 assert isinstance(modifications['FOO'][0], SetEnv) assert modifications['FOO'][0].value == 'intel64' # Unset variables assert len(modifications['EMPTY_PATH_LIST']) == 1 assert isinstance(modifications['EMPTY_PATH_LIST'][0], UnsetEnv) # Modified variables assert len(modifications['UNSET_ME']) == 1 assert isinstance(modifications['UNSET_ME'][0], SetEnv) assert modifications['UNSET_ME'][0].value == 'overridden' assert len(modifications['PATH_LIST']) == 3 assert isinstance(modifications['PATH_LIST'][0], RemovePath) assert modifications['PATH_LIST'][0].value == '/path/third' assert isinstance(modifications['PATH_LIST'][1], AppendPath) assert modifications['PATH_LIST'][1].value == '/path/fourth' assert isinstance(modifications['PATH_LIST'][2], PrependPath) assert modifications['PATH_LIST'][2].value == '/path/first'
class Executable(object): """Class representing a program that can be run on the command line.""" def __init__(self, name): self.exe = shlex.split(str(name)) self.default_env = {} from spack.util.environment import EnvironmentModifications # no cycle self.default_envmod = EnvironmentModifications() self.returncode = None if not self.exe: raise ProcessError("Cannot construct executable for '%s'" % name) def add_default_arg(self, arg): """Add a default argument to the command.""" self.exe.append(arg) def add_default_env(self, key, value): """Set an environment variable when the command is run. Parameters: key: The environment variable to set value: The value to set it to """ self.default_env[key] = value def add_default_envmod(self, envmod): """Set an EnvironmentModifications to use when the command is run.""" self.default_envmod.extend(envmod) @property def command(self): """The command-line string. Returns: str: The executable and default arguments """ return ' '.join(self.exe) @property def name(self): """The executable name. Returns: str: The basename of the executable """ return os.path.basename(self.path) @property def path(self): """The path to the executable. Returns: str: The path to the executable """ return self.exe[0] def __call__(self, *args, **kwargs): """Run this executable in a subprocess. Parameters: *args (str): Command-line arguments to the executable to run Keyword Arguments: _dump_env (dict): Dict to be set to the environment actually used (envisaged for testing purposes only) env (dict or EnvironmentModifications): The environment with which to run the executable extra_env (dict or EnvironmentModifications): Extra items to add to the environment (neither requires nor precludes env) fail_on_error (bool): Raise an exception if the subprocess returns an error. Default is True. The return code is available as ``exe.returncode`` ignore_errors (int or list): A list of error codes to ignore. If these codes are returned, this process will not raise an exception even if ``fail_on_error`` is set to ``True`` ignore_quotes (bool): If False, warn users that quotes are not needed as Spack does not use a shell. Defaults to False. input: Where to read stdin from output: Where to send stdout error: Where to send stderr Accepted values for input, output, and error: * python streams, e.g. open Python file objects, or ``os.devnull`` * filenames, which will be automatically opened for writing * ``str``, as in the Python string type. If you set these to ``str``, output and error will be written to pipes and returned as a string. If both ``output`` and ``error`` are set to ``str``, then one string is returned containing output concatenated with error. Not valid for ``input`` * ``str.split``, as in the ``split`` method of the Python string type. Behaves the same as ``str``, except that value is also written to ``stdout`` or ``stderr``. By default, the subprocess inherits the parent's file descriptors. """ # Environment env_arg = kwargs.get('env', None) # Setup default environment env = os.environ.copy() if env_arg is None else {} self.default_envmod.apply_modifications(env) env.update(self.default_env) from spack.util.environment import EnvironmentModifications # no cycle # Apply env argument if isinstance(env_arg, EnvironmentModifications): env_arg.apply_modifications(env) elif env_arg: env.update(env_arg) # Apply extra env extra_env = kwargs.get('extra_env', {}) if isinstance(extra_env, EnvironmentModifications): extra_env.apply_modifications(env) else: env.update(extra_env) if '_dump_env' in kwargs: kwargs['_dump_env'].clear() kwargs['_dump_env'].update(env) fail_on_error = kwargs.pop('fail_on_error', True) ignore_errors = kwargs.pop('ignore_errors', ()) ignore_quotes = kwargs.pop('ignore_quotes', False) # If they just want to ignore one error code, make it a tuple. if isinstance(ignore_errors, int): ignore_errors = (ignore_errors, ) input = kwargs.pop('input', None) output = kwargs.pop('output', None) error = kwargs.pop('error', None) if input is str: raise ValueError('Cannot use `str` as input stream.') def streamify(arg, mode): if isinstance(arg, string_types): return open(arg, mode), True elif arg in (str, str.split): return subprocess.PIPE, False else: return arg, False ostream, close_ostream = streamify(output, 'w') estream, close_estream = streamify(error, 'w') istream, close_istream = streamify(input, 'r') if not ignore_quotes: quoted_args = [arg for arg in args if re.search(r'^"|^\'|"$|\'$', arg)] if quoted_args: tty.warn( "Quotes in command arguments can confuse scripts like" " configure.", "The following arguments may cause problems when executed:", str("\n".join([" " + arg for arg in quoted_args])), "Quotes aren't needed because spack doesn't use a shell. " "Consider removing them.", "If multiple levels of quotation are required, use " "`ignore_quotes=True`.") cmd = self.exe + list(args) cmd_line = "'%s'" % "' '".join( map(lambda arg: arg.replace("'", "'\"'\"'"), cmd)) tty.debug(cmd_line) try: proc = subprocess.Popen( cmd, stdin=istream, stderr=estream, stdout=ostream, env=env) out, err = proc.communicate() result = None if output in (str, str.split) or error in (str, str.split): result = '' if output in (str, str.split): outstr = text_type(out.decode('utf-8')) result += outstr if output is str.split: sys.stdout.write(outstr) if error in (str, str.split): errstr = text_type(err.decode('utf-8')) result += errstr if error is str.split: sys.stderr.write(errstr) rc = self.returncode = proc.returncode if fail_on_error and rc != 0 and (rc not in ignore_errors): long_msg = cmd_line if result: # If the output is not captured in the result, it will have # been stored either in the specified files (e.g. if # 'output' specifies a file) or written to the parent's # stdout/stderr (e.g. if 'output' is not specified) long_msg += '\n' + result raise ProcessError('Command exited with status %d:' % proc.returncode, long_msg) return result except OSError as e: raise ProcessError( '%s: %s' % (self.exe[0], e.strerror), 'Command: ' + cmd_line) except subprocess.CalledProcessError as e: if fail_on_error: raise ProcessError( str(e), '\nExit status %d when invoking command: %s' % (proc.returncode, cmd_line)) finally: if close_ostream: ostream.close() if close_estream: estream.close() if close_istream: istream.close() def __eq__(self, other): return hasattr(other, 'exe') and self.exe == other.exe def __neq__(self, other): return not (self == other) def __hash__(self): return hash((type(self), ) + tuple(self.exe)) def __repr__(self): return '<exe: %s>' % self.exe def __str__(self): return ' '.join(self.exe)
def get_executable(exe, spec=None, install=False): """Find an executable named exe, either in PATH or in Spack Args: exe (str): needed executable name spec (Spec or str): spec to search for exe in (default exe) install (bool): install spec if not available When ``install`` is True, Spack will use the python used to run Spack as an external. The ``install`` option should only be used with packages that install quickly (when using external python) or are guaranteed by Spack organization to be in a binary mirror (clingo).""" # Search the system first runner = spack.util.executable.which(exe) if runner: return runner # Check whether it's already installed spec = spack.spec.Spec(spec or exe) installed_specs = spack.store.db.query(spec, installed=True) for ispec in installed_specs: # filter out directories of the same name as the executable exe_path = [exe_p for exe_p in fs.find(ispec.prefix, exe) if fs.is_exe(exe_p)] if exe_path: ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in ispec.traverse(root=True, order='post'): envmod.extend(uenv.environment_modifications_for_spec(dep)) ret.add_default_envmod(envmod) return ret else: tty.warn('Exe %s not found in prefix %s' % (exe, ispec.prefix)) def _raise_error(executable, exe_spec): error_msg = 'cannot find the executable "{0}"'.format(executable) if exe_spec: error_msg += ' from spec "{0}'.format(exe_spec) raise RuntimeError(error_msg) # If we're not allowed to install this for ourselves, we can't find it if not install: _raise_error(exe, spec) with spack_python_interpreter(): # We will install for ourselves, using this python if needed # Concretize the spec spec.concretize() spec.package.do_install() # filter out directories of the same name as the executable exe_path = [exe_p for exe_p in fs.find(spec.prefix, exe) if fs.is_exe(exe_p)] if exe_path: ret = spack.util.executable.Executable(exe_path[0]) envmod = EnvironmentModifications() for dep in spec.traverse(root=True, order='post'): envmod.extend(uenv.environment_modifications_for_spec(dep)) ret.add_default_envmod(envmod) return ret _raise_error(exe, spec)
def env_activate(args): if not args.activate_env and not args.dir and not args.temp: tty.die('spack env activate requires an environment name, directory, or --temp') if not args.shell: spack.cmd.common.shell_init_instructions( "spack env activate", " eval `spack env activate {sh_arg} [...]`", ) return 1 # Error out when -e, -E, -D flags are given, cause they are ambiguous. if args.env or args.no_env or args.env_dir: tty.die('Calling spack env activate with --env, --env-dir and --no-env ' 'is ambiguous') env_name_or_dir = args.activate_env or args.dir # Temporary environment if args.temp: env = create_temp_env_directory() env_path = os.path.abspath(env) short_name = os.path.basename(env_path) ev.Environment(env).write(regenerate=False) # Named environment elif ev.exists(env_name_or_dir) and not args.dir: env_path = ev.root(env_name_or_dir) short_name = env_name_or_dir # Environment directory elif ev.is_env_dir(env_name_or_dir): env_path = os.path.abspath(env_name_or_dir) short_name = os.path.basename(env_path) else: tty.die("No such environment: '%s'" % env_name_or_dir) env_prompt = '[%s]' % short_name # We only support one active environment at a time, so deactivate the current one. if ev.active_environment() is None: cmds = '' env_mods = EnvironmentModifications() else: cmds = spack.environment.shell.deactivate_header(shell=args.shell) env_mods = spack.environment.shell.deactivate() # Activate new environment active_env = ev.Environment(env_path) cmds += spack.environment.shell.activate_header( env=active_env, shell=args.shell, prompt=env_prompt if args.prompt else None ) env_mods.extend(spack.environment.shell.activate( env=active_env, add_view=args.with_view )) cmds += env_mods.shell_modifications(args.shell) sys.stdout.write(cmds)