def create_dev_env(parser, args): env_path = create_env.create_env(parser, args) env = ev.Environment(env_path) ev.activate(env) specs = spack.cmd.parse_specs(args.spec) for s in specs: # check that all specs were concrete if not s.versions.concrete: print('\nWarning: {spec} is not concrete and will not ' 'be setup as a develop spec.' '\nAll specs must be concrete for ' '\'spack manager create-dev-env\' to clone for you i.e. at ' 'least [package]@[version].\nTo learn what versions are' ' available type \'spack info [package]\'' '\nSome common exawind versions are: exawind@master, ' 'amr-wind@main and nalu-wind@master\n'.format(spec=s)) continue dev_args = [] # kind of hacky, but spack will try to re-clone # if we don't give the --path argument even though # it is already in the spack.yaml if env.is_develop(s) and 'path' in env.yaml['spack']['develop'][str( s.name)]: dev_args.extend( ['--path', env.yaml['spack']['develop'][str(s.name)]['path']]) if 'trilinos' in str(s.name): dev_args.extend([ '-rb', '[email protected]:trilinos/trilinos.git', str(s.version) ]) dev_args.append(str(s)) develop(dev_args) ev.deactivate()
def use_develop_specs(env, specs): # we have to concretize to solve the dependency tree to extract # the top level dependencies and make them develop specs. # anything that is not a develop spec is not gauranteed to get installed # since spack can reuse them for matching hashes print('\nSetting up develop specs') dev_specs = list( dict.fromkeys([s.format('{name}{@version}') for s in specs])) ev.activate(env) for spec_string in dev_specs: # special treatment for trilinos since its clone fails # with standard spack develop if 'trilinos' in spec_string: branch = spec_string.split('@')[-1] command(manager, 'develop', '--shallow', '-rb', 'https://github.com/trilinos/trilinos', branch, spec_string) elif 'openfast' in spec_string: # skip openfast. we never want to dev build it # because it takes so long to compile continue elif 'cmake' in spec_string: continue else: command(manager, 'develop', '--shallow', spec_string) ev.deactivate()
def clean_user_environment(): env_var = ev.spack_env_var in os.environ active = ev._active_environment if env_var: spack_env_value = os.environ.pop(ev.spack_env_var) if active: ev.deactivate() yield if env_var: os.environ[ev.spack_env_var] = spack_env_value if active: ev.activate(active)
def env_activate(args): env = args.activate_env if not args.shell: spack.cmd.common.shell_init_instructions( "spack env activate", " eval `spack env activate {sh_arg} %s`" % env, ) return 1 if ev.exists(env) and not args.dir: spack_env = ev.root(env) short_name = env env_prompt = '[%s]' % env elif ev.is_env_dir(env): spack_env = os.path.abspath(env) short_name = os.path.basename(os.path.abspath(env)) env_prompt = '[%s]' % short_name else: tty.die("No such environment: '%s'" % env) if spack_env == os.environ.get('SPACK_ENV'): tty.die("Environment %s is already active" % args.activate_env) active_env = ev.get_env(namedtuple('args', ['env'])(env), 'activate') cmds = ev.activate(active_env, add_view=args.with_view, shell=args.shell, prompt=env_prompt if args.prompt else None) sys.stdout.write(cmds)
def create_snapshots(args): machine = find_machine(verbose=False) extension = path_extension(args.name, args.use_machine_name) env_path = os.path.join(os.environ['SPACK_MANAGER'], 'environments', extension) print('\nCreating snapshot environment') command(manager, 'create-env', '-d', env_path) e = ev.Environment(env_path) with e.write_transaction(): e.yaml['spack']['concretizer'] = {'unify': 'when_possible'} e.write() spec_data = machine_specs[machine] add_view(e, extension, args.link_type) for s in spec_data: add_spec(e, extension, s, args.modules) top_specs = get_top_level_specs(e) if args.stop_after == 'create_env': return if args.use_develop: use_develop_specs(e, top_specs) else: use_latest_git_hashes(e, top_specs) if args.stop_after == 'mod_specs': return ev.activate(e) print('\nConcretize') command(concretize, '-f') if args.stop_after == 'concretize': return print('\nInstall') with multiprocessing.Pool(args.num_threads): spack_install_cmd(args.spack_install_args) if args.modules: print('\nGenerate module files') command(module, 'tcl', 'refresh', '-y') return env_path
def get_top_level_specs(env, blacklist=blacklist): ev.activate(env) print('\nInitial concretize') command(concretize) top_specs = [] for root in env.roots(): if root.name in blacklist: continue top_specs.append(root) for dep in root.dependencies(): if dep.name not in blacklist: top_specs.append(dep) # remove any duplicates top_specs = list(dict.fromkeys(top_specs)) print('\nTop Level Specs:', [s.name for s in top_specs]) ev.deactivate() return top_specs
def activate(env, use_env_repo=False, add_view=True): """ Activate an environment and append environment modifications To activate an environment, we add its configuration scope to the existing Spack configuration, and we set active to the current environment. Arguments: env (spack.environment.Environment): the environment to activate use_env_repo (bool): use the packages exactly as they appear in the environment's repository add_view (bool): generate commands to add view to path variables Returns: spack.util.environment.EnvironmentModifications: Environment variables modifications to activate environment. """ ev.activate(env, use_env_repo=use_env_repo) env_mods = EnvironmentModifications() # # NOTE in the fish-shell: Path variables are a special kind of variable # used to support colon-delimited path lists including PATH, CDPATH, # MANPATH, PYTHONPATH, etc. All variables that end in PATH (case-sensitive) # become PATH variables. # try: if add_view and ev.default_view_name in env.views: with spack.store.db.read_transaction(): env.add_default_view_to_env(env_mods) except (spack.repo.UnknownPackageError, spack.repo.UnknownNamespaceError) as e: tty.error(e) tty.die( 'Environment view is broken due to a missing package or repo.\n', ' To activate without views enabled, activate with:\n', ' spack env activate -V {0}\n'.format(env.name), ' To remove it and resolve the issue, ' 'force concretize with the command:\n', ' spack -e {0} concretize --force'.format(env.name)) return env_mods
def add_spec(env, extension, data, create_modules): ev.activate(env) add(data.spec) ev.deactivate() excludes = view_excludes(data) with open(env.manifest_path, 'r') as f: yaml = syaml.load(f) if create_modules: module_excludes = excludes.copy() module_path = os.path.join(os.environ['SPACK_MANAGER'], 'modules') module_dict = { data.id: { 'enable': ['tcl'], 'use_view': data.id, 'prefix_inspections': { 'bin': ['PATH'] }, 'roots': { 'tcl': module_path }, 'arch_folder': False, 'tcl': { 'projections': { 'all': '%s/{name}-%s' % (extension, data.id) }, 'hash_length': 0, 'blacklist_implicits': True, 'blacklist': module_excludes } } } try: yaml['spack']['modules'].update(module_dict) except KeyError: yaml['spack']['modules'] = module_dict with open(env.manifest_path, 'w') as f: syaml.dump(yaml, stream=f, default_flow_style=False)
def env_activate(args): env = args.activate_env if not args.shell: msg = [ "This command works best with Spack's shell support", "" ] + spack.cmd.common.shell_init_instructions + [ 'Or, if you want to use `spack env activate` without initializing', 'shell support, you can run one of these:', '', ' eval `spack env activate --sh %s` # for bash/sh' % env, ' eval `spack env activate --csh %s` # for csh/tcsh' % env, ] tty.msg(*msg) return 1 if ev.exists(env) and not args.dir: spack_env = ev.root(env) short_name = env env_prompt = '[%s]' % env elif ev.is_env_dir(env): spack_env = os.path.abspath(env) short_name = os.path.basename(os.path.abspath(env)) env_prompt = '[%s]' % short_name else: tty.die("No such environment: '%s'" % env) if spack_env == os.environ.get('SPACK_ENV'): tty.die("Environment %s is already active" % args.activate_env) active_env = ev.get_env(namedtuple('args', ['env'])(env), 'activate') cmds = ev.activate( active_env, add_view=args.with_view, shell=args.shell, prompt=env_prompt if args.prompt else None ) sys.stdout.write(cmds)
def env_activate(args): env = args.activate_env if not args.shell: spack.cmd.common.shell_init_instructions( "spack env activate", " eval `spack env activate {sh_arg} %s`" % env, ) return 1 # Error out when -e, -E, -D flags are given, cause they are ambiguous. if args.env or args.no_env or args.env_dir: tty.die( 'Calling spack env activate with --env, --env-dir and --no-env ' 'is ambiguous') if ev.exists(env) and not args.dir: spack_env = ev.root(env) short_name = env env_prompt = '[%s]' % env elif ev.is_env_dir(env): spack_env = os.path.abspath(env) short_name = os.path.basename(os.path.abspath(env)) env_prompt = '[%s]' % short_name else: tty.die("No such environment: '%s'" % env) if spack_env == os.environ.get('SPACK_ENV'): tty.debug("Environment %s is already active" % args.activate_env) return cmds = ev.activate(ev.Environment(spack_env), add_view=args.with_view, shell=args.shell, prompt=env_prompt if args.prompt else None) sys.stdout.write(cmds)
def main(argv=None): """This is the entry point for the Spack command. Args: argv (list of str or None): command line arguments, NOT including the executable name. If None, parses from sys.argv. """ # Create a parser with a simple positional argument first. We'll # lazily load the subcommand(s) we need later. This allows us to # avoid loading all the modules from spack.cmd when we don't need # them, which reduces startup latency. parser = make_argument_parser() parser.add_argument('command', nargs=argparse.REMAINDER) args, unknown = parser.parse_known_args(argv) # Recover stored LD_LIBRARY_PATH variables from spack shell function # This is necessary because MacOS System Integrity Protection clears # (DY?)LD_LIBRARY_PATH variables on process start. # Spack clears these variables before building and installing packages, # but needs to know the prior state for commands like `spack load` and # `spack env activate that modify the user environment. recovered_vars = ('LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'DYLD_FALLBACK_LIBRARY_PATH') for var in recovered_vars: stored_var_name = 'SPACK_%s' % var if stored_var_name in os.environ: os.environ[var] = os.environ[stored_var_name] # make spack.config aware of any command line configuration scopes if args.config_scopes: spack.config.command_line_scopes = args.config_scopes # activate an environment if one was specified on the command line if not args.no_env: env = ev.find_environment(args) if env: ev.activate(env, args.use_env_repo, add_view=False) if args.print_shell_vars: print_setup_info(*args.print_shell_vars.split(',')) return 0 # Just print help and exit if run with no arguments at all no_args = (len(sys.argv) == 1) if argv is None else (len(argv) == 0) if no_args: parser.print_help() return 1 # -h, -H, and -V are special as they do not require a command, but # all the other options do nothing without a command. if args.version: print(get_version()) return 0 elif args.help: sys.stdout.write(parser.format_help(level=args.help)) return 0 elif not args.command: parser.print_help() return 1 try: # ensure options on spack command come before everything setup_main_options(args) # Try to load the particular command the caller asked for. cmd_name = args.command[0] cmd_name = aliases.get(cmd_name, cmd_name) command = parser.add_command(cmd_name) # Re-parse with the proper sub-parser added. args, unknown = parser.parse_known_args() # many operations will fail without a working directory. set_working_dir() # now we can actually execute the command. if args.spack_profile or args.sorted_profile: _profile_wrapper(command, parser, args, unknown) elif args.pdb: import pdb pdb.runctx('_invoke_command(command, parser, args, unknown)', globals(), locals()) return 0 else: return _invoke_command(command, parser, args, unknown) except SpackError as e: tty.debug(e) e.die() # gracefully die on any SpackErrors except Exception as e: if spack.config.get('config:debug'): raise tty.die(e) except KeyboardInterrupt: if spack.config.get('config:debug'): raise sys.stderr.write('\n') tty.die("Keyboard interrupt.") except SystemExit as e: if spack.config.get('config:debug'): traceback.print_exc() return e.code
def test_activate_should_require_an_env(): with pytest.raises(TypeError): ev.activate(env='name') with pytest.raises(TypeError): ev.activate(env=None)
def _main(argv=None): """Logic for the main entry point for the Spack command. ``main()`` calls ``_main()`` and catches any errors that emerge. ``_main()`` handles: 1. Parsing arguments; 2. Setting up configuration; and 3. Finding and executing a Spack command. Args: argv (list or None): command line arguments, NOT including the executable name. If None, parses from ``sys.argv``. """ # ------------------------------------------------------------------------ # main() is tricky to get right, so be careful where you put things. # # Things in this first part of `main()` should *not* require any # configuration. This doesn't include much -- setting up th parser, # restoring some key environment variables, very simple CLI options, etc. # ------------------------------------------------------------------------ # Create a parser with a simple positional argument first. We'll # lazily load the subcommand(s) we need later. This allows us to # avoid loading all the modules from spack.cmd when we don't need # them, which reduces startup latency. parser = make_argument_parser() parser.add_argument('command', nargs=argparse.REMAINDER) args, unknown = parser.parse_known_args(argv) # Recover stored LD_LIBRARY_PATH variables from spack shell function # This is necessary because MacOS System Integrity Protection clears # (DY?)LD_LIBRARY_PATH variables on process start. # Spack clears these variables before building and installing packages, # but needs to know the prior state for commands like `spack load` and # `spack env activate that modify the user environment. recovered_vars = ('LD_LIBRARY_PATH', 'DYLD_LIBRARY_PATH', 'DYLD_FALLBACK_LIBRARY_PATH') for var in recovered_vars: stored_var_name = 'SPACK_%s' % var if stored_var_name in os.environ: os.environ[var] = os.environ[stored_var_name] # Just print help and exit if run with no arguments at all no_args = (len(sys.argv) == 1) if argv is None else (len(argv) == 0) if no_args: parser.print_help() return 1 # -h, -H, and -V are special as they do not require a command, but # all the other options do nothing without a command. if args.version: print(get_version()) return 0 elif args.help: sys.stdout.write(parser.format_help(level=args.help)) return 0 # ------------------------------------------------------------------------ # This part of the `main()` sets up Spack's configuration. # # We set command line options (like --debug), then command line config # scopes, then environment configuration here. # ------------------------------------------------------------------------ # make spack.config aware of any command line configuration scopes if args.config_scopes: spack.config.command_line_scopes = args.config_scopes # ensure options on spack command come before everything setup_main_options(args) # activate an environment if one was specified on the command line env_format_error = None if not args.no_env: try: env = spack.cmd.find_environment(args) if env: ev.activate(env, args.use_env_repo) except spack.config.ConfigFormatError as e: # print the context but delay this exception so that commands like # `spack config edit` can still work with a bad environment. e.print_context() env_format_error = e # ------------------------------------------------------------------------ # Things that require configuration should go below here # ------------------------------------------------------------------------ if args.print_shell_vars: print_setup_info(*args.print_shell_vars.split(',')) return 0 # At this point we've considered all the options to spack itself, so we # need a command or we're done. if not args.command: parser.print_help() return 1 # Try to load the particular command the caller asked for. cmd_name = args.command[0] cmd_name = aliases.get(cmd_name, cmd_name) command = parser.add_command(cmd_name) # Re-parse with the proper sub-parser added. args, unknown = parser.parse_known_args() # Now that we know what command this is and what its args are, determine # whether we can continue with a bad environment and raise if not. if env_format_error: subcommand = getattr(args, "config_command", None) if (cmd_name, subcommand) != ("config", "edit"): raise env_format_error # many operations will fail without a working directory. set_working_dir() # now we can actually execute the command. if args.spack_profile or args.sorted_profile: _profile_wrapper(command, parser, args, unknown) elif args.pdb: import pdb pdb.runctx('_invoke_command(command, parser, args, unknown)', globals(), locals()) return 0 else: return _invoke_command(command, parser, args, unknown)
def main(argv=None): """This is the entry point for the Spack command. Args: argv (list of str or None): command line arguments, NOT including the executable name. If None, parses from sys.argv. """ # Create a parser with a simple positional argument first. We'll # lazily load the subcommand(s) we need later. This allows us to # avoid loading all the modules from spack.cmd when we don't need # them, which reduces startup latency. parser = make_argument_parser() parser.add_argument('command', nargs=argparse.REMAINDER) args, unknown = parser.parse_known_args(argv) # activate an environment if one was specified on the command line if not args.no_env: env = ev.find_environment(args) if env: ev.activate(env, args.use_env_repo) # make spack.config aware of any command line configuration scopes if args.config_scopes: spack.config.command_line_scopes = args.config_scopes if args.print_shell_vars: print_setup_info(*args.print_shell_vars.split(',')) return 0 # Just print help and exit if run with no arguments at all no_args = (len(sys.argv) == 1) if argv is None else (len(argv) == 0) if no_args: parser.print_help() return 1 # -h, -H, and -V are special as they do not require a command, but # all the other options do nothing without a command. if args.version: print(spack.spack_version) return 0 elif args.help: sys.stdout.write(parser.format_help(level=args.help)) return 0 elif not args.command: parser.print_help() return 1 try: # ensure options on spack command come before everything setup_main_options(args) # Try to load the particular command the caller asked for. If there # is no module for it, just die. cmd_name = args.command[0] cmd_name = aliases.get(cmd_name, cmd_name) try: command = parser.add_command(cmd_name) except ImportError: if spack.config.get('config:debug'): raise tty.die("Unknown command: %s" % args.command[0]) # Re-parse with the proper sub-parser added. args, unknown = parser.parse_known_args() # many operations will fail without a working directory. set_working_dir() # pre-run hooks happen after we know we have a valid working dir spack.hooks.pre_run() # now we can actually execute the command. if args.spack_profile or args.sorted_profile: _profile_wrapper(command, parser, args, unknown) elif args.pdb: import pdb pdb.runctx('_invoke_command(command, parser, args, unknown)', globals(), locals()) return 0 else: return _invoke_command(command, parser, args, unknown) except SpackError as e: e.die() # gracefully die on any SpackErrors except Exception as e: if spack.config.get('config:debug'): raise tty.die(str(e)) except KeyboardInterrupt: sys.stderr.write('\n') tty.die("Keyboard interrupt.") except SystemExit as e: return e.code
def main(argv=None): """This is the entry point for the Spack command. Args: argv (list of str or None): command line arguments, NOT including the executable name. If None, parses from sys.argv. """ # Create a parser with a simple positional argument first. We'll # lazily load the subcommand(s) we need later. This allows us to # avoid loading all the modules from spack.cmd when we don't need # them, which reduces startup latency. parser = make_argument_parser() parser.add_argument('command', nargs=argparse.REMAINDER) args, unknown = parser.parse_known_args(argv) # activate an environment if one was specified on the command line if not args.no_env: env = ev.find_environment(args) if env: ev.activate(env, args.use_env_repo) # make spack.config aware of any command line configuration scopes if args.config_scopes: spack.config.command_line_scopes = args.config_scopes if args.print_shell_vars: print_setup_info(*args.print_shell_vars.split(',')) return 0 # Just print help and exit if run with no arguments at all no_args = (len(sys.argv) == 1) if argv is None else (len(argv) == 0) if no_args: parser.print_help() return 1 # -h, -H, and -V are special as they do not require a command, but # all the other options do nothing without a command. if args.version: print(spack.spack_version) return 0 elif args.help: sys.stdout.write(parser.format_help(level=args.help)) return 0 elif not args.command: parser.print_help() return 1 try: # ensure options on spack command come before everything setup_main_options(args) # Try to load the particular command the caller asked for. If there # is no module for it, just die. cmd_name = args.command[0] cmd_name = aliases.get(cmd_name, cmd_name) command = parser.add_command(cmd_name) # Re-parse with the proper sub-parser added. args, unknown = parser.parse_known_args() # many operations will fail without a working directory. set_working_dir() # pre-run hooks happen after we know we have a valid working dir spack.hooks.pre_run() # now we can actually execute the command. if args.spack_profile or args.sorted_profile: _profile_wrapper(command, parser, args, unknown) elif args.pdb: import pdb pdb.runctx('_invoke_command(command, parser, args, unknown)', globals(), locals()) return 0 else: return _invoke_command(command, parser, args, unknown) except SpackError as e: tty.debug(e) e.die() # gracefully die on any SpackErrors except Exception as e: if spack.config.get('config:debug'): raise tty.die(e) except KeyboardInterrupt: if spack.config.get('config:debug'): raise sys.stderr.write('\n') tty.die("Keyboard interrupt.") except SystemExit as e: if spack.config.get('config:debug'): traceback.print_exc() return e.code
def main(argv=None): """This is the entry point for the Spack command. Args: argv (list of str or None): command line arguments, NOT including the executable name. If None, parses from sys.argv. """ # Create a parser with a simple positional argument first. We'll # lazily load the subcommand(s) we need later. This allows us to # avoid loading all the modules from spack.cmd when we don't need # them, which reduces startup latency. parser = make_argument_parser() parser.add_argument('command', nargs=argparse.REMAINDER) args, unknown = parser.parse_known_args(argv) # activate an environment if one was specified on the command line if not args.no_env: env = ev.find_environment(args) if env: ev.activate(env, args.use_env_repo) # make spack.config aware of any command line configuration scopes if args.config_scopes: spack.config.command_line_scopes = args.config_scopes if args.print_shell_vars: print_setup_info(*args.print_shell_vars.split(',')) return 0 # from spack.spec import Spec # #import llnl.util.tty as tty # import time # import llnl.util.lock as lk # # tty.set_verbose(args.verbose) # tty.set_debug(args.debug) # # def use_llnl_lock(path='/tmp/lockfile', start=10000, len=100): # mylk = lk.Lock(path, start, len) # mylk.acquire_write() # tty.warn('ENTERED LOCK') # time.sleep(1) # tty.warn('EXITING LOCK') # mylk.release_write() # # # def use_db_lock(): # s = Spec('m4') # s.concretize() # #tty.warn('Locking phase...\n') # time.sleep(0.1) # with spack.store.db.prefix_write_lock(s): # tty.warn('ENTERED LOCK') # time.sleep(1) # tty.warn('EXITING LOCK') # # #use_db_lock() # use_llnl_lock( # '/home/sknigh/code/github/spack/opt/spack/.spack-db/prefix_lock', # 4651886554793840719, 1) # exit() # Just print help and exit if run with no arguments at all no_args = (len(sys.argv) == 1) if argv is None else (len(argv) == 0) if no_args: parser.print_help() return 1 # -h, -H, and -V are special as they do not require a command, but # all the other options do nothing without a command. if args.version: print(spack.spack_version) return 0 elif args.help: sys.stdout.write(parser.format_help(level=args.help)) return 0 elif not args.command: parser.print_help() return 1 try: # ensure options on spack command come before everything setup_main_options(args) # Try to load the particular command the caller asked for. If there # is no module for it, just die. cmd_name = args.command[0] cmd_name = aliases.get(cmd_name, cmd_name) try: command = parser.add_command(cmd_name) except ImportError: if spack.config.get('config:debug'): raise tty.die("Unknown command: %s" % args.command[0]) # Re-parse with the proper sub-parser added. args, unknown = parser.parse_known_args() # many operations will fail without a working directory. set_working_dir() # pre-run hooks happen after we know we have a valid working dir spack.hooks.pre_run() # now we can actually execute the command. if args.spack_profile or args.sorted_profile: _profile_wrapper(command, parser, args, unknown) elif args.pdb: import pdb pdb.runctx('_invoke_command(command, parser, args, unknown)', globals(), locals()) return 0 else: return _invoke_command(command, parser, args, unknown) except SpackError as e: e.die() # gracefully die on any SpackErrors except Exception as e: if spack.config.get('config:debug'): raise tty.die(str(e)) except KeyboardInterrupt: sys.stderr.write('\n') tty.die("Keyboard interrupt.") except SystemExit as e: return e.code
parser.add_argument('--dev_name', required=False, help='directory name for the developer env') parser.add_argument('--num_threads', required=False, help='DAG paralleization') parser.set_defaults(view_name='default', snap_name='test', dev_name='test_external', num_threads=1) args = parser.parse_args() # set up the snapshot print('Create Snapshot') args = snapshot_creator.parse([ '--use_develop', '--modules', '--name', args.snap_name, '--num_threads', args.num_threads ]) snapshot_path = snapshot_creator.create_snapshots(args) print('Snapshot created at', snapshot_path) # set up the user environment ev.deactivate() env_path = os.path.join(os.environ['SPACK_MANAGER'], 'environments', args.dev_name) command(manager, 'create-env', '--directory', env_path, '--spec', 'nalu-wind') ev.activate(ev.Environment(env_path)) command(manager, 'external', snapshot_path, '-v', args.view_name, '--blacklist', 'nalu-wind') command(manager, 'develop', 'nalu-wind@master') command(concretize) command(install)