def test_hash_change_no_rehash_concrete(tmpdir, mock_packages, config):
# create an environment
env_path = tmpdir.mkdir('env_dir').strpath
env = ev.Environment(env_path)
env.write()
# add a spec with a rewritten build hash
spec = spack.spec.Spec('mpileaks')
env.add(spec)
env.concretize()
# rewrite the hash
old_hash = env.concretized_order[0]
new_hash = 'abc'
env.specs_by_hash[old_hash]._build_hash = new_hash
env.concretized_order[0] = new_hash
env.specs_by_hash[new_hash] = env.specs_by_hash[old_hash]
del env.specs_by_hash[old_hash]
env.write()
# Read environment
read_in = ev.Environment(env_path)
# Ensure read hashes are used (rewritten hash seen on read)
assert read_in.concretized_order
assert read_in.concretized_order[0] in read_in.specs_by_hash
assert read_in.specs_by_hash[
read_in.concretized_order[0]]._build_hash == new_hash
def env_activate(args):
if not args.activate_env and not args.dir and not args.temp:
tty.die('spack env activate requires an environment name, directory, or --temp')
if not args.shell:
spack.cmd.common.shell_init_instructions(
"spack env activate",
" eval `spack env activate {sh_arg} [...]`",
)
return 1
# Error out when -e, -E, -D flags are given, cause they are ambiguous.
if args.env or args.no_env or args.env_dir:
tty.die('Calling spack env activate with --env, --env-dir and --no-env '
'is ambiguous')
env_name_or_dir = args.activate_env or args.dir
# Temporary environment
if args.temp:
env = create_temp_env_directory()
spack_env = os.path.abspath(env)
short_name = os.path.basename(spack_env)
ev.Environment(env).write(regenerate=False)
# Named environment
elif ev.exists(env_name_or_dir) and not args.dir:
spack_env = ev.root(env_name_or_dir)
short_name = env_name_or_dir
# Environment directory
elif ev.is_env_dir(env_name_or_dir):
spack_env = os.path.abspath(env_name_or_dir)
short_name = os.path.basename(spack_env)
else:
tty.die("No such environment: '%s'" % env_name_or_dir)
env_prompt = '[%s]' % short_name
if spack_env == os.environ.get('SPACK_ENV'):
tty.debug("Environment is already active")
return
# Activate new environment
active_env = ev.Environment(spack_env)
cmds = spack.environment.shell.activate_header(
env=active_env,
shell=args.shell,
prompt=env_prompt if args.prompt else None
)
env_mods = spack.environment.shell.activate(
env=active_env,
add_view=args.with_view
)
cmds += env_mods.shell_modifications(args.shell)
sys.stdout.write(cmds)
def GetValidEnvironment(env):
try:
return ev.read(env)
except:
try:
ev.Environment(env)
except:
raise ev.SpackEnvironmentErrror(
'%s is not a valid environment' % env)
finally:
return ev.Environment(env)
return None
def test_specsCanBeAddedToExisitingYaml(tmpdir):
with tmpdir.as_cwd():
preset_yaml = """
spack:
develop:
amr-wind:
spec: amr-wind@main
path: /tst/dir"""
with open('test.yaml', 'w') as fyaml:
fyaml.write(preset_yaml)
env_root = str(tmpdir.join('dev'))
os.makedirs(env_root)
assert os.path.isfile('test.yaml')
manager('create-env', '-d', env_root, '-m', 'darwin',
'-y', 'test.yaml', '-s', 'amr-wind', 'nalu-wind')
e = env.Environment(env_root)
assert e.yaml['spack']['specs'][0] == 'amr-wind'
assert e.yaml['spack']['specs'][1] == 'nalu-wind'
assert e.yaml['spack']['develop']['amr-wind']['spec'] == 'amr-wind@main'
assert e.yaml['spack']['develop']['amr-wind']['path'] == '/tst/dir'
assert not e.yaml['spack']['view']
def _env_create(name_or_path, init_file=None, dir=False, with_view=None,
keep_relative=False):
"""Create a new environment, with an optional yaml description.
Arguments:
name_or_path (str): name of the environment to create, or path to it
init_file (str or file): optional initialization file -- can be
spack.yaml or spack.lock
dir (bool): if True, create an environment in a directory instead
of a named environment
keep_relative (bool): if True, develop paths are copied verbatim into
the new environment file, otherwise they may be made absolute if the
new environment is in a different location
"""
if dir:
env = ev.Environment(name_or_path, init_file, with_view, keep_relative)
env.write()
tty.msg("Created environment in %s" % env.path)
tty.msg("You can activate this environment with:")
tty.msg(" spack env activate %s" % env.path)
else:
env = ev.create(name_or_path, init_file, with_view, keep_relative)
env.write()
tty.msg("Created environment '%s' in %s" % (name_or_path, env.path))
tty.msg("You can activate this environment with:")
tty.msg(" spack env activate %s" % (name_or_path))
return env
def create_dev_env(parser, args):
env_path = create_env.create_env(parser, args)
env = ev.Environment(env_path)
ev.activate(env)
specs = spack.cmd.parse_specs(args.spec)
for s in specs:
# check that all specs were concrete
if not s.versions.concrete:
print('\nWarning: {spec} is not concrete and will not '
'be setup as a develop spec.'
'\nAll specs must be concrete for '
'\'spack manager create-dev-env\' to clone for you i.e. at '
'least [package]@[version].\nTo learn what versions are'
' available type \'spack info [package]\''
'\nSome common exawind versions are: exawind@master, '
'amr-wind@main and nalu-wind@master\n'.format(spec=s))
continue
dev_args = []
# kind of hacky, but spack will try to re-clone
# if we don't give the --path argument even though
# it is already in the spack.yaml
if env.is_develop(s) and 'path' in env.yaml['spack']['develop'][str(
s.name)]:
dev_args.extend(
['--path', env.yaml['spack']['develop'][str(s.name)]['path']])
if 'trilinos' in str(s.name):
dev_args.extend([
'-rb', '[email protected]:trilinos/trilinos.git',
str(s.version)
])
dev_args.append(str(s))
develop(dev_args)
ev.deactivate()
def test_config_add_to_env_preserve_comments(mutable_empty_config,
mutable_mock_env_path, tmpdir):
filepath = str(tmpdir.join('spack.yaml'))
manifest = """# comment
spack: # comment
# comment
specs: # comment
- foo # comment
# comment
view: true # comment
packages: # comment
# comment
all: # comment
# comment
compiler: [gcc] # comment
"""
with open(filepath, 'w') as f:
f.write(manifest)
env = ev.Environment(str(tmpdir))
with env:
config('add', 'config:dirty:true')
output = config('get')
expected = manifest
expected += """ config:
dirty: true
"""
assert output == expected
def test_missingReferenceYamlFilesDontBreakEnv(monkeypatch):
TESTMACHINE = 'test_machine'
with TemporaryDirectory() as tmpdir:
# setup a mirror configuration of spack-manager
link_dir = os.path.join(os.environ['SPACK_MANAGER'], 'configs', 'base')
os.mkdir(os.path.join(tmpdir, 'configs'))
os.symlink(link_dir,
os.path.join(tmpdir, 'configs', 'base'))
os.mkdir(os.path.join(tmpdir, 'configs', TESTMACHINE))
# monkeypatches
envVars = {'SPACK_MANAGER': tmpdir}
monkeypatch.setattr(os, 'environ', envVars)
def MockFindMachine(verbose=True):
if verbose:
print(TESTMACHINE)
return TESTMACHINE
monkeypatch.setattr(create_env, 'find_machine', MockFindMachine)
# create spack.yaml
manager('create-env', '-d', tmpdir)
# ensure that this environment can be created
# missing includes will cause a failure
env.Environment(tmpdir)
def test_noSpecsIsNotAnErrorGivesBlankEnv(mock_develop, tmpdir):
with tmpdir.as_cwd():
manager('create-dev-env', '-d', tmpdir.strpath)
assert not mock_develop.called
e = ev.Environment(tmpdir.strpath)
assert len(e.user_specs) == 0
assert e.yaml['spack']['specs'] == []
def test_environment_status(capfd, tmpdir):
with capfd.disabled():
with tmpdir.as_cwd():
assert 'No active environment' in env('status')
with ev.create('test'):
assert 'In environment test' in env('status')
with ev.Environment('local_dir'):
assert os.path.join(os.getcwd(), 'local_dir') in env('status')
e = ev.Environment('myproject')
e.write()
with tmpdir.join('myproject').as_cwd():
with e:
assert 'in current directory' in env('status')
def test_nonConcreteSpecsDontGetCloned(mock_dev, tmpdir):
with tmpdir.as_cwd():
manager('create-dev-env', '-s', 'amr-wind', 'nalu-wind',
'exawind@master', '-d', tmpdir.strpath)
mock_dev.assert_called_once_with(['exawind@master'])
e = ev.Environment(tmpdir.strpath)
assert 'nalu-wind' in e.yaml['spack']['specs']
assert 'exawind@master' in e.yaml['spack']['specs']
assert 'amr-wind' in e.yaml['spack']['specs']
def create_test_env(args):
if args.test_nalu:
manager('create-env', '-d', args.directory, '-s',
'nalu-wind-nightly' + args.nalu_wind_vars)
with env.Environment(args.directory):
test_package = 'nalu-wind-nightly@master'
spec = test_package + args.nalu_wind_vars
variant_factory(spec)
print('cloning nalu-wind')
manager('develop', '-rb', 'https://github.com/Exawind/nalu-wind',
'master', test_package)
def check_config_concretizations(name, ref_yaml):
print('Concretization test for', name)
with TemporaryDirectory() as tmpdir:
try:
manager('create-env', '-y', ref_yaml, '-d', tmpdir, '-m', name)
with env.Environment(tmpdir) as this_env:
this_env.concretize()
except Exception as err:
raise Exception(name + ' failed to concretize.'
' Additional exception: ' + str(err))
print(name, 'concretized succesfully')
def test_errorsIfThereIsNoView(tmpdir):
yaml_file = """spack:
view: true
specs: [mpileaks]"""
with tmpdir.as_cwd():
ext_env = setupExternalEnv(tmpdir, False)
path = tmpdir.join('spack.yaml')
with open(str(path), 'w') as f:
f.write(yaml_file)
env('create', '-d', 'test', 'spack.yaml', )
args = ParserMock(ext_env)
with pytest.raises(SystemExit):
with ev.Environment('test'):
manager_cmds.external.external(None, args)
def test_unify_in_yaml_preserved(tmpdir):
with tmpdir.as_cwd():
preset_yaml = """
spack:
specs: [amr-wind, nalu-wind]
concretizer:
unify: when_possible"""
with open('template.yaml', 'w') as f:
f.write(preset_yaml)
manager('create-env', '-y', 'template.yaml')
e = env.Environment(tmpdir.strpath)
assert 'when_possible' == e.yaml['spack']['concretizer']['unify']
def test_modules_relative_to_view(self, tmpdir, modulefile_content,
module_configuration, install_mockery):
with ev.Environment(str(tmpdir), with_view=True) as e:
module_configuration('with_view')
install('cmake')
spec = spack.spec.Spec('cmake').concretized()
content = modulefile_content('cmake')
expected = e.default_view.view().get_projection_for_spec(spec)
# Rather than parse all lines, ensure all prefixes in the content
# point to the right one
assert any(expected in line for line in content)
assert not any(spec.prefix in line for line in content)
def create_snapshots(args):
machine = find_machine(verbose=False)
extension = path_extension(args.name, args.use_machine_name)
env_path = os.path.join(os.environ['SPACK_MANAGER'], 'environments',
extension)
print('\nCreating snapshot environment')
command(manager, 'create-env', '-d', env_path)
e = ev.Environment(env_path)
with e.write_transaction():
e.yaml['spack']['concretizer'] = {'unify': 'when_possible'}
e.write()
spec_data = machine_specs[machine]
add_view(e, extension, args.link_type)
for s in spec_data:
add_spec(e, extension, s, args.modules)
top_specs = get_top_level_specs(e)
if args.stop_after == 'create_env':
return
if args.use_develop:
use_develop_specs(e, top_specs)
else:
use_latest_git_hashes(e, top_specs)
if args.stop_after == 'mod_specs':
return
ev.activate(e)
print('\nConcretize')
command(concretize, '-f')
if args.stop_after == 'concretize':
return
print('\nInstall')
with multiprocessing.Pool(args.num_threads):
spack_install_cmd(args.spack_install_args)
if args.modules:
print('\nGenerate module files')
command(module, 'tcl', 'refresh', '-y')
return env_path
def test_newEnvironmentKeepingUserSpecifiedYAML(mock_dev, tmpdir):
with tmpdir.as_cwd():
amr_path = tmpdir.join('test_amr-wind')
nalu_path = tmpdir.join('test_nalu-wind')
os.makedirs(amr_path.strpath)
os.makedirs(nalu_path.strpath)
assert os.path.isdir(amr_path.strpath)
assert os.path.isdir(nalu_path.strpath)
user_spec_yaml = """spack:
develop:
amr-wind:
spec: amr-wind@main
path: {amr}
nalu-wind:
spec: nalu-wind@master
path: {nalu}""".format(amr=amr_path.strpath, nalu=nalu_path.strpath)
with open(tmpdir.join('user.yaml'), 'w') as yaml:
yaml.write(user_spec_yaml)
print(
manager('create-dev-env',
'-d',
tmpdir.strpath,
'-s',
'amr-wind@main',
'nalu-wind@master',
'trilinos@master',
'-y',
str(tmpdir.join('user.yaml')),
fail_on_error=False))
e = ev.Environment(tmpdir.strpath)
print(e.yaml)
print(list(e.user_specs))
assert os.path.isfile(str(tmpdir.join('spack.yaml')))
assert os.path.isfile(str(tmpdir.join('include.yaml')))
assert 'path' in e.yaml['spack']['develop']['amr-wind']
assert 'path' in e.yaml['spack']['develop']['nalu-wind']
# mocked out call that would update yaml with trilinos info but
# assuming it works fine
mock_dev.assert_any_call(['--path', amr_path.strpath, 'amr-wind@main'])
mock_dev.assert_any_call(
['--path', nalu_path.strpath, 'nalu-wind@master'])
mock_dev.assert_called_with([
'-rb', '[email protected]:trilinos/trilinos.git', 'master',
'trilinos@master'
])
def _env_create(name_or_path, init_file=None, dir=False, with_view=None):
"""Create a new environment, with an optional yaml description.
Arguments:
name_or_path (str): name of the environment to create, or path to it
init_file (str or file): optional initialization file -- can be
spack.yaml or spack.lock
dir (bool): if True, create an environment in a directory instead
of a named environment
"""
if dir:
env = ev.Environment(name_or_path, init_file, with_view)
env.write()
tty.msg("Created environment in %s" % env.path)
else:
env = ev.create(name_or_path, init_file, with_view)
env.write()
tty.msg("Created environment '%s' in %s" % (name_or_path, env.path))
return env
def find_environment(args):
"""Find active environment from args or environment variable.
Check for an environment in this order:
1. via ``spack -e ENV`` or ``spack -D DIR`` (arguments)
2. via a path in the spack.environment.spack_env_var environment variable.
If an environment is found, read it in. If not, return None.
Arguments:
args (argparse.Namespace): argparse namespace with command arguments
Returns:
(spack.environment.Environment): a found environment, or ``None``
"""
# treat env as a name
env = args.env
if env:
if ev.exists(env):
return ev.read(env)
else:
# if env was specified, see if it is a directory otherwise, look
# at env_dir (env and env_dir are mutually exclusive)
env = args.env_dir
# if no argument, look for the environment variable
if not env:
env = os.environ.get(ev.spack_env_var)
# nothing was set; there's no active environment
if not env:
return None
# if we get here, env isn't the name of a spack environment; it has
# to be a path to an environment, or there is something wrong.
if ev.is_env_dir(env):
return ev.Environment(env)
raise ev.SpackEnvironmentError('no environment in %s' % env)
def test_init_from_yaml(tmpdir):
"""Test that an environment can be instantiated from a lockfile."""
initial_yaml = StringIO("""\
env:
specs:
- mpileaks
- hypre
- libelf
""")
e1 = ev.create('test', initial_yaml)
e1.concretize()
e1.write()
e2 = ev.Environment(str(tmpdir), e1.manifest_path)
for s1, s2 in zip(e1.user_specs, e2.user_specs):
assert s1 == s2
assert not e2.concretized_order
assert not e2.concretized_user_specs
assert not e2.specs_by_hash
def evaluate_external(tmpdir, yaml_file):
ext_path = setupExternalEnv(tmpdir)
manifest = tmpdir.join('spack.yaml')
with open(str(manifest), 'w') as f:
f.write(yaml_file)
env('create', '-d', 'test', 'spack.yaml', )
assert os.path.isfile('test/spack.yaml')
with ev.Environment('test') as e:
args = ParserMock(ext_path, merge=True)
with patch("manager_cmds.external.write_spec",
return_value=str(
ExtPackage(
'cmake', '[email protected]', '/path/top/some/view')
)) as mock_write:
manager_cmds.external.external(None, args)
# check that the include entry was added to the spack.yaml
assert mock_write.called_once()
includes = config_dict(e.yaml).get('include', [])
assert 1 == len(includes)
assert 'externals.yaml' == str(includes[0])
def test_existingYamlViewIsNotOverwritten(tmpdir):
with tmpdir.as_cwd():
preset_yaml = """
spack:
view: true
develop:
amr-wind:
spec: amr-wind@main
path: /tst/dir"""
with open('test.yaml', 'w') as fyaml:
fyaml.write(preset_yaml)
env_root = str(tmpdir.join('dev'))
os.makedirs(env_root)
assert os.path.isfile('test.yaml')
manager('create-env', '-d', env_root, '-m', 'darwin',
'-y', 'test.yaml', '-s', 'amr-wind', 'nalu-wind')
e = env.Environment(env_root)
assert e.yaml['spack']['view']
def env_activate(args):
env = args.activate_env
if not args.shell:
spack.cmd.common.shell_init_instructions(
"spack env activate",
" eval `spack env activate {sh_arg} %s`" % env,
)
return 1
# Error out when -e, -E, -D flags are given, cause they are ambiguous.
if args.env or args.no_env or args.env_dir:
tty.die(
'Calling spack env activate with --env, --env-dir and --no-env '
'is ambiguous')
if ev.exists(env) and not args.dir:
spack_env = ev.root(env)
short_name = env
env_prompt = '[%s]' % env
elif ev.is_env_dir(env):
spack_env = os.path.abspath(env)
short_name = os.path.basename(os.path.abspath(env))
env_prompt = '[%s]' % short_name
else:
tty.die("No such environment: '%s'" % env)
if spack_env == os.environ.get('SPACK_ENV'):
tty.debug("Environment %s is already active" % args.activate_env)
return
cmds = ev.activate(ev.Environment(spack_env),
add_view=args.with_view,
shell=args.shell,
prompt=env_prompt if args.prompt else None)
sys.stdout.write(cmds)
def test_read_old_lock_creates_backup(tmpdir):
"""When reading a version-1 lockfile, make sure that a backup of that file
is created.
"""
y = MockPackage('y', [], [])
mock_repo = MockPackageMultiRepo([y])
with spack.repo.swap(mock_repo):
y = Spec('y')
y.concretize()
test_lockfile_dict = create_v1_lockfile_dict([y], [y])
env_root = tmpdir.mkdir('test-root')
test_lockfile_path = str(env_root.join(ev.lockfile_name))
with open(test_lockfile_path, 'w') as f:
sjson.dump(test_lockfile_dict, stream=f)
e = ev.Environment(str(env_root))
assert os.path.exists(e._lock_backup_v1_path)
with open(e._lock_backup_v1_path, 'r') as backup_v1_file:
lockfile_dict_v1 = sjson.load(backup_v1_file)
# Make sure that the backup file follows the v1 hash scheme
assert y.dag_hash() in lockfile_dict_v1['concrete_specs']
def external(parser, args):
extern_dir = get_external_dir()
if args.list:
snaps = get_all_snapshots()
dated = get_ordered_dated_snapshots()
if snaps and dated:
non_dated = list(set(snaps) - set(dated))
def print_snapshots(snaps):
for s in snaps:
env_dir = os.path.join(extern_dir, s)
print(' - {path}'.format(path=env_dir))
print('-' * 54)
print('Available snapshot directories are:')
print('-' * 54)
if dated:
print('\nDated Snapshots (ordered)')
print('-' * 54)
print_snapshots(dated)
if non_dated:
print('\nAdditional Snapshots (unordered)')
print('-' * 54)
print_snapshots(non_dated)
return
env = ev.active_environment()
if not env:
tty.die('spack manager external requires an active environment')
if args.latest:
snaps = get_ordered_dated_snapshots()
if not snaps:
print('WARNING: No \'externals.yaml\' created because no valid '
'snapshots were found. \n'
' If you are trying to use a system level snapshot make '
'sure you have SPACK_MANAGER_EXTERNAL pointing to '
'spack-manager directory for the system.\n')
return
else:
snap_path = os.path.join(extern_dir, snaps[0])
else:
snap_path = args.path
# check that directory of ext view exists
if not snap_path or not ev.is_env_dir(snap_path):
tty.die('External path must point to a spack environment with a view. '
'Auto detection of the latest dated snapshot can be achived'
' with the \'--latest\' flag.')
snap_env = ev.Environment(snap_path)
snap_env.check_views()
if not snap_env.views:
tty.die('Environments used to create externals must have at least 1'
' associated view')
# copy the file and overwrite any that may exist (or merge?)
inc_name_abs = os.path.abspath(os.path.join(env.path, args.name))
try:
detected = assemble_dict_of_detected_externals(snap_env,
args.blacklist,
args.whitelist)
src = create_yaml_from_detected_externals(detected)
except ev.SpackEnvironmentError as e:
tty.die(e.long_message)
if include_entry_exists(env, args.name):
if args.merge:
# merge the existing includes with the new one
# giving precedent to the new data coming in
dest = spack.config.read_config_file(
inc_name_abs, spack.config.section_schemas['packages'])
combined = spack.config.merge_yaml(src, dest)
final = combined
else:
final = src
else:
add_include_entry(env, args.name)
final = src
with open(inc_name_abs, 'w') as fout:
syaml.dump_config(final, stream=fout, default_flow_style=False)
env.write()
parser.add_argument('--dev_name',
required=False,
help='directory name for the developer env')
parser.add_argument('--num_threads', required=False, help='DAG paralleization')
parser.set_defaults(view_name='default',
snap_name='test',
dev_name='test_external',
num_threads=1)
args = parser.parse_args()
# set up the snapshot
print('Create Snapshot')
args = snapshot_creator.parse([
'--use_develop', '--modules', '--name', args.snap_name, '--num_threads',
args.num_threads
])
snapshot_path = snapshot_creator.create_snapshots(args)
print('Snapshot created at', snapshot_path)
# set up the user environment
ev.deactivate()
env_path = os.path.join(os.environ['SPACK_MANAGER'], 'environments',
args.dev_name)
command(manager, 'create-env', '--directory', env_path, '--spec', 'nalu-wind')
ev.activate(ev.Environment(env_path))
command(manager, 'external', snapshot_path, '-v', args.view_name,
'--blacklist', 'nalu-wind')
command(manager, 'develop', 'nalu-wind@master')
command(concretize)
command(install)