def test_user_removed_spec():
"""Ensure a user can remove from any position in the spack.yaml file."""
initial_yaml = StringIO("""\
env:
specs:
- mpileaks
- hypre
- libelf
""")
before = ev.create('test', initial_yaml)
before.concretize()
before.write()
# user modifies yaml externally to spack and removes hypre
with open(before.manifest_path, 'w') as f:
f.write("""\
env:
specs:
- mpileaks
- libelf
""")
after = ev.read('test')
after.concretize()
after.write()
env_specs = after._get_environment_specs()
read = ev.read('test')
env_specs = read._get_environment_specs()
assert not any(x.name == 'hypre' for x in env_specs)
def test_env_install_single_spec(install_mockery, mock_fetch):
env('create', 'test')
install = SpackCommand('install')
e = ev.read('test')
with e:
install('cmake-client')
e = ev.read('test')
assert e.user_specs[0].name == 'cmake-client'
assert e.concretized_user_specs[0].name == 'cmake-client'
assert e.specs_by_hash[e.concretized_order[0]].name == 'cmake-client'
def test_env_with_included_config_scope():
config_scope_path = os.path.join(ev.root('test'), 'config')
test_config = """\
env:
include:
- %s
specs:
- mpileaks
""" % config_scope_path
spack.package_prefs.PackagePrefs.clear_caches()
_env_create('test', StringIO(test_config))
e = ev.read('test')
fs.mkdirp(config_scope_path)
with open(os.path.join(config_scope_path, 'packages.yaml'), 'w') as f:
f.write("""\
packages:
mpileaks:
version: [2.2]
""")
ev.prepare_config_scope(e)
e.concretize()
assert any(x.satisfies('[email protected]')
for x in e._get_environment_specs())
def env_remove(args):
"""Remove a *named* environment.
This removes an environment managed by Spack. Directory environments
and `spack.yaml` files embedded in repositories should be removed
manually.
"""
read_envs = []
for env_name in args.rm_env:
env = ev.read(env_name)
read_envs.append(env)
if not args.yes_to_all:
answer = tty.get_yes_or_no(
'Really remove %s %s?' % (
string.plural(len(args.rm_env), 'environment', show_n=False),
string.comma_and(args.rm_env)),
default=False)
if not answer:
tty.die("Will not remove any environments")
for env in read_envs:
if env.active:
tty.die("Environment %s can't be removed while activated.")
env.destroy()
tty.msg("Successfully removed environment '%s'" % env.name)
def test_env_blocks_uninstall(mock_stage, mock_fetch, install_mockery):
env('create', 'test')
with ev.read('test'):
add('mpileaks')
install('--fake')
out = uninstall('mpileaks', fail_on_error=False)
assert uninstall.returncode == 1
assert 'used by the following environments' in out
def test_uninstall_removes_from_env(mock_stage, mock_fetch, install_mockery):
env('create', 'test')
with ev.read('test'):
add('mpileaks')
add('libelf')
install('--fake')
test = ev.read('test')
assert any(s.name == 'mpileaks' for s in test.specs_by_hash.values())
assert any(s.name == 'libelf' for s in test.specs_by_hash.values())
with ev.read('test'):
uninstall('-ya')
test = ev.read('test')
assert not test.specs_by_hash
assert not test.concretized_order
assert not test.user_specs
def test_env_install_same_spec_twice(install_mockery, mock_fetch, capfd):
env('create', 'test')
e = ev.read('test')
with capfd.disabled():
with e:
install('cmake-client')
out = install('cmake-client')
assert 'is already installed in' in out
def test_env_loads(install_mockery, mock_fetch):
env('create', 'test')
with ev.read('test'):
add('mpileaks')
concretize()
install('--fake')
with ev.read('test'):
env('loads', 'test')
e = ev.read('test')
loads_file = os.path.join(e.path, 'loads')
assert os.path.exists(loads_file)
with open(loads_file) as f:
contents = f.read()
assert 'module load mpileaks' in contents
def test_env_repo():
e = ev.create('test')
e.add('mpileaks')
e.write()
with ev.read('test'):
concretize()
package = e.repo.get('mpileaks')
assert package.name == 'mpileaks'
assert package.namespace == 'spack.pkg.builtin.mock'
def test_stage(mock_stage, mock_fetch, install_mockery):
env('create', 'test')
with ev.read('test'):
add('mpileaks')
add('zmpi')
concretize()
stage()
root = str(mock_stage)
def check_stage(spec):
spec = Spec(spec).concretized()
for dep in spec.traverse():
stage_name = "%s-%s-%s" % (dep.name, dep.version, dep.dag_hash())
assert os.path.isdir(os.path.join(root, stage_name))
check_stage('mpileaks')
check_stage('zmpi')
def test_env_with_config():
test_config = """\
env:
specs:
- mpileaks
packages:
mpileaks:
version: [2.2]
"""
spack.package_prefs.PackagePrefs.clear_caches()
_env_create('test', StringIO(test_config))
e = ev.read('test')
ev.prepare_config_scope(e)
e.concretize()
assert any(x.satisfies('[email protected]')
for x in e._get_environment_specs())
def test_install_only_dependencies_of_all_in_env(tmpdir, mock_fetch,
install_mockery,
mutable_mock_env_path):
env('create', '--without-view', 'test')
with ev.read('test'):
roots = [
Spec('[email protected]').concretized(),
Spec('[email protected]').concretized()
]
add('[email protected]')
add('[email protected]')
install('--only', 'dependencies')
for root in roots:
assert not os.path.exists(root.prefix)
for dep in root.traverse(root=False):
assert os.path.exists(dep.prefix)
def test_stack_definition_conditional_with_variable(tmpdir):
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [libelf, mpileaks]
- packages: [callpath]
when: platform == 'test'
specs:
- $packages
""")
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
test = ev.read('test')
assert Spec('libelf') in test.user_specs
assert Spec('mpileaks') in test.user_specs
assert Spec('callpath') in test.user_specs
def tests_buildcache_create_env(
install_mockery, mock_fetch, monkeypatch,
tmpdir, mutable_mock_env_path):
""""Ensure that buildcache create creates output files from env"""
pkg = 'trivial-install-test-package'
env('create', 'test')
with ev.read('test'):
add(pkg)
install()
buildcache('create', '-d', str(tmpdir), '--unsigned')
spec = Spec(pkg).concretized()
tarball_path = spack.binary_distribution.tarball_path_name(spec, '.spack')
tarball = spack.binary_distribution.tarball_name(spec, '.spec.yaml')
assert os.path.exists(
os.path.join(str(tmpdir), 'build_cache', tarball_path))
assert os.path.exists(
os.path.join(str(tmpdir), 'build_cache', tarball))
def test_stack_definition_complex_conditional(tmpdir):
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
env:
definitions:
- packages: [libelf, mpileaks]
- packages: [callpath]
when: re.search(r'foo', hostname) and env['test'] == 'THISSHOULDBEFALSE'
specs:
- $packages
""")
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
test = ev.read('test')
assert Spec('libelf') in test.user_specs
assert Spec('mpileaks') in test.user_specs
assert Spec('callpath') not in test.user_specs
def find_environment(args):
"""Find active environment from args or environment variable.
Check for an environment in this order:
1. via ``spack -e ENV`` or ``spack -D DIR`` (arguments)
2. via a path in the spack.environment.spack_env_var environment variable.
If an environment is found, read it in. If not, return None.
Arguments:
args (argparse.Namespace): argparse namespace with command arguments
Returns:
(spack.environment.Environment): a found environment, or ``None``
"""
# treat env as a name
env = args.env
if env:
if ev.exists(env):
return ev.read(env)
else:
# if env was specified, see if it is a directory otherwise, look
# at env_dir (env and env_dir are mutually exclusive)
env = args.env_dir
# if no argument, look for the environment variable
if not env:
env = os.environ.get(ev.spack_env_var)
# nothing was set; there's no active environment
if not env:
return None
# if we get here, env isn't the name of a spack environment; it has
# to be a path to an environment, or there is something wrong.
if ev.is_env_dir(env):
return ev.Environment(env)
raise ev.SpackEnvironmentError('no environment in %s' % env)
def test_ci_generate_with_external_pkg(tmpdir, mutable_mock_env_path,
env_deactivate, install_mockery,
mock_packages):
"""Make sure we do not generate jobs for external pkgs"""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
specs:
- archive-files
- externaltest
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
mappings:
- match:
- archive-files
- externaltest
runner-attributes:
tags:
- donotcare
image: donotcare
""")
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as f:
contents = f.read()
print('generated contents: ')
print(contents)
yaml_contents = syaml.load(contents)
for ci_key in yaml_contents.keys():
if 'externaltool' in ci_key:
print('Erroneously staged "externaltool" pkg')
assert (False)
def test_release_jobs_with_env(tmpdir, mutable_mock_env_path, env_deactivate,
install_mockery, mock_packages):
"""Make sure we can get a .gitlab-ci.yml from an environment file
which has the gitlab-ci, cdash, and mirrors sections."""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
definitions:
- packages: [archive-files]
specs:
- $packages
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
mappings:
- match:
- archive-files
runner-attributes:
tags:
- donotcare
image: donotcare
cdash:
build-group: Not important
url: https://my.fake.cdash
project: Not used
site: Nothing
""")
with tmpdir.as_cwd():
env('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
release_jobs('--output-file', outputfile)
with open(outputfile) as f:
contents = f.read()
assert('archive-files' in contents)
assert('stages: [stage-0' in contents)
def test_env_with_included_config_file():
test_config = """\
env:
include:
- ./included-config.yaml
specs:
- mpileaks
"""
_env_create('test', StringIO(test_config))
e = ev.read('test')
with open(os.path.join(e.path, 'included-config.yaml'), 'w') as f:
f.write("""\
packages:
mpileaks:
version: [2.2]
""")
with e:
e.concretize()
assert any(x.satisfies('[email protected]') for x in e._get_environment_specs())
def test_ci_generate_with_env_missing_section(tmpdir, mutable_mock_env_path,
env_deactivate, install_mockery,
mock_packages):
"""Make sure we get a reasonable message if we omit gitlab-ci section"""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
specs:
- archive-files
mirrors:
some-mirror: https://my.fake.mirror
""")
expect_out = 'Error: Environment yaml does not have "gitlab-ci" section'
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
with ev.read('test'):
output = ci_cmd('generate', fail_on_error=False, output=str)
assert (expect_out in output)
def test_ci_subcommands_without_mirror(tmpdir, mutable_mock_env_path,
env_deactivate, mock_packages,
install_mockery):
"""Make sure we catch if there is not a mirror and report an error"""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
specs:
- archive-files
gitlab-ci:
mappings:
- match:
- archive-files
runner-attributes:
tags:
- donotcare
image: donotcare
""")
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
# Check the 'generate' subcommand
output = ci_cmd('generate',
'--output-file',
outputfile,
output=str,
fail_on_error=False)
ex = 'spack ci generate requires an env containing a mirror'
assert (ex in output)
# Also check the 'rebuild-index' subcommand
output = ci_cmd('rebuild-index', output=str, fail_on_error=False)
ex = 'spack ci rebuild-index requires an env containing a mirror'
assert (ex in output)
def test_included_config_precedence():
test_config = """\
env:
include:
- ./high-config.yaml # this one should take precedence
- ./low-config.yaml
specs:
- mpileaks
"""
spack.package_prefs.PackagePrefs.clear_caches()
_env_create('test', StringIO(test_config))
e = ev.read('test')
with open(os.path.join(e.path, 'high-config.yaml'), 'w') as f:
f.write("""\
packages:
libelf:
version: [0.8.10] # this should override libelf version below
""")
with open(os.path.join(e.path, 'low-config.yaml'), 'w') as f:
f.write("""\
packages:
mpileaks:
version: [2.2]
libelf:
version: [0.8.12]
""")
ev.prepare_config_scope(e)
e.concretize()
assert any(
x.satisfies('[email protected]') for x in e._get_environment_specs())
assert any(
[x.satisfies('[email protected]') for x in e._get_environment_specs()])
def test_included_config_precedence():
test_config = """\
env:
include:
- ./high-config.yaml # this one should take precedence
- ./low-config.yaml
specs:
- mpileaks
"""
spack.package_prefs.PackagePrefs.clear_caches()
_env_create('test', StringIO(test_config))
e = ev.read('test')
with open(os.path.join(e.path, 'high-config.yaml'), 'w') as f:
f.write("""\
packages:
libelf:
version: [0.8.10] # this should override libelf version below
""")
with open(os.path.join(e.path, 'low-config.yaml'), 'w') as f:
f.write("""\
packages:
mpileaks:
version: [2.2]
libelf:
version: [0.8.12]
""")
ev.prepare_config_scope(e)
e.concretize()
assert any(
x.satisfies('[email protected]') for x in e._get_environment_specs())
assert any(
[x.satisfies('[email protected]') for x in e._get_environment_specs()])
def test_ci_generate_with_external_pkg(tmpdir, mutable_mock_env_path,
env_deactivate, install_mockery,
mock_packages, monkeypatch):
"""Make sure we do not generate jobs for external pkgs"""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
specs:
- archive-files
- externaltest
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
mappings:
- match:
- archive-files
- externaltest
runner-attributes:
tags:
- donotcare
image: donotcare
""")
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
monkeypatch.setattr(ci, 'SPACK_PR_MIRRORS_ROOT_URL',
r"file:///fake/mirror")
ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as f:
yaml_contents = syaml.load(f)
# Check that the "externaltool" package was not erroneously staged
assert not any('externaltool' in key for key in yaml_contents)
def test_init_with_file_and_remove(tmpdir):
"""Ensure a user can remove from any position in the spack.yaml file."""
path = tmpdir.join('spack.yaml')
with tmpdir.as_cwd():
with open(str(path), 'w') as f:
f.write("""\
env:
specs:
- mpileaks
""")
env('create', 'test', 'spack.yaml')
out = env('list')
assert 'test' in out
with ev.read('test'):
assert 'mpileaks' in find()
env('remove', '-y', 'test')
out = env('list')
assert 'test' not in out
def test_env_config_precedence():
test_config = """\
env:
packages:
libelf:
version: [0.8.12]
include:
- ./included-config.yaml
specs:
- mpileaks
"""
spack.package_prefs.PackagePrefs.clear_caches()
_env_create('test', StringIO(test_config))
e = ev.read('test')
with open(os.path.join(e.path, 'included-config.yaml'), 'w') as f:
f.write("""\
packages:
mpileaks:
version: [2.2]
libelf:
version: [0.8.11]
""")
ev.prepare_config_scope(e)
e.concretize()
# ensure included scope took effect
assert any(
x.satisfies('[email protected]') for x in e._get_environment_specs())
# ensure env file takes precedence
assert any(
x.satisfies('[email protected]') for x in e._get_environment_specs())
def test_env_config_precedence():
test_config = """\
env:
packages:
libelf:
version: [0.8.12]
include:
- ./included-config.yaml
specs:
- mpileaks
"""
spack.package_prefs.PackagePrefs.clear_caches()
_env_create('test', StringIO(test_config))
e = ev.read('test')
with open(os.path.join(e.path, 'included-config.yaml'), 'w') as f:
f.write("""\
packages:
mpileaks:
version: [2.2]
libelf:
version: [0.8.11]
""")
ev.prepare_config_scope(e)
e.concretize()
# ensure included scope took effect
assert any(
x.satisfies('[email protected]') for x in e._get_environment_specs())
# ensure env file takes precedence
assert any(
x.satisfies('[email protected]') for x in e._get_environment_specs())
def test_env_remove(capfd):
env('create', 'foo')
env('create', 'bar')
out = env('list')
assert 'foo' in out
assert 'bar' in out
foo = ev.read('foo')
with foo:
with pytest.raises(spack.main.SpackCommandError):
with capfd.disabled():
env('remove', '-y', 'foo')
assert 'foo' in env('list')
env('remove', '-y', 'foo')
out = env('list')
assert 'foo' not in out
assert 'bar' in out
env('remove', '-y', 'bar')
out = env('list')
assert 'foo' not in out
assert 'bar' not in out
def test_env_remove(capfd):
env('create', 'foo')
env('create', 'bar')
out = env('list')
assert 'foo' in out
assert 'bar' in out
foo = ev.read('foo')
with foo:
with pytest.raises(spack.main.SpackCommandError):
with capfd.disabled():
env('remove', '-y', 'foo')
assert 'foo' in env('list')
env('remove', '-y', 'foo')
out = env('list')
assert 'foo' not in out
assert 'bar' in out
env('remove', '-y', 'bar')
out = env('list')
assert 'foo' not in out
assert 'bar' not in out
def test_init_with_file_and_remove(tmpdir):
"""Ensure a user can remove from any position in the spack.yaml file."""
path = tmpdir.join('spack.yaml')
with tmpdir.as_cwd():
with open(str(path), 'w') as f:
f.write("""\
env:
specs:
- mpileaks
""")
env('create', 'test', 'spack.yaml')
out = env('list')
assert 'test' in out
with ev.read('test'):
assert 'mpileaks' in find()
env('remove', '-y', 'test')
out = env('list')
assert 'test' not in out
def test_remove_command():
env('create', 'test')
assert 'test' in env('list')
with ev.read('test'):
add('mpileaks')
assert 'mpileaks' in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' not in find('--show-concretized')
with ev.read('test'):
remove('mpileaks')
assert 'mpileaks' not in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' not in find('--show-concretized')
with ev.read('test'):
add('mpileaks')
assert 'mpileaks' in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' not in find('--show-concretized')
with ev.read('test'):
concretize()
assert 'mpileaks' in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' in find('--show-concretized')
with ev.read('test'):
remove('mpileaks')
assert 'mpileaks' not in find()
# removed but still in last concretized specs
assert 'mpileaks@' in find('--show-concretized')
with ev.read('test'):
concretize()
assert 'mpileaks' not in find()
assert 'mpileaks@' not in find()
# now the lockfile is regenerated and it's gone.
assert 'mpileaks@' not in find('--show-concretized')
def test_remove_command():
env('create', 'test')
assert 'test' in env('list')
with ev.read('test'):
add('mpileaks')
assert 'mpileaks' in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' not in find('--show-concretized')
with ev.read('test'):
remove('mpileaks')
assert 'mpileaks' not in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' not in find('--show-concretized')
with ev.read('test'):
add('mpileaks')
assert 'mpileaks' in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' not in find('--show-concretized')
with ev.read('test'):
concretize()
assert 'mpileaks' in find()
assert 'mpileaks@' not in find()
assert 'mpileaks@' in find('--show-concretized')
with ev.read('test'):
remove('mpileaks')
assert 'mpileaks' not in find()
# removed but still in last concretized specs
assert 'mpileaks@' in find('--show-concretized')
with ev.read('test'):
concretize()
assert 'mpileaks' not in find()
assert 'mpileaks@' not in find()
# now the lockfile is regenerated and it's gone.
assert 'mpileaks@' not in find('--show-concretized')
def test_push_mirror_contents(tmpdir, mutable_mock_env_path, env_deactivate,
install_mockery, mock_packages, mock_fetch,
mock_stage, mock_gnupghome):
working_dir = tmpdir.join('working_dir')
mirror_dir = working_dir.join('mirror')
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
signing_key_dir = spack_paths.mock_gpg_keys_path
signing_key_path = os.path.join(signing_key_dir, 'package-signing-key')
with open(signing_key_path) as fd:
signing_key = fd.read()
ci.import_signing_key(signing_key)
spack_yaml_contents = """
spack:
definitions:
- packages: [patchelf]
specs:
- $packages
mirrors:
test-mirror: {0}
gitlab-ci:
enable-artifacts-buildcache: True
mappings:
- match:
- patchelf
runner-attributes:
tags:
- donotcare
image: donotcare
service-job-attributes:
tags:
- nonbuildtag
image: basicimage
""".format(mirror_url)
print('spack.yaml:\n{0}\n'.format(spack_yaml_contents))
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write(spack_yaml_contents)
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
with ev.read('test') as env:
spec_map = ci.get_concrete_specs('patchelf', 'patchelf', '',
'FIND_ANY')
concrete_spec = spec_map['patchelf']
spec_yaml = concrete_spec.to_yaml(hash=ht.build_hash)
yaml_path = str(tmpdir.join('spec.yaml'))
with open(yaml_path, 'w') as ypfd:
ypfd.write(spec_yaml)
install_cmd('--keep-stage', yaml_path)
# env, spec, yaml_path, mirror_url, build_id, sign_binaries
ci.push_mirror_contents(env, concrete_spec, yaml_path, mirror_url,
'42', True)
buildcache_path = os.path.join(mirror_dir.strpath, 'build_cache')
# Now test the --prune-dag (default) option of spack ci generate
mirror_cmd('add', 'test-ci', mirror_url)
outputfile_pruned = str(tmpdir.join('pruned_pipeline.yml'))
ci_cmd('generate', '--output-file', outputfile_pruned)
with open(outputfile_pruned) as f:
contents = f.read()
yaml_contents = syaml.load(contents)
assert ('no-specs-to-rebuild' in yaml_contents)
# Make sure there are no other spec jobs or rebuild-index
assert (len(yaml_contents.keys()) == 1)
the_elt = yaml_contents['no-specs-to-rebuild']
assert ('tags' in the_elt)
assert ('nonbuildtag' in the_elt['tags'])
assert ('image' in the_elt)
assert (the_elt['image'] == 'basicimage')
outputfile_not_pruned = str(tmpdir.join('unpruned_pipeline.yml'))
ci_cmd('generate', '--no-prune-dag', '--output-file',
outputfile_not_pruned)
# Test the --no-prune-dag option of spack ci generate
with open(outputfile_not_pruned) as f:
contents = f.read()
yaml_contents = syaml.load(contents)
found_spec_job = False
for ci_key in yaml_contents.keys():
if '(specs) patchelf' in ci_key:
the_elt = yaml_contents[ci_key]
assert ('variables' in the_elt)
job_vars = the_elt['variables']
assert ('SPACK_SPEC_NEEDS_REBUILD' in job_vars)
assert (
job_vars['SPACK_SPEC_NEEDS_REBUILD'] == 'False')
found_spec_job = True
assert (found_spec_job)
mirror_cmd('rm', 'test-ci')
# Test generating buildcache index while we have bin mirror
buildcache_cmd('update-index', '--mirror-url', mirror_url)
index_path = os.path.join(buildcache_path, 'index.json')
with open(index_path) as idx_fd:
index_object = json.load(idx_fd)
validate(index_object, db_idx_schema)
# Now that index is regenerated, validate "buildcache list" output
buildcache_list_output = buildcache_cmd('list', output=str)
assert ('patchelf' in buildcache_list_output)
# Also test buildcache_spec schema
bc_files_list = os.listdir(buildcache_path)
for file_name in bc_files_list:
if file_name.endswith('.spec.yaml'):
spec_yaml_path = os.path.join(buildcache_path, file_name)
with open(spec_yaml_path) as yaml_fd:
yaml_object = syaml.load(yaml_fd)
validate(yaml_object, spec_yaml_schema)
logs_dir = working_dir.join('logs_dir')
if not os.path.exists(logs_dir.strpath):
os.makedirs(logs_dir.strpath)
ci.copy_stage_logs_to_artifacts(concrete_spec, logs_dir.strpath)
logs_dir_list = os.listdir(logs_dir.strpath)
assert ('spack-build-out.txt' in logs_dir_list)
# Also just make sure that if something goes wrong with the
# stage logs copy, no exception is thrown
ci.copy_stage_logs_to_artifacts(None, logs_dir.strpath)
dl_dir = working_dir.join('download_dir')
if not os.path.exists(dl_dir.strpath):
os.makedirs(dl_dir.strpath)
buildcache_cmd('download', '--spec-yaml', yaml_path, '--path',
dl_dir.strpath, '--require-cdashid')
dl_dir_list = os.listdir(dl_dir.strpath)
assert (len(dl_dir_list) == 3)
def test_ci_generate_bootstrap_artifacts_buildcache(tmpdir,
mutable_mock_env_path,
env_deactivate,
install_mockery,
mock_packages):
"""Test that we can bootstrap a compiler when artifacts buildcache
is turned on"""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
definitions:
- bootstrap:
- [email protected]
specs:
- dyninst%[email protected]
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
bootstrap:
- name: bootstrap
compiler-agnostic: true
mappings:
- match:
- arch=test-debian6-x86_64
runner-attributes:
tags:
- donotcare
enable-artifacts-buildcache: True
""")
needs_graph = {
'(bootstrap) conflict': [],
'(bootstrap) gcc': [
'(bootstrap) conflict',
],
'(specs) libelf': [
'(bootstrap) gcc',
'(bootstrap) conflict',
],
'(specs) libdwarf': [
'(bootstrap) gcc',
'(bootstrap) conflict',
'(specs) libelf',
],
'(specs) dyninst': [
'(bootstrap) gcc',
'(bootstrap) conflict',
'(specs) libelf',
'(specs) libdwarf',
],
}
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as f:
contents = f.read()
yaml_contents = syaml.load(contents)
_validate_needs_graph(yaml_contents, needs_graph, True)
def test_ci_generate_with_env(tmpdir, mutable_mock_env_path, env_deactivate,
install_mockery, mock_packages):
"""Make sure we can get a .gitlab-ci.yml from an environment file
which has the gitlab-ci, cdash, and mirrors sections."""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
definitions:
- bootstrap:
- [email protected]
- old-gcc-pkgs:
- archive-files
- callpath
# specify ^openblas-with-lapack to ensure that builtin.mock repo flake8
# package (which can also provide lapack) is not chosen, as it violates
# a package-level check which requires exactly one fetch strategy (this
# is apparently not an issue for other tests that use it).
- [email protected] ^openblas-with-lapack
specs:
- matrix:
- [$old-gcc-pkgs]
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
bootstrap:
- name: bootstrap
compiler-agnostic: true
mappings:
- match:
- arch=test-debian6-core2
runner-attributes:
tags:
- donotcare
image: donotcare
final-stage-rebuild-index:
image: donotcare
tags: [donotcare]
cdash:
build-group: Not important
url: https://my.fake.cdash
project: Not used
site: Nothing
""")
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as f:
contents = f.read()
yaml_contents = syaml.load(contents)
found_spec = False
for ci_key in yaml_contents.keys():
if '(bootstrap)' in ci_key:
found_spec = True
assert ('cmake' in ci_key)
assert (found_spec)
assert ('stages' in yaml_contents)
assert (len(yaml_contents['stages']) == 6)
assert (yaml_contents['stages'][0] == 'stage-0')
assert (yaml_contents['stages'][5] == 'stage-rebuild-index')
def test_env_add_nonexistant_fails():
env('create', 'test')
e = ev.read('test')
with pytest.raises(ev.SpackEnvironmentError, match=r'no such package'):
e.add('thispackagedoesnotexist')
def test_push_mirror_contents(tmpdir, mutable_mock_env_path, env_deactivate,
install_mockery, mock_packages, mock_fetch,
mock_stage, mock_gnupghome):
working_dir = tmpdir.join('working_dir')
mirror_dir = working_dir.join('mirror')
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
signing_key_dir = spack_paths.mock_gpg_keys_path
signing_key_path = os.path.join(signing_key_dir, 'package-signing-key')
with open(signing_key_path) as fd:
signing_key = fd.read()
ci.import_signing_key(signing_key)
spack_yaml_contents = """
spack:
definitions:
- packages: [patchelf]
specs:
- $packages
mirrors:
test-mirror: {0}
""".format(mirror_url)
print('spack.yaml:\n{0}\n'.format(spack_yaml_contents))
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write(spack_yaml_contents)
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
with ev.read('test') as env:
spec_map = ci.get_concrete_specs(
'patchelf', 'patchelf', '', 'FIND_ANY')
concrete_spec = spec_map['patchelf']
spec_yaml = concrete_spec.to_yaml(hash=ht.build_hash)
yaml_path = str(tmpdir.join('spec.yaml'))
with open(yaml_path, 'w') as ypfd:
ypfd.write(spec_yaml)
install_cmd('--keep-stage', yaml_path)
# env, spec, yaml_path, mirror_url, build_id
ci.push_mirror_contents(
env, concrete_spec, yaml_path, mirror_url, '42')
buildcache_list_output = buildcache_cmd('list', output=str)
assert('patchelf' in buildcache_list_output)
logs_dir = working_dir.join('logs_dir')
if not os.path.exists(logs_dir.strpath):
os.makedirs(logs_dir.strpath)
ci.copy_stage_logs_to_artifacts(concrete_spec, logs_dir.strpath)
logs_dir_list = os.listdir(logs_dir.strpath)
assert('spack-build-env.txt' in logs_dir_list)
assert('spack-build-out.txt' in logs_dir_list)
# Also just make sure that if something goes wrong with the
# stage logs copy, no exception is thrown
ci.copy_stage_logs_to_artifacts(None, logs_dir.strpath)
dl_dir = working_dir.join('download_dir')
if not os.path.exists(dl_dir.strpath):
os.makedirs(dl_dir.strpath)
buildcache_cmd('download', '--spec-yaml', yaml_path, '--path',
dl_dir.strpath, '--require-cdashid')
dl_dir_list = os.listdir(dl_dir.strpath)
assert(len(dl_dir_list) == 3)
def test_push_mirror_contents(tmpdir, mutable_mock_env_path, env_deactivate,
install_mockery, mock_packages, mock_fetch,
mock_stage, mock_gnupghome):
working_dir = tmpdir.join('working_dir')
mirror_dir = working_dir.join('mirror')
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
signing_key_dir = spack_paths.mock_gpg_keys_path
signing_key_path = os.path.join(signing_key_dir, 'package-signing-key')
with open(signing_key_path) as fd:
signing_key = fd.read()
ci.import_signing_key(signing_key)
spack_yaml_contents = """
spack:
definitions:
- packages: [patchelf]
specs:
- $packages
mirrors:
test-mirror: {0}
""".format(mirror_url)
print('spack.yaml:\n{0}\n'.format(spack_yaml_contents))
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write(spack_yaml_contents)
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
with ev.read('test') as env:
spec_map = ci.get_concrete_specs('patchelf', 'patchelf', '',
'FIND_ANY')
concrete_spec = spec_map['patchelf']
spec_yaml = concrete_spec.to_yaml(hash=ht.build_hash)
yaml_path = str(tmpdir.join('spec.yaml'))
with open(yaml_path, 'w') as ypfd:
ypfd.write(spec_yaml)
install_cmd('--keep-stage', yaml_path)
# env, spec, yaml_path, mirror_url, build_id, sign_binaries
ci.push_mirror_contents(env, concrete_spec, yaml_path, mirror_url,
'42', True)
buildcache_path = os.path.join(mirror_dir.strpath, 'build_cache')
# Test generating buildcache index while we have bin mirror
buildcache_cmd('update-index', '--mirror-url', mirror_url)
index_path = os.path.join(buildcache_path, 'index.json')
with open(index_path) as idx_fd:
index_object = json.load(idx_fd)
validate(index_object, db_idx_schema)
# Now that index is regenerated, validate "buildcache list" output
buildcache_list_output = buildcache_cmd('list', output=str)
assert ('patchelf' in buildcache_list_output)
# Also test buildcache_spec schema
bc_files_list = os.listdir(buildcache_path)
for file_name in bc_files_list:
if file_name.endswith('.spec.yaml'):
spec_yaml_path = os.path.join(buildcache_path, file_name)
with open(spec_yaml_path) as yaml_fd:
yaml_object = syaml.load(yaml_fd)
validate(yaml_object, spec_yaml_schema)
logs_dir = working_dir.join('logs_dir')
if not os.path.exists(logs_dir.strpath):
os.makedirs(logs_dir.strpath)
ci.copy_stage_logs_to_artifacts(concrete_spec, logs_dir.strpath)
logs_dir_list = os.listdir(logs_dir.strpath)
assert ('spack-build-out.txt' in logs_dir_list)
# Also just make sure that if something goes wrong with the
# stage logs copy, no exception is thrown
ci.copy_stage_logs_to_artifacts(None, logs_dir.strpath)
dl_dir = working_dir.join('download_dir')
if not os.path.exists(dl_dir.strpath):
os.makedirs(dl_dir.strpath)
buildcache_cmd('download', '--spec-yaml', yaml_path, '--path',
dl_dir.strpath, '--require-cdashid')
dl_dir_list = os.listdir(dl_dir.strpath)
assert (len(dl_dir_list) == 3)
def test_develop(self):
env('create', 'test')
with ev.read('test') as e:
develop('[email protected]')
self.check_develop(e, spack.spec.Spec('[email protected]'))
def test_ci_generate_bootstrap_prune_dag(install_mockery_mutable_config,
mock_packages, mock_fetch,
mock_archive, mutable_config,
monkeypatch, tmpdir,
mutable_mock_env_path,
env_deactivate):
"""Test compiler bootstrapping with DAG pruning. Specifically, make
sure that if we detect the bootstrapped compiler needs to be rebuilt,
we ensure the spec we want to build with that compiler is scheduled
for rebuild as well."""
# Create a temp mirror directory for buildcache usage
mirror_dir = tmpdir.join('mirror_dir')
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
# Install a compiler, because we want to put it in a buildcache
install_cmd('[email protected]%[email protected]')
# Put installed compiler in the buildcache
buildcache_cmd('create', '-u', '-a', '-f', '-d', mirror_dir.strpath,
'[email protected]%[email protected]')
# Now uninstall the compiler
uninstall_cmd('-y', '[email protected]%[email protected]')
monkeypatch.setattr(spack.concretize.Concretizer,
'check_for_compiler_existence', False)
spack.config.set('config:install_missing_compilers', True)
assert CompilerSpec('[email protected]') not in compilers.all_compiler_specs()
# Configure the mirror where we put that buildcache w/ the compiler
mirror_cmd('add', 'test-mirror', mirror_url)
install_cmd('--no-check-signature', 'a%[email protected]')
# Put spec built with installed compiler in the buildcache
buildcache_cmd('create', '-u', '-a', '-f', '-d', mirror_dir.strpath,
'a%[email protected]')
# Now uninstall the spec
uninstall_cmd('-y', 'a%[email protected]')
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
definitions:
- bootstrap:
- [email protected]%[email protected]
specs:
- a%[email protected]
mirrors:
atestm: {0}
gitlab-ci:
bootstrap:
- name: bootstrap
compiler-agnostic: true
mappings:
- match:
- arch=test-debian6-x86_64
runner-attributes:
tags:
- donotcare
- match:
- arch=test-debian6-core2
runner-attributes:
tags:
- meh
""".format(mirror_url))
# Without this monkeypatch, pipeline generation process would think that
# nothing in the environment needs rebuilding. With the monkeypatch, the
# process sees the compiler as needing a rebuild, which should then result
# in the specs built with that compiler needing a rebuild too.
def fake_get_mirrors_for_spec(spec=None,
full_hash_match=False,
mirrors_to_check=None,
index_only=False):
if spec.name == 'gcc':
return []
else:
return [{
'spec': spec,
'mirror_url': mirror_url,
}]
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
monkeypatch.setattr(ci, 'SPACK_PR_MIRRORS_ROOT_URL',
r"file:///fake/mirror")
ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as of:
yaml_contents = of.read()
original_yaml_contents = syaml.load(yaml_contents)
# without the monkeypatch, everything appears up to date and no
# rebuild jobs are generated.
assert (original_yaml_contents)
assert ('no-specs-to-rebuild' in original_yaml_contents)
monkeypatch.setattr(spack.binary_distribution,
'get_mirrors_for_spec',
fake_get_mirrors_for_spec)
ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as of:
yaml_contents = of.read()
new_yaml_contents = syaml.load(yaml_contents)
assert (new_yaml_contents)
# This 'needs' graph reflects that even though specs 'a' and 'b' do
# not otherwise need to be rebuilt (thanks to DAG pruning), they
# both end up in the generated pipeline because the compiler they
# depend on is bootstrapped, and *does* need to be rebuilt.
needs_graph = {
'(bootstrap) gcc': [],
'(specs) b': [
'(bootstrap) gcc',
],
'(specs) a': [
'(bootstrap) gcc',
'(specs) b',
],
}
_validate_needs_graph(new_yaml_contents, needs_graph, False)
def test_ci_generate_with_custom_scripts(tmpdir, mutable_mock_env_path,
env_deactivate, install_mockery,
mock_packages, monkeypatch):
"""Test use of user-provided scripts"""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
specs:
- archive-files
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
mappings:
- match:
- archive-files
runner-attributes:
tags:
- donotcare
variables:
ONE: plain-string-value
TWO: ${INTERP_ON_BUILD}
before_script:
- mkdir /some/path
- pushd /some/path
- git clone ${SPACK_REPO}
- cd spack
- git checkout ${SPACK_REF}
- popd
script:
- spack -d ci rebuild
after_script:
- rm -rf /some/path/spack
""")
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
monkeypatch.setattr(spack.main, 'get_version', lambda: '0.15.3')
ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as f:
contents = f.read()
yaml_contents = syaml.load(contents)
found_it = False
assert ('variables' in yaml_contents)
global_vars = yaml_contents['variables']
assert ('SPACK_VERSION' in global_vars)
assert (global_vars['SPACK_VERSION'] == '0.15.3')
assert ('SPACK_CHECKOUT_VERSION' in global_vars)
assert (global_vars['SPACK_CHECKOUT_VERSION'] == 'v0.15.3')
for ci_key in yaml_contents.keys():
ci_obj = yaml_contents[ci_key]
if 'archive-files' in ci_key:
# Ensure we have variables, possibly interpolated
assert ('variables' in ci_obj)
var_d = ci_obj['variables']
assert ('ONE' in var_d)
assert (var_d['ONE'] == 'plain-string-value')
assert ('TWO' in var_d)
assert (var_d['TWO'] == '${INTERP_ON_BUILD}')
# Ensure we have scripts verbatim
assert ('before_script' in ci_obj)
before_script = ci_obj['before_script']
assert (before_script[0] == 'mkdir /some/path')
assert (before_script[1] == 'pushd /some/path')
assert (before_script[2] == 'git clone ${SPACK_REPO}')
assert (before_script[3] == 'cd spack')
assert (
before_script[4] == 'git checkout ${SPACK_REF}')
assert (before_script[5] == 'popd')
assert ('script' in ci_obj)
assert (ci_obj['script'][0] == 'spack -d ci rebuild')
assert ('after_script' in ci_obj)
after_script = ci_obj['after_script'][0]
assert (after_script == 'rm -rf /some/path/spack')
found_it = True
assert (found_it)
def test_ci_generate_for_pr_pipeline(tmpdir, mutable_mock_env_path,
env_deactivate, install_mockery,
mock_packages, monkeypatch):
"""Test that PR pipelines do not include a final stage job for
rebuilding the mirror index, even if that job is specifically
configured"""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
specs:
- flatten-deps
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
enable-artifacts-buildcache: True
mappings:
- match:
- flatten-deps
runner-attributes:
tags:
- donotcare
- match:
- dependency-install
runner-attributes:
tags:
- donotcare
service-job-attributes:
image: donotcare
tags: [donotcare]
rebuild-index: False
""")
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
os.environ['SPACK_IS_PR_PIPELINE'] = 'True'
os.environ['SPACK_PR_BRANCH'] = 'fake-test-branch'
monkeypatch.setattr(ci, 'SPACK_PR_MIRRORS_ROOT_URL',
r"file:///fake/mirror")
try:
ci_cmd('generate', '--output-file', outputfile)
finally:
del os.environ['SPACK_IS_PR_PIPELINE']
del os.environ['SPACK_PR_BRANCH']
with open(outputfile) as f:
contents = f.read()
print('generated contents: ')
print(contents)
yaml_contents = syaml.load(contents)
assert ('rebuild-index' not in yaml_contents)
for ci_key in yaml_contents.keys():
if ci_key.startswith('(specs) '):
job_object = yaml_contents[ci_key]
job_vars = job_object['variables']
assert ('SPACK_IS_PR_PIPELINE' in job_vars)
assert (job_vars['SPACK_IS_PR_PIPELINE'] == 'True')
def test_ci_rebuild_basic(tmpdir, mutable_mock_env_path, env_deactivate,
install_mockery, mock_packages, mock_gnupghome):
working_dir = tmpdir.join('working_dir')
mirror_dir = working_dir.join('mirror')
mirror_url = 'file://{0}'.format(mirror_dir.strpath)
signing_key_dir = spack_paths.mock_gpg_keys_path
signing_key_path = os.path.join(signing_key_dir, 'package-signing-key')
with open(signing_key_path) as fd:
signing_key = fd.read()
spack_yaml_contents = """
spack:
definitions:
- packages: [archive-files]
specs:
- $packages
mirrors:
test-mirror: {0}
gitlab-ci:
enable-artifacts-buildcache: True
mappings:
- match:
- archive-files
runner-attributes:
tags:
- donotcare
image: donotcare
cdash:
build-group: Not important
url: https://my.fake.cdash
project: Not used
site: Nothing
""".format(mirror_url)
print('spack.yaml:\n{0}\n'.format(spack_yaml_contents))
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write(spack_yaml_contents)
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
with ev.read('test'):
root_spec = ('eJyNjsGOwyAMRO/5Ct96alRFFK34ldUqcohJ6BJAQFHUry9Nk66'
'UXNY3v5mxJ3qSojoDBjnqTGelDUVRQZlMIWpnBZya+nJa0Mv1Fg'
'G8waRcmAQkimkHWxcF9NRptHyVEoaBkoD5i7ecLVC6yZd/YTtpc'
'SIBg5Tr/mnA6mt9qTZL9CiLr7trk7StJyd/F81jKGoqoe2gVAaH'
'0uT7ZwPeH9A875HaA9MfidHdHxgxjgJuTGVtIrvfHGtynjkGyzi'
'xRrkHy94t1lftvv1n4AkVK3kQ')
# Create environment variables as gitlab would do it
set_env_var('CI_PROJECT_DIR', working_dir.strpath)
set_env_var('SPACK_SIGNING_KEY', signing_key)
set_env_var('SPACK_ROOT_SPEC', root_spec)
set_env_var('SPACK_JOB_SPEC_PKG_NAME', 'archive-files')
set_env_var('SPACK_COMPILER_ACTION', 'NONE')
set_env_var('SPACK_CDASH_BUILD_NAME', '(specs) archive-files')
set_env_var('SPACK_RELATED_BUILDS_CDASH', '')
rebuild_output = ci_cmd('rebuild', fail_on_error=False, output=str)
print(rebuild_output)
def test_develop_no_clone(self, tmpdir):
env('create', 'test')
with ev.read('test') as e:
develop('--no-clone', '-p', str(tmpdir), '[email protected]')
self.check_develop(e, spack.spec.Spec('[email protected]'), str(tmpdir))
def test_ci_generate_override_runner_attrs(tmpdir, mutable_mock_env_path,
env_deactivate, install_mockery,
mock_packages, monkeypatch):
"""Test that we get the behavior we want with respect to the provision
of runner attributes like tags, variables, and scripts, both when we
inherit them from the top level, as well as when we override one or
more at the runner level"""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
specs:
- flatten-deps
- a
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
tags:
- toplevel
variables:
ONE: toplevelvarone
TWO: toplevelvartwo
before_script:
- pre step one
- pre step two
script:
- main step
after_script:
- post step one
mappings:
- match:
- flatten-deps
runner-attributes:
tags:
- specific-one
variables:
THREE: specificvarthree
- match:
- dependency-install
- match:
- a
runner-attributes:
tags:
- specific-a
- toplevel
variables:
ONE: specificvarone
TWO: specificvartwo
before_script:
- custom pre step one
script:
- custom main step
after_script:
- custom post step one
final-stage-rebuild-index:
image: donotcare
tags: [donotcare]
""")
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outputfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
monkeypatch.setattr(spack.main, 'get_version',
lambda: '0.15.3-416-12ad69eb1')
ci_cmd('generate', '--output-file', outputfile)
with open(outputfile) as f:
contents = f.read()
print('generated contents: ')
print(contents)
yaml_contents = syaml.load(contents)
assert ('variables' in yaml_contents)
global_vars = yaml_contents['variables']
assert ('SPACK_VERSION' in global_vars)
assert (global_vars['SPACK_VERSION'] == '0.15.3-416-12ad69eb1')
assert ('SPACK_CHECKOUT_VERSION' in global_vars)
assert (global_vars['SPACK_CHECKOUT_VERSION'] == '12ad69eb1')
for ci_key in yaml_contents.keys():
if '(specs) b' in ci_key:
print('Should not have staged "b" w/out a match')
assert (False)
if '(specs) a' in ci_key:
# Make sure a's attributes override variables, and all the
# scripts. Also, make sure the 'toplevel' tag doesn't
# appear twice, but that a's specific extra tag does appear
the_elt = yaml_contents[ci_key]
assert (the_elt['variables']['ONE'] == 'specificvarone')
assert (the_elt['variables']['TWO'] == 'specificvartwo')
assert ('THREE' not in the_elt['variables'])
assert (len(the_elt['tags']) == 2)
assert ('specific-a' in the_elt['tags'])
assert ('toplevel' in the_elt['tags'])
assert (len(the_elt['before_script']) == 1)
assert (
the_elt['before_script'][0] == 'custom pre step one')
assert (len(the_elt['script']) == 1)
assert (the_elt['script'][0] == 'custom main step')
assert (len(the_elt['after_script']) == 1)
assert (
the_elt['after_script'][0] == 'custom post step one')
if '(specs) dependency-install' in ci_key:
# Since the dependency-install match omits any
# runner-attributes, make sure it inherited all the
# top-level attributes.
the_elt = yaml_contents[ci_key]
assert (the_elt['variables']['ONE'] == 'toplevelvarone')
assert (the_elt['variables']['TWO'] == 'toplevelvartwo')
assert ('THREE' not in the_elt['variables'])
assert (len(the_elt['tags']) == 1)
assert (the_elt['tags'][0] == 'toplevel')
assert (len(the_elt['before_script']) == 2)
assert (the_elt['before_script'][0] == 'pre step one')
assert (the_elt['before_script'][1] == 'pre step two')
assert (len(the_elt['script']) == 1)
assert (the_elt['script'][0] == 'main step')
assert (len(the_elt['after_script']) == 1)
assert (the_elt['after_script'][0] == 'post step one')
if '(specs) flatten-deps' in ci_key:
# The flatten-deps match specifies that we keep the two
# top level variables, but add a third specifc one. It
# also adds a custom tag which should be combined with
# the top-level tag.
the_elt = yaml_contents[ci_key]
assert (the_elt['variables']['ONE'] == 'toplevelvarone')
assert (the_elt['variables']['TWO'] == 'toplevelvartwo')
assert (
the_elt['variables']['THREE'] == 'specificvarthree')
assert (len(the_elt['tags']) == 2)
assert ('specific-one' in the_elt['tags'])
assert ('toplevel' in the_elt['tags'])
assert (len(the_elt['before_script']) == 2)
assert (the_elt['before_script'][0] == 'pre step one')
assert (the_elt['before_script'][1] == 'pre step two')
assert (len(the_elt['script']) == 1)
assert (the_elt['script'][0] == 'main step')
assert (len(the_elt['after_script']) == 1)
assert (the_elt['after_script'][0] == 'post step one')
def test_ci_generate_debug_with_custom_spack(tmpdir, mutable_mock_env_path,
env_deactivate, install_mockery,
mock_packages):
"""Make sure we generate cloning of spack in job script if needed"""
filename = str(tmpdir.join('spack.yaml'))
with open(filename, 'w') as f:
f.write("""\
spack:
specs:
- archive-files
mirrors:
some-mirror: https://my.fake.mirror
gitlab-ci:
enable-artifacts-buildcache: True
enable-debug-messages: True
mappings:
- match:
- archive-files
runner-attributes:
tags:
- donotcare
image: donotcare
""")
with tmpdir.as_cwd():
env_cmd('create', 'test', './spack.yaml')
outfile = str(tmpdir.join('.gitlab-ci.yml'))
with ev.read('test'):
spack_repo = 'https://github.com/usera/spack.git'
spack_ref = 'custom-branch'
expected_clone_str = 'git clone "{0}"'.format(spack_repo)
ci_cmd('generate', '--output-file', outfile, '--spack-repo',
spack_repo, '--spack-ref', spack_ref)
with open(outfile) as f:
contents = f.read()
yaml_contents = syaml.load(contents)
for ci_key in yaml_contents.keys():
if '(specs)' in ci_key:
next_job = yaml_contents[ci_key]
print(next_job)
assert ('before_script' in next_job)
before_script = next_job['before_script']
for step in before_script:
if expected_clone_str in step:
break
else:
msg = 'job "{0}" did not clone spack repo'.format(
ci_key)
print(msg)
assert (False)
assert ('script' in next_job)
script = next_job['script']
for step in script:
if 'spack -d ci rebuild' in step:
break
else:
msg = 'job {0} missing rebuild command'.format(
ci_key)
print(msg)
assert (False)