def restore_impl(name, the_collection):
# First unload all loaded modules
pymod.environ.unset(pymod.names.loaded_collection)
pymod.mc.purge(load_after_purge=False)
# clear the modulepath
orig_path = pymod.modulepath.path()
path = pymod.modulepath.Modulepath([])
pymod.modulepath.set_path(path)
# Now load the collection, one module at a time
for (directory, archives) in the_collection:
pymod.mc.use(directory, append=True)
for ar in archives:
try:
module = pymod.mc.unarchive_module(ar)
tty.verbose("Loading part of collection: {0}".format(module))
except pymod.error.ModuleNotFoundError:
raise pymod.error.CollectionModuleNotFoundError(
ar["fullname"], ar["filename"])
pymod.mc.load_impl(module)
module.acquired_as = module.fullname
assert module.is_loaded
pymod.environ.set(pymod.names.loaded_collection, name)
for p in orig_path:
if not pymod.modulepath.contains(p): # pragma: no cover
pymod.mc.use(p, append=True)
return None
def factory(root, path):
filename = os.path.join(root, path)
if not os.path.isfile(filename): # pragma: no cover
tty.verbose("{0} does not exist".format(filename))
return None
elif filename.endswith(("~", )) or filename.startswith(
(".#", )): # pragma: no cover
# Don't read backup files
return None
if filename.endswith(".py"):
module_type = PyModule
elif is_tcl_module(filename):
module_type = TclModule
else:
return None
module = module_type(root, path)
if pymod.config.get("debug"): # pragma: no cover
if module_type == TclModule and "gcc" in filename:
tty.debug(module.name)
tty.debug(module.modulepath)
tty.debug(module.filename, "\n")
return module
def on_install_success(spec):
"""On the success of an install (after everything is complete)
"""
if not spack.monitor.cli:
return
tty.debug("Running on_install_success for %s" % spec)
result = spack.monitor.cli.update_build(spec, status="SUCCESS")
tty.verbose(result.get('message'))
def on_install_failure(spec):
"""Triggered on failure of an install
"""
if not spack.monitor.cli:
return
tty.debug("Running on_install_failure for %s" % spec)
result = spack.monitor.cli.fail_task(spec)
tty.verbose(result.get('message'))
def on_phase_success(pkg, phase_name, log_file):
"""Triggered on a phase success
"""
if not spack.monitor.cli:
return
tty.debug("Running on_phase_success %s, phase %s" % (pkg.name, phase_name))
result = spack.monitor.cli.send_phase(pkg, phase_name, log_file, "SUCCESS")
tty.verbose(result.get('message'))
def on_phase_error(pkg, phase_name, log_file):
"""Triggered on a phase error
"""
if not spack.monitor.cli:
return
tty.debug("Running on_phase_error %s, phase %s" % (pkg.name, phase_name))
result = spack.monitor.cli.send_phase(pkg, phase_name, log_file, "ERROR")
tty.verbose(result.get('message'))
def on_install_start(spec):
"""On start of an install, we want to ping the server if it exists
"""
if not spack.monitor.cli:
return
tty.debug("Running on_install_start for %s" % spec)
build_id = spack.monitor.cli.new_build(spec)
tty.verbose("Build created with id %s" % build_id)
def pop_marked_default(dirname, versions):
dirname = os.path.realpath(dirname)
assert os.path.isdir(dirname)
linked_default = pop_linked_default(dirname, versions)
versioned_default = pop_versioned_default(dirname, versions)
if linked_default and versioned_default:
tty.verbose("A linked and versioned default exist in {0}, "
"choosing the linked".format(dirname))
return linked_default
return linked_default or versioned_default
def refresh():
"""Unload all modules from environment and reload them"""
loaded_modules = pymod.mc.get_loaded_modules()
for module in loaded_modules[::-1]:
tty.verbose("Unloading {0}".format(module))
if module.is_loaded:
pymod.mc.unload_impl(module)
for module in loaded_modules:
tty.verbose("Loading {0}".format(module))
if not module.is_loaded:
pymod.mc.load_impl(module)
def append_path(self, dirname):
dirname = Path.expand_name(dirname)
if dirname in self:
return
path = Path(dirname)
if not path.modules:
tty.verbose("No modules found in {0}".format(path.path))
return
self.path.append(path)
self.path_modified()
return path.modules
def mark_explicit_defaults(modules, defaults):
for (name, filename) in defaults.items():
mods = modules.get(name)
if mods is None:
tty.debug("There is no module named {0}".format(name))
continue
for module in mods:
if os.path.realpath(module.filename) == os.path.realpath(filename):
module.marked_as_default = True
break
else:
tty.verbose(
"No matching module to mark default for {0}".format(name))
def pop_versioned_default(dirname, files):
"""TCL modules .version scheme"""
version_file_name = ".version"
try:
files.remove(version_file_name)
except ValueError:
return None
version_file = os.path.join(dirname, version_file_name)
version = read_tcl_default_version(version_file)
if version is None:
tty.verbose(
"Could not determine .version default in {0}".format(dirname))
else:
default_file = os.path.join(dirname, version)
if os.path.exists(default_file):
return default_file
tty.verbose(
"{0!r}: version default does not exist".format(default_file))
def __init__(self, args):
Reporter.__init__(self, args)
self.success = True
# Posixpath is used here to support the underlying template enginge
# Jinja2, which expects `/` path separators
self.template_dir = posixpath.join('reports', 'cdash')
self.cdash_upload_url = args.cdash_upload_url
if self.cdash_upload_url:
self.buildid_regexp = re.compile("<buildId>([0-9]+)</buildId>")
self.phase_regexp = re.compile(r"Executing phase: '(.*)'")
self.authtoken = None
if 'SPACK_CDASH_AUTH_TOKEN' in os.environ:
tty.verbose("Using CDash auth token from environment")
self.authtoken = os.environ.get('SPACK_CDASH_AUTH_TOKEN')
if getattr(args, 'spec', ''):
packages = args.spec
elif getattr(args, 'specs', ''):
packages = args.specs
else:
packages = []
for file in args.specfiles:
with open(file, 'r') as f:
s = spack.spec.Spec.from_yaml(f)
packages.append(s.format())
self.install_command = ' '.join(packages)
self.base_buildname = args.cdash_build or self.install_command
self.site = args.cdash_site or socket.gethostname()
self.osname = platform.system()
self.endtime = int(time.time())
if args.cdash_buildstamp:
self.buildstamp = args.cdash_buildstamp
else:
buildstamp_format = "%Y%m%d-%H%M-{0}".format(args.cdash_track)
self.buildstamp = time.strftime(buildstamp_format,
time.localtime(self.endtime))
self.buildIds = collections.OrderedDict()
self.revision = ''
git = which('git')
with working_dir(spack.paths.spack_root):
self.revision = git('rev-parse', 'HEAD', output=str).strip()
self.multiple_packages = False
def pop_linked_default(dirname, files):
"""Look for a file named `default` that is a symlink to a module file"""
linked_default_name = "default"
try:
files.remove(linked_default_name)
except ValueError:
return None
linked_default_file = os.path.join(dirname, linked_default_name)
if not os.path.islink(linked_default_file):
tty.verbose("Modulepath: expected file named `default` in {0} "
"to be a link to a modulefile".format(dirname))
return None
linked_default_source = os.path.realpath(linked_default_file)
if not os.path.dirname(linked_default_source) == dirname:
tty.verbose(
"Modulepath: expected file named `default` in {0} to be "
"a link to a modulefile in the same directory".format(dirname))
return None
return linked_default_source
def find_modules(directory):
directory = os.path.expanduser(directory)
if directory == "/":
tty.verbose("Requesting to find modules in root directory")
return None
if not os.access(directory, os.R_OK):
tty.verbose("{0!r} is not an accessible directory".format(directory))
return None
if not os.path.isdir(directory): # pragma: no cover
# This should be redundant because of the previous check
tty.verbose("{0!r} is not a directory".format(directory))
return None
return _find(directory)
def load(name, opts=None, insert_at=None, caller="command_line"):
"""Load the module given by `name`
This is a higher level interface to `load_impl` that gets the actual module
object from `name`
Parameters
----------
name : string_like
Module name, full name, or file path
opts : dict
(Optional) options to send to module
insert_at : int
Load the module as the `insert_at`th module.
caller : str
Who is calling. If modulefile, the reference count will be incremented
if the module is already loaded.
Returns
-------
module : Module
If the `name` was loaded (or is already loaded), return its module.
Raises
------
ModuleNotFoundError
"""
tty.verbose("Loading {0}".format(name))
# Execute the module
module = pymod.modulepath.get(name, use_file_modulepath=True) # caller=="command_line")
if module is None:
if caller == "command_line":
collection = pymod.collection.get(name)
if collection is not None:
return pymod.mc.collection.restore_impl(name, collection)
raise ModuleNotFoundError(name, mp=pymod.modulepath)
# Set the command line options
if opts:
module.opts = opts
if module.is_loaded:
if caller == "modulefile":
pymod.mc.increment_refcount(module)
else:
tty.warn(
"{0} is already loaded, use 'module reload' to reload".format(
module.fullname
)
)
return module
if pymod.environ.get(pymod.names.loaded_collection): # pragma: no cover
collection = pymod.environ.get(pymod.names.loaded_collection)
tty.debug(
"Loading {0} on top of loaded collection {1}. "
"Removing the collection name from the environment".format(
module.fullname, collection
)
)
pymod.environ.unset(pymod.names.loaded_collection)
if insert_at is not None:
load_inserted_impl(module, insert_at)
else:
load_impl(module)
return module
def generate_gitlab_ci_yaml(env,
print_summary,
output_file,
custom_spack_repo=None,
custom_spack_ref=None):
# FIXME: What's the difference between one that opens with 'spack'
# and one that opens with 'env'? This will only handle the former.
with spack.concretize.disable_compiler_existence_check():
env.concretize()
yaml_root = ev.config_dict(env.yaml)
if 'gitlab-ci' not in yaml_root:
tty.die('Environment yaml does not have "gitlab-ci" section')
gitlab_ci = yaml_root['gitlab-ci']
ci_mappings = gitlab_ci['mappings']
final_job_config = None
if 'final-stage-rebuild-index' in gitlab_ci:
final_job_config = gitlab_ci['final-stage-rebuild-index']
build_group = None
enable_cdash_reporting = False
cdash_auth_token = None
if 'cdash' in yaml_root:
enable_cdash_reporting = True
ci_cdash = yaml_root['cdash']
build_group = ci_cdash['build-group']
cdash_url = ci_cdash['url']
cdash_project = ci_cdash['project']
cdash_site = ci_cdash['site']
if 'SPACK_CDASH_AUTH_TOKEN' in os.environ:
tty.verbose("Using CDash auth token from environment")
cdash_auth_token = os.environ.get('SPACK_CDASH_AUTH_TOKEN')
# Make sure we use a custom spack if necessary
before_script = None
after_script = None
if custom_spack_repo:
if not custom_spack_ref:
custom_spack_ref = 'master'
before_script = [
('git clone "{0}" --branch "{1}" --depth 1 '
'--single-branch'.format(custom_spack_repo, custom_spack_ref)),
# Next line just shows spack version in pipeline output
'pushd ./spack && git rev-parse HEAD && popd',
'. "./spack/share/spack/setup-env.sh"',
]
after_script = ['rm -rf "./spack"']
ci_mirrors = yaml_root['mirrors']
mirror_urls = [url for url in ci_mirrors.values()]
enable_artifacts_buildcache = False
if 'enable-artifacts-buildcache' in gitlab_ci:
enable_artifacts_buildcache = gitlab_ci['enable-artifacts-buildcache']
bootstrap_specs = []
phases = []
if 'bootstrap' in gitlab_ci:
for phase in gitlab_ci['bootstrap']:
try:
phase_name = phase.get('name')
strip_compilers = phase.get('compiler-agnostic')
except AttributeError:
phase_name = phase
strip_compilers = False
phases.append({
'name': phase_name,
'strip-compilers': strip_compilers,
})
for bs in env.spec_lists[phase_name]:
bootstrap_specs.append({
'spec': bs,
'phase-name': phase_name,
'strip-compilers': strip_compilers,
})
phases.append({
'name': 'specs',
'strip-compilers': False,
})
staged_phases = {}
for phase in phases:
phase_name = phase['name']
with spack.concretize.disable_compiler_existence_check():
staged_phases[phase_name] = stage_spec_jobs(
env.spec_lists[phase_name])
if print_summary:
for phase in phases:
phase_name = phase['name']
tty.msg('Stages for phase "{0}"'.format(phase_name))
phase_stages = staged_phases[phase_name]
print_staging_summary(*phase_stages)
all_job_names = []
output_object = {}
job_id = 0
stage_id = 0
stage_names = []
for phase in phases:
phase_name = phase['name']
strip_compilers = phase['strip-compilers']
main_phase = is_main_phase(phase_name)
spec_labels, dependencies, stages = staged_phases[phase_name]
for stage_jobs in stages:
stage_name = 'stage-{0}'.format(stage_id)
stage_names.append(stage_name)
stage_id += 1
for spec_label in stage_jobs:
root_spec = spec_labels[spec_label]['rootSpec']
pkg_name = pkg_name_from_spec_label(spec_label)
release_spec = root_spec[pkg_name]
runner_attribs = find_matching_config(release_spec,
ci_mappings)
if not runner_attribs:
tty.warn('No match found for {0}, skipping it'.format(
release_spec))
continue
tags = [tag for tag in runner_attribs['tags']]
variables = {}
if 'variables' in runner_attribs:
variables.update(runner_attribs['variables'])
image_name = None
image_entry = None
if 'image' in runner_attribs:
build_image = runner_attribs['image']
try:
image_name = build_image.get('name')
entrypoint = build_image.get('entrypoint')
image_entry = [p for p in entrypoint]
except AttributeError:
image_name = build_image
osname = str(release_spec.architecture)
job_name = get_job_name(phase_name, strip_compilers,
release_spec, osname, build_group)
debug_flag = ''
if 'enable-debug-messages' in gitlab_ci:
debug_flag = '-d '
job_scripts = ['spack {0}ci rebuild'.format(debug_flag)]
compiler_action = 'NONE'
if len(phases) > 1:
compiler_action = 'FIND_ANY'
if is_main_phase(phase_name):
compiler_action = 'INSTALL_MISSING'
job_vars = {
'SPACK_ROOT_SPEC':
format_root_spec(root_spec, main_phase, strip_compilers),
'SPACK_JOB_SPEC_PKG_NAME':
release_spec.name,
'SPACK_COMPILER_ACTION':
compiler_action,
}
job_dependencies = []
if spec_label in dependencies:
for dep_label in dependencies[spec_label]:
dep_pkg = pkg_name_from_spec_label(dep_label)
dep_spec = spec_labels[dep_label]['rootSpec'][dep_pkg]
dep_job_name = get_job_name(phase_name,
strip_compilers, dep_spec,
osname, build_group)
job_dependencies.append(dep_job_name)
# This next section helps gitlab make sure the right
# bootstrapped compiler exists in the artifacts buildcache by
# creating an artificial dependency between this spec and its
# compiler. So, if we are in the main phase, and if the
# compiler we are supposed to use is listed in any of the
# bootstrap spec lists, then we will add one more dependency to
# "job_dependencies" (that compiler).
if is_main_phase(phase_name):
compiler_pkg_spec = compilers.pkg_spec_for_compiler(
release_spec.compiler)
for bs in bootstrap_specs:
bs_arch = bs['spec'].architecture
if (bs['spec'].satisfies(compiler_pkg_spec)
and bs_arch == release_spec.architecture):
c_job_name = get_job_name(bs['phase-name'],
bs['strip-compilers'],
bs['spec'], str(bs_arch),
build_group)
job_dependencies.append(c_job_name)
if enable_cdash_reporting:
cdash_build_name = get_cdash_build_name(
release_spec, build_group)
all_job_names.append(cdash_build_name)
related_builds = [] # Used for relating CDash builds
if spec_label in dependencies:
related_builds = ([
spec_labels[d]['spec'].name
for d in dependencies[spec_label]
])
job_vars['SPACK_CDASH_BUILD_NAME'] = cdash_build_name
job_vars['SPACK_RELATED_BUILDS_CDASH'] = ';'.join(
related_builds)
variables.update(job_vars)
artifact_paths = [
'jobs_scratch_dir',
'cdash_report',
]
if enable_artifacts_buildcache:
artifact_paths.append('local_mirror/build_cache')
job_object = {
'stage': stage_name,
'variables': variables,
'script': job_scripts,
'tags': tags,
'artifacts': {
'paths': artifact_paths,
'when': 'always',
},
'dependencies': job_dependencies,
}
if before_script:
job_object['before_script'] = before_script
if after_script:
job_object['after_script'] = after_script
if image_name:
job_object['image'] = image_name
if image_entry is not None:
job_object['image'] = {
'name': image_name,
'entrypoint': image_entry,
}
output_object[job_name] = job_object
job_id += 1
tty.debug('{0} build jobs generated in {1} stages'.format(
job_id, stage_id))
# Use "all_job_names" to populate the build group for this set
if enable_cdash_reporting and cdash_auth_token:
try:
populate_buildgroup(all_job_names, build_group, cdash_project,
cdash_site, cdash_auth_token, cdash_url)
except (SpackError, HTTPError, URLError) as err:
tty.warn('Problem populating buildgroup: {0}'.format(err))
else:
tty.warn('Unable to populate buildgroup without CDash credentials')
if final_job_config:
# Add an extra, final job to regenerate the index
final_stage = 'stage-rebuild-index'
final_job = {
'stage': final_stage,
'script':
'spack buildcache update-index -d {0}'.format(mirror_urls[0]),
'tags': final_job_config['tags']
}
if 'image' in final_job_config:
final_job['image'] = final_job_config['image']
if before_script:
final_job['before_script'] = before_script
if after_script:
final_job['after_script'] = after_script
output_object['rebuild-index'] = final_job
stage_names.append(final_stage)
output_object['stages'] = stage_names
with open(output_file, 'w') as outf:
outf.write(syaml.dump_config(output_object, default_flow_style=True))
def generate_gitlab_ci_yaml(env,
print_summary,
output_file,
run_optimizer=False,
use_dependencies=False):
# FIXME: What's the difference between one that opens with 'spack'
# and one that opens with 'env'? This will only handle the former.
with spack.concretize.disable_compiler_existence_check():
env.concretize()
yaml_root = ev.config_dict(env.yaml)
if 'gitlab-ci' not in yaml_root:
tty.die('Environment yaml does not have "gitlab-ci" section')
gitlab_ci = yaml_root['gitlab-ci']
final_job_config = None
if 'final-stage-rebuild-index' in gitlab_ci:
final_job_config = gitlab_ci['final-stage-rebuild-index']
build_group = None
enable_cdash_reporting = False
cdash_auth_token = None
if 'cdash' in yaml_root:
enable_cdash_reporting = True
ci_cdash = yaml_root['cdash']
build_group = ci_cdash['build-group']
cdash_url = ci_cdash['url']
cdash_project = ci_cdash['project']
cdash_site = ci_cdash['site']
if 'SPACK_CDASH_AUTH_TOKEN' in os.environ:
tty.verbose("Using CDash auth token from environment")
cdash_auth_token = os.environ.get('SPACK_CDASH_AUTH_TOKEN')
is_pr_pipeline = (os.environ.get('SPACK_IS_PR_PIPELINE',
'').lower() == 'true')
ci_mirrors = yaml_root['mirrors']
mirror_urls = [url for url in ci_mirrors.values()]
enable_artifacts_buildcache = False
if 'enable-artifacts-buildcache' in gitlab_ci:
enable_artifacts_buildcache = gitlab_ci['enable-artifacts-buildcache']
bootstrap_specs = []
phases = []
if 'bootstrap' in gitlab_ci:
for phase in gitlab_ci['bootstrap']:
try:
phase_name = phase.get('name')
strip_compilers = phase.get('compiler-agnostic')
except AttributeError:
phase_name = phase
strip_compilers = False
phases.append({
'name': phase_name,
'strip-compilers': strip_compilers,
})
for bs in env.spec_lists[phase_name]:
bootstrap_specs.append({
'spec': bs,
'phase-name': phase_name,
'strip-compilers': strip_compilers,
})
phases.append({
'name': 'specs',
'strip-compilers': False,
})
staged_phases = {}
for phase in phases:
phase_name = phase['name']
with spack.concretize.disable_compiler_existence_check():
staged_phases[phase_name] = stage_spec_jobs(
env.spec_lists[phase_name])
if print_summary:
for phase in phases:
phase_name = phase['name']
tty.msg('Stages for phase "{0}"'.format(phase_name))
phase_stages = staged_phases[phase_name]
print_staging_summary(*phase_stages)
all_job_names = []
output_object = {}
job_id = 0
stage_id = 0
stage_names = []
max_length_needs = 0
max_needs_job = ''
for phase in phases:
phase_name = phase['name']
strip_compilers = phase['strip-compilers']
main_phase = is_main_phase(phase_name)
spec_labels, dependencies, stages = staged_phases[phase_name]
for stage_jobs in stages:
stage_name = 'stage-{0}'.format(stage_id)
stage_names.append(stage_name)
stage_id += 1
for spec_label in stage_jobs:
root_spec = spec_labels[spec_label]['rootSpec']
pkg_name = pkg_name_from_spec_label(spec_label)
release_spec = root_spec[pkg_name]
runner_attribs = find_matching_config(release_spec, gitlab_ci)
if not runner_attribs:
tty.warn('No match found for {0}, skipping it'.format(
release_spec))
continue
tags = [tag for tag in runner_attribs['tags']]
variables = {}
if 'variables' in runner_attribs:
variables.update(runner_attribs['variables'])
image_name = None
image_entry = None
if 'image' in runner_attribs:
build_image = runner_attribs['image']
try:
image_name = build_image.get('name')
entrypoint = build_image.get('entrypoint')
image_entry = [p for p in entrypoint]
except AttributeError:
image_name = build_image
job_script = [
'spack env activate --without-view .',
'spack ci rebuild',
]
if 'script' in runner_attribs:
job_script = [s for s in runner_attribs['script']]
before_script = None
if 'before_script' in runner_attribs:
before_script = [
s for s in runner_attribs['before_script']
]
after_script = None
if 'after_script' in runner_attribs:
after_script = [s for s in runner_attribs['after_script']]
osname = str(release_spec.architecture)
job_name = get_job_name(phase_name, strip_compilers,
release_spec, osname, build_group)
compiler_action = 'NONE'
if len(phases) > 1:
compiler_action = 'FIND_ANY'
if is_main_phase(phase_name):
compiler_action = 'INSTALL_MISSING'
job_vars = {
'SPACK_ROOT_SPEC':
format_root_spec(root_spec, main_phase, strip_compilers),
'SPACK_JOB_SPEC_PKG_NAME':
release_spec.name,
'SPACK_COMPILER_ACTION':
compiler_action,
'SPACK_IS_PR_PIPELINE':
str(is_pr_pipeline),
}
job_dependencies = []
if spec_label in dependencies:
if enable_artifacts_buildcache:
dep_jobs = [
d for d in release_spec.traverse(deptype=all,
root=False)
]
else:
dep_jobs = []
for dep_label in dependencies[spec_label]:
dep_pkg = pkg_name_from_spec_label(dep_label)
dep_root = spec_labels[dep_label]['rootSpec']
dep_jobs.append(dep_root[dep_pkg])
job_dependencies.extend(
format_job_needs(phase_name, strip_compilers, dep_jobs,
osname, build_group,
enable_artifacts_buildcache))
# This next section helps gitlab make sure the right
# bootstrapped compiler exists in the artifacts buildcache by
# creating an artificial dependency between this spec and its
# compiler. So, if we are in the main phase, and if the
# compiler we are supposed to use is listed in any of the
# bootstrap spec lists, then we will add more dependencies to
# the job (that compiler and maybe it's dependencies as well).
if is_main_phase(phase_name):
compiler_pkg_spec = compilers.pkg_spec_for_compiler(
release_spec.compiler)
for bs in bootstrap_specs:
bs_arch = bs['spec'].architecture
if (bs['spec'].satisfies(compiler_pkg_spec)
and bs_arch == release_spec.architecture):
# We found the bootstrap compiler this release spec
# should be built with, so for DAG scheduling
# purposes, we will at least add the compiler spec
# to the jobs "needs". But if artifact buildcache
# is enabled, we'll have to add all transtive deps
# of the compiler as well.
dep_jobs = [bs['spec']]
if enable_artifacts_buildcache:
dep_jobs = [
d for d in bs['spec'].traverse(deptype=all)
]
job_dependencies.extend(
format_job_needs(bs['phase-name'],
bs['strip-compilers'],
dep_jobs, str(bs_arch),
build_group,
enable_artifacts_buildcache))
if enable_cdash_reporting:
cdash_build_name = get_cdash_build_name(
release_spec, build_group)
all_job_names.append(cdash_build_name)
related_builds = [] # Used for relating CDash builds
if spec_label in dependencies:
related_builds = ([
spec_labels[d]['spec'].name
for d in dependencies[spec_label]
])
job_vars['SPACK_CDASH_BUILD_NAME'] = cdash_build_name
job_vars['SPACK_RELATED_BUILDS_CDASH'] = ';'.join(
sorted(related_builds))
variables.update(job_vars)
artifact_paths = [
'jobs_scratch_dir',
'cdash_report',
]
if enable_artifacts_buildcache:
bc_root = 'local_mirror/build_cache'
artifact_paths.extend([
os.path.join(bc_root, p) for p in [
bindist.tarball_name(release_spec, '.spec.yaml'),
bindist.tarball_name(release_spec, '.cdashid'),
bindist.tarball_directory_name(release_spec),
]
])
job_object = {
'stage': stage_name,
'variables': variables,
'script': job_script,
'tags': tags,
'artifacts': {
'paths': artifact_paths,
'when': 'always',
},
'needs': sorted(job_dependencies, key=lambda d: d['job']),
'retry': {
'max': 2,
'when': JOB_RETRY_CONDITIONS,
}
}
length_needs = len(job_dependencies)
if length_needs > max_length_needs:
max_length_needs = length_needs
max_needs_job = job_name
if before_script:
job_object['before_script'] = before_script
if after_script:
job_object['after_script'] = after_script
if image_name:
job_object['image'] = image_name
if image_entry is not None:
job_object['image'] = {
'name': image_name,
'entrypoint': image_entry,
}
output_object[job_name] = job_object
job_id += 1
tty.debug('{0} build jobs generated in {1} stages'.format(
job_id, stage_id))
tty.debug('The max_needs_job is {0}, with {1} needs'.format(
max_needs_job, max_length_needs))
# Use "all_job_names" to populate the build group for this set
if enable_cdash_reporting and cdash_auth_token:
try:
populate_buildgroup(all_job_names, build_group, cdash_project,
cdash_site, cdash_auth_token, cdash_url)
except (SpackError, HTTPError, URLError) as err:
tty.warn('Problem populating buildgroup: {0}'.format(err))
else:
tty.warn('Unable to populate buildgroup without CDash credentials')
if final_job_config and not is_pr_pipeline:
# Add an extra, final job to regenerate the index
final_stage = 'stage-rebuild-index'
final_job = {
'stage':
final_stage,
'script':
'spack buildcache update-index --keys -d {0}'.format(
mirror_urls[0]),
'tags':
final_job_config['tags'],
'when':
'always'
}
if 'image' in final_job_config:
final_job['image'] = final_job_config['image']
if before_script:
final_job['before_script'] = before_script
if after_script:
final_job['after_script'] = after_script
output_object['rebuild-index'] = final_job
stage_names.append(final_stage)
output_object['stages'] = stage_names
# Capture the version of spack used to generate the pipeline, transform it
# into a value that can be passed to "git checkout", and save it in a
# global yaml variable
spack_version = spack.main.get_version()
version_to_clone = None
v_match = re.match(r"^\d+\.\d+\.\d+$", spack_version)
if v_match:
version_to_clone = 'v{0}'.format(v_match.group(0))
else:
v_match = re.match(r"^[^-]+-[^-]+-([a-f\d]+)$", spack_version)
if v_match:
version_to_clone = v_match.group(1)
else:
version_to_clone = spack_version
output_object['variables'] = {
'SPACK_VERSION': spack_version,
'SPACK_CHECKOUT_VERSION': version_to_clone,
}
sorted_output = {}
for output_key, output_value in sorted(output_object.items()):
sorted_output[output_key] = output_value
# TODO(opadron): remove this or refactor
if run_optimizer:
import spack.ci_optimization as ci_opt
sorted_output = ci_opt.optimizer(sorted_output)
# TODO(opadron): remove this or refactor
if use_dependencies:
import spack.ci_needs_workaround as cinw
sorted_output = cinw.needs_to_dependencies(sorted_output)
with open(output_file, 'w') as outf:
outf.write(syaml.dump_config(sorted_output, default_flow_style=True))
def _log_command_output(self, out):
if tty.is_verbose():
fmt = self.command_name + ': {0}'
for ln in out.getvalue().split('\n'):
if len(ln) > 0:
tty.verbose(fmt.format(ln.replace('==> ', '')))
def generate_gitlab_ci_yaml(env,
print_summary,
output_file,
prune_dag=False,
check_index_only=False,
run_optimizer=False,
use_dependencies=False):
# FIXME: What's the difference between one that opens with 'spack'
# and one that opens with 'env'? This will only handle the former.
with spack.concretize.disable_compiler_existence_check():
env.concretize()
yaml_root = ev.config_dict(env.yaml)
if 'gitlab-ci' not in yaml_root:
tty.die('Environment yaml does not have "gitlab-ci" section')
gitlab_ci = yaml_root['gitlab-ci']
build_group = None
enable_cdash_reporting = False
cdash_auth_token = None
if 'cdash' in yaml_root:
enable_cdash_reporting = True
ci_cdash = yaml_root['cdash']
build_group = ci_cdash['build-group']
cdash_url = ci_cdash['url']
cdash_project = ci_cdash['project']
cdash_site = ci_cdash['site']
if 'SPACK_CDASH_AUTH_TOKEN' in os.environ:
tty.verbose("Using CDash auth token from environment")
cdash_auth_token = os.environ.get('SPACK_CDASH_AUTH_TOKEN')
is_pr_pipeline = (os.environ.get('SPACK_IS_PR_PIPELINE',
'').lower() == 'true')
spack_pr_branch = os.environ.get('SPACK_PR_BRANCH', None)
pr_mirror_url = None
if spack_pr_branch:
pr_mirror_url = url_util.join(SPACK_PR_MIRRORS_ROOT_URL,
spack_pr_branch)
if 'mirrors' not in yaml_root or len(yaml_root['mirrors'].values()) < 1:
tty.die('spack ci generate requires an env containing a mirror')
ci_mirrors = yaml_root['mirrors']
mirror_urls = [url for url in ci_mirrors.values()]
enable_artifacts_buildcache = False
if 'enable-artifacts-buildcache' in gitlab_ci:
enable_artifacts_buildcache = gitlab_ci['enable-artifacts-buildcache']
rebuild_index_enabled = True
if 'rebuild-index' in gitlab_ci and gitlab_ci['rebuild-index'] is False:
rebuild_index_enabled = False
temp_storage_url_prefix = None
if 'temporary-storage-url-prefix' in gitlab_ci:
temp_storage_url_prefix = gitlab_ci['temporary-storage-url-prefix']
bootstrap_specs = []
phases = []
if 'bootstrap' in gitlab_ci:
for phase in gitlab_ci['bootstrap']:
try:
phase_name = phase.get('name')
strip_compilers = phase.get('compiler-agnostic')
except AttributeError:
phase_name = phase
strip_compilers = False
phases.append({
'name': phase_name,
'strip-compilers': strip_compilers,
})
for bs in env.spec_lists[phase_name]:
bootstrap_specs.append({
'spec': bs,
'phase-name': phase_name,
'strip-compilers': strip_compilers,
})
phases.append({
'name': 'specs',
'strip-compilers': False,
})
# Add this mirror if it's enabled, as some specs might be up to date
# here and thus not need to be rebuilt.
if pr_mirror_url:
add_pr_mirror(pr_mirror_url)
# Speed up staging by first fetching binary indices from all mirrors
# (including the per-PR mirror we may have just added above).
bindist.binary_index.update()
staged_phases = {}
try:
for phase in phases:
phase_name = phase['name']
with spack.concretize.disable_compiler_existence_check():
staged_phases[phase_name] = stage_spec_jobs(
env.spec_lists[phase_name],
check_index_only=check_index_only)
finally:
# Clean up PR mirror if enabled
if pr_mirror_url:
remove_pr_mirror()
all_job_names = []
output_object = {}
job_id = 0
stage_id = 0
stage_names = []
max_length_needs = 0
max_needs_job = ''
before_script, after_script = None, None
for phase in phases:
phase_name = phase['name']
strip_compilers = phase['strip-compilers']
main_phase = is_main_phase(phase_name)
spec_labels, dependencies, stages = staged_phases[phase_name]
for stage_jobs in stages:
stage_name = 'stage-{0}'.format(stage_id)
stage_names.append(stage_name)
stage_id += 1
for spec_label in stage_jobs:
spec_record = spec_labels[spec_label]
root_spec = spec_record['rootSpec']
pkg_name = pkg_name_from_spec_label(spec_label)
release_spec = root_spec[pkg_name]
runner_attribs = find_matching_config(release_spec, gitlab_ci)
if not runner_attribs:
tty.warn('No match found for {0}, skipping it'.format(
release_spec))
continue
tags = [tag for tag in runner_attribs['tags']]
variables = {}
if 'variables' in runner_attribs:
variables.update(runner_attribs['variables'])
image_name = None
image_entry = None
if 'image' in runner_attribs:
build_image = runner_attribs['image']
try:
image_name = build_image.get('name')
entrypoint = build_image.get('entrypoint')
image_entry = [p for p in entrypoint]
except AttributeError:
image_name = build_image
job_script = [
'spack env activate --without-view .',
'spack ci rebuild',
]
if 'script' in runner_attribs:
job_script = [s for s in runner_attribs['script']]
before_script = None
if 'before_script' in runner_attribs:
before_script = [
s for s in runner_attribs['before_script']
]
after_script = None
if 'after_script' in runner_attribs:
after_script = [s for s in runner_attribs['after_script']]
osname = str(release_spec.architecture)
job_name = get_job_name(phase_name, strip_compilers,
release_spec, osname, build_group)
compiler_action = 'NONE'
if len(phases) > 1:
compiler_action = 'FIND_ANY'
if is_main_phase(phase_name):
compiler_action = 'INSTALL_MISSING'
job_vars = {
'SPACK_ROOT_SPEC':
format_root_spec(root_spec, main_phase, strip_compilers),
'SPACK_JOB_SPEC_PKG_NAME':
release_spec.name,
'SPACK_COMPILER_ACTION':
compiler_action,
'SPACK_IS_PR_PIPELINE':
str(is_pr_pipeline),
}
job_dependencies = []
if spec_label in dependencies:
if enable_artifacts_buildcache:
# Get dependencies transitively, so they're all
# available in the artifacts buildcache.
dep_jobs = [
d for d in release_spec.traverse(deptype=all,
root=False)
]
else:
# In this case, "needs" is only used for scheduling
# purposes, so we only get the direct dependencies.
dep_jobs = []
for dep_label in dependencies[spec_label]:
dep_pkg = pkg_name_from_spec_label(dep_label)
dep_root = spec_labels[dep_label]['rootSpec']
dep_jobs.append(dep_root[dep_pkg])
job_dependencies.extend(
format_job_needs(phase_name, strip_compilers, dep_jobs,
osname, build_group, prune_dag,
spec_labels,
enable_artifacts_buildcache))
rebuild_spec = spec_record['needs_rebuild']
# This next section helps gitlab make sure the right
# bootstrapped compiler exists in the artifacts buildcache by
# creating an artificial dependency between this spec and its
# compiler. So, if we are in the main phase, and if the
# compiler we are supposed to use is listed in any of the
# bootstrap spec lists, then we will add more dependencies to
# the job (that compiler and maybe it's dependencies as well).
if is_main_phase(phase_name):
spec_arch_family = (release_spec.architecture.target.
microarchitecture.family)
compiler_pkg_spec = compilers.pkg_spec_for_compiler(
release_spec.compiler)
for bs in bootstrap_specs:
c_spec = bs['spec']
bs_arch = c_spec.architecture
bs_arch_family = (
bs_arch.target.microarchitecture.family)
if (c_spec.satisfies(compiler_pkg_spec)
and bs_arch_family == spec_arch_family):
# We found the bootstrap compiler this release spec
# should be built with, so for DAG scheduling
# purposes, we will at least add the compiler spec
# to the jobs "needs". But if artifact buildcache
# is enabled, we'll have to add all transtive deps
# of the compiler as well.
# Here we check whether the bootstrapped compiler
# needs to be rebuilt. Until compilers are proper
# dependencies, we artificially force the spec to
# be rebuilt if the compiler targeted to build it
# needs to be rebuilt.
bs_specs, _, _ = staged_phases[bs['phase-name']]
c_spec_key = spec_deps_key(c_spec)
rbld_comp = bs_specs[c_spec_key]['needs_rebuild']
rebuild_spec = rebuild_spec or rbld_comp
# Also update record so dependents do not fail to
# add this spec to their "needs"
spec_record['needs_rebuild'] = rebuild_spec
dep_jobs = [c_spec]
if enable_artifacts_buildcache:
dep_jobs = [
d for d in c_spec.traverse(deptype=all)
]
job_dependencies.extend(
format_job_needs(bs['phase-name'],
bs['strip-compilers'],
dep_jobs, str(bs_arch),
build_group, prune_dag,
bs_specs,
enable_artifacts_buildcache))
else:
debug_msg = ''.join([
'Considered compiler {0} for spec ',
'{1}, but rejected it either because it was ',
'not the compiler required by the spec, or ',
'because the target arch families of the ',
'spec and the compiler did not match'
]).format(c_spec, release_spec)
tty.debug(debug_msg)
if prune_dag and not rebuild_spec:
continue
job_vars['SPACK_SPEC_NEEDS_REBUILD'] = str(rebuild_spec)
if enable_cdash_reporting:
cdash_build_name = get_cdash_build_name(
release_spec, build_group)
all_job_names.append(cdash_build_name)
related_builds = [] # Used for relating CDash builds
if spec_label in dependencies:
related_builds = ([
spec_labels[d]['spec'].name
for d in dependencies[spec_label]
])
job_vars['SPACK_CDASH_BUILD_NAME'] = cdash_build_name
job_vars['SPACK_RELATED_BUILDS_CDASH'] = ';'.join(
sorted(related_builds))
variables.update(job_vars)
artifact_paths = [
'jobs_scratch_dir',
'cdash_report',
]
if enable_artifacts_buildcache:
bc_root = 'local_mirror/build_cache'
artifact_paths.extend([
os.path.join(bc_root, p) for p in [
bindist.tarball_name(release_spec, '.spec.yaml'),
bindist.tarball_name(release_spec, '.cdashid'),
bindist.tarball_directory_name(release_spec),
]
])
job_object = {
'stage': stage_name,
'variables': variables,
'script': job_script,
'tags': tags,
'artifacts': {
'paths': artifact_paths,
'when': 'always',
},
'needs': sorted(job_dependencies, key=lambda d: d['job']),
'retry': {
'max': 2,
'when': JOB_RETRY_CONDITIONS,
},
'interruptible': True
}
length_needs = len(job_dependencies)
if length_needs > max_length_needs:
max_length_needs = length_needs
max_needs_job = job_name
if before_script:
job_object['before_script'] = before_script
if after_script:
job_object['after_script'] = after_script
if image_name:
job_object['image'] = image_name
if image_entry is not None:
job_object['image'] = {
'name': image_name,
'entrypoint': image_entry,
}
output_object[job_name] = job_object
job_id += 1
if print_summary:
for phase in phases:
phase_name = phase['name']
tty.msg('Stages for phase "{0}"'.format(phase_name))
phase_stages = staged_phases[phase_name]
print_staging_summary(*phase_stages)
tty.debug('{0} build jobs generated in {1} stages'.format(
job_id, stage_id))
if job_id > 0:
tty.debug('The max_needs_job is {0}, with {1} needs'.format(
max_needs_job, max_length_needs))
# Use "all_job_names" to populate the build group for this set
if enable_cdash_reporting and cdash_auth_token:
try:
populate_buildgroup(all_job_names, build_group, cdash_project,
cdash_site, cdash_auth_token, cdash_url)
except (SpackError, HTTPError, URLError) as err:
tty.warn('Problem populating buildgroup: {0}'.format(err))
else:
tty.warn('Unable to populate buildgroup without CDash credentials')
service_job_config = None
if 'service-job-attributes' in gitlab_ci:
service_job_config = gitlab_ci['service-job-attributes']
default_attrs = [
'image',
'tags',
'variables',
'before_script',
# 'script',
'after_script',
]
if job_id > 0:
if temp_storage_url_prefix:
# There were some rebuild jobs scheduled, so we will need to
# schedule a job to clean up the temporary storage location
# associated with this pipeline.
stage_names.append('cleanup-temp-storage')
cleanup_job = {}
if service_job_config:
copy_attributes(default_attrs, service_job_config, cleanup_job)
cleanup_job['stage'] = 'cleanup-temp-storage'
cleanup_job['script'] = [
'spack -d mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'.
format(temp_storage_url_prefix)
]
cleanup_job['when'] = 'always'
output_object['cleanup'] = cleanup_job
if rebuild_index_enabled:
# Add a final job to regenerate the index
stage_names.append('stage-rebuild-index')
final_job = {}
if service_job_config:
copy_attributes(default_attrs, service_job_config, final_job)
index_target_mirror = mirror_urls[0]
if is_pr_pipeline:
index_target_mirror = pr_mirror_url
final_job['stage'] = 'stage-rebuild-index'
final_job['script'] = [
'spack buildcache update-index --keys -d {0}'.format(
index_target_mirror)
]
final_job['when'] = 'always'
output_object['rebuild-index'] = final_job
output_object['stages'] = stage_names
# Capture the version of spack used to generate the pipeline, transform it
# into a value that can be passed to "git checkout", and save it in a
# global yaml variable
spack_version = spack.main.get_version()
version_to_clone = None
v_match = re.match(r"^\d+\.\d+\.\d+$", spack_version)
if v_match:
version_to_clone = 'v{0}'.format(v_match.group(0))
else:
v_match = re.match(r"^[^-]+-[^-]+-([a-f\d]+)$", spack_version)
if v_match:
version_to_clone = v_match.group(1)
else:
version_to_clone = spack_version
output_object['variables'] = {
'SPACK_VERSION': spack_version,
'SPACK_CHECKOUT_VERSION': version_to_clone,
}
if pr_mirror_url:
output_object['variables']['SPACK_PR_MIRROR_URL'] = pr_mirror_url
sorted_output = {}
for output_key, output_value in sorted(output_object.items()):
sorted_output[output_key] = output_value
# TODO(opadron): remove this or refactor
if run_optimizer:
import spack.ci_optimization as ci_opt
sorted_output = ci_opt.optimizer(sorted_output)
# TODO(opadron): remove this or refactor
if use_dependencies:
import spack.ci_needs_workaround as cinw
sorted_output = cinw.needs_to_dependencies(sorted_output)
else:
# No jobs were generated
tty.debug('No specs to rebuild, generating no-op job')
noop_job = {}
if service_job_config:
copy_attributes(default_attrs, service_job_config, noop_job)
if 'script' not in noop_job:
noop_job['script'] = [
'echo "All specs already up to date, nothing to rebuild."',
]
sorted_output = {'no-specs-to-rebuild': noop_job}
with open(output_file, 'w') as outf:
outf.write(syaml.dump_config(sorted_output, default_flow_style=True))
