def mirror_create(args): """Create a directory to be used as a spack mirror, and fill it with package archives.""" mirror_specs = _determine_specs_to_mirror(args) mirror = spack.mirror.Mirror(args.directory or spack.config.get('config:source_cache')) directory = url_util.format(mirror.push_url) existed = web_util.url_exists(directory) # Actually do the work to create the mirror present, mirrored, error = spack.mirror.create(directory, mirror_specs, args.skip_unstable_versions) p, m, e = len(present), len(mirrored), len(error) verb = "updated" if existed else "created" tty.msg("Successfully %s mirror in %s" % (verb, directory), "Archive stats:", " %-4d already present" % p, " %-4d added" % m, " %-4d failed to fetch." % e) if error: tty.error("Failed downloads:") colify(s.cformat("{name}{@version}") for s in error) sys.exit(1)
def get_keys(install=False, trust=False, force=False, mirrors=None): """Get pgp public keys available on mirror with suffix .pub """ mirror_collection = (mirrors or spack.mirror.MirrorCollection()) if not mirror_collection: tty.die("Please add a spack mirror to allow " + "download of build caches.") for mirror in mirror_collection.values(): fetch_url = mirror.fetch_url keys_url = url_util.join(fetch_url, _build_cache_relative_path, _build_cache_keys_relative_path) keys_index = url_util.join(keys_url, 'index.json') tty.debug('Finding public keys in {0}'.format( url_util.format(fetch_url))) try: _, _, json_file = web_util.read_from_url(keys_index) json_index = sjson.load(codecs.getreader('utf-8')(json_file)) except (URLError, web_util.SpackWebError) as url_err: if web_util.url_exists(keys_index): err_msg = [ 'Unable to find public keys in {0},', ' caught exception attempting to read from {1}.', ] tty.error(''.join(err_msg).format(url_util.format(fetch_url), url_util.format(keys_index))) tty.debug(url_err) continue for fingerprint, key_attributes in json_index['keys'].items(): link = os.path.join(keys_url, fingerprint + '.pub') with Stage(link, name="build_cache", keep=True) as stage: if os.path.exists(stage.save_filename) and force: os.remove(stage.save_filename) if not os.path.exists(stage.save_filename): try: stage.fetch() except fs.FetchError: continue tty.debug('Found key {0}'.format(fingerprint)) if install: if trust: spack.util.gpg.trust(stage.save_filename) tty.debug('Added this key to trusted keys.') else: tty.debug('Will not add this key to trusted keys.' 'Use -t to install all downloaded keys')
def ci_rebuild(args): """Check a single spec against the remote mirror, and rebuild it from source if the mirror does not contain the full hash match of the spec as computed locally. """ env = ev.get_env(args, 'ci rebuild', required=True) # Make sure the environment is "gitlab-enabled", or else there's nothing # to do. yaml_root = ev.config_dict(env.yaml) gitlab_ci = None if 'gitlab-ci' in yaml_root: gitlab_ci = yaml_root['gitlab-ci'] if not gitlab_ci: tty.die('spack ci rebuild requires an env containing gitlab-ci cfg') # Grab the environment variables we need. These either come from the # pipeline generation step ("spack ci generate"), where they were written # out as variables, or else provided by GitLab itself. pipeline_artifacts_dir = get_env_var('SPACK_ARTIFACTS_ROOT') job_log_dir = get_env_var('SPACK_JOB_LOG_DIR') repro_dir = get_env_var('SPACK_JOB_REPRO_DIR') local_mirror_dir = get_env_var('SPACK_LOCAL_MIRROR_DIR') concrete_env_dir = get_env_var('SPACK_CONCRETE_ENV_DIR') ci_pipeline_id = get_env_var('CI_PIPELINE_ID') ci_job_name = get_env_var('CI_JOB_NAME') signing_key = get_env_var('SPACK_SIGNING_KEY') root_spec = get_env_var('SPACK_ROOT_SPEC') job_spec_pkg_name = get_env_var('SPACK_JOB_SPEC_PKG_NAME') compiler_action = get_env_var('SPACK_COMPILER_ACTION') cdash_build_name = get_env_var('SPACK_CDASH_BUILD_NAME') related_builds = get_env_var('SPACK_RELATED_BUILDS_CDASH') spack_pipeline_type = get_env_var('SPACK_PIPELINE_TYPE') pr_mirror_url = get_env_var('SPACK_PR_MIRROR_URL') remote_mirror_url = get_env_var('SPACK_REMOTE_MIRROR_URL') # Construct absolute paths relative to current $CI_PROJECT_DIR ci_project_dir = get_env_var('CI_PROJECT_DIR') pipeline_artifacts_dir = os.path.join(ci_project_dir, pipeline_artifacts_dir) job_log_dir = os.path.join(ci_project_dir, job_log_dir) repro_dir = os.path.join(ci_project_dir, repro_dir) local_mirror_dir = os.path.join(ci_project_dir, local_mirror_dir) concrete_env_dir = os.path.join(ci_project_dir, concrete_env_dir) # Debug print some of the key environment variables we should have received tty.debug('pipeline_artifacts_dir = {0}'.format(pipeline_artifacts_dir)) tty.debug('root_spec = {0}'.format(root_spec)) tty.debug('remote_mirror_url = {0}'.format(remote_mirror_url)) tty.debug('job_spec_pkg_name = {0}'.format(job_spec_pkg_name)) tty.debug('compiler_action = {0}'.format(compiler_action)) # Query the environment manifest to find out whether we're reporting to a # CDash instance, and if so, gather some information from the manifest to # support that task. enable_cdash = False if 'cdash' in yaml_root: enable_cdash = True ci_cdash = yaml_root['cdash'] job_spec_buildgroup = ci_cdash['build-group'] cdash_base_url = ci_cdash['url'] cdash_project = ci_cdash['project'] proj_enc = urlencode({'project': cdash_project}) eq_idx = proj_enc.find('=') + 1 cdash_project_enc = proj_enc[eq_idx:] cdash_site = ci_cdash['site'] tty.debug('cdash_base_url = {0}'.format(cdash_base_url)) tty.debug('cdash_project = {0}'.format(cdash_project)) tty.debug('cdash_project_enc = {0}'.format(cdash_project_enc)) tty.debug('cdash_build_name = {0}'.format(cdash_build_name)) tty.debug('cdash_site = {0}'.format(cdash_site)) tty.debug('related_builds = {0}'.format(related_builds)) tty.debug('job_spec_buildgroup = {0}'.format(job_spec_buildgroup)) # Is this a pipeline run on a spack PR or a merge to develop? It might # be neither, e.g. a pipeline run on some environment repository. spack_is_pr_pipeline = spack_pipeline_type == 'spack_pull_request' spack_is_develop_pipeline = spack_pipeline_type == 'spack_protected_branch' tty.debug('Pipeline type - PR: {0}, develop: {1}'.format( spack_is_pr_pipeline, spack_is_develop_pipeline)) # Figure out what is our temporary storage mirror: Is it artifacts # buildcache? Or temporary-storage-url-prefix? In some cases we need to # force something or pipelines might not have a way to propagate build # artifacts from upstream to downstream jobs. pipeline_mirror_url = None temp_storage_url_prefix = None if 'temporary-storage-url-prefix' in gitlab_ci: temp_storage_url_prefix = gitlab_ci['temporary-storage-url-prefix'] pipeline_mirror_url = url_util.join(temp_storage_url_prefix, ci_pipeline_id) enable_artifacts_mirror = False if 'enable-artifacts-buildcache' in gitlab_ci: enable_artifacts_mirror = gitlab_ci['enable-artifacts-buildcache'] if (enable_artifacts_mirror or (spack_is_pr_pipeline and not enable_artifacts_mirror and not temp_storage_url_prefix)): # If you explicitly enabled the artifacts buildcache feature, or # if this is a PR pipeline but you did not enable either of the # per-pipeline temporary storage features, we force the use of # artifacts buildcache. Otherwise jobs will not have binary # dependencies from previous stages available since we do not # allow pushing binaries to the remote mirror during PR pipelines. enable_artifacts_mirror = True pipeline_mirror_url = 'file://' + local_mirror_dir mirror_msg = 'artifact buildcache enabled, mirror url: {0}'.format( pipeline_mirror_url) tty.debug(mirror_msg) # Whatever form of root_spec we got, use it to get a map giving us concrete # specs for this job and all of its dependencies. spec_map = spack_ci.get_concrete_specs(env, root_spec, job_spec_pkg_name, related_builds, compiler_action) job_spec = spec_map[job_spec_pkg_name] job_spec_yaml_file = '{0}.yaml'.format(job_spec_pkg_name) job_spec_yaml_path = os.path.join(repro_dir, job_spec_yaml_file) # To provide logs, cdash reports, etc for developer download/perusal, # these things have to be put into artifacts. This means downstream # jobs that "need" this job will get those artifacts too. So here we # need to clean out the artifacts we may have got from upstream jobs. cdash_report_dir = os.path.join(pipeline_artifacts_dir, 'cdash_report') if os.path.exists(cdash_report_dir): shutil.rmtree(cdash_report_dir) if os.path.exists(job_log_dir): shutil.rmtree(job_log_dir) if os.path.exists(repro_dir): shutil.rmtree(repro_dir) # Now that we removed them if they existed, create the directories we # need for storing artifacts. The cdash_report directory will be # created internally if needed. os.makedirs(job_log_dir) os.makedirs(repro_dir) # Copy the concrete environment files to the repro directory so we can # expose them as artifacts and not conflict with the concrete environment # files we got as artifacts from the upstream pipeline generation job. # Try to cast a slightly wider net too, and hopefully get the generated # pipeline yaml. If we miss it, the user will still be able to go to the # pipeline generation job and get it from there. target_dirs = [concrete_env_dir, pipeline_artifacts_dir] for dir_to_list in target_dirs: for file_name in os.listdir(dir_to_list): src_file = os.path.join(dir_to_list, file_name) if os.path.isfile(src_file): dst_file = os.path.join(repro_dir, file_name) shutil.copyfile(src_file, dst_file) # If signing key was provided via "SPACK_SIGNING_KEY", then try to # import it. if signing_key: spack_ci.import_signing_key(signing_key) # Depending on the specifics of this job, we might need to turn on the # "config:install_missing compilers" option (to build this job spec # with a bootstrapped compiler), or possibly run "spack compiler find" # (to build a bootstrap compiler or one of its deps in a # compiler-agnostic way), or maybe do nothing at all (to build a spec # using a compiler already installed on the target system). spack_ci.configure_compilers(compiler_action) # Write this job's spec yaml into the reproduction directory, and it will # also be used in the generated "spack install" command to install the spec tty.debug('job concrete spec path: {0}'.format(job_spec_yaml_path)) with open(job_spec_yaml_path, 'w') as fd: fd.write(job_spec.to_yaml(hash=ht.build_hash)) # Write the concrete root spec yaml into the reproduction directory root_spec_yaml_path = os.path.join(repro_dir, 'root.yaml') with open(root_spec_yaml_path, 'w') as fd: fd.write(spec_map['root'].to_yaml(hash=ht.build_hash)) # Write some other details to aid in reproduction into an artifact repro_file = os.path.join(repro_dir, 'repro.json') repro_details = { 'job_name': ci_job_name, 'job_spec_yaml': job_spec_yaml_file, 'root_spec_yaml': 'root.yaml', 'ci_project_dir': ci_project_dir } with open(repro_file, 'w') as fd: fd.write(json.dumps(repro_details)) # Write information about spack into an artifact in the repro dir spack_info = spack_ci.get_spack_info() spack_info_file = os.path.join(repro_dir, 'spack_info.txt') with open(spack_info_file, 'w') as fd: fd.write('\n{0}\n'.format(spack_info)) # If we decided there should be a temporary storage mechanism, add that # mirror now so it's used when we check for a full hash match already # built for this spec. if pipeline_mirror_url: spack.mirror.add(spack_ci.TEMP_STORAGE_MIRROR_NAME, pipeline_mirror_url, cfg.default_modify_scope()) cdash_build_id = None cdash_build_stamp = None # Check configured mirrors for a built spec with a matching full hash matches = bindist.get_mirrors_for_spec(job_spec, full_hash_match=True, index_only=False) if matches: # Got a full hash match on at least one configured mirror. All # matches represent the fully up-to-date spec, so should all be # equivalent. If artifacts mirror is enabled, we just pick one # of the matches and download the buildcache files from there to # the artifacts, so they're available to be used by dependent # jobs in subsequent stages. tty.msg('No need to rebuild {0}, found full hash match at: '.format( job_spec_pkg_name)) for match in matches: tty.msg(' {0}'.format(match['mirror_url'])) if enable_artifacts_mirror: matching_mirror = matches[0]['mirror_url'] build_cache_dir = os.path.join(local_mirror_dir, 'build_cache') tty.debug('Getting {0} buildcache from {1}'.format( job_spec_pkg_name, matching_mirror)) tty.debug('Downloading to {0}'.format(build_cache_dir)) buildcache.download_buildcache_files(job_spec, build_cache_dir, False, matching_mirror) # Now we are done and successful sys.exit(0) # No full hash match anywhere means we need to rebuild spec # Start with spack arguments install_args = [base_arg for base_arg in CI_REBUILD_INSTALL_BASE_ARGS] config = cfg.get('config') if not config['verify_ssl']: install_args.append('-k') install_args.extend([ 'install', '--keep-stage', '--require-full-hash-match', ]) can_verify = spack_ci.can_verify_binaries() verify_binaries = can_verify and spack_is_pr_pipeline is False if not verify_binaries: install_args.append('--no-check-signature') # If CDash reporting is enabled, we first register this build with # the specified CDash instance, then relate the build to those of # its dependencies. if enable_cdash: tty.debug('CDash: Registering build') (cdash_build_id, cdash_build_stamp) = spack_ci.register_cdash_build( cdash_build_name, cdash_base_url, cdash_project, cdash_site, job_spec_buildgroup) if cdash_build_id is not None: cdash_upload_url = '{0}/submit.php?project={1}'.format( cdash_base_url, cdash_project_enc) install_args.extend([ '--cdash-upload-url', cdash_upload_url, '--cdash-build', cdash_build_name, '--cdash-site', cdash_site, '--cdash-buildstamp', cdash_build_stamp, ]) tty.debug('CDash: Relating build with dependency builds') spack_ci.relate_cdash_builds( spec_map, cdash_base_url, cdash_build_id, cdash_project, [pipeline_mirror_url, pr_mirror_url, remote_mirror_url]) # A compiler action of 'FIND_ANY' means we are building a bootstrap # compiler or one of its deps. # TODO: when compilers are dependencies, we should include --no-add if compiler_action != 'FIND_ANY': install_args.append('--no-add') # TODO: once we have the concrete spec registry, use the DAG hash # to identify the spec to install, rather than the concrete spec # yaml file. install_args.extend(['-f', job_spec_yaml_path]) tty.debug('Installing {0} from source'.format(job_spec.name)) tty.debug('spack install arguments: {0}'.format(install_args)) # Write the install command to a shell script with open('install.sh', 'w') as fd: fd.write('#!/bin/bash\n\n') fd.write('\n# spack install command\n') fd.write(' '.join(['"{0}"'.format(i) for i in install_args])) fd.write('\n') st = os.stat('install.sh') os.chmod('install.sh', st.st_mode | stat.S_IEXEC) install_copy_path = os.path.join(repro_dir, 'install.sh') shutil.copyfile('install.sh', install_copy_path) # Run the generated install.sh shell script as if it were being run in # a login shell. try: install_process = subprocess.Popen(['bash', '-l', './install.sh']) install_process.wait() install_exit_code = install_process.returncode except (ValueError, subprocess.CalledProcessError, OSError) as inst: tty.error('Encountered error running install script') tty.error(inst) # Now do the post-install tasks tty.debug('spack install exited {0}'.format(install_exit_code)) # If a spec fails to build in a spack develop pipeline, we add it to a # list of known broken full hashes. This allows spack PR pipelines to # avoid wasting compute cycles attempting to build those hashes. if install_exit_code == INSTALL_FAIL_CODE and spack_is_develop_pipeline: tty.debug('Install failed on develop') if 'broken-specs-url' in gitlab_ci: broken_specs_url = gitlab_ci['broken-specs-url'] dev_fail_hash = job_spec.full_hash() broken_spec_path = url_util.join(broken_specs_url, dev_fail_hash) tty.msg('Reporting broken develop build as: {0}'.format( broken_spec_path)) tmpdir = tempfile.mkdtemp() empty_file_path = os.path.join(tmpdir, 'empty.txt') broken_spec_details = { 'broken-spec': { 'job-url': get_env_var('CI_JOB_URL'), 'pipeline-url': get_env_var('CI_PIPELINE_URL'), 'concrete-spec-yaml': job_spec.to_dict(hash=ht.full_hash) } } try: with open(empty_file_path, 'w') as efd: efd.write(syaml.dump(broken_spec_details)) web_util.push_to_url(empty_file_path, broken_spec_path, keep_original=False, extra_args={'ContentType': 'text/plain'}) except Exception as err: # If we got some kind of S3 (access denied or other connection # error), the first non boto-specific class in the exception # hierarchy is Exception. Just print a warning and return msg = 'Error writing to broken specs list {0}: {1}'.format( broken_spec_path, err) tty.warn(msg) finally: shutil.rmtree(tmpdir) # We generated the "spack install ..." command to "--keep-stage", copy # any logs from the staging directory to artifacts now spack_ci.copy_stage_logs_to_artifacts(job_spec, job_log_dir) # Create buildcache on remote mirror, either on pr-specific mirror or # on the main mirror defined in the gitlab-enabled spack environment if spack_is_pr_pipeline: buildcache_mirror_url = pr_mirror_url else: buildcache_mirror_url = remote_mirror_url # If the install succeeded, create a buildcache entry for this job spec # and push it to one or more mirrors. If the install did not succeed, # print out some instructions on how to reproduce this build failure # outside of the pipeline environment. if install_exit_code == 0: can_sign = spack_ci.can_sign_binaries() sign_binaries = can_sign and spack_is_pr_pipeline is False # Create buildcache in either the main remote mirror, or in the # per-PR mirror, if this is a PR pipeline if buildcache_mirror_url: spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, buildcache_mirror_url, sign_binaries) if cdash_build_id: tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( cdash_build_id, buildcache_mirror_url)) spack_ci.write_cdashid_to_mirror(cdash_build_id, job_spec, buildcache_mirror_url) # Create another copy of that buildcache in the per-pipeline # temporary storage mirror (this is only done if either # artifacts buildcache is enabled or a temporary storage url # prefix is set) if pipeline_mirror_url: spack_ci.push_mirror_contents(env, job_spec, job_spec_yaml_path, pipeline_mirror_url, sign_binaries) if cdash_build_id: tty.debug('Writing cdashid ({0}) to remote mirror: {1}'.format( cdash_build_id, pipeline_mirror_url)) spack_ci.write_cdashid_to_mirror(cdash_build_id, job_spec, pipeline_mirror_url) # If this is a develop pipeline, check if the spec that we just built is # on the broken-specs list. If so, remove it. if spack_is_develop_pipeline and 'broken-specs-url' in gitlab_ci: broken_specs_url = gitlab_ci['broken-specs-url'] just_built_hash = job_spec.full_hash() broken_spec_path = url_util.join(broken_specs_url, just_built_hash) if web_util.url_exists(broken_spec_path): tty.msg('Removing {0} from the list of broken specs'.format( broken_spec_path)) try: web_util.remove_url(broken_spec_path) except Exception as err: # If we got some kind of S3 (access denied or other connection # error), the first non boto-specific class in the exception # hierarchy is Exception. Just print a warning and return msg = 'Error removing {0} from broken specs list: {1}'.format( broken_spec_path, err) tty.warn(msg) else: tty.debug('spack install exited non-zero, will not create buildcache') api_root_url = get_env_var('CI_API_V4_URL') ci_project_id = get_env_var('CI_PROJECT_ID') ci_job_id = get_env_var('CI_JOB_ID') repro_job_url = '{0}/projects/{1}/jobs/{2}/artifacts'.format( api_root_url, ci_project_id, ci_job_id) # Control characters cause this to be printed in blue so it stands out reproduce_msg = """ \033[34mTo reproduce this build locally, run: spack ci reproduce-build {0} [--working-dir <dir>] If this project does not have public pipelines, you will need to first: export GITLAB_PRIVATE_TOKEN=<generated_token> ... then follow the printed instructions.\033[0;0m """.format(repro_job_url) print(reproduce_msg) # Tie job success/failure to the success/failure of building the spec return install_exit_code
def build_tarball(spec, outdir, force=False, rel=False, unsigned=False, allow_root=False, key=None, regenerate_index=False): """ Build a tarball from given spec and put it into the directory structure used at the mirror (following <tarball_directory_name>). """ if not spec.concrete: raise ValueError('spec must be concrete to build tarball') # set up some paths tmpdir = tempfile.mkdtemp() cache_prefix = build_cache_prefix(tmpdir) tarfile_name = tarball_name(spec, '.tar.gz') tarfile_dir = os.path.join(cache_prefix, tarball_directory_name(spec)) tarfile_path = os.path.join(tarfile_dir, tarfile_name) spackfile_path = os.path.join(cache_prefix, tarball_path_name(spec, '.spack')) remote_spackfile_path = url_util.join( outdir, os.path.relpath(spackfile_path, tmpdir)) mkdirp(tarfile_dir) if web_util.url_exists(remote_spackfile_path): if force: web_util.remove_url(remote_spackfile_path) else: raise NoOverwriteException(url_util.format(remote_spackfile_path)) # need to copy the spec file so the build cache can be downloaded # without concretizing with the current spack packages # and preferences spec_file = os.path.join(spec.prefix, ".spack", "spec.yaml") specfile_name = tarball_name(spec, '.spec.yaml') specfile_path = os.path.realpath(os.path.join(cache_prefix, specfile_name)) remote_specfile_path = url_util.join( outdir, os.path.relpath(specfile_path, os.path.realpath(tmpdir))) if web_util.url_exists(remote_specfile_path): if force: web_util.remove_url(remote_specfile_path) else: raise NoOverwriteException(url_util.format(remote_specfile_path)) # make a copy of the install directory to work with workdir = os.path.join(tempfile.mkdtemp(), os.path.basename(spec.prefix)) install_tree(spec.prefix, workdir, symlinks=True) # create info for later relocation and create tar write_buildinfo_file(spec.prefix, workdir, rel=rel) # optionally make the paths in the binaries relative to each other # in the spack install tree before creating tarball if rel: try: make_package_relative(workdir, spec, allow_root) except Exception as e: shutil.rmtree(workdir) shutil.rmtree(tarfile_dir) shutil.rmtree(tmpdir) tty.die(e) else: try: make_package_placeholder(workdir, spec, allow_root) except Exception as e: shutil.rmtree(workdir) shutil.rmtree(tarfile_dir) shutil.rmtree(tmpdir) tty.die(e) # create compressed tarball of the install prefix with closing(tarfile.open(tarfile_path, 'w:gz')) as tar: tar.add(name='%s' % workdir, arcname='%s' % os.path.basename(spec.prefix)) # remove copy of install directory shutil.rmtree(workdir) # get the sha256 checksum of the tarball checksum = checksum_tarball(tarfile_path) # add sha256 checksum to spec.yaml with open(spec_file, 'r') as inputfile: content = inputfile.read() spec_dict = yaml.load(content) bchecksum = {} bchecksum['hash_algorithm'] = 'sha256' bchecksum['hash'] = checksum spec_dict['binary_cache_checksum'] = bchecksum # Add original install prefix relative to layout root to spec.yaml. # This will be used to determine is the directory layout has changed. buildinfo = {} buildinfo['relative_prefix'] = os.path.relpath(spec.prefix, spack.store.layout.root) spec_dict['buildinfo'] = buildinfo spec_dict['full_hash'] = spec.full_hash() tty.debug('The full_hash ({0}) of {1} will be written into {2}'.format( spec_dict['full_hash'], spec.name, url_util.format(remote_specfile_path))) tty.debug(spec.tree()) with open(specfile_path, 'w') as outfile: outfile.write(syaml.dump(spec_dict)) # sign the tarball and spec file with gpg if not unsigned: sign_tarball(key, force, specfile_path) # put tarball, spec and signature files in .spack archive with closing(tarfile.open(spackfile_path, 'w')) as tar: tar.add(name='%s' % tarfile_path, arcname='%s' % tarfile_name) tar.add(name='%s' % specfile_path, arcname='%s' % specfile_name) if not unsigned: tar.add(name='%s.asc' % specfile_path, arcname='%s.asc' % specfile_name) # cleanup file moved to archive os.remove(tarfile_path) if not unsigned: os.remove('%s.asc' % specfile_path) web_util.push_to_url(spackfile_path, remote_spackfile_path, keep_original=False) web_util.push_to_url(specfile_path, remote_specfile_path, keep_original=False) try: # create an index.html for the build_cache directory so specs can be # found if regenerate_index: generate_package_index( url_util.join(outdir, os.path.relpath(cache_prefix, tmpdir))) finally: shutil.rmtree(tmpdir) return None
def mirror_create(args): """Create a directory to be used as a spack mirror, and fill it with package archives.""" if args.specs and args.all: raise SpackError("Cannot specify specs on command line if you" " chose to mirror all specs with '--all'") elif args.file and args.all: raise SpackError("Cannot specify specs with a file ('-f') if you" " chose to mirror all specs with '--all'") if not args.versions_per_spec: num_versions = 1 elif args.versions_per_spec == 'all': num_versions = 'all' else: try: num_versions = int(args.versions_per_spec) except ValueError: raise SpackError("'--versions-per-spec' must be a number or 'all'," " got '{0}'".format(args.versions_per_spec)) # try to parse specs from the command line first. with spack.concretize.disable_compiler_existence_check(): specs = spack.cmd.parse_specs(args.specs, concretize=True) # If there is a file, parse each line as a spec and add it to the list. if args.file: if specs: tty.die("Cannot pass specs on the command line with --file.") specs = _read_specs_from_file(args.file) if not specs: # If nothing is passed, use environment or all if no active env if not args.all: tty.die( "No packages were specified.", "To mirror all packages, use the '--all' option" " (this will require significant time and space).") env = ev.get_env(args, 'mirror') if env: mirror_specs = env.specs_by_hash.values() else: specs = [Spec(n) for n in spack.repo.all_package_names()] mirror_specs = spack.mirror.get_all_versions(specs) mirror_specs.sort(key=lambda s: (s.name, s.version)) else: # If the user asked for dependencies, traverse spec DAG get them. if args.dependencies: new_specs = set() for spec in specs: spec.concretize() for s in spec.traverse(): new_specs.add(s) specs = list(new_specs) # Skip external specs, as they are already installed external_specs = [s for s in specs if s.external] specs = [s for s in specs if not s.external] for spec in external_specs: msg = 'Skipping {0} as it is an external spec.' tty.msg(msg.format(spec.cshort_spec)) if num_versions == 'all': mirror_specs = spack.mirror.get_all_versions(specs) else: mirror_specs = spack.mirror.get_matching_versions( specs, num_versions=num_versions) mirror = spack.mirror.Mirror(args.directory or spack.config.get('config:source_cache')) directory = url_util.format(mirror.push_url) existed = web_util.url_exists(directory) # Actually do the work to create the mirror present, mirrored, error = spack.mirror.create(directory, mirror_specs) p, m, e = len(present), len(mirrored), len(error) verb = "updated" if existed else "created" tty.msg("Successfully %s mirror in %s" % (verb, directory), "Archive stats:", " %-4d already present" % p, " %-4d added" % m, " %-4d failed to fetch." % e) if error: tty.error("Failed downloads:") colify(s.cformat("{name}{@version}") for s in error) sys.exit(1)
def generate_gitlab_ci_yaml(env, print_summary, output_file, prune_dag=False, check_index_only=False, run_optimizer=False, use_dependencies=False): # FIXME: What's the difference between one that opens with 'spack' # and one that opens with 'env'? This will only handle the former. with spack.concretize.disable_compiler_existence_check(): env.concretize() yaml_root = ev.config_dict(env.yaml) if 'gitlab-ci' not in yaml_root: tty.die('Environment yaml does not have "gitlab-ci" section') gitlab_ci = yaml_root['gitlab-ci'] build_group = None enable_cdash_reporting = False cdash_auth_token = None if 'cdash' in yaml_root: enable_cdash_reporting = True ci_cdash = yaml_root['cdash'] build_group = ci_cdash['build-group'] cdash_url = ci_cdash['url'] cdash_project = ci_cdash['project'] cdash_site = ci_cdash['site'] if 'SPACK_CDASH_AUTH_TOKEN' in os.environ: tty.verbose("Using CDash auth token from environment") cdash_auth_token = os.environ.get('SPACK_CDASH_AUTH_TOKEN') is_pr_pipeline = (os.environ.get('SPACK_IS_PR_PIPELINE', '').lower() == 'true') spack_pr_branch = os.environ.get('SPACK_PR_BRANCH', None) pr_mirror_url = None if spack_pr_branch: pr_mirror_url = url_util.join(SPACK_PR_MIRRORS_ROOT_URL, spack_pr_branch) if 'mirrors' not in yaml_root or len(yaml_root['mirrors'].values()) < 1: tty.die('spack ci generate requires an env containing a mirror') ci_mirrors = yaml_root['mirrors'] mirror_urls = [url for url in ci_mirrors.values()] # Check for a list of "known broken" specs that we should not bother # trying to build. broken_specs_url = '' known_broken_specs_encountered = [] if 'broken-specs-url' in gitlab_ci: broken_specs_url = gitlab_ci['broken-specs-url'] enable_artifacts_buildcache = False if 'enable-artifacts-buildcache' in gitlab_ci: enable_artifacts_buildcache = gitlab_ci['enable-artifacts-buildcache'] rebuild_index_enabled = True if 'rebuild-index' in gitlab_ci and gitlab_ci['rebuild-index'] is False: rebuild_index_enabled = False temp_storage_url_prefix = None if 'temporary-storage-url-prefix' in gitlab_ci: temp_storage_url_prefix = gitlab_ci['temporary-storage-url-prefix'] bootstrap_specs = [] phases = [] if 'bootstrap' in gitlab_ci: for phase in gitlab_ci['bootstrap']: try: phase_name = phase.get('name') strip_compilers = phase.get('compiler-agnostic') except AttributeError: phase_name = phase strip_compilers = False phases.append({ 'name': phase_name, 'strip-compilers': strip_compilers, }) for bs in env.spec_lists[phase_name]: bootstrap_specs.append({ 'spec': bs, 'phase-name': phase_name, 'strip-compilers': strip_compilers, }) phases.append({ 'name': 'specs', 'strip-compilers': False, }) # Add this mirror if it's enabled, as some specs might be up to date # here and thus not need to be rebuilt. if pr_mirror_url: add_pr_mirror(pr_mirror_url) # Speed up staging by first fetching binary indices from all mirrors # (including the per-PR mirror we may have just added above). bindist.binary_index.update() staged_phases = {} try: for phase in phases: phase_name = phase['name'] with spack.concretize.disable_compiler_existence_check(): staged_phases[phase_name] = stage_spec_jobs( env.spec_lists[phase_name], check_index_only=check_index_only) finally: # Clean up PR mirror if enabled if pr_mirror_url: remove_pr_mirror() all_job_names = [] output_object = {} job_id = 0 stage_id = 0 stage_names = [] max_length_needs = 0 max_needs_job = '' before_script, after_script = None, None for phase in phases: phase_name = phase['name'] strip_compilers = phase['strip-compilers'] main_phase = is_main_phase(phase_name) spec_labels, dependencies, stages = staged_phases[phase_name] for stage_jobs in stages: stage_name = 'stage-{0}'.format(stage_id) stage_names.append(stage_name) stage_id += 1 for spec_label in stage_jobs: spec_record = spec_labels[spec_label] root_spec = spec_record['rootSpec'] pkg_name = pkg_name_from_spec_label(spec_label) release_spec = root_spec[pkg_name] # Check if this spec is in our list of known failures. if broken_specs_url: full_hash = release_spec.full_hash() broken_spec_path = url_util.join(broken_specs_url, full_hash) if web_util.url_exists(broken_spec_path): known_broken_specs_encountered.append( '{0} ({1})'.format(release_spec, full_hash)) runner_attribs = find_matching_config(release_spec, gitlab_ci) if not runner_attribs: tty.warn('No match found for {0}, skipping it'.format( release_spec)) continue tags = [tag for tag in runner_attribs['tags']] variables = {} if 'variables' in runner_attribs: variables.update(runner_attribs['variables']) image_name = None image_entry = None if 'image' in runner_attribs: build_image = runner_attribs['image'] try: image_name = build_image.get('name') entrypoint = build_image.get('entrypoint') image_entry = [p for p in entrypoint] except AttributeError: image_name = build_image job_script = [ 'spack env activate --without-view .', 'spack ci rebuild', ] if 'script' in runner_attribs: job_script = [s for s in runner_attribs['script']] before_script = None if 'before_script' in runner_attribs: before_script = [ s for s in runner_attribs['before_script'] ] after_script = None if 'after_script' in runner_attribs: after_script = [s for s in runner_attribs['after_script']] osname = str(release_spec.architecture) job_name = get_job_name(phase_name, strip_compilers, release_spec, osname, build_group) compiler_action = 'NONE' if len(phases) > 1: compiler_action = 'FIND_ANY' if is_main_phase(phase_name): compiler_action = 'INSTALL_MISSING' job_vars = { 'SPACK_ROOT_SPEC': format_root_spec(root_spec, main_phase, strip_compilers), 'SPACK_JOB_SPEC_PKG_NAME': release_spec.name, 'SPACK_COMPILER_ACTION': compiler_action, 'SPACK_IS_PR_PIPELINE': str(is_pr_pipeline), } job_dependencies = [] if spec_label in dependencies: if enable_artifacts_buildcache: # Get dependencies transitively, so they're all # available in the artifacts buildcache. dep_jobs = [ d for d in release_spec.traverse(deptype=all, root=False) ] else: # In this case, "needs" is only used for scheduling # purposes, so we only get the direct dependencies. dep_jobs = [] for dep_label in dependencies[spec_label]: dep_pkg = pkg_name_from_spec_label(dep_label) dep_root = spec_labels[dep_label]['rootSpec'] dep_jobs.append(dep_root[dep_pkg]) job_dependencies.extend( format_job_needs(phase_name, strip_compilers, dep_jobs, osname, build_group, prune_dag, spec_labels, enable_artifacts_buildcache)) rebuild_spec = spec_record['needs_rebuild'] # This next section helps gitlab make sure the right # bootstrapped compiler exists in the artifacts buildcache by # creating an artificial dependency between this spec and its # compiler. So, if we are in the main phase, and if the # compiler we are supposed to use is listed in any of the # bootstrap spec lists, then we will add more dependencies to # the job (that compiler and maybe it's dependencies as well). if is_main_phase(phase_name): spec_arch_family = (release_spec.architecture.target. microarchitecture.family) compiler_pkg_spec = compilers.pkg_spec_for_compiler( release_spec.compiler) for bs in bootstrap_specs: c_spec = bs['spec'] bs_arch = c_spec.architecture bs_arch_family = ( bs_arch.target.microarchitecture.family) if (c_spec.satisfies(compiler_pkg_spec) and bs_arch_family == spec_arch_family): # We found the bootstrap compiler this release spec # should be built with, so for DAG scheduling # purposes, we will at least add the compiler spec # to the jobs "needs". But if artifact buildcache # is enabled, we'll have to add all transtive deps # of the compiler as well. # Here we check whether the bootstrapped compiler # needs to be rebuilt. Until compilers are proper # dependencies, we artificially force the spec to # be rebuilt if the compiler targeted to build it # needs to be rebuilt. bs_specs, _, _ = staged_phases[bs['phase-name']] c_spec_key = spec_deps_key(c_spec) rbld_comp = bs_specs[c_spec_key]['needs_rebuild'] rebuild_spec = rebuild_spec or rbld_comp # Also update record so dependents do not fail to # add this spec to their "needs" spec_record['needs_rebuild'] = rebuild_spec dep_jobs = [c_spec] if enable_artifacts_buildcache: dep_jobs = [ d for d in c_spec.traverse(deptype=all) ] job_dependencies.extend( format_job_needs(bs['phase-name'], bs['strip-compilers'], dep_jobs, str(bs_arch), build_group, prune_dag, bs_specs, enable_artifacts_buildcache)) else: debug_msg = ''.join([ 'Considered compiler {0} for spec ', '{1}, but rejected it either because it was ', 'not the compiler required by the spec, or ', 'because the target arch families of the ', 'spec and the compiler did not match' ]).format(c_spec, release_spec) tty.debug(debug_msg) if prune_dag and not rebuild_spec: continue job_vars['SPACK_SPEC_NEEDS_REBUILD'] = str(rebuild_spec) if enable_cdash_reporting: cdash_build_name = get_cdash_build_name( release_spec, build_group) all_job_names.append(cdash_build_name) related_builds = [] # Used for relating CDash builds if spec_label in dependencies: related_builds = ([ spec_labels[d]['spec'].name for d in dependencies[spec_label] ]) job_vars['SPACK_CDASH_BUILD_NAME'] = cdash_build_name job_vars['SPACK_RELATED_BUILDS_CDASH'] = ';'.join( sorted(related_builds)) variables.update(job_vars) artifact_paths = [ 'jobs_scratch_dir', 'cdash_report', ] if enable_artifacts_buildcache: bc_root = 'local_mirror/build_cache' artifact_paths.extend([ os.path.join(bc_root, p) for p in [ bindist.tarball_name(release_spec, '.spec.yaml'), bindist.tarball_name(release_spec, '.cdashid'), bindist.tarball_directory_name(release_spec), ] ]) job_object = { 'stage': stage_name, 'variables': variables, 'script': job_script, 'tags': tags, 'artifacts': { 'paths': artifact_paths, 'when': 'always', }, 'needs': sorted(job_dependencies, key=lambda d: d['job']), 'retry': { 'max': 2, 'when': JOB_RETRY_CONDITIONS, }, 'interruptible': True } length_needs = len(job_dependencies) if length_needs > max_length_needs: max_length_needs = length_needs max_needs_job = job_name if before_script: job_object['before_script'] = before_script if after_script: job_object['after_script'] = after_script if image_name: job_object['image'] = image_name if image_entry is not None: job_object['image'] = { 'name': image_name, 'entrypoint': image_entry, } output_object[job_name] = job_object job_id += 1 if print_summary: for phase in phases: phase_name = phase['name'] tty.msg('Stages for phase "{0}"'.format(phase_name)) phase_stages = staged_phases[phase_name] print_staging_summary(*phase_stages) tty.debug('{0} build jobs generated in {1} stages'.format( job_id, stage_id)) if job_id > 0: tty.debug('The max_needs_job is {0}, with {1} needs'.format( max_needs_job, max_length_needs)) # Use "all_job_names" to populate the build group for this set if enable_cdash_reporting and cdash_auth_token: try: populate_buildgroup(all_job_names, build_group, cdash_project, cdash_site, cdash_auth_token, cdash_url) except (SpackError, HTTPError, URLError) as err: tty.warn('Problem populating buildgroup: {0}'.format(err)) else: tty.warn('Unable to populate buildgroup without CDash credentials') service_job_config = None if 'service-job-attributes' in gitlab_ci: service_job_config = gitlab_ci['service-job-attributes'] default_attrs = [ 'image', 'tags', 'variables', 'before_script', # 'script', 'after_script', ] if job_id > 0: if temp_storage_url_prefix: # There were some rebuild jobs scheduled, so we will need to # schedule a job to clean up the temporary storage location # associated with this pipeline. stage_names.append('cleanup-temp-storage') cleanup_job = {} if service_job_config: copy_attributes(default_attrs, service_job_config, cleanup_job) cleanup_job['stage'] = 'cleanup-temp-storage' cleanup_job['script'] = [ 'spack -d mirror destroy --mirror-url {0}/$CI_PIPELINE_ID'. format(temp_storage_url_prefix) ] cleanup_job['when'] = 'always' output_object['cleanup'] = cleanup_job if rebuild_index_enabled: # Add a final job to regenerate the index stage_names.append('stage-rebuild-index') final_job = {} if service_job_config: copy_attributes(default_attrs, service_job_config, final_job) index_target_mirror = mirror_urls[0] if is_pr_pipeline: index_target_mirror = pr_mirror_url final_job['stage'] = 'stage-rebuild-index' final_job['script'] = [ 'spack buildcache update-index --keys -d {0}'.format( index_target_mirror) ] final_job['when'] = 'always' output_object['rebuild-index'] = final_job output_object['stages'] = stage_names # Capture the version of spack used to generate the pipeline, transform it # into a value that can be passed to "git checkout", and save it in a # global yaml variable spack_version = spack.main.get_version() version_to_clone = None v_match = re.match(r"^\d+\.\d+\.\d+$", spack_version) if v_match: version_to_clone = 'v{0}'.format(v_match.group(0)) else: v_match = re.match(r"^[^-]+-[^-]+-([a-f\d]+)$", spack_version) if v_match: version_to_clone = v_match.group(1) else: version_to_clone = spack_version output_object['variables'] = { 'SPACK_VERSION': spack_version, 'SPACK_CHECKOUT_VERSION': version_to_clone, } if pr_mirror_url: output_object['variables']['SPACK_PR_MIRROR_URL'] = pr_mirror_url sorted_output = {} for output_key, output_value in sorted(output_object.items()): sorted_output[output_key] = output_value # TODO(opadron): remove this or refactor if run_optimizer: import spack.ci_optimization as ci_opt sorted_output = ci_opt.optimizer(sorted_output) # TODO(opadron): remove this or refactor if use_dependencies: import spack.ci_needs_workaround as cinw sorted_output = cinw.needs_to_dependencies(sorted_output) else: # No jobs were generated tty.debug('No specs to rebuild, generating no-op job') noop_job = {} if service_job_config: copy_attributes(default_attrs, service_job_config, noop_job) if 'script' not in noop_job: noop_job['script'] = [ 'echo "All specs already up to date, nothing to rebuild."', ] sorted_output = {'no-specs-to-rebuild': noop_job} if known_broken_specs_encountered: error_msg = ( 'Pipeline generation failed due to the presence of the ' 'following specs that are known to be broken in develop:\n') for broken_spec in known_broken_specs_encountered: error_msg += '* {0}\n'.format(broken_spec) tty.die(error_msg) with open(output_file, 'w') as outf: outf.write(syaml.dump_config(sorted_output, default_flow_style=True))