def read_cdashid_from_mirror(spec, mirror_url): if not spec.concrete: tty.die('Can only read cdashid for concrete spec from mirror') buildcache_name = bindist.tarball_name(spec, '') cdashid_file_name = '{0}.cdashid'.format(buildcache_name) url = os.path.join(mirror_url, bindist.build_cache_relative_path(), cdashid_file_name) resp_url, resp_headers, response = web_util.read_from_url(url) contents = response.fp.read() return int(contents)
def get_specs(allarch=False): """ Get spec.yaml's for build caches available on mirror """ global _cached_specs arch = architecture.Arch(architecture.platform(), 'default_os', 'default_target') if not spack.mirror.MirrorCollection(): tty.debug("No Spack mirrors are currently configured") return {} for mirror in spack.mirror.MirrorCollection().values(): fetch_url_build_cache = url_util.join(mirror.fetch_url, _build_cache_relative_path) tty.debug('Finding buildcaches at {0}'.format( url_util.format(fetch_url_build_cache))) index_url = url_util.join(fetch_url_build_cache, 'index.json') try: _, _, file_stream = web_util.read_from_url(index_url, 'application/json') index_object = codecs.getreader('utf-8')(file_stream).read() except (URLError, web_util.SpackWebError) as url_err: tty.error('Failed to read index {0}'.format(index_url)) tty.debug(url_err) # Continue on to the next mirror continue tmpdir = tempfile.mkdtemp() index_file_path = os.path.join(tmpdir, 'index.json') with open(index_file_path, 'w') as fd: fd.write(index_object) db_root_dir = os.path.join(tmpdir, 'db_root') db = spack_db.Database(None, db_dir=db_root_dir, enable_transaction_locking=False) db._read_from_file(index_file_path) spec_list = db.query_local(installed=False) for indexed_spec in spec_list: spec_arch = architecture.arch_for_spec(indexed_spec.architecture) if (allarch is True or spec_arch == arch): _cached_specs.add(indexed_spec) return _cached_specs
def generate_package_index(cache_prefix): """Create the build cache index page. Creates (or replaces) the "index.json" page at the location given in cache_prefix. This page contains a link for each binary package (*.yaml) and public key (*.key) under cache_prefix. """ tmpdir = tempfile.mkdtemp() db_root_dir = os.path.join(tmpdir, 'db_root') db = spack_db.Database(None, db_dir=db_root_dir, enable_transaction_locking=False, record_fields=['spec', 'ref_count']) file_list = ( entry for entry in web_util.list_url(cache_prefix) if entry.endswith('.yaml')) tty.debug('Retrieving spec.yaml files from {0} to build index'.format( cache_prefix)) for file_path in file_list: try: yaml_url = url_util.join(cache_prefix, file_path) tty.debug('fetching {0}'.format(yaml_url)) _, _, yaml_file = web_util.read_from_url(yaml_url) yaml_contents = codecs.getreader('utf-8')(yaml_file).read() # yaml_obj = syaml.load(yaml_contents) # s = Spec.from_yaml(yaml_obj) s = Spec.from_yaml(yaml_contents) db.add(s, None) except (URLError, web_util.SpackWebError) as url_err: tty.error('Error reading spec.yaml: {0}'.format(file_path)) tty.error(url_err) try: index_json_path = os.path.join(db_root_dir, 'index.json') with open(index_json_path, 'w') as f: db._write_to_file(f) web_util.push_to_url( index_json_path, url_util.join(cache_prefix, 'index.json'), keep_original=False, extra_args={'ContentType': 'application/json'}) finally: shutil.rmtree(tmpdir)
def needs_rebuild(spec, mirror_url, rebuild_on_errors=False): if not spec.concrete: raise ValueError('spec must be concrete to check against mirror') pkg_name = spec.name pkg_version = spec.version pkg_hash = spec.dag_hash() pkg_full_hash = spec.full_hash() tty.debug('Checking {0}-{1}, dag_hash = {2}, full_hash = {3}'.format( pkg_name, pkg_version, pkg_hash, pkg_full_hash)) tty.debug(spec.tree()) # Try to retrieve the .spec.yaml directly, based on the known # format of the name, in order to determine if the package # needs to be rebuilt. cache_prefix = build_cache_prefix(mirror_url) spec_yaml_file_name = tarball_name(spec, '.spec.yaml') file_path = os.path.join(cache_prefix, spec_yaml_file_name) result_of_error = 'Package ({0}) will {1}be rebuilt'.format( spec.short_spec, '' if rebuild_on_errors else 'not ') try: _, _, yaml_file = web_util.read_from_url(file_path) yaml_contents = codecs.getreader('utf-8')(yaml_file).read() except (URLError, web_util.SpackWebError) as url_err: err_msg = [ 'Unable to determine whether {0} needs rebuilding,', ' caught exception attempting to read from {1}.', ] tty.error(''.join(err_msg).format(spec.short_spec, file_path)) tty.debug(url_err) tty.warn(result_of_error) return rebuild_on_errors if not yaml_contents: tty.error('Reading {0} returned nothing'.format(file_path)) tty.warn(result_of_error) return rebuild_on_errors spec_yaml = syaml.load(yaml_contents) # If either the full_hash didn't exist in the .spec.yaml file, or it # did, but didn't match the one we computed locally, then we should # just rebuild. This can be simplified once the dag_hash and the # full_hash become the same thing. if ('full_hash' not in spec_yaml or spec_yaml['full_hash'] != pkg_full_hash): if 'full_hash' in spec_yaml: reason = 'hash mismatch, remote = {0}, local = {1}'.format( spec_yaml['full_hash'], pkg_full_hash) else: reason = 'full_hash was missing from remote spec.yaml' tty.msg('Rebuilding {0}, reason: {1}'.format( spec.short_spec, reason)) tty.msg(spec.tree()) return True return False