def test_fetchstrategy_bad_url_scheme(): """Ensure that trying to make a fetch strategy from a URL with an unsupported scheme fails as expected.""" with pytest.raises(ValueError): fetcher = from_url_scheme( # noqa: F841 'bogus-scheme://example.com/a/b/c')
def fetch(self, mirror_only=False): """Downloads an archive or checks out code from a repository.""" fetchers = [] if not mirror_only: fetchers.append(self.default_fetcher) # TODO: move mirror logic out of here and clean it up! # TODO: Or @alalazo may have some ideas about how to use a # TODO: CompositeFetchStrategy here. self.skip_checksum_for_mirror = True if self.mirror_paths: # Join URLs of mirror roots with mirror paths. Because # urljoin() will strip everything past the final '/' in # the root, so we add a '/' if it is not present. urls = [] for mirror in spack.mirror.MirrorCollection().values(): for rel_path in self.mirror_paths: urls.append(url_util.join(mirror.fetch_url, rel_path)) # If this archive is normally fetched from a tarball URL, # then use the same digest. `spack mirror` ensures that # the checksum will be the same. digest = None expand = True extension = None if isinstance(self.default_fetcher, fs.URLFetchStrategy): digest = self.default_fetcher.digest expand = self.default_fetcher.expand_archive extension = self.default_fetcher.extension # Have to skip the checksum for things archived from # repositories. How can this be made safer? self.skip_checksum_for_mirror = not bool(digest) # Add URL strategies for all the mirrors with the digest for url in urls: fetchers.insert( 0, fs.from_url_scheme(url, digest, expand=expand, extension=extension)) if self.default_fetcher.cachable: for rel_path in reversed(list(self.mirror_paths)): cache_fetcher = spack.caches.fetch_cache.fetcher( rel_path, digest, expand=expand, extension=extension) fetchers.insert(0, cache_fetcher) def generate_fetchers(): for fetcher in fetchers: yield fetcher # The search function may be expensive, so wait until now to # call it so the user can stop if a prior fetcher succeeded if self.search_fn and not mirror_only: dynamic_fetchers = self.search_fn() for fetcher in dynamic_fetchers: yield fetcher for fetcher in generate_fetchers(): try: fetcher.set_stage(self) self.fetcher = fetcher self.fetcher.fetch() break except spack.fetch_strategy.NoCacheError: # Don't bother reporting when something is not cached. continue except spack.error.SpackError as e: tty.msg("Fetching from %s failed." % fetcher) tty.debug(e) continue else: err_msg = "All fetchers failed for %s" % self.name self.fetcher = self.default_fetcher raise fs.FetchError(err_msg, None)
def __init__(self, url_or_fetch_strategy, name=None, mirror_paths=None, keep=False, path=None, lock=True, search_fn=None): """Create a stage object. Parameters: url_or_fetch_strategy URL of the archive to be downloaded into this stage, OR a valid FetchStrategy. name If a name is provided, then this stage is a named stage and will persist between runs (or if you construct another stage object later). If name is not provided, then this stage will be given a unique name automatically. mirror_paths If provided, Stage will search Spack's mirrors for this archive at each of the provided relative mirror paths before using the default fetch strategy. keep By default, when used as a context manager, the Stage is deleted on exit when no exceptions are raised. Pass True to keep the stage intact even if no exceptions are raised. path If provided, the stage path to use for associated builds. lock True if the stage directory file lock is to be used, False otherwise. search_fn The search function that provides the fetch strategy instance. """ # TODO: fetch/stage coupling needs to be reworked -- the logic # TODO: here is convoluted and not modular enough. if isinstance(url_or_fetch_strategy, string_types): self.fetcher = fs.from_url_scheme(url_or_fetch_strategy) elif isinstance(url_or_fetch_strategy, fs.FetchStrategy): self.fetcher = url_or_fetch_strategy else: raise ValueError( "Can't construct Stage without url or fetch strategy") self.fetcher.set_stage(self) # self.fetcher can change with mirrors. self.default_fetcher = self.fetcher self.search_fn = search_fn # used for mirrored archives of repositories. self.skip_checksum_for_mirror = True self.srcdir = None # TODO: This uses a protected member of tempfile, but seemed the only # TODO: way to get a temporary name. It won't be the same as the # TODO: temporary stage area in _stage_root. self.name = name if name is None: self.name = stage_prefix + next(tempfile._get_candidate_names()) self.mirror_paths = mirror_paths # Use the provided path or construct an optionally named stage path. if path is not None: self.path = path else: self.path = os.path.join(get_stage_root(), self.name) # Flag to decide whether to delete the stage folder on exit or not self.keep = keep # File lock for the stage directory. We use one file for all # stage locks. See spack.database.Database.prefix_lock for # details on this approach. self._lock = None if lock: if self.name not in Stage.stage_locks: sha1 = hashlib.sha1(self.name.encode('utf-8')).digest() lock_id = prefix_bits(sha1, bit_length(sys.maxsize)) stage_lock_path = os.path.join(get_stage_root(), '.lock') Stage.stage_locks[self.name] = spack.util.lock.Lock( stage_lock_path, lock_id, 1) self._lock = Stage.stage_locks[self.name] # When stages are reused, we need to know whether to re-create # it. This marks whether it has been created/destroyed. self.created = False
def fetch(self, mirror_only=False, err_msg=None): """Retrieves the code or archive Args: mirror_only (bool): only fetch from a mirror err_msg (str or None): the error message to display if all fetchers fail or ``None`` for the default fetch failure message """ fetchers = [] if not mirror_only: fetchers.append(self.default_fetcher) # TODO: move mirror logic out of here and clean it up! # TODO: Or @alalazo may have some ideas about how to use a # TODO: CompositeFetchStrategy here. self.skip_checksum_for_mirror = True if self.mirror_paths: # Join URLs of mirror roots with mirror paths. Because # urljoin() will strip everything past the final '/' in # the root, so we add a '/' if it is not present. mirror_urls = {} for mirror in spack.mirror.MirrorCollection().values(): for rel_path in self.mirror_paths: mirror_url = url_util.join(mirror.fetch_url, rel_path) mirror_urls[mirror_url] = {} if mirror.get_access_pair("fetch") or \ mirror.get_access_token("fetch") or \ mirror.get_profile("fetch"): mirror_urls[mirror_url] = { "access_token": mirror.get_access_token("fetch"), "access_pair": mirror.get_access_pair("fetch"), "access_profile": mirror.get_profile("fetch"), "endpoint_url": mirror.get_endpoint_url("fetch") } # If this archive is normally fetched from a tarball URL, # then use the same digest. `spack mirror` ensures that # the checksum will be the same. digest = None expand = True extension = None if isinstance(self.default_fetcher, fs.URLFetchStrategy): digest = self.default_fetcher.digest expand = self.default_fetcher.expand_archive extension = self.default_fetcher.extension # Have to skip the checksum for things archived from # repositories. How can this be made safer? self.skip_checksum_for_mirror = not bool(digest) # Add URL strategies for all the mirrors with the digest # Insert fetchers in the order that the URLs are provided. for url in reversed(list(mirror_urls.keys())): fetchers.insert( 0, fs.from_url_scheme(url, digest, expand=expand, extension=extension, connection=mirror_urls[url])) if self.default_fetcher.cachable: for rel_path in reversed(list(self.mirror_paths)): cache_fetcher = spack.caches.fetch_cache.fetcher( rel_path, digest, expand=expand, extension=extension) fetchers.insert(0, cache_fetcher) def generate_fetchers(): for fetcher in fetchers: yield fetcher # The search function may be expensive, so wait until now to # call it so the user can stop if a prior fetcher succeeded if self.search_fn and not mirror_only: dynamic_fetchers = self.search_fn() for fetcher in dynamic_fetchers: yield fetcher def print_errors(errors): for msg in errors: tty.debug(msg) errors = [] for fetcher in generate_fetchers(): try: fetcher.stage = self self.fetcher = fetcher self.fetcher.fetch() break except spack.fetch_strategy.NoCacheError: # Don't bother reporting when something is not cached. continue except spack.error.SpackError as e: errors.append('Fetching from {0} failed.'.format(fetcher)) tty.debug(e) continue else: print_errors(errors) self.fetcher = self.default_fetcher default_msg = 'All fetchers failed for {0}'.format(self.name) raise fs.FetchError(err_msg or default_msg, None) print_errors(errors)