def pick_digest(impl): from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair best = None for digest in impl.digests: alg_name, digest_value = parse_algorithm_digest_pair(digest) alg = manifest.algorithms.get(alg_name, None) if alg and (best is None or best.rating < alg.rating): best = alg required_digest = digest if best is None: if not impl.digests: raise SafeException( "No <manifest-digest> given for '%(implementation)s' version %(version)s" % { 'implementation': impl.feed.get_name(), 'version': impl.get_version() }) raise SafeException( "Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s" % { 'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version() }) return required_digest
def splitID(id): """Take an ID in the form 'alg=value' and return a tuple (alg, value). @type id: str @rtype: (L{Algorithm}, str) @raise BadDigest: if the algorithm isn't known or the ID has the wrong format.""" alg, digest = parse_algorithm_digest_pair(id) return (get_algorithm(alg), digest)
def do_audit(args): """audit [DIRECTORY]""" if len(args) == 0: audit_stores = stores.stores else: audit_stores = [zerostore.Store(x) for x in args] audit_ls = [] total = 0 for a in audit_stores: if os.path.isdir(a.dir): items = sorted(os.listdir(a.dir)) audit_ls.append((a.dir, items)) total += len(items) elif len(args): raise SafeException(_("No such directory '%s'") % a.dir) verified = 0 failures = [] i = 0 for root, impls in audit_ls: print(_("Scanning %s") % root) for required_digest in impls: path = os.path.join(root, required_digest) try: (alg, digest ) = zerostore.parse_algorithm_digest_pair(required_digest) except zerostore.BadDigest: print(_("Skipping non-implementation directory %s") % path) continue i += 1 try: msg = _("[%(done)d / %(total)d] Verifying %(digest)s") % { 'done': i, 'total': total, 'digest': required_digest } print(msg, end='') sys.stdout.flush() verify(path, required_digest) print("\r" + (" " * len(msg)) + "\r", end='') verified += 1 except zerostore.BadDigest as ex: print() failures.append(path) print(str(ex)) if ex.detail: print() print(ex.detail) if failures: print('\n' + _("List of corrupted or modified implementations:")) for x in failures: print(x) print() print(_("Checked %d items") % i) print(_("Successfully verified implementations: %d") % verified) print(_("Corrupted or modified implementations: %d") % len(failures)) if failures: sys.exit(1)
def do_audit(args): """audit [DIRECTORY]""" if len(args) == 0: audit_stores = stores.stores else: audit_stores = [zerostore.Store(x) for x in args] audit_ls = [] total = 0 for a in audit_stores: if os.path.isdir(a.dir): items = sorted(os.listdir(a.dir)) audit_ls.append((a.dir, items)) total += len(items) elif len(args): raise SafeException(_("No such directory '%s'") % a.dir) verified = 0 failures = [] i = 0 for root, impls in audit_ls: print(_("Scanning %s") % root) for required_digest in impls: path = os.path.join(root, required_digest) try: (alg, digest) = zerostore.parse_algorithm_digest_pair(required_digest) except zerostore.BadDigest: print(_("Skipping non-implementation directory %s") % path) continue i += 1 try: msg = _("[%(done)d / %(total)d] Verifying %(digest)s") % { "done": i, "total": total, "digest": required_digest, } print(msg, end="") sys.stdout.flush() verify(path, required_digest) print("\r" + (" " * len(msg)) + "\r", end="") verified += 1 except zerostore.BadDigest as ex: print() failures.append(path) print(str(ex)) if ex.detail: print() print(ex.detail) if failures: print("\n" + _("List of corrupted or modified implementations:")) for x in failures: print(x) print() print(_("Checked %d items") % i) print(_("Successfully verified implementations: %d") % verified) print(_("Corrupted or modified implementations: %d") % len(failures)) if failures: sys.exit(1)
def pick_digest(impl): from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair best = None for digest in impl.digests: alg_name, digest_value = parse_algorithm_digest_pair(digest) alg = manifest.algorithms.get(alg_name, None) if alg and (best is None or best.rating < alg.rating): best = alg required_digest = digest if best is None: if not impl.digests: raise SafeException("No <manifest-digest> given for '%(implementation)s' version %(version)s" % {'implementation': impl.feed.get_name(), 'version': impl.get_version()}) raise SafeException("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s" % {'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version()}) return required_digest
def check_manifest_and_rename(self, required_digest, unpack_dir, dry_run = False): implementation = self.impl sha1new = get_digest(unpack_dir, 'sha1new') if not implementation.getAttribute('id'): implementation.setAttribute('id', sha1new) digests = [sha1new] def add_digest(alg_name): digest_id = get_digest(unpack_dir, alg_name) digests.append(digest_id) name, value = zerostore.parse_algorithm_digest_pair(digest_id) elem.setAttribute(alg_name, value) have_manifest_elem = False for elem in implementation.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'manifest-digest'): have_manifest_elem = True have_digests = False for attr_name, value in elem.attributes.items(): if value: continue add_digest(attr_name) have_digests = True if not have_digests: add_digest('sha256new') if not have_manifest_elem: print("WARNING: no <manifest-digest> element found") best_rating = -1 best_digest = None for digest_id in digests: alg_name, value = zerostore.parse_algorithm_digest_pair(digest_id) alg = manifest.get_algorithm(alg_name) if alg.rating > best_rating: best_rating = alg.rating best_digest = digest_id # Cache if necessary (saves downloading it again later) stores = self.real_stores if stores.lookup_maybe(digests) is None: stores.add_dir_to_cache(best_digest, unpack_dir)
def splitID(id): """Take an ID in the form 'alg=value' and return a tuple (alg, value), where 'alg' is an instance of Algorithm and 'value' is a string. @raise BadDigest: if the algorithm isn't known or the ID has the wrong format.""" alg, digest = parse_algorithm_digest_pair(id) return (get_algorithm(alg), digest)
def optimise(impl_dir): """Scan an implementation cache directory for duplicate files, and hard-link any duplicates together to save space. @param impl_dir: a $cache/0install.net/implementations directory @type impl_dir: str @return: (unique bytes, duplicated bytes, already linked, manifest size) @rtype: (int, int, int, int)""" first_copy = {} # TypeDigest -> Path dup_size = uniq_size = already_linked = man_size = 0 import random from zeroinstall.zerostore import BadDigest, parse_algorithm_digest_pair for x in range(10): tmpfile = os.path.join(impl_dir, 'optimise-%d' % random.randint(0, 1000000)) if not os.path.exists(tmpfile): break else: raise Exception(_("Can't generate unused tempfile name!")) dirs = os.listdir(impl_dir) total = len(dirs) msg = "" def clear(): print("\r" + (" " * len(msg)) + "\r", end='') for i, impl in enumerate(dirs): clear() msg = _("[%(done)d / %(total)d] Reading manifests...") % {'done': i, 'total': total} print(msg, end='') sys.stdout.flush() try: alg, manifest_digest = parse_algorithm_digest_pair(impl) except BadDigest: logger.warning(_("Skipping non-implementation '%s'"), impl) continue manifest_path = os.path.join(impl_dir, impl, '.manifest') try: ms = open(manifest_path, 'rt') except OSError as ex: logger.warning(_("Failed to read manifest file '%(manifest_path)s': %(exception)s"), {'manifest': manifest_path, 'exception': str(ex)}) continue if alg == 'sha1': ms.close() continue man_size += os.path.getsize(manifest_path) dir = "" for line in ms: if line[0] == 'D': itype, path = line.split(' ', 1) assert path.startswith('/') dir = path[1:-1] # Strip slash and newline continue if line[0] == "S": itype, digest, size, rest = line.split(' ', 3) uniq_size += int(size) continue assert line[0] in "FX" itype, digest, mtime, size, path = line.split(' ', 4) path = path[:-1] # Strip newline size = int(size) key = (itype, digest, mtime, size) loc_path = (impl, dir, path) first_loc = first_copy.get(key, None) if first_loc: first_full = os.path.join(impl_dir, *first_loc) new_full = os.path.join(impl_dir, *loc_path) if _already_linked(first_full, new_full): already_linked += size else: _link(first_full, new_full, tmpfile) dup_size += size else: first_copy[key] = loc_path uniq_size += size ms.close() clear() return (uniq_size, dup_size, already_linked, man_size)
def toDOM(self): """Create a DOM document for the selected implementations. The document gives the URI of the root, plus each selected implementation. For each selected implementation, we record the ID, the version, the URI and (if different) the feed URL. We also record all the bindings needed. @return: a new DOM Document""" from xml.dom import minidom, XMLNS_NAMESPACE assert self.interface impl = minidom.getDOMImplementation() doc = impl.createDocument(XMLNS_IFACE, "selections", None) root = doc.documentElement root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE) root.setAttributeNS(None, 'interface', self.interface) root.setAttributeNS(None, 'command', self.command or "") prefixes = Prefixes(XMLNS_IFACE) for iface, selection in sorted(self.selections.items()): selection_elem = doc.createElementNS(XMLNS_IFACE, 'selection') selection_elem.setAttributeNS(None, 'interface', selection.interface) root.appendChild(selection_elem) if selection.quick_test_file: selection_elem.setAttributeNS(None, 'quick-test-file', selection.quick_test_file) if selection.quick_test_mtime: selection_elem.setAttributeNS( None, 'quick-test-mtime', str(selection.quick_test_mtime)) for name, value in selection.attrs.items(): if ' ' in name: ns, localName = name.split(' ', 1) prefixes.setAttributeNS(selection_elem, ns, localName, value) elif name == 'stability': pass elif name == 'from-feed': # Don't bother writing from-feed attr if it's the same as the interface if value != selection.attrs['interface']: selection_elem.setAttributeNS(None, name, value) elif name not in ('main', 'self-test'): # (replaced by <command>) selection_elem.setAttributeNS(None, name, value) if selection.digests: manifest_digest = doc.createElementNS(XMLNS_IFACE, 'manifest-digest') for digest in selection.digests: aname, avalue = zerostore.parse_algorithm_digest_pair( digest) assert ':' not in aname manifest_digest.setAttribute(aname, avalue) selection_elem.appendChild(manifest_digest) for b in selection.bindings: selection_elem.appendChild(b._toxml(doc, prefixes)) for dep in selection.dependencies: if not isinstance(dep, model.InterfaceDependency): continue dep_elem = doc.createElementNS(XMLNS_IFACE, 'requires') dep_elem.setAttributeNS(None, 'interface', dep.interface) selection_elem.appendChild(dep_elem) for m in dep.metadata: parts = m.split(' ', 1) if len(parts) == 1: ns = None localName = parts[0] dep_elem.setAttributeNS(None, localName, dep.metadata[m]) else: ns, localName = parts prefixes.setAttributeNS(dep_elem, ns, localName, dep.metadata[m]) for b in dep.bindings: dep_elem.appendChild(b._toxml(doc, prefixes)) for command in selection.get_commands().values(): selection_elem.appendChild(command._toxml(doc, prefixes)) for ns, prefix in prefixes.prefixes.items(): root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:' + prefix, ns) return doc
def add_digest(alg_name): digest_id = get_digest(unpack_dir, alg_name) digests.append(digest_id) name, value = zerostore.parse_algorithm_digest_pair(digest_id) elem.setAttribute(alg_name, value)
def toDOM(self): """Create a DOM document for the selected implementations. The document gives the URI of the root, plus each selected implementation. For each selected implementation, we record the ID, the version, the URI and (if different) the feed URL. We also record all the bindings needed. @return: a new DOM Document""" from xml.dom import minidom, XMLNS_NAMESPACE assert self.interface impl = minidom.getDOMImplementation() doc = impl.createDocument(XMLNS_IFACE, "selections", None) root = doc.documentElement root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE) root.setAttributeNS(None, 'interface', self.interface) root.setAttributeNS(None, 'command', self.command or "") prefixes = Prefixes(XMLNS_IFACE) for iface, selection in sorted(self.selections.items()): selection_elem = doc.createElementNS(XMLNS_IFACE, 'selection') selection_elem.setAttributeNS(None, 'interface', selection.interface) root.appendChild(selection_elem) for name, value in selection.attrs.items(): if ' ' in name: ns, localName = name.split(' ', 1) prefixes.setAttributeNS(selection_elem, ns, localName, value) elif name == 'stability': pass elif name == 'from-feed': # Don't bother writing from-feed attr if it's the same as the interface if value != selection.attrs['interface']: selection_elem.setAttributeNS(None, name, value) elif name not in ('main', 'self-test'): # (replaced by <command>) selection_elem.setAttributeNS(None, name, value) if selection.digests: manifest_digest = doc.createElementNS(XMLNS_IFACE, 'manifest-digest') for digest in selection.digests: aname, avalue = zerostore.parse_algorithm_digest_pair(digest) assert ':' not in aname manifest_digest.setAttribute(aname, avalue) selection_elem.appendChild(manifest_digest) for b in selection.bindings: selection_elem.appendChild(b._toxml(doc, prefixes)) for dep in selection.dependencies: if not isinstance(dep, model.InterfaceDependency): continue dep_elem = doc.createElementNS(XMLNS_IFACE, 'requires') dep_elem.setAttributeNS(None, 'interface', dep.interface) selection_elem.appendChild(dep_elem) for m in dep.metadata: parts = m.split(' ', 1) if len(parts) == 1: ns = None localName = parts[0] dep_elem.setAttributeNS(None, localName, dep.metadata[m]) else: ns, localName = parts prefixes.setAttributeNS(dep_elem, ns, localName, dep.metadata[m]) for b in dep.bindings: dep_elem.appendChild(b._toxml(doc, prefixes)) for command in selection.get_commands().values(): selection_elem.appendChild(command._toxml(doc, prefixes)) for ns, prefix in prefixes.prefixes.items(): root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:' + prefix, ns) return doc
def download_impl(self, impl, retrieval_method, stores, force=False): """Download an implementation. @param impl: the selected implementation @type impl: L{model.ZeroInstallImplementation} @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe) @type retrieval_method: L{model.RetrievalMethod} @param stores: where to store the downloaded implementation @type stores: L{zerostore.Stores} @rtype: L{tasks.Blocker}""" assert impl assert retrieval_method if isinstance(retrieval_method, DistributionSource): return retrieval_method.install(self.handler) from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair best = None for digest in impl.digests: alg_name, digest_value = parse_algorithm_digest_pair(digest) alg = manifest.algorithms.get(alg_name, None) if alg and (best is None or best.rating < alg.rating): best = alg required_digest = digest if best is None: if not impl.digests: raise SafeException( _("No <manifest-digest> given for '%(implementation)s' version %(version)s") % {"implementation": impl.feed.get_name(), "version": impl.get_version()} ) raise SafeException( _("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s") % {"algorithms": impl.digests, "implementation": impl.feed.get_name(), "version": impl.get_version()} ) @tasks.async def download_impl(method): original_exception = None while True: try: if isinstance(method, DownloadSource): blocker, stream = self.download_archive( method, impl_hint=impl, may_use_mirror=original_exception is None ) try: yield blocker tasks.check(blocker) stream.seek(0) if self.external_store: self._add_to_external_store(required_digest, [method], [stream]) else: self._add_to_cache(required_digest, stores, method, stream) finally: stream.close() elif isinstance(method, Recipe): blocker = self.cook(required_digest, method, stores, impl_hint=impl) yield blocker tasks.check(blocker) else: raise Exception(_("Unknown download type for '%s'") % method) except download.DownloadError as ex: if original_exception: logger.info("Error from mirror: %s", ex) raise original_exception else: original_exception = ex mirror_url = self._get_impl_mirror(impl) if mirror_url is not None: logger.info("%s: trying implementation mirror at %s", ex, mirror_url) method = model.DownloadSource( impl, mirror_url, None, None, type="application/x-bzip-compressed-tar" ) continue # Retry raise break self.handler.impl_added_to_store(impl) return download_impl(retrieval_method)
def download_impl(self, impl, retrieval_method, stores, force=False): """Download an implementation. @param impl: the selected implementation @type impl: L{model.ZeroInstallImplementation} @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe) @type retrieval_method: L{model.RetrievalMethod} @param stores: where to store the downloaded implementation @type stores: L{zerostore.Stores} @rtype: L{tasks.Blocker}""" assert impl assert retrieval_method if isinstance(retrieval_method, DistributionSource): return retrieval_method.install(self.handler) from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair best = None for digest in impl.digests: alg_name, digest_value = parse_algorithm_digest_pair(digest) alg = manifest.algorithms.get(alg_name, None) if alg and (best is None or best.rating < alg.rating): best = alg required_digest = digest if best is None: if not impl.digests: raise SafeException( _("No <manifest-digest> given for '%(implementation)s' version %(version)s" ) % { 'implementation': impl.feed.get_name(), 'version': impl.get_version() }) raise SafeException( _("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s" ) % { 'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version() }) @tasks. async def download_impl(method): original_exception = None while True: try: if isinstance(method, DownloadSource): blocker, stream = self.download_archive( method, impl_hint=impl, may_use_mirror=original_exception is None) try: yield blocker tasks.check(blocker) stream.seek(0) if self.external_store: self._add_to_external_store( required_digest, [method], [stream]) else: self._add_to_cache(required_digest, stores, method, stream) finally: stream.close() elif isinstance(method, Recipe): blocker = self.cook(required_digest, method, stores, impl_hint=impl) yield blocker tasks.check(blocker) else: raise Exception( _("Unknown download type for '%s'") % method) except download.DownloadError as ex: if original_exception: logger.info("Error from mirror: %s", ex) raise original_exception else: original_exception = ex mirror_url = self._get_impl_mirror(impl) if mirror_url is not None: logger.info("%s: trying implementation mirror at %s", ex, mirror_url) method = model.DownloadSource( impl, mirror_url, None, None, type='application/x-bzip-compressed-tar') continue # Retry raise break self.handler.impl_added_to_store(impl) return download_impl(retrieval_method)
def optimise(impl_dir): """Scan an implementation cache directory for duplicate files, and hard-link any duplicates together to save space. @param impl_dir: a $cache/0install.net/implementations directory @type impl_dir: str @return: (unique bytes, duplicated bytes, already linked, manifest size) @rtype: (int, int, int, int)""" first_copy = {} # TypeDigest -> Path dup_size = uniq_size = already_linked = man_size = 0 import random from zeroinstall.zerostore import BadDigest, parse_algorithm_digest_pair for x in range(10): tmpfile = os.path.join(impl_dir, 'optimise-%d' % random.randint(0, 1000000)) if not os.path.exists(tmpfile): break else: raise Exception(_("Can't generate unused tempfile name!")) dirs = os.listdir(impl_dir) total = len(dirs) msg = "" def clear(): print("\r" + (" " * len(msg)) + "\r", end='') for i, impl in enumerate(dirs): clear() msg = _("[%(done)d / %(total)d] Reading manifests...") % {'done': i, 'total': total} print(msg, end='') sys.stdout.flush() try: alg, manifest_digest = parse_algorithm_digest_pair(impl) except BadDigest: logger.warn(_("Skipping non-implementation '%s'"), impl) continue manifest_path = os.path.join(impl_dir, impl, '.manifest') try: ms = open(manifest_path, 'rt') except OSError as ex: logger.warn(_("Failed to read manifest file '%(manifest_path)s': %(exception)s"), {'manifest': manifest_path, 'exception': str(ex)}) continue if alg == 'sha1': continue man_size += os.path.getsize(manifest_path) dir = "" for line in ms: if line[0] == 'D': itype, path = line.split(' ', 1) assert path.startswith('/') dir = path[1:-1] # Strip slash and newline continue if line[0] == "S": itype, digest, size, rest = line.split(' ', 3) uniq_size += int(size) continue assert line[0] in "FX" itype, digest, mtime, size, path = line.split(' ', 4) path = path[:-1] # Strip newline size = int(size) key = (itype, digest, mtime, size) loc_path = (impl, dir, path) first_loc = first_copy.get(key, None) if first_loc: first_full = os.path.join(impl_dir, *first_loc) new_full = os.path.join(impl_dir, *loc_path) if _already_linked(first_full, new_full): already_linked += size else: _link(first_full, new_full, tmpfile) dup_size += size else: first_copy[key] = loc_path uniq_size += size ms.close() clear() return (uniq_size, dup_size, already_linked, man_size)
def download_impl(self, impl, retrieval_method, stores, force=False): """Download an implementation. @param impl: the selected implementation @type impl: L{model.ZeroInstallImplementation} @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe) @type retrieval_method: L{model.RetrievalMethod} @param stores: where to store the downloaded implementation @type stores: L{zerostore.Stores} @type force: bool @rtype: L{tasks.Blocker}""" assert impl assert retrieval_method if isinstance(retrieval_method, DistributionSource): return retrieval_method.install(self.handler) from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair best = None for digest in impl.digests: alg_name, digest_value = parse_algorithm_digest_pair(digest) alg = manifest.algorithms.get(alg_name, None) if alg and (best is None or best.rating < alg.rating): best = alg required_digest = digest if best is None: if not impl.digests: raise SafeException( _("No <manifest-digest> given for '%(implementation)s' version %(version)s" ) % { 'implementation': impl.feed.get_name(), 'version': impl.get_version() }) raise SafeException( _("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s" ) % { 'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version() }) @tasks. async def download_impl(method): original_exception = None while True: if not isinstance(method, Recipe): # turn an individual method into a single-step Recipe step = method method = Recipe() method.steps.append(step) try: blocker = self.cook( required_digest, method, stores, impl_hint=impl, dry_run=self.handler.dry_run, may_use_mirror=original_exception is None) yield blocker tasks.check(blocker) except download.DownloadError as ex: if original_exception: logger.info("Error from mirror: %s", ex) raise original_exception else: original_exception = ex mirror_url = self._get_impl_mirror(impl) if mirror_url is not None: logger.info("%s: trying implementation mirror at %s", ex, mirror_url) method = model.DownloadSource( impl, mirror_url, None, None, type='application/x-bzip-compressed-tar') continue # Retry raise except SafeException as ex: raise SafeException( "Error fetching {url} {version}: {ex}".format( url=impl.feed.url, version=impl.get_version(), ex=ex)) break self.handler.impl_added_to_store(impl) return download_impl(retrieval_method)
def download_impl(self, impl, retrieval_method, stores, force = False): """Download an implementation. @param impl: the selected implementation @type impl: L{model.ZeroInstallImplementation} @param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe) @type retrieval_method: L{model.RetrievalMethod} @param stores: where to store the downloaded implementation @type stores: L{zerostore.Stores} @type force: bool @rtype: L{tasks.Blocker}""" assert impl assert retrieval_method if isinstance(retrieval_method, DistributionSource): return retrieval_method.install(self.handler) from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair best = None for digest in impl.digests: alg_name, digest_value = parse_algorithm_digest_pair(digest) alg = manifest.algorithms.get(alg_name, None) if alg and (best is None or best.rating < alg.rating): best = alg required_digest = digest if best is None: if not impl.digests: raise SafeException(_("No <manifest-digest> given for '%(implementation)s' version %(version)s") % {'implementation': impl.feed.get_name(), 'version': impl.get_version()}) raise SafeException(_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s") % {'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version()}) @tasks.async def download_impl(method): original_exception = None while True: if not isinstance(method, Recipe): # turn an individual method into a single-step Recipe step = method method = Recipe() method.steps.append(step) try: blocker = self.cook(required_digest, method, stores, impl_hint = impl, dry_run = self.handler.dry_run, may_use_mirror = original_exception is None) yield blocker tasks.check(blocker) except download.DownloadError as ex: if original_exception: logger.info("Error from mirror: %s", ex) raise original_exception else: original_exception = ex mirror_url = self._get_impl_mirror(impl) if mirror_url is not None: logger.info("%s: trying implementation mirror at %s", ex, mirror_url) method = model.DownloadSource(impl, mirror_url, None, None, type = 'application/x-bzip-compressed-tar') continue # Retry raise except SafeException as ex: raise SafeException("Error fetching {url} {version}: {ex}".format( url = impl.feed.url, version = impl.get_version(), ex = ex)) break self.handler.impl_added_to_store(impl) return download_impl(retrieval_method)