def pick_digest(impl):
from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair
best = None
for digest in impl.digests:
alg_name, digest_value = parse_algorithm_digest_pair(digest)
alg = manifest.algorithms.get(alg_name, None)
if alg and (best is None or best.rating < alg.rating):
best = alg
required_digest = digest
if best is None:
if not impl.digests:
raise SafeException(
"No <manifest-digest> given for '%(implementation)s' version %(version)s"
% {
'implementation': impl.feed.get_name(),
'version': impl.get_version()
})
raise SafeException(
"Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s"
% {
'algorithms': impl.digests,
'implementation': impl.feed.get_name(),
'version': impl.get_version()
})
return required_digest
def splitID(id):
"""Take an ID in the form 'alg=value' and return a tuple (alg, value).
@type id: str
@rtype: (L{Algorithm}, str)
@raise BadDigest: if the algorithm isn't known or the ID has the wrong format."""
alg, digest = parse_algorithm_digest_pair(id)
return (get_algorithm(alg), digest)
def splitID(id):
"""Take an ID in the form 'alg=value' and return a tuple (alg, value).
@type id: str
@rtype: (L{Algorithm}, str)
@raise BadDigest: if the algorithm isn't known or the ID has the wrong format."""
alg, digest = parse_algorithm_digest_pair(id)
return (get_algorithm(alg), digest)
def do_audit(args):
"""audit [DIRECTORY]"""
if len(args) == 0:
audit_stores = stores.stores
else:
audit_stores = [zerostore.Store(x) for x in args]
audit_ls = []
total = 0
for a in audit_stores:
if os.path.isdir(a.dir):
items = sorted(os.listdir(a.dir))
audit_ls.append((a.dir, items))
total += len(items)
elif len(args):
raise SafeException(_("No such directory '%s'") % a.dir)
verified = 0
failures = []
i = 0
for root, impls in audit_ls:
print(_("Scanning %s") % root)
for required_digest in impls:
path = os.path.join(root, required_digest)
try:
(alg, digest
) = zerostore.parse_algorithm_digest_pair(required_digest)
except zerostore.BadDigest:
print(_("Skipping non-implementation directory %s") % path)
continue
i += 1
try:
msg = _("[%(done)d / %(total)d] Verifying %(digest)s") % {
'done': i,
'total': total,
'digest': required_digest
}
print(msg, end='')
sys.stdout.flush()
verify(path, required_digest)
print("\r" + (" " * len(msg)) + "\r", end='')
verified += 1
except zerostore.BadDigest as ex:
print()
failures.append(path)
print(str(ex))
if ex.detail:
print()
print(ex.detail)
if failures:
print('\n' + _("List of corrupted or modified implementations:"))
for x in failures:
print(x)
print()
print(_("Checked %d items") % i)
print(_("Successfully verified implementations: %d") % verified)
print(_("Corrupted or modified implementations: %d") % len(failures))
if failures:
sys.exit(1)
def do_audit(args):
"""audit [DIRECTORY]"""
if len(args) == 0:
audit_stores = stores.stores
else:
audit_stores = [zerostore.Store(x) for x in args]
audit_ls = []
total = 0
for a in audit_stores:
if os.path.isdir(a.dir):
items = sorted(os.listdir(a.dir))
audit_ls.append((a.dir, items))
total += len(items)
elif len(args):
raise SafeException(_("No such directory '%s'") % a.dir)
verified = 0
failures = []
i = 0
for root, impls in audit_ls:
print(_("Scanning %s") % root)
for required_digest in impls:
path = os.path.join(root, required_digest)
try:
(alg, digest) = zerostore.parse_algorithm_digest_pair(required_digest)
except zerostore.BadDigest:
print(_("Skipping non-implementation directory %s") % path)
continue
i += 1
try:
msg = _("[%(done)d / %(total)d] Verifying %(digest)s") % {
"done": i,
"total": total,
"digest": required_digest,
}
print(msg, end="")
sys.stdout.flush()
verify(path, required_digest)
print("\r" + (" " * len(msg)) + "\r", end="")
verified += 1
except zerostore.BadDigest as ex:
print()
failures.append(path)
print(str(ex))
if ex.detail:
print()
print(ex.detail)
if failures:
print("\n" + _("List of corrupted or modified implementations:"))
for x in failures:
print(x)
print()
print(_("Checked %d items") % i)
print(_("Successfully verified implementations: %d") % verified)
print(_("Corrupted or modified implementations: %d") % len(failures))
if failures:
sys.exit(1)
def pick_digest(impl):
from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair
best = None
for digest in impl.digests:
alg_name, digest_value = parse_algorithm_digest_pair(digest)
alg = manifest.algorithms.get(alg_name, None)
if alg and (best is None or best.rating < alg.rating):
best = alg
required_digest = digest
if best is None:
if not impl.digests:
raise SafeException("No <manifest-digest> given for '%(implementation)s' version %(version)s" %
{'implementation': impl.feed.get_name(), 'version': impl.get_version()})
raise SafeException("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s" %
{'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version()})
return required_digest
def check_manifest_and_rename(self, required_digest, unpack_dir, dry_run = False):
implementation = self.impl
sha1new = get_digest(unpack_dir, 'sha1new')
if not implementation.getAttribute('id'):
implementation.setAttribute('id', sha1new)
digests = [sha1new]
def add_digest(alg_name):
digest_id = get_digest(unpack_dir, alg_name)
digests.append(digest_id)
name, value = zerostore.parse_algorithm_digest_pair(digest_id)
elem.setAttribute(alg_name, value)
have_manifest_elem = False
for elem in implementation.getElementsByTagNameNS(namespaces.XMLNS_IFACE, 'manifest-digest'):
have_manifest_elem = True
have_digests = False
for attr_name, value in elem.attributes.items():
if value: continue
add_digest(attr_name)
have_digests = True
if not have_digests:
add_digest('sha256new')
if not have_manifest_elem:
print("WARNING: no <manifest-digest> element found")
best_rating = -1
best_digest = None
for digest_id in digests:
alg_name, value = zerostore.parse_algorithm_digest_pair(digest_id)
alg = manifest.get_algorithm(alg_name)
if alg.rating > best_rating:
best_rating = alg.rating
best_digest = digest_id
# Cache if necessary (saves downloading it again later)
stores = self.real_stores
if stores.lookup_maybe(digests) is None:
stores.add_dir_to_cache(best_digest, unpack_dir)
def splitID(id): """Take an ID in the form 'alg=value' and return a tuple (alg, value), where 'alg' is an instance of Algorithm and 'value' is a string. @raise BadDigest: if the algorithm isn't known or the ID has the wrong format.""" alg, digest = parse_algorithm_digest_pair(id) return (get_algorithm(alg), digest)
def optimise(impl_dir):
"""Scan an implementation cache directory for duplicate files, and
hard-link any duplicates together to save space.
@param impl_dir: a $cache/0install.net/implementations directory
@type impl_dir: str
@return: (unique bytes, duplicated bytes, already linked, manifest size)
@rtype: (int, int, int, int)"""
first_copy = {} # TypeDigest -> Path
dup_size = uniq_size = already_linked = man_size = 0
import random
from zeroinstall.zerostore import BadDigest, parse_algorithm_digest_pair
for x in range(10):
tmpfile = os.path.join(impl_dir, 'optimise-%d' % random.randint(0, 1000000))
if not os.path.exists(tmpfile):
break
else:
raise Exception(_("Can't generate unused tempfile name!"))
dirs = os.listdir(impl_dir)
total = len(dirs)
msg = ""
def clear():
print("\r" + (" " * len(msg)) + "\r", end='')
for i, impl in enumerate(dirs):
clear()
msg = _("[%(done)d / %(total)d] Reading manifests...") % {'done': i, 'total': total}
print(msg, end='')
sys.stdout.flush()
try:
alg, manifest_digest = parse_algorithm_digest_pair(impl)
except BadDigest:
logger.warning(_("Skipping non-implementation '%s'"), impl)
continue
manifest_path = os.path.join(impl_dir, impl, '.manifest')
try:
ms = open(manifest_path, 'rt')
except OSError as ex:
logger.warning(_("Failed to read manifest file '%(manifest_path)s': %(exception)s"), {'manifest': manifest_path, 'exception': str(ex)})
continue
if alg == 'sha1':
ms.close()
continue
man_size += os.path.getsize(manifest_path)
dir = ""
for line in ms:
if line[0] == 'D':
itype, path = line.split(' ', 1)
assert path.startswith('/')
dir = path[1:-1] # Strip slash and newline
continue
if line[0] == "S":
itype, digest, size, rest = line.split(' ', 3)
uniq_size += int(size)
continue
assert line[0] in "FX"
itype, digest, mtime, size, path = line.split(' ', 4)
path = path[:-1] # Strip newline
size = int(size)
key = (itype, digest, mtime, size)
loc_path = (impl, dir, path)
first_loc = first_copy.get(key, None)
if first_loc:
first_full = os.path.join(impl_dir, *first_loc)
new_full = os.path.join(impl_dir, *loc_path)
if _already_linked(first_full, new_full):
already_linked += size
else:
_link(first_full, new_full, tmpfile)
dup_size += size
else:
first_copy[key] = loc_path
uniq_size += size
ms.close()
clear()
return (uniq_size, dup_size, already_linked, man_size)
def toDOM(self):
"""Create a DOM document for the selected implementations.
The document gives the URI of the root, plus each selected implementation.
For each selected implementation, we record the ID, the version, the URI and
(if different) the feed URL. We also record all the bindings needed.
@return: a new DOM Document"""
from xml.dom import minidom, XMLNS_NAMESPACE
assert self.interface
impl = minidom.getDOMImplementation()
doc = impl.createDocument(XMLNS_IFACE, "selections", None)
root = doc.documentElement
root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE)
root.setAttributeNS(None, 'interface', self.interface)
root.setAttributeNS(None, 'command', self.command or "")
prefixes = Prefixes(XMLNS_IFACE)
for iface, selection in sorted(self.selections.items()):
selection_elem = doc.createElementNS(XMLNS_IFACE, 'selection')
selection_elem.setAttributeNS(None, 'interface',
selection.interface)
root.appendChild(selection_elem)
if selection.quick_test_file:
selection_elem.setAttributeNS(None, 'quick-test-file',
selection.quick_test_file)
if selection.quick_test_mtime:
selection_elem.setAttributeNS(
None, 'quick-test-mtime',
str(selection.quick_test_mtime))
for name, value in selection.attrs.items():
if ' ' in name:
ns, localName = name.split(' ', 1)
prefixes.setAttributeNS(selection_elem, ns, localName,
value)
elif name == 'stability':
pass
elif name == 'from-feed':
# Don't bother writing from-feed attr if it's the same as the interface
if value != selection.attrs['interface']:
selection_elem.setAttributeNS(None, name, value)
elif name not in ('main',
'self-test'): # (replaced by <command>)
selection_elem.setAttributeNS(None, name, value)
if selection.digests:
manifest_digest = doc.createElementNS(XMLNS_IFACE,
'manifest-digest')
for digest in selection.digests:
aname, avalue = zerostore.parse_algorithm_digest_pair(
digest)
assert ':' not in aname
manifest_digest.setAttribute(aname, avalue)
selection_elem.appendChild(manifest_digest)
for b in selection.bindings:
selection_elem.appendChild(b._toxml(doc, prefixes))
for dep in selection.dependencies:
if not isinstance(dep, model.InterfaceDependency): continue
dep_elem = doc.createElementNS(XMLNS_IFACE, 'requires')
dep_elem.setAttributeNS(None, 'interface', dep.interface)
selection_elem.appendChild(dep_elem)
for m in dep.metadata:
parts = m.split(' ', 1)
if len(parts) == 1:
ns = None
localName = parts[0]
dep_elem.setAttributeNS(None, localName,
dep.metadata[m])
else:
ns, localName = parts
prefixes.setAttributeNS(dep_elem, ns, localName,
dep.metadata[m])
for b in dep.bindings:
dep_elem.appendChild(b._toxml(doc, prefixes))
for command in selection.get_commands().values():
selection_elem.appendChild(command._toxml(doc, prefixes))
for ns, prefix in prefixes.prefixes.items():
root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:' + prefix, ns)
return doc
def add_digest(alg_name): digest_id = get_digest(unpack_dir, alg_name) digests.append(digest_id) name, value = zerostore.parse_algorithm_digest_pair(digest_id) elem.setAttribute(alg_name, value)
def toDOM(self):
"""Create a DOM document for the selected implementations.
The document gives the URI of the root, plus each selected implementation.
For each selected implementation, we record the ID, the version, the URI and
(if different) the feed URL. We also record all the bindings needed.
@return: a new DOM Document"""
from xml.dom import minidom, XMLNS_NAMESPACE
assert self.interface
impl = minidom.getDOMImplementation()
doc = impl.createDocument(XMLNS_IFACE, "selections", None)
root = doc.documentElement
root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns', XMLNS_IFACE)
root.setAttributeNS(None, 'interface', self.interface)
root.setAttributeNS(None, 'command', self.command or "")
prefixes = Prefixes(XMLNS_IFACE)
for iface, selection in sorted(self.selections.items()):
selection_elem = doc.createElementNS(XMLNS_IFACE, 'selection')
selection_elem.setAttributeNS(None, 'interface', selection.interface)
root.appendChild(selection_elem)
for name, value in selection.attrs.items():
if ' ' in name:
ns, localName = name.split(' ', 1)
prefixes.setAttributeNS(selection_elem, ns, localName, value)
elif name == 'stability':
pass
elif name == 'from-feed':
# Don't bother writing from-feed attr if it's the same as the interface
if value != selection.attrs['interface']:
selection_elem.setAttributeNS(None, name, value)
elif name not in ('main', 'self-test'): # (replaced by <command>)
selection_elem.setAttributeNS(None, name, value)
if selection.digests:
manifest_digest = doc.createElementNS(XMLNS_IFACE, 'manifest-digest')
for digest in selection.digests:
aname, avalue = zerostore.parse_algorithm_digest_pair(digest)
assert ':' not in aname
manifest_digest.setAttribute(aname, avalue)
selection_elem.appendChild(manifest_digest)
for b in selection.bindings:
selection_elem.appendChild(b._toxml(doc, prefixes))
for dep in selection.dependencies:
if not isinstance(dep, model.InterfaceDependency): continue
dep_elem = doc.createElementNS(XMLNS_IFACE, 'requires')
dep_elem.setAttributeNS(None, 'interface', dep.interface)
selection_elem.appendChild(dep_elem)
for m in dep.metadata:
parts = m.split(' ', 1)
if len(parts) == 1:
ns = None
localName = parts[0]
dep_elem.setAttributeNS(None, localName, dep.metadata[m])
else:
ns, localName = parts
prefixes.setAttributeNS(dep_elem, ns, localName, dep.metadata[m])
for b in dep.bindings:
dep_elem.appendChild(b._toxml(doc, prefixes))
for command in selection.get_commands().values():
selection_elem.appendChild(command._toxml(doc, prefixes))
for ns, prefix in prefixes.prefixes.items():
root.setAttributeNS(XMLNS_NAMESPACE, 'xmlns:' + prefix, ns)
return doc
def download_impl(self, impl, retrieval_method, stores, force=False):
"""Download an implementation.
@param impl: the selected implementation
@type impl: L{model.ZeroInstallImplementation}
@param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
@type retrieval_method: L{model.RetrievalMethod}
@param stores: where to store the downloaded implementation
@type stores: L{zerostore.Stores}
@rtype: L{tasks.Blocker}"""
assert impl
assert retrieval_method
if isinstance(retrieval_method, DistributionSource):
return retrieval_method.install(self.handler)
from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair
best = None
for digest in impl.digests:
alg_name, digest_value = parse_algorithm_digest_pair(digest)
alg = manifest.algorithms.get(alg_name, None)
if alg and (best is None or best.rating < alg.rating):
best = alg
required_digest = digest
if best is None:
if not impl.digests:
raise SafeException(
_("No <manifest-digest> given for '%(implementation)s' version %(version)s")
% {"implementation": impl.feed.get_name(), "version": impl.get_version()}
)
raise SafeException(
_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s")
% {"algorithms": impl.digests, "implementation": impl.feed.get_name(), "version": impl.get_version()}
)
@tasks.async
def download_impl(method):
original_exception = None
while True:
try:
if isinstance(method, DownloadSource):
blocker, stream = self.download_archive(
method, impl_hint=impl, may_use_mirror=original_exception is None
)
try:
yield blocker
tasks.check(blocker)
stream.seek(0)
if self.external_store:
self._add_to_external_store(required_digest, [method], [stream])
else:
self._add_to_cache(required_digest, stores, method, stream)
finally:
stream.close()
elif isinstance(method, Recipe):
blocker = self.cook(required_digest, method, stores, impl_hint=impl)
yield blocker
tasks.check(blocker)
else:
raise Exception(_("Unknown download type for '%s'") % method)
except download.DownloadError as ex:
if original_exception:
logger.info("Error from mirror: %s", ex)
raise original_exception
else:
original_exception = ex
mirror_url = self._get_impl_mirror(impl)
if mirror_url is not None:
logger.info("%s: trying implementation mirror at %s", ex, mirror_url)
method = model.DownloadSource(
impl, mirror_url, None, None, type="application/x-bzip-compressed-tar"
)
continue # Retry
raise
break
self.handler.impl_added_to_store(impl)
return download_impl(retrieval_method)
def download_impl(self, impl, retrieval_method, stores, force=False):
"""Download an implementation.
@param impl: the selected implementation
@type impl: L{model.ZeroInstallImplementation}
@param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
@type retrieval_method: L{model.RetrievalMethod}
@param stores: where to store the downloaded implementation
@type stores: L{zerostore.Stores}
@rtype: L{tasks.Blocker}"""
assert impl
assert retrieval_method
if isinstance(retrieval_method, DistributionSource):
return retrieval_method.install(self.handler)
from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair
best = None
for digest in impl.digests:
alg_name, digest_value = parse_algorithm_digest_pair(digest)
alg = manifest.algorithms.get(alg_name, None)
if alg and (best is None or best.rating < alg.rating):
best = alg
required_digest = digest
if best is None:
if not impl.digests:
raise SafeException(
_("No <manifest-digest> given for '%(implementation)s' version %(version)s"
) % {
'implementation': impl.feed.get_name(),
'version': impl.get_version()
})
raise SafeException(
_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s"
) % {
'algorithms': impl.digests,
'implementation': impl.feed.get_name(),
'version': impl.get_version()
})
@tasks. async
def download_impl(method):
original_exception = None
while True:
try:
if isinstance(method, DownloadSource):
blocker, stream = self.download_archive(
method,
impl_hint=impl,
may_use_mirror=original_exception is None)
try:
yield blocker
tasks.check(blocker)
stream.seek(0)
if self.external_store:
self._add_to_external_store(
required_digest, [method], [stream])
else:
self._add_to_cache(required_digest, stores,
method, stream)
finally:
stream.close()
elif isinstance(method, Recipe):
blocker = self.cook(required_digest,
method,
stores,
impl_hint=impl)
yield blocker
tasks.check(blocker)
else:
raise Exception(
_("Unknown download type for '%s'") % method)
except download.DownloadError as ex:
if original_exception:
logger.info("Error from mirror: %s", ex)
raise original_exception
else:
original_exception = ex
mirror_url = self._get_impl_mirror(impl)
if mirror_url is not None:
logger.info("%s: trying implementation mirror at %s",
ex, mirror_url)
method = model.DownloadSource(
impl,
mirror_url,
None,
None,
type='application/x-bzip-compressed-tar')
continue # Retry
raise
break
self.handler.impl_added_to_store(impl)
return download_impl(retrieval_method)
def optimise(impl_dir):
"""Scan an implementation cache directory for duplicate files, and
hard-link any duplicates together to save space.
@param impl_dir: a $cache/0install.net/implementations directory
@type impl_dir: str
@return: (unique bytes, duplicated bytes, already linked, manifest size)
@rtype: (int, int, int, int)"""
first_copy = {} # TypeDigest -> Path
dup_size = uniq_size = already_linked = man_size = 0
import random
from zeroinstall.zerostore import BadDigest, parse_algorithm_digest_pair
for x in range(10):
tmpfile = os.path.join(impl_dir, 'optimise-%d' % random.randint(0, 1000000))
if not os.path.exists(tmpfile):
break
else:
raise Exception(_("Can't generate unused tempfile name!"))
dirs = os.listdir(impl_dir)
total = len(dirs)
msg = ""
def clear():
print("\r" + (" " * len(msg)) + "\r", end='')
for i, impl in enumerate(dirs):
clear()
msg = _("[%(done)d / %(total)d] Reading manifests...") % {'done': i, 'total': total}
print(msg, end='')
sys.stdout.flush()
try:
alg, manifest_digest = parse_algorithm_digest_pair(impl)
except BadDigest:
logger.warn(_("Skipping non-implementation '%s'"), impl)
continue
manifest_path = os.path.join(impl_dir, impl, '.manifest')
try:
ms = open(manifest_path, 'rt')
except OSError as ex:
logger.warn(_("Failed to read manifest file '%(manifest_path)s': %(exception)s"), {'manifest': manifest_path, 'exception': str(ex)})
continue
if alg == 'sha1': continue
man_size += os.path.getsize(manifest_path)
dir = ""
for line in ms:
if line[0] == 'D':
itype, path = line.split(' ', 1)
assert path.startswith('/')
dir = path[1:-1] # Strip slash and newline
continue
if line[0] == "S":
itype, digest, size, rest = line.split(' ', 3)
uniq_size += int(size)
continue
assert line[0] in "FX"
itype, digest, mtime, size, path = line.split(' ', 4)
path = path[:-1] # Strip newline
size = int(size)
key = (itype, digest, mtime, size)
loc_path = (impl, dir, path)
first_loc = first_copy.get(key, None)
if first_loc:
first_full = os.path.join(impl_dir, *first_loc)
new_full = os.path.join(impl_dir, *loc_path)
if _already_linked(first_full, new_full):
already_linked += size
else:
_link(first_full, new_full, tmpfile)
dup_size += size
else:
first_copy[key] = loc_path
uniq_size += size
ms.close()
clear()
return (uniq_size, dup_size, already_linked, man_size)
def splitID(id):
"""Take an ID in the form 'alg=value' and return a tuple (alg, value),
where 'alg' is an instance of Algorithm and 'value' is a string.
@raise BadDigest: if the algorithm isn't known or the ID has the wrong format."""
alg, digest = parse_algorithm_digest_pair(id)
return (get_algorithm(alg), digest)
def download_impl(self, impl, retrieval_method, stores, force=False):
"""Download an implementation.
@param impl: the selected implementation
@type impl: L{model.ZeroInstallImplementation}
@param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
@type retrieval_method: L{model.RetrievalMethod}
@param stores: where to store the downloaded implementation
@type stores: L{zerostore.Stores}
@type force: bool
@rtype: L{tasks.Blocker}"""
assert impl
assert retrieval_method
if isinstance(retrieval_method, DistributionSource):
return retrieval_method.install(self.handler)
from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair
best = None
for digest in impl.digests:
alg_name, digest_value = parse_algorithm_digest_pair(digest)
alg = manifest.algorithms.get(alg_name, None)
if alg and (best is None or best.rating < alg.rating):
best = alg
required_digest = digest
if best is None:
if not impl.digests:
raise SafeException(
_("No <manifest-digest> given for '%(implementation)s' version %(version)s"
) % {
'implementation': impl.feed.get_name(),
'version': impl.get_version()
})
raise SafeException(
_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s"
) % {
'algorithms': impl.digests,
'implementation': impl.feed.get_name(),
'version': impl.get_version()
})
@tasks. async
def download_impl(method):
original_exception = None
while True:
if not isinstance(method, Recipe):
# turn an individual method into a single-step Recipe
step = method
method = Recipe()
method.steps.append(step)
try:
blocker = self.cook(
required_digest,
method,
stores,
impl_hint=impl,
dry_run=self.handler.dry_run,
may_use_mirror=original_exception is None)
yield blocker
tasks.check(blocker)
except download.DownloadError as ex:
if original_exception:
logger.info("Error from mirror: %s", ex)
raise original_exception
else:
original_exception = ex
mirror_url = self._get_impl_mirror(impl)
if mirror_url is not None:
logger.info("%s: trying implementation mirror at %s",
ex, mirror_url)
method = model.DownloadSource(
impl,
mirror_url,
None,
None,
type='application/x-bzip-compressed-tar')
continue # Retry
raise
except SafeException as ex:
raise SafeException(
"Error fetching {url} {version}: {ex}".format(
url=impl.feed.url,
version=impl.get_version(),
ex=ex))
break
self.handler.impl_added_to_store(impl)
return download_impl(retrieval_method)
def download_impl(self, impl, retrieval_method, stores, force = False):
"""Download an implementation.
@param impl: the selected implementation
@type impl: L{model.ZeroInstallImplementation}
@param retrieval_method: a way of getting the implementation (e.g. an Archive or a Recipe)
@type retrieval_method: L{model.RetrievalMethod}
@param stores: where to store the downloaded implementation
@type stores: L{zerostore.Stores}
@type force: bool
@rtype: L{tasks.Blocker}"""
assert impl
assert retrieval_method
if isinstance(retrieval_method, DistributionSource):
return retrieval_method.install(self.handler)
from zeroinstall.zerostore import manifest, parse_algorithm_digest_pair
best = None
for digest in impl.digests:
alg_name, digest_value = parse_algorithm_digest_pair(digest)
alg = manifest.algorithms.get(alg_name, None)
if alg and (best is None or best.rating < alg.rating):
best = alg
required_digest = digest
if best is None:
if not impl.digests:
raise SafeException(_("No <manifest-digest> given for '%(implementation)s' version %(version)s") %
{'implementation': impl.feed.get_name(), 'version': impl.get_version()})
raise SafeException(_("Unknown digest algorithms '%(algorithms)s' for '%(implementation)s' version %(version)s") %
{'algorithms': impl.digests, 'implementation': impl.feed.get_name(), 'version': impl.get_version()})
@tasks.async
def download_impl(method):
original_exception = None
while True:
if not isinstance(method, Recipe):
# turn an individual method into a single-step Recipe
step = method
method = Recipe()
method.steps.append(step)
try:
blocker = self.cook(required_digest, method, stores,
impl_hint = impl,
dry_run = self.handler.dry_run,
may_use_mirror = original_exception is None)
yield blocker
tasks.check(blocker)
except download.DownloadError as ex:
if original_exception:
logger.info("Error from mirror: %s", ex)
raise original_exception
else:
original_exception = ex
mirror_url = self._get_impl_mirror(impl)
if mirror_url is not None:
logger.info("%s: trying implementation mirror at %s", ex, mirror_url)
method = model.DownloadSource(impl, mirror_url,
None, None, type = 'application/x-bzip-compressed-tar')
continue # Retry
raise
except SafeException as ex:
raise SafeException("Error fetching {url} {version}: {ex}".format(
url = impl.feed.url,
version = impl.get_version(),
ex = ex))
break
self.handler.impl_added_to_store(impl)
return download_impl(retrieval_method)
