def fetch_archives(
self,
progress=None, # type: Optional[AcquireProgress]
fetcher=None, # type: Optional[apt_pkg.Acquire]
allow_unauthenticated=None, # type: Optional[bool]
):
# type: (...) -> int
"""Fetch the archives for all packages marked for install/upgrade.
You can specify either an :class:`apt.progress.base.AcquireProgress()`
object for the parameter *progress*, or specify an already
existing :class:`apt_pkg.Acquire` object for the parameter *fetcher*.
The return value of the function is undefined. If an error occurred,
an exception of type :class:`FetchFailedException` or
:class:`FetchCancelledException` is raised.
The keyword-only parameter *allow_unauthenticated* specifies whether
to allow unauthenticated downloads. If not specified, it defaults to
the configuration option `APT::Get::AllowUnauthenticated`.
.. versionadded:: 0.8.0
"""
if progress is not None and fetcher is not None:
raise ValueError("Takes a progress or a an Acquire object")
if progress is None:
progress = apt.progress.text.AcquireProgress()
if fetcher is None:
fetcher = apt_pkg.Acquire(progress)
with self._archive_lock:
return self._fetch_archives(fetcher,
apt_pkg.PackageManager(self._depcache),
allow_unauthenticated)
def TSexec_download(self):
downloaded = []
success = False
if not self.updatePkgDatabase():
self.logger.error("Could not update apt package database")
else:
self.TSpurge_pkgs()
self.update_progress("Downloading")
self.apt_cache.upgrade()
required = self.apt_cache.required_download / (1024 * 1024)
downloaded = [pkg.name for pkg in self.apt_cache.get_changes()]
self.logger.debug(
"%d upgradable packages, %.1fMB required download space" %
(len(downloaded), required))
# Download packages
pm = apt_pkg.PackageManager(self.apt_cache._depcache)
fetcher = apt_pkg.Acquire(GetAcquireProgress(self))
try:
self.apt_cache._fetch_archives(fetcher, pm)
success = True
except Exception:
self.logger.error(traceback.format_exc())
if success:
self.update_progress("Ready to install")
self.logger.info("Successfully downloaded %s packages" %
len(downloaded))
else:
self.update_progress("Download failure")
self.logger.error("Failed downloading packages!")
return downloaded
def fetch_archives(self, progress=None, fetcher=None):
# type: (AcquireProgress, apt_pkg.Acquire) -> int
"""Fetch the archives for all packages marked for install/upgrade.
You can specify either an :class:`apt.progress.base.AcquireProgress()`
object for the parameter *progress*, or specify an already
existing :class:`apt_pkg.Acquire` object for the parameter *fetcher*.
The return value of the function is undefined. If an error occurred,
an exception of type :class:`FetchFailedException` or
:class:`FetchCancelledException` is raised.
.. versionadded:: 0.8.0
"""
if progress is not None and fetcher is not None:
raise ValueError("Takes a progress or a an Acquire object")
if progress is None:
progress = apt.progress.text.AcquireProgress()
if fetcher is None:
fetcher = apt_pkg.Acquire(progress)
self._get_archive_lock(fetcher)
return self._fetch_archives(fetcher,
apt_pkg.PackageManager(self._depcache))
def commit(
self,
fetch_progress=None, # type: Optional[AcquireProgress]
install_progress=None, # type: Optional[InstallProgress]
allow_unauthenticated=None, # type: Optional[bool]
):
# type: (...) -> bool
"""Apply the marked changes to the cache.
The first parameter, *fetch_progress*, refers to a FetchProgress()
object as found in apt.progress, the default being
apt.progress.FetchProgress().
The second parameter, *install_progress*, is a
apt.progress.InstallProgress() object.
The keyword-only parameter *allow_unauthenticated* specifies whether
to allow unauthenticated downloads. If not specified, it defaults to
the configuration option `APT::Get::AllowUnauthenticated`.
"""
# FIXME:
# use the new acquire/pkgmanager interface here,
# raise exceptions when a download or install fails
# and send proper error strings to the application.
# Current a failed download will just display "error"
# which is less than optimal!
if fetch_progress is None:
fetch_progress = apt.progress.base.AcquireProgress()
if install_progress is None:
install_progress = apt.progress.base.InstallProgress()
assert install_progress is not None
with apt_pkg.SystemLock():
pm = apt_pkg.PackageManager(self._depcache)
fetcher = apt_pkg.Acquire(fetch_progress)
with self._archive_lock:
while True:
# fetch archives first
res = self._fetch_archives(fetcher, pm,
allow_unauthenticated)
# then install
res = self.install_archives(pm, install_progress)
if res == pm.RESULT_COMPLETED:
break
elif res == pm.RESULT_FAILED:
raise SystemError("installArchives() failed")
elif res == pm.RESULT_INCOMPLETE:
pass
else:
raise SystemError("internal-error: unknown result "
"code from InstallArchives: %s" %
res)
# reload the fetcher for media swaping
fetcher.shutdown()
return (res == pm.RESULT_COMPLETED)
def required_download(self):
"""Get the size of the packages that are required to download."""
if self._records is None:
raise CacheClosedException(
"Cache object used after close() called")
pm = apt_pkg.PackageManager(self._depcache)
fetcher = apt_pkg.Acquire()
pm.get_archives(fetcher, self._list, self._records)
return fetcher.fetch_needed
def downloadArchives(self):
pkgAcquire, pkgSourceList = apt_pkg.Acquire(), apt_pkg.SourceList()
self.logger.verbose("Reading main source lists...")
pkgSourceList.read_main_list()
self.logger.verbose("Downloading archives...")
self.pkgManager = apt_pkg.PackageManager(self.depCache)
self.pkgManager.get_archives(pkgAcquire, pkgSourceList,
apt_pkg.PackageRecords(self.pkgCache))
pkgAcquire.run()
[package.update(install=False) for package in self.pkgList.values()]
def _test():
# type: () -> None
"""Internal test code."""
print("Cache self test")
apt_pkg.init()
cache = Cache(apt.progress.text.OpProgress())
cache.connect2("cache_pre_change", cache_pre_changed)
cache.connect2("cache_post_change", cache_post_changed)
print(("aptitude" in cache))
pkg = cache["aptitude"]
print(pkg.name)
print(len(cache))
for pkgname in cache.keys():
assert cache[pkgname].name == pkgname
cache.upgrade()
changes = cache.get_changes()
print(len(changes))
for pkg in changes:
assert pkg.name
# see if fetching works
for dirname in ["/tmp/pytest", "/tmp/pytest/partial"]:
if not os.path.exists(dirname):
os.mkdir(dirname)
apt_pkg.config.set("Dir::Cache::Archives", "/tmp/pytest")
pm = apt_pkg.PackageManager(cache._depcache)
fetcher = apt_pkg.Acquire(apt.progress.text.AcquireProgress())
cache._fetch_archives(fetcher, pm, None)
#sys.exit(1)
print("Testing filtered cache (argument is old cache)")
filtered = FilteredCache(cache)
filtered.cache.connect2("cache_pre_change", cache_pre_changed)
filtered.cache.connect2("cache_post_change", cache_post_changed)
filtered.cache.upgrade()
filtered.set_filter(MarkedChangesFilter())
print(len(filtered))
for pkgname in filtered.keys():
assert pkgname == filtered[pkgname].name
print(len(filtered))
print("Testing filtered cache (no argument)")
filtered = FilteredCache(progress=apt.progress.base.OpProgress())
filtered.cache.connect2("cache_pre_change", cache_pre_changed)
filtered.cache.connect2("cache_post_change", cache_post_changed)
filtered.cache.upgrade()
filtered.set_filter(MarkedChangesFilter())
print(len(filtered))
for pkgname in filtered.keys():
assert pkgname == filtered[pkgname].name
print(len(filtered))
def commit(self, fetch_progress=None, install_progress=None):
# type: (AcquireProgress, InstallProgress) -> bool
"""Apply the marked changes to the cache.
The first parameter, *fetch_progress*, refers to a FetchProgress()
object as found in apt.progress, the default being
apt.progress.FetchProgress().
The second parameter, *install_progress*, is a
apt.progress.InstallProgress() object.
"""
# FIXME:
# use the new acquire/pkgmanager interface here,
# raise exceptions when a download or install fails
# and send proper error strings to the application.
# Current a failed download will just display "error"
# which is less than optimal!
if fetch_progress is None:
fetch_progress = apt.progress.base.AcquireProgress()
if install_progress is None:
install_progress = apt.progress.base.InstallProgress()
assert install_progress is not None
with apt_pkg.SystemLock():
pm = apt_pkg.PackageManager(self._depcache)
fetcher = apt_pkg.Acquire(fetch_progress)
self._get_archive_lock(fetcher)
while True:
# fetch archives first
res = self._fetch_archives(fetcher, pm)
# then install
res = self.install_archives(pm, install_progress)
if res == pm.RESULT_COMPLETED:
break
elif res == pm.RESULT_FAILED:
raise SystemError("installArchives() failed")
elif res == pm.RESULT_INCOMPLETE:
pass
else:
raise SystemError("internal-error: unknown result code "
"from InstallArchives: %s" % res)
# reload the fetcher for media swaping
fetcher.shutdown()
return (res == pm.RESULT_COMPLETED)
def TSdownload_pkgs(self, pkglist):
'''Uses python-apt to download available packages'''
results = {}
self.set_state('DL')
apt_cache = self.apt_cache
apt_cache.open(None)
pm = apt_pkg.PackageManager(apt_cache._depcache)
fetcher = apt_pkg.Acquire(GetAcquireProgress(self))
for pkg_name in pkglist:
# mark packages for upgrade/install
pkg = apt_cache[pkg_name]
if pkg.isUpgradable:
pkg.markUpgrade()
elif not pkg.isInstalled:
pkg.markInstall()
try:
apt_cache._fetch_archives(fetcher, pm)
return True
except:
self.logger.error(traceback.format_exc())
return False
def commit_with_verify(self, cache, fetch_progress, install_progress):
# Hack around occasional undetected download errors in apt by doing
# our own verification pass at the end. See
# https://bugs.launchpad.net/bugs/922949. Unfortunately this means
# clone-and-hacking most of cache.commit ...
pm = apt_pkg.PackageManager(cache._depcache)
fetcher = apt_pkg.Acquire(fetch_progress)
while True:
# fetch archives first
res = cache._fetch_archives(fetcher, pm)
# manually verify all the downloads
syslog.syslog('Verifying downloads ...')
for item in fetcher.items:
with open(item.destfile, 'rb') as destfile:
st = os.fstat(destfile.fileno())
if st.st_size != item.filesize:
osextras.unlink_force(item.destfile)
raise IOError(
"%s size mismatch: %ld != %ld" %
(item.destfile, st.st_size, item.filesize))
# Mapping back to the package object is an utter pain.
# If we fail to find one, it's entirely possible it's a
# programming error and not a download error, so skip
# verification in such cases rather than failing.
destfile_base = os.path.basename(item.destfile)
try:
name, version, arch = destfile_base.split('_')
version = version.replace('%3a', ':')
arch = arch.split('.')[0]
if arch == 'all':
fullname = name
else:
fullname = '%s:%s' % (name, arch)
# This syntax only works on systems configured
# for multiarch, so check and fall back to the
# single-architecture syntax.
if fullname not in cache:
fullname = name
candidate = cache[fullname].versions[version]
except (KeyError, ValueError) as e:
syslog.syslog(
'Failed to find package object for %s: %s' %
(item.destfile, e))
continue
if candidate.sha256 is not None:
sha256 = hashlib.sha256()
for chunk in iter(lambda: destfile.read(16384), b''):
sha256.update(chunk)
if sha256.hexdigest() != candidate.sha256:
osextras.unlink_force(item.destfile)
raise IOError(
"%s SHA256 checksum mismatch: %s != %s" %
(item.destfile, sha256.hexdigest(),
candidate.sha256))
syslog.syslog('Downloads verified successfully')
# then install
res = cache.install_archives(pm, install_progress)
if res == pm.RESULT_COMPLETED:
break
elif res == pm.RESULT_FAILED:
raise SystemError("installArchives() failed")
elif res == pm.RESULT_INCOMPLETE:
pass
else:
raise SystemError("internal-error: unknown result code "
"from InstallArchives: %s" % res)
# reload the fetcher for media swapping
fetcher.shutdown()
return (res == pm.RESULT_COMPLETED)
def required_download(self):
""" get the size of the packages that are required to download """
pm = apt_pkg.PackageManager(self._depcache)
fetcher = apt_pkg.Acquire()
pm.get_archives(fetcher, self._list, self._records)
return fetcher.fetch_needed
def install(self):
"""
Installs the updates.
"""
if not self.cache:
return False
package_manager = apt_pkg.PackageManager(self.cache._depcache)
# Once the installation has been completed, there are three
# possible actions to do:
# - If the status is pm.RESULT_COMPLETED, the installation
# was successful and we can go home
# - If the status is pm.RESULT_INCOMPLETE or pm.RESULT_FAILED,
# we try again, fixing the possible broken packages.
# libchannels will re-try the operation for maximum 5 tries.
# - If the status is unknown, listeners are notified and
# everything ends here.
# Notify progress listeners that the installation is starting
self.packages_install_progress.start_update()
tries = 0
while tries < self.MAX_TRIES:
# Run the fetch operation again, just to be sure
if not self.fetch(package_manager):
return
# Then run the actual installation process
res = self.packages_install_progress.run(package_manager)
#res = package_manager.do_install()
if res in (package_manager.RESULT_INCOMPLETE,
package_manager.RESULT_FAILED):
logger.warning(
"System update %s, trying again (%s/%s)" %
("incomplete" if res == package_manager.RESULT_INCOMPLETE
else "failed", tries + 1, self.MAX_TRIES))
# Dpkg journal dirty?
if self.cache.dpkg_journal_dirty:
subprocess.call(["dpkg", "configure", "--all"])
# FIXME: add check for core packages
# Broken packages?
#if self.cache._depcache.broken_count > 0:
if True:
fixer = apt_pkg.ProblemResolver(self.cache._depcache)
fixer.resolve(True)
# FIXME: should notify this?
self.fetch(package_manager)
self.packages_install_progress.run(package_manager)
# Restore working state for the next upgrade run
self.restore_working_state()
elif res == package_manager.RESULT_COMPLETED:
# Everything completed successfully
logger.info("Clearing cache")
self.clear()
logger.info("System update completed")
self.packages_install_progress.finish_update()
return
else:
# Unknown error.
logger.error("Unknown error: %s" % res)
self.packages_install_failure_callback("Unknown error: %s" %
res)
return
tries += 1
# If we are here, the installation process failed and we were
# unable to continue.
logger.error("System upgrade failed: MAX_TRIES reached")
self.packages_install_failure_callback(
"System upgrade failed: MAX_TRIES reached")
import apt_pkg
deps = []
misspkg = "bison"
# reporting progress
progress = apt.progress.base.OpProgress()
# apt cache
cache = apt_pkg.Cache(progress)
# get the package
pkg = cache[misspkg]
deps.append(misspkg)
depcache = apt_pkg.DepCache(cache)
depcache.init(progress)
pkgmanager = apt_pkg.PackageManager(depcache)
def resolve(pkg, store):
versions = pkg.version_list
print(" - Got total {} versions of the package {}".format(
len(versions), pkg.name))
for i, v in enumerate(versions[0:1]):
# print("\nThe {} of all versions: --- {} to be download--- \n"
# .format(i+1, v.downloadable))
# get dependencies of thie package version
depends = v.depends_list.get("Depends")
if depends:
for d in depends:
# get name of the pkg
#apt_pkg.config.set("Debug::pkgDPkgPM","1");
#apt_pkg.config.set("Debug::pkgPackageManager","1");
#apt_pkg.config.set("Debug::pkgDPkgProgressReporting","1");
cache = apt_pkg.Cache()
depcache = apt_pkg.DepCache(cache)
recs = apt_pkg.PackageRecords(cache)
list = apt_pkg.SourceList()
list.read_main_list()
# show the amount fetch needed for a dist-upgrade
depcache.upgrade(True)
progress = apt.progress.text.AcquireProgress()
fetcher = apt_pkg.Acquire(progress)
pm = apt_pkg.PackageManager(depcache)
pm.get_archives(fetcher, list, recs)
print "%s (%s)" % (apt_pkg.size_to_str(
fetcher.fetch_needed), fetcher.fetch_needed)
actiongroup = apt_pkg.ActionGroup(depcache)
for pkg in cache.packages:
depcache.mark_keep(pkg)
try:
os.mkdir("/tmp/pyapt-test")
os.mkdir("/tmp/pyapt-test/partial")
except OSError:
pass
apt_pkg.config.set("Dir::Cache::archives", "/tmp/pyapt-test")
pkg = cache["3ddesktop"]
