def generate_blacklist(self): manifest_remove = os.path.join(self.casper_path, 'filesystem.manifest-remove') manifest_desktop = os.path.join(self.casper_path, 'filesystem.manifest-desktop') manifest = os.path.join(self.casper_path, 'filesystem.manifest') if os.path.exists(manifest_remove) and os.path.exists(manifest): difference = set() with open(manifest_remove) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] difference.add(pkg.split()[0]) live_packages = set() with open(manifest) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] live_packages.add(pkg.split()[0]) desktop_packages = live_packages - difference elif os.path.exists(manifest_desktop) and os.path.exists(manifest): desktop_packages = set() with open(manifest_desktop) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] desktop_packages.add(pkg.split()[0]) live_packages = set() with open(manifest) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] live_packages.add(pkg.split()[0]) difference = live_packages - desktop_packages else: difference = set() cache = Cache() use_restricted = True try: if self.db.get('apt-setup/restricted') == 'false': use_restricted = False except debconf.DebconfError: pass if not use_restricted: for pkg in cache.keys(): if (cache[pkg].is_installed and cache[pkg].section.startswith('restricted/')): difference.add(pkg) # Keep packages we explicitly installed. keep = install_misc.query_recorded_installed() arch, subarch = install_misc.archdetect() # Less than ideal. Since we cannot know which bootloader we'll need # at file copy time, we should figure out why grub still fails when # apt-install-direct is present during configure_bootloader (code # removed). if arch in ('amd64', 'i386'): if subarch == 'efi': keep.add('grub-efi') keep.add('grub-efi-amd64') keep.add('grub-efi-amd64-signed') keep.add('shim-signed') keep.add('mokutil') keep.add('fwupdate-signed') install_misc.record_installed(['fwupdate-signed']) try: altmeta = self.db.get('base-installer/kernel/altmeta') if altmeta: altmeta = '-%s' % altmeta except debconf.DebconfError: altmeta = '' keep.add('linux-signed-generic%s' % altmeta) else: keep.add('grub') keep.add('grub-pc') elif (arch in ('armel', 'armhf') and subarch in ('omap', 'omap4', 'mx5')): keep.add('flash-kernel') keep.add('u-boot-tools') elif arch == 'powerpc': keep.add('yaboot') keep.add('hfsutils') # Even adding ubiquity as a depends to oem-config-{gtk,kde} doesn't # appear to force ubiquity and libdebian-installer4 to copy all of # their files, so this does the trick. try: if self.db.get('oem-config/enable') == 'true': keep.add('ubiquity') except (debconf.DebconfError, IOError): pass difference -= install_misc.expand_dependencies_simple( cache, keep, difference) # Consider only packages that don't have a prerm, and which can # therefore have their files removed without any preliminary work. difference = { x for x in difference if not os.path.exists('/var/lib/dpkg/info/%s.prerm' % x) } confirmed_remove = set() with cache.actiongroup(): for pkg in sorted(difference): if pkg in confirmed_remove: continue would_remove = install_misc.get_remove_list(cache, [pkg], recursive=True) if would_remove <= difference: confirmed_remove |= would_remove # Leave these marked for removal in the apt cache to # speed up further calculations. else: for removedpkg in would_remove: cachedpkg = install_misc.get_cache_pkg( cache, removedpkg) cachedpkg.mark_keep() difference = confirmed_remove if len(difference) == 0: del cache self.blacklist = {} return cmd = ['dpkg', '-L'] cmd.extend(difference) subp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) res = subp.communicate()[0].splitlines() u = {} for x in res: u[x] = 1 self.blacklist = u
def generate_blacklist(self): manifest_remove = os.path.join(self.casper_path, 'filesystem.manifest-remove') manifest_desktop = os.path.join(self.casper_path, 'filesystem.manifest-desktop') manifest = os.path.join(self.casper_path, 'filesystem.manifest') if os.path.exists(manifest_remove) and os.path.exists(manifest): difference = set() with open(manifest_remove) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] difference.add(pkg.split()[0]) live_packages = set() with open(manifest) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] live_packages.add(pkg.split()[0]) desktop_packages = live_packages - difference elif os.path.exists(manifest_desktop) and os.path.exists(manifest): desktop_packages = set() with open(manifest_desktop) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] desktop_packages.add(pkg.split()[0]) live_packages = set() with open(manifest) as manifest_file: for line in manifest_file: if line.strip() != '' and not line.startswith('#'): pkg = line.split(':')[0] live_packages.add(pkg.split()[0]) difference = live_packages - desktop_packages else: difference = set() cache = Cache() use_restricted = True try: if self.db.get('apt-setup/restricted') == 'false': use_restricted = False except debconf.DebconfError: pass if not use_restricted: for pkg in cache.keys(): if (cache[pkg].is_installed and cache[pkg].section.startswith('restricted/')): difference.add(pkg) # Keep packages we explicitly installed. keep = install_misc.query_recorded_installed() arch, subarch = install_misc.archdetect() # Less than ideal. Since we cannot know which bootloader we'll need # at file copy time, we should figure out why grub still fails when # apt-install-direct is present during configure_bootloader (code # removed). if arch in ('amd64', 'i386'): if subarch == 'efi': keep.add('grub-efi') keep.add('grub-efi-amd64') keep.add('grub-efi-amd64-signed') keep.add('shim-signed') keep.add('mokutil') keep.add('fwupdate-signed') install_misc.record_installed(['fwupdate-signed']) try: altmeta = self.db.get( 'base-installer/kernel/altmeta') if altmeta: altmeta = '-%s' % altmeta except debconf.DebconfError: altmeta = '' keep.add('linux-signed-generic%s' % altmeta) else: keep.add('grub') keep.add('grub-pc') elif (arch in ('armel', 'armhf') and subarch in ('omap', 'omap4', 'mx5')): keep.add('flash-kernel') keep.add('u-boot-tools') elif arch == 'powerpc': keep.add('yaboot') keep.add('hfsutils') # Even adding ubiquity as a depends to oem-config-{gtk,kde} doesn't # appear to force ubiquity and libdebian-installer4 to copy all of # their files, so this does the trick. try: if self.db.get('oem-config/enable') == 'true': keep.add('ubiquity') except (debconf.DebconfError, IOError): pass difference -= install_misc.expand_dependencies_simple( cache, keep, difference) # Consider only packages that don't have a prerm, and which can # therefore have their files removed without any preliminary work. difference = { x for x in difference if not os.path.exists('/var/lib/dpkg/info/%s.prerm' % x)} confirmed_remove = set() with cache.actiongroup(): for pkg in sorted(difference): if pkg in confirmed_remove: continue would_remove = install_misc.get_remove_list( cache, [pkg], recursive=True) if would_remove <= difference: confirmed_remove |= would_remove # Leave these marked for removal in the apt cache to # speed up further calculations. else: for removedpkg in would_remove: cachedpkg = install_misc.get_cache_pkg( cache, removedpkg) cachedpkg.mark_keep() difference = confirmed_remove if len(difference) == 0: del cache self.blacklist = {} return cmd = ['dpkg', '-L'] cmd.extend(difference) subp = subprocess.Popen( cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) res = subp.communicate()[0].splitlines() u = {} for x in res: u[x] = 1 self.blacklist = u
class GDebiCommon(object): # cprogress may be different in child classes def __init__(self, datadir, options, file=""): self.cprogress = None self._cache = None self.deps = "" self.version_info_title = "" self.version_info_msg = "" self._deb = None self._options = options self.install = [] self.remove = [] self.unauthenticated = 0 def openCache(self): self._cache = Cache(self.cprogress) if self._cache._depcache.broken_count > 0: self.error_header = _("Broken dependencies") self.error_body = _("Your system has broken dependencies. " "This application can not continue until " "this is fixed. " "To fix it run 'pkexec synaptic' or " "'sudo apt-get install -f' " "in a terminal window.") return False return True def open(self, file, downloaded=False): file = os.path.abspath(file) klass = DebPackage if file.endswith(".click"): klass = ClickPackage try: self._deb = klass(file, self._cache, downloaded) except (IOError, SystemError, ValueError) as e: logging.debug("open failed with %s" % e) mimetype = guess_type(file) if (mimetype[0] != None and mimetype[0] != "application/vnd.debian.binary-package"): self.error_header = _( "'%s' is not a Debian package") % os.path.basename(file) self.error_body = _( "The MIME type of this file is '%s' " "and can not be installed on this system.") % mimetype[0] return False else: self.error_header = _( "Could not open '%s'") % os.path.basename(file) self.error_body = _( "The package might be corrupted or you are not " "allowed to open the file. Check the permissions " "of the file.") return False def compareDebWithCache(self): # check if the package is available in the normal sources as well res = self._deb.compare_to_version_in_cache(use_installed=False) if not self._options.non_interactive and res != DebPackage.VERSION_NONE: try: pkg = self._cache[self._deb.pkgname] except (KeyError, TypeError): return if self._deb.downloaded: self.version_info_title = "" self.version_info_msg = "" return # FIXME: make this strs better if res == DebPackage.VERSION_SAME: if pkg.candidate and pkg.candidate.downloadable: self.version_info_title = _( "Same version is available in a software channel") self.version_info_msg = _( "You are recommended to install the software " "from the channel instead.") elif res == DebPackage.VERSION_NEWER: if pkg.candidate and pkg.candidate.downloadable: self.version_info_title = _( "An older version is available in a software channel") self.version_info_msg = _( "Generally you are recommended to install " "the version from the software channel, since " "it is usually better supported.") elif res == DebPackage.VERSION_OUTDATED: if pkg.candidate and pkg.candidate.downloadable: self.version_info_title = _( "A later version is available in a software " "channel") self.version_info_msg = _( "You are strongly advised to install " "the version from the software channel, since " "it is usually better supported.") def compareProvides(self): provides = set() broken_provides = set() try: pkg = self._cache[self._deb.pkgname].installed except (KeyError, TypeError): pkg = None if pkg: if pkg.provides: for p in self._deb.provides: for i in p: provides.add(i[0]) provides = set(pkg.provides).difference(provides) if provides: for package in list(self._cache.keys()): if self._cache[package].installed: for dep in self._cache[package].installed.dependencies: for d in dep.or_dependencies: if d.name in provides: broken_provides.add(d.name) return broken_provides def download_package(self): dirname = os.path.abspath(os.path.dirname(self._deb.filename)) package = self._cache[self._deb.pkgname].candidate pkgname = os.path.basename(package.filename) if package.downloadable: if not os.access(dirname, os.W_OK): dirname = "/tmp" if not os.path.exists(os.path.join(dirname, pkgname)): package.fetch_binary(dirname) self.open(os.path.join(dirname, pkgname), True) return True def get_changes(self): (self.install, self.remove, self.unauthenticated) = self._deb.required_changes self.deps = "" if len(self.remove) == len(self.install) == 0: self.deps = _("All dependencies are satisfied") if len(self.remove) > 0: # FIXME: use ngettext here self.deps += _("Requires the <b>removal</b> of %s packages\n" ) % len(self.remove) if len(self.install) > 0: self.deps += _("Requires the installation of %s packages") % len( self.install) return True
class GDebiCommon(object): # cprogress may be different in child classes def __init__(self, datadir, options, file=""): self.cprogress = None self._cache = None self.deps = "" self.version_info_title = "" self.version_info_msg = "" self._deb = None self._options = options self.install = [] self.remove = [] self.unauthenticated = 0 def openCache(self): self._cache = Cache(self.cprogress) if self._cache._depcache.broken_count > 0: self.error_header = _("Broken dependencies") self.error_body = _("Your system has broken dependencies. " "This application can not continue until " "this is fixed. " "To fix it run 'gksudo synaptic' or " "'sudo apt-get install -f' " "in a terminal window.") return False return True def open(self, file, downloaded=False): file = os.path.abspath(file) klass = DebPackage if file.endswith(".click"): klass = ClickPackage try: self._deb = klass(file, self._cache, downloaded) except (IOError, SystemError, ValueError) as e: logging.debug("open failed with %s" % e) mimetype=guess_type(file) if (mimetype[0] != None and mimetype[0] != "application/vnd.debian.binary-package"): self.error_header = _("'%s' is not a Debian package") % os.path.basename(file) self.error_body = _("The MIME type of this file is '%s' " "and can not be installed on this system.") % mimetype[0] return False else: self.error_header = _("Could not open '%s'") % os.path.basename(file) self.error_body = _("The package might be corrupted or you are not " "allowed to open the file. Check the permissions " "of the file.") return False def compareDebWithCache(self): # check if the package is available in the normal sources as well res = self._deb.compare_to_version_in_cache(use_installed=False) if not self._options.non_interactive and res != DebPackage.VERSION_NONE: try: pkg = self._cache[self._deb.pkgname] except (KeyError, TypeError): return if self._deb.downloaded: self.version_info_title = "" self.version_info_msg = "" return # FIXME: make this strs better if res == DebPackage.VERSION_SAME: if pkg.candidate and pkg.candidate.downloadable: self.version_info_title = _("Same version is available in a software channel") self.version_info_msg = _("You are recommended to install the software " "from the channel instead.") elif res == DebPackage.VERSION_NEWER: if pkg.candidate and pkg.candidate.downloadable: self.version_info_title = _("An older version is available in a software channel") self.version_info_msg = _("Generally you are recommended to install " "the version from the software channel, since " "it is usually better supported.") elif res == DebPackage.VERSION_OUTDATED: if pkg.candidate and pkg.candidate.downloadable: self.version_info_title = _("A later version is available in a software " "channel") self.version_info_msg = _("You are strongly advised to install " "the version from the software channel, since " "it is usually better supported.") def compareProvides(self): provides = set() broken_provides = set() try: pkg = self._cache[self._deb.pkgname].installed except (KeyError, TypeError): pkg = None if pkg: if pkg.provides: for p in self._deb.provides: for i in p: provides.add(i[0]) provides = set(pkg.provides).difference(provides) if provides: for package in list(self._cache.keys()): if self._cache[package].installed: for dep in self._cache[package].installed.dependencies: for d in dep.or_dependencies: if d.name in provides: broken_provides.add(d.name) return broken_provides def download_package(self): dirname = os.path.abspath(os.path.dirname(self._deb.filename)) package = self._cache[self._deb.pkgname].candidate pkgname = os.path.basename(package.filename) if package.downloadable: if not os.access(dirname, os.W_OK): dirname = "/tmp" if not os.path.exists(os.path.join(dirname, pkgname)): package.fetch_binary(dirname) self.open(os.path.join(dirname, pkgname), True) return True def get_changes(self): (self.install, self.remove, self.unauthenticated) = self._deb.required_changes self.deps = "" if len(self.remove) == len(self.install) == 0: self.deps = _("All dependencies are satisfied") if len(self.remove) > 0: # FIXME: use ngettext here self.deps += _("Requires the <b>removal</b> of %s packages\n") % len(self.remove) if len(self.install) > 0: self.deps += _("Requires the installation of %s packages") % len(self.install) return True def try_acquire_lock(self): " check if we can lock the apt database " try: apt_pkg.pkgsystem_lock() except SystemError: self.error_header = _("Only one software management tool is allowed to" " run at the same time") self.error_body = _("Please close the other application e.g. 'Update " "Manager', 'aptitude' or 'Synaptic' first.") return False apt_pkg.pkgsystem_unlock() return True def acquire_lock(self): " lock the pkgsystem for install " # sanity check ( moved here ) if self._deb is None: return False # check if we can lock the apt database try: apt_pkg.pkgsystem_lock() except SystemError: self.error_header = _("Only one software management tool is allowed to" " run at the same time") self.error_body = _("Please close the other application e.g. 'Update " "Manager', 'aptitude' or 'Synaptic' first.") return False return True def release_lock(self): " release the pkgsystem lock " apt_pkg.pkgsystem_lock() return True
class _AptChangelog(): def __init__(self, interactive:bool=False): self.interactive = interactive # constants # apt uses MB rather than MiB, so let's stay consistent self.MB = 1000 ** 2 # downloads larger than this require confirmation or fail self.max_download_size_default = 1.5 * self.MB self.max_download_size = self.max_download_size_default max_download_size_msg_template = "\ To retrieve the full changelog, %s MB have to be downloaded.\n%s\ \n\ Proceed with the download?" self.max_download_size_msg_lc = max_download_size_msg_template % ("%.1f", "Otherwise we will try to retrieve just the last change.\n") self.max_download_size_msg = max_download_size_msg_template % ("%.1f","") self.max_download_size_msg_unknown = max_download_size_msg_template % ("an unknown amount of", "") self.apt_cache = None self.apt_cache_date = None self.candidate = None # get apt's configuration apt_pkg.init_config() if apt_pkg.config.exists("Acquire::Changelogs::URI::Origin"): self.apt_origins = apt_pkg.config.subtree("Acquire::Changelogs::URI::Origin") else: self.apt_origins = None if apt_pkg.config.exists("Dir::Cache::pkgcache"): self.apt_cache_path = apt_pkg.config.find_dir("Dir::Cache") self.pkgcache = apt_pkg.config.find_file("Dir::Cache::pkgcache") else: self.apt_cache = "invalid" if (self.apt_cache or not os.path.isdir(self.apt_cache_path) or not os.path.isfile(self.pkgcache) ): print("E: Invalid APT configuration found, try to run `apt update` first", file=sys.stderr) self.close(99) def get_cache_date(self): if os.path.isfile(self.pkgcache): return os.path.getmtime(self.pkgcache) return None def refresh_cache(self): cache_date = self.get_cache_date() if not self.apt_cache: self.apt_cache = Cache() self.apt_cache_date = cache_date elif cache_date != self.apt_cache_date: self.apt_cache.open(None) self.apt_cache_date = cache_date def drop_cache(self): if self.candidate: self.candidate = None self.apt_cache = None def get_changelog(self, pkg_name:str, no_local:bool=False): self.refresh_cache() self.candidate = self.parse_package_metadata(pkg_name) # parse the package's origin if not self.candidate.downloadable: origin = "local_package" elif self.candidate.origin == "linuxmint": origin = "linuxmint" elif self.candidate.origin.startswith("LP-PPA-"): origin = "LP-PPA" elif self.apt_origins and self.candidate.origin in self.apt_origins.list(): origin = "APT" else: origin = "unsupported" # Check for changelog of installed package first has_local_changelog = False uri = None if not no_local and self.candidate.is_installed: if _DEBUG: print("Package is installed...") uri = self.get_changelog_from_filelist( self.candidate.installed_files, local=True) # Ubuntu kernel workarounds if self.candidate.origin == "Ubuntu": if self.candidate.source_name == "linux-signed": uri = uri.replace("linux-image","linux-modules") if self.candidate.source_name == "linux-meta": uri = None if uri and not os.path.isfile(uri): uri = None # Do nothing if local changelog exists if uri: has_local_changelog = True # all origins that APT supports elif origin == 'APT': uri = self.get_apt_changelog_uri( self.apt_origins.get(self.candidate.origin)) r = self.check_url(uri) if not r: self.exit_on_fail(2) # Linux Mint repo elif origin == 'linuxmint': # Mint repos don't have .debian.tar.xz files, only full packages, so # check the package cache first base_uri, _ = os.path.split(self.candidate.uri) r, uri = self.get_changelog_uri(base_uri) if not r: # fall back to last change info for the source package # Mint's naming scheme seems to be using amd64 unless source # is i386 only, we always check amd64 first base_uri = "http://packages.linuxmint.com/dev/%s_%s_%s.changes" uri = base_uri % (self.candidate.source_name, self.candidate.source_version, "amd64") r = self.check_url(uri, False) if not r: uri = base_uri % (self.candidate.source_name, self.candidate.source_version, "i386") r = self.check_url(uri, False) if not r: self.exit_on_fail(3) # Launchpad PPA elif origin == 'LP-PPA': ppa_owner, ppa_name, _ = \ self.candidate.uri.split("ppa.launchpad.net/")[1].split("/", 2) base_uri = "http://ppa.launchpad.net/%(owner)s/%(name)s/ubuntu/pool/main/%(source_prefix)s/%(source_name)s" % \ { "owner": ppa_owner, "name": ppa_name, "source_prefix": self.source_prefix(), "source_name": self.candidate.source_name } r, uri = self.get_changelog_uri(base_uri) if not r: # fall back to last change info only uri = "https://launchpad.net/~%(owner)s/+archive/ubuntu/%(name)s/+files/%(source_name)s_%(source_version)s_source.changes" % \ { "owner" : ppa_owner, "name": ppa_name, "source_name": self.candidate.source_name, "source_version": self.candidate.source_version } r = self.check_url(uri, False) if not r: self.exit_on_fail(4) # Not supported origin elif origin == 'unsupported': if _DEBUG: print("Unsupported Package") base_uri, _ = os.path.split(self.candidate.uri) r, uri = self.get_changelog_uri(base_uri) if not r: self.exit_on_fail(5) # Locally installed package without local changelog or remote # source, hope it's cached and contains a changelog elif origin == 'local_package': uri = self.apt_cache_path + self.candidate.filename if not os.path.isfile(uri): self.exit_on_fail(6) # Changelog downloading, extracting and processing: changelog = "" # local changelog if has_local_changelog and not no_local: if _DEBUG: print("Using local changelog:",uri) try: filename = os.path.basename(uri) # determine file type by name/extension # as per debian policy 4.4 the encoding must be UTF-8 # as per policy 12.7 the name must be changelog.Debian.gz or # changelog.gz (deprecated) if filename.lower().endswith('.gz'): changelog = gzip.open(uri,'r').read().decode('utf-8') elif filename.lower().endswith('.xz'): # just in case / future proofing changelog = lzma.open(uri,'r').read().decode('utf-8') elif filename.lower() == 'changelog': changelog = open(uri, 'r').read().encode().decode('utf-8') else: raise ValueError('Unknown changelog format') except Exception as e: _generic_exception_handler(e) self.exit_on_fail(1) # APT-format changelog, download directly # - unfortunately this is slow since the servers support no compression elif origin == "APT": if _DEBUG: print("Downloading: %s (%.2f MB)" % (uri, r.length / self.MB)) changelog = r.text r.close() # last change changelog, download directly elif uri.endswith('.changes'): if _DEBUG: print("Downloading: %s (%.2f MB)" % (uri, r.length / self.MB)) changes = r.text.split("Changes:")[1].split("Checksums")[0].split("\n") r.close() for change in changes: change = change.strip() if change: if change == ".": change = "" changelog += change + "\n" # compressed binary source, download and extract changelog else: source_is_cache = uri.startswith(self.apt_cache_path) if _DEBUG: print("Using cached package:" if source_is_cache else "Downloading: %s (%.2f MB)" % (uri, r.length / self.MB)) try: if not source_is_cache: # download stream to temporary file tmpFile = tempfile.NamedTemporaryFile(prefix="apt-changelog-") if self.interactive and r.length: # download chunks with progress indicator recv_length = 0 blocks = 60 for data in r.iter_content(chunk_size=16384): recv_length += len(data) tmpFile.write(data) recv_pct = recv_length / r.length recv_blocks = int(blocks * recv_pct) print("\r[%(progress)s%(spacer)s] %(percentage).1f%%" % { "progress": "=" * recv_blocks, "spacer": " " * (blocks - recv_blocks), "percentage": recv_pct * 100 }, end="", flush=True) # clear progress bar when done print("\r" + " " * (blocks + 10), end="\r", flush=True) else: # no content-length or non-interactive, download in one go # up to the configured max_download_size, ask only when # exceeded r.raw.decode_content = True size = 0 size_exceeded = False while True: buf = r.raw.read(16*1024) if not size_exceeded: size += len(buf) if size > self.max_download_size: if not self.user_confirm(self.max_download_size_msg_unknown): r.close() tmpFile.close() return "" else: size_exceeded = True if not buf: break tmpFile.write(buf) r.close() tmpFile.seek(0) if uri.endswith(".deb"): # process .deb file if source_is_cache: f = uri else: f = tmpFile.name # We could copy the downloaded .deb files to the apt # cache here but then we'd need to run the script elevated: # shutil.copy(f, self.apt_cache_path + os.path.basename(uri)) deb = DebPackage(f) changelog_file = self.get_changelog_from_filelist(deb.filelist) if changelog_file: changelog = deb.data_content(changelog_file) if changelog.startswith('Automatically decompressed:'): changelog = changelog[29:] else: raise ValueError('Malformed Debian package') elif uri.endswith(".diff.gz"): # Ubuntu partner repo has .diff.gz files, # we can extract a changelog from that data = gzip.open(tmpFile.name, "r").read().decode('utf-8') additions = data.split("+++") for addition in additions: lines = addition.split("\n") if "/debian/changelog" in lines[0]: for line in lines[2:]: if line.startswith("+"): changelog += "%s\n" % line[1:] else: break if not changelog: raise ValueError('No changelog in .diff.gz') else: # process .tar.xz file with tarfile.open(fileobj=tmpFile, mode="r:xz") as tar: changelog_file = self.get_changelog_from_filelist( [s.name for s in tar.getmembers() if s.type in (b"0", b"2")]) if changelog_file: changelog = tar.extractfile(changelog_file).read().decode() else: raise ValueError('No changelog in source package') except Exception as e: _generic_exception_handler(e) self.exit_on_fail(520) if 'tmpFile' in vars(): try: tmpFile.close() except Exception as e: _generic_exception_handler(e) # ALL DONE return changelog def parse_package_metadata(self, pkg_name:str): """ Creates the self.candidate object based on package name=version/release Wildcard matching is only used for version and release, and only the first match is processed. """ # parse =version declaration if "=" in pkg_name: (pkg_name, pkg_version) = pkg_name.split("=", 1) pkg_release = None # parse /release declaration (only if no version specified) elif "/" in pkg_name: (pkg_name, pkg_release) = pkg_name.split("/", 1) pkg_version = None else: pkg_version = None pkg_release = None # check if pkg_name exists # unlike apt no pattern matching, a single exact match only if pkg_name in self.apt_cache.keys(): pkg = self.apt_cache[pkg_name] else: print("E: Unable to locate package %s" % pkg_name, file=sys.stderr) self.close(13) # get package data _candidate = None candidate = None if pkg_release or pkg_version: match_found = False for _pkg in pkg.versions: if pkg_version: if fnmatch.fnmatch(_pkg.version, pkg_version): match_found = True else: for _origin in _pkg.origins: if fnmatch.fnmatch(_origin.archive, pkg_release): match_found = True if match_found: _candidate = _pkg break if not match_found: if pkg_release: print('E: Release "%s" is unavailable for "%s"' % (pkg_release, pkg.name), file=sys.stderr) else: print('E: Version "%s" is unavailable for "%s"' % (pkg_version, pkg.name), file=sys.stderr) self.close(14) else: _candidate = pkg.candidate candidate = _Package( version = _candidate.version, name = _candidate.package.name, fullname = None, architecture = pkg.architecture, source_name = _candidate.source_name, source_version = _candidate.source_version, uri = _candidate.uri, filename = os.path.basename(_candidate.filename), origin = _candidate.origins[0].origin, component = _candidate.origins[0].component, downloadable = _candidate.downloadable, is_installed = _candidate.is_installed, dependencies = _candidate.dependencies ) if candidate.is_installed: candidate.installed_files = pkg.installed_files candidate.source_version_raw = candidate.source_version if ":" in candidate.source_version: candidate.source_version = candidate.source_version.split(":", 1)[1] return candidate def check_url(self, url:str, check_size:bool=True, stream:bool=True, msg:str=None): """ True if url can be downloaded and fits size requirements """ if _DEBUG: print("Checking:", url) try: _r = requests.get(url, stream=stream, timeout=5) except Exception as e: _generic_exception_handler(e) else: if _r: length = _r.headers.get("Content-Length") if length: _r.length = int(length) else: _r.length = 0 if (not check_size or not (check_size and _r.length > self.max_download_size and not self.user_confirm( (self.max_download_size_msg_lc if not msg else msg) % (_r.length / self.MB)) )): return _r if '_r' in vars(): _r.close() return False @staticmethod def close(err:int=0): """ Exit """ sys.exit(err) def exit_on_fail(self, err:int=404): """ Prints error message and calls self.close() """ try: details = "Changelog unavailable for %s=%s" %\ (self.candidate.source_name, self.candidate.source_version_raw) except AttributeError: details = "" print("E: Failed to fetch changelog. %s" % details, file=sys.stderr) self.close(err) @staticmethod def strtobool (val): val = val.lower() if val in ('y', 'yes'): return True elif val in ('n', 'no'): return False else: raise ValueError("Invalid response value %r" % (val,)) def user_confirm(self, q:str): """ returns bool (always False in non-interactive mode) """ if not self.interactive: if _DEBUG: print("Maximum size exceeded, skipping in non-interactive mode") return False print('%s [y/n] ' % q, end='') while True: try: response = self.strtobool(input()) print("") return response except ValueError: print('Invalid response. Try again [y/n]: ', end='') except KeyboardInterrupt: pass def get_deb_or_tar(self, uri_tar:str=None): """ Returns request and URI of the preferred source The choice is made based on availability and size. If .deb is smaller than comparison_trigger_size, or if check_tar is False, then .deb is always selected. """ comparison_trigger_size = 50000 r_deb = self.check_url(self.candidate.uri, False) if r_deb: if uri_tar and r_deb.length > comparison_trigger_size: # try for .tar.xz r_tar = self.check_url(uri_tar, False) # validate and compare sizes if r_tar and r_tar.length < r_deb.length: _r = r_tar r_deb.close() else: _r = r_deb if r_tar: r_tar.close() else: _r = r_deb if (not _r.length > self.max_download_size or self.user_confirm(self.max_download_size_msg_lc % (_r.length / self.MB)) ): return (_r, _r.url) return (False, "") def get_changelog_from_filelist(self, filelist:list, local:bool=False): """ Returns hopefully the correct "changelog" or an empty string. We should not need to be searching because the debian policy says it must be at debian/changelog for source packages but not all seem to adhere to the policy: https://www.debian.org/doc/debian-policy/ch-source.html#debian-changelog-debian-changelog """ files = [s for s in filelist if "changelog" in s.lower()] if local: testpath = '/usr/share/doc/%s/changelog' % self.candidate.name for item in files: if item.lower().startswith(testpath): return item else: testpath = 'debian/changelog' if testpath in files: return testpath testpath = 'recipe/debian/changelog' if testpath in files: return testpath testpath = 'usr/share/doc/%s/changelog' % self.candidate.name for item in files: if item.lower().startswith(testpath): return item # no hits in the standard locations, let's try our luck in # random locations at the risk of getting the wrong file for item in files: if os.path.basename(item).lower().startswith("changelog"): return item return None def get_apt_changelog_uri(self, uri_template:str): """ Returns URI based on provided apt changelog URI template. Emulates apt's std::string pkgAcqChangelog::URI The template must contain the @CHANGEPATH@ variable, which will be expanded to COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER Component is omitted for releases without one (= flat-style repositories). """ source_version = self.candidate.source_version def get_kernel_version_from_meta_package(pkg): for dependency in pkg.dependencies: if not dependency.target_versions: if _DEBUG: print("W: Kernel dependency not found:", dependency) return None deppkg = dependency.target_versions[0] if deppkg.source_name in ("linux", "linux-signed"): return deppkg.source_version if deppkg.source_name == "linux-meta": _pkg = self.parse_package_metadata(str(deppkg)) return get_kernel_version_from_meta_package(_pkg) return None # Ubuntu kernel meta package workaround if (self.candidate.origin == "Ubuntu" and self.candidate.source_name == "linux-meta" ): _source_version = get_kernel_version_from_meta_package(self.candidate) if _source_version: source_version = _source_version self.candidate.source_name = "linux" # Ubuntu signed kernel workaround if (self.candidate.origin == "Ubuntu" and self.candidate.source_name == "linux-signed" ): self.candidate.source_name = "linux" # XXX: Debian does not seem to reliably keep changelogs for previous # (kernel) versions, so should we always look for the latest # version instead on Debian? apt does not do this but the # packages.debian.org website shows the latest version in the # selected archive # strip epoch if ":" in source_version: source_version = source_version.split(":", 1)[1] # the path is: COMPONENT/SRC/SRCNAME/SRCNAME_SRCVER, e.g. # main/a/apt/apt_1.1 or contrib/liba/libapt/libapt_2.0 return uri_template.replace('@CHANGEPATH@', "%(component)s%(source_prefix)s/%(source_name)s/%(source_name)s_%(source_version)s" % { "component": self.candidate.component + "/" if \ self.candidate.component and \ self.candidate.component != "" else "", "source_prefix": self.source_prefix(), "source_name": self.candidate.source_name, "source_version": source_version }) def source_prefix(self, source_name:str=None): """ Return prefix used for build repository URL """ if not source_name: source_name = self.candidate.source_name return source_name[0] if not source_name.startswith("lib") else \ source_name[:4] def parse_dsc(self, url:str): """ Returns filename or None """ _r = self.check_url(url, False, False) if _r: target = "" lines = _r.text.split("Files:", 1)[1].split(":", 1)[0].split("-----BEGIN", 1)[0].split("\n") target = [s.strip() for s in lines if s.strip().lower().endswith('.debian.tar.xz')] if not target: target = [s.strip() for s in lines if s.strip().lower().endswith('.diff.gz')] if not target: target = [s.strip() for s in lines if s.strip().lower().endswith('.tar.xz')] # don't even test for .tar.gz, it will be too big compared to the .deb # if not target: # target = [s.strip() for s in lines if s.strip().lower().endswith('.tar.gz')] if target: return target[0].split()[-1] elif _DEBUG: print(".dsc parse error for", url) return None def get_changelog_uri(self, base_uri:str): """ Tries to find a changelog in files listed in .dsc, locally cached packages as well as the remote .deb file Returns r and uri """ uri = None # XXX: For APT sources we could just read the apt_pkg.SourceRecords() # directly, if available, which it is not for most users, so # probably not worth it target_filename = self.parse_dsc("%s/%s_%s.dsc" % (base_uri, self.candidate.source_name, self.candidate.source_version)) # get .debian.tar.xz or .diff.gz as a priority as the smallest options if (base_uri and target_filename and ( target_filename.lower().endswith('.debian.tar.xz') or target_filename.lower().endswith('.diff.gz') )): uri = "%s/%s" % (base_uri, target_filename) target_filename = None r = self.check_url(uri, msg = self.max_download_size_msg) else: r = None if not r: # fall back to cached local package uri = self.apt_cache_path + self.candidate.filename if not os.path.isfile(uri): # cache miss, download the full source package or the .deb, # depending on size and availability if target_filename: uri_tar = "%s/%s" % (base_uri, target_filename) else: uri_tar = None r, uri = self.get_deb_or_tar(uri_tar) return (r, uri)
def generate_blacklist(self): manifest_remove = os.path.join(self.casper_path, "filesystem.manifest-remove") manifest_desktop = os.path.join(self.casper_path, "filesystem.manifest-desktop") manifest = os.path.join(self.casper_path, "filesystem.manifest") if os.path.exists(manifest_remove) and os.path.exists(manifest): difference = set() with open(manifest_remove) as manifest_file: for line in manifest_file: if line.strip() != "" and not line.startswith("#"): difference.add(line.split()[0]) live_packages = set() with open(manifest) as manifest_file: for line in manifest_file: if line.strip() != "" and not line.startswith("#"): live_packages.add(line.split()[0]) desktop_packages = live_packages - difference elif os.path.exists(manifest_desktop) and os.path.exists(manifest): desktop_packages = set() with open(manifest_desktop) as manifest_file: for line in manifest_file: if line.strip() != "" and not line.startswith("#"): desktop_packages.add(line.split()[0]) live_packages = set() with open(manifest) as manifest_file: for line in manifest_file: if line.strip() != "" and not line.startswith("#"): live_packages.add(line.split()[0]) difference = live_packages - desktop_packages else: difference = set() cache = Cache() use_restricted = True try: if self.db.get("apt-setup/restricted") == "false": use_restricted = False except debconf.DebconfError: pass if not use_restricted: for pkg in cache.keys(): if cache[pkg].is_installed and cache[pkg].section.startswith("restricted/"): difference.add(pkg) # Keep packages we explicitly installed. keep = install_misc.query_recorded_installed() arch, subarch = install_misc.archdetect() # Less than ideal. Since we cannot know which bootloader we'll need # at file copy time, we should figure out why grub still fails when # apt-install-direct is present during configure_bootloader (code # removed). if arch in ("amd64", "i386"): if subarch == "efi": keep.add("grub-efi") keep.add("grub-efi-amd64") efi_vars = "/sys/firmware/efi/vars" sb_var = os.path.join(efi_vars, "SecureBoot-8be4df61-93ca-11d2-aa0d-00e098032b8c", "data") if os.path.exists(sb_var): with open(sb_var, "rb") as sb_var_file: if sb_var_file.read(1) == b"\x01": keep.add("grub-efi-amd64-signed") keep.add("shim-signed") try: altmeta = self.db.get("base-installer/kernel/altmeta") if altmeta: altmeta = "-%s" % altmeta except debconf.DebconfError: altmeta = "" keep.add("linux-signed-generic%s" % altmeta) else: keep.add("grub") keep.add("grub-pc") elif arch in ("armel", "armhf") and subarch in ("omap", "omap4", "mx5"): keep.add("flash-kernel") keep.add("u-boot-tools") elif arch == "powerpc": keep.add("yaboot") keep.add("hfsutils") # Even adding ubiquity as a depends to oem-config-{gtk,kde} doesn't # appear to force ubiquity and libdebian-installer4 to copy all of # their files, so this does the trick. try: if self.db.get("oem-config/enable") == "true": keep.add("ubiquity") except (debconf.DebconfError, IOError): pass difference -= install_misc.expand_dependencies_simple(cache, keep, difference) # Consider only packages that don't have a prerm, and which can # therefore have their files removed without any preliminary work. difference = {x for x in difference if not os.path.exists("/var/lib/dpkg/info/%s.prerm" % x)} confirmed_remove = set() with cache.actiongroup(): for pkg in sorted(difference): if pkg in confirmed_remove: continue would_remove = install_misc.get_remove_list(cache, [pkg], recursive=True) if would_remove <= difference: confirmed_remove |= would_remove # Leave these marked for removal in the apt cache to # speed up further calculations. else: for removedpkg in would_remove: cachedpkg = install_misc.get_cache_pkg(cache, removedpkg) cachedpkg.mark_keep() difference = confirmed_remove if len(difference) == 0: del cache self.blacklist = {} return cmd = ["dpkg", "-L"] cmd.extend(difference) subp = subprocess.Popen(cmd, stdout=subprocess.PIPE, stderr=subprocess.PIPE, universal_newlines=True) res = subp.communicate()[0].splitlines() u = {} for x in res: u[x] = 1 self.blacklist = u
def fail(self, msg, returncode=1): """ Print package versions, traceback and error message. """ apt_cache = AptCache() self.log.error('\n%s\n%s%s', '=' * 79, ''.join(traceback.format_stack()), '=' * 79) pck_s = ['{:<40} {}'.format(pck, apt_cache[pck].installed.version if apt_cache[pck].is_installed else 'Not installed') for pck in sorted([pck for pck in apt_cache.keys() if 'school' in pck])] self.log.info('Installed package versions:\n{}'.format('\n'.join(pck_s))) utils.fail(msg, returncode)
# dashes with periods (so those become wildcards in the regex). The following package # Debian package version strings are considered equivalent: # ii linux-generic-lts-xenial 4.4.0.53.40 # ii linux-headers-4.4.0-53 4.4.0-53.74~14.04.1 # Note the search roots, '4.4.0.53' vs '4.4.0-53'! Tricky shit! # current = m.group(1).replace('.', '\.').replace('-', '.') current_str = m.group(1) # This will hold all purgable Kernel packages and the keepers purgable = [] keepers = [] # Open the APT cache cache = Cache() for p in cache.keys(): if cache[p].installed is None: continue m = match('^linux-(image|headers|virtual|generic).*$', p) if m is None: continue m = search(current, cache[p].installed.version) if m is None or args.all == True: purgable.append(p) else: keepers.append(p) # Summarize the result purgable.sort() keepers.sort()