def configure_grub(bootdev): print("Configuring GRUB.") os.system("mkdir /etc/default/grub.d") if len(bootdev) == 0: fipscfg = open("/etc/default/grub.d/99-fips.cfg", "w+") fipscfg.write( "GRUB_CMDLINE_LINUX_DEFAULT=\"$GRUB_CMDLINE_LINUX_DEFAULT fips=1\"" ) fipscfg.close() else: fipscfg = open("/etc/default/grub.d/99-fips.cfg", "w+") fipscfg.write( "GRUB_CMDLINE_LINUX_DEFAULT=\"$GRUB_CMDLINE_LINUX_DEFAULT fips=1 bootdev=" + bootdev + "\"") fipscfg.close() cache = Cache() cache.open() kernelnumbers = str(str( cache["linux-fips"].versions[0]).split("=")[1]).split(".") kernelversion = str(kernelnumbers[0]) + "." + str( kernelnumbers[1]) + "." + str(kernelnumbers[2]) + "-" + str( kernelnumbers[3]) + "-fips" os.system("mv /etc/default/grub /etc/default/grub.pre-fips") grubcfg = open("/etc/default/grub.pre-fips", "r") newgrubcfg = open("/etc/default/grub", "w+") for line in grubcfg.readlines(): if line.startswith("GRUB_DEFAULT"): newgrubcfg.write("#" + line) newgrubcfg.write( "GRUB_DEFAULT=\"Advanced options for Ubuntu>Ubuntu, with Linux " + str(kernelversion) + "\"\n") else: newgrubcfg.write(line) grubcfg.close() newgrubcfg.close()
def updates_check(self) -> Dict[str, List[Tuple[str, str]]]: """ Returns the list of packages to be updated/installed by a dist-upgrade. """ install = [] update = [] remove = [] apt = Cache(memonly=True) apt.update() apt.open() apt.clear() apt.upgrade(dist_upgrade=True) for pkg in apt.get_changes(): if pkg.marked_install: install.append((pkg.name, pkg.candidate.version)) if pkg.marked_upgrade: update.append((pkg.name, pkg.candidate.version)) if pkg.marked_delete: remove.append((pkg.name, pkg.installed.version)) return dict( update=sorted(update), install=sorted(install), remove=sorted(remove), )
def get_initvm_pkglist(): with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) cache = Cache() cache.open() pkglist = [APTPackage(p) for p in cache if p.is_installed] return pkglist
def mk_source_cdrom(rfs, arch, codename, init_codename, target, log, cdrom_size=CDROM_SIZE): hostfs.mkdir_p('/var/cache/elbe/sources') rfs.mkdir_p('/var/cache/elbe/sources') repo = CdromSrcRepo(codename, init_codename, os.path.join(target, "srcrepo"), log, cdrom_size) cache = get_rpcaptcache(rfs, "aptcache.log", arch) pkglist = cache.get_installed_pkgs() for pkg in pkglist: try: dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources') repo.includedsc(dsc) except ValueError as ve: log.printo("No sources for Package " + pkg.name + "-" + pkg.installed_version) except FetchError as fe: log.printo("Source for Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded") pkglist = get_initvm_pkglist() cache = Cache() cache.open() for pkg in pkglist: try: p = cache[pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions[0] else: pkgver = p.installed dsc = pkgver.fetch_source('/var/cache/elbe/sources', ElbeAcquireProgress(cb=None), unpack=False) repo.includedsc(dsc) except ValueError as ve: log.printo("No sources for Package " + pkg.name + "-" + str(pkg.installed_version)) except FetchError as fe: log.printo("Source for Package " + pkg.name + "-" + pkgver.version + " could not be downloaded") return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def get_initvm_pkglist (): with warnings.catch_warnings(): warnings.filterwarnings("ignore",category=DeprecationWarning) cache = Cache () cache.open () pkglist = [APTPackage (p) for p in cache if p.is_installed] try: eb = APTPackage( cache ['elbe-bootstrap'] ) pkglist.append (eb) # elbe bootstrap is not installed on pc running elbe except KeyError: pass return pkglist
def mk_source_cdrom(rfs, arch, codename, init_codename, target, log, cdrom_size=CDROM_SIZE): hostfs.mkdir_p( '/var/cache/elbe/sources' ) rfs.mkdir_p( '/var/cache/elbe/sources' ) repo = CdromSrcRepo( codename, init_codename, os.path.join( target, "srcrepo" ), log, cdrom_size ) cache = get_rpcaptcache( rfs, "aptcache.log", arch ) pkglist = cache.get_installed_pkgs() for pkg in pkglist: try: dsc = cache.download_source( pkg.name, '/var/cache/elbe/sources' ) repo.includedsc( dsc ) except ValueError as ve: log.printo( "No sources for Package " + pkg.name + "-" + pkg.installed_version ) except FetchError as fe: log.printo( "Source for Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded" ) pkglist = get_initvm_pkglist() cache = Cache () cache.open () for pkg in pkglist: try: p = cache[pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions [0] else: pkgver = p.installed dsc = pkgver.fetch_source ('/var/cache/elbe/sources', ElbeAcquireProgress (cb=None), unpack=False) repo.includedsc( dsc ) except ValueError as ve: log.printo( "No sources for Package " + pkg.name + "-" + str(pkg.installed_version) ) except FetchError as fe: log.printo( "Source for Package " + pkg.name + "-" + pkgver.version + " could not be downloaded" ) return repo.buildiso( os.path.join( target, "src-cdrom.iso" ) )
class RPCAPTCache(InChRootObject): # pylint: disable=too-many-public-methods def __init__(self, rfs, arch, notifier=None, norecommend=False, noauth=True): # pylint: disable=too-many-arguments InChRootObject.__init__(self, rfs) self.notifier = notifier config.set("APT::Architecture", arch) if norecommend: config.set("APT::Install-Recommends", "0") else: config.set("APT::Install-Recommends", "1") if noauth: config.set("APT::Get::AllowUnauthenticated", "1") else: config.set("APT::Get::AllowUnauthenticated", "0") self.cache = Cache(progress=ElbeOpProgress()) self.cache.open(progress=ElbeOpProgress()) def dbg_dump(self, filename): ts = time.localtime() filename = filename + ('_%02d%02d%02d' % (ts.tm_hour, ts.tm_min, ts.tm_sec)) with open(filename, 'w') as dbg: for p in self.cache: dbg.write( '%s %s %d %d %d %d %d %d %d %d %d %d %d %d\n' % (p.name, p.candidate.version, p.marked_keep, p.marked_delete, p.marked_upgrade, p.marked_downgrade, p.marked_install, p.marked_reinstall, p.is_auto_installed, p.is_installed, p.is_auto_removable, p.is_now_broken, p.is_inst_broken, p.is_upgradable)) def get_sections(self): ret = list({p.section for p in self.cache}) ret.sort() return ret def get_pkglist(self, section): if section == 'all': ret = [APTPackage(p) for p in self.cache] else: ret = [APTPackage(p) for p in self.cache if p.section == section] return ret def mark_install(self, pkgname, version, from_user=True, nodeps=False): print('Mark for install "%s"' % pkgname) p = self.cache[pkgname] if version: p.candidate = p.versions[version] p.mark_install(auto_fix=not nodeps, auto_inst=not nodeps, from_user=from_user) def mark_install_devpkgs(self, ignore_pkgs, ignore_dev_pkgs): ignore_pkgs.discard('libc6') # we don't want to ignore libc ignore_pkgs.discard('libstdc++5') ignore_pkgs.discard('libstdc++6') # list all debian src packages of all installed packages that don't # come from debootstrap src_list = [ p.candidate.source_name for p in self.cache if (p.is_installed and p.name not in ignore_pkgs) ] version_dict = { p.name: p.candidate.version for p in self.cache if (p.is_installed and p.name not in ignore_pkgs) } # go through all packages, remember package if its source package # matches one of the installed packages and the binary package is a # '-dev' package dev_list = [ s for s in self.cache if (s.candidate.source_name in src_list and ( s.name.endswith('-dev'))) ] for p in dev_list: if p.name not in ignore_dev_pkgs: name_no_suffix = p.name[:-len('-dev')] if name_no_suffix in version_dict: version = version_dict[name_no_suffix] candidate = p.versions.get(version) if candidate: p.candidate = candidate p.mark_install() # ensure that the symlinks package will be installed (it's needed for # fixing links inside the sysroot self.cache['symlinks'].mark_install() for p in ignore_dev_pkgs: self.cache[p].mark_delete() dbgsym_list = [ s.name + '-dbgsym' for s in self.cache if (s.is_installed or s.marked_install) ] for p in dbgsym_list: if p in self.cache: pkg = self.cache[p] name_no_suffix = pkg.name[:-len('-dbgsym')] if name_no_suffix in version_dict: version = version_dict[name_no_suffix] candidate = pkg.versions.get(version) if candidate: pkg.candidate = candidate pkg.mark_install() def cleanup(self, exclude_pkgs): for p in self.cache: if p.is_installed and not \ p.is_auto_installed or \ p.is_auto_removable: remove = True for x in exclude_pkgs: if x == p.name: remove = False if remove: p.mark_delete(auto_fix=True, purge=True) def mark_upgrade(self, pkgname, version): p = self.cache[pkgname] if version: p.candidate = p.versions[version] p.mark_upgrade() def mark_delete(self, pkgname): p = self.cache[pkgname] p.mark_delete(purge=True) def mark_keep(self, pkgname, _version): p = self.cache[pkgname] p.mark_keep() def update(self): self.cache.update(fetch_progress=ElbeAcquireProgress()) self.cache.open(progress=ElbeOpProgress()) def commit(self): os.environ["DEBIAN_FRONTEND"] = "noninteractive" os.environ["DEBONF_NONINTERACTIVE_SEEN"] = "true" print("Commiting changes ...") self.cache.commit(ElbeAcquireProgress(), ElbeInstallProgress(fileno=sys.stdout.fileno())) self.cache.open(progress=ElbeOpProgress()) def clear(self): self.cache.clear() def get_dependencies(self, pkgname): deps = getalldeps(self.cache, pkgname) return [APTPackage(p, cache=self.cache) for p in deps] def get_installed_pkgs(self, section='all'): if section == 'all': pl = [APTPackage(p) for p in self.cache if p.is_installed] else: pl = [ APTPackage(p) for p in self.cache if (p.section == section and p.is_installed) ] return pl def get_fileindex(self): index = {} for p in self.cache: if p.is_installed: for f in p.installed_files: index[f] = p.name return index def get_marked_install(self, section='all'): if section == 'all': ret = [APTPackage(p) for p in self.cache if p.marked_install] else: ret = [ APTPackage(p) for p in self.cache if (p.section == section and p.marked_install) ] return ret def get_upgradeable(self, section='all'): if section == 'all': ret = [APTPackage(p) for p in self.cache if p.is_upgradable] else: ret = [ APTPackage(p) for p in self.cache if (p.section == section and p.is_upgradable) ] return ret def upgrade(self, dist_upgrade=False): self.cache.upgrade(dist_upgrade) def get_changes(self): changes = self.cache.get_changes() return [APTPackage(p) for p in changes] def has_pkg(self, pkgname): return pkgname in self.cache def is_installed(self, pkgname): if pkgname not in self.cache: return False return self.cache[pkgname].is_installed def get_pkg(self, pkgname): return APTPackage(self.cache[pkgname]) def get_pkgs(self, pkgname): return [ APTPackage(self.cache[p]) for p in sorted(self.cache.keys()) if pkgname in p.lower() ] def get_corresponding_source_packages(self, pkg_lst=None): if pkg_lst is None: pkg_lst = {p.name for p in self.cache if p.is_installed} src_set = set() with TagFile('/var/lib/dpkg/status') as tagfile: for section in tagfile: pkg = section['Package'] if pkg not in pkg_lst: continue tmp = self.cache[pkg].installed or self.cache[pkg].candidate src_set.add((tmp.source_name, tmp.source_version)) if "Built-Using" not in section: continue built_using_lst = section["Built-Using"].split(', ') for built_using in built_using_lst: name, version = built_using.split(' ', 1) version = version.strip('(= )') src_set.add((name, version)) return list(src_set) @staticmethod def compare_versions(self, ver1, ver2): return version_compare(ver1, ver2) def download_binary(self, pkgname, path, version=None): p = self.cache[pkgname] if version is None: pkgver = p.installed else: pkgver = p.versions[version] rel_filename = fetch_binary(pkgver, path, ElbeAcquireProgress()) return self.rfs.fname(rel_filename) def download_source(self, src_name, src_version, dest_dir): allow_untrusted = config.find_b("APT::Get::AllowUnauthenticated", False) rec = SourceRecords() acq = Acquire(ElbeAcquireProgress()) # poorman's iterator while True: next_p = rec.lookup(src_name) # End of the list? if not next_p: raise ("No source found for %s_%s" % (src_name, src_version)) if src_version == rec.version: break # We don't allow untrusted package and the package is not # marks as trusted if not (allow_untrusted or rec.index.is_trusted): raise FetchError( "Can't fetch source %s_%s; Source %r is not trusted" % (src_name, src_version, rec.index.describe)) # Copy from src to dst all files of the source package dsc = None files = [] for _file in rec.files: src = os.path.basename(_file.path) dst = os.path.join(dest_dir, src) if 'dsc' == _file.type: dsc = dst if not (allow_untrusted or _file.hashes.usable): raise FetchError( "Can't fetch file %s. No trusted hash found." % dst) # acq is accumlating the AcquireFile, the files list only # exists to prevent Python from GC the object .. I guess. # Anyway, if we don't keep the list, We will get an empty # directory files.append( AcquireFile(acq, rec.index.archive_uri(_file.path), _file.hashes, _file.size, src, destfile=dst)) acq.run() if dsc is None: raise ValueError("No source found for %s_%s" % (src_name, src_version)) for item in acq.items: if item.STAT_DONE != item.status: raise FetchError("Can't fetch item %s: %s" % (item.destfile, item.error_text)) return self.rfs.fname(os.path.abspath(dsc))
class RPCAPTCache(InChRootObject): def __init__(self, rfs, log, arch, notifier=None, norecommend=False, noauth=True): sys.stdout = open(log, 'a', buffering=0) sys.stderr = open(log, 'a', buffering=0) self.logfile = open(log, 'a', buffering=0) InChRootObject.__init__(self, rfs) self.notifier = notifier config.set("APT::Architecture", arch) if norecommend: config.set("APT::Install-Recommends", "1") else: config.set("APT::Install-Recommends", "0") if noauth: config.set("APT::Get::AllowUnauthenticated", "1") else: config.set("APT::Get::AllowUnauthenticated", "0") self.cache = Cache(progress=ElbeOpProgress()) self.cache.open(progress=ElbeOpProgress()) def dbg_dump(self, filename): ts = time.localtime() filename = filename + ('_%02d%02d%02d' % (ts.tm_hour, ts.tm_min, ts.tm_sec)) with open(filename, 'w') as dbg: for p in self.cache: dbg.write( '%s %s %d %d %d %d %d %d %d %d %d %d %d %d\n' % (p.name, p.candidate.version, p.marked_keep, p.marked_delete, p.marked_upgrade, p.marked_downgrade, p.marked_install, p.marked_reinstall, p.is_auto_installed, p.is_installed, p.is_auto_removable, p.is_now_broken, p.is_inst_broken, p.is_upgradable)) def get_sections(self): ret = list(set([p.section for p in self.cache])) ret.sort() return ret def get_pkglist(self, section): if section == 'all': ret = [APTPackage(p) for p in self.cache] else: ret = [APTPackage(p) for p in self.cache if p.section == section] return ret def mark_install(self, pkgname, version, from_user=True, nodeps=False): p = self.cache[pkgname] if version: p.candidate = p.versions[version] p.mark_install(auto_fix=not nodeps, auto_inst=not nodeps, from_user=from_user) def mark_install_devpkgs(self, ignore_pkgs, ignore_dev_pkgs): ignore_pkgs.remove('libc6') # we don't want to ignore libc # we don't want to ignore libstdc++ try: ignore_pkgs.remove('libstdc++5') except: pass try: ignore_pkgs.remove('libstdc++6') except: pass # list all debian src packages of all installed packages that don't # come from debootstrap src_list = [ p.candidate.source_name for p in self.cache if p.is_installed and p.name not in ignore_pkgs ] # go through all packages, remember package if its source package # matches one of the installed packages and the binary package is a # '-dev' package dev_list = [ s for s in self.cache if (s.candidate.source_name in src_list and s.name.endswith('-dev') ) ] for p in dev_list: if p.name not in ignore_dev_pkgs: p.mark_install() # ensure that the symlinks package will be installed (it's needed for # fixing links inside the sysroot self.cache['symlinks'].mark_install() for p in ignore_dev_pkgs: self.cache[p].mark_delete() def cleanup(self, exclude_pkgs): for p in self.cache: if (p.is_installed and not p.is_auto_installed) or p.is_auto_removable: remove = True for x in exclude_pkgs: if x == p.name: remove = False if remove: p.mark_delete(auto_fix=True, purge=True) def mark_upgrade(self, pkgname, version): p = self.cache[pkgname] if version: p.candidate = p.versions[version] p.mark_upgrade() def mark_delete(self, pkgname, version): p = self.cache[pkgname] p.mark_delete(purge=True) def mark_keep(self, pkgname, version): p = self.cache[pkgname] p.mark_keep() def update(self): self.cache.update(fetch_progress=ElbeAcquireProgress()) self.cache.open(progress=ElbeOpProgress()) def commit(self): os.environ["DEBIAN_FRONTEND"] = "noninteractive" os.environ["DEBONF_NONINTERACTIVE_SEEN"] = "true" self.cache.commit(ElbeAcquireProgress(), ElbeInstallProgress(fileno=self.logfile.fileno())) self.cache.open(progress=ElbeOpProgress()) def clear(self): self.cache.clear() def get_dependencies(self, pkgname): deps = getalldeps(self.cache, pkgname) return [APTPackage(p, cache=self.cache) for p in deps] def get_installed_pkgs(self, section='all'): # avoid DeprecationWarning: MD5Hash is deprecated, use Hashes instead # triggerd by python-apt with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) if section == 'all': pl = [APTPackage(p) for p in self.cache if p.is_installed] else: pl = [ APTPackage(p) for p in self.cache if (p.section == section and p.is_installed) ] return pl def get_fileindex(self): index = {} for p in self.cache: if p.is_installed: for f in p.installed_files: index[f] = p.name return index def get_marked_install(self, section='all'): if section == 'all': ret = [APTPackage(p) for p in self.cache if p.marked_install] else: ret = [ APTPackage(p) for p in self.cache if (p.section == section and p.marked_install) ] return ret def get_upgradeable(self, section='all'): if section == 'all': ret = [APTPackage(p) for p in self.cache if p.is_upgradable] else: ret = [ APTPackage(p) for p in self.cache if (p.section == section and p.is_upgradable) ] return ret def upgrade(self, dist_upgrade=False): self.cache.upgrade(dist_upgrade) def get_changes(self): changes = self.cache.get_changes() return [APTPackage(p) for p in changes] def has_pkg(self, pkgname): return pkgname in self.cache def is_installed(self, pkgname): if not pkgname in self.cache: return False return self.cache[pkgname].is_installed def get_pkg(self, pkgname): return APTPackage(self.cache[pkgname]) def get_pkgs(self, pkgname): return [ APTPackage(self.cache[p]) for p in sorted(self.cache.keys()) if pkgname in p.lower() ] def compare_versions(self, ver1, ver2): return version_compare(ver1, ver2) def download_binary(self, pkgname, path, version=None): p = self.cache[pkgname] if version is None: pkgver = p.installed else: pkgver = p.versions[version] rel_filename = pkgver.fetch_binary(path, ElbeAcquireProgress()) return self.rfs.fname(rel_filename) def download_source(self, pkgname, path, version=None): p = self.cache[pkgname] if version is None: pkgver = p.installed else: pkgver = p.versions[version] rel_filename = pkgver.fetch_source(path, ElbeAcquireProgress(), unpack=False) return self.rfs.fname(rel_filename)
def get_initvm_pkglist(): cache = Cache() cache.open() pkglist = [APTPackage(p) for p in cache if p.is_installed] pkglist.append(APTPackage(cache['elbe-bootstrap'])) return pkglist
def get_initvm_pkglist(): cache = Cache() cache.open() pkglist = [APTPackage(p) for p in cache if p.is_installed] return pkglist
def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target, log, cdrom_size=CDROM_SIZE): rfs.mkdir_p('/var/cache/elbe/binaries/added') rfs.mkdir_p('/var/cache/elbe/binaries/main') hostfs.mkdir_p('/var/cache/elbe/binaries/main') if not xml is None: mirror = xml.get_primary_mirror(rfs.fname("cdrom")) else: mirror = 'http://ftp.debian.org/debian' repo_path = os.path.join(target, "binrepo") repo = CdromBinRepo(arch, codename, init_codename, repo_path, log, cdrom_size, mirror) if not xml is None: pkglist = get_initvm_pkglist() cache = Cache() cache.open() for pkg in pkglist: try: p = cache[pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions[0] else: pkgver = p.installed deb = pkgver.fetch_binary('/var/cache/elbe/binaries/main', ElbeAcquireProgress(cb=None)) repo.include_init_deb(deb, 'main') except ValueError as ve: log.printo("No Package " + pkg.name + "-" + str(pkg.installed_version)) except FetchError as fe: log.printo("Package " + pkg.name + "-" + pkgver.version + " could not be downloaded") except TypeError as te: log.printo("Package " + pkg.name + "-" + str(pkg.installed_version) + " missing name or version") cache = get_rpcaptcache(rfs, "aptcache.log", arch) for p in xml.node("debootstrappkgs"): pkg = XMLPackage(p, arch) try: deb = cache.download_binary(pkg.name, '/var/cache/elbe/binaries/main', pkg.installed_version) repo.includedeb(deb, 'main') except ValueError as ve: log.printo("No Package " + pkg.name + "-" + pkg.installed_version) except FetchError as fe: log.printo("Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded") except TypeError as te: log.printo("Package " + pkg.name + "-" + pkg.installed_version + " missing name or version") cache = get_rpcaptcache(rfs, "aptcache.log", arch) pkglist = cache.get_installed_pkgs() for pkg in pkglist: try: deb = cache.download_binary(pkg.name, '/var/cache/elbe/binaries/added', pkg.installed_version) repo.includedeb(deb, 'added') except KeyError as ke: log.printo(str(ke)) except ValueError as ve: log.printo("No Package " + pkg.name + "-" + pkg.installed_version) except FetchError as fe: log.printo("Package " + pkg.name + "-" + str(pkg.installed_version) + " could not be downloaded") except TypeError as te: log.printo("Package " + pkg.name + "-" + pkg.installed_version + " missing name or version") # Mark the binary repo with the necessary Files # to make the installer accept this as a CDRom repo_fs = Filesystem(repo_path) repo_fs.mkdir_p(".disk") repo_fs.write_file(".disk/base_installable", 0644, "main\n") repo_fs.write_file(".disk/base_components", 0644, "main\n") repo_fs.write_file(".disk/cd_type", 0644, "not_complete\n") repo_fs.write_file(".disk/info", 0644, "elbe inst cdrom - full cd\n") repo_fs.symlink(".", "debian", allow_exists=True) repo_fs.write_file("md5sum.txt", 0644, "") # write source xml onto cdrom xml.xml.write(repo_fs.fname('source.xml')) return repo.buildiso(os.path.join(target, "bin-cdrom.iso"))
class RPCAPTCache(InChRootObject): # pylint: disable=too-many-public-methods def __init__( self, rfs, log, arch, notifier=None, norecommend=False, noauth=True): # pylint: disable=too-many-arguments sys.stdout = open(log, 'a', buffering=0) sys.stderr = open(log, 'a', buffering=0) self.logfile = open(log, 'a', buffering=0) InChRootObject.__init__(self, rfs) self.notifier = notifier config.set("APT::Architecture", arch) if norecommend: config.set("APT::Install-Recommends", "0") else: config.set("APT::Install-Recommends", "1") if noauth: config.set("APT::Get::AllowUnauthenticated", "1") else: config.set("APT::Get::AllowUnauthenticated", "0") self.cache = Cache(progress=ElbeOpProgress()) self.cache.open(progress=ElbeOpProgress()) def dbg_dump(self, filename): ts = time.localtime() filename = filename + ( '_%02d%02d%02d' % (ts.tm_hour, ts.tm_min, ts.tm_sec)) with open(filename, 'w') as dbg: for p in self.cache: dbg.write( '%s %s %d %d %d %d %d %d %d %d %d %d %d %d\n' % (p.name, p.candidate.version, p.marked_keep, p.marked_delete, p.marked_upgrade, p.marked_downgrade, p.marked_install, p.marked_reinstall, p.is_auto_installed, p.is_installed, p.is_auto_removable, p.is_now_broken, p.is_inst_broken, p.is_upgradable)) def get_sections(self): ret = list(set([p.section for p in self.cache])) ret.sort() return ret def get_pkglist(self, section): if section == 'all': ret = [APTPackage(p) for p in self.cache] else: ret = [APTPackage(p) for p in self.cache if p.section == section] return ret def mark_install(self, pkgname, version, from_user=True, nodeps=False): p = self.cache[pkgname] if version: p.candidate = p.versions[version] p.mark_install(auto_fix=not nodeps, auto_inst=not nodeps, from_user=from_user) def mark_install_devpkgs(self, ignore_pkgs, ignore_dev_pkgs): ignore_pkgs.discard('libc6') # we don't want to ignore libc ignore_pkgs.discard('libstdc++5') ignore_pkgs.discard('libstdc++6') # list all debian src packages of all installed packages that don't # come from debootstrap src_list = [ p.candidate.source_name for p in self.cache if ( p.is_installed and p.name not in ignore_pkgs)] # go through all packages, remember package if its source package # matches one of the installed packages and the binary package is a # '-dev' package dev_list = [ s for s in self.cache if ( s.candidate.source_name in src_list and ( s.name.endswith('-dev')))] for p in dev_list: if p.name not in ignore_dev_pkgs: p.mark_install() # ensure that the symlinks package will be installed (it's needed for # fixing links inside the sysroot self.cache['symlinks'].mark_install() for p in ignore_dev_pkgs: self.cache[p].mark_delete() def cleanup(self, exclude_pkgs): for p in self.cache: if p.is_installed and not \ p.is_auto_installed or \ p.is_auto_removable: remove = True for x in exclude_pkgs: if x == p.name: remove = False if remove: p.mark_delete(auto_fix=True, purge=True) def mark_upgrade(self, pkgname, version): p = self.cache[pkgname] if version: p.candidate = p.versions[version] p.mark_upgrade() def mark_delete(self, pkgname): p = self.cache[pkgname] p.mark_delete(purge=True) def mark_keep(self, pkgname, version): p = self.cache[pkgname] p.mark_keep() def update(self): self.cache.update(fetch_progress=ElbeAcquireProgress()) self.cache.open(progress=ElbeOpProgress()) def commit(self): os.environ["DEBIAN_FRONTEND"] = "noninteractive" os.environ["DEBONF_NONINTERACTIVE_SEEN"] = "true" self.cache.commit(ElbeAcquireProgress(), ElbeInstallProgress(fileno=self.logfile.fileno())) self.cache.open(progress=ElbeOpProgress()) def clear(self): self.cache.clear() def get_dependencies(self, pkgname): deps = getalldeps(self.cache, pkgname) return [APTPackage(p, cache=self.cache) for p in deps] def get_installed_pkgs(self, section='all'): # avoid DeprecationWarning: MD5Hash is deprecated, use Hashes instead # triggerd by python-apt with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) if section == 'all': pl = [APTPackage(p) for p in self.cache if p.is_installed] else: pl = [APTPackage(p) for p in self.cache if ( p.section == section and p.is_installed)] return pl def get_fileindex(self): index = {} for p in self.cache: if p.is_installed: for f in p.installed_files: index[f] = p.name return index def get_marked_install(self, section='all'): if section == 'all': ret = [APTPackage(p) for p in self.cache if p.marked_install] else: ret = [APTPackage(p) for p in self.cache if (p.section == section and p.marked_install)] return ret def get_upgradeable(self, section='all'): if section == 'all': ret = [APTPackage(p) for p in self.cache if p.is_upgradable] else: ret = [APTPackage(p) for p in self.cache if (p.section == section and p.is_upgradable)] return ret def upgrade(self, dist_upgrade=False): self.cache.upgrade(dist_upgrade) def get_changes(self): changes = self.cache.get_changes() return [APTPackage(p) for p in changes] def has_pkg(self, pkgname): return pkgname in self.cache def is_installed(self, pkgname): if pkgname not in self.cache: return False return self.cache[pkgname].is_installed def get_pkg(self, pkgname): return APTPackage(self.cache[pkgname]) def get_pkgs(self, pkgname): return [ APTPackage( self.cache[p]) for p in sorted( self.cache.keys()) if pkgname in p.lower()] def compare_versions(self, ver1, ver2): return version_compare(ver1, ver2) def download_binary(self, pkgname, path, version=None): p = self.cache[pkgname] if version is None: pkgver = p.installed else: pkgver = p.versions[version] # avoid DeprecationWarning: # "MD5Hash is deprecated, use Hashes instead" # triggerd by python-apt with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) rel_filename = pkgver.fetch_binary(path, ElbeAcquireProgress()) return self.rfs.fname(rel_filename) def download_source(self, pkgname, path, version=None): p = self.cache[pkgname] if version is None: pkgver = p.installed else: pkgver = p.versions[version] # avoid DeprecationWarning: # "MD5Hash is deprecated, use Hashes instead" # triggerd by python-apt with warnings.catch_warnings(): warnings.filterwarnings("ignore", category=DeprecationWarning) rel_filename = pkgver.fetch_source(path, ElbeAcquireProgress(), unpack=False) return self.rfs.fname(rel_filename)
def mk_source_cdrom(rfs, arch, codename, init_codename, target, log, cdrom_size=CDROM_SIZE, xml=None): hostfs.mkdir_p('/var/cache/elbe/sources') rfs.mkdir_p('/var/cache/elbe/sources') repo = CdromSrcRepo(codename, init_codename, os.path.join(target, "srcrepo"), log, cdrom_size) cache = get_rpcaptcache(rfs, "aptcache.log", arch) pkglist = cache.get_installed_pkgs() forbiddenPackages = [] if xml != None and xml.has('target/pkg-list'): for i in xml.node('target/pkg-list'): try: if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False': forbiddenPackages.append(i.text('.').strip()) except KeyError: pass for pkg in pkglist: # Do not include forbidden packages in src cdrom if pkg.name in forbiddenPackages: continue try: dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources') repo.includedsc(dsc, force=True) except ValueError as ve: log.printo("No sources for Package " + pkg.name + "-" + pkg.installed_version) except FetchError as fe: log.printo("Source for Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded") repo.finalize() pkglist = get_initvm_pkglist() cache = Cache() cache.open() for pkg in pkglist: # Do not include forbidden packages in src cdrom if pkg.name in forbiddenPackages: continue try: p = cache[pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions[0] else: pkgver = p.installed dsc = pkgver.fetch_source('/var/cache/elbe/sources', ElbeAcquireProgress(cb=None), unpack=False) repo.includedsc(dsc) except ValueError as ve: log.printo("No sources for Package " + pkg.name + "-" + str(pkg.installed_version)) except FetchError as fe: log.printo("Source for Package " + pkg.name + "-" + pkgver.version + " could not be downloaded") repo.finalize() return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target, log, cdrom_size=CDROM_SIZE): rfs.mkdir_p( '/var/cache/elbe/binaries/added' ) rfs.mkdir_p( '/var/cache/elbe/binaries/main' ) hostfs.mkdir_p( '/var/cache/elbe/binaries/main' ) if not xml is None: mirror = xml.get_primary_mirror (rfs.fname("cdrom")) else: mirror='http://ftp.debian.org/debian' repo_path = os.path.join (target, "binrepo") repo = CdromBinRepo (arch, codename, init_codename, repo_path, log, cdrom_size, mirror) if not xml is None: pkglist = get_initvm_pkglist() cache = Cache () cache.open () for pkg in pkglist: try: p = cache [pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions [0] else: pkgver = p.installed deb = pkgver.fetch_binary ('/var/cache/elbe/binaries/main', ElbeAcquireProgress (cb=None) ) repo.include_init_deb(deb, 'main') except ValueError as ve: log.printo( "No Package " + pkg.name + "-" + str(pkg.installed_version) ) except FetchError as fe: log.printo( "Package " + pkg.name + "-" + pkgver.version + " could not be downloaded" ) except TypeError as te: log.printo( "Package " + pkg.name + "-" + str(pkg.installed_version) + " missing name or version" ) cache = get_rpcaptcache( rfs, "aptcache.log", arch ) for p in xml.node("debootstrappkgs"): pkg = XMLPackage(p, arch) try: deb = cache.download_binary( pkg.name, '/var/cache/elbe/binaries/main', pkg.installed_version ) repo.includedeb(deb, 'main') except ValueError as ve: log.printo( "No Package " + pkg.name + "-" + pkg.installed_version ) except FetchError as fe: log.printo( "Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded" ) except TypeError as te: log.printo( "Package " + pkg.name + "-" + pkg.installed_version + " missing name or version" ) cache = get_rpcaptcache( rfs, "aptcache.log", arch ) pkglist = cache.get_installed_pkgs () for pkg in pkglist: try: deb = cache.download_binary( pkg.name, '/var/cache/elbe/binaries/added', pkg.installed_version ) repo.includedeb(deb, 'added') except KeyError as ke: log.printo( str (ke) ) except ValueError as ve: log.printo( "No Package " + pkg.name + "-" + pkg.installed_version ) except FetchError as fe: log.printo( "Package " + pkg.name + "-" + str (pkg.installed_version) + " could not be downloaded" ) except TypeError as te: log.printo( "Package " + pkg.name + "-" + pkg.installed_version + " missing name or version" ) # Mark the binary repo with the necessary Files # to make the installer accept this as a CDRom repo_fs = Filesystem( repo_path ) repo_fs.mkdir_p (".disk") repo_fs.write_file (".disk/base_installable", 0644, "main\n") repo_fs.write_file (".disk/base_components", 0644, "main\n") repo_fs.write_file (".disk/cd_type", 0644, "not_complete\n") repo_fs.write_file (".disk/info", 0644, "elbe inst cdrom - full cd\n") repo_fs.symlink (".", "debian", allow_exists=True) repo_fs.write_file ("md5sum.txt", 0644, "") # write source xml onto cdrom xml.xml.write (repo_fs.fname ('source.xml')) return repo.buildiso( os.path.join( target, "bin-cdrom.iso" ) )
def run_command(argv): # TODO - Set threshold and remove pylint directives # # We might want to make the threshold higher for certain # files/directories or just globaly. # pylint: disable=too-many-locals # pylint: disable=too-many-branches # pylint: disable=too-many-statements oparser = OptionParser( usage="usage: %prog fetch_initvm_pkgs [options] <xmlfile>") oparser.add_option("-b", "--binrepo", dest="binrepo", default="/var/cache/elbe/initvm-bin-repo", help="directory where the bin repo should reside") oparser.add_option("-s", "--srcrepo", dest="srcrepo", default="/var/cache/elbe/initvm-src-repo", help="directory where the src repo should reside") oparser.add_option("--skip-validation", action="store_true", dest="skip_validation", default=False, help="Skip xml schema validation") oparser.add_option("--cdrom-mount-path", dest="cdrom_path", help="path where cdrom is mounted") oparser.add_option("--cdrom-device", dest="cdrom_device", help="cdrom device, in case it has to be mounted") oparser.add_option("--apt-archive", dest="archive", default="/var/cache/elbe/binaries/main", help="path where binary packages are downloaded to.") oparser.add_option("--src-archive", dest="srcarchive", default="/var/cache/elbe/sources", help="path where src packages are downloaded to.") oparser.add_option("--skip-build-sources", action="store_false", dest="build_sources", default=True, help="Skip downloading Source Packages") oparser.add_option("--skip-build-bin", action="store_false", dest="build_bin", default=True, help="Skip downloading binary packages") (opt, args) = oparser.parse_args(argv) if len(args) != 1: print("wrong number of arguments") oparser.print_help() sys.exit(20) try: xml = ElbeXML(args[0], skip_validate=opt.skip_validation) except ValidationError as e: print(str(e)) print("xml validation failed. Bailing out") sys.exit(20) with elbe_logging({"streams": sys.stdout}): if opt.cdrom_path: if opt.cdrom_device: do('mount "%s" "%s"' % (opt.cdrom_device, opt.cdrom_path)) # a cdrom build is identified by the cdrom option # the xml file that is copied into the initvm # by the initrd does not have the cdrom tags setup. mirror = "file://%s" % opt.cdrom_path else: mirror = xml.get_initvm_primary_mirror(opt.cdrom_path) init_codename = xml.get_initvm_codename() # Binary Repo # repo = CdromInitRepo(init_codename, opt.binrepo, mirror) hostfs.mkdir_p(opt.archive) if opt.build_bin: pkglist = get_initvm_pkglist() cache = Cache() cache.open() for pkg in pkglist: pkg_id = "%s-%s" % (pkg.name, pkg.installed_version) try: p = cache[pkg.name] pkgver = p.installed deb = fetch_binary(pkgver, opt.archive, ElbeAcquireProgress(cb=None)) repo.includedeb(deb, 'main') except ValueError: logging.exception('No package "%s"', pkg_id) except FetchError: logging.exception( 'Package "%s-%s" could not be downloaded', pkg.name, pkgver.version) except TypeError: logging.exception('Package "%s" missing name or version', pkg_id) repo.finalize() # Source Repo # repo = CdromSrcRepo(init_codename, init_codename, opt.srcrepo, 0, mirror) hostfs.mkdir_p(opt.srcarchive) # a cdrom build does not have sources # skip adding packages to the source repo # # FIXME: we need a way to add source cdroms later on if opt.cdrom_path: opt.build_sources = False if opt.build_sources: for pkg in pkglist: pkg_id = "%s-%s" % (pkg.name, pkg.installed_version) try: p = cache[pkg.name] pkgver = p.installed dsc = pkgver.fetch_source(opt.srcarchive, ElbeAcquireProgress(cb=None), unpack=False) repo.include_init_dsc(dsc, 'initvm') except ValueError: logging.exception('No package "%s"', pkg_id) except FetchError: logging.exception( 'Package "%s-%s" could not be downloaded', pkg.name, pkgver.version) except TypeError: logging.exception('Package "%s" missing name or version', pkg_id) repo.finalize() if opt.cdrom_device: do('umount "%s"' % opt.cdrom_device)
class RPCAPTCache(InChRootObject): # pylint: disable=too-many-public-methods def __init__(self, rfs, arch, notifier=None, norecommend=False, noauth=True): # pylint: disable=too-many-arguments InChRootObject.__init__(self, rfs) self.notifier = notifier config.set("APT::Architecture", arch) if norecommend: config.set("APT::Install-Recommends", "0") else: config.set("APT::Install-Recommends", "1") if noauth: config.set("APT::Get::AllowUnauthenticated", "1") else: config.set("APT::Get::AllowUnauthenticated", "0") self.cache = Cache(progress=ElbeOpProgress()) self.cache.open(progress=ElbeOpProgress()) def dbg_dump(self, filename): ts = time.localtime() filename = filename + ('_%02d%02d%02d' % (ts.tm_hour, ts.tm_min, ts.tm_sec)) with open(filename, 'w') as dbg: for p in self.cache: dbg.write( '%s %s %d %d %d %d %d %d %d %d %d %d %d %d\n' % (p.name, p.candidate.version, p.marked_keep, p.marked_delete, p.marked_upgrade, p.marked_downgrade, p.marked_install, p.marked_reinstall, p.is_auto_installed, p.is_installed, p.is_auto_removable, p.is_now_broken, p.is_inst_broken, p.is_upgradable)) def get_sections(self): ret = list(set([p.section for p in self.cache])) ret.sort() return ret def get_pkglist(self, section): if section == 'all': ret = [APTPackage(p) for p in self.cache] else: ret = [APTPackage(p) for p in self.cache if p.section == section] return ret def mark_install(self, pkgname, version, from_user=True, nodeps=False): print('Mark for install "%s"' % pkgname) p = self.cache[pkgname] if version: p.candidate = p.versions[version] p.mark_install(auto_fix=not nodeps, auto_inst=not nodeps, from_user=from_user) def mark_install_devpkgs(self, ignore_pkgs, ignore_dev_pkgs): ignore_pkgs.discard('libc6') # we don't want to ignore libc ignore_pkgs.discard('libstdc++5') ignore_pkgs.discard('libstdc++6') # list all debian src packages of all installed packages that don't # come from debootstrap src_list = [ p.candidate.source_name for p in self.cache if (p.is_installed and p.name not in ignore_pkgs) ] version_dict = { p.name: p.candidate.version for p in self.cache if (p.is_installed and p.name not in ignore_pkgs) } # go through all packages, remember package if its source package # matches one of the installed packages and the binary package is a # '-dev' package dev_list = [ s for s in self.cache if (s.candidate.source_name in src_list and ( s.name.endswith('-dev'))) ] for p in dev_list: if p.name not in ignore_dev_pkgs: name_no_suffix = p.name[:-len('-dev')] if name_no_suffix in version_dict: version = version_dict[name_no_suffix] candidate = p.versions.get(version) if candidate: p.candidate = candidate p.mark_install() # ensure that the symlinks package will be installed (it's needed for # fixing links inside the sysroot self.cache['symlinks'].mark_install() for p in ignore_dev_pkgs: self.cache[p].mark_delete() dbgsym_list = [ s.name + '-dbgsym' for s in self.cache if (s.is_installed or s.marked_install) ] for p in dbgsym_list: if p in self.cache: pkg = self.cache[p] name_no_suffix = pkg.name[:-len('-dbgsym')] if name_no_suffix in version_dict: version = version_dict[name_no_suffix] candidate = pkg.versions.get(version) if candidate: pkg.candidate = candidate pkg.mark_install() def cleanup(self, exclude_pkgs): for p in self.cache: if p.is_installed and not \ p.is_auto_installed or \ p.is_auto_removable: remove = True for x in exclude_pkgs: if x == p.name: remove = False if remove: p.mark_delete(auto_fix=True, purge=True) def mark_upgrade(self, pkgname, version): p = self.cache[pkgname] if version: p.candidate = p.versions[version] p.mark_upgrade() def mark_delete(self, pkgname): p = self.cache[pkgname] p.mark_delete(purge=True) def mark_keep(self, pkgname, version): p = self.cache[pkgname] p.mark_keep() def update(self): self.cache.update(fetch_progress=ElbeAcquireProgress()) self.cache.open(progress=ElbeOpProgress()) def commit(self): os.environ["DEBIAN_FRONTEND"] = "noninteractive" os.environ["DEBONF_NONINTERACTIVE_SEEN"] = "true" print("Commiting changes ...") self.cache.commit(ElbeAcquireProgress(), ElbeInstallProgress(fileno=sys.stdout.fileno())) self.cache.open(progress=ElbeOpProgress()) def clear(self): self.cache.clear() def get_dependencies(self, pkgname): deps = getalldeps(self.cache, pkgname) return [APTPackage(p, cache=self.cache) for p in deps] def get_installed_pkgs(self, section='all'): if section == 'all': pl = [APTPackage(p) for p in self.cache if p.is_installed] else: pl = [ APTPackage(p) for p in self.cache if (p.section == section and p.is_installed) ] return pl def get_fileindex(self): index = {} for p in self.cache: if p.is_installed: for f in p.installed_files: index[f] = p.name return index def get_marked_install(self, section='all'): if section == 'all': ret = [APTPackage(p) for p in self.cache if p.marked_install] else: ret = [ APTPackage(p) for p in self.cache if (p.section == section and p.marked_install) ] return ret def get_upgradeable(self, section='all'): if section == 'all': ret = [APTPackage(p) for p in self.cache if p.is_upgradable] else: ret = [ APTPackage(p) for p in self.cache if (p.section == section and p.is_upgradable) ] return ret def upgrade(self, dist_upgrade=False): self.cache.upgrade(dist_upgrade) def get_changes(self): changes = self.cache.get_changes() return [APTPackage(p) for p in changes] def has_pkg(self, pkgname): return pkgname in self.cache def is_installed(self, pkgname): if pkgname not in self.cache: return False return self.cache[pkgname].is_installed def get_pkg(self, pkgname): return APTPackage(self.cache[pkgname]) def get_pkgs(self, pkgname): return [ APTPackage(self.cache[p]) for p in sorted(self.cache.keys()) if pkgname in p.lower() ] def compare_versions(self, ver1, ver2): return version_compare(ver1, ver2) def download_binary(self, pkgname, path, version=None): p = self.cache[pkgname] if version is None: pkgver = p.installed else: pkgver = p.versions[version] rel_filename = fetch_binary(pkgver, path, ElbeAcquireProgress()) return self.rfs.fname(rel_filename) def download_source(self, pkgname, path, version=None): p = self.cache[pkgname] if version is None: pkgver = p.installed else: pkgver = p.versions[version] rel_filename = pkgver.fetch_source(path, ElbeAcquireProgress(), unpack=False) return self.rfs.fname(rel_filename)
def get_initvm_pkglist (): cache = Cache () cache.open () pkglist = [APTPackage (p) for p in cache if p.is_installed] pkglist.append ( APTPackage( cache ['elbe-bootstrap'] ) ) return pkglist