def mk_source_cdrom(rfs, arch, codename, init_codename, target, log, cdrom_size=CDROM_SIZE): hostfs.mkdir_p('/var/cache/elbe/sources') rfs.mkdir_p('/var/cache/elbe/sources') repo = CdromSrcRepo(codename, init_codename, os.path.join(target, "srcrepo"), log, cdrom_size) cache = get_rpcaptcache(rfs, "aptcache.log", arch) pkglist = cache.get_installed_pkgs() for pkg in pkglist: try: dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources') repo.includedsc(dsc) except ValueError as ve: log.printo("No sources for Package " + pkg.name + "-" + pkg.installed_version) except FetchError as fe: log.printo("Source for Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded") pkglist = get_initvm_pkglist() cache = Cache() cache.open() for pkg in pkglist: try: p = cache[pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions[0] else: pkgver = p.installed dsc = pkgver.fetch_source('/var/cache/elbe/sources', ElbeAcquireProgress(cb=None), unpack=False) repo.includedsc(dsc) except ValueError as ve: log.printo("No sources for Package " + pkg.name + "-" + str(pkg.installed_version)) except FetchError as fe: log.printo("Source for Package " + pkg.name + "-" + pkgver.version + " could not be downloaded") return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def mk_source_cdrom(rfs, arch, codename, init_codename, target, log, cdrom_size=CDROM_SIZE): hostfs.mkdir_p( '/var/cache/elbe/sources' ) rfs.mkdir_p( '/var/cache/elbe/sources' ) repo = CdromSrcRepo( codename, init_codename, os.path.join( target, "srcrepo" ), log, cdrom_size ) cache = get_rpcaptcache( rfs, "aptcache.log", arch ) pkglist = cache.get_installed_pkgs() for pkg in pkglist: try: dsc = cache.download_source( pkg.name, '/var/cache/elbe/sources' ) repo.includedsc( dsc ) except ValueError as ve: log.printo( "No sources for Package " + pkg.name + "-" + pkg.installed_version ) except FetchError as fe: log.printo( "Source for Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded" ) pkglist = get_initvm_pkglist() cache = Cache () cache.open () for pkg in pkglist: try: p = cache[pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions [0] else: pkgver = p.installed dsc = pkgver.fetch_source ('/var/cache/elbe/sources', ElbeAcquireProgress (cb=None), unpack=False) repo.includedsc( dsc ) except ValueError as ve: log.printo( "No sources for Package " + pkg.name + "-" + str(pkg.installed_version) ) except FetchError as fe: log.printo( "Source for Package " + pkg.name + "-" + pkgver.version + " could not be downloaded" ) return repo.buildiso( os.path.join( target, "src-cdrom.iso" ) )
def run_command(argv): # TODO - Set threshold and remove pylint directives # # We might want to make the threshold higher for certain # files/directories or just globaly. # pylint: disable=too-many-locals # pylint: disable=too-many-branches # pylint: disable=too-many-statements oparser = OptionParser( usage="usage: %prog fetch_initvm_pkgs [options] <xmlfile>") oparser.add_option("-b", "--binrepo", dest="binrepo", default="/var/cache/elbe/initvm-bin-repo", help="directory where the bin repo should reside") oparser.add_option("-s", "--srcrepo", dest="srcrepo", default="/var/cache/elbe/initvm-src-repo", help="directory where the src repo should reside") oparser.add_option("--skip-validation", action="store_true", dest="skip_validation", default=False, help="Skip xml schema validation") oparser.add_option("--cdrom-mount-path", dest="cdrom_path", help="path where cdrom is mounted") oparser.add_option("--cdrom-device", dest="cdrom_device", help="cdrom device, in case it has to be mounted") oparser.add_option("--apt-archive", dest="archive", default="/var/cache/elbe/binaries/main", help="path where binary packages are downloaded to.") oparser.add_option("--src-archive", dest="srcarchive", default="/var/cache/elbe/sources", help="path where src packages are downloaded to.") oparser.add_option("--skip-build-sources", action="store_false", dest="build_sources", default=True, help="Skip downloading Source Packages") oparser.add_option("--skip-build-bin", action="store_false", dest="build_bin", default=True, help="Skip downloading binary packages") (opt, args) = oparser.parse_args(argv) if len(args) != 1: print("wrong number of arguments") oparser.print_help() sys.exit(20) try: xml = ElbeXML(args[0], skip_validate=opt.skip_validation) except ValidationError as e: print(str(e)) print("xml validation failed. Bailing out") sys.exit(20) with elbe_logging({"streams": sys.stdout}): if opt.cdrom_path: if opt.cdrom_device: do('mount "%s" "%s"' % (opt.cdrom_device, opt.cdrom_path)) # a cdrom build is identified by the cdrom option # the xml file that is copied into the initvm # by the initrd does not have the cdrom tags setup. mirror = "file://%s" % opt.cdrom_path else: mirror = xml.get_initvm_primary_mirror(opt.cdrom_path) init_codename = xml.get_initvm_codename() # Binary Repo # repo = CdromInitRepo(init_codename, opt.binrepo, mirror) hostfs.mkdir_p(opt.archive) if opt.build_bin: pkglist = get_initvm_pkglist() cache = Cache() cache.open() for pkg in pkglist: pkg_id = "%s-%s" % (pkg.name, pkg.installed_version) try: p = cache[pkg.name] pkgver = p.installed deb = fetch_binary(pkgver, opt.archive, ElbeAcquireProgress(cb=None)) repo.includedeb(deb, 'main') except ValueError: logging.exception('No package "%s"', pkg_id) except FetchError: logging.exception( 'Package "%s-%s" could not be downloaded', pkg.name, pkgver.version) except TypeError: logging.exception('Package "%s" missing name or version', pkg_id) repo.finalize() # Source Repo # repo = CdromSrcRepo(init_codename, init_codename, opt.srcrepo, 0, mirror) hostfs.mkdir_p(opt.srcarchive) # a cdrom build does not have sources # skip adding packages to the source repo # # FIXME: we need a way to add source cdroms later on if opt.cdrom_path: opt.build_sources = False if opt.build_sources: for pkg in pkglist: pkg_id = "%s-%s" % (pkg.name, pkg.installed_version) try: p = cache[pkg.name] pkgver = p.installed dsc = pkgver.fetch_source(opt.srcarchive, ElbeAcquireProgress(cb=None), unpack=False) repo.include_init_dsc(dsc, 'initvm') except ValueError: logging.exception('No package "%s"', pkg_id) except FetchError: logging.exception( 'Package "%s-%s" could not be downloaded', pkg.name, pkgver.version) except TypeError: logging.exception('Package "%s" missing name or version', pkg_id) repo.finalize() if opt.cdrom_device: do('umount "%s"' % opt.cdrom_device)
def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target, log, cdrom_size=CDROM_SIZE): rfs.mkdir_p('/var/cache/elbe/binaries/added') rfs.mkdir_p('/var/cache/elbe/binaries/main') hostfs.mkdir_p('/var/cache/elbe/binaries/main') if not xml is None: mirror = xml.get_primary_mirror(rfs.fname("cdrom")) else: mirror = 'http://ftp.debian.org/debian' repo_path = os.path.join(target, "binrepo") repo = CdromBinRepo(arch, codename, init_codename, repo_path, log, cdrom_size, mirror) if not xml is None: pkglist = get_initvm_pkglist() cache = Cache() cache.open() for pkg in pkglist: try: p = cache[pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions[0] else: pkgver = p.installed deb = pkgver.fetch_binary('/var/cache/elbe/binaries/main', ElbeAcquireProgress(cb=None)) repo.include_init_deb(deb, 'main') except ValueError as ve: log.printo("No Package " + pkg.name + "-" + str(pkg.installed_version)) except FetchError as fe: log.printo("Package " + pkg.name + "-" + pkgver.version + " could not be downloaded") except TypeError as te: log.printo("Package " + pkg.name + "-" + str(pkg.installed_version) + " missing name or version") cache = get_rpcaptcache(rfs, "aptcache.log", arch) for p in xml.node("debootstrappkgs"): pkg = XMLPackage(p, arch) try: deb = cache.download_binary(pkg.name, '/var/cache/elbe/binaries/main', pkg.installed_version) repo.includedeb(deb, 'main') except ValueError as ve: log.printo("No Package " + pkg.name + "-" + pkg.installed_version) except FetchError as fe: log.printo("Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded") except TypeError as te: log.printo("Package " + pkg.name + "-" + pkg.installed_version + " missing name or version") cache = get_rpcaptcache(rfs, "aptcache.log", arch) pkglist = cache.get_installed_pkgs() for pkg in pkglist: try: deb = cache.download_binary(pkg.name, '/var/cache/elbe/binaries/added', pkg.installed_version) repo.includedeb(deb, 'added') except KeyError as ke: log.printo(str(ke)) except ValueError as ve: log.printo("No Package " + pkg.name + "-" + pkg.installed_version) except FetchError as fe: log.printo("Package " + pkg.name + "-" + str(pkg.installed_version) + " could not be downloaded") except TypeError as te: log.printo("Package " + pkg.name + "-" + pkg.installed_version + " missing name or version") # Mark the binary repo with the necessary Files # to make the installer accept this as a CDRom repo_fs = Filesystem(repo_path) repo_fs.mkdir_p(".disk") repo_fs.write_file(".disk/base_installable", 0644, "main\n") repo_fs.write_file(".disk/base_components", 0644, "main\n") repo_fs.write_file(".disk/cd_type", 0644, "not_complete\n") repo_fs.write_file(".disk/info", 0644, "elbe inst cdrom - full cd\n") repo_fs.symlink(".", "debian", allow_exists=True) repo_fs.write_file("md5sum.txt", 0644, "") # write source xml onto cdrom xml.xml.write(repo_fs.fname('source.xml')) return repo.buildiso(os.path.join(target, "bin-cdrom.iso"))
def mk_source_cdrom(rfs, arch, codename, init_codename, target, log, cdrom_size=CDROM_SIZE, xml=None): hostfs.mkdir_p('/var/cache/elbe/sources') rfs.mkdir_p('/var/cache/elbe/sources') repo = CdromSrcRepo(codename, init_codename, os.path.join(target, "srcrepo"), log, cdrom_size) cache = get_rpcaptcache(rfs, "aptcache.log", arch) pkglist = cache.get_installed_pkgs() forbiddenPackages = [] if xml != None and xml.has('target/pkg-list'): for i in xml.node('target/pkg-list'): try: if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False': forbiddenPackages.append(i.text('.').strip()) except KeyError: pass for pkg in pkglist: # Do not include forbidden packages in src cdrom if pkg.name in forbiddenPackages: continue try: dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources') repo.includedsc(dsc, force=True) except ValueError as ve: log.printo("No sources for Package " + pkg.name + "-" + pkg.installed_version) except FetchError as fe: log.printo("Source for Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded") repo.finalize() pkglist = get_initvm_pkglist() cache = Cache() cache.open() for pkg in pkglist: # Do not include forbidden packages in src cdrom if pkg.name in forbiddenPackages: continue try: p = cache[pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions[0] else: pkgver = p.installed dsc = pkgver.fetch_source('/var/cache/elbe/sources', ElbeAcquireProgress(cb=None), unpack=False) repo.includedsc(dsc) except ValueError as ve: log.printo("No sources for Package " + pkg.name + "-" + str(pkg.installed_version)) except FetchError as fe: log.printo("Source for Package " + pkg.name + "-" + pkgver.version + " could not be downloaded") repo.finalize() return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target, log, cdrom_size=CDROM_SIZE): rfs.mkdir_p( '/var/cache/elbe/binaries/added' ) rfs.mkdir_p( '/var/cache/elbe/binaries/main' ) hostfs.mkdir_p( '/var/cache/elbe/binaries/main' ) if not xml is None: mirror = xml.get_primary_mirror (rfs.fname("cdrom")) else: mirror='http://ftp.debian.org/debian' repo_path = os.path.join (target, "binrepo") repo = CdromBinRepo (arch, codename, init_codename, repo_path, log, cdrom_size, mirror) if not xml is None: pkglist = get_initvm_pkglist() cache = Cache () cache.open () for pkg in pkglist: try: p = cache [pkg.name] if pkg.name == 'elbe-bootstrap': pkgver = p.versions [0] else: pkgver = p.installed deb = pkgver.fetch_binary ('/var/cache/elbe/binaries/main', ElbeAcquireProgress (cb=None) ) repo.include_init_deb(deb, 'main') except ValueError as ve: log.printo( "No Package " + pkg.name + "-" + str(pkg.installed_version) ) except FetchError as fe: log.printo( "Package " + pkg.name + "-" + pkgver.version + " could not be downloaded" ) except TypeError as te: log.printo( "Package " + pkg.name + "-" + str(pkg.installed_version) + " missing name or version" ) cache = get_rpcaptcache( rfs, "aptcache.log", arch ) for p in xml.node("debootstrappkgs"): pkg = XMLPackage(p, arch) try: deb = cache.download_binary( pkg.name, '/var/cache/elbe/binaries/main', pkg.installed_version ) repo.includedeb(deb, 'main') except ValueError as ve: log.printo( "No Package " + pkg.name + "-" + pkg.installed_version ) except FetchError as fe: log.printo( "Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded" ) except TypeError as te: log.printo( "Package " + pkg.name + "-" + pkg.installed_version + " missing name or version" ) cache = get_rpcaptcache( rfs, "aptcache.log", arch ) pkglist = cache.get_installed_pkgs () for pkg in pkglist: try: deb = cache.download_binary( pkg.name, '/var/cache/elbe/binaries/added', pkg.installed_version ) repo.includedeb(deb, 'added') except KeyError as ke: log.printo( str (ke) ) except ValueError as ve: log.printo( "No Package " + pkg.name + "-" + pkg.installed_version ) except FetchError as fe: log.printo( "Package " + pkg.name + "-" + str (pkg.installed_version) + " could not be downloaded" ) except TypeError as te: log.printo( "Package " + pkg.name + "-" + pkg.installed_version + " missing name or version" ) # Mark the binary repo with the necessary Files # to make the installer accept this as a CDRom repo_fs = Filesystem( repo_path ) repo_fs.mkdir_p (".disk") repo_fs.write_file (".disk/base_installable", 0644, "main\n") repo_fs.write_file (".disk/base_components", 0644, "main\n") repo_fs.write_file (".disk/cd_type", 0644, "not_complete\n") repo_fs.write_file (".disk/info", 0644, "elbe inst cdrom - full cd\n") repo_fs.symlink (".", "debian", allow_exists=True) repo_fs.write_file ("md5sum.txt", 0644, "") # write source xml onto cdrom xml.xml.write (repo_fs.fname ('source.xml')) return repo.buildiso( os.path.join( target, "bin-cdrom.iso" ) )