def execute(self, client, opt, args):
if len (args) != 1:
print ("usage: elbe control get_files <project_dir>", file=sys.stderr)
sys.exit(20)
builddir = args[0]
files = client.service.get_files (builddir)
nfiles = 0
for f in files.SoapFile:
if opt.pbuilder_only and not f.name.startswith ('pbuilder'):
continue
nfiles += 1
if f.description:
print ("%s \t(%s)" % (f.name, f.description))
else:
print ("%s" % (f.name))
if opt.output:
fs = Filesystem ('/')
dst = os.path.abspath (opt.output)
fs.mkdir_p (dst)
dst_fname = str (os.path.join (dst, os.path.basename (f.name)))
client.download_file (builddir, f.name, dst_fname)
if nfiles == 0:
sys.exit (10)
def execute(self, client, opt, args):
if len(args) != 1:
print(
"usage: elbe control get_files <project_dir>",
file=sys.stderr)
sys.exit(20)
builddir = args[0]
files = client.service.get_files(builddir)
nfiles = 0
for f in files[0]:
if (opt.pbuilder_only and not f.name.startswith('pbuilder_cross')):
continue
if opt.matches and not fnmatch.fnmatch(f.name, opt.matches):
continue
nfiles += 1
try:
print("%s \t(%s)" % (f.name, f.description))
except AttributeError:
print("%s" % (f.name))
if opt.output:
fs = Filesystem('/')
dst = os.path.abspath(opt.output)
fs.mkdir_p(dst)
dst_fname = str(os.path.join(dst, os.path.basename(f.name)))
client.download_file(builddir, f.name, dst_fname)
if nfiles == 0:
sys.exit(10)
def execute(self, client, opt, args):
if len(args) != 1:
print("usage: elbe control get_build_chroot_tarball <project_dir>",
file=sys.stderr)
sys.exit(20)
builddir = args[0]
dst_fname = "build_chroot.tar.xz"
if opt.output:
fs = Filesystem('/')
dst = os.path.abspath(opt.output)
fs.mkdir_p(dst)
dst_fname = str(os.path.join(dst, filename))
with file(dst_fname, "w") as fp:
part = 0
while True:
ret = client.service.get_build_chroot_tarball(builddir, part)
if ret == "EndOfFile":
break
fp.write(binascii.a2b_base64(ret))
part = part + 1
print(dst_fname + " saved", file=sys.stderr)
def run_command(argv):
oparser = OptionParser(usage="usage: %prog genlicence [options] <rfs>")
oparser.add_option("--output", dest="output", help="outputfilename")
oparser.add_option("--xml",
dest="xml",
default=None,
help="xml outputfilename")
(opt, args) = oparser.parse_args(argv)
if len(args) != 1:
print "wrong number of arguments"
oparser.print_help()
sys.exit(20)
chroot = os.path.abspath(args[0])
rfs = Filesystem(chroot)
log = StdoutLog()
if opt.output:
f = io.open(opt.output, "w+", encoding='utf-8')
else:
f = io.open('licence.txt', "w+", encoding='utf-8')
rfs.write_licenses(f, log, opt.xml)
f.close()
def get_volume_fs(self, volume):
if self.maxsize:
if volume >= 0:
volume_no = volume
else:
# negative numbers represent the volumes counted from last
# (-1: last, -2: second last, ...)
volume_no = self.volume_count + 1 + volume
volname = os.path.join(self.vol_path, "vol%02d" % volume_no)
return Filesystem(volname)
return Filesystem(self.vol_path)
def get_apt_keys(builddir, xml):
if xml.prj is None:
return (["# No Project"], [])
if not xml.prj.has("mirror") and not xml.prj.has("mirror/cdrom"):
return (["# No mirrors configured"], [])
keys = [Filesystem(builddir).read_file("repo/repo.pub")]
if xml.prj.has("mirror/primary_host") and xml.prj.has("mirror/url-list"):
for url in xml.prj.node("mirror/url-list"):
if url.has("options"):
options = "[%s]" % ' '.join([opt.et.text.strip(' \t\n')
for opt
in url.all("options/option")])
else:
options = ""
if url.has("raw-key") and not "trusted=yes" in options:
key = "\n".join(line.strip(" \t")
for line
in url.text('raw-key').splitlines()[1:-1])
keys.append(key)
return (keys)
def execute(self, client, opt, args):
if len (args) != 2:
print ("usage: elbe control get_file <project_dir> <file>", file=sys.stderr)
sys.exit(20)
builddir = args[0]
filename = args[1]
dst_fname = filename
if opt.output:
fs = Filesystem ('/')
dst = os.path.abspath (opt.output)
fs.mkdir_p (dst)
dst_fname = str (os.path.join (dst, filename))
client.download_file (builddir, filename, dst_fname)
print (dst_fname + " saved", file=sys.stderr)
def execute(self, client, opt, args):
if len(args) != 2:
print("usage: elbe control get_file <project_dir> <file>",
file=sys.stderr)
sys.exit(20)
builddir = args[0]
filename = args[1]
dst_fname = filename
if opt.output:
fs = Filesystem('/')
dst = os.path.abspath(opt.output)
fs.mkdir_p(dst)
dst_fname = str(os.path.join(dst, filename)).encode()
client.download_file(builddir, filename, dst_fname)
print("%s saved" % dst_fname)
def execute(self, client, opt, args):
if len (args) != 1:
print ("usage: elbe control get_build_chroot_tarball <project_dir>", file=sys.stderr)
sys.exit(20)
builddir = args[0]
dst_fname = "build_chroot.tar.xz"
if opt.output:
fs = Filesystem ('/')
dst = os.path.abspath (opt.output)
fs.mkdir_p (dst)
dst_fname = str (os.path.join (dst, filename))
with file (dst_fname, "w") as fp:
part = 0
while True:
ret = client.service.get_build_chroot_tarball (builddir, part)
if ret == "EndOfFile":
break
fp.write (binascii.a2b_base64 (ret))
part = part + 1
print (dst_fname + " saved", file=sys.stderr)
def run_command( argv ):
oparser = OptionParser(usage="usage: %prog genlicence [options] <rfs>")
oparser.add_option( "--output", dest="output",
help="outputfilename" )
(opt,args) = oparser.parse_args(argv)
if len(args) != 1:
print "wrong number of arguments"
oparser.print_help()
sys.exit(20)
chroot = os.path.abspath(args[0])
rfs = Filesystem( chroot )
log = StdoutLog()
if opt.output:
f = open( opt.output, "w+" )
else:
f = open( 'licence.txt', "w+" )
rfs.write_licenses(f, log)
f.close()
def install(self, target):
if '/' not in self.fs:
return
imagemnt = os.path.join(target, "imagemnt")
imagemntfs = Filesystem(imagemnt)
try:
loopdev = self.losetup(self.fs['/'].filename)
loopnum = loopdev.replace("/dev/loop", "")
poopdev = "/dev/poop" + loopnum
do('cp -a %s %s' % (loopdev, poopdev))
do('kpartx -as %s' % poopdev)
for entry in self.fs.depthlist():
do('mount /dev/mapper/poop%sp%d %s' %
(loopnum, entry.partnum, imagemntfs.fname(
entry.mountpoint)))
do("mount --bind /dev %s" % imagemntfs.fname("dev"))
do("mount --bind /proc %s" % imagemntfs.fname("proc"))
do("mount --bind /sys %s" % imagemntfs.fname("sys"))
do('mkdir -p "%s"' % imagemntfs.fname("boot/grub"))
devmap = open(imagemntfs.fname("boot/grub/device.map"), "w")
devmap.write("(hd0) %s\n" % poopdev)
devmap.close()
chroot(imagemnt, "update-initramfs -u -k all")
# Replace groot and kopt because
# else they will be given bad values
do('chroot %s sed -in "s/^# groot=.*$/# groot=\(hd0,%d\)/" %s' %
(imagemnt, int(entry.partnum) - 1, "/boot/grub/menu.lst"))
do('chroot %s sed -in "s/^# kopt=.*$/# kopt=root=LABEL=%s/" %s' %
(imagemnt, entry.label, "/boot/grub/menu.lst"))
chroot(imagemnt, "update-grub")
do("chroot %s grub-install --no-floppy %s" % (imagemnt, poopdev))
except Exception as E:
logging.exception(E)
finally:
os.unlink(imagemntfs.fname("boot/grub/device.map"))
do("umount %s" % imagemntfs.fname("dev"), allow_fail=True)
do("umount %s" % imagemntfs.fname("proc"), allow_fail=True)
do("umount %s" % imagemntfs.fname("sys"), allow_fail=True)
for entry in reversed(self.fs.depthlist()):
do('umount /dev/mapper/poop%sp%d' % (loopnum, entry.partnum),
allow_fail=True)
do("kpartx -d %s" % poopdev, allow_fail=True)
do("losetup -d %s" % poopdev, allow_fail=True)
def install(self, target):
if '/' not in self.fs:
return
imagemnt = os.path.join(target, "imagemnt")
imagemntfs = Filesystem(imagemnt)
try:
loopdev = self.losetup(self.fs['/'].filename)
loopnum = loopdev.replace("/dev/loop", "")
poopdev = "/dev/poop" + loopnum
do('cp -a %s %s' % (loopdev, poopdev))
do('kpartx -as %s' % poopdev)
for entry in self.fs.depthlist():
do('mount /dev/mapper/poop%sp%d %s' %
(loopnum, entry.partnum, imagemntfs.fname(
entry.mountpoint)))
do("mount --bind /dev %s" % imagemntfs.fname("dev"))
do("mount --bind /proc %s" % imagemntfs.fname("proc"))
do("mount --bind /sys %s" % imagemntfs.fname("sys"))
do('mkdir -p "%s"' % imagemntfs.fname("boot/grub"))
devmap = open(imagemntfs.fname("boot/grub/device.map"), "w")
devmap.write("(hd0) %s\n" % poopdev)
devmap.close()
chroot(imagemnt, "update-initramfs -u -k all")
chroot(imagemnt, "update-grub2")
if "efi" in self.fw_type:
grub_tgt = next(t for t in self.fw_type if t.endswith("-efi"))
do("chroot %s grub-install --target=%s --removable "
"--no-floppy %s" % (imagemnt, grub_tgt, poopdev))
if "shimfix" in self.fw_type:
# grub-install is heavily dependent on the running system having
# a BIOS or EFI. The initvm is BIOS-based, so fix the resulting
# shim installation.
do("chroot %s /bin/bash -c '"
"cp -r /boot/efi/EFI/BOOT /boot/efi/EFI/debian && "
"cd /usr/lib/shim && f=( shim*.efi.signed ) && cp "
"${f[0]} /boot/efi/EFI/debian/${f[0]%%.signed}'" % imagemnt)
if not self.fw_type or "bios" in self.fw_type:
do("chroot %s grub-install --no-floppy %s" %
(imagemnt, poopdev))
except Exception as E:
logging.exception(E)
finally:
os.unlink(imagemntfs.fname("boot/grub/device.map"))
do("umount %s" % imagemntfs.fname("dev"), allow_fail=True)
do("umount %s" % imagemntfs.fname("proc"), allow_fail=True)
do("umount %s" % imagemntfs.fname("sys"), allow_fail=True)
for entry in reversed(self.fs.depthlist()):
do('umount /dev/mapper/poop%sp%d' % (loopnum, entry.partnum),
allow_fail=True)
do("kpartx -d %s" % poopdev, allow_fail=True)
do("losetup -d %s" % poopdev, allow_fail=True)
def run_command(argv):
# pylint: disable=too-many-locals
# pylint: disable=too-many-statements
# pylint: disable=too-many-branches
oparser = OptionParser(usage="usage: %prog init [options] <filename>")
oparser.add_option("--skip-validation", action="store_true",
dest="skip_validation", default=False,
help="Skip xml schema validation")
oparser.add_option("--directory", dest="directory", default="./build",
help="Working directory (default is build)",
metavar="FILE")
oparser.add_option(
"--cdrom",
dest="cdrom",
help="Use FILE as cdrom iso, and use that to build the initvm",
metavar="FILE")
oparser.add_option("--buildtype", dest="buildtype",
help="Override the buildtype")
oparser.add_option(
"--debug",
dest="debug",
action="store_true",
default=False,
help="start qemu in graphical mode to enable console switch")
oparser.add_option(
"--devel",
dest="devel",
action="store_true",
default=False,
help="use devel mode, and install current builddir inside initvm")
oparser.add_option(
"--nesting",
dest="nesting",
action="store_true",
default=False,
help="allow initvm to support nested kvm. "
"This makes /proc/cpuinfo inside initvm differ per host.")
oparser.add_option(
"--skip-build-bin",
action="store_false",
dest="build_bin",
default=True,
help="Skip building Binary Repository CDROM, for exact Reproduction")
oparser.add_option(
"--skip-build-sources",
action="store_false",
dest="build_sources",
default=True,
help="Skip building Source CDROM")
(opt, args) = oparser.parse_args(argv)
if not args:
print("no filename specified")
oparser.print_help()
sys.exit(20)
elif len(args) > 1:
print("too many filenames specified")
oparser.print_help()
sys.exit(20)
with elbe_logging({"files": None}):
if opt.devel:
if not os.path.isdir(os.path.join(elbe_dir, "elbepack")):
logging.error("Devel Mode only valid, "
"when running from elbe checkout")
sys.exit(20)
if not opt.skip_validation:
validation = validate_xml(args[0])
if validation:
logging.error("xml validation failed. Bailing out")
for i in validation:
logging.error(i)
sys.exit(20)
xml = etree(args[0])
if not xml.has("initvm"):
logging.error("fatal error: "
"xml missing mandatory section 'initvm'")
sys.exit(20)
if opt.buildtype:
buildtype = opt.buildtype
elif xml.has("initvm/buildtype"):
buildtype = xml.text("/initvm/buildtype")
else:
buildtype = "nodefaults"
defs = ElbeDefaults(buildtype)
http_proxy = xml.text("/initvm/mirror/primary_proxy", default="")
http_proxy = http_proxy.strip().replace("LOCALMACHINE", "localhost")
if opt.cdrom:
mirror = xml.node("initvm/mirror")
mirror.clear()
cdrom = mirror.ensure_child("cdrom")
cdrom.set_text(os.path.abspath(opt.cdrom))
# this is a workaround for
# http://lists.linutronix.de/pipermail/elbe-devel/2017-July/000541.html
_, virt = command_out('test -x /usr/bin/systemd-detect-virt && '
'/usr/bin/systemd-detect-virt')
_, dist = command_out('lsb_release -cs')
if 'vmware' in virt and 'stretch' in dist:
machine_type = 'pc-i440fx-2.6'
else:
machine_type = 'pc'
try:
os.makedirs(opt.directory)
except OSError as e:
logging.error("unable to create project directory: %s (%s)",
opt.directory,
e.strerror)
sys.exit(30)
out_path = os.path.join(opt.directory, ".elbe-in")
try:
os.makedirs(out_path)
except OSError as e:
logging.error("unable to create subdirectory: %s (%s)",
out_path,
e.strerror)
sys.exit(30)
initvm_http_proxy = http_proxy.replace('http://localhost:',
'http://10.0.2.2:')
d = {"elbe_version": elbe_version,
"defs": defs,
"opt": opt,
"xml": xml,
"prj": xml.node("/initvm"),
"http_proxy": initvm_http_proxy,
"pkgs": xml.node("/initvm/pkg-list") or [],
"preseed": get_initvm_preseed(xml),
"machine_type": machine_type,
"cfg": cfg}
if http_proxy != "":
os.putenv("http_proxy", http_proxy)
os.putenv("https_proxy", http_proxy)
os.putenv("no_proxy", "localhost,127.0.0.1")
try:
copy_kinitrd(xml.node("/initvm"), out_path)
except NoKinitrdException as e:
msg = str(e)
logging.error("Failure to download kernel/initrd debian Package:")
logging.error("")
logging.error(msg)
logging.error("")
logging.error("Check Mirror configuration")
if 'SHA256SUMS' in msg:
logging.error("If you use debmirror please read "
"https://github.com/Linutronix/elbe/issues/188 "
"on how to work around the issue")
sys.exit(20)
templates = os.listdir(init_template_dir)
make_executable = ["init-elbe.sh.mako",
"preseed.cfg.mako"]
for t in templates:
o = t.replace(".mako", "")
if t in ("Makefile.mako", "libvirt.xml.mako"):
write_template(
os.path.join(
opt.directory, o), os.path.join(
init_template_dir, t), d, linebreak=True)
else:
write_template(
os.path.join(
out_path, o), os.path.join(
init_template_dir, t), d, linebreak=False)
if t in make_executable:
os.chmod(os.path.join(out_path, o), 0o755)
shutil.copyfile(args[0],
os.path.join(out_path, "source.xml"))
if opt.cdrom:
system('7z x -o%s "%s" elbe-keyring.gpg' % (out_path, opt.cdrom))
else:
keys = []
for key in xml.all(".//initvm/mirror/url-list/url/raw-key"):
keys.append(key.et.text)
import_keyring = os.path.join(out_path, "elbe-keyring")
do('gpg --no-options \
--no-default-keyring \
--keyring %s --import' % import_keyring,
stdin="".join(keys).encode('ascii'),
allow_fail=True,
env_add={'GNUPGHOME': out_path})
export_keyring = import_keyring + ".gpg"
# No need to set GNUPGHOME because both input and output
# keyring files are specified.
do('gpg --no-options \
--no-default-keyring \
--keyring %s \
--export \
--output %s' % (import_keyring, export_keyring))
if opt.devel:
out_real = os.path.realpath(out_path)
opts = []
if out_real.startswith(elbe_dir + os.sep):
opts.append('--exclude "%s"' %
os.path.relpath(out_path, start=elbe_dir))
opts.append("--exclude-vcs")
opts.append("--exclude-vcs-ignores")
opts.append("--exclude='elbe-build*'")
opts.append("--exclude='docs/*'")
tar_fname = os.path.join(out_path, "elbe-devel.tar.bz2")
system('tar cfj "%s" %s -C "%s" .' % (tar_fname,
" ".join(opts),
elbe_dir))
to_cpy = [("apt.conf", "etc/apt"),
("init-elbe.sh", ""),
("source.xml", ""),
("initrd-cdrom.gz", ""),
("vmlinuz", ""),
("preseed.cfg", "")]
elbe_in = Filesystem(out_path)
if opt.devel:
to_cpy.append(("elbe-devel.tar.bz2", ""))
# Convert relative rfs path to absolute in the system
to_cpy = [(elbe_in.fname(src), elbe_in.fname(os.path.join("initrd-tree", dst)))
for src, dst
in to_cpy]
# These are already absolute path!
keyrings = elbe_in.fname(os.path.join("initrd-tree", "usr/share/keyrings"))
for gpg in elbe_in.glob("*.gpg"):
to_cpy.append((gpg, keyrings))
for src, dst in to_cpy:
try:
os.makedirs(dst)
except FileExistsError:
pass
shutil.copy(src, dst)
def install(self, target):
if '/' not in self.fs:
return
imagemnt = os.path.join(target, "imagemnt")
imagemntfs = Filesystem(imagemnt)
try:
do('cp -a /dev/loop0 /dev/poop0')
do('losetup /dev/poop0 "%s"' % self.fs['/'].filename)
do('kpartx -as /dev/poop0')
for entry in self.fs.depthlist():
do('mount /dev/mapper/poop0p%d %s' %
(entry.partnum, imagemntfs.fname(entry.mountpoint)))
do("mount --bind /dev %s" % imagemntfs.fname("dev"))
do("mount --bind /proc %s" % imagemntfs.fname("proc"))
do("mount --bind /sys %s" % imagemntfs.fname("sys"))
do('mkdir -p "%s"' % imagemntfs.fname("boot/grub"))
devmap = open(imagemntfs.fname("boot/grub/device.map"), "w")
devmap.write("(hd0) /dev/poop0\n")
devmap.close()
chroot(imagemnt, "update-initramfs -u -k all")
# Replace groot and kopt because
# else they will be given bad values
do('chroot %s sed -in "s/^# groot=.*$/# groot=\(hd0,%d\)/" %s' %
(imagemnt, int(entry.partnum) - 1, "/boot/grub/menu.lst"))
do('chroot %s sed -in "s/^# kopt=.*$/# kopt=root=LABEL=%s/" %s' %
(imagemnt, entry.label, "/boot/grub/menu.lst"))
chroot(imagemnt, "update-grub")
do("chroot %s grub-install --no-floppy /dev/poop0" % (imagemnt))
finally:
os.unlink(imagemntfs.fname("boot/grub/device.map"))
do("umount %s" % imagemntfs.fname("dev"), allow_fail=True)
do("umount %s" % imagemntfs.fname("proc"), allow_fail=True)
do("umount %s" % imagemntfs.fname("sys"), allow_fail=True)
for entry in reversed(self.fs.depthlist()):
do('umount /dev/mapper/poop0p%d' % entry.partnum,
allow_fail=True)
do('kpartx -d /dev/poop0', allow_fail=True)
do("losetup -d /dev/poop0", allow_fail=True)
def __init__(self, path, clean=False):
Filesystem.__init__(self, path, clean)
def get_volume_fs(self, volume):
if self.maxsize:
volname = os.path.join(self.vol_path, "vol%02d" % volume)
return Filesystem(volname)
else:
return Filesystem(self.vol_path)
def mk_binary_cdrom(
rfs,
arch,
codename,
init_codename,
xml,
target,
log,
cdrom_size=CDROM_SIZE):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
rfs.mkdir_p('/var/cache/elbe/binaries/added')
rfs.mkdir_p('/var/cache/elbe/binaries/main')
if xml is not None:
mirror = xml.get_primary_mirror(rfs.fname("cdrom"))
else:
mirror = 'http://ftp.de.debian.org/debian'
repo_path = os.path.join(target, "binrepo")
target_repo_path = os.path.join(repo_path, 'targetrepo')
# initvm repo has been built upon initvm creation
# just copy it. the repo __init__() afterwards will
# not touch the repo config, nor generate a new key.
try:
log.do('cp -av /var/cache/elbe/initvm-bin-repo "%s"' % repo_path)
except CommandError:
# When /var/cache/elbe/initvm-bin-repo has not been created
# (because the initvm install was an old version or somthing,
# log an error, and continue with an empty directory.
log.printo('ERROR: /var/cache/elbe/initvm-bin-repo does not exist')
log.printo(' The generated CDROM will not contain initvm pkgs')
log.printo(' This happened because the initvm was probably')
log.printo(' generated with --skip-build-bin')
log.do('mkdir -p "%s"' % repo_path)
repo = CdromInitRepo(init_codename, repo_path, log, cdrom_size, mirror)
target_repo = CdromBinRepo(arch, codename, None,
target_repo_path, log, cdrom_size, mirror)
if xml is not None:
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
for p in xml.node("debootstrappkgs"):
pkg = XMLPackage(p, arch)
try:
deb = cache.download_binary(pkg.name,
'/var/cache/elbe/binaries/main',
pkg.installed_version)
target_repo.includedeb(deb, 'main')
except ValueError:
log.printo(
"No Package " +
pkg.name +
"-" +
pkg.installed_version)
except FetchError:
log.printo(
"Package " +
pkg.name +
"-" +
pkg.installed_version +
" could not be downloaded")
except TypeError:
log.printo(
"Package " +
pkg.name +
"-" +
pkg.installed_version +
" missing name or version")
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
pkglist = cache.get_installed_pkgs()
for pkg in pkglist:
try:
deb = cache.download_binary(pkg.name,
'/var/cache/elbe/binaries/added',
pkg.installed_version)
target_repo.includedeb(deb, 'added', pkg.name, True)
except KeyError as ke:
log.printo(str(ke))
except ValueError:
log.printo("No Package " + pkg.name + "-" + pkg.installed_version)
except FetchError:
log.printo("Package " +
pkg.name +
"-" +
str(pkg.installed_version) +
" could not be downloaded")
except TypeError:
log.printo(
"Package " +
pkg.name +
"-" +
pkg.installed_version +
" missing name or version")
target_repo.finalize()
# Mark the binary repo with the necessary Files
# to make the installer accept this as a CDRom
repo_fs = Filesystem(repo_path)
repo_fs.mkdir_p(".disk")
repo_fs.write_file(".disk/base_installable", 0o644, "main\n")
repo_fs.write_file(".disk/base_components", 0o644, "main\n")
repo_fs.write_file(".disk/cd_type", 0o644, "not_complete\n")
repo_fs.write_file(".disk/info", 0o644, "elbe inst cdrom - full cd\n")
repo_fs.symlink(".", "debian", allow_exists=True)
repo_fs.write_file("md5sum.txt", 0o644, "")
# write source xml onto cdrom
xml.xml.write(repo_fs.fname('source.xml'))
# copy initvm-cdrom.gz and vmlinuz
copyfile('/var/cache/elbe/installer/initrd-cdrom.gz',
repo_fs.fname('initrd-cdrom.gz'))
copyfile('/var/cache/elbe/installer/vmlinuz',
repo_fs.fname('vmlinuz'))
target_repo_fs = Filesystem(target_repo_path)
target_repo_fs.write_file(".aptignr", 0o644, "")
return repo.buildiso(os.path.join(target, "bin-cdrom.iso"))
def mk_source_cdrom(
rfs,
arch,
codename,
init_codename,
target,
log,
cdrom_size=CDROM_SIZE,
xml=None):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
if xml is not None:
mirror = xml.get_primary_mirror(rfs.fname("cdrom"))
else:
mirror = 'http://ftp.de.debian.org/debian'
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"),
log,
cdrom_size,
mirror)
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
cache.update()
pkglist = cache.get_installed_pkgs()
forbiddenPackages = []
if xml is not None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
for pkg in pkglist:
# Do not include forbidden packages in src cdrom
if pkg.name in forbiddenPackages:
continue
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc, force=True)
except ValueError:
log.printo(
"No sources for Package " +
pkg.name +
"-" +
pkg.installed_version)
except FetchError:
log.printo(
"Source for Package " +
pkg.name +
"-" +
pkg.installed_version +
" could not be downloaded")
# elbe fetch_initvm_pkgs has downloaded all sources to
# /var/cache/elbe/sources
# use walk_files to scan it, and add all dsc files.
#
# we can not just copy the source repo, like we do
# with the bin repo, because the src cdrom can be split
# into multiple cdroms
initvm_repo = Filesystem('/var/cache/elbe/sources')
for _ , dsc_real in initvm_repo.walk_files():
if not dsc_real.endswith('.dsc'):
continue
repo.include_init_dsc(dsc_real, 'initvm')
repo.finalize()
return repo.buildiso(os.path.join(target, "src-cdrom.iso"))
def mk_source_cdrom(rfs, arch, codename, init_codename, target,
cdrom_size=CDROM_SIZE, xml=None):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
hostfs.mkdir_p('/var/cache/elbe/sources')
rfs.mkdir_p('/var/cache/elbe/sources')
if xml is not None:
mirror = xml.get_primary_mirror(rfs.fname("cdrom"))
else:
mirror = 'http://ftp.de.debian.org/debian'
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo"),
cdrom_size,
mirror)
cache = get_rpcaptcache(rfs, arch)
cache.update()
pkglist = cache.get_installed_pkgs()
forbiddenPackages = []
if xml is not None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
for pkg in pkglist:
# Do not include forbidden packages in src cdrom
if pkg.name in forbiddenPackages:
continue
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
dsc = cache.download_source(pkg.name, '/var/cache/elbe/sources')
repo.includedsc(dsc, force=True)
except ValueError:
logging.error("No sources for package '%s'", pkg_id)
except FetchError:
logging.error("Source for package '%s' could not be downloaded", pkg_id)
# elbe fetch_initvm_pkgs has downloaded all sources to
# /var/cache/elbe/sources
# use walk_files to scan it, and add all dsc files.
#
# we can not just copy the source repo, like we do
# with the bin repo, because the src cdrom can be split
# into multiple cdroms
initvm_repo = Filesystem('/var/cache/elbe/sources')
for _ , dsc_real in initvm_repo.walk_files():
if not dsc_real.endswith('.dsc'):
continue
repo.include_init_dsc(dsc_real, 'initvm')
repo.finalize()
if xml is not None:
options = get_iso_options(xml)
for arch_vol in xml.all('src-cdrom/archive'):
volume_attr = arch_vol.et.get('volume')
if volume_attr == 'all':
volume_list = repo.volume_indexes
else:
volume_list = [int(v) for v in volume_attr.split(",")]
for volume_number in volume_list:
with archive_tmpfile(arch_vol.text(".")) as fp:
if volume_number in repo.volume_indexes:
do('tar xvfj "%s" -h -C "%s"' % (fp.name,
repo.get_volume_fs(volume_number).path))
else:
logging.warning("The src-cdrom archive's volume value "
"is not contained in the actual volumes")
else:
options = ""
return repo.buildiso(os.path.join(target, "src-cdrom.iso"), options=options)
def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
# pylint: disable=too-many-statements
rfs.mkdir_p('/var/cache/elbe/binaries/added')
rfs.mkdir_p('/var/cache/elbe/binaries/main')
if xml is not None:
mirror = xml.get_primary_mirror(rfs.fname("cdrom"))
else:
mirror = 'http://ftp.de.debian.org/debian'
repo_path = os.path.join(target, "binrepo")
target_repo_path = os.path.join(repo_path, 'targetrepo')
# initvm repo has been built upon initvm creation
# just copy it. the repo __init__() afterwards will
# not touch the repo config, nor generate a new key.
try:
do('cp -av /var/cache/elbe/initvm-bin-repo "%s"' % repo_path)
except CommandError:
# When /var/cache/elbe/initvm-bin-repo has not been created
# (because the initvm install was an old version or somthing,
# log an error, and continue with an empty directory.
logging.exception("/var/cache/elbe/initvm-bin-repo does not exist\n"
"The generated CDROM will not contain initvm pkgs\n"
"This happened because the initvm was probably\n"
"generated with --skip-build-bin")
do('mkdir -p "%s"' % repo_path)
repo = CdromInitRepo(init_codename, repo_path, mirror)
target_repo = CdromBinRepo(arch, codename, None,
target_repo_path, mirror)
if xml is not None:
cache = get_rpcaptcache(rfs, arch)
for p in xml.node("debootstrappkgs"):
pkg = XMLPackage(p, arch)
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
deb = cache.download_binary(pkg.name,
'/var/cache/elbe/binaries/main',
pkg.installed_version)
target_repo.includedeb(deb, 'main')
except ValueError:
logging.error("No package '%s'", pkg_id)
except FetchError:
logging.error("Package '%s' could not be downloaded", pkg_id)
except TypeError:
logging.error("Package '%s' missing name or version", pkg_id)
cache = get_rpcaptcache(rfs, arch)
pkglist = cache.get_installed_pkgs()
for pkg in pkglist:
pkg_id = "%s-%s" % (pkg.name, pkg.installed_version)
try:
deb = cache.download_binary(pkg.name,
'/var/cache/elbe/binaries/added',
pkg.installed_version)
target_repo.includedeb(deb, 'added', pkg.name, True)
except KeyError as ke:
logging.error(str(ke))
except ValueError:
logging.error("No package '%s'", pkg_id)
except FetchError:
logging.error("Package '%s' could not be downloaded", pkg_id)
except TypeError:
logging.error("Package '%s' missing name or version", pkg_id)
target_repo.finalize()
# Mark the binary repo with the necessary Files
# to make the installer accept this as a CDRom
repo_fs = Filesystem(repo_path)
repo_fs.mkdir_p(".disk")
repo_fs.write_file(".disk/base_installable", 0o644, "main\n")
repo_fs.write_file(".disk/base_components", 0o644, "main\n")
repo_fs.write_file(".disk/cd_type", 0o644, "not_complete\n")
repo_fs.write_file(".disk/info", 0o644, "elbe inst cdrom - full cd\n")
repo_fs.symlink(".", "debian", allow_exists=True)
repo_fs.write_file("md5sum.txt", 0o644, "")
# write source xml onto cdrom
xml.xml.write(repo_fs.fname('source.xml'))
# copy initvm-cdrom.gz and vmlinuz
copyfile('/var/cache/elbe/installer/initrd-cdrom.gz',
repo_fs.fname('initrd-cdrom.gz'))
copyfile('/var/cache/elbe/installer/vmlinuz',
repo_fs.fname('vmlinuz'))
target_repo_fs = Filesystem(target_repo_path)
target_repo_fs.write_file(".aptignr", 0o644, "")
return repo.buildiso(os.path.join(target, "bin-cdrom.iso"))
def mk_binary_cdrom(rfs, arch, codename, init_codename, xml, target, log, cdrom_size=CDROM_SIZE):
rfs.mkdir_p( '/var/cache/elbe/binaries/added' )
rfs.mkdir_p( '/var/cache/elbe/binaries/main' )
hostfs.mkdir_p( '/var/cache/elbe/binaries/main' )
if not xml is None:
mirror = xml.get_primary_mirror (rfs.fname("cdrom"))
else:
mirror='http://ftp.debian.org/debian'
repo_path = os.path.join (target, "binrepo")
repo = CdromBinRepo (arch, codename, init_codename,
repo_path, log, cdrom_size, mirror)
if not xml is None:
pkglist = get_initvm_pkglist()
cache = Cache ()
cache.open ()
for pkg in pkglist:
try:
p = cache [pkg.name]
if pkg.name == 'elbe-bootstrap':
pkgver = p.versions [0]
else:
pkgver = p.installed
deb = pkgver.fetch_binary ('/var/cache/elbe/binaries/main',
ElbeAcquireProgress (cb=None) )
repo.include_init_deb(deb, 'main')
except ValueError as ve:
log.printo( "No Package " + pkg.name + "-" + str(pkg.installed_version) )
except FetchError as fe:
log.printo( "Package " + pkg.name + "-" + pkgver.version + " could not be downloaded" )
except TypeError as te:
log.printo( "Package " + pkg.name + "-" + str(pkg.installed_version) + " missing name or version" )
cache = get_rpcaptcache( rfs, "aptcache.log", arch )
for p in xml.node("debootstrappkgs"):
pkg = XMLPackage(p, arch)
try:
deb = cache.download_binary( pkg.name,
'/var/cache/elbe/binaries/main',
pkg.installed_version )
repo.includedeb(deb, 'main')
except ValueError as ve:
log.printo( "No Package " + pkg.name + "-" + pkg.installed_version )
except FetchError as fe:
log.printo( "Package " + pkg.name + "-" + pkg.installed_version + " could not be downloaded" )
except TypeError as te:
log.printo( "Package " + pkg.name + "-" + pkg.installed_version + " missing name or version" )
cache = get_rpcaptcache( rfs, "aptcache.log", arch )
pkglist = cache.get_installed_pkgs ()
for pkg in pkglist:
try:
deb = cache.download_binary( pkg.name,
'/var/cache/elbe/binaries/added',
pkg.installed_version )
repo.includedeb(deb, 'added')
except KeyError as ke:
log.printo( str (ke) )
except ValueError as ve:
log.printo( "No Package " + pkg.name + "-" + pkg.installed_version )
except FetchError as fe:
log.printo( "Package " + pkg.name + "-" + str (pkg.installed_version) + " could not be downloaded" )
except TypeError as te:
log.printo( "Package " + pkg.name + "-" + pkg.installed_version + " missing name or version" )
# Mark the binary repo with the necessary Files
# to make the installer accept this as a CDRom
repo_fs = Filesystem( repo_path )
repo_fs.mkdir_p (".disk")
repo_fs.write_file (".disk/base_installable", 0644, "main\n")
repo_fs.write_file (".disk/base_components", 0644, "main\n")
repo_fs.write_file (".disk/cd_type", 0644, "not_complete\n")
repo_fs.write_file (".disk/info", 0644, "elbe inst cdrom - full cd\n")
repo_fs.symlink (".", "debian", allow_exists=True)
repo_fs.write_file ("md5sum.txt", 0644, "")
# write source xml onto cdrom
xml.xml.write (repo_fs.fname ('source.xml'))
return repo.buildiso( os.path.join( target, "bin-cdrom.iso" ) )
def mk_binary_cdrom(rfs,
arch,
codename,
init_codename,
xml,
target,
log,
cdrom_size=CDROM_SIZE):
rfs.mkdir_p('/var/cache/elbe/binaries/added')
rfs.mkdir_p('/var/cache/elbe/binaries/main')
hostfs.mkdir_p('/var/cache/elbe/binaries/main')
if not xml is None:
mirror = xml.get_primary_mirror(rfs.fname("cdrom"))
else:
mirror = 'http://ftp.debian.org/debian'
repo_path = os.path.join(target, "binrepo")
repo = CdromBinRepo(arch, codename, init_codename, repo_path, log,
cdrom_size, mirror)
if not xml is None:
pkglist = get_initvm_pkglist()
cache = Cache()
cache.open()
for pkg in pkglist:
try:
p = cache[pkg.name]
if pkg.name == 'elbe-bootstrap':
pkgver = p.versions[0]
else:
pkgver = p.installed
deb = pkgver.fetch_binary('/var/cache/elbe/binaries/main',
ElbeAcquireProgress(cb=None))
repo.include_init_deb(deb, 'main')
except ValueError as ve:
log.printo("No Package " + pkg.name + "-" +
str(pkg.installed_version))
except FetchError as fe:
log.printo("Package " + pkg.name + "-" + pkgver.version +
" could not be downloaded")
except TypeError as te:
log.printo("Package " + pkg.name + "-" +
str(pkg.installed_version) +
" missing name or version")
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
for p in xml.node("debootstrappkgs"):
pkg = XMLPackage(p, arch)
try:
deb = cache.download_binary(pkg.name,
'/var/cache/elbe/binaries/main',
pkg.installed_version)
repo.includedeb(deb, 'main')
except ValueError as ve:
log.printo("No Package " + pkg.name + "-" +
pkg.installed_version)
except FetchError as fe:
log.printo("Package " + pkg.name + "-" +
pkg.installed_version + " could not be downloaded")
except TypeError as te:
log.printo("Package " + pkg.name + "-" +
pkg.installed_version + " missing name or version")
cache = get_rpcaptcache(rfs, "aptcache.log", arch)
pkglist = cache.get_installed_pkgs()
for pkg in pkglist:
try:
deb = cache.download_binary(pkg.name,
'/var/cache/elbe/binaries/added',
pkg.installed_version)
repo.includedeb(deb, 'added')
except KeyError as ke:
log.printo(str(ke))
except ValueError as ve:
log.printo("No Package " + pkg.name + "-" + pkg.installed_version)
except FetchError as fe:
log.printo("Package " + pkg.name + "-" +
str(pkg.installed_version) + " could not be downloaded")
except TypeError as te:
log.printo("Package " + pkg.name + "-" + pkg.installed_version +
" missing name or version")
# Mark the binary repo with the necessary Files
# to make the installer accept this as a CDRom
repo_fs = Filesystem(repo_path)
repo_fs.mkdir_p(".disk")
repo_fs.write_file(".disk/base_installable", 0644, "main\n")
repo_fs.write_file(".disk/base_components", 0644, "main\n")
repo_fs.write_file(".disk/cd_type", 0644, "not_complete\n")
repo_fs.write_file(".disk/info", 0644, "elbe inst cdrom - full cd\n")
repo_fs.symlink(".", "debian", allow_exists=True)
repo_fs.write_file("md5sum.txt", 0644, "")
# write source xml onto cdrom
xml.xml.write(repo_fs.fname('source.xml'))
return repo.buildiso(os.path.join(target, "bin-cdrom.iso"))
def __init__(self, path, clean=False):
Filesystem.__init__(self,path,clean)
def mk_source_cdrom(components, codename,
init_codename, target,
cdrom_size=CDROM_SIZE, xml=None,
mirror='http://ftp.de.debian.org/debian'):
# pylint: disable=too-many-arguments
# pylint: disable=too-many-locals
# pylint: disable=too-many-branches
hostfs.mkdir_p('/var/cache/elbe/sources')
forbiddenPackages = []
if xml is not None and xml.has('target/pkg-list'):
for i in xml.node('target/pkg-list'):
try:
if i.tag == 'pkg' and i.et.attrib['on_src_cd'] == 'False':
forbiddenPackages.append(i.text('.').strip())
except KeyError:
pass
repos = {}
for component in components.keys():
rfs, cache, pkg_lst = components[component]
logging.info("Adding %s component", component)
rfs.mkdir_p("/var/cache/elbe/sources")
repo = CdromSrcRepo(codename, init_codename,
os.path.join(target, "srcrepo-%s" % component),
cdrom_size, mirror)
repos[component] = repo
for pkg, version in pkg_lst:
add_source_pkg(repo, component,
cache, pkg, version,
forbiddenPackages)
# elbe fetch_initvm_pkgs has downloaded all sources to
# /var/cache/elbe/sources
# use walk_files to scan it, and add all dsc files.
#
# we can not just copy the source repo, like we do
# with the bin repo, because the src cdrom can be split
# into multiple cdroms
initvm_repo = Filesystem('/var/cache/elbe/sources')
for _ , dsc_real in initvm_repo.walk_files():
if not dsc_real.endswith('.dsc'):
continue
repos["main"].include_init_dsc(dsc_real, "initvm")
for repo in repos.values():
repo.finalize()
if xml is not None:
options = get_iso_options(xml)
for arch_vol in xml.all('src-cdrom/archive'):
volume_attr = arch_vol.et.get('volume')
for repo in repos.values():
if volume_attr == 'all':
volume_list = repo.volume_indexes
else:
volume_list = [int(v) for v in volume_attr.split(",")]
for volume_number in volume_list:
with archive_tmpfile(arch_vol.text(".")) as fp:
if volume_number in repo.volume_indexes:
do('tar xvfj "%s" -h -C "%s"' % (fp.name,
repo.get_volume_fs(volume_number).path))
else:
logging.warning("The src-cdrom archive's volume value "
"is not contained in the actual volumes")
else:
options = ""
return [(repo.buildiso(os.path.join(target, "src-cdrom-%s.iso" % component),
options=options)) for component, repo in repos.items()]
