class RuntimeBuilder(object):
'''Builds the anaconda runtime image.'''
def __init__(self,
product,
arch,
dbo,
templatedir=None,
installpkgs=None,
add_templates=None,
add_template_vars=None):
root = dbo.conf.installroot
# use a copy of product so we can modify it locally
product = product.copy()
product.name = product.name.lower()
self.vars = DataHolder(arch=arch,
product=product,
dbo=dbo,
root=root,
basearch=arch.basearch,
libdir=arch.libdir)
self.dbo = dbo
self._runner = LoraxTemplateRunner(inroot=root,
outroot=root,
dbo=dbo,
templatedir=templatedir)
self.add_templates = add_templates or []
self.add_template_vars = add_template_vars or {}
self._installpkgs = installpkgs or []
self._runner.defaults = self.vars
self.dbo.reset()
def _install_branding(self):
release = None
q = self.dbo.sack.query()
a = q.available()
for pkg in a.filter(provides='/etc/system-release'):
if pkg.name.startswith('generic'):
continue
else:
release = pkg.name
break
if not release:
logger.error('could not get the release')
return
# release
logger.info('got release: %s', release)
self._runner.installpkg(release)
# logos
release, _suffix = release.split('-', 1)
self._runner.installpkg('%s-logos' % release)
def install(self):
'''Install packages and do initial setup with runtime-install.tmpl'''
self._install_branding()
if len(self._installpkgs) > 0:
self._runner.installpkg(*self._installpkgs)
self._runner.run("runtime-install.tmpl")
for tmpl in self.add_templates:
self._runner.run(tmpl, **self.add_template_vars)
def writepkglists(self, pkglistdir):
'''debugging data: write out lists of package contents'''
if not os.path.isdir(pkglistdir):
os.makedirs(pkglistdir)
q = self.dbo.sack.query()
for pkgobj in q.installed():
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
for fname in pkgobj.files:
fobj.write("{0}\n".format(fname))
def postinstall(self):
'''Do some post-install setup work with runtime-postinstall.tmpl'''
# copy configdir into runtime root beforehand
configdir = joinpaths(self._runner.templatedir, "config_files")
configdir_path = "tmp/config_files"
fullpath = joinpaths(self.vars.root, configdir_path)
if os.path.exists(fullpath):
remove(fullpath)
copytree(configdir, fullpath)
self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)
def cleanup(self):
'''Remove unneeded packages and files with runtime-cleanup.tmpl'''
self._runner.run("runtime-cleanup.tmpl")
def verify(self):
'''Ensure that contents of the installroot can run'''
status = True
ELF_MAGIC = b'\x7fELF'
# Iterate over all files in /usr/bin and /usr/sbin
# For ELF files, gather them into a list and we'll check them all at
# the end. For files with a #!, check them as we go
elf_files = []
usr_bin = Path(self.vars.root + '/usr/bin')
usr_sbin = Path(self.vars.root + '/usr/sbin')
for path in (str(x) for x in itertools.chain(usr_bin.iterdir(), usr_sbin.iterdir()) \
if x.is_file()):
with open(path, "rb") as f:
magic = f.read(4)
if magic == ELF_MAGIC:
# Save the path, minus the chroot prefix
elf_files.append(path[len(self.vars.root):])
elif magic[:2] == b'#!':
# Reopen the file as text and read the first line.
# Open as latin-1 so that stray 8-bit characters don't make
# things blow up. We only really care about ASCII parts.
with open(path, "rt", encoding="latin-1") as f_text:
# Remove the #!, split on space, and take the first part
shabang = f_text.readline()[2:].split()[0]
# Does the path exist?
if not os.path.exists(self.vars.root + shabang):
logger.error('%s, needed by %s, does not exist',
shabang, path)
status = False
# Now, run ldd on all the ELF files
# Just run ldd once on everything so it isn't logged a million times.
# At least one thing in the list isn't going to be a dynamic executable,
# so use execWithCapture to ignore the exit code.
filename = ''
for line in execWithCapture('ldd',
elf_files,
root=self.vars.root,
log_output=False,
filter_stderr=True).split('\n'):
if line and not line[0].isspace():
# New filename header, strip the : at the end and save
filename = line[:-1]
elif 'not found' in line:
logger.error('%s, needed by %s, not found',
line.split()[0], filename)
status = False
return status
def writepkgsizes(self, pkgsizefile):
'''debugging data: write a big list of pkg sizes'''
fobj = open(pkgsizefile, "w")
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
q = self.dbo.sack.query()
for p in sorted(q.installed()):
pkgsize = sum(
getsize(joinpaths(self.vars.root, f)) for f in p.files)
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
def generate_module_data(self):
root = self.vars.root
moddir = joinpaths(root, "lib/modules/")
for kver in os.listdir(moddir):
ksyms = joinpaths(root, "boot/System.map-%s" % kver)
logger.info("doing depmod and module-info for %s", kver)
runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kver])
generate_module_info(moddir + kver, outfile=moddir + "module-info")
def create_runtime(self,
outfile="/var/tmp/squashfs.img",
compression="xz",
compressargs=None,
size=2):
# make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
compressargs = compressargs or []
workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
os.makedirs(joinpaths(workdir, "LiveOS"))
imgutils.mkrootfsimg(self.vars.root,
joinpaths(workdir, "LiveOS/rootfs.img"),
"Anaconda",
size=size)
# squash the live rootfs and clean up workdir
imgutils.mksquashfs(workdir, outfile, compression, compressargs)
remove(workdir)
def finished(self):
""" Done using RuntimeBuilder
Close the dnf base object
"""
self.dbo.close()
class RuntimeBuilder(object):
'''Builds the anaconda runtime image.'''
def __init__(self, product, arch, dbo, templatedir=None,
installpkgs=None, excludepkgs=None,
add_templates=None,
add_template_vars=None,
skip_branding=False):
root = dbo.conf.installroot
self.dbo = dbo
self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
dbo=dbo, templatedir=templatedir)
self.add_templates = add_templates or []
self.add_template_vars = add_template_vars or {}
self._installpkgs = installpkgs or []
self._excludepkgs = excludepkgs or []
self.dbo.reset()
# use a copy of product so we can modify it locally
product = product.copy()
product.name = product.name.lower()
self._branding = self.get_branding(skip_branding, product)
self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
basearch=arch.basearch, libdir=arch.libdir,
branding=self._branding)
self._runner.defaults = self.vars
def get_branding(self, skip, product):
"""Select the branding from the available 'system-release' packages
The *best* way to control this is to have a single package in the repo provide 'system-release'
When there are more than 1 package it will:
- Make a list of the available packages
- If variant is set look for a package ending with lower(variant) and use that
- If there are one or more non-generic packages, use the first one after sorting
Returns the package names of the system-release and release logos package
"""
if skip:
return DataHolder(release=None, logos=None)
release = None
q = self.dbo.sack.query()
a = q.available()
pkgs = sorted([p.name for p in a.filter(provides='system-release')
if not p.name.startswith("generic")])
if not pkgs:
logger.error("No system-release packages found, could not get the release")
return DataHolder(release=None, logos=None)
logger.debug("system-release packages: %s", pkgs)
if product.variant:
variant = [p for p in pkgs if p.endswith("-"+product.variant.lower())]
if variant:
release = variant[0]
if not release:
release = pkgs[0]
# release
logger.info('got release: %s', release)
# logos uses the basename from release (fedora, redhat, centos, ...)
logos, _suffix = release.split('-', 1)
return DataHolder(release=release, logos=logos+"-logos")
def install(self):
'''Install packages and do initial setup with runtime-install.tmpl'''
if self._branding.release:
self._runner.installpkg(self._branding.release)
if self._branding.logos:
self._runner.installpkg(self._branding.logos)
if len(self._installpkgs) > 0:
self._runner.installpkg(*self._installpkgs)
if len(self._excludepkgs) > 0:
self._runner.removepkg(*self._excludepkgs)
self._runner.run("runtime-install.tmpl")
for tmpl in self.add_templates:
self._runner.run(tmpl, **self.add_template_vars)
def writepkglists(self, pkglistdir):
'''debugging data: write out lists of package contents'''
if not os.path.isdir(pkglistdir):
os.makedirs(pkglistdir)
q = self.dbo.sack.query()
for pkgobj in q.installed():
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
for fname in pkgobj.files:
fobj.write("{0}\n".format(fname))
def postinstall(self):
'''Do some post-install setup work with runtime-postinstall.tmpl'''
# copy configdir into runtime root beforehand
configdir = joinpaths(self._runner.templatedir,"config_files")
configdir_path = "tmp/config_files"
fullpath = joinpaths(self.vars.root, configdir_path)
if os.path.exists(fullpath):
remove(fullpath)
copytree(configdir, fullpath)
self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)
def cleanup(self):
'''Remove unneeded packages and files with runtime-cleanup.tmpl'''
self._runner.run("runtime-cleanup.tmpl")
def verify(self):
'''Ensure that contents of the installroot can run'''
status = True
ELF_MAGIC = b'\x7fELF'
# Iterate over all files in /usr/bin and /usr/sbin
# For ELF files, gather them into a list and we'll check them all at
# the end. For files with a #!, check them as we go
elf_files = []
usr_bin = Path(self.vars.root + '/usr/bin')
usr_sbin = Path(self.vars.root + '/usr/sbin')
for path in (str(x) for x in itertools.chain(usr_bin.iterdir(), usr_sbin.iterdir()) \
if x.is_file()):
with open(path, "rb") as f:
magic = f.read(4)
if magic == ELF_MAGIC:
# Save the path, minus the chroot prefix
elf_files.append(path[len(self.vars.root):])
elif magic[:2] == b'#!':
# Reopen the file as text and read the first line.
# Open as latin-1 so that stray 8-bit characters don't make
# things blow up. We only really care about ASCII parts.
with open(path, "rt", encoding="latin-1") as f_text:
# Remove the #!, split on space, and take the first part
shabang = f_text.readline()[2:].split()[0]
# Does the path exist?
if not os.path.exists(self.vars.root + shabang):
logger.error('%s, needed by %s, does not exist', shabang, path)
status = False
# Now, run ldd on all the ELF files
# Just run ldd once on everything so it isn't logged a million times.
# At least one thing in the list isn't going to be a dynamic executable,
# so use execWithCapture to ignore the exit code.
filename = ''
for line in execWithCapture('ldd', elf_files, root=self.vars.root,
log_output=False, filter_stderr=True).split('\n'):
if line and not line[0].isspace():
# New filename header, strip the : at the end and save
filename = line[:-1]
elif 'not found' in line:
logger.error('%s, needed by %s, not found', line.split()[0], filename)
status = False
return status
def writepkgsizes(self, pkgsizefile):
'''debugging data: write a big list of pkg sizes'''
fobj = open(pkgsizefile, "w")
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
q = self.dbo.sack.query()
for p in sorted(q.installed()):
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
def generate_module_data(self):
root = self.vars.root
moddir = joinpaths(root, "lib/modules/")
for kernel in findkernels(root=root):
ksyms = joinpaths(root, "boot/System.map-%s" % kernel.version)
logger.info("doing depmod and module-info for %s", kernel.version)
runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kernel.version])
generate_module_info(moddir+kernel.version, outfile=moddir+"module-info")
def create_squashfs_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
"""Create a plain squashfs runtime"""
compressargs = compressargs or []
os.makedirs(os.path.dirname(outfile))
# squash the rootfs
return imgutils.mksquashfs(self.vars.root, outfile, compression, compressargs)
def create_ext4_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
"""Create a squashfs compressed ext4 runtime"""
# make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
compressargs = compressargs or []
workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
os.makedirs(joinpaths(workdir, "LiveOS"))
# Catch problems with the rootfs being too small and clearly log them
try:
imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
"Anaconda", size=size)
except CalledProcessError as e:
if e.stdout and "No space left on device" in e.stdout:
logger.error("The rootfs ran out of space with size=%d", size)
raise
# squash the live rootfs and clean up workdir
rc = imgutils.mksquashfs(workdir, outfile, compression, compressargs)
remove(workdir)
return rc
def finished(self):
""" Done using RuntimeBuilder
Close the dnf base object
"""
self.dbo.close()
class RuntimeBuilder(object):
'''Builds the anaconda runtime image.'''
def __init__(self, product, arch, dbo, templatedir=None,
installpkgs=None,
add_templates=None,
add_template_vars=None):
root = dbo.conf.installroot
# use a copy of product so we can modify it locally
product = product.copy()
product.name = product.name.lower()
self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
basearch=arch.basearch, libdir=arch.libdir)
self.dbo = dbo
self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
dbo=dbo, templatedir=templatedir)
self.add_templates = add_templates or []
self.add_template_vars = add_template_vars or {}
self._installpkgs = installpkgs or []
self._runner.defaults = self.vars
self.dbo.reset()
def _install_branding(self):
release = None
q = self.dbo.sack.query()
a = q.available()
for pkg in a.filter(provides='/etc/system-release'):
if pkg.name.startswith('generic'):
continue
else:
release = pkg.name
break
if not release:
logger.error('could not get the release')
return
# release
logger.info('got release: %s', release)
self._runner.installpkg(release)
# logos
release, _suffix = release.split('-', 1)
self._runner.installpkg('%s-logos' % release)
def install(self):
'''Install packages and do initial setup with runtime-install.tmpl'''
self._install_branding()
if len(self._installpkgs) > 0:
self._runner.installpkg(*self._installpkgs)
self._runner.run("runtime-install.tmpl")
for tmpl in self.add_templates:
self._runner.run(tmpl, **self.add_template_vars)
def writepkglists(self, pkglistdir):
'''debugging data: write out lists of package contents'''
if not os.path.isdir(pkglistdir):
os.makedirs(pkglistdir)
q = self.dbo.sack.query()
for pkgobj in q.installed():
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
for fname in pkgobj.files:
fobj.write("{0}\n".format(fname))
def postinstall(self):
'''Do some post-install setup work with runtime-postinstall.tmpl'''
# copy configdir into runtime root beforehand
configdir = joinpaths(self._runner.templatedir,"config_files")
configdir_path = "tmp/config_files"
fullpath = joinpaths(self.vars.root, configdir_path)
if os.path.exists(fullpath):
remove(fullpath)
copytree(configdir, fullpath)
self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)
def cleanup(self):
'''Remove unneeded packages and files with runtime-cleanup.tmpl'''
self._runner.run("runtime-cleanup.tmpl")
def verify(self):
'''Ensure that contents of the installroot can run'''
status = True
ELF_MAGIC = b'\x7fELF'
# Iterate over all files in /usr/bin and /usr/sbin
# For ELF files, gather them into a list and we'll check them all at
# the end. For files with a #!, check them as we go
elf_files = []
usr_bin = Path(self.vars.root + '/usr/bin')
usr_sbin = Path(self.vars.root + '/usr/sbin')
for path in (str(x) for x in itertools.chain(usr_bin.iterdir(), usr_sbin.iterdir()) \
if x.is_file()):
with open(path, "rb") as f:
magic = f.read(4)
if magic == ELF_MAGIC:
# Save the path, minus the chroot prefix
elf_files.append(path[len(self.vars.root):])
elif magic[:2] == b'#!':
# Reopen the file as text and read the first line.
# Open as latin-1 so that stray 8-bit characters don't make
# things blow up. We only really care about ASCII parts.
with open(path, "rt", encoding="latin-1") as f_text:
# Remove the #!, split on space, and take the first part
shabang = f_text.readline()[2:].split()[0]
# Does the path exist?
if not os.path.exists(self.vars.root + shabang):
logger.error('%s, needed by %s, does not exist', shabang, path)
status = False
# Now, run ldd on all the ELF files
# Just run ldd once on everything so it isn't logged a million times.
# At least one thing in the list isn't going to be a dynamic executable,
# so use execWithCapture to ignore the exit code.
filename = ''
for line in execWithCapture('ldd', elf_files, root=self.vars.root,
log_output=False, filter_stderr=True).split('\n'):
if line and not line[0].isspace():
# New filename header, strip the : at the end and save
filename = line[:-1]
elif 'not found' in line:
logger.error('%s, needed by %s, not found', line.split()[0], filename)
status = False
return status
def writepkgsizes(self, pkgsizefile):
'''debugging data: write a big list of pkg sizes'''
fobj = open(pkgsizefile, "w")
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
q = self.dbo.sack.query()
for p in sorted(q.installed()):
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
def generate_module_data(self):
root = self.vars.root
moddir = joinpaths(root, "lib/modules/")
for kver in os.listdir(moddir):
ksyms = joinpaths(root, "boot/System.map-%s" % kver)
logger.info("doing depmod and module-info for %s", kver)
runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kver])
generate_module_info(moddir+kver, outfile=moddir+"module-info")
def create_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
# make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
compressargs = compressargs or []
workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
os.makedirs(joinpaths(workdir, "LiveOS"))
imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
"Anaconda", size=size)
# squash the live rootfs and clean up workdir
imgutils.mksquashfs(workdir, outfile, compression, compressargs)
remove(workdir)
def finished(self):
""" Done using RuntimeBuilder
Close the dnf base object
"""
self.dbo.close()
class RuntimeBuilder(object):
'''Builds the anaconda runtime image.'''
def __init__(self, product, arch, dbo, templatedir=None,
installpkgs=None,
add_templates=None,
add_template_vars=None):
root = dbo.conf.installroot
# use a copy of product so we can modify it locally
product = product.copy()
product.name = product.name.lower()
self.vars = DataHolder(arch=arch, product=product, dbo=dbo, root=root,
basearch=arch.basearch, libdir=arch.libdir)
self.dbo = dbo
self._runner = LoraxTemplateRunner(inroot=root, outroot=root,
dbo=dbo, templatedir=templatedir)
self.add_templates = add_templates or []
self.add_template_vars = add_template_vars or {}
self._installpkgs = installpkgs or []
self._runner.defaults = self.vars
self.dbo.reset()
def _install_branding(self):
release = None
q = self.dbo.sack.query()
a = q.available()
for pkg in a.filter(provides='/etc/system-release'):
if pkg.name.startswith('generic'):
continue
else:
release = pkg.name
break
if not release:
logger.error('could not get the release')
return
# release
logger.info('got release: %s', release)
self._runner.installpkg(release)
# logos
release, _suffix = release.split('-', 1)
self._runner.installpkg('%s-logos' % release)
def install(self):
'''Install packages and do initial setup with runtime-install.tmpl'''
self._install_branding()
if len(self._installpkgs) > 0:
self._runner.installpkg(*self._installpkgs)
self._runner.run("runtime-install.tmpl")
for tmpl in self.add_templates:
self._runner.run(tmpl, **self.add_template_vars)
def writepkglists(self, pkglistdir):
'''debugging data: write out lists of package contents'''
if not os.path.isdir(pkglistdir):
os.makedirs(pkglistdir)
q = self.dbo.sack.query()
for pkgobj in q.installed():
with open(joinpaths(pkglistdir, pkgobj.name), "w") as fobj:
for fname in pkgobj.files:
fobj.write("{0}\n".format(fname))
def postinstall(self):
'''Do some post-install setup work with runtime-postinstall.tmpl'''
# copy configdir into runtime root beforehand
configdir = joinpaths(self._runner.templatedir,"config_files")
configdir_path = "tmp/config_files"
fullpath = joinpaths(self.vars.root, configdir_path)
if os.path.exists(fullpath):
remove(fullpath)
copytree(configdir, fullpath)
self._runner.run("runtime-postinstall.tmpl", configdir=configdir_path)
def cleanup(self):
'''Remove unneeded packages and files with runtime-cleanup.tmpl'''
self._runner.run("runtime-cleanup.tmpl")
def writepkgsizes(self, pkgsizefile):
'''debugging data: write a big list of pkg sizes'''
fobj = open(pkgsizefile, "w")
getsize = lambda f: os.lstat(f).st_size if os.path.exists(f) else 0
q = self.dbo.sack.query()
for p in sorted(q.installed()):
pkgsize = sum(getsize(joinpaths(self.vars.root,f)) for f in p.files)
fobj.write("{0.name}.{0.arch}: {1}\n".format(p, pkgsize))
def generate_module_data(self):
root = self.vars.root
moddir = joinpaths(root, "lib/modules/")
for kver in os.listdir(moddir):
ksyms = joinpaths(root, "boot/System.map-%s" % kver)
logger.info("doing depmod and module-info for %s", kver)
runcmd(["depmod", "-a", "-F", ksyms, "-b", root, kver])
generate_module_info(moddir+kver, outfile=moddir+"module-info")
def create_runtime(self, outfile="/var/tmp/squashfs.img", compression="xz", compressargs=None, size=2):
# make live rootfs image - must be named "LiveOS/rootfs.img" for dracut
compressargs = compressargs or []
workdir = joinpaths(os.path.dirname(outfile), "runtime-workdir")
os.makedirs(joinpaths(workdir, "LiveOS"))
imgutils.mkrootfsimg(self.vars.root, joinpaths(workdir, "LiveOS/rootfs.img"),
"Anaconda", size=size)
# squash the live rootfs and clean up workdir
imgutils.mksquashfs(workdir, outfile, compression, compressargs)
remove(workdir)
def finished(self):
""" Done using RuntimeBuilder
Close the dnf base object
"""
self.dbo.close()