def findkernels(root="/", kdir="boot"):
# To find possible flavors, awk '/BuildKernel/ { print $4 }' kernel.spec
flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen', 'lpae', 'tegra')
kre = re.compile(r"vmlinuz-(?P<version>.+?\.(?P<arch>[a-z0-9_]+)"
r"(\.(?P<flavor>{0}))?)$".format("|".join(flavors)))
kernels = []
bootfiles = os.listdir(joinpaths(root, kdir))
for f in bootfiles:
match = kre.match(f)
if match:
kernel = DataHolder(path=joinpaths(kdir, f))
kernel.update(match.groupdict()) # sets version, arch, flavor
kernels.append(kernel)
# look for associated initrd/initramfs/etc.
for kernel in kernels:
for f in bootfiles:
if f.endswith('-' + kernel.version + '.img'):
imgtype, rest = f.split('-', 1)
# special backwards-compat case
if imgtype == 'initramfs':
imgtype = 'initrd'
kernel[imgtype] = DataHolder(path=joinpaths(kdir, f))
logger.debug("kernels=%s" % kernels)
return kernels
def findkernels(root="/", kdir="boot"):
# To find possible flavors, awk '/BuildKernel/ { print $4 }' kernel.spec
flavors = ('debug', 'PAE', 'PAEdebug', 'smp', 'xen', 'lpae', 'tegra')
kre = re.compile(r"vmlinuz-(?P<version>.+?\.(?P<arch>[a-z0-9_]+)"
r"(\.(?P<flavor>{0}))?)$".format("|".join(flavors)))
kernels = []
bootfiles = os.listdir(joinpaths(root, kdir))
for f in bootfiles:
match = kre.match(f)
if match:
kernel = DataHolder(path=joinpaths(kdir, f))
kernel.update(match.groupdict()) # sets version, arch, flavor
kernels.append(kernel)
# look for associated initrd/initramfs/etc.
for kernel in kernels:
for f in bootfiles:
if f.endswith('-'+kernel.version+'.img'):
imgtype, rest = f.split('-',1)
# special backwards-compat case
if imgtype == 'initramfs':
imgtype = 'initrd'
kernel[imgtype] = DataHolder(path=joinpaths(kdir, f))
logger.debug("kernels=%s" % kernels)
return kernels
def __init__(self, inroot, outroot, yum=None, fatalerrors=True,
templatedir=None, defaults={}):
self.inroot = inroot
self.outroot = outroot
self.yum = yum
self.fatalerrors = fatalerrors
self.templatedir = templatedir or "/usr/share/lorax"
# some builtin methods
self.builtins = DataHolder(exists=lambda p: rexists(p, root=inroot),
glob=lambda g: list(rglob(g, root=inroot)))
self.defaults = defaults
self.results = DataHolder(treeinfo=dict()) # just treeinfo for now
def __init__(self,
product,
arch,
yum,
templatedir=None,
add_templates=None,
add_template_vars=None):
root = yum.conf.installroot
# use a copy of product so we can modify it locally
product = product.copy()
product.name = product.name.lower()
self.vars = DataHolder(arch=arch,
product=product,
yum=yum,
root=root,
basearch=arch.basearch,
libdir=arch.libdir)
self.yum = yum
self._runner = LoraxTemplateRunner(inroot=root,
outroot=root,
yum=yum,
templatedir=templatedir)
self.add_templates = add_templates or []
self.add_template_vars = add_template_vars or {}
self._runner.defaults = self.vars
def __init__(self,
product,
arch,
inroot,
outroot,
runtime,
isolabel,
domacboot=False,
doupgrade=True,
templatedir=None):
# NOTE: if you pass an arg named "runtime" to a mako template it'll
# clobber some mako internal variables - hence "runtime_img".
self.vars = DataHolder(arch=arch,
product=product,
runtime_img=runtime,
runtime_base=basename(runtime),
inroot=inroot,
outroot=outroot,
basearch=arch.basearch,
libdir=arch.libdir,
isolabel=isolabel,
udev=udev_escape,
domacboot=domacboot,
doupgrade=doupgrade)
self._runner = LoraxTemplateRunner(inroot,
outroot,
templatedir=templatedir)
self._runner.defaults = self.vars
self.templatedir = templatedir
def __init__(self, inroot, outroot, yum=None, fatalerrors=True,
templatedir=None, defaults={}):
self.inroot = inroot
self.outroot = outroot
self.yum = yum
self.fatalerrors = fatalerrors
self.templatedir = templatedir or "/usr/share/lorax"
# some builtin methods
self.builtins = DataHolder(exists=lambda p: rexists(p, root=inroot),
glob=lambda g: list(rglob(g, root=inroot)))
self.defaults = defaults
self.results = DataHolder(treeinfo=dict()) # just treeinfo for now
class LoraxTemplateRunner(object):
'''
This class parses and executes Lorax templates. Sample usage:
# install a bunch of packages
runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, yum=yum_obj)
runner.run("install-packages.ltmpl")
# modify a runtime dir
runner = LoraxTemplateRunner(inroot=rundir, outroot=newrun)
runner.run("runtime-transmogrify.ltmpl")
NOTES:
* Parsing procedure is roughly:
1. Mako template expansion (on the whole file)
2. For each line of the result,
a. Whitespace splitting (using shlex.split())
b. Brace expansion (using brace_expand())
c. If the first token is the name of a function, call that function
with the rest of the line as arguments
* Parsing and execution are *separate* passes - so you can't use the result
of a command in an %if statement (or any other control statements)!
* Commands that run external programs (systemctl, gconfset) currently use
the *host*'s copy of that program, which may cause problems if there's a
big enough difference between the host and the image you're modifying.
* The commands are not executed under a real chroot, so absolute symlinks
will point *outside* the inroot/outroot. Be careful with symlinks!
ADDING NEW COMMANDS:
* Each template command is just a method of the LoraxTemplateRunner
object - so adding a new command is as easy as adding a new function.
* Each function gets arguments that correspond to the rest of the tokens
on that line (after word splitting and brace expansion)
* Commands should raise exceptions for errors - don't use sys.exit()
'''
def __init__(self, inroot, outroot, yum=None, fatalerrors=True,
templatedir=None, defaults={}):
self.inroot = inroot
self.outroot = outroot
self.yum = yum
self.fatalerrors = fatalerrors
self.templatedir = templatedir or "/usr/share/lorax"
# some builtin methods
self.builtins = DataHolder(exists=lambda p: rexists(p, root=inroot),
glob=lambda g: list(rglob(g, root=inroot)))
self.defaults = defaults
self.results = DataHolder(treeinfo=dict()) # just treeinfo for now
# TODO: set up custom logger with a filter to add line info
def _out(self, path):
return joinpaths(self.outroot, path)
def _in(self, path):
return joinpaths(self.inroot, path)
def _filelist(self, *pkgs):
pkglist = self.yum.doPackageLists(pkgnarrow="installed", patterns=pkgs)
return set([f for pkg in pkglist.installed for f in pkg.filelist+pkg.ghostlist])
def _getsize(self, *files):
return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
def run(self, templatefile, **variables):
for k,v in self.defaults.items() + self.builtins.items():
variables.setdefault(k,v)
logger.debug("parsing %s", templatefile)
self.templatefile = templatefile
t = LoraxTemplate(directories=[self.templatedir])
commands = t.parse(templatefile, variables)
self._run(commands)
def _run(self, parsed_template):
logger.info("running %s", self.templatefile)
for (num, line) in enumerate(parsed_template,1):
logger.debug("template line %i: %s", num, " ".join(line))
skiperror = False
(cmd, args) = (line[0], line[1:])
# Following Makefile convention, if the command is prefixed with
# a dash ('-'), we'll ignore any errors on that line.
if cmd.startswith('-'):
cmd = cmd[1:]
skiperror = True
try:
# grab the method named in cmd and pass it the given arguments
f = getattr(self, cmd, None)
if cmd[0] == '_' or cmd == 'run' or not callable(f):
raise ValueError, "unknown command %s" % cmd
f(*args)
except Exception:
if skiperror:
logger.debug("ignoring error")
continue
logger.error("template command error in %s:", self.templatefile)
logger.error(" %s", " ".join(line))
# format the exception traceback
exclines = traceback.format_exception(*sys.exc_info())
# skip the bit about "ltmpl.py, in _run()" - we know that
exclines.pop(1)
# log the "ErrorType: this is what happened" line
logger.error(" " + exclines[-1].strip())
# and log the entire traceback to the debug log
for line in ''.join(exclines).splitlines():
logger.debug(" " + line)
if self.fatalerrors:
raise
def install(self, srcglob, dest):
'''
install SRC DEST
Copy the given file (or files, if a glob is used) from the input
tree to the given destination in the output tree.
The path to DEST must exist in the output tree.
If DEST is a directory, SRC will be copied into that directory.
If DEST doesn't exist, SRC will be copied to a file with that name,
assuming the rest of the path exists.
This is pretty much like how the 'cp' command works.
Examples:
install usr/share/myconfig/grub.conf /boot
install /usr/share/myconfig/grub.conf.in /boot/grub.conf
'''
for src in rglob(self._in(srcglob), fatal=True):
cpfile(src, self._out(dest))
def mkdir(self, *dirs):
'''
mkdir DIR [DIR ...]
Create the named DIR(s). Will create leading directories as needed.
Example:
mkdir /images
'''
for d in dirs:
d = self._out(d)
if not isdir(d):
os.makedirs(d)
def replace(self, pat, repl, *fileglobs):
'''
replace PATTERN REPLACEMENT FILEGLOB [FILEGLOB ...]
Find-and-replace the given PATTERN (Python-style regex) with the given
REPLACEMENT string for each of the files listed.
Example:
replace @VERSION@ ${product.version} /boot/grub.conf /boot/isolinux.cfg
'''
match = False
for g in fileglobs:
for f in rglob(self._out(g)):
match = True
replace(f, pat, repl)
if not match:
raise IOError, "no files matched %s" % " ".join(fileglobs)
def append(self, filename, data):
'''
append FILE STRING
Append STRING (followed by a newline character) to FILE.
Python character escape sequences ('\\n', '\\t', etc.) will be
converted to the appropriate characters.
Examples:
append /etc/depmod.d/dd.conf "search updates built-in"
append /etc/resolv.conf ""
'''
with open(self._out(filename), "a") as fobj:
fobj.write(data.decode('string_escape')+"\n")
def treeinfo(self, section, key, *valuetoks):
'''
treeinfo SECTION KEY ARG [ARG ...]
Add an item to the treeinfo data store.
The given SECTION will have a new item added where
KEY = ARG ARG ...
Example:
treeinfo images-${kernel.arch} boot.iso images/boot.iso
'''
if section not in self.results.treeinfo:
self.results.treeinfo[section] = dict()
self.results.treeinfo[section][key] = " ".join(valuetoks)
def installkernel(self, section, src, dest):
'''
installkernel SECTION SRC DEST
Install the kernel from SRC in the input tree to DEST in the output
tree, and then add an item to the treeinfo data store, in the named
SECTION, where "kernel" = DEST.
Equivalent to:
install SRC DEST
treeinfo SECTION kernel DEST
'''
self.install(src, dest)
self.treeinfo(section, "kernel", dest)
def installinitrd(self, section, src, dest):
'''
installinitrd SECTION SRC DEST
Same as installkernel, but for "initrd".
'''
self.install(src, dest)
self.chmod(dest, '644')
self.treeinfo(section, "initrd", dest)
def installupgradeinitrd(self, section, src, dest):
'''
installupgradeinitrd SECTION SRC DEST
Same as installkernel, but for "upgrade".
'''
self.install(src, dest)
self.chmod(dest, '644')
self.treeinfo(section, "upgrade", dest)
def hardlink(self, src, dest):
'''
hardlink SRC DEST
Create a hardlink at DEST which is linked to SRC.
'''
if isdir(self._out(dest)):
dest = joinpaths(dest, basename(src))
if os.path.exists(self._out(dest)):
logger.debug("file %s exists...removing before linking" % (self._out(dest)))
os.remove(self._out(dest))
os.link(self._out(src), self._out(dest))
def symlink(self, target, dest):
'''
symlink SRC DEST
Create a symlink at DEST which points to SRC.
'''
if rexists(self._out(dest)):
self.remove(dest)
os.symlink(target, self._out(dest))
def copy(self, src, dest):
'''
copy SRC DEST
Copy SRC to DEST.
If DEST is a directory, SRC will be copied inside it.
If DEST doesn't exist, SRC will be copied to a file with
that name, if the path leading to it exists.
'''
cpfile(self._out(src), self._out(dest))
def move(self, src, dest):
'''
move SRC DEST
Move SRC to DEST.
'''
mvfile(self._out(src), self._out(dest))
def remove(self, *fileglobs):
'''
remove FILEGLOB [FILEGLOB ...]
Remove all the named files or directories.
Will *not* raise exceptions if the file(s) are not found.
'''
for g in fileglobs:
for f in rglob(self._out(g)):
remove(f)
logger.debug("removed %s", f)
def chmod(self, fileglob, mode):
'''
chmod FILEGLOB OCTALMODE
Change the mode of all the files matching FILEGLOB to OCTALMODE.
'''
for f in rglob(self._out(fileglob), fatal=True):
os.chmod(f, int(mode,8))
# TODO: do we need a new command for gsettings?
def gconfset(self, path, keytype, value, outfile=None):
'''
gconfset PATH KEYTYPE VALUE [OUTFILE]
Set the given gconf PATH, with type KEYTYPE, to the given value.
OUTFILE defaults to /etc/gconf/gconf.xml.defaults if not given.
Example:
gconfset /apps/metacity/general/num_workspaces int 1
'''
if outfile is None:
outfile = self._out("etc/gconf/gconf.xml.defaults")
cmd = ["gconftool-2", "--direct",
"--config-source=xml:readwrite:%s" % outfile,
"--set", "--type", keytype, path, value]
runcmd(cmd)
def log(self, msg):
'''
log MESSAGE
Emit the given log message. Be sure to put it in quotes!
Example:
log "Reticulating splines, please wait..."
'''
logger.info(msg)
# TODO: add ssh-keygen, mkisofs(?), find, and other useful commands
def runcmd(self, *cmdlist):
'''
runcmd CMD [--chdir=DIR] [ARG ...]
Run the given command with the given arguments.
If "--chdir=DIR" is given, change to the named directory
before executing the command.
NOTE: All paths given MUST be COMPLETE, ABSOLUTE PATHS to the file
or files mentioned. ${root}/${inroot}/${outroot} are good for
constructing these paths.
FURTHER NOTE: Please use this command only as a last resort!
Whenever possible, you should use the existing template commands.
If the existing commands don't do what you need, fix them!
Examples:
(this should be replaced with a "find" function)
runcmd find ${root} -name "*.pyo" -type f -delete
%for f in find(root, name="*.pyo"):
remove ${f}
%endfor
'''
cwd = None
cmd = cmdlist
logger.debug('running command: %s', cmd)
if cmd[0].startswith("--chdir="):
cwd = cmd[0].split('=',1)[1]
cmd = cmd[1:]
try:
output = runcmd_output(cmd, cwd=cwd)
if output:
logger.debug('command output:\n%s', output)
logger.debug("command finished successfully")
except CalledProcessError as e:
if e.output:
logger.debug('command output:\n%s', e.output)
logger.debug('command returned failure (%d)', e.returncode)
raise
def installpkg(self, *pkgs):
'''
installpkg [--required] PKGGLOB [PKGGLOB ...]
Request installation of all packages matching the given globs.
Note that this is just a *request* - nothing is *actually* installed
until the 'run_pkg_transaction' command is given.
'''
required = False
if pkgs[0] == '--required':
pkgs = pkgs[1:]
required = True
for p in pkgs:
try:
self.yum.install(pattern=p)
except Exception as e:
# FIXME: save exception and re-raise after the loop finishes
logger.error("installpkg %s failed: %s",p,str(e))
if required:
raise
def removepkg(self, *pkgs):
'''
removepkg PKGGLOB [PKGGLOB...]
Delete the named package(s).
IMPLEMENTATION NOTES:
RPM scriptlets (%preun/%postun) are *not* run.
Files are deleted, but directories are left behind.
'''
for p in pkgs:
filepaths = [f.lstrip('/') for f in self._filelist(p)]
# TODO: also remove directories that aren't owned by anything else
if filepaths:
logger.debug("removepkg %s: %ikb", p, self._getsize(*filepaths)/1024)
self.remove(*filepaths)
else:
logger.debug("removepkg %s: no files to remove!", p)
def run_pkg_transaction(self):
'''
run_pkg_transaction
Actually install all the packages requested by previous 'installpkg'
commands.
'''
self.yum.buildTransaction()
self.yum.repos.setProgressBar(LoraxDownloadCallback())
self.yum.processTransaction(callback=LoraxTransactionCallback(),
rpmDisplay=LoraxRpmCallback())
# verify if all packages that were supposed to be installed,
# are really installed
errs = [t.po for t in self.yum.tsInfo if not self.yum.rpmdb.contains(po=t.po)]
for po in errs:
logger.error("package '%s' was not installed", po)
self.yum.closeRpmDB()
def removefrom(self, pkg, *globs):
'''
removefrom PKGGLOB [--allbut] FILEGLOB [FILEGLOB...]
Remove all files matching the given file globs from the package
(or packages) named.
If '--allbut' is used, all the files from the given package(s) will
be removed *except* the ones which match the file globs.
Examples:
removefrom usbutils /usr/bin/*
removefrom xfsprogs --allbut /sbin/*
'''
cmd = "%s %s" % (pkg, " ".join(globs)) # save for later logging
keepmatches = False
if globs[0] == '--allbut':
keepmatches = True
globs = globs[1:]
# get pkg filelist and find files that match the globs
filelist = self._filelist(pkg)
matches = set()
for g in globs:
globs_re = re.compile(fnmatch.translate(g))
m = filter(globs_re.match, filelist)
if m:
matches.update(m)
else:
logger.debug("removefrom %s %s: no files matched!", pkg, g)
# are we removing the matches, or keeping only the matches?
if keepmatches:
remove = filelist.difference(matches)
else:
remove = matches
# remove the files
if remove:
logger.debug("%s: removed %i/%i files, %ikb/%ikb", cmd,
len(remove), len(filelist),
self._getsize(*remove)/1024, self._getsize(*filelist)/1024)
self.remove(*remove)
else:
logger.debug("%s: no files to remove!", cmd)
def createaddrsize(self, addr, src, dest):
'''
createaddrsize INITRD_ADDRESS INITRD ADDRSIZE
Create the initrd.addrsize file required in LPAR boot process.
Examples:
createaddrsize ${INITRD_ADDRESS} ${outroot}/${BOOTDIR}/initrd.img ${outroot}/${BOOTDIR}/initrd.addrsize
'''
addrsize = open(dest, "wb")
addrsize_data = struct.pack(">iiii", 0, int(addr, 16), 0, os.stat(src).st_size)
addrsize.write(addrsize_data)
addrsize.close()
def systemctl(self, cmd, *units):
'''
systemctl [enable|disable|mask] UNIT [UNIT...]
Enable, disable, or mask the given systemd units.
Examples:
systemctl disable lvm2-monitor.service
systemctl mask fedora-storage-init.service fedora-configure.service
'''
if cmd not in ('enable', 'disable', 'mask'):
raise ValueError('unsupported systemctl cmd: %s' % cmd)
if not units:
logger.debug("systemctl: no units given for %s, ignoring", cmd)
return
self.mkdir("/run/systemd/system") # XXX workaround for systemctl bug
systemctl = ('systemctl', '--root', self.outroot, '--no-reload',
'--quiet', cmd)
# XXX for some reason 'systemctl enable/disable' always returns 1
try:
cmd = systemctl + units
runcmd(cmd)
except CalledProcessError:
pass
class LoraxTemplateRunner(object):
'''
This class parses and executes Lorax templates. Sample usage:
# install a bunch of packages
runner = LoraxTemplateRunner(inroot=rundir, outroot=rundir, yum=yum_obj)
runner.run("install-packages.ltmpl")
# modify a runtime dir
runner = LoraxTemplateRunner(inroot=rundir, outroot=newrun)
runner.run("runtime-transmogrify.ltmpl")
NOTES:
* Parsing procedure is roughly:
1. Mako template expansion (on the whole file)
2. For each line of the result,
a. Whitespace splitting (using shlex.split())
b. Brace expansion (using brace_expand())
c. If the first token is the name of a function, call that function
with the rest of the line as arguments
* Parsing and execution are *separate* passes - so you can't use the result
of a command in an %if statement (or any other control statements)!
* Commands that run external programs (systemctl, gconfset) currently use
the *host*'s copy of that program, which may cause problems if there's a
big enough difference between the host and the image you're modifying.
* The commands are not executed under a real chroot, so absolute symlinks
will point *outside* the inroot/outroot. Be careful with symlinks!
ADDING NEW COMMANDS:
* Each template command is just a method of the LoraxTemplateRunner
object - so adding a new command is as easy as adding a new function.
* Each function gets arguments that correspond to the rest of the tokens
on that line (after word splitting and brace expansion)
* Commands should raise exceptions for errors - don't use sys.exit()
'''
def __init__(self, inroot, outroot, yum=None, fatalerrors=True,
templatedir=None, defaults={}):
self.inroot = inroot
self.outroot = outroot
self.yum = yum
self.fatalerrors = fatalerrors
self.templatedir = templatedir or "/usr/share/lorax"
# some builtin methods
self.builtins = DataHolder(exists=lambda p: rexists(p, root=inroot),
glob=lambda g: list(rglob(g, root=inroot)))
self.defaults = defaults
self.results = DataHolder(treeinfo=dict()) # just treeinfo for now
# TODO: set up custom logger with a filter to add line info
def _out(self, path):
return joinpaths(self.outroot, path)
def _in(self, path):
return joinpaths(self.inroot, path)
def _filelist(self, *pkgs):
pkglist = self.yum.doPackageLists(pkgnarrow="installed", patterns=pkgs)
return set([f for pkg in pkglist.installed for f in pkg.filelist+pkg.ghostlist])
def _getsize(self, *files):
return sum(os.path.getsize(self._out(f)) for f in files if os.path.isfile(self._out(f)))
def run(self, templatefile, **variables):
for k,v in self.defaults.items() + self.builtins.items():
variables.setdefault(k,v)
logger.debug("executing {0} with variables={1}".format(templatefile, variables))
self.templatefile = templatefile
t = LoraxTemplate(directories=[self.templatedir])
commands = t.parse(templatefile, variables)
self._run(commands)
def _run(self, parsed_template):
logger.info("running %s", self.templatefile)
for (num, line) in enumerate(parsed_template,1):
logger.debug("template line %i: %s", num, " ".join(line))
skiperror = False
(cmd, args) = (line[0], line[1:])
# Following Makefile convention, if the command is prefixed with
# a dash ('-'), we'll ignore any errors on that line.
if cmd.startswith('-'):
cmd = cmd[1:]
skiperror = True
try:
# grab the method named in cmd and pass it the given arguments
f = getattr(self, cmd, None)
if cmd[0] == '_' or cmd == 'run' or not callable(f):
raise ValueError, "unknown command %s" % cmd
f(*args)
except Exception:
if skiperror:
logger.debug("ignoring error")
continue
logger.error("template command error in %s:", self.templatefile)
logger.error(" %s", " ".join(line))
# format the exception traceback
exclines = traceback.format_exception(*sys.exc_info())
# skip the bit about "ltmpl.py, in _run()" - we know that
exclines.pop(1)
# log the "ErrorType: this is what happened" line
logger.error(" " + exclines[-1].strip())
# and log the entire traceback to the debug log
for line in ''.join(exclines).splitlines():
logger.debug(" " + line)
if self.fatalerrors:
raise
def install(self, srcglob, dest):
'''
install SRC DEST
Copy the given file (or files, if a glob is used) from the input
tree to the given destination in the output tree.
The path to DEST must exist in the output tree.
If DEST is a directory, SRC will be copied into that directory.
If DEST doesn't exist, SRC will be copied to a file with that name,
assuming the rest of the path exists.
This is pretty much like how the 'cp' command works.
Examples:
install usr/share/myconfig/grub.conf /boot
install /usr/share/myconfig/grub.conf.in /boot/grub.conf
'''
for src in rglob(self._in(srcglob), fatal=True):
cpfile(src, self._out(dest))
def installimg(self, srcdir, destfile):
'''
installimg SRCDIR DESTFILE
Create a compressed cpio archive of the contents of SRCDIR and place
it in DESTFILE.
If SRCDIR doesn't exist or is empty nothing is created.
Examples:
installimg ${LORAXDIR}/product/ images/product.img
installimg ${LORAXDIR}/updates/ images/updates.img
'''
if not os.path.isdir(self._in(srcdir)) or not os.listdir(self._in(srcdir)):
return
logger.info("Creating image file %s from contents of %s", self._out(destfile), self._in(srcdir))
mkcpio(self._in(srcdir), self._out(destfile))
def mkdir(self, *dirs):
'''
mkdir DIR [DIR ...]
Create the named DIR(s). Will create leading directories as needed.
Example:
mkdir /images
'''
for d in dirs:
d = self._out(d)
if not isdir(d):
os.makedirs(d)
def replace(self, pat, repl, *fileglobs):
'''
replace PATTERN REPLACEMENT FILEGLOB [FILEGLOB ...]
Find-and-replace the given PATTERN (Python-style regex) with the given
REPLACEMENT string for each of the files listed.
Example:
replace @VERSION@ ${product.version} /boot/grub.conf /boot/isolinux.cfg
'''
match = False
for g in fileglobs:
for f in rglob(self._out(g)):
match = True
replace(f, pat, repl)
if not match:
raise IOError, "no files matched %s" % " ".join(fileglobs)
def append(self, filename, data):
'''
append FILE STRING
Append STRING (followed by a newline character) to FILE.
Python character escape sequences ('\\n', '\\t', etc.) will be
converted to the appropriate characters.
Examples:
append /etc/depmod.d/dd.conf "search updates built-in"
append /etc/resolv.conf ""
'''
with open(self._out(filename), "a") as fobj:
fobj.write(data.decode('string_escape')+"\n")
def treeinfo(self, section, key, *valuetoks):
'''
treeinfo SECTION KEY ARG [ARG ...]
Add an item to the treeinfo data store.
The given SECTION will have a new item added where
KEY = ARG ARG ...
Example:
treeinfo images-${kernel.arch} boot.iso images/boot.iso
'''
if section not in self.results.treeinfo:
self.results.treeinfo[section] = dict()
self.results.treeinfo[section][key] = " ".join(valuetoks)
def installkernel(self, section, src, dest):
'''
installkernel SECTION SRC DEST
Install the kernel from SRC in the input tree to DEST in the output
tree, and then add an item to the treeinfo data store, in the named
SECTION, where "kernel" = DEST.
Equivalent to:
install SRC DEST
treeinfo SECTION kernel DEST
'''
self.install(src, dest)
self.treeinfo(section, "kernel", dest)
def installinitrd(self, section, src, dest):
'''
installinitrd SECTION SRC DEST
Same as installkernel, but for "initrd".
'''
self.install(src, dest)
self.chmod(dest, '644')
self.treeinfo(section, "initrd", dest)
def installupgradeinitrd(self, section, src, dest):
'''
installupgradeinitrd SECTION SRC DEST
Same as installkernel, but for "upgrade".
'''
self.install(src, dest)
self.chmod(dest, '644')
self.treeinfo(section, "upgrade", dest)
def hardlink(self, src, dest):
'''
hardlink SRC DEST
Create a hardlink at DEST which is linked to SRC.
'''
if isdir(self._out(dest)):
dest = joinpaths(dest, basename(src))
if os.path.exists(self._out(dest)):
logger.debug("file %s exists...removing before linking" % (self._out(dest)))
os.remove(self._out(dest))
os.link(self._out(src), self._out(dest))
def symlink(self, target, dest):
'''
symlink SRC DEST
Create a symlink at DEST which points to SRC.
'''
if rexists(self._out(dest)):
self.remove(dest)
os.symlink(target, self._out(dest))
def copy(self, src, dest):
'''
copy SRC DEST
Copy SRC to DEST.
If DEST is a directory, SRC will be copied inside it.
If DEST doesn't exist, SRC will be copied to a file with
that name, if the path leading to it exists.
'''
cpfile(self._out(src), self._out(dest))
def move(self, src, dest):
'''
move SRC DEST
Move SRC to DEST.
'''
mvfile(self._out(src), self._out(dest))
def remove(self, *fileglobs):
'''
remove FILEGLOB [FILEGLOB ...]
Remove all the named files or directories.
Will *not* raise exceptions if the file(s) are not found.
'''
for g in fileglobs:
for f in rglob(self._out(g)):
remove(f)
logger.debug("removed %s", f)
def chmod(self, fileglob, mode):
'''
chmod FILEGLOB OCTALMODE
Change the mode of all the files matching FILEGLOB to OCTALMODE.
'''
for f in rglob(self._out(fileglob), fatal=True):
os.chmod(f, int(mode,8))
# TODO: do we need a new command for gsettings?
def gconfset(self, path, keytype, value, outfile=None):
'''
gconfset PATH KEYTYPE VALUE [OUTFILE]
Set the given gconf PATH, with type KEYTYPE, to the given value.
OUTFILE defaults to /etc/gconf/gconf.xml.defaults if not given.
Example:
gconfset /apps/metacity/general/num_workspaces int 1
'''
if outfile is None:
outfile = self._out("etc/gconf/gconf.xml.defaults")
cmd = ["gconftool-2", "--direct",
"--config-source=xml:readwrite:%s" % outfile,
"--set", "--type", keytype, path, value]
runcmd(cmd)
def log(self, msg):
'''
log MESSAGE
Emit the given log message. Be sure to put it in quotes!
Example:
log "Reticulating splines, please wait..."
'''
logger.info(msg)
# TODO: add ssh-keygen, mkisofs(?), find, and other useful commands
def runcmd(self, *cmdlist):
'''
runcmd CMD [--chdir=DIR] [ARG ...]
Run the given command with the given arguments.
If "--chdir=DIR" is given, change to the named directory
before executing the command.
NOTE: All paths given MUST be COMPLETE, ABSOLUTE PATHS to the file
or files mentioned. ${root}/${inroot}/${outroot} are good for
constructing these paths.
FURTHER NOTE: Please use this command only as a last resort!
Whenever possible, you should use the existing template commands.
If the existing commands don't do what you need, fix them!
Examples:
(this should be replaced with a "find" function)
runcmd find ${root} -name "*.pyo" -type f -delete
%for f in find(root, name="*.pyo"):
remove ${f}
%endfor
'''
cwd = None
cmd = cmdlist
logger.debug('running command: %s', cmd)
if cmd[0].startswith("--chdir="):
cwd = cmd[0].split('=',1)[1]
cmd = cmd[1:]
try:
output = runcmd_output(cmd, cwd=cwd)
if output:
logger.debug('command output:\n%s', output)
logger.debug("command finished successfully")
except CalledProcessError as e:
if e.output:
logger.debug('command output:\n%s', e.output)
logger.debug('command returned failure (%d)', e.returncode)
raise
def installpkg(self, *pkgs):
'''
installpkg [--required] PKGGLOB [PKGGLOB ...]
Request installation of all packages matching the given globs.
Note that this is just a *request* - nothing is *actually* installed
until the 'run_pkg_transaction' command is given.
'''
required = False
if pkgs[0] == '--required':
pkgs = pkgs[1:]
required = True
for p in pkgs:
try:
self.yum.install(pattern=p)
except Exception as e:
# FIXME: save exception and re-raise after the loop finishes
logger.error("installpkg %s failed: %s",p,str(e))
if required:
raise
def removepkg(self, *pkgs):
'''
removepkg PKGGLOB [PKGGLOB...]
Delete the named package(s).
IMPLEMENTATION NOTES:
RPM scriptlets (%preun/%postun) are *not* run.
Files are deleted, but directories are left behind.
'''
for p in pkgs:
filepaths = [f.lstrip('/') for f in self._filelist(p)]
# TODO: also remove directories that aren't owned by anything else
if filepaths:
logger.debug("removepkg %s: %ikb", p, self._getsize(*filepaths)/1024)
self.remove(*filepaths)
else:
logger.debug("removepkg %s: no files to remove!", p)
def run_pkg_transaction(self):
'''
run_pkg_transaction
Actually install all the packages requested by previous 'installpkg'
commands.
'''
self.yum.buildTransaction()
self.yum.repos.setProgressBar(LoraxDownloadCallback())
self.yum.processTransaction(callback=LoraxTransactionCallback(),
rpmDisplay=LoraxRpmCallback())
# verify if all packages that were supposed to be installed,
# are really installed
errs = [t.po for t in self.yum.tsInfo if not self.yum.rpmdb.contains(po=t.po)]
for po in errs:
logger.error("package '%s' was not installed", po)
self.yum.closeRpmDB()
def removefrom(self, pkg, *globs):
'''
removefrom PKGGLOB [--allbut] FILEGLOB [FILEGLOB...]
Remove all files matching the given file globs from the package
(or packages) named.
If '--allbut' is used, all the files from the given package(s) will
be removed *except* the ones which match the file globs.
Examples:
removefrom usbutils /usr/bin/*
removefrom xfsprogs --allbut /sbin/*
'''
cmd = "%s %s" % (pkg, " ".join(globs)) # save for later logging
keepmatches = False
if globs[0] == '--allbut':
keepmatches = True
globs = globs[1:]
# get pkg filelist and find files that match the globs
filelist = self._filelist(pkg)
matches = set()
for g in globs:
globs_re = re.compile(fnmatch.translate(g))
m = filter(globs_re.match, filelist)
if m:
matches.update(m)
else:
logger.debug("removefrom %s %s: no files matched!", pkg, g)
# are we removing the matches, or keeping only the matches?
if keepmatches:
remove = filelist.difference(matches)
else:
remove = matches
# remove the files
if remove:
logger.debug("%s: removed %i/%i files, %ikb/%ikb", cmd,
len(remove), len(filelist),
self._getsize(*remove)/1024, self._getsize(*filelist)/1024)
self.remove(*remove)
else:
logger.debug("removefrom %s: no files to remove!", cmd)
def removekmod(self, *globs):
'''
removekmod GLOB [GLOB...] [--allbut] KEEPGLOB [KEEPGLOB...]
Remove all files and directories matching the given file globs from the kernel
modules directory.
If '--allbut' is used, all the files from the modules will be removed *except*
the ones which match the file globs. There must be at least one initial GLOB
to search and one KEEPGLOB to keep. The KEEPGLOB is expanded to be *KEEPGLOB*
so that it will match anywhere in the path.
This only removes files from under /lib/modules/*/kernel/
Examples:
removekmod sound drivers/media drivers/hwmon drivers/video
removekmod drivers/char --allbut virtio_console hw_random
'''
cmd = " ".join(globs)
if "--allbut" in globs:
idx = globs.index("--allbut")
if idx == 0:
raise ValueError("removekmod needs at least one GLOB before --allbut")
# Apply keepglobs anywhere they appear in the path
keepglobs = globs[idx+1:]
if len(keepglobs) == 0:
raise ValueError("removekmod needs at least one GLOB after --allbut")
globs = globs[:idx]
else:
# Nothing to keep
keepglobs = []
filelist = set()
for g in globs:
for top_dir in rglob(self._out("/lib/modules/*/kernel/"+g)):
for root, _dirs, files in os.walk(top_dir):
filelist.update(root+"/"+f for f in files)
# Remove anything matching keepglobs from the list
matches = set()
for g in keepglobs:
globs_re = re.compile(fnmatch.translate("*"+g+"*"))
m = filter(globs_re.match, filelist)
if m:
matches.update(m)
else:
logger.debug("removekmod %s: no files matched!", g)
remove_files = filelist.difference(matches)
if remove_files:
logger.debug("removekmod: removing %d files", len(remove_files))
map(remove, remove_files)
else:
logger.debug("removekmod %s: no files to remove!", cmd)
def createaddrsize(self, addr, src, dest):
'''
createaddrsize INITRD_ADDRESS INITRD ADDRSIZE
Create the initrd.addrsize file required in LPAR boot process.
Examples:
createaddrsize ${INITRD_ADDRESS} ${outroot}/${BOOTDIR}/initrd.img ${outroot}/${BOOTDIR}/initrd.addrsize
'''
addrsize = open(dest, "wb")
addrsize_data = struct.pack(">iiii", 0, int(addr, 16), 0, os.stat(src).st_size)
addrsize.write(addrsize_data)
addrsize.close()
def systemctl(self, cmd, *units):
'''
systemctl [enable|disable|mask] UNIT [UNIT...]
Enable, disable, or mask the given systemd units.
Examples:
systemctl disable lvm2-monitor.service
systemctl mask fedora-storage-init.service fedora-configure.service
'''
if cmd not in ('enable', 'disable', 'mask'):
raise ValueError('unsupported systemctl cmd: %s' % cmd)
if not units:
logger.debug("systemctl: no units given for %s, ignoring", cmd)
return
self.mkdir("/run/systemd/system") # XXX workaround for systemctl bug
systemctl = ('systemctl', '--root', self.outroot, '--no-reload',
'--quiet', cmd)
# XXX for some reason 'systemctl enable/disable' always returns 1
try:
cmd = systemctl + units
runcmd(cmd)
except CalledProcessError:
pass
def run(self, ybo, product, version, release, variant="", bugurl="",
isfinal=False, workdir=None, outputdir=None, buildarch=None, volid=None,
domacboot=False, doupgrade=True, remove_temp=False,
installpkgs=None,
ssss=None,
size=2,
add_templates=None,
add_template_vars=None,
add_arch_templates=None,
add_arch_template_vars=None,
template_tempdir=None):
assert self._configured
installpkgs = installpkgs or []
# get lorax version
try:
import pylorax.version
except ImportError:
vernum = "devel"
else:
vernum = pylorax.version.num
if domacboot:
try:
runcmd(["rpm", "-q", "hfsplus-tools"])
except CalledProcessError:
logger.critical("you need to install hfsplus-tools to create mac images")
sys.exit(1)
# set up work directory
self.workdir = workdir or tempfile.mkdtemp(prefix="pylorax.work.")
if not os.path.isdir(self.workdir):
os.makedirs(self.workdir)
# set up log directory
logdir = '/var/log/lorax'
if not os.path.isdir(logdir):
os.makedirs(logdir)
self.init_stream_logging()
self.init_file_logging(logdir)
logger.debug("version is {0}".format(vernum))
logger.debug("using work directory {0.workdir}".format(self))
logger.debug("using log directory {0}".format(logdir))
# set up output directory
self.outputdir = outputdir or tempfile.mkdtemp(prefix="pylorax.out.")
if not os.path.isdir(self.outputdir):
os.makedirs(self.outputdir)
logger.debug("using output directory {0.outputdir}".format(self))
# do we have root privileges?
logger.info("checking for root privileges")
if not os.geteuid() == 0:
logger.critical("no root privileges")
sys.exit(1)
# is selinux disabled?
# With selinux in enforcing mode the rpcbind package required for
# dracut nfs module, which is in turn required by anaconda module,
# will not get installed, because it's preinstall scriptlet fails,
# resulting in an incomplete initial ramdisk image.
# The reason is that the scriptlet runs tools from the shadow-utils
# package in chroot, particularly groupadd and useradd to add the
# required rpc group and rpc user. This operation fails, because
# the selinux context on files in the chroot, that the shadow-utils
# tools need to access (/etc/group, /etc/passwd, /etc/shadow etc.),
# is wrong and selinux therefore disallows access to these files.
logger.info("checking the selinux mode")
if selinux.is_selinux_enabled() and selinux.security_getenforce():
logger.critical("selinux must be disabled or in Permissive mode")
sys.exit(1)
# do we have a proper yum base object?
logger.info("checking yum base object")
if not isinstance(ybo, yum.YumBase):
logger.critical("no yum base object")
sys.exit(1)
self.inroot = ybo.conf.installroot
logger.debug("using install root: {0}".format(self.inroot))
if not buildarch:
buildarch = get_buildarch(ybo)
logger.info("setting up build architecture")
self.arch = ArchData(buildarch)
for attr in ('buildarch', 'basearch', 'libdir'):
logger.debug("self.arch.%s = %s", attr, getattr(self.arch,attr))
logger.info("setting up build parameters")
product = DataHolder(name=product, version=version, release=release,
variant=variant, bugurl=bugurl, isfinal=isfinal)
self.product = product
logger.debug("product data: %s" % product)
# NOTE: if you change isolabel, you need to change pungi to match, or
# the pungi images won't boot.
isolabel = volid or "{0.name} {0.version} {1.basearch}".format(self.product,
self.arch)
if len(isolabel) > 32:
logger.fatal("the volume id cannot be longer than 32 characters")
sys.exit(1)
templatedir = self.conf.get("lorax", "sharedir")
# NOTE: rb.root = ybo.conf.installroot (== self.inroot)
rb = RuntimeBuilder(product=self.product, arch=self.arch,
yum=ybo, templatedir=templatedir,
installpkgs=installpkgs,
add_templates=add_templates,
add_template_vars=add_template_vars)
logger.info("installing runtime packages")
rb.yum.conf.skip_broken = self.conf.getboolean("yum", "skipbroken")
rb.install()
# write .buildstamp
buildstamp = BuildStamp(self.product.name, self.product.version,
self.product.bugurl, self.product.isfinal, self.arch.buildarch)
buildstamp.write(joinpaths(self.inroot, ".buildstamp"))
if self.debug:
rb.writepkglists(joinpaths(logdir, "pkglists"))
rb.writepkgsizes(joinpaths(logdir, "original-pkgsizes.txt"))
logger.info("doing post-install configuration")
rb.postinstall()
# write .discinfo
discinfo = DiscInfo(self.product.release, self.arch.basearch)
discinfo.write(joinpaths(self.outputdir, ".discinfo"))
logger.info("backing up installroot")
installroot = joinpaths(self.workdir, "installroot")
linktree(self.inroot, installroot)
logger.info("generating kernel module metadata")
rb.generate_module_data()
logger.info("cleaning unneeded files")
rb.cleanup()
if self.debug:
rb.writepkgsizes(joinpaths(logdir, "final-pkgsizes.txt"))
logger.info("creating the runtime image")
runtime = "images/install.img"
compression = self.conf.get("compression", "type")
compressargs = self.conf.get("compression", "args").split()
if self.conf.getboolean("compression", "bcj"):
if self.arch.bcj:
compressargs += ["-Xbcj", self.arch.bcj]
else:
logger.info("no BCJ filter for arch %s", self.arch.basearch)
rb.create_runtime(joinpaths(installroot,runtime),
compression=compression, compressargs=compressargs)
logger.info("preparing to build output tree and boot images")
treebuilder = TreeBuilder(product=self.product, arch=self.arch,
inroot=installroot, outroot=self.outputdir, ssss=ssss,
runtime=runtime, isolabel=isolabel,
domacboot=domacboot, doupgrade=doupgrade,
templatedir=templatedir,
add_templates=add_arch_templates,
add_template_vars=add_arch_template_vars,
workdir=self.workdir)
logger.info("rebuilding initramfs images")
dracut_args = ["--xz", "--install", "/.buildstamp", "--no-early-microcode"]
anaconda_args = dracut_args + ["--add", "anaconda pollcdrom"]
# ppc64 cannot boot an initrd > 32MiB so remove some drivers
if self.arch.basearch in ("ppc64", "ppc64le"):
dracut_args.extend(["--omit-drivers", REMOVE_PPC64_DRIVERS])
# Only omit dracut modules from the initrd so that they're kept for
# upgrade.img
anaconda_args.extend(["--omit", REMOVE_PPC64_MODULES])
treebuilder.rebuild_initrds(add_args=anaconda_args)
if doupgrade:
# Build upgrade.img. It'd be nice if these could coexist in the same
# image, but that would increase the size of the anaconda initramfs,
# which worries some people (esp. PPC tftpboot). So they're separate.
try:
# If possible, use the 'redhat-upgrade-tool' plymouth theme
themes = runcmd_output(['plymouth-set-default-theme', '--list'],
root=installroot)
if 'redhat-upgrade-tool' in themes.splitlines():
os.environ['PLYMOUTH_THEME_NAME'] = 'redhat-upgrade-tool'
except RuntimeError:
pass
upgrade_args = dracut_args + ["--add", "system-upgrade convertfs"]
treebuilder.rebuild_initrds(add_args=upgrade_args, prefix="upgrade")
logger.info("populating output tree and building boot images")
treebuilder.build()
# write .treeinfo file and we're done
treeinfo = TreeInfo(self.product.name, self.product.version,
self.product.variant, self.arch.basearch)
for section, data in treebuilder.treeinfo_data.items():
treeinfo.add_section(section, data)
treeinfo.write(joinpaths(self.outputdir, ".treeinfo"))
# cleanup
if remove_temp:
remove(self.workdir)