def ungrab_pm(self, *ignored, **kwignored):
if self.cachedir:
with lockfile(j(self.cachedir, "ifz-lockfile")):
while self.cachemounts:
while ismount(self.cachemounts[-1]):
logger.debug("Unmounting %s", self.cachemounts[-1])
umount(self.cachemounts[-1])
os.rmdir(self.cachemounts[-1])
self.cachemounts.pop()
if self.pkgmgr_config:
self.pkgmgr_config.close()
self.pkgmgr_config = None
def ungrab_pm(self, *ignored, **kwignored):
if self.cachedir:
with lockfile(j(self.cachedir, "ifz-lockfile")):
while self.cachemounts:
while ismount(self.cachemounts[-1]):
logger.debug("Unmounting %s", self.cachemounts[-1])
umount(self.cachemounts[-1])
os.rmdir(self.cachemounts[-1])
self.cachemounts.pop()
if self.pkgmgr_config:
self.pkgmgr_config.close()
self.pkgmgr_config = None
def undo(self):
for n, (typ, o) in reversed(list(enumerate(self.actions[:]))):
if typ == "unmount":
umount(o)
if typ == "rmrf":
shutil.rmtree(o)
if typ == "rmdir":
os.rmdir(o)
if typ == "export":
check_call(["sync"])
check_call(["zpool", "export", o])
if typ == "luks_close":
check_call(["sync"])
check_call(["cryptsetup", "luksClose", o])
if typ == "un_losetup":
check_call(["sync"])
if get_associated_lodev(o):
check_call(["losetup", "-d", o])
if get_associated_lodev(o):
logging.error("losetup -d failed with device %s", o)
self.actions.pop(n)
def grab_pm(self, method):
if self.cachemounts or self.pkgmgr_config:
assert 0, "programming error, invalid state, cannot enter without exiting first"
if method == "in_chroot":
dirforconfig = self.chroot
if os.path.isfile(j(self.chroot, "etc", "dnf", "dnf.conf")):
sourceconf = j(self.chroot, "etc", "dnf", "dnf.conf")
pkgmgr = "dnf"
elif os.path.isfile(j(self.chroot, "etc", "yum.conf")):
sourceconf = j(self.chroot, "etc", "yum.conf")
pkgmgr = "yum"
else:
raise Exception("Cannot use in_chroot method without a working yum or DNF inside the chroot")
ver = self.releasever
elif method == "out_of_chroot":
dirforconfig = os.getenv("TMPDIR") or "/tmp"
if os.path.exists("/etc/dnf/dnf.conf"):
sourceconf = "/etc/dnf/dnf.conf"
pkgmgr = "dnf"
elif os.path.exists("/etc/yum.conf"):
sourceconf = "/etc/yum.conf"
pkgmgr = "yum"
else:
raise Exception("Cannot use out_of_chroot method without a working yum or DNF installed on your system")
ver = self.myreleasever
else:
assert 0, "method unknown: %r" % method
parms = dict(
source=sourceconf,
directory=dirforconfig,
logfile="/dev/null",
debuglevel=2,
reposdir="/nonexistent",
include=None,
keepcache=1 if pkgmgr == "yum" else True,
)
# /yumcache
if self.cachedir:
makedirs([self.cachedir])
with lockfile(j(self.cachedir, "ifz-lockfile")):
# /yumcache/(dnf|yum)/(ver)/(lib|cache)
persistdir = j(self.cachedir, pkgmgr, str(ver), "lib")
cachedir = j(self.cachedir, pkgmgr, str(ver), "cache")
makedirs([persistdir, cachedir])
# /yumcache/(dnf|yum)/(ver)/lock
# /chroot/var/(lib|cache)/(dnf|yum)
persistin, cachein = makedirs([
j(self.chroot, "tmp-%s-%s" % (pkgmgr, x))
for x in ["lib", "cache"]
])
maybemounted = [persistin, cachein]
while maybemounted:
while ismount(maybemounted[-1]):
logger.debug("Preemptively unmounting %s", maybemounted[-1])
umount(maybemounted[-1])
maybemounted.pop()
for x, y in ([persistdir, persistin], [cachedir, cachein]):
logger.debug("Mounting %s to %s", x, y)
self.cachemounts.append(bindmount(x, y))
# /var/(lib|cache)/(dnf|yum)
parms["persistdir"] = persistin[len(self.chroot):]
parms["cachedir"] = cachein[len(self.chroot):]
lock = lockfile(j(self.cachedir, pkgmgr, str(ver), "lock"))
else:
lock = dummylock()
self.pkgmgr_config = make_temp_yum_config(**parms)
return pkgmgr, self.pkgmgr_config, lock
def grab_pm(self, method):
if self.cachemounts or self.pkgmgr_config:
assert 0, "programming error, invalid state, cannot enter without exiting first"
if method == "in_chroot":
dirforconfig = self.chroot
if os.path.isfile(j(self.chroot, "etc", "dnf", "dnf.conf")):
sourceconf = j(self.chroot, "etc", "dnf", "dnf.conf")
pkgmgr = "dnf"
elif os.path.isfile(j(self.chroot, "etc", "yum.conf")):
sourceconf = j(self.chroot, "etc", "yum.conf")
pkgmgr = "yum"
else:
raise Exception("Cannot use in_chroot method without a working yum or DNF inside the chroot")
ver = self.releasever
elif method == "out_of_chroot":
dirforconfig = os.getenv("TMPDIR") or "/tmp"
if os.path.exists("/etc/dnf/dnf.conf"):
sourceconf = "/etc/dnf/dnf.conf"
pkgmgr = "dnf"
elif os.path.exists("/etc/yum.conf"):
sourceconf = "/etc/yum.conf"
pkgmgr = "yum"
else:
raise Exception("Cannot use out_of_chroot method without a working yum or DNF installed on your system")
ver = self.myreleasever
else:
assert 0, "method unknown: %r" % method
parms = dict(
source=sourceconf,
directory=dirforconfig,
logfile="/dev/null",
debuglevel=2,
reposdir="/nonexistent",
include=None,
keepcache=1 if pkgmgr == "yum" else True,
)
# /yumcache
if self.cachedir:
makedirs([self.cachedir])
with lockfile(j(self.cachedir, "ifz-lockfile")):
# /yumcache/(dnf|yum)/(ver)/(lib|cache)
persistdir = j(self.cachedir, pkgmgr, str(ver), "lib")
cachedir = j(self.cachedir, pkgmgr, str(ver), "cache")
makedirs([persistdir, cachedir])
# /yumcache/(dnf|yum)/(ver)/lock
lock = lockfile(j(self.cachedir, pkgmgr, str(ver), "lock"))
# /chroot/var/(lib|cache)/(dnf|yum)
persistin, cachein = makedirs([
j(self.chroot, "tmp-%s-%s" % (pkgmgr, x))
for x in ["lib", "cache"]
])
maybemounted = [persistin, cachein]
while maybemounted:
while ismount(maybemounted[-1]):
logger.debug("Preemptively unmounting %s", maybemounted[-1])
umount(maybemounted[-1])
maybemounted.pop()
for x, y in ([persistdir, persistin], [cachedir, cachein]):
logger.debug("Mounting %s to %s", x, y)
self.cachemounts.append(bindmount(x, y))
# /var/(lib|cache)/(dnf|yum)
parms["persistdir"] = persistin[len(self.chroot):]
parms["cachedir"] = cachein[len(self.chroot):]
else:
lock = dummylock()
self.pkgmgr_config = make_temp_yum_config(**parms)
return pkgmgr, self.pkgmgr_config, lock
def deploy_zfs_in_machine(p, in_chroot, pkgmgr, branch,
prebuilt_rpms_path, break_before, to_unmount, to_rmdir):
arch = platform.machine()
stringtoexclude = "debuginfo"
# check for stage stop
if break_before == "install_prebuilt_rpms":
raise BreakingBefore(break_before)
if prebuilt_rpms_path:
target_rpms_path = p(j("tmp","zfs-fedora-installer-prebuilt-rpms"))
if not os.path.isdir(target_rpms_path):
os.mkdir(target_rpms_path)
if ismount(target_rpms_path):
if os.stat(prebuilt_rpms_path).st_ino != os.stat(target_rpms_path):
umount(target_rpms_path)
bindmount(os.path.abspath(prebuilt_rpms_path), target_rpms_path)
else:
bindmount(os.path.abspath(prebuilt_rpms_path), target_rpms_path)
if os.path.isdir(target_rpms_path):
to_rmdir.append(target_rpms_path)
if ismount(target_rpms_path):
to_unmount.append(target_rpms_path)
prebuilt_rpms_to_install = glob.glob(j(prebuilt_rpms_path,"*%s.rpm"%(arch,))) + glob.glob(j(prebuilt_rpms_path,"*%s.rpm"%("noarch",)))
prebuilt_rpms_to_install = set([
os.path.basename(s)
for s in prebuilt_rpms_to_install
if stringtoexclude not in os.path.basename(s)
])
else:
target_rpms_path = None
prebuilt_rpms_to_install = set()
if prebuilt_rpms_to_install:
logging.info(
"Installing available prebuilt RPMs: %s",
prebuilt_rpms_to_install
)
files_to_install = [
j(target_rpms_path, s)
for s in prebuilt_rpms_to_install
]
pkgmgr.install_local_packages(files_to_install)
if target_rpms_path:
umount(target_rpms_path)
to_unmount.remove(target_rpms_path)
check_call(["rmdir", target_rpms_path])
to_rmdir.remove(target_rpms_path)
# check for stage stop
if break_before == "install_grub_zfs_fixer":
raise BreakingBefore(break_before)
for project, patterns in (
(
"grub-zfs-fixer",
(
"RPMS/%s/*.%s.rpm" % ("noarch", "noarch"),
"RPMS/*.%s.rpm" % ("noarch",),
),
),
):
grubzfsfixerpath = j(os.path.dirname(__file__), os.path.pardir, os.path.pardir, "grub-zfs-fixer")
class FixerNotInstalledYet(Exception): pass
try:
logging.info("Checking if %s has the GRUB ZFS fixer installed", project)
try:
fixerlines = file(j(grubzfsfixerpath, "grub-zfs-fixer.spec")).readlines()
fixerversion = [ x.split()[1] for x in fixerlines if x.startswith("Version:") ][0]
fixerrelease = [ x.split()[1] for x in fixerlines if x.startswith("Release:") ][0]
check_output(in_chroot([
"rpm",
"-q",
"grub-zfs-fixer-%s-%s" % (fixerversion, fixerrelease)
]))
except subprocess.CalledProcessError:
raise FixerNotInstalledYet()
except FixerNotInstalledYet:
logging.info("%s does not have the GRUB ZFS fixer, building", project)
project_dir = p(j("usr","src",project))
def getrpms(pats, directory):
therpms = [
rpmfile
for pat in pats
for rpmfile in glob.glob(j(directory, pat))
if stringtoexclude not in os.path.basename(rpmfile)
]
return therpms
files_to_install = getrpms(patterns, project_dir)
if not files_to_install:
if not os.path.isdir(project_dir):
os.mkdir(project_dir)
pkgmgr.ensure_packages_installed(
[
"rpm-build", "tar", "gzip",
],
)
logging.info("Tarring %s tarball", project)
check_call(['tar', 'cvzf', j(project_dir, "%s.tar.gz" % project), project],
cwd=j(grubzfsfixerpath, os.path.pardir))
logging.info("Building project: %s", project)
project_dir_in_chroot = project_dir[len(p(""))-1:]
check_call(in_chroot(["rpmbuild", "--define", "_topdir %s"%(project_dir_in_chroot,), "-ta", j(project_dir_in_chroot,"%s.tar.gz" % project)]))
files_to_install = getrpms(patterns, project_dir)
logging.info("Installing built RPMs: %s", files_to_install)
pkgmgr.install_local_packages(files_to_install)
# Check we have a patched grub2-mkconfig.
mkconfig_file = p(j("usr", "sbin", "grub2-mkconfig"))
mkconfig_text = file(mkconfig_file).read()
if "This program was patched by fix-grub-mkconfig" not in mkconfig_text:
raise ZFSBuildFailure("expected to find patched %s but could not find it. Perhaps the grub-zfs-fixer RPM was never installed?" % mkconfig_file)
for project, patterns, keystonepkgs, mindeps in (
(
"spl",
(
"spl-*.%s.rpm" % arch,
"spl-dkms-*.noarch.rpm",
),
('spl', 'spl-dkms'),
[
"make", "autoconf", "automake", "gcc", "libtool", "git",
"rpm-build", "dkms",
],
),
(
"zfs",
(
"zfs-dkms-*.noarch.rpm",
"libnvpair*.%s.rpm" % arch,
"libuutil*.%s.rpm" % arch,
"libzfs?-[0123456789]*.%s.rpm" % arch,
"libzfs?-devel-[0123456789]*.%s.rpm" % arch,
"libzpool*.%s.rpm" % arch,
"zfs-[0123456789]*.%s.rpm" % arch,
"zfs-dracut-*.%s.rpm" % arch,
),
('zfs', 'zfs-dkms', 'zfs-dracut'),
[
"zlib-devel", "libuuid-devel", "bc", "libblkid-devel",
"libattr-devel", "lsscsi", "mdadm", "parted",
"libudev-devel",
],
),
):
# check for stage stop
if break_before == "deploy_%s" % project:
raise BreakingBefore(break_before)
try:
logging.info("Checking if keystone packages %s are installed", ", ".join(keystonepkgs))
check_call(in_chroot(["rpm", "-q"] + list(keystonepkgs)),
stdout=file(os.devnull,"w"), stderr=subprocess.STDOUT)
except subprocess.CalledProcessError:
logging.info("Package %s-dkms is not installed, building", project)
project_dir = p(j("usr","src",project))
def getrpms(pats, directory):
therpms = [
rpmfile
for pat in pats
for rpmfile in glob.glob(j(directory, pat))
if stringtoexclude not in os.path.basename(rpmfile)
]
return therpms
files_to_install = getrpms(patterns, project_dir)
if not files_to_install:
if not os.path.isdir(project_dir):
repo = "https://github.com/Rudd-O/%s" % project
logging.info("Cloning git repository: %s", repo)
cmd = ["git", "clone", repo, project_dir]
check_call(cmd)
cmd = ["git", "checkout", branch]
check_call(cmd, cwd= project_dir)
cmd = ["git", "--no-pager", "show"]
check_call(cmd, cwd= project_dir)
pkgmgr.ensure_packages_installed(mindeps)
logging.info("Building project: %s", project)
cores = multiprocessing.cpu_count()
cmd = in_chroot(["bash", "-c",
(
"cd /usr/src/%s && "
"./autogen.sh && "
"./configure --with-config=user && "
"make -j%s rpm-utils && "
"make -j%s rpm-dkms" % (project, cores, cores)
)
])
check_call(cmd)
files_to_install = getrpms(patterns, project_dir)
logging.info("Installing built RPMs: %s", files_to_install)
pkgmgr.install_local_packages(files_to_install)
# Check we have a ZFS.ko for at least one kernel.
modules_dir = p(j("usr", "lib", "modules", "*", "*", "zfs.ko"))
modules_files = glob.glob(modules_dir)
if not modules_files:
raise ZFSBuildFailure("expected to find but could not find module zfs.ko in %s. Perhaps the ZFS source you used is too old to work with the kernel this program installed?" % modules_dir)
def cleanup():
for fs in reversed(to_unmount):
umount(fs)
for filename in to_rmdir:
os.rmdir(filename)