class LogMonitor(object):
def __init__(self, install_log, timeout, log_request_handler_class=None):
"""Monitor VM logs and reacts properly on what happens there.
:param str install_log: path where to store installation log
:param int timeout: set timeout in minutes before killing the VM
:param log_request_handler_class: class compatible with
pylorax.monitor.LogRequestHandler
Use VirtualLogRequestHandler if not set.
"""
self._install_log = install_log
self._timeout = timeout
if log_request_handler_class:
self._log_request_handler_class = log_request_handler_class
else:
self._log_request_handler_class = VirtualLogRequestHandler
if not is_dry_run():
self._lorax_log_monitor = LoraxLogMonitor(
self._install_log,
timeout=self._timeout,
log_request_handler_class=self._log_request_handler_class)
@property
def host(self):
if is_dry_run():
return "DRY_RUN_HOST"
return self._lorax_log_monitor.host
@property
def port(self):
if is_dry_run():
return "DRY_RUN_PORT"
return self._lorax_log_monitor.port
@property
def error_line(self):
if is_dry_run():
return None
return self._lorax_log_monitor.server.error_line
@disable_on_dry_run(False)
def log_check(self):
return self._lorax_log_monitor.server.log_check()
@disable_on_dry_run
def shutdown(self):
self._lorax_log_monitor.shutdown()
def _start_virt_install(self, install_log):
"""
Use virt-install to install to a disk image
:param str install_log: The path to write the log from virt-install
This uses virt-install with a boot.iso and a kickstart to create a disk
image.
"""
iso_mount = IsoMountpoint(self._conf.iso_path, self._conf.location)
log_monitor = LogMonitor(install_log, timeout=self._conf.timeout)
kernel_args = ""
if self._conf.kernel_args:
kernel_args += self._conf.kernel_args
if self._conf.proxy:
kernel_args += " proxy=" + self._conf.proxy
try:
log.info("Starting virtual machine")
virt = VirtualInstall(iso_mount, self._conf.ks_paths,
disk_paths=self._conf.disk_paths,
kernel_args=kernel_args,
vcpu_count=self._conf.vcpu_count,
memory=self._conf.ram,
vnc=self._conf.vnc,
log_check=log_monitor.server.log_check,
virtio_host=log_monitor.host,
virtio_port=log_monitor.port,
nics=self._conf.networks,
boot=self._conf.boot_image)
virt.run()
virt.destroy(os.path.basename(self._conf.temp_dir))
log_monitor.shutdown()
except InstallError as e:
log.error("VirtualInstall failed: %s", e)
raise
finally:
log.info("unmounting the iso")
iso_mount.umount()
if log_monitor.server.log_check():
if not log_monitor.server.error_line and self._conf.timeout:
msg = "Test timed out"
else:
msg = "Test failed on line: %s" % log_monitor.server.error_line
raise InstallError(msg)
def test_monitor_timeout(self):
# Timeout is in minutes so to shorten the test we pass 0.1
monitor = LogMonitor(timeout=0.1)
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((monitor.host, monitor.port))
s.sendall(
"Just a test string\nwith two and a half\nlines in it".
encode("utf8"))
time.sleep(1)
self.assertFalse(monitor.server.log_check())
time.sleep(7)
self.assertTrue(monitor.server.log_check())
self.assertEqual(monitor.server.error_line, "")
finally:
monitor.shutdown()
def test_monitor_utf8(self):
## If a utf8 character spans the end of the 4096 byte buffer it will fail to
## decode. Test to make sure it is reassembled correctly.
monitor = LogMonitor(timeout=1)
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((monitor.host, monitor.port))
# Simulate a UTF8 character that gets broken into parts by buffering, etc.
data = "Just a test string\nTraceback (Not a real traceback)\nWith A"
s.sendall(data.encode("utf8") + b"\xc3")
time.sleep(1)
self.assertTrue(monitor.server.log_check())
self.assertEqual(monitor.server.error_line,
"Traceback (Not a real traceback)")
finally:
monitor.shutdown()
def test_monitor(self):
monitor = LogMonitor(timeout=1)
try:
with socket.socket(socket.AF_INET, socket.SOCK_STREAM) as s:
s.connect((monitor.host, monitor.port))
s.sendall(
"Just a test string\nwith two and a half\nlines in it".
encode("utf8"))
time.sleep(1)
self.assertFalse(monitor.server.log_check())
s.sendall("\nAnother line\nTraceback (Not a real traceback)\n".
encode("utf8"))
time.sleep(1)
self.assertTrue(monitor.server.log_check())
self.assertEqual(monitor.server.error_line,
"Traceback (Not a real traceback)")
finally:
monitor.shutdown()
def virt_install(opts, install_log, disk_img, disk_size, cancel_func=None):
"""
Use qemu to install to a disk image
:param opts: options passed to livemedia-creator
:type opts: argparse options
:param str install_log: The path to write the log from qemu
:param str disk_img: The full path to the disk image to be created
:param int disk_size: The size of the disk_img in MiB
:param cancel_func: Function that returns True to cancel build
:type cancel_func: function
This uses qemu with a boot.iso and a kickstart to create a disk
image and then optionally, based on the opts passed, creates tarfile.
"""
iso_mount = IsoMountpoint(opts.iso, opts.location)
if not iso_mount.stage2:
iso_mount.umount()
raise InstallError("ISO is missing stage2, cannot continue")
log_monitor = LogMonitor(install_log, timeout=opts.timeout)
cancel_funcs = [log_monitor.server.log_check]
if cancel_func is not None:
cancel_funcs.append(cancel_func)
kernel_args = ""
if opts.kernel_args:
kernel_args += opts.kernel_args
if opts.proxy:
kernel_args += " proxy=" + opts.proxy
if opts.image_type and not opts.make_fsimage:
qemu_args = []
for arg in opts.qemu_args:
qemu_args += arg.split(" ", 1)
if "-f" not in qemu_args:
qemu_args += ["-f", opts.image_type]
mkqemu_img(disk_img, disk_size * 1024**2, qemu_args)
if opts.make_fsimage or opts.make_tar or opts.make_oci:
diskimg_path = tempfile.mktemp(prefix="lmc-disk-", suffix=".img")
else:
diskimg_path = disk_img
try:
QEMUInstall(opts,
iso_mount,
opts.ks,
diskimg_path,
disk_size,
kernel_args,
opts.ram,
opts.vcpus,
opts.vnc,
opts.arch,
cancel_func=lambda: any(f() for f in cancel_funcs),
virtio_host=log_monitor.host,
virtio_port=log_monitor.port,
image_type=opts.image_type,
boot_uefi=opts.virt_uefi,
ovmf_path=opts.ovmf_path)
log_monitor.shutdown()
except InstallError as e:
log.error("VirtualInstall failed: %s", e)
raise
finally:
log.info("unmounting the iso")
iso_mount.umount()
if log_monitor.server.log_check():
if not log_monitor.server.error_line and opts.timeout:
msg = "virt_install failed due to timeout"
else:
msg = "virt_install failed on line: %s" % log_monitor.server.error_line
raise InstallError(msg)
elif cancel_func and cancel_func():
raise InstallError("virt_install canceled by cancel_func")
if opts.make_fsimage:
mkfsimage_from_disk(diskimg_path,
disk_img,
disk_size,
label=opts.fs_label)
os.unlink(diskimg_path)
elif opts.make_tar:
compress_args = []
for arg in opts.compress_args:
compress_args += arg.split(" ", 1)
with PartitionMount(diskimg_path) as img_mount:
if img_mount and img_mount.mount_dir:
rc = mktar(img_mount.mount_dir, disk_img, opts.compression,
compress_args)
else:
rc = 1
os.unlink(diskimg_path)
if rc:
raise InstallError("virt_install failed")
elif opts.make_oci:
# An OCI image places the filesystem under /rootfs/ and adds the json files at the top
# And then creates a tar of the whole thing.
compress_args = []
for arg in opts.compress_args:
compress_args += arg.split(" ", 1)
with PartitionMount(diskimg_path, submount="rootfs") as img_mount:
if img_mount and img_mount.temp_dir:
shutil.copy2(opts.oci_config, img_mount.temp_dir)
shutil.copy2(opts.oci_runtime, img_mount.temp_dir)
rc = mktar(img_mount.temp_dir, disk_img, opts.compression,
compress_args)
else:
rc = 1
os.unlink(diskimg_path)
if rc:
raise InstallError("virt_install failed")
elif opts.make_vagrant:
compress_args = []
for arg in opts.compress_args:
compress_args += arg.split(" ", 1)
vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
metadata_path = joinpaths(vagrant_dir, "metadata.json")
execWithRedirect(
"mv",
["-f", disk_img, joinpaths(vagrant_dir, "box.img")],
raise_err=True)
if opts.vagrant_metadata:
shutil.copy2(opts.vagrant_metadata, metadata_path)
else:
create_vagrant_metadata(metadata_path)
update_vagrant_metadata(metadata_path, disk_size)
if opts.vagrantfile:
shutil.copy2(opts.vagrantfile, joinpaths(vagrant_dir,
"vagrantfile"))
rc = mktar(vagrant_dir,
disk_img,
opts.compression,
compress_args,
selinux=False)
if rc:
raise InstallError("virt_install failed")
shutil.rmtree(vagrant_dir)
def novirt_install(opts, disk_img, disk_size, cancel_func=None):
"""
Use Anaconda to install to a disk image
:param opts: options passed to livemedia-creator
:type opts: argparse options
:param str disk_img: The full path to the disk image to be created
:param int disk_size: The size of the disk_img in MiB
:param cancel_func: Function that returns True to cancel build
:type cancel_func: function
This method runs anaconda to create the image and then based on the opts
passed creates a qemu disk image or tarfile.
"""
dirinstall_path = ROOT_PATH
# Clean up /tmp/ from previous runs to prevent stale info from being used
for path in ["/tmp/yum.repos.d/", "/tmp/yum.cache/"]:
if os.path.isdir(path):
shutil.rmtree(path)
args = ["--kickstart", opts.ks[0], "--cmdline", "--loglevel", "debug"]
if opts.anaconda_args:
for arg in opts.anaconda_args:
args += arg.split(" ", 1)
if opts.proxy:
args += ["--proxy", opts.proxy]
if opts.armplatform:
args += ["--armplatform", opts.armplatform]
if opts.make_iso or opts.make_fsimage or opts.make_pxe_live:
# Make a blank fs image
args += ["--dirinstall"]
mkext4img(None,
disk_img,
label=opts.fs_label,
size=disk_size * 1024**2)
if not os.path.isdir(dirinstall_path):
os.mkdir(dirinstall_path)
mount(disk_img, opts="loop", mnt=dirinstall_path)
elif opts.make_tar or opts.make_oci:
# Install under dirinstall_path, make sure it starts clean
if os.path.exists(dirinstall_path):
shutil.rmtree(dirinstall_path)
if opts.make_oci:
# OCI installs under /rootfs/
dirinstall_path = joinpaths(dirinstall_path, "rootfs")
args += ["--dirinstall", dirinstall_path]
else:
args += ["--dirinstall"]
os.makedirs(dirinstall_path)
else:
args += ["--image", disk_img]
# Create the sparse image
mksparse(disk_img, disk_size * 1024**2)
log_monitor = LogMonitor(timeout=opts.timeout)
args += ["--remotelog", "%s:%s" % (log_monitor.host, log_monitor.port)]
cancel_funcs = [log_monitor.server.log_check]
if cancel_func is not None:
cancel_funcs.append(cancel_func)
# Make sure anaconda has the right product and release
log.info("Running anaconda.")
try:
unshare_args = [
"--pid", "--kill-child", "--mount", "--propagation", "unchanged",
"anaconda"
] + args
for line in execReadlines(
"unshare",
unshare_args,
reset_lang=False,
env_add={
"ANACONDA_PRODUCTNAME": opts.project,
"ANACONDA_PRODUCTVERSION": opts.releasever
},
callback=lambda p: not novirt_cancel_check(cancel_funcs, p)):
log.info(line)
# Make sure the new filesystem is correctly labeled
setfiles_args = [
"-e", "/proc", "-e", "/sys",
"/etc/selinux/targeted/contexts/files/file_contexts", "/"
]
if "--dirinstall" in args:
# setfiles may not be available, warn instead of fail
try:
execWithRedirect("setfiles",
setfiles_args,
root=dirinstall_path)
except (subprocess.CalledProcessError, OSError) as e:
log.warning("Running setfiles on install tree failed: %s",
str(e))
else:
with PartitionMount(disk_img) as img_mount:
if img_mount and img_mount.mount_dir:
try:
execWithRedirect("setfiles",
setfiles_args,
root=img_mount.mount_dir)
except (subprocess.CalledProcessError, OSError) as e:
log.warning(
"Running setfiles on install tree failed: %s",
str(e))
# For image installs, run fstrim to discard unused blocks. This way
# unused blocks do not need to be allocated for sparse image types
execWithRedirect("fstrim", [img_mount.mount_dir])
except (subprocess.CalledProcessError, OSError) as e:
log.error("Running anaconda failed: %s", e)
raise InstallError("novirt_install failed")
finally:
log_monitor.shutdown()
# Move the anaconda logs over to a log directory
log_dir = os.path.abspath(os.path.dirname(opts.logfile))
log_anaconda = joinpaths(log_dir, "anaconda")
if not os.path.isdir(log_anaconda):
os.mkdir(log_anaconda)
for l in glob.glob("/tmp/*log") + glob.glob("/tmp/anaconda-tb-*"):
shutil.copy2(l, log_anaconda)
os.unlink(l)
# Make sure any leftover anaconda mounts have been cleaned up
if not anaconda_cleanup(dirinstall_path):
raise InstallError(
"novirt_install cleanup of anaconda mounts failed.")
if not opts.make_iso and not opts.make_fsimage and not opts.make_pxe_live:
dm_name = os.path.splitext(os.path.basename(disk_img))[0]
# Remove device-mapper for partitions and disk
log.debug("Removing device-mapper setup on %s", dm_name)
for d in sorted(glob.glob("/dev/mapper/" + dm_name + "*"),
reverse=True):
dm_detach(d)
log.debug("Removing loop device for %s", disk_img)
loop_detach("/dev/" + get_loop_name(disk_img))
# qemu disk image is used by bare qcow2 images and by Vagrant
if opts.image_type:
log.info("Converting %s to %s", disk_img, opts.image_type)
qemu_args = []
for arg in opts.qemu_args:
qemu_args += arg.split(" ", 1)
# convert the image to the selected format
if "-O" not in qemu_args:
qemu_args.extend(["-O", opts.image_type])
qemu_img = tempfile.mktemp(prefix="lmc-disk-", suffix=".img")
execWithRedirect("qemu-img",
["convert"] + qemu_args + [disk_img, qemu_img],
raise_err=True)
if not opts.make_vagrant:
execWithRedirect("mv", ["-f", qemu_img, disk_img], raise_err=True)
else:
# Take the new qcow2 image and package it up for Vagrant
compress_args = []
for arg in opts.compress_args:
compress_args += arg.split(" ", 1)
vagrant_dir = tempfile.mkdtemp(prefix="lmc-tmpdir-")
metadata_path = joinpaths(vagrant_dir, "metadata.json")
execWithRedirect(
"mv", ["-f", qemu_img,
joinpaths(vagrant_dir, "box.img")],
raise_err=True)
if opts.vagrant_metadata:
shutil.copy2(opts.vagrant_metadata, metadata_path)
else:
create_vagrant_metadata(metadata_path)
update_vagrant_metadata(metadata_path, disk_size)
if opts.vagrantfile:
shutil.copy2(opts.vagrantfile,
joinpaths(vagrant_dir, "vagrantfile"))
log.info("Creating Vagrant image")
rc = mktar(vagrant_dir,
disk_img,
opts.compression,
compress_args,
selinux=False)
if rc:
raise InstallError("novirt_install mktar failed: rc=%s" % rc)
shutil.rmtree(vagrant_dir)
elif opts.make_tar:
compress_args = []
for arg in opts.compress_args:
compress_args += arg.split(" ", 1)
rc = mktar(dirinstall_path, disk_img, opts.compression, compress_args)
shutil.rmtree(dirinstall_path)
if rc:
raise InstallError("novirt_install mktar failed: rc=%s" % rc)
elif opts.make_oci:
# An OCI image places the filesystem under /rootfs/ and adds the json files at the top
# And then creates a tar of the whole thing.
compress_args = []
for arg in opts.compress_args:
compress_args += arg.split(" ", 1)
shutil.copy2(opts.oci_config, ROOT_PATH)
shutil.copy2(opts.oci_runtime, ROOT_PATH)
rc = mktar(ROOT_PATH, disk_img, opts.compression, compress_args)
if rc:
raise InstallError("novirt_install mktar failed: rc=%s" % rc)
else:
# For raw disk images, use fallocate to deallocate unused space
execWithRedirect("fallocate", ["--dig-holes", disk_img],
raise_err=True)
