def __init__(self, devices=None):
"""Operate on mounted Linux systems
Accepts a collection of devices to operate upon
"""
debug(f"Initiating LinuxWorker with devices: {devices}")
self.devices = devices
# - property value placeholders -
# This pattern should help to prevent repeated system queries and improve debug clarity
self._was_root_mounted = None # Bool
self._fdisk_partitions = []
self._lvm_pvs = []
self._lvm_lvs = {}
self._blkid = {}
self._data_volumes = []
self._root_volume = ""
self._boot_partition_is_on_root_volume = False
self._has_run_dir = None # boolean when set
self._fstab = []
self._boot_volume = ""
self._boot_mode = ""
self.debug_task = [
] # Keeps track of current state for troubleshooting
# - constants -
self.MOUNT_BASE = "/convert"
self.ROOT_MOUNT = f"{self.MOUNT_BASE}/root"
# init
self._was_root_mounted = self.was_root_mounted
def blkid(self):
"""Return the blkid output of each device in devices
{"<device>": {"UUID": "<UUID">", "TYPE": "<TYPE>"}
blkid is used to get the filesystem and UUID of a block device
"""
if self._blkid:
return self._blkid
self.debug_action(action="GET BLKID DATA")
_blkid = {}
blkid_lines = run("blkid")
def blkid_val(line, prop):
""" Return the blkid property value if present else none """
word = next(
(elem for elem in line.split(" ") if f"{prop}=" in elem), None)
if word:
return word.split("=")[1].replace('"', "")
return None
for line in blkid_lines:
split = line.split(" ")
if not split[0]:
continue
path = split[0].replace(":", "")
uuid = blkid_val(line, "UUID")
type_ = blkid_val(line, "TYPE")
_blkid[path] = {"UUID": uuid, "TYPE": type_}
self._blkid = _blkid
for device in _blkid:
debug(f"{device}: {_blkid[device]}")
self.debug_action(end=True)
return _blkid
def lvm_lvs(self):
"""Return a dict of of LVM logical volumes on the given devices
{"<device mapper path>": { name: "<name>", devices: [<partitions/PVs>] }}
"""
if self._lvm_lvs:
return self._lvm_lvs
self.debug_action(action="FIND LVM LV's")
lvs = {}
for lvm_pv in self.lvm_pvs:
pv_lv_lines = grep(f"pvdisplay -m {lvm_pv}", "Logical volume")
# pv_lv_lines looks like this: [' Logical volume\t/dev/vg_rhel610/lv_root']
for pv_lv in pv_lv_lines:
# example's name is /dev/vg_rhel610/lv_root
name = pv_lv.strip().split("\t")[1]
dm_path = run(f"lvdisplay -C -o lv_dm_path {name}")[1].strip()
if dm_path not in lvs:
# First time encountering this LV
lvs[dm_path] = {"name": name, "devices": [lvm_pv]}
else:
# This LV was in a prior device, just add this device to it
lvs[dm_path]["devices"].append(lvm_pv)
self._lvm_lvs = lvs
debug(f"lvs: {list(lvs)}")
self.debug_action(end=True)
return lvs
def unmount_volumes(self, prompt=False, print_progress=False):
""" Unmount the /etc/fstab and device volumes from the chroot root dir """
self.debug_action(action="UNMOUNT ALL VOLUMES")
for mount_opts in self.get_ordered_mount_opts(reverse=True):
debug(f"Unmount: {mount_opts['mnt_to']}")
if print_progress:
print(f"umount {mount_opts['mnt_to']}")
unmount(mount_opts["mnt_to"], prompt=prompt, fail=prompt)
self.debug_action(end=True)
def set_netplan_interface(
self,
interface_name,
is_dhcp,
mac_addr,
ip_addr=None,
prefix=None,
gateway=None,
dns=(),
domain=None,
):
""" Deploy a netplan styled interface file """
netplan_dir_path = f"{self.ROOT_MOUNT}/etc/netplan"
netplan_file_path = f"{netplan_dir_path}/{interface_name}.yaml"
self.debug_action(action=f"SET NETPLAN FILE: {netplan_file_path}")
# check the yaml files in the netplan dir to see if this interface is already defined
dir_files = os.listdir(netplan_dir_path)
yaml_files = [fil for fil in dir_files if fil.endswith(".yaml")]
# make sure this interface is not defined elsewhere
for filename in yaml_files:
file_path = f"{netplan_dir_path}/{filename}"
debug(f"Ensuring {interface_name} is not in {file_path}")
if f"{interface_name}:" in get_file_contents(file_path):
error(f"ERROR: {interface_name} found in {file_path}",
exit=True)
# write the new yaml file
iface_lines = [f"network:", f" ethernets:", f" {interface_name}:"]
if ip_addr and prefix:
iface_lines.append(f" dhcp4: no")
iface_lines.append(f" addresses: [\"{ip_addr}/{prefix}\"]")
if gateway:
iface_lines.append(f" gateway4: {gateway}")
if dns:
nameservers = ', '.join('"{0}"'.format(entry) for entry in dns)
iface_lines.append(f" nameservers:")
iface_lines.append(f" addresses: [{nameservers}]")
if domain:
iface_lines.append(f" search: [\"{domain}\"]")
elif is_dhcp:
iface_lines.append(f" dhcp4: yes")
with open(netplan_file_path, "a+") as iface_file:
for line in iface_lines:
iface_file.write(line + "\n")
print(f"# {netplan_file_path}")
with open(netplan_file_path, "r") as iface_file:
print(iface_file.read())
self.debug_action(end=True)
def lvm_pvs(self):
""" Return a list of physical volumes (partitions) from LVM that match given devices """
if self._lvm_pvs:
return self._lvm_pvs
self.debug_action(action="FIND LVM PV's")
pvs = []
pvs_lines = run("pvs")
for line in pvs_lines:
partition = line.strip().split(" ")[0]
if "/dev/" not in partition:
debug(f"Skipping {partition} - does not contain /dev")
continue
if partition in self.fdisk_partitions:
pvs.append(partition)
self._lvm_pvs = pvs
self.debug_action(end=True)
return pvs
def set_ifupdown_interface(
self,
interface_name,
is_dhcp,
mac_addr,
ip_addr=None,
prefix=None,
gateway=None,
dns=(),
domain=None,
):
""" Deploy an ifupdown styled interface file """
debug("Setting ifupdown file")
path = f"{self.ROOT_MOUNT}/etc/network/interfaces"
interfaces_contents = get_file_contents(path)
if interface_name in interfaces_contents:
error(
f"ERROR: {interface_name} is already in {path} - cannot continue",
exit=True)
self.debug_action(action=f"SET IFUPDOWN FILE: {path}")
mode = "dhcp" if is_dhcp else "static"
iface_lines = [
f"auto {interface_name}", f"iface {interface_name} inet {mode}"
]
if ip_addr and prefix:
iface_lines.append(f" address {ip_addr}")
ip_obj = IPv4Interface(f"{ip_addr}/{prefix}")
iface_lines.append(f" netmask {ip_obj.netmask}")
if gateway:
iface_lines.append(f" gateway {gateway}")
if dns:
nameservers = " ".join(dns)
iface_lines.append(f" dns-nameservers {nameservers}")
if domain:
iface_lines.append(f" dns-search {domain}")
print(f"Writing: {path}")
with open(path, "a+") as iface_file:
iface_file.write("\n")
for line in iface_lines:
iface_file.write(line + "\n")
with open(path, "r") as iface_file:
print(iface_file.read())
self.debug_action(end=True)
def data_volumes(self):
"""Return a list of valid data volume paths:
Physical (fdisk) partitions that are not LVM PV's, and LVM LVs - no swap
"""
if self._data_volumes:
return self._data_volumes
self.debug_action(action="FIND DATA VOLUMES")
data_volumes = []
all_vols = [vol for vol in self.fdisk_partitions if vol in self.blkid
] + list(self.lvm_lvs.keys())
debug(f"All volumes: {all_vols}")
# if TYPE is swap, you can't mount it
# if TYPE is None (false), cant mount - could be Ubuntu's BIOS BOOT partition
self._data_volumes = [
vol for vol in all_vols if not vol in self.lvm_pvs and vol
and self.blkid[vol]["TYPE"] != "swap" and self.blkid[vol]["TYPE"]
]
debug(f"Data volumes: {self._data_volumes}")
self.debug_action(end=True)
return self._data_volumes
def fdisk_partitions(self):
""" return list of partitions on devices """
if self._fdisk_partitions:
return self._fdisk_partitions
self.debug_action(action="FIND FDISK PARTITIONS")
partitions = []
if not self.devices:
error(
"ERROR: Cannot list partitions when devices are not specified",
exit=True)
for device in self.devices:
fdisk = run(f"fdisk -l {device}")
partition_lines = (line for line in fdisk
if line.startswith(device))
for partition_line in partition_lines:
partitions.append(partition_line.split(" ")[0])
self._fdisk_partitions = partitions
debug(f"fdisk_partitions: {partitions}")
self.debug_action(end=True)
return partitions
def debug_action(self, action=None, end=False):
""" Write a debug message tracking what's going on here """
if end:
breadcrumbs = " > ".join(self.debug_task)
debug(f"---- DONE: {breadcrumbs}")
self.debug_task.pop()
if self.debug_task:
breadcrumbs = " > ".join(self.debug_task)
debug(f"---- CONT: {breadcrumbs}")
else:
debug("---- DONE!")
else:
self.debug_task.append(action.upper())
breadcrumbs = " > ".join(self.debug_task)
debug(f"---- START: {breadcrumbs}")
def mount_volumes(self, print_progress=False):
""" Mount the /etc/fstab and device volumes into the chroot root dir """
self.debug_action(action="MOUNT ALL VOLUMES")
# Mount the root volume
debug("Collect the ordered mount options")
ordered_mount_opts = self.get_ordered_mount_opts() # unmounts root
debug(f"Mounting root volume {self.root_volume} to {self.ROOT_MOUNT}")
mount(self.root_volume, self.ROOT_MOUNT)
if print_progress:
print(f"mount {self.root_volume} {self.ROOT_MOUNT}")
# Mount the other volumes
for mount_opts in ordered_mount_opts:
if mount_opts["mnt_to"] == self.ROOT_MOUNT:
continue
if print_progress:
bind = "--bind" if mount_opts["bind"] else ""
print(
f"mount {mount_opts['mnt_from']} {mount_opts['mnt_to']} {bind}"
)
mount(mount_opts["mnt_from"],
mount_opts["mnt_to"],
bind=mount_opts["bind"])
self.debug_action(end=True)
def set_interface(
self,
interface_name,
is_dhcp,
mac_addr,
ip_addr=None,
prefix=None,
gateway=None,
dns=(),
domain=None,
):
""" Find the interface type (netplan/ifupdown) then deploy the file """
if not is_mounted(self.ROOT_MOUNT):
error(
f"ERROR: Cannot set interface file, {self.ROOT_MOUNT} is not mounted",
exit=True)
if Path(f"{self.ROOT_MOUNT}/etc/netplan").exists():
# This is (probably) a newer Ubuntu OS that uses Netplan
debug("Using Netplan")
self.set_netplan_interface(interface_name,
is_dhcp,
mac_addr,
ip_addr=ip_addr,
prefix=prefix,
gateway=gateway,
dns=dns,
domain=domain)
else:
# This is (probably) an older Ubuntu OS that uses ifupdown
self.set_ifupdown_interface(interface_name,
is_dhcp,
mac_addr,
ip_addr=ip_addr,
prefix=prefix,
gateway=gateway,
dns=dns,
domain=domain)
def add_virtio_drivers(self, force=False):
""" Install VirtIO drivers to mounted system """
if not self.was_root_mounted:
error("ERROR: You must mount the volumes before you can add virtio drivers", exit=True)
self.debug_action(action="ADD VIRTIO DRIVERS")
ls_boot_lines = run(f"ls {self.ROOT_MOUNT}/boot")
initram_lines = [
line
for line in ls_boot_lines
if line.startswith("initramfs-") and line.endswith(".img") and "dump" not in line
]
for filename in initram_lines:
if "rescue" in filename or "kdump" in filename:
debug(f"Skipping rescue/kdump file: {filename}")
continue
kernel_version = filename.replace("initramfs-", "").replace(".img", "")
debug("Running lsinitrd to check for virtio drivers")
lsinitrd = self.chroot_run(f"lsinitrd /boot/{filename}")
virtio_line = next((line for line in lsinitrd if "virtio" in line.lower()), None)
if virtio_line is not None:
print(f"{filename} already has virtio drivers")
if force:
print("force=true, reinstalling")
else:
continue
print(f"Adding virtio drivers to {filename}")
drivers = "virtio_blk virtio_net virtio_scsi virtio_balloon"
cmd = f'dracut --add-drivers "{drivers}" -f /boot/{filename} {kernel_version}'
# Python+chroot causes dracut space delimiter to break - use a script file
script_file = f"{self.ROOT_MOUNT}/virtio.sh"
debug(f"writing script file: {script_file}")
debug(f"script file contents: {cmd}")
set_file_contents(script_file, cmd)
self.chroot_run("bash /virtio.sh")
debug(f"deleting script file: {script_file}")
os.remove(script_file)
self.debug_action(end=True)
def root_volume(self):
""" Return the path to the volume: the volume that contains /etc/fstab """
if self._root_volume:
# self._root_volume can be set here or during __init__()
return self._root_volume
self.debug_action(action="FIND ROOT VOLUME")
fstab_path = f"{self.ROOT_MOUNT}/etc/fstab"
if is_mounted(self.ROOT_MOUNT):
# something's already mounted to ROOT_MOUNT, validate it
fstab_contents = get_file_contents(fstab_path)
if not fstab_contents:
error("ERROR: Mounted root volume has no /etc/fstab",
exit=True)
device = get_mount(self.ROOT_MOUNT)["device"]
self._root_volume = device
debug(device)
self.debug_action(end=True)
return device
if self.devices is None:
error(
f"ERROR: Failed to find root partition - no devices specified",
exit=True)
_root_volume = None
# root volume wasn't mounted, mount each data volume until you find /etc/fstab
for vol_path in self.data_volumes:
debug(f"Checking for /etc/fstab in {vol_path}")
try:
mount(vol_path, self.ROOT_MOUNT)
if get_file_contents(fstab_path):
self._root_volume = vol_path
unmount(self.ROOT_MOUNT)
_root_volume = vol_path
break
finally:
unmount(self.ROOT_MOUNT)
debug(f"> root volume = {_root_volume}")
self.debug_action(end=True)
if _root_volume is None:
error(
f"ERROR: Failed to find a root volume on devices: {self.devices}",
exit=True)
self._root_volume = _root_volume
return _root_volume
def fstab(self):
"""Return the parsed content of the root volume's /etc/fstab file.
Parses UUIDs into device paths, quits with an error if that fails.
Return value is a list of dicts with the following keys:
- path
- mountpoint
- fstype
- options
"""
if self._fstab:
return self._fstab
self.debug_action(action="PARSE FSTAB")
_fstab = []
try:
if not self.was_root_mounted:
self.mount_root()
fstab_lines = get_file_contents(
f"{self.ROOT_MOUNT}/etc/fstab").replace("\t", "")
debug("/etc/fstab contents:")
debug(fstab_lines)
for line in fstab_lines.split("\n"):
# Skip comments, swap tabs with spaces
line = line.strip().replace("\t", "")
if line.startswith("#"):
continue
split = [word for word in line.split(" ") if word]
if len(split) < 3:
continue
path = split[0]
if path.startswith("UUID="):
uuid = path.split("=")[1]
debug(f"fstab line has UUID: {uuid}")
debug(line)
path = next((path for path in self.blkid
if self.blkid[path]["UUID"] == uuid), None)
if path is None:
error(
f"ERROR: Failed to find path to fstab UUID in {line}",
exit=True)
debug(f"Mapped UUID {uuid} to device path: {path}")
elif not path.startswith("/"):
debug(f"Skipping /etc/fstab system path: {path}")
continue
_fstab.append({
"path": path,
"mountpoint": split[1],
"fstype": split[2],
"options": split[3] if len(split) > 3 else "",
})
finally:
if not self.was_root_mounted:
self.unmount_root()
self.debug_action(end=True)
self._fstab = _fstab
return _fstab
def __init__(self, devices=None):
"""Operate on mounted RedHat systems
Accepts a collection of devices to operate upon
"""
debug(f"Initiating RhelWorker with devices: {devices}")
super().__init__(devices=devices)
def get_ordered_mount_opts(self, reverse=False):
"""Return the order of volumes to be mounted/unmounted, in the order ftab returned them
This is the lengthy logic where the /etc/fstab file gets parsed out
Returns list of dicts with these keys:
{ "mnt_from": "<path>", "mnt_to": "<path>", "bind": <bool> }
"""
self.debug_action(action="GET ORDERED MOUNT OPTIONS")
mount_opts = []
try:
if not self.was_root_mounted:
self.mount_root()
mountpoints = [
entry["mountpoint"] for entry in self.fstab
if entry["mountpoint"] != "swap"
and entry["mountpoint"].startswith("/")
]
for mpoint in mountpoints:
fstab_entry = next(entry for entry in self.fstab
if entry["mountpoint"] == mpoint)
if fstab_entry["mountpoint"] == "/":
# Handle root mount differently - it goes to ROOT_MOUNT and doesn't have a bind
mount_opts.append({
"mnt_from": fstab_entry["path"],
"mnt_to": self.ROOT_MOUNT,
"bind": False
})
continue
debug(f"FSTAB ENTRY: {fstab_entry}")
if "bind" not in fstab_entry["options"]:
device = fstab_entry["path"]
# Before vol can be mounted to the chroot it needs to be mounted to the worker
# the sys_mountpoint of /var/tmp would be /convert/var_tmp
subpath = fstab_entry["mountpoint"][1:].replace("/", "_")
sys_mountpoint = f"{self.MOUNT_BASE}/{subpath}"
mount_opts.append({
"mnt_from": fstab_entry["path"],
"mnt_to": sys_mountpoint,
"bind": False
})
# then bind-mind the volume into the chroot (remove first char / from mpoint)
chroot_bind_path = f"{self.ROOT_MOUNT}/{fstab_entry['mountpoint'][1:]}"
mount_opts.append({
"mnt_from": sys_mountpoint,
"mnt_to": chroot_bind_path,
"bind": True
})
else:
# This is a bind mount, so just link the dirs in the chroot
chroot_src = f"{self.ROOT_MOUNT}/{fstab_entry['path']}"
chroot_dest = f"{self.ROOT_MOUNT}/{fstab_entry['mountpoint']}"
mount_opts.append({
"mnt_from": chroot_src,
"mnt_to": chroot_dest,
"bind": True
})
devpaths = ["/sys", "/proc", "/dev"]
if self._has_run_dir:
devpaths.append("/run")
for devpath in devpaths:
chroot_devpath = f"{self.ROOT_MOUNT}{devpath}"
mount_opts.append({
"mnt_from": devpath,
"mnt_to": chroot_devpath,
"bind": True
})
finally:
if not self.was_root_mounted:
self.unmount_root()
if reverse:
mount_opts.reverse()
self.debug_action(end=True)
return mount_opts
