def _disable_lvm2_lvmetad(ssh):
"""Disables lvm2-lvmetad service. This service is responsible
for automatically activating LVM2 volume groups when a disk is
attached or when a volume group is created. During disk replication
this service needs to be disabled.
"""
cfg = "/etc/lvm/lvm.conf"
if utils.test_ssh_path(ssh, cfg):
utils.exec_ssh_cmd(
ssh,
'sudo sed -i "s/use_lvmetad.*=.*1/use_lvmetad = 0/g" '
'%s' % cfg, get_pty=True)
# NOTE: lvm2-lvmetad is the name of the lvmetad service
# on both debian and RHEL based systems. It needs to be stopped
# before we begin disk replication. We disable it in the config
# just in case some other process starts the daemon later on, as
# a dependency. As the service may not actually exist, even though
# the config is present, we ignore errors when stopping it.
utils.ignore_exceptions(utils.exec_ssh_cmd)(
ssh, "sudo service lvm2-lvmetad stop", get_pty=True)
# disable volume groups. Any volume groups that have volumes in use
# will remain online. However, volume groups belonging to disks
# that have been synced at least once, will be deactivated.
utils.ignore_exceptions(utils.exec_ssh_cmd)(
ssh, "sudo vgchange -an", get_pty=True)
def _find_and_mount_root(self, devices):
files = ["etc", "bin", "sbin", "boot"]
os_root_dir = None
os_root_device = self._find_dev_with_contents(devices, all_files=files)
if os_root_device is None:
raise exception.OperatingSystemNotFound(
"Coriolis was unable to identify the root partition of the OS "
"being migrated for mounting during OSMorphing. Please ensure "
"that the source VM's root partition(s) are not encrypted, "
"and that they are using a filesystem type and version which "
"is supported by the OS images used for the OSMorphing minion "
"machine. Also ensure that the source VM's mountpoint "
"declarations in '/etc/fstab' are all correct, and are "
"declared using '/dev/disk/by-uuid/' or 'UUID=' notation. "
"If all else fails, please retry while using an OSMorphing "
"minion machine image which is the same OS release as the VM "
"being migrated.")
try:
tmp_dir = self._exec_cmd('mktemp -d').decode().splitlines()[0]
self._exec_cmd('sudo mount %s %s' % (os_root_device, tmp_dir))
os_root_dir = tmp_dir
except Exception as ex:
self._event_manager.progress_update(
"Exception occurred while Coriolis was attempting to mount the"
" root device (%s) of the OS being migrated for OSMorphing. "
"Please ensure that the source VM's root partition(s) are "
"using a filesystem type and version which is supported by the"
" OS images used for the OSMorphing minion machine. Also "
"ensure that the source VM's mountpoint declarations in "
"'/etc/fstab' are all correct, and are declared using "
"'/dev/disk/by-uuid/' or 'UUID=' notation. If all else fails, "
"please retry while using an OSMorphing minion machine image "
"which is the same OS release as the VM being migrated. Error "
"was: %s" % (os_root_device, str(ex)))
LOG.error(ex)
LOG.warn("Failed to mount root device '%s':\n%s", os_root_device,
utils.get_exception_details())
utils.ignore_exceptions(self._exec_cmd)("sudo umount %s" % tmp_dir)
utils.ignore_exceptions(self._exec_cmd)("sudo rmdir %s" % tmp_dir)
raise
for directory in ['proc', 'sys', 'dev', 'run']:
mount_dir = os.path.join(os_root_dir, directory)
if not utils.test_ssh_path(self._ssh, mount_dir):
LOG.info("No '%s' directory in mounted OS. Skipping mount.",
directory)
continue
self._exec_cmd('sudo mount -o bind /%(dir)s/ %(mount_dir)s' % {
'dir': directory,
'mount_dir': mount_dir
})
if os_root_device in devices:
devices.remove(os_root_device)
return os_root_dir, os_root_device
def _find_and_mount_root(self, devices):
files = ["etc", "bin", "sbin", "boot"]
os_root_dir = None
os_root_device = self._find_dev_with_contents(
devices, all_files=files)
if os_root_device is None:
raise exception.OperatingSystemNotFound(
"root partition not found")
try:
tmp_dir = self._exec_cmd('mktemp -d').decode().splitlines()[0]
self._exec_cmd('sudo mount %s %s' % (os_root_device, tmp_dir))
os_root_dir = tmp_dir
except Exception:
self._event_manager.progress_update(
"Failed to mount root device '%s'" % os_root_device)
LOG.warn(
"Failed to mount root device '%s':\n%s",
os_root_device, utils.get_exception_details())
utils.ignore_exceptions(self._exec_cmd)(
"sudo umount %s" % tmp_dir
)
utils.ignore_exceptions(self._exec_cmd)(
"sudo rmdir %s" % tmp_dir
)
raise
for directory in ['proc', 'sys', 'dev', 'run']:
mount_dir = os.path.join(os_root_dir, directory)
if not utils.test_ssh_path(self._ssh, mount_dir):
LOG.info(
"No '%s' directory in mounted OS. Skipping mount.",
directory)
continue
self._exec_cmd(
'sudo mount -o bind /%(dir)s/ %(mount_dir)s' %
{'dir': directory, 'mount_dir': mount_dir})
if os_root_device in devices:
devices.remove(os_root_device)
return os_root_dir, os_root_device
def _setup_certificates(self, ssh):
remote_base_dir = "/etc/coriolis-writer"
ca_crt_name = "ca-cert.pem"
client_crt_name = "client-cert.pem"
client_key_name = "client-key.pem"
srv_crt_name = "srv-cert.pem"
srv_key_name = "srv-key.pem"
remote_ca_crt = os.path.join(remote_base_dir, ca_crt_name)
remote_client_crt = os.path.join(remote_base_dir, client_crt_name)
remote_client_key = os.path.join(remote_base_dir, client_key_name)
remote_srv_crt = os.path.join(remote_base_dir, srv_crt_name)
remote_srv_key = os.path.join(remote_base_dir, srv_key_name)
exist = []
for i in (remote_ca_crt, remote_client_crt, remote_client_key,
remote_srv_crt, remote_srv_key):
exist.append(utils.test_ssh_path(ssh, i))
if not all(exist):
utils.exec_ssh_cmd(ssh,
"sudo mkdir -p %s" % remote_base_dir,
get_pty=True)
utils.exec_ssh_cmd(
ssh,
"sudo %(writer_cmd)s generate-certificates -output-dir "
"%(cert_dir)s -certificate-hosts %(extra_hosts)s" % {
"writer_cmd": self._writer_cmd,
"cert_dir": remote_base_dir,
"extra_hosts": self._ip,
},
get_pty=True)
return {
"srv_crt": remote_srv_crt,
"srv_key": remote_srv_key,
"ca_crt": remote_ca_crt,
"client_crt": remote_client_crt,
"client_key": remote_client_key
}
def _setup_certificates(self, ssh, args):
# TODO(gsamfira): coriolis-replicator and coriolis-writer share
# the functionality of being able to generate certificates
# This will either be replaced with proper certificate management
# in Coriolis, and the needed files will be pushed to the services
# that need them (userdata or ssh), or the two applications will be
# merged into one, and we will deduplicate this functionallity.
remote_base_dir = REPLICATOR_DIR
ip = args["ip"]
ca_crt_name = "ca-cert.pem"
client_crt_name = "client-cert.pem"
client_key_name = "client-key.pem"
srv_crt_name = "srv-cert.pem"
srv_key_name = "srv-key.pem"
remote_ca_crt = os.path.join(remote_base_dir, ca_crt_name)
remote_client_crt = os.path.join(remote_base_dir, client_crt_name)
remote_client_key = os.path.join(remote_base_dir, client_key_name)
remote_srv_crt = os.path.join(remote_base_dir, srv_crt_name)
remote_srv_key = os.path.join(remote_base_dir, srv_key_name)
ca_crt = os.path.join(self._cert_dir, ca_crt_name)
client_crt = os.path.join(self._cert_dir, client_crt_name)
client_key = os.path.join(self._cert_dir, client_key_name)
exist = []
for i in (remote_ca_crt, remote_client_crt, remote_client_key,
remote_srv_crt, remote_srv_key):
exist.append(utils.test_ssh_path(ssh, i))
force_fetch = False
if not all(exist):
utils.exec_ssh_cmd(ssh,
"sudo mkdir -p %s" % remote_base_dir,
get_pty=True)
utils.exec_ssh_cmd(
ssh,
"sudo %(replicator_cmd)s gen-certs -output-dir "
"%(cert_dir)s -certificate-hosts %(extra_hosts)s" % {
"replicator_cmd": REPLICATOR_PATH,
"cert_dir": remote_base_dir,
"extra_hosts": ip,
},
get_pty=True)
utils.exec_ssh_cmd(
ssh,
"sudo chown -R %(user)s:%(group)s %(cert_dir)s" % {
"cert_dir": remote_base_dir,
"user": REPLICATOR_USERNAME,
"group": REPLICATOR_GROUP_NAME
},
get_pty=True)
utils.exec_ssh_cmd(ssh,
"sudo chmod -R g+r %(cert_dir)s" % {
"cert_dir": remote_base_dir,
},
get_pty=True)
force_fetch = True
exists = []
for i in (ca_crt, client_crt, client_key):
exists.append(os.path.isfile(i))
if not all(exists) or force_fetch:
# certificates either are missing, or have been regenerated
# on the writer worker. We need to fetch them.
self._fetch_remote_file(ssh, remote_ca_crt, ca_crt)
self._fetch_remote_file(ssh, remote_client_crt, client_crt)
self._fetch_remote_file(ssh, remote_client_key, client_key)
return {
"local": {
"client_cert": client_crt,
"client_key": client_key,
"ca_cert": ca_crt,
},
"remote": {
"srv_crt": remote_srv_crt,
"srv_key": remote_srv_key,
"ca_crt": remote_ca_crt,
},
}
def _test_path(self, chroot_path):
path = os.path.join(self._os_root_dir, chroot_path)
return utils.test_ssh_path(self._ssh, path)
def _check_mount_fstab_partitions(self,
os_root_dir,
skip_mounts=["/", "/boot"],
skip_filesystems=["swap"],
mountable_lvm_devs=None):
""" Reads the contents of /etc/fstab from the VM's root directory and
tries to mount all clearly identified (by UUID or path) filesystems.
Returns the list of the new directories which were mounted.
param: skip_mounts: list(str()): list of directories (inside the
chroot) to not try to mount.
param: skip_filesystems: list(str()): list of filesystem types to skip
mounting entirely
param: mountable_lvm_devs: list(str()): list of LVM device paths which
exist and are mountable should they appear in /etc/fstab
"""
new_mountpoints = []
etc_fstab_path = os.path.join(os_root_dir, "etc/fstab")
if not utils.test_ssh_path(self._ssh, etc_fstab_path):
LOG.warn(
"etc/fstab file not found in '%s'. Cannot mount non-root dirs",
os_root_dir)
return []
etc_fstab_raw = utils.read_ssh_file(self._ssh, etc_fstab_path)
etc_fstab = etc_fstab_raw.decode('utf-8')
LOG.debug("Mounting non-root partitions from fstab file: %s" %
(base64.b64encode(etc_fstab_raw)))
# dictionary of the form {"mountpoint":
# {"device": "<dev>", "filesystem": "<fs>", "options": "<opts>"}}
mounts = {}
# fstab entry format:
# <device> <mountpoint> <filesystem> <options> <dump> <fsck>
fstab_entry_regex = (
"^(\s*([^#\s]+)\s+(\S+)\s+(\S+)\s+(\S+)(\s+(\d)(\s+(\d))?)?\s*)$")
for line in etc_fstab.splitlines():
match = re.match(fstab_entry_regex, line)
if not match:
LOG.warn("Skipping unparseable /etc/fstab line: '%s'", line)
continue
device = match.group(2)
mountpoint = match.group(3)
if mountpoint in mounts:
raise exception.CoriolisException(
"Mountpoint '%s' appears to be mounted twice in "
"'/etc/fstab'" % mountpoint)
mounts[mountpoint] = {
"device": device,
"filesystem": match.group(4),
"options": match.group(5)
}
# NOTE: we sort the mountpoints based on length to ensure
# they get mounted in the correct order:
mounts = collections.OrderedDict(
(mountpoint, mounts[mountpoint])
for mountpoint in sorted(mounts, key=len))
# regexes for supported fstab device references:
uuid_char_regex = "[0-9a-fA-F]"
fs_uuid_regex = ("%(char)s{8}-%(char)s{4}-%(char)s{4}-"
"%(char)s{4}-%(char)s{12}") % {
"char": uuid_char_regex
}
fs_uuid_entry_regex = "^(UUID=%s)$" % fs_uuid_regex
by_uuid_entry_regex = "^(/dev/disk/by-uuid/%s)$" % fs_uuid_regex
if not mountable_lvm_devs:
mountable_lvm_devs = []
device_paths = self._get_device_file_paths(mountable_lvm_devs)
for (mountpoint, details) in mounts.items():
device = details['device']
if (re.match(fs_uuid_entry_regex, device) is None
and re.match(by_uuid_entry_regex, device) is None):
device_file_path = self._get_symlink_target(device)
if device not in mountable_lvm_devs and (device_file_path
not in device_paths):
LOG.warn(
"Found fstab entry for dir %s which references device "
"%s. Only LVM volumes or devices referenced by UUID=* "
"or /dev/disk/by-uuid/* notation are supported. "
"Devicemapper paths for LVM volumes are also "
"supported. Skipping mounting directory." %
(mountpoint, device))
continue
if mountpoint in skip_mounts:
LOG.debug("Skipping undesired mount: %s: %s", mountpoint,
details)
continue
if details["filesystem"] in skip_filesystems:
LOG.debug("Skipping mounting undesired FS for device %s: %s",
device, details)
continue
LOG.debug("Attempting to mount fstab device: %s: %s", device,
details)
# NOTE: `mountpoint` should always be an absolute path:
chroot_mountpoint = "%s%s" % (os_root_dir, mountpoint)
mountcmd = "sudo mount -t %s -o %s %s '%s'" % (
details["filesystem"], details["options"], device,
chroot_mountpoint)
try:
self._exec_cmd(mountcmd)
new_mountpoints.append(chroot_mountpoint)
except Exception:
LOG.warn(
"Failed to run fstab filesystem mount command: '%s'. "
"Skipping mount. Error details: %s", mountcmd,
utils.get_exception_details())
if new_mountpoints:
LOG.info(
"The following new /etc/fstab entries were successfully "
"mounted: %s", new_mountpoints)
return new_mountpoints
def _test_path(self, chroot_path):
path = os.path.join(self._os_root_dir, chroot_path)
return utils.test_ssh_path(self._ssh, path)
def _setup_certificates(self, ssh):
remote_base_dir = "/etc/coriolis-writer"
ca_crt_name = "ca-cert.pem"
client_crt_name = "client-cert.pem"
client_key_name = "client-key.pem"
srv_crt_name = "srv-cert.pem"
srv_key_name = "srv-key.pem"
remote_ca_crt = os.path.join(remote_base_dir, ca_crt_name)
remote_client_crt = os.path.join(remote_base_dir, client_crt_name)
remote_client_key = os.path.join(remote_base_dir, client_key_name)
remote_srv_crt = os.path.join(remote_base_dir, srv_crt_name)
remote_srv_key = os.path.join(remote_base_dir, srv_key_name)
ca_crt = os.path.join(self._crt_dir, ca_crt_name)
client_crt = os.path.join(self._crt_dir, client_crt_name)
client_key = os.path.join(self._crt_dir, client_key_name)
exist = []
for i in (remote_ca_crt, remote_client_crt, remote_client_key,
remote_srv_crt, remote_srv_key):
exist.append(utils.test_ssh_path(ssh, i))
force_fetch = False
if not all(exist):
utils.exec_ssh_cmd(ssh,
"sudo mkdir -p %s" % remote_base_dir,
get_pty=True)
utils.exec_ssh_cmd(
ssh,
"sudo %(writer_cmd)s generate-certificates -output-dir "
"%(cert_dir)s -certificate-hosts %(extra_hosts)s" % {
"writer_cmd": self._writer_cmd,
"cert_dir": remote_base_dir,
"extra_hosts": self._ip,
},
get_pty=True)
force_fetch = True
exists = []
for i in (ca_crt, client_crt, client_key):
exists.append(os.path.isfile(i))
if not all(exists) or force_fetch:
# certificates either are missing, or have been regenerated
# on the writer worker. We need to fetch them.
self._fetch_remote_file(ssh, remote_ca_crt, ca_crt)
self._fetch_remote_file(ssh, remote_client_crt, client_crt)
self._fetch_remote_file(ssh, remote_client_key, client_key)
return {
"local": {
"client_crt": client_crt,
"client_key": client_key,
"ca_crt": ca_crt,
},
"remote": {
"srv_crt": remote_srv_crt,
"srv_key": remote_srv_key,
"ca_crt": remote_ca_crt,
},
}
def _test_path(self, chroot_path):
full_path = os.path.join(self._os_root_dir, chroot_path)
return utils.test_ssh_path(self._conn, full_path)
def mount_os(self):
dev_paths = []
mounted_devs = self._get_mounted_devices()
volume_devs = self._get_volume_block_devices()
for volume_dev in volume_devs:
self._exec_cmd("sudo partx -v -a %s || true" % volume_dev)
dev_paths += self._exec_cmd("sudo ls %s*" %
volume_dev).decode().split('\n')[:-1]
pvs = self._get_pvs()
for vg_name in self._get_vgnames():
found = False
for pv in pvs[vg_name]:
if pv in dev_paths:
found = True
break
if not found:
continue
self._exec_cmd("sudo vgchange -ay %s" % vg_name)
lvm_dev_paths = self._exec_cmd("sudo ls /dev/%s/*" %
vg_name).decode().split('\n')[:-1]
dev_paths += lvm_dev_paths
valid_filesystems = ['ext2', 'ext3', 'ext4', 'xfs', 'btrfs']
dev_paths_to_mount = []
for dev_path in dev_paths:
if dev_path in mounted_devs:
# this device is already mounted. Skip it, as it most likely
# means this device belongs to the worker VM.
continue
fs_type = self._exec_cmd("sudo blkid -o value -s TYPE %s || true" %
dev_path).decode().split('\n')[0]
if fs_type in valid_filesystems:
if fs_type == "xfs":
utils.run_xfs_repair(self._ssh, dev_path)
else:
utils.check_fs(self._ssh, fs_type, dev_path)
dev_paths_to_mount.append(dev_path)
os_boot_device = None
os_root_device = None
os_root_dir = None
for dev_path in dev_paths_to_mount:
dirs = None
tmp_dir = self._exec_cmd('mktemp -d').decode().split('\n')[0]
try:
self._exec_cmd('sudo mount %s %s' % (dev_path, tmp_dir))
# NOTE: it's possible that the device was mounted successfully
# but an I/O error occurs later along the line:
dirs = self._exec_cmd('ls %s' % tmp_dir).decode().split('\n')
except Exception:
self._event_manager.progress_update(
"Failed to mount and scan device '%s'" % dev_path)
LOG.warn("Failed to mount and scan device '%s':\n%s", dev_path,
utils.get_exception_details())
continue
LOG.debug("Contents of device %s:\n%s", dev_path, dirs)
# TODO(alexpilotti): better ways to check for a linux root?
if (not os_root_dir and 'etc' in dirs and 'bin' in dirs
and 'sbin' in dirs):
os_root_dir = tmp_dir
os_root_device = dev_path
LOG.info("OS root device: %s", dev_path)
continue
elif (not os_boot_device and ('grub' in dirs or 'grub2' in dirs)):
os_boot_device = dev_path
LOG.info("OS boot device: %s", dev_path)
self._exec_cmd('sudo umount %s' % tmp_dir)
else:
self._exec_cmd('sudo umount %s' % tmp_dir)
if not os_root_dir:
raise exception.OperatingSystemNotFound("root partition not found")
if os_boot_device:
LOG.debug("Mounting boot device '%s'", os_boot_device)
self._exec_cmd('sudo mount %s "%s/boot"' %
(os_boot_device, os_root_dir))
lvm_devs = list(set(self._get_lv_paths()) - set(mounted_devs))
self._check_mount_fstab_partitions(os_root_dir,
mountable_lvm_devs=lvm_devs)
for dir in set(dirs).intersection(['proc', 'sys', 'dev', 'run']):
mount_dir = os.path.join(os_root_dir, dir)
if not utils.test_ssh_path(self._ssh, mount_dir):
LOG.info("No '%s' directory in mounted OS. Skipping mount.",
dir)
continue
self._exec_cmd('sudo mount -o bind /%(dir)s/ %(mount_dir)s' % {
'dir': dir,
'mount_dir': mount_dir
})
return os_root_dir, os_root_device
def mount_os(self):
dev_paths = []
other_mounted_dirs = []
volume_devs = self._get_volume_block_devices()
for volume_dev in volume_devs:
self._exec_cmd("sudo partx -v -a %s || true" % volume_dev)
dev_paths += self._exec_cmd("sudo ls %s*" %
volume_dev).decode().split('\n')[:-1]
pvs = self._get_pvs()
for vg_name in self._get_vgnames():
found = False
for pv in pvs[vg_name]:
if pv in dev_paths:
found = True
break
if not found:
continue
self._exec_cmd("sudo vgchange -ay %s" % vg_name)
lvm_dev_paths = self._exec_cmd("sudo ls /dev/%s/*" %
vg_name).decode().split('\n')[:-1]
dev_paths += lvm_dev_paths
valid_filesystems = ['ext2', 'ext3', 'ext4', 'xfs', 'btrfs']
dev_paths_to_mount = []
for dev_path in dev_paths:
fs_type = self._exec_cmd("sudo blkid -o value -s TYPE %s || true" %
dev_path).decode().split('\n')[0]
if fs_type in valid_filesystems:
if fs_type == "xfs":
utils.run_xfs_repair(self._ssh, dev_path)
else:
utils.check_fs(self._ssh, fs_type, dev_path)
dev_paths_to_mount.append(dev_path)
os_root_device = None
os_root_dir = None
for dev_path in dev_paths_to_mount:
tmp_dir = self._exec_cmd('mktemp -d').decode().split('\n')[0]
self._exec_cmd('sudo mount %s %s' % (dev_path, tmp_dir))
dirs = self._exec_cmd('ls %s' % tmp_dir).decode().split('\n')
# TODO(alexpilotti): better ways to check for a linux root?
if (not os_root_dir and 'etc' in dirs and 'bin' in dirs
and 'sbin' in dirs):
os_root_dir = tmp_dir
os_root_device = dev_path
LOG.info("OS root device: %s", dev_path)
break
else:
self._exec_cmd('sudo umount %s' % tmp_dir)
if not os_root_dir:
raise exception.OperatingSystemNotFound("root partition not found")
other_mounted_dirs.extend(
self._check_mount_fstab_partitions(os_root_dir))
for dir in set(dirs).intersection(['proc', 'sys', 'dev', 'run']):
mount_dir = os.path.join(os_root_dir, dir)
if not utils.test_ssh_path(self._ssh, mount_dir):
LOG.info("No '%s' directory in mounted OS. Skipping mount.",
dir)
continue
self._exec_cmd('sudo mount -o bind /%(dir)s/ %(mount_dir)s' % {
'dir': dir,
'mount_dir': mount_dir
})
other_mounted_dirs.append(mount_dir)
return os_root_dir, other_mounted_dirs, os_root_device
