def _disable_lvm2_lvmetad(ssh): """Disables lvm2-lvmetad service. This service is responsible for automatically activating LVM2 volume groups when a disk is attached or when a volume group is created. During disk replication this service needs to be disabled. """ cfg = "/etc/lvm/lvm.conf" if utils.test_ssh_path(ssh, cfg): utils.exec_ssh_cmd( ssh, 'sudo sed -i "s/use_lvmetad.*=.*1/use_lvmetad = 0/g" ' '%s' % cfg, get_pty=True) # NOTE: lvm2-lvmetad is the name of the lvmetad service # on both debian and RHEL based systems. It needs to be stopped # before we begin disk replication. We disable it in the config # just in case some other process starts the daemon later on, as # a dependency. As the service may not actually exist, even though # the config is present, we ignore errors when stopping it. utils.ignore_exceptions(utils.exec_ssh_cmd)( ssh, "sudo service lvm2-lvmetad stop", get_pty=True) # disable volume groups. Any volume groups that have volumes in use # will remain online. However, volume groups belonging to disks # that have been synced at least once, will be deactivated. utils.ignore_exceptions(utils.exec_ssh_cmd)( ssh, "sudo vgchange -an", get_pty=True)
def _find_and_mount_root(self, devices): files = ["etc", "bin", "sbin", "boot"] os_root_dir = None os_root_device = self._find_dev_with_contents(devices, all_files=files) if os_root_device is None: raise exception.OperatingSystemNotFound( "Coriolis was unable to identify the root partition of the OS " "being migrated for mounting during OSMorphing. Please ensure " "that the source VM's root partition(s) are not encrypted, " "and that they are using a filesystem type and version which " "is supported by the OS images used for the OSMorphing minion " "machine. Also ensure that the source VM's mountpoint " "declarations in '/etc/fstab' are all correct, and are " "declared using '/dev/disk/by-uuid/' or 'UUID=' notation. " "If all else fails, please retry while using an OSMorphing " "minion machine image which is the same OS release as the VM " "being migrated.") try: tmp_dir = self._exec_cmd('mktemp -d').decode().splitlines()[0] self._exec_cmd('sudo mount %s %s' % (os_root_device, tmp_dir)) os_root_dir = tmp_dir except Exception as ex: self._event_manager.progress_update( "Exception occurred while Coriolis was attempting to mount the" " root device (%s) of the OS being migrated for OSMorphing. " "Please ensure that the source VM's root partition(s) are " "using a filesystem type and version which is supported by the" " OS images used for the OSMorphing minion machine. Also " "ensure that the source VM's mountpoint declarations in " "'/etc/fstab' are all correct, and are declared using " "'/dev/disk/by-uuid/' or 'UUID=' notation. If all else fails, " "please retry while using an OSMorphing minion machine image " "which is the same OS release as the VM being migrated. Error " "was: %s" % (os_root_device, str(ex))) LOG.error(ex) LOG.warn("Failed to mount root device '%s':\n%s", os_root_device, utils.get_exception_details()) utils.ignore_exceptions(self._exec_cmd)("sudo umount %s" % tmp_dir) utils.ignore_exceptions(self._exec_cmd)("sudo rmdir %s" % tmp_dir) raise for directory in ['proc', 'sys', 'dev', 'run']: mount_dir = os.path.join(os_root_dir, directory) if not utils.test_ssh_path(self._ssh, mount_dir): LOG.info("No '%s' directory in mounted OS. Skipping mount.", directory) continue self._exec_cmd('sudo mount -o bind /%(dir)s/ %(mount_dir)s' % { 'dir': directory, 'mount_dir': mount_dir }) if os_root_device in devices: devices.remove(os_root_device) return os_root_dir, os_root_device
def _find_and_mount_root(self, devices): files = ["etc", "bin", "sbin", "boot"] os_root_dir = None os_root_device = self._find_dev_with_contents( devices, all_files=files) if os_root_device is None: raise exception.OperatingSystemNotFound( "root partition not found") try: tmp_dir = self._exec_cmd('mktemp -d').decode().splitlines()[0] self._exec_cmd('sudo mount %s %s' % (os_root_device, tmp_dir)) os_root_dir = tmp_dir except Exception: self._event_manager.progress_update( "Failed to mount root device '%s'" % os_root_device) LOG.warn( "Failed to mount root device '%s':\n%s", os_root_device, utils.get_exception_details()) utils.ignore_exceptions(self._exec_cmd)( "sudo umount %s" % tmp_dir ) utils.ignore_exceptions(self._exec_cmd)( "sudo rmdir %s" % tmp_dir ) raise for directory in ['proc', 'sys', 'dev', 'run']: mount_dir = os.path.join(os_root_dir, directory) if not utils.test_ssh_path(self._ssh, mount_dir): LOG.info( "No '%s' directory in mounted OS. Skipping mount.", directory) continue self._exec_cmd( 'sudo mount -o bind /%(dir)s/ %(mount_dir)s' % {'dir': directory, 'mount_dir': mount_dir}) if os_root_device in devices: devices.remove(os_root_device) return os_root_dir, os_root_device
def _setup_certificates(self, ssh): remote_base_dir = "/etc/coriolis-writer" ca_crt_name = "ca-cert.pem" client_crt_name = "client-cert.pem" client_key_name = "client-key.pem" srv_crt_name = "srv-cert.pem" srv_key_name = "srv-key.pem" remote_ca_crt = os.path.join(remote_base_dir, ca_crt_name) remote_client_crt = os.path.join(remote_base_dir, client_crt_name) remote_client_key = os.path.join(remote_base_dir, client_key_name) remote_srv_crt = os.path.join(remote_base_dir, srv_crt_name) remote_srv_key = os.path.join(remote_base_dir, srv_key_name) exist = [] for i in (remote_ca_crt, remote_client_crt, remote_client_key, remote_srv_crt, remote_srv_key): exist.append(utils.test_ssh_path(ssh, i)) if not all(exist): utils.exec_ssh_cmd(ssh, "sudo mkdir -p %s" % remote_base_dir, get_pty=True) utils.exec_ssh_cmd( ssh, "sudo %(writer_cmd)s generate-certificates -output-dir " "%(cert_dir)s -certificate-hosts %(extra_hosts)s" % { "writer_cmd": self._writer_cmd, "cert_dir": remote_base_dir, "extra_hosts": self._ip, }, get_pty=True) return { "srv_crt": remote_srv_crt, "srv_key": remote_srv_key, "ca_crt": remote_ca_crt, "client_crt": remote_client_crt, "client_key": remote_client_key }
def _setup_certificates(self, ssh, args): # TODO(gsamfira): coriolis-replicator and coriolis-writer share # the functionality of being able to generate certificates # This will either be replaced with proper certificate management # in Coriolis, and the needed files will be pushed to the services # that need them (userdata or ssh), or the two applications will be # merged into one, and we will deduplicate this functionallity. remote_base_dir = REPLICATOR_DIR ip = args["ip"] ca_crt_name = "ca-cert.pem" client_crt_name = "client-cert.pem" client_key_name = "client-key.pem" srv_crt_name = "srv-cert.pem" srv_key_name = "srv-key.pem" remote_ca_crt = os.path.join(remote_base_dir, ca_crt_name) remote_client_crt = os.path.join(remote_base_dir, client_crt_name) remote_client_key = os.path.join(remote_base_dir, client_key_name) remote_srv_crt = os.path.join(remote_base_dir, srv_crt_name) remote_srv_key = os.path.join(remote_base_dir, srv_key_name) ca_crt = os.path.join(self._cert_dir, ca_crt_name) client_crt = os.path.join(self._cert_dir, client_crt_name) client_key = os.path.join(self._cert_dir, client_key_name) exist = [] for i in (remote_ca_crt, remote_client_crt, remote_client_key, remote_srv_crt, remote_srv_key): exist.append(utils.test_ssh_path(ssh, i)) force_fetch = False if not all(exist): utils.exec_ssh_cmd(ssh, "sudo mkdir -p %s" % remote_base_dir, get_pty=True) utils.exec_ssh_cmd( ssh, "sudo %(replicator_cmd)s gen-certs -output-dir " "%(cert_dir)s -certificate-hosts %(extra_hosts)s" % { "replicator_cmd": REPLICATOR_PATH, "cert_dir": remote_base_dir, "extra_hosts": ip, }, get_pty=True) utils.exec_ssh_cmd( ssh, "sudo chown -R %(user)s:%(group)s %(cert_dir)s" % { "cert_dir": remote_base_dir, "user": REPLICATOR_USERNAME, "group": REPLICATOR_GROUP_NAME }, get_pty=True) utils.exec_ssh_cmd(ssh, "sudo chmod -R g+r %(cert_dir)s" % { "cert_dir": remote_base_dir, }, get_pty=True) force_fetch = True exists = [] for i in (ca_crt, client_crt, client_key): exists.append(os.path.isfile(i)) if not all(exists) or force_fetch: # certificates either are missing, or have been regenerated # on the writer worker. We need to fetch them. self._fetch_remote_file(ssh, remote_ca_crt, ca_crt) self._fetch_remote_file(ssh, remote_client_crt, client_crt) self._fetch_remote_file(ssh, remote_client_key, client_key) return { "local": { "client_cert": client_crt, "client_key": client_key, "ca_cert": ca_crt, }, "remote": { "srv_crt": remote_srv_crt, "srv_key": remote_srv_key, "ca_crt": remote_ca_crt, }, }
def _test_path(self, chroot_path): path = os.path.join(self._os_root_dir, chroot_path) return utils.test_ssh_path(self._ssh, path)
def _check_mount_fstab_partitions(self, os_root_dir, skip_mounts=["/", "/boot"], skip_filesystems=["swap"], mountable_lvm_devs=None): """ Reads the contents of /etc/fstab from the VM's root directory and tries to mount all clearly identified (by UUID or path) filesystems. Returns the list of the new directories which were mounted. param: skip_mounts: list(str()): list of directories (inside the chroot) to not try to mount. param: skip_filesystems: list(str()): list of filesystem types to skip mounting entirely param: mountable_lvm_devs: list(str()): list of LVM device paths which exist and are mountable should they appear in /etc/fstab """ new_mountpoints = [] etc_fstab_path = os.path.join(os_root_dir, "etc/fstab") if not utils.test_ssh_path(self._ssh, etc_fstab_path): LOG.warn( "etc/fstab file not found in '%s'. Cannot mount non-root dirs", os_root_dir) return [] etc_fstab_raw = utils.read_ssh_file(self._ssh, etc_fstab_path) etc_fstab = etc_fstab_raw.decode('utf-8') LOG.debug("Mounting non-root partitions from fstab file: %s" % (base64.b64encode(etc_fstab_raw))) # dictionary of the form {"mountpoint": # {"device": "<dev>", "filesystem": "<fs>", "options": "<opts>"}} mounts = {} # fstab entry format: # <device> <mountpoint> <filesystem> <options> <dump> <fsck> fstab_entry_regex = ( "^(\s*([^#\s]+)\s+(\S+)\s+(\S+)\s+(\S+)(\s+(\d)(\s+(\d))?)?\s*)$") for line in etc_fstab.splitlines(): match = re.match(fstab_entry_regex, line) if not match: LOG.warn("Skipping unparseable /etc/fstab line: '%s'", line) continue device = match.group(2) mountpoint = match.group(3) if mountpoint in mounts: raise exception.CoriolisException( "Mountpoint '%s' appears to be mounted twice in " "'/etc/fstab'" % mountpoint) mounts[mountpoint] = { "device": device, "filesystem": match.group(4), "options": match.group(5) } # NOTE: we sort the mountpoints based on length to ensure # they get mounted in the correct order: mounts = collections.OrderedDict( (mountpoint, mounts[mountpoint]) for mountpoint in sorted(mounts, key=len)) # regexes for supported fstab device references: uuid_char_regex = "[0-9a-fA-F]" fs_uuid_regex = ("%(char)s{8}-%(char)s{4}-%(char)s{4}-" "%(char)s{4}-%(char)s{12}") % { "char": uuid_char_regex } fs_uuid_entry_regex = "^(UUID=%s)$" % fs_uuid_regex by_uuid_entry_regex = "^(/dev/disk/by-uuid/%s)$" % fs_uuid_regex if not mountable_lvm_devs: mountable_lvm_devs = [] device_paths = self._get_device_file_paths(mountable_lvm_devs) for (mountpoint, details) in mounts.items(): device = details['device'] if (re.match(fs_uuid_entry_regex, device) is None and re.match(by_uuid_entry_regex, device) is None): device_file_path = self._get_symlink_target(device) if device not in mountable_lvm_devs and (device_file_path not in device_paths): LOG.warn( "Found fstab entry for dir %s which references device " "%s. Only LVM volumes or devices referenced by UUID=* " "or /dev/disk/by-uuid/* notation are supported. " "Devicemapper paths for LVM volumes are also " "supported. Skipping mounting directory." % (mountpoint, device)) continue if mountpoint in skip_mounts: LOG.debug("Skipping undesired mount: %s: %s", mountpoint, details) continue if details["filesystem"] in skip_filesystems: LOG.debug("Skipping mounting undesired FS for device %s: %s", device, details) continue LOG.debug("Attempting to mount fstab device: %s: %s", device, details) # NOTE: `mountpoint` should always be an absolute path: chroot_mountpoint = "%s%s" % (os_root_dir, mountpoint) mountcmd = "sudo mount -t %s -o %s %s '%s'" % ( details["filesystem"], details["options"], device, chroot_mountpoint) try: self._exec_cmd(mountcmd) new_mountpoints.append(chroot_mountpoint) except Exception: LOG.warn( "Failed to run fstab filesystem mount command: '%s'. " "Skipping mount. Error details: %s", mountcmd, utils.get_exception_details()) if new_mountpoints: LOG.info( "The following new /etc/fstab entries were successfully " "mounted: %s", new_mountpoints) return new_mountpoints
def _setup_certificates(self, ssh): remote_base_dir = "/etc/coriolis-writer" ca_crt_name = "ca-cert.pem" client_crt_name = "client-cert.pem" client_key_name = "client-key.pem" srv_crt_name = "srv-cert.pem" srv_key_name = "srv-key.pem" remote_ca_crt = os.path.join(remote_base_dir, ca_crt_name) remote_client_crt = os.path.join(remote_base_dir, client_crt_name) remote_client_key = os.path.join(remote_base_dir, client_key_name) remote_srv_crt = os.path.join(remote_base_dir, srv_crt_name) remote_srv_key = os.path.join(remote_base_dir, srv_key_name) ca_crt = os.path.join(self._crt_dir, ca_crt_name) client_crt = os.path.join(self._crt_dir, client_crt_name) client_key = os.path.join(self._crt_dir, client_key_name) exist = [] for i in (remote_ca_crt, remote_client_crt, remote_client_key, remote_srv_crt, remote_srv_key): exist.append(utils.test_ssh_path(ssh, i)) force_fetch = False if not all(exist): utils.exec_ssh_cmd(ssh, "sudo mkdir -p %s" % remote_base_dir, get_pty=True) utils.exec_ssh_cmd( ssh, "sudo %(writer_cmd)s generate-certificates -output-dir " "%(cert_dir)s -certificate-hosts %(extra_hosts)s" % { "writer_cmd": self._writer_cmd, "cert_dir": remote_base_dir, "extra_hosts": self._ip, }, get_pty=True) force_fetch = True exists = [] for i in (ca_crt, client_crt, client_key): exists.append(os.path.isfile(i)) if not all(exists) or force_fetch: # certificates either are missing, or have been regenerated # on the writer worker. We need to fetch them. self._fetch_remote_file(ssh, remote_ca_crt, ca_crt) self._fetch_remote_file(ssh, remote_client_crt, client_crt) self._fetch_remote_file(ssh, remote_client_key, client_key) return { "local": { "client_crt": client_crt, "client_key": client_key, "ca_crt": ca_crt, }, "remote": { "srv_crt": remote_srv_crt, "srv_key": remote_srv_key, "ca_crt": remote_ca_crt, }, }
def _test_path(self, chroot_path): full_path = os.path.join(self._os_root_dir, chroot_path) return utils.test_ssh_path(self._conn, full_path)
def mount_os(self): dev_paths = [] mounted_devs = self._get_mounted_devices() volume_devs = self._get_volume_block_devices() for volume_dev in volume_devs: self._exec_cmd("sudo partx -v -a %s || true" % volume_dev) dev_paths += self._exec_cmd("sudo ls %s*" % volume_dev).decode().split('\n')[:-1] pvs = self._get_pvs() for vg_name in self._get_vgnames(): found = False for pv in pvs[vg_name]: if pv in dev_paths: found = True break if not found: continue self._exec_cmd("sudo vgchange -ay %s" % vg_name) lvm_dev_paths = self._exec_cmd("sudo ls /dev/%s/*" % vg_name).decode().split('\n')[:-1] dev_paths += lvm_dev_paths valid_filesystems = ['ext2', 'ext3', 'ext4', 'xfs', 'btrfs'] dev_paths_to_mount = [] for dev_path in dev_paths: if dev_path in mounted_devs: # this device is already mounted. Skip it, as it most likely # means this device belongs to the worker VM. continue fs_type = self._exec_cmd("sudo blkid -o value -s TYPE %s || true" % dev_path).decode().split('\n')[0] if fs_type in valid_filesystems: if fs_type == "xfs": utils.run_xfs_repair(self._ssh, dev_path) else: utils.check_fs(self._ssh, fs_type, dev_path) dev_paths_to_mount.append(dev_path) os_boot_device = None os_root_device = None os_root_dir = None for dev_path in dev_paths_to_mount: dirs = None tmp_dir = self._exec_cmd('mktemp -d').decode().split('\n')[0] try: self._exec_cmd('sudo mount %s %s' % (dev_path, tmp_dir)) # NOTE: it's possible that the device was mounted successfully # but an I/O error occurs later along the line: dirs = self._exec_cmd('ls %s' % tmp_dir).decode().split('\n') except Exception: self._event_manager.progress_update( "Failed to mount and scan device '%s'" % dev_path) LOG.warn("Failed to mount and scan device '%s':\n%s", dev_path, utils.get_exception_details()) continue LOG.debug("Contents of device %s:\n%s", dev_path, dirs) # TODO(alexpilotti): better ways to check for a linux root? if (not os_root_dir and 'etc' in dirs and 'bin' in dirs and 'sbin' in dirs): os_root_dir = tmp_dir os_root_device = dev_path LOG.info("OS root device: %s", dev_path) continue elif (not os_boot_device and ('grub' in dirs or 'grub2' in dirs)): os_boot_device = dev_path LOG.info("OS boot device: %s", dev_path) self._exec_cmd('sudo umount %s' % tmp_dir) else: self._exec_cmd('sudo umount %s' % tmp_dir) if not os_root_dir: raise exception.OperatingSystemNotFound("root partition not found") if os_boot_device: LOG.debug("Mounting boot device '%s'", os_boot_device) self._exec_cmd('sudo mount %s "%s/boot"' % (os_boot_device, os_root_dir)) lvm_devs = list(set(self._get_lv_paths()) - set(mounted_devs)) self._check_mount_fstab_partitions(os_root_dir, mountable_lvm_devs=lvm_devs) for dir in set(dirs).intersection(['proc', 'sys', 'dev', 'run']): mount_dir = os.path.join(os_root_dir, dir) if not utils.test_ssh_path(self._ssh, mount_dir): LOG.info("No '%s' directory in mounted OS. Skipping mount.", dir) continue self._exec_cmd('sudo mount -o bind /%(dir)s/ %(mount_dir)s' % { 'dir': dir, 'mount_dir': mount_dir }) return os_root_dir, os_root_device
def mount_os(self): dev_paths = [] other_mounted_dirs = [] volume_devs = self._get_volume_block_devices() for volume_dev in volume_devs: self._exec_cmd("sudo partx -v -a %s || true" % volume_dev) dev_paths += self._exec_cmd("sudo ls %s*" % volume_dev).decode().split('\n')[:-1] pvs = self._get_pvs() for vg_name in self._get_vgnames(): found = False for pv in pvs[vg_name]: if pv in dev_paths: found = True break if not found: continue self._exec_cmd("sudo vgchange -ay %s" % vg_name) lvm_dev_paths = self._exec_cmd("sudo ls /dev/%s/*" % vg_name).decode().split('\n')[:-1] dev_paths += lvm_dev_paths valid_filesystems = ['ext2', 'ext3', 'ext4', 'xfs', 'btrfs'] dev_paths_to_mount = [] for dev_path in dev_paths: fs_type = self._exec_cmd("sudo blkid -o value -s TYPE %s || true" % dev_path).decode().split('\n')[0] if fs_type in valid_filesystems: if fs_type == "xfs": utils.run_xfs_repair(self._ssh, dev_path) else: utils.check_fs(self._ssh, fs_type, dev_path) dev_paths_to_mount.append(dev_path) os_root_device = None os_root_dir = None for dev_path in dev_paths_to_mount: tmp_dir = self._exec_cmd('mktemp -d').decode().split('\n')[0] self._exec_cmd('sudo mount %s %s' % (dev_path, tmp_dir)) dirs = self._exec_cmd('ls %s' % tmp_dir).decode().split('\n') # TODO(alexpilotti): better ways to check for a linux root? if (not os_root_dir and 'etc' in dirs and 'bin' in dirs and 'sbin' in dirs): os_root_dir = tmp_dir os_root_device = dev_path LOG.info("OS root device: %s", dev_path) break else: self._exec_cmd('sudo umount %s' % tmp_dir) if not os_root_dir: raise exception.OperatingSystemNotFound("root partition not found") other_mounted_dirs.extend( self._check_mount_fstab_partitions(os_root_dir)) for dir in set(dirs).intersection(['proc', 'sys', 'dev', 'run']): mount_dir = os.path.join(os_root_dir, dir) if not utils.test_ssh_path(self._ssh, mount_dir): LOG.info("No '%s' directory in mounted OS. Skipping mount.", dir) continue self._exec_cmd('sudo mount -o bind /%(dir)s/ %(mount_dir)s' % { 'dir': dir, 'mount_dir': mount_dir }) other_mounted_dirs.append(mount_dir) return os_root_dir, other_mounted_dirs, os_root_device