def force_devmapper_symlinks():
"""Check if /dev/mapper/mpath* files are symlinks, if not trigger udev."""
LOG.debug('Verifying /dev/mapper/mpath* files are symlinks')
needs_trigger = []
for mp_id, dm_dev in dmname_to_blkdev_mapping().items():
if mp_id.startswith('mpath'):
mapper_path = '/dev/mapper/' + mp_id
if not os.path.islink(mapper_path):
LOG.warning(
'Found invalid device mapper mp path: %s, removing',
mapper_path)
util.del_file(mapper_path)
needs_trigger.append((mapper_path, dm_dev))
if len(needs_trigger):
for (mapper_path, dm_dev) in needs_trigger:
LOG.debug('multipath: regenerating symlink for %s (%s)',
mapper_path, dm_dev)
util.subp([
'udevadm', 'trigger', '--subsystem-match=block',
'--action=add', '/sys/class/block/' + os.path.basename(dm_dev)
])
udev.udevadm_settle(exists=mapper_path)
if not os.path.islink(mapper_path):
LOG.error('Failed to regenerate udev symlink %s', mapper_path)
def mdadm_assemble(md_devname=None, devices=[], spares=[], scan=False,
ignore_errors=False):
# md_devname is a /dev/XXXX
# devices is non-empty list of /dev/xxx
# if spares is non-empt list append of /dev/xxx
cmd = ["mdadm", "--assemble"]
if scan:
cmd += ['--scan', '-v']
else:
valid_mdname(md_devname)
cmd += [md_devname, "--run"] + devices
if spares:
cmd += spares
try:
# mdadm assemble returns 1 when no arrays are found. this might not be
# an error depending on the situation this function was called in, so
# accept a return code of 1
# mdadm assemble returns 2 when called on an array that is already
# assembled. this is not an error, so accept return code of 2
# all other return codes can be accepted with ignore_error set to true
scan, err = util.subp(cmd, capture=True, rcs=[0, 1, 2])
LOG.debug('mdadm assemble scan results:\n%s\n%s', scan, err)
scan, err = util.subp(['mdadm', '--detail', '--scan', '-v'],
capture=True, rcs=[0, 1])
LOG.debug('mdadm detail scan after assemble:\n%s\n%s',
scan, err)
except util.ProcessExecutionError:
LOG.warning("mdadm_assemble had unexpected return code")
if not ignore_errors:
raise
udev.udevadm_settle()
def iscsiadm_logout(target, portal):
LOG.debug('iscsiadm_logout: target=%s portal=%s', target, portal)
cmd = ['iscsiadm', '--mode=node', '--targetname=%s' % target,
'--portal=%s' % portal, '--logout']
util.subp(cmd, capture=True, log_captured=True)
udev.udevadm_settle()
def ensure_bcache_is_registered(bcache_device, expected, retry=None):
""" Test that bcache_device is found at an expected path and
re-register the device if it's not ready.
Retry the validation and registration as needed.
"""
if not retry:
retry = BCACHE_REGISTRATION_RETRY
for attempt, wait in enumerate(retry):
# find the actual bcache device name via sysfs using the
# backing device's holders directory.
LOG.debug(
'check just created bcache %s if it is registered,'
' try=%s', bcache_device, attempt + 1)
try:
udevadm_settle()
if os.path.exists(expected):
LOG.debug('Found bcache dev %s at expected path %s',
bcache_device, expected)
validate_bcache_ready(bcache_device, expected)
else:
msg = 'bcache device path not found: %s' % expected
LOG.debug(msg)
raise ValueError(msg)
# if bcache path exists and holders are > 0 we can return
LOG.debug(
'bcache dev %s at path %s successfully registered'
' on attempt %s/%s', bcache_device, expected, attempt + 1,
len(retry))
return
except (OSError, IndexError, ValueError):
# Some versions of bcache-tools will register the bcache device
# as soon as we run make-bcache using udev rules, so wait for
# udev to settle, then try to locate the dev, on older versions
# we need to register it manually though
LOG.debug(
'bcache device was not registered, registering %s '
'at /sys/fs/bcache/register', bcache_device)
try:
register_bcache(bcache_device)
except IOError:
# device creation is notoriously racy and this can trigger
# "Invalid argument" IOErrors if it got created in "the
# meantime" - just restart the function a few times to
# check it all again
pass
LOG.debug("bcache dev %s not ready, waiting %ss", bcache_device, wait)
time.sleep(wait)
# we've exhausted our retries
LOG.warning('Repetitive error registering the bcache dev %s',
bcache_device)
raise RuntimeError("bcache device %s can't be registered" % bcache_device)
def remove_partition(devpath, retries=10):
LOG.debug('multipath: removing multipath partition: %s', devpath)
for _ in range(0, retries):
util.subp(['dmsetup', 'remove', devpath], rcs=[0, 1])
udev.udevadm_settle()
if not os.path.exists(devpath):
return
util.wait_for_removal(devpath)
def remove_partition(devpath, retries=10):
""" Remove a multipath partition mapping. """
LOG.debug('multipath: removing multipath partition: %s', devpath)
for _ in range(0, retries):
util.subp(['dmsetup', 'remove', '--force', '--retry', devpath])
udev.udevadm_settle()
if not os.path.exists(devpath):
return
util.wait_for_removal(devpath)
def remove_map(map_id, retries=10):
LOG.debug('multipath: removing multipath map: %s', map_id)
devpath = '/dev/mapper/%s' % map_id
for _ in range(0, retries):
util.subp(['multipath', '-f', map_id], rcs=[0, 1])
udev.udevadm_settle()
if not os.path.exists(devpath):
return
util.wait_for_removal(devpath)
def connect(self):
if self.target not in iscsiadm_sessions():
iscsiadm_discovery(self.portal)
iscsiadm_authenticate(self.target, self.portal, self.user,
self.password, self.iuser, self.ipassword)
iscsiadm_login(self.target, self.portal)
udev.udevadm_settle(self.devdisk_path)
# always set automatic mode
iscsiadm_set_automatic(self.target, self.portal)
def find_mpath_members(multipath_id, paths=None):
""" Return a list of device path for each member of aspecified mpath_id."""
if not paths:
paths = show_paths()
for retry in range(0, 5):
orphans = [path for path in paths if 'orphan' in path['multipath']]
if len(orphans):
udev.udevadm_settle()
paths = show_paths()
else:
break
members = [
'/dev/' + path['device'] for path in paths
if path['multipath'] == multipath_id
]
return members
def rescan_block_devices(devices=None, warn_on_fail=True):
"""
run 'blockdev --rereadpt' for all block devices not currently mounted
"""
if not devices:
unused = get_unused_blockdev_info()
devices = []
for devname, data in unused.items():
if data.get('RM') == "1":
continue
if data.get('RO') != "0" or data.get('TYPE') != "disk":
continue
devices.append(data['device_path'])
if not devices:
LOG.debug("no devices found to rescan")
return
# blockdev needs /dev/ parameters, convert if needed
cmd = ['blockdev', '--rereadpt'] + [
dev if dev.startswith('/dev/') else sysfs_to_devpath(dev)
for dev in devices
]
try:
util.subp(cmd, capture=True)
except util.ProcessExecutionError as e:
if warn_on_fail:
# FIXME: its less than ideal to swallow this error, but until
# we fix LP: #1489521 we kind of need to.
LOG.warn(
"Error rescanning devices, possibly known issue LP: #1489521")
# Reformatting the exception output so as to not trigger
# vmtest scanning for Unexepected errors in install logfile
LOG.warn("cmd: %s\nstdout:%s\nstderr:%s\nexit_code:%s", e.cmd,
e.stdout, e.stderr, e.exit_code)
udevadm_settle()
return
def mdadm_create(md_devname, raidlevel, devices, spares=None, container=None,
md_name="", metadata=None):
LOG.debug('mdadm_create: ' +
'md_name=%s raidlevel=%s ' % (md_devname, raidlevel) +
' devices=%s spares=%s name=%s' % (devices, spares, md_name))
assert_valid_devpath(md_devname)
if not metadata:
metadata = 'default'
if raidlevel not in VALID_RAID_LEVELS:
raise ValueError('Invalid raidlevel: [{}]'.format(raidlevel))
min_devices = md_minimum_devices(raidlevel)
devcnt = len(devices) if not container else \
len(md_get_devices_list(container))
if devcnt < min_devices:
err = 'Not enough devices (' + str(devcnt) + ') '
err += 'for raidlevel: ' + str(raidlevel)
err += ' minimum devices needed: ' + str(min_devices)
raise ValueError(err)
if spares and raidlevel not in SPARE_RAID_LEVELS:
err = ('Raidlevel does not support spare devices: ' + str(raidlevel))
raise ValueError(err)
(hostname, _err) = util.subp(["hostname", "-s"], rcs=[0], capture=True)
cmd = ["mdadm", "--create", md_devname, "--run",
"--homehost=%s" % hostname.strip(),
"--raid-devices=%s" % devcnt]
if not container:
cmd.append("--metadata=%s" % metadata)
if raidlevel != 'container':
cmd.append("--level=%s" % raidlevel)
if md_name:
cmd.append("--name=%s" % md_name)
if container:
cmd.append(container)
for device in devices:
holders = get_holders(device)
if len(holders) > 0:
LOG.warning('Detected holders during mdadm creation: %s', holders)
raise OSError('Failed to remove holders from %s', device)
zero_device(device)
cmd.append(device)
if spares:
cmd.append("--spare-devices=%s" % len(spares))
for device in spares:
zero_device(device)
cmd.append(device)
# Create the raid device
udev.udevadm_settle()
util.subp(["udevadm", "control", "--stop-exec-queue"])
try:
util.subp(cmd, capture=True)
except util.ProcessExecutionError:
# frequent issues by modules being missing (LP: #1519470) - add debug
LOG.debug('mdadm_create failed - extra debug regarding md modules')
(out, _err) = util.subp(["lsmod"], capture=True)
if not _err:
LOG.debug('modules loaded: \n%s' % out)
raidmodpath = '/lib/modules/%s/kernel/drivers/md' % os.uname()[2]
(out, _err) = util.subp(["find", raidmodpath],
rcs=[0, 1], capture=True)
if out:
LOG.debug('available md modules: \n%s' % out)
else:
LOG.debug('no available md modules found')
for dev in devices + spares:
h = get_holders(dev)
LOG.debug('Device %s has holders: %s', dev, h)
raise
util.subp(["udevadm", "control", "--start-exec-queue"])
udev.udevadm_settle(exists=md_devname)
def shutdown_bcache(device):
"""
Shut down bcache for specified bcache device
1. wipe the bcache device contents
2. extract the cacheset uuid (if cached)
3. extract the backing device
4. stop cacheset (if present)
5. stop the bcacheN device
6. wait for removal of sysfs path to bcacheN, bcacheN/bcache and
backing/bcache to go away
"""
if not device.startswith('/sys/class/block'):
raise ValueError(
'Invalid Device (%s): '
'Device path must start with /sys/class/block/', device)
# bcache device removal should be fast but in an extreme
# case, might require the cache device to flush large
# amounts of data to a backing device. The strategy here
# is to wait for approximately 30 seconds but to check
# frequently since curtin cannot proceed until devices
# cleared.
bcache_shutdown_message = ('shutdown_bcache running on {} has determined '
'that the device has already been shut down '
'during handling of another bcache dev. '
'skipping'.format(device))
if not os.path.exists(device):
LOG.info(bcache_shutdown_message)
return
LOG.info('Wiping superblock on bcache device: %s', device)
_wipe_superblock(block.sysfs_to_devpath(device), exclusive=False)
# collect required information before stopping bcache device
# UUID from /sys/fs/cache/UUID
cset_uuid = bcache.get_attached_cacheset(device)
# /sys/class/block/vdX which is a backing dev of device (bcacheN)
backing_sysfs = bcache.get_backing_device(block.path_to_kname(device))
# /sys/class/block/bcacheN/bache
bcache_sysfs = bcache.sysfs_path(device, strict=False)
# stop cacheset if one is presennt
if cset_uuid:
LOG.info('%s was attached to cacheset %s, stopping cacheset', device,
cset_uuid)
bcache.stop_cacheset(cset_uuid)
# let kernel settle before the next remove
udev.udevadm_settle()
LOG.info('bcache cacheset stopped: %s', cset_uuid)
# test and log whether the device paths are still present
to_check = [bcache_sysfs, backing_sysfs]
found_devs = [os.path.exists(p) for p in to_check]
LOG.debug('os.path.exists on blockdevs:\n%s',
list(zip(to_check, found_devs)))
if not any(found_devs):
LOG.info('bcache backing device already removed: %s (%s)',
bcache_sysfs, device)
LOG.debug('bcache backing device checked: %s', backing_sysfs)
else:
LOG.info('stopping bcache backing device at: %s', bcache_sysfs)
bcache.stop_device(bcache_sysfs)
return
def shutdown_bcache(device):
"""
Shut down bcache for specified bcache device
1. Stop the cacheset that `device` is connected to
2. Stop the 'device'
"""
if not device.startswith('/sys/class/block'):
raise ValueError(
'Invalid Device (%s): '
'Device path must start with /sys/class/block/', device)
LOG.info('Wiping superblock on bcache device: %s', device)
_wipe_superblock(block.sysfs_to_devpath(device), exclusive=False)
# bcache device removal should be fast but in an extreme
# case, might require the cache device to flush large
# amounts of data to a backing device. The strategy here
# is to wait for approximately 30 seconds but to check
# frequently since curtin cannot proceed until devices
# cleared.
removal_retries = [0.2] * 150 # 30 seconds total
bcache_shutdown_message = ('shutdown_bcache running on {} has determined '
'that the device has already been shut down '
'during handling of another bcache dev. '
'skipping'.format(device))
if not os.path.exists(device):
LOG.info(bcache_shutdown_message)
return
# get slaves [vdb1, vdc], allow for slaves to not have bcache dir
slave_paths = [
get_bcache_sys_path(k, strict=False)
for k in os.listdir(os.path.join(device, 'slaves'))
]
# stop cacheset if it exists
bcache_cache_sysfs = get_bcache_using_dev(device, strict=False)
if not os.path.exists(bcache_cache_sysfs):
LOG.info('bcache cacheset already removed: %s',
os.path.basename(bcache_cache_sysfs))
else:
LOG.info('stopping bcache cacheset at: %s', bcache_cache_sysfs)
maybe_stop_bcache_device(bcache_cache_sysfs)
try:
util.wait_for_removal(bcache_cache_sysfs, retries=removal_retries)
except OSError:
LOG.info('Failed to stop bcache cacheset %s', bcache_cache_sysfs)
raise
# let kernel settle before the next remove
udev.udevadm_settle()
# after stopping cache set, we may need to stop the device
# both the dev and sysfs entry should be gone.
# we know the bcacheN device is really gone when we've removed:
# /sys/class/block/{bcacheN}
# /sys/class/block/slaveN1/bcache
# /sys/class/block/slaveN2/bcache
bcache_block_sysfs = get_bcache_sys_path(device, strict=False)
to_check = [device] + slave_paths
found_devs = [os.path.exists(p) for p in to_check]
LOG.debug('os.path.exists on blockdevs:\n%s',
list(zip(to_check, found_devs)))
if not any(found_devs):
LOG.info('bcache backing device already removed: %s (%s)',
bcache_block_sysfs, device)
LOG.debug('bcache slave paths checked: %s', slave_paths)
return
else:
LOG.info('stopping bcache backing device at: %s', bcache_block_sysfs)
maybe_stop_bcache_device(bcache_block_sysfs)
try:
# wait for them all to go away
for dev in [device, bcache_block_sysfs] + slave_paths:
util.wait_for_removal(dev, retries=removal_retries)
except OSError:
LOG.info('Failed to stop bcache backing device %s',
bcache_block_sysfs)
raise
return
def start_clear_holders_deps():
"""
prepare system for clear holders to be able to scan old devices
"""
# a mdadm scan has to be started in case there is a md device that needs to
# be detected. if the scan fails, it is either because there are no mdadm
# devices on the system, or because there is a mdadm device in a damaged
# state that could not be started. due to the nature of mdadm tools, it is
# difficult to know which is the case. if any errors did occur, then ignore
# them, since no action needs to be taken if there were no mdadm devices on
# the system, and in the case where there is some mdadm metadata on a disk,
# but there was not enough to start the array, the call to wipe_volume on
# all disks and partitions should be sufficient to remove the mdadm
# metadata
mdadm.mdadm_assemble(scan=True, ignore_errors=True)
# collect detail on any assembling arrays
for md in [md for md in glob.glob('/dev/md*')
if not os.path.isdir(md) and not identify_partition(md)]:
mdstat = None
if os.path.exists('/proc/mdstat'):
mdstat = util.load_file('/proc/mdstat')
LOG.debug("/proc/mdstat:\n%s", mdstat)
found = [line for line in mdstat.splitlines()
if os.path.basename(md) in line]
# in some cases we have a /dev/md0 device node
# but the kernel has already renamed the device /dev/md127
if len(found) == 0:
LOG.debug('Ignoring md device %s, not present in mdstat', md)
continue
# give it a second poke to encourage running
try:
LOG.debug('Activating mdadm array %s', md)
(out, err) = mdadm.mdadm_run(md)
LOG.debug('MDADM run on %s stdout:\n%s\nstderr:\n%s', md, out, err)
except util.ProcessExecutionError:
LOG.debug('Non-fatal error when starting mdadm device %s', md)
# extract details if we can
try:
(out, err) = mdadm.mdadm_query_detail(md, export=False,
rawoutput=True)
LOG.debug('MDADM detail on %s stdout:\n%s\nstderr:\n%s',
md, out, err)
except util.ProcessExecutionError:
LOG.debug('Non-fatal error when querying mdadm detail on %s', md)
mp_support = multipath.multipath_supported()
if mp_support:
LOG.debug('Detected multipath support, reload maps')
multipath.reload()
multipath.force_devmapper_symlinks()
# scan and activate for logical volumes
lvm.lvm_scan(multipath=mp_support)
try:
lvm.activate_volgroups(multipath=mp_support)
except util.ProcessExecutionError:
# partial vg may not come up due to missing members, that's OK
pass
udev.udevadm_settle()
# the bcache module needs to be present to properly detect bcache devs
# on some systems (precise without hwe kernel) it may not be possible to
# lad the bcache module bcause it is not present in the kernel. if this
# happens then there is no need to halt installation, as the bcache devices
# will never appear and will never prevent the disk from being reformatted
util.load_kernel_module('bcache')
if not zfs.zfs_supported():
LOG.warning('zfs filesystem is not supported in this environment')
def mdadm_create(md_devname, raidlevel, devices, spares=None, md_name=""):
LOG.debug('mdadm_create: ' + 'md_name=%s raidlevel=%s ' %
(md_devname, raidlevel) + ' devices=%s spares=%s name=%s' %
(devices, spares, md_name))
assert_valid_devpath(md_devname)
if raidlevel not in VALID_RAID_LEVELS:
raise ValueError('Invalid raidlevel: [{}]'.format(raidlevel))
min_devices = md_minimum_devices(raidlevel)
if len(devices) < min_devices:
err = 'Not enough devices for raidlevel: ' + str(raidlevel)
err += ' minimum devices needed: ' + str(min_devices)
raise ValueError(err)
if spares and raidlevel not in SPARE_RAID_LEVELS:
err = ('Raidlevel does not support spare devices: ' + str(raidlevel))
raise ValueError(err)
(hostname, _err) = util.subp(["hostname", "-s"], rcs=[0], capture=True)
cmd = [
"mdadm", "--create", md_devname, "--run",
"--homehost=%s" % hostname.strip(),
"--level=%s" % raidlevel,
"--raid-devices=%s" % len(devices)
]
if md_name:
cmd.append("--name=%s" % md_name)
for device in devices:
# mdadm can be sticky, double check
holders = get_holders(device)
if len(holders) > 0:
LOG.warning('Detected holders during mdadm creation: %s', holders)
raise OSError('Failed to remove holders from %s', device)
# Zero out device superblock just in case device has been used for raid
# before, as this will cause many issues
util.subp(["mdadm", "--zero-superblock", device], capture=True)
cmd.append(device)
if spares:
cmd.append("--spare-devices=%s" % len(spares))
for device in spares:
util.subp(["mdadm", "--zero-superblock", device], capture=True)
cmd.append(device)
# Create the raid device
udev.udevadm_settle()
util.subp(["udevadm", "control", "--stop-exec-queue"])
try:
util.subp(cmd, capture=True)
except util.ProcessExecutionError:
# frequent issues by modules being missing (LP: #1519470) - add debug
LOG.debug('mdadm_create failed - extra debug regarding md modules')
(out, _err) = util.subp(["lsmod"], capture=True)
if not _err:
LOG.debug('modules loaded: \n%s' % out)
raidmodpath = '/lib/modules/%s/kernel/drivers/md' % os.uname()[2]
(out, _err) = util.subp(["find", raidmodpath],
rcs=[0, 1],
capture=True)
if out:
LOG.debug('available md modules: \n%s' % out)
else:
LOG.debug('no available md modules found')
for dev in devices + spares:
h = get_holders(dev)
LOG.debug('Device %s has holders: %s', dev, h)
raise
util.subp(["udevadm", "control", "--start-exec-queue"])
udev.udevadm_settle(exists=md_devname)
