def remove_device(mddev, arraydev):
assert_valid_devpath(mddev)
LOG.info("mdadm remove %s from array %s", arraydev, mddev)
out, err = util.subp(["mdadm", "--remove", mddev, arraydev],
rcs=[0],
capture=True)
LOG.debug("mdadm remove:\n%s\n%s", out, err)
def fail_device(mddev, arraydev):
assert_valid_devpath(mddev)
LOG.info("mdadm mark faulty: %s in array %s", arraydev, mddev)
out, err = util.subp(["mdadm", "--fail", mddev, arraydev],
rcs=[0],
capture=True)
LOG.debug("mdadm mark faulty:\n%s\n%s", out, err)
def udevadm_info(path=None):
""" Return a dictionary populated by properties of the device specified
in the `path` variable via querying udev 'property' database.
:params: path: path to device, either /dev or /sys
:returns: dictionary of key=value pairs as exported from the udev database
:raises: ValueError path is None, ProcessExecutionError on exec error.
"""
if not path:
raise ValueError('Invalid path: "%s"' % path)
info_cmd = ['udevadm', 'info', '--query=property', '--export', path]
output, _ = util.subp(info_cmd, capture=True)
# strip for trailing empty line
info = {}
for line in output.splitlines():
if not line:
continue
# maxsplit=1 gives us key and remaininng part of line is value
# py2.7 on Trusty doesn't have keyword, pass as argument
key, value = line.split('=', 1)
if not value:
value = None
if value:
# preserve spaces in values to match udev database
try:
parsed = shlex.split(value)
except ValueError:
# strip the leading/ending single tick from udev output before
# escaping the value to prevent their inclusion in the result.
trimmed_value = value[1:-1]
try:
quoted = shlex_quote(trimmed_value)
LOG.debug(
'udevadm_info: quoting shell-escape chars '
'in %s=%s -> %s', key, value, quoted)
parsed = shlex.split(quoted)
except ValueError:
escaped_value = (trimmed_value.replace("'", "_").replace(
'"', "_"))
LOG.debug(
'udevadm_info: replacing shell-escape chars '
'in %s=%s -> %s', key, value, escaped_value)
parsed = shlex.split(escaped_value)
if ' ' not in value:
info[key] = parsed[0]
else:
# special case some known entries with spaces, e.g. ID_SERIAL
# and DEVLINKS, see tests/unittests/test_udev.py
if key == "DEVLINKS":
info[key] = shlex.split(parsed[0])
elif key == 'ID_SERIAL':
info[key] = parsed[0]
else:
info[key] = parsed
return info
def mdadm_stop(devpath, retries=None):
assert_valid_devpath(devpath)
if not retries:
retries = [0.2] * 60
sync_action = md_sysfs_attr_path(devpath, 'sync_action')
sync_max = md_sysfs_attr_path(devpath, 'sync_max')
sync_min = md_sysfs_attr_path(devpath, 'sync_min')
LOG.info("mdadm stopping: %s" % devpath)
for (attempt, wait) in enumerate(retries):
try:
LOG.debug('mdadm: stop on %s attempt %s', devpath, attempt)
# An array in 'resync' state may not be stoppable, attempt to
# cancel an ongoing resync
val = md_sysfs_attr(devpath, 'sync_action')
LOG.debug('%s/sync_max = %s', sync_action, val)
if val != "idle":
LOG.debug("mdadm: setting array sync_action=idle")
try:
util.write_file(sync_action, content="idle")
except (IOError, OSError) as e:
LOG.debug("mdadm: (non-fatal) write to %s failed %s",
sync_action, e)
# Setting the sync_{max,min} may can help prevent the array from
# changing back to 'resync' which may prevent the array from being
# stopped
val = md_sysfs_attr(devpath, 'sync_max')
LOG.debug('%s/sync_max = %s', sync_max, val)
if val != "0":
LOG.debug("mdadm: setting array sync_{min,max}=0")
try:
for sync_file in [sync_max, sync_min]:
util.write_file(sync_file, content="0")
except (IOError, OSError) as e:
LOG.debug('mdadm: (non-fatal) write to %s failed %s',
sync_file, e)
# one wonders why this command doesn't do any of the above itself?
out, err = util.subp(["mdadm", "--manage", "--stop", devpath],
capture=True)
LOG.debug("mdadm stop command output:\n%s\n%s", out, err)
LOG.info("mdadm: successfully stopped %s after %s attempt(s)",
devpath, attempt+1)
return
except util.ProcessExecutionError:
LOG.warning("mdadm stop failed, retrying ")
if os.path.isfile('/proc/mdstat'):
LOG.critical("/proc/mdstat:\n%s",
util.load_file('/proc/mdstat'))
LOG.debug("mdadm: stop failed, retrying in %s seconds", wait)
time.sleep(wait)
pass
raise OSError('Failed to stop mdadm device %s', devpath)
def get_dmsetup_uuid(device):
"""
get the dm uuid for a specified dmsetup device
"""
blockdev = block.sysfs_to_devpath(device)
(out, _) = util.subp(
['dmsetup', 'info', blockdev, '-C', '-o', 'uuid', '--noheadings'],
capture=True)
return out.strip()
def test_subp_combined_stderr_stdout(self):
"""Providing combine_capture as True redirects stderr to stdout."""
data = b'hello world'
(out, err) = util.subp(self.stdin2err,
combine_capture=True,
decode=False,
data=data)
self.assertEqual(err, b'')
self.assertEqual(out, data)
def get_volume_uuid(path):
"""
Get uuid of disk with given path. This address uniquely identifies
the device and remains consistant across reboots
"""
(out, _err) = util.subp(["blkid", "-o", "export", path], capture=True)
for line in out.splitlines():
if "UUID" in line:
return line.split('=')[-1]
return ''
def _extract_mpath_data(cmd, show_verb):
data, _err = util.subp(cmd, capture=True)
result = []
for line in data.splitlines():
mp_dict = util.load_shell_content(line, add_empty=True)
LOG.debug('Extracted multipath %s fields: %s', show_verb, mp_dict)
if mp_dict:
result.append(mp_dict)
return result
def iscsiadm_discovery(portal):
# only supported type for now
type = 'sendtargets'
if not portal:
raise ValueError("Portal must be specified for discovery")
cmd = [
"iscsiadm", "--mode=discovery",
"--type=%s" % type,
"--portal=%s" % portal
]
try:
util.subp(cmd, capture=True, log_captured=True)
except util.ProcessExecutionError as e:
LOG.warning("iscsiadm_discovery to %s failed with exit code %d",
portal, e.exit_code)
raise
def export_armour(key):
"""Export gpg key, armoured key gets returned"""
try:
(armour, _) = util.subp(["gpg", "--export", "--armour", key],
capture=True)
except util.ProcessExecutionError as error:
# debug, since it happens for any key not on the system initially
LOG.debug('Failed to export armoured key "%s": %s', key, error)
armour = None
return armour
def _subp_wrap_popen(self,
cmd,
kwargs,
stdout=b'',
stderr=b'',
returncodes=None):
# mocks the subprocess.Popen as expected from subp
# checks that subp returned the output of 'communicate' and
# returns the (args, kwargs) that Popen() was called with.
# returncodes is a list to cover, one for each expected call
if returncodes is None:
returncodes = [0]
capture = kwargs.get('capture')
mreturncodes = mock.PropertyMock(side_effect=iter(returncodes))
with mock.patch("curtin.util.subprocess.Popen") as m_popen:
sp = mock.Mock()
m_popen.return_value = sp
if capture:
sp.communicate.return_value = (stdout, stderr)
else:
sp.communicate.return_value = (None, None)
type(sp).returncode = mreturncodes
ret = util.subp(cmd, **kwargs)
# popen may be called once or > 1 for retries, but must be called.
self.assertTrue(m_popen.called)
# communicate() needs to have been called.
self.assertTrue(sp.communicate.called)
if capture:
# capture response is decoded if decode is not False
decode = kwargs.get('decode', "replace")
if decode is False:
self.assertEqual(stdout.decode(stdout, stderr), ret)
else:
self.assertEqual((stdout.decode(errors=decode),
stderr.decode(errors=decode)), ret)
else:
# if capture is false, then return is None, None
self.assertEqual((None, None), ret)
# if target is not provided or is /, chroot should not be used
calls = m_popen.call_args_list
popen_args, popen_kwargs = calls[-1]
target = paths.target_path(kwargs.get('target', None))
unshcmd = self.mock_get_unshare_pid_args.return_value
if target == "/":
self.assertEqual(unshcmd + list(cmd), popen_args[0])
else:
self.assertEqual(unshcmd + ['chroot', target] + list(cmd),
popen_args[0])
return calls
def zkey_supported(strict=True):
""" Return True if zkey cmd present and can generate keys, else False."""
LOG.debug('Checking if zkey encryption is supported...')
try:
util.load_kernel_module('pkey')
except util.ProcessExecutionError as err:
msg = "Failed to load 'pkey' kernel module"
LOG.error(msg + ": %s" % err) if strict else LOG.warning(msg)
return False
try:
with tempfile.NamedTemporaryFile() as tf:
util.subp(['zkey', 'generate', tf.name], capture=True)
LOG.debug('zkey encryption supported.')
return True
except util.ProcessExecutionError as err:
msg = "zkey not supported"
LOG.error(msg + ": %s" % err) if strict else LOG.warning(msg)
return False
def stop_all_unused_multipath_devices():
"""
Stop all unused multipath devices.
"""
multipath = util.which('multipath')
# Command multipath is not available only when multipath-tools package
# is not installed. Nothing needs to be done in this case because system
# doesn't create multipath devices without this package installed and we
# have nothing to stop.
if not multipath:
return
# Command multipath -F flushes all unused multipath device maps
cmd = [multipath, '-F']
try:
# unless multipath cleared *everything* it will exit with 1
util.subp(cmd, rcs=[0, 1])
except util.ProcessExecutionError as e:
LOG.warn("Failed to stop multipath devices: %s", e)
def zpool_list():
"""
Return a list of zfs pool names which have been imported
:returns: List of strings
"""
# -H drops the header, -o specifies an attribute to fetch
out, _err = util.subp(['zpool', 'list', '-H', '-o', 'name'], capture=True)
return out.splitlines()
def split_lvm_name(full):
"""
split full lvm name into tuple of (volgroup, lv_name)
"""
# 'dmsetup splitname' is the authoratative source for lvm name parsing
(out, _) = util.subp([
'dmsetup', 'splitname', full, '-c', '--noheadings', '--separator',
_SEP, '-o', 'vg_name,lv_name'
],
capture=True)
return out.strip().split(_SEP)
def zfs_mount(poolname, volume):
"""
Mount zfs pool/volume
:param poolname: String used to specify the pool in which to create the
filesystem.
:param volume: String used as the name of the filesystem.
:returns: None
:raises: ValueError: raises exceptions on missing/bad input.
:raises: ProcessExecutionError: raised on unhandled exceptions from
invoking `zfs mount`.
"""
if not isinstance(poolname, util.string_types) or not poolname:
raise ValueError("Invalid poolname: %s", poolname)
if not isinstance(volume, util.string_types) or not volume:
raise ValueError("Invalid volume: %s", volume)
cmd = ['zfs', 'mount', _join_pool_volume(poolname, volume)]
util.subp(cmd, capture=True)
def lvm_scan(activate=True):
"""
run full scan for volgroups, logical volumes and physical volumes
"""
# prior to xenial, lvmetad is not packaged, so even if a tool supports
# flag --cache it has no effect. In Xenial and newer the --cache flag is
# used (if lvmetad is running) to ensure that the data cached by
# lvmetad is updated.
# before appending the cache flag though, check if lvmetad is running. this
# ensures that we do the right thing even if lvmetad is supported but is
# not running
release = distro.lsb_release().get('codename')
if release in [None, 'UNAVAILABLE']:
LOG.warning('unable to find release number, assuming xenial or later')
release = 'xenial'
for cmd in [['pvscan'], ['vgscan', '--mknodes']]:
if release != 'precise' and lvmetad_running():
cmd.append('--cache')
util.subp(cmd, capture=True)
def apply_power_state(pstate):
"""
power_state:
delay: 5
mode: poweroff
message: Bye Bye
"""
cmd = load_power_state(pstate)
if not cmd:
return
LOG.info("powering off with %s", cmd)
fid = os.fork()
if fid == 0:
try:
util.subp(cmd)
os._exit(0)
except Exception as e:
LOG.warn("%s returned non-zero: %s" % (cmd, e))
os._exit(1)
return
def run_python(self, args):
start_dir = os.getcwd()
cmd = [sys.executable]
for i in args:
cmd.append(i)
env = os.environ.copy()
env['CURTIN_STACKTRACE'] = "1"
try:
os.chdir(self.extract_dir)
return util.subp(cmd, capture=True, env=env)
finally:
os.chdir(start_dir)
def dmname_to_blkdev_mapping():
""" Use dmsetup ls output to build a dict of DM_NAME, /dev/dm-x values."""
data, _err = util.subp(['dmsetup', 'ls', '-o', 'blkdevname'], capture=True)
mapping = {}
if data and data.strip() != "No devices found":
LOG.debug('multipath: dmsetup ls output:\n%s', data)
for line in data.splitlines():
if line:
dm_name, blkdev = line.split('\t')
# (dm-1) -> /dev/dm-1
mapping[dm_name] = '/dev/' + blkdev.strip('()')
return mapping
def tar_xattr_opts(cmd=None):
# if tar cmd supports xattrs, return the required flags to extract them.
if cmd is None:
cmd = ['tar']
if isinstance(cmd, str):
cmd = [cmd]
(out, _err) = util.subp(cmd + ['--help'], capture=True)
if "xattr" in out:
return ['--xattrs', '--xattrs-include=*']
return []
def rescan_block_devices(devices=None, warn_on_fail=True):
"""
run 'blockdev --rereadpt' for all block devices not currently mounted
"""
if not devices:
unused = get_unused_blockdev_info()
devices = []
for devname, data in unused.items():
if data.get('RM') == "1":
continue
if data.get('RO') != "0" or data.get('TYPE') != "disk":
continue
devices.append(data['device_path'])
if not devices:
LOG.debug("no devices found to rescan")
return
# blockdev needs /dev/ parameters, convert if needed
cmd = ['blockdev', '--rereadpt'] + [
dev if dev.startswith('/dev/') else sysfs_to_devpath(dev)
for dev in devices
]
try:
util.subp(cmd, capture=True)
except util.ProcessExecutionError as e:
if warn_on_fail:
# FIXME: its less than ideal to swallow this error, but until
# we fix LP: #1489521 we kind of need to.
LOG.warn(
"Error rescanning devices, possibly known issue LP: #1489521")
# Reformatting the exception output so as to not trigger
# vmtest scanning for Unexepected errors in install logfile
LOG.warn("cmd: %s\nstdout:%s\nstderr:%s\nexit_code:%s", e.cmd,
e.stdout, e.stderr, e.exit_code)
udevadm_settle()
return
def create_backing_device(backing_device, cache_device, cache_mode, cset_uuid):
backing_device_sysfs = sys_block_path(backing_device)
target_sysfs_path = os.path.join(backing_device_sysfs, "bcache")
# there should not be any pre-existing bcache device
bdir = os.path.join(backing_device_sysfs, "bcache")
if os.path.exists(bdir):
raise RuntimeError('Unexpected old bcache device: %s', backing_device)
LOG.debug('Creating a backing device on %s', backing_device)
util.subp(["make-bcache", "-B", backing_device])
ensure_bcache_is_registered(backing_device, target_sysfs_path)
# via the holders we can identify which bcache device we just created
# for a given backing device
from .clear_holders import get_holders
holders = get_holders(backing_device)
if len(holders) != 1:
err = ('Invalid number {} of holding devices:'
' "{}"'.format(len(holders), holders))
LOG.error(err)
raise ValueError(err)
[bcache_dev] = holders
LOG.debug('The just created bcache device is {}'.format(holders))
if cache_device:
# if we specify both then we need to attach backing to cache
if cset_uuid:
attach_backing_to_cacheset(backing_device, cache_device, cset_uuid)
else:
msg = "Invalid cset_uuid: {}".format(cset_uuid)
LOG.error(msg)
raise ValueError(msg)
if cache_mode:
set_cache_mode(bcache_dev, cache_mode)
return dev_path(bcache_dev)
def wipe_volume(path, mode="superblock", exclusive=True, strict=False):
"""wipe a volume/block device
:param path: a path to a block device
:param mode: how to wipe it.
pvremove: wipe a lvm physical volume
zero: write zeros to the entire volume
random: write random data (/dev/urandom) to the entire volume
superblock: zero the beginning and the end of the volume
superblock-recursive: zero the beginning of the volume, the end of the
volume and beginning and end of any partitions that are
known to be on this device.
:param exclusive: boolean to control how path is opened
:param strict: boolean to control when to raise errors on write failures
"""
if mode == "pvremove":
# We need to use --force --force in case it's already in a volgroup and
# pvremove doesn't want to remove it
# If pvremove is run and there is no label on the system,
# then it exits with 5. That is also okay, because we might be
# wiping something that is already blank
util.subp(['pvremove', '--force', '--force', '--yes', path],
rcs=[0, 5],
capture=True)
lvm.lvm_scan()
elif mode == "zero":
wipe_file(path, exclusive=exclusive)
elif mode == "random":
with open("/dev/urandom", "rb") as reader:
wipe_file(path, reader=reader.read, exclusive=exclusive)
elif mode == "superblock":
quick_zero(path, partitions=False, exclusive=exclusive, strict=strict)
elif mode == "superblock-recursive":
quick_zero(path, partitions=True, exclusive=exclusive, strict=strict)
else:
raise ValueError("wipe mode %s not supported" % mode)
def get_scsi_wwid(device, replace_whitespace=False):
"""
Issue a call to scsi_id utility to get WWID of the device.
"""
cmd = ['/lib/udev/scsi_id', '--whitelisted', '--device=%s' % device]
if replace_whitespace:
cmd.append('--replace-whitespace')
try:
(out, err) = util.subp(cmd, capture=True)
LOG.debug("scsi_id output raw:\n%s\nerror:\n%s", out, err)
scsi_wwid = out.rstrip('\n')
return scsi_wwid
except util.ProcessExecutionError as e:
LOG.warn("Failed to get WWID: %s", e)
return None
def mdadm_assemble(md_devname=None,
devices=[],
spares=[],
scan=False,
ignore_errors=False):
# md_devname is a /dev/XXXX
# devices is non-empty list of /dev/xxx
# if spares is non-empt list append of /dev/xxx
cmd = ["mdadm", "--assemble"]
if scan:
cmd += ['--scan', '-v']
else:
valid_mdname(md_devname)
cmd += [md_devname, "--run"] + devices
if spares:
cmd += spares
try:
# mdadm assemble returns 1 when no arrays are found. this might not be
# an error depending on the situation this function was called in, so
# accept a return code of 1
# mdadm assemble returns 2 when called on an array that is already
# assembled. this is not an error, so accept return code of 2
# all other return codes can be accepted with ignore_error set to true
scan, err = util.subp(cmd, capture=True, rcs=[0, 1, 2])
LOG.debug('mdadm assemble scan results:\n%s\n%s', scan, err)
scan, err = util.subp(['mdadm', '--detail', '--scan', '-v'],
capture=True,
rcs=[0, 1])
LOG.debug('mdadm detail scan after assemble:\n%s\n%s', scan, err)
except util.ProcessExecutionError:
LOG.warning("mdadm_assemble had unexpected return code")
if not ignore_errors:
raise
udev.udevadm_settle()
def _filter_lvm_info(lvtool, match_field, query_field, match_key):
"""
filter output of pv/vg/lvdisplay tools
"""
(out, _) = util.subp([
lvtool, '-C', '--separator', _SEP, '--noheadings', '-o', ','.join(
[match_field, query_field])
],
capture=True)
return [
qf
for (mf,
qf) in [l.strip().split(_SEP) for l in out.strip().splitlines()]
if mf == match_key
]
def mdadm_query_detail(md_devname, export=MDADM_USE_EXPORT):
valid_mdname(md_devname)
cmd = ["mdadm", "--query", "--detail"]
if export:
cmd.extend(["--export"])
cmd.extend([md_devname])
(out, _err) = util.subp(cmd, capture=True)
if export:
data = __mdadm_export_to_dict(out)
else:
data = __upgrade_detail_dict(__mdadm_detail_to_dict(out))
return data
def superblock_asdict(device=None, data=None):
""" Convert output from bcache-super-show into a dictionary"""
if not device and not data:
raise ValueError('Supply a device name, or data to parse')
if not data:
data, _err = util.subp(['bcache-super-show', device], capture=True)
bcache_super = {}
for line in data.splitlines():
if not line:
continue
values = [val for val in line.split('\t') if val]
bcache_super.update({values[0]: values[1]})
return bcache_super
def dasdview(devname, rawoutput=False):
''' Run dasdview on devname and return dictionary of data.
dasdview --extended has 3 sections
general (2:6), geometry (8:12), extended (14:)
'''
if not os.path.exists(devname):
raise ValueError("Invalid dasd device name: '%s'" % devname)
out, err = util.subp(['dasdview', '--extended', devname], capture=True)
if rawoutput:
return (out, err)
return _parse_dasdview(out)
