def remove_device(mddev, arraydev): assert_valid_devpath(mddev) LOG.info("mdadm remove %s from array %s", arraydev, mddev) out, err = util.subp(["mdadm", "--remove", mddev, arraydev], rcs=[0], capture=True) LOG.debug("mdadm remove:\n%s\n%s", out, err)
def fail_device(mddev, arraydev): assert_valid_devpath(mddev) LOG.info("mdadm mark faulty: %s in array %s", arraydev, mddev) out, err = util.subp(["mdadm", "--fail", mddev, arraydev], rcs=[0], capture=True) LOG.debug("mdadm mark faulty:\n%s\n%s", out, err)
def udevadm_info(path=None): """ Return a dictionary populated by properties of the device specified in the `path` variable via querying udev 'property' database. :params: path: path to device, either /dev or /sys :returns: dictionary of key=value pairs as exported from the udev database :raises: ValueError path is None, ProcessExecutionError on exec error. """ if not path: raise ValueError('Invalid path: "%s"' % path) info_cmd = ['udevadm', 'info', '--query=property', '--export', path] output, _ = util.subp(info_cmd, capture=True) # strip for trailing empty line info = {} for line in output.splitlines(): if not line: continue # maxsplit=1 gives us key and remaininng part of line is value # py2.7 on Trusty doesn't have keyword, pass as argument key, value = line.split('=', 1) if not value: value = None if value: # preserve spaces in values to match udev database try: parsed = shlex.split(value) except ValueError: # strip the leading/ending single tick from udev output before # escaping the value to prevent their inclusion in the result. trimmed_value = value[1:-1] try: quoted = shlex_quote(trimmed_value) LOG.debug( 'udevadm_info: quoting shell-escape chars ' 'in %s=%s -> %s', key, value, quoted) parsed = shlex.split(quoted) except ValueError: escaped_value = (trimmed_value.replace("'", "_").replace( '"', "_")) LOG.debug( 'udevadm_info: replacing shell-escape chars ' 'in %s=%s -> %s', key, value, escaped_value) parsed = shlex.split(escaped_value) if ' ' not in value: info[key] = parsed[0] else: # special case some known entries with spaces, e.g. ID_SERIAL # and DEVLINKS, see tests/unittests/test_udev.py if key == "DEVLINKS": info[key] = shlex.split(parsed[0]) elif key == 'ID_SERIAL': info[key] = parsed[0] else: info[key] = parsed return info
def mdadm_stop(devpath, retries=None): assert_valid_devpath(devpath) if not retries: retries = [0.2] * 60 sync_action = md_sysfs_attr_path(devpath, 'sync_action') sync_max = md_sysfs_attr_path(devpath, 'sync_max') sync_min = md_sysfs_attr_path(devpath, 'sync_min') LOG.info("mdadm stopping: %s" % devpath) for (attempt, wait) in enumerate(retries): try: LOG.debug('mdadm: stop on %s attempt %s', devpath, attempt) # An array in 'resync' state may not be stoppable, attempt to # cancel an ongoing resync val = md_sysfs_attr(devpath, 'sync_action') LOG.debug('%s/sync_max = %s', sync_action, val) if val != "idle": LOG.debug("mdadm: setting array sync_action=idle") try: util.write_file(sync_action, content="idle") except (IOError, OSError) as e: LOG.debug("mdadm: (non-fatal) write to %s failed %s", sync_action, e) # Setting the sync_{max,min} may can help prevent the array from # changing back to 'resync' which may prevent the array from being # stopped val = md_sysfs_attr(devpath, 'sync_max') LOG.debug('%s/sync_max = %s', sync_max, val) if val != "0": LOG.debug("mdadm: setting array sync_{min,max}=0") try: for sync_file in [sync_max, sync_min]: util.write_file(sync_file, content="0") except (IOError, OSError) as e: LOG.debug('mdadm: (non-fatal) write to %s failed %s', sync_file, e) # one wonders why this command doesn't do any of the above itself? out, err = util.subp(["mdadm", "--manage", "--stop", devpath], capture=True) LOG.debug("mdadm stop command output:\n%s\n%s", out, err) LOG.info("mdadm: successfully stopped %s after %s attempt(s)", devpath, attempt+1) return except util.ProcessExecutionError: LOG.warning("mdadm stop failed, retrying ") if os.path.isfile('/proc/mdstat'): LOG.critical("/proc/mdstat:\n%s", util.load_file('/proc/mdstat')) LOG.debug("mdadm: stop failed, retrying in %s seconds", wait) time.sleep(wait) pass raise OSError('Failed to stop mdadm device %s', devpath)
def get_dmsetup_uuid(device): """ get the dm uuid for a specified dmsetup device """ blockdev = block.sysfs_to_devpath(device) (out, _) = util.subp( ['dmsetup', 'info', blockdev, '-C', '-o', 'uuid', '--noheadings'], capture=True) return out.strip()
def test_subp_combined_stderr_stdout(self): """Providing combine_capture as True redirects stderr to stdout.""" data = b'hello world' (out, err) = util.subp(self.stdin2err, combine_capture=True, decode=False, data=data) self.assertEqual(err, b'') self.assertEqual(out, data)
def get_volume_uuid(path): """ Get uuid of disk with given path. This address uniquely identifies the device and remains consistant across reboots """ (out, _err) = util.subp(["blkid", "-o", "export", path], capture=True) for line in out.splitlines(): if "UUID" in line: return line.split('=')[-1] return ''
def _extract_mpath_data(cmd, show_verb): data, _err = util.subp(cmd, capture=True) result = [] for line in data.splitlines(): mp_dict = util.load_shell_content(line, add_empty=True) LOG.debug('Extracted multipath %s fields: %s', show_verb, mp_dict) if mp_dict: result.append(mp_dict) return result
def iscsiadm_discovery(portal): # only supported type for now type = 'sendtargets' if not portal: raise ValueError("Portal must be specified for discovery") cmd = [ "iscsiadm", "--mode=discovery", "--type=%s" % type, "--portal=%s" % portal ] try: util.subp(cmd, capture=True, log_captured=True) except util.ProcessExecutionError as e: LOG.warning("iscsiadm_discovery to %s failed with exit code %d", portal, e.exit_code) raise
def export_armour(key): """Export gpg key, armoured key gets returned""" try: (armour, _) = util.subp(["gpg", "--export", "--armour", key], capture=True) except util.ProcessExecutionError as error: # debug, since it happens for any key not on the system initially LOG.debug('Failed to export armoured key "%s": %s', key, error) armour = None return armour
def _subp_wrap_popen(self, cmd, kwargs, stdout=b'', stderr=b'', returncodes=None): # mocks the subprocess.Popen as expected from subp # checks that subp returned the output of 'communicate' and # returns the (args, kwargs) that Popen() was called with. # returncodes is a list to cover, one for each expected call if returncodes is None: returncodes = [0] capture = kwargs.get('capture') mreturncodes = mock.PropertyMock(side_effect=iter(returncodes)) with mock.patch("curtin.util.subprocess.Popen") as m_popen: sp = mock.Mock() m_popen.return_value = sp if capture: sp.communicate.return_value = (stdout, stderr) else: sp.communicate.return_value = (None, None) type(sp).returncode = mreturncodes ret = util.subp(cmd, **kwargs) # popen may be called once or > 1 for retries, but must be called. self.assertTrue(m_popen.called) # communicate() needs to have been called. self.assertTrue(sp.communicate.called) if capture: # capture response is decoded if decode is not False decode = kwargs.get('decode', "replace") if decode is False: self.assertEqual(stdout.decode(stdout, stderr), ret) else: self.assertEqual((stdout.decode(errors=decode), stderr.decode(errors=decode)), ret) else: # if capture is false, then return is None, None self.assertEqual((None, None), ret) # if target is not provided or is /, chroot should not be used calls = m_popen.call_args_list popen_args, popen_kwargs = calls[-1] target = paths.target_path(kwargs.get('target', None)) unshcmd = self.mock_get_unshare_pid_args.return_value if target == "/": self.assertEqual(unshcmd + list(cmd), popen_args[0]) else: self.assertEqual(unshcmd + ['chroot', target] + list(cmd), popen_args[0]) return calls
def zkey_supported(strict=True): """ Return True if zkey cmd present and can generate keys, else False.""" LOG.debug('Checking if zkey encryption is supported...') try: util.load_kernel_module('pkey') except util.ProcessExecutionError as err: msg = "Failed to load 'pkey' kernel module" LOG.error(msg + ": %s" % err) if strict else LOG.warning(msg) return False try: with tempfile.NamedTemporaryFile() as tf: util.subp(['zkey', 'generate', tf.name], capture=True) LOG.debug('zkey encryption supported.') return True except util.ProcessExecutionError as err: msg = "zkey not supported" LOG.error(msg + ": %s" % err) if strict else LOG.warning(msg) return False
def stop_all_unused_multipath_devices(): """ Stop all unused multipath devices. """ multipath = util.which('multipath') # Command multipath is not available only when multipath-tools package # is not installed. Nothing needs to be done in this case because system # doesn't create multipath devices without this package installed and we # have nothing to stop. if not multipath: return # Command multipath -F flushes all unused multipath device maps cmd = [multipath, '-F'] try: # unless multipath cleared *everything* it will exit with 1 util.subp(cmd, rcs=[0, 1]) except util.ProcessExecutionError as e: LOG.warn("Failed to stop multipath devices: %s", e)
def zpool_list(): """ Return a list of zfs pool names which have been imported :returns: List of strings """ # -H drops the header, -o specifies an attribute to fetch out, _err = util.subp(['zpool', 'list', '-H', '-o', 'name'], capture=True) return out.splitlines()
def split_lvm_name(full): """ split full lvm name into tuple of (volgroup, lv_name) """ # 'dmsetup splitname' is the authoratative source for lvm name parsing (out, _) = util.subp([ 'dmsetup', 'splitname', full, '-c', '--noheadings', '--separator', _SEP, '-o', 'vg_name,lv_name' ], capture=True) return out.strip().split(_SEP)
def zfs_mount(poolname, volume): """ Mount zfs pool/volume :param poolname: String used to specify the pool in which to create the filesystem. :param volume: String used as the name of the filesystem. :returns: None :raises: ValueError: raises exceptions on missing/bad input. :raises: ProcessExecutionError: raised on unhandled exceptions from invoking `zfs mount`. """ if not isinstance(poolname, util.string_types) or not poolname: raise ValueError("Invalid poolname: %s", poolname) if not isinstance(volume, util.string_types) or not volume: raise ValueError("Invalid volume: %s", volume) cmd = ['zfs', 'mount', _join_pool_volume(poolname, volume)] util.subp(cmd, capture=True)
def lvm_scan(activate=True): """ run full scan for volgroups, logical volumes and physical volumes """ # prior to xenial, lvmetad is not packaged, so even if a tool supports # flag --cache it has no effect. In Xenial and newer the --cache flag is # used (if lvmetad is running) to ensure that the data cached by # lvmetad is updated. # before appending the cache flag though, check if lvmetad is running. this # ensures that we do the right thing even if lvmetad is supported but is # not running release = distro.lsb_release().get('codename') if release in [None, 'UNAVAILABLE']: LOG.warning('unable to find release number, assuming xenial or later') release = 'xenial' for cmd in [['pvscan'], ['vgscan', '--mknodes']]: if release != 'precise' and lvmetad_running(): cmd.append('--cache') util.subp(cmd, capture=True)
def apply_power_state(pstate): """ power_state: delay: 5 mode: poweroff message: Bye Bye """ cmd = load_power_state(pstate) if not cmd: return LOG.info("powering off with %s", cmd) fid = os.fork() if fid == 0: try: util.subp(cmd) os._exit(0) except Exception as e: LOG.warn("%s returned non-zero: %s" % (cmd, e)) os._exit(1) return
def run_python(self, args): start_dir = os.getcwd() cmd = [sys.executable] for i in args: cmd.append(i) env = os.environ.copy() env['CURTIN_STACKTRACE'] = "1" try: os.chdir(self.extract_dir) return util.subp(cmd, capture=True, env=env) finally: os.chdir(start_dir)
def dmname_to_blkdev_mapping(): """ Use dmsetup ls output to build a dict of DM_NAME, /dev/dm-x values.""" data, _err = util.subp(['dmsetup', 'ls', '-o', 'blkdevname'], capture=True) mapping = {} if data and data.strip() != "No devices found": LOG.debug('multipath: dmsetup ls output:\n%s', data) for line in data.splitlines(): if line: dm_name, blkdev = line.split('\t') # (dm-1) -> /dev/dm-1 mapping[dm_name] = '/dev/' + blkdev.strip('()') return mapping
def tar_xattr_opts(cmd=None): # if tar cmd supports xattrs, return the required flags to extract them. if cmd is None: cmd = ['tar'] if isinstance(cmd, str): cmd = [cmd] (out, _err) = util.subp(cmd + ['--help'], capture=True) if "xattr" in out: return ['--xattrs', '--xattrs-include=*'] return []
def rescan_block_devices(devices=None, warn_on_fail=True): """ run 'blockdev --rereadpt' for all block devices not currently mounted """ if not devices: unused = get_unused_blockdev_info() devices = [] for devname, data in unused.items(): if data.get('RM') == "1": continue if data.get('RO') != "0" or data.get('TYPE') != "disk": continue devices.append(data['device_path']) if not devices: LOG.debug("no devices found to rescan") return # blockdev needs /dev/ parameters, convert if needed cmd = ['blockdev', '--rereadpt'] + [ dev if dev.startswith('/dev/') else sysfs_to_devpath(dev) for dev in devices ] try: util.subp(cmd, capture=True) except util.ProcessExecutionError as e: if warn_on_fail: # FIXME: its less than ideal to swallow this error, but until # we fix LP: #1489521 we kind of need to. LOG.warn( "Error rescanning devices, possibly known issue LP: #1489521") # Reformatting the exception output so as to not trigger # vmtest scanning for Unexepected errors in install logfile LOG.warn("cmd: %s\nstdout:%s\nstderr:%s\nexit_code:%s", e.cmd, e.stdout, e.stderr, e.exit_code) udevadm_settle() return
def create_backing_device(backing_device, cache_device, cache_mode, cset_uuid): backing_device_sysfs = sys_block_path(backing_device) target_sysfs_path = os.path.join(backing_device_sysfs, "bcache") # there should not be any pre-existing bcache device bdir = os.path.join(backing_device_sysfs, "bcache") if os.path.exists(bdir): raise RuntimeError('Unexpected old bcache device: %s', backing_device) LOG.debug('Creating a backing device on %s', backing_device) util.subp(["make-bcache", "-B", backing_device]) ensure_bcache_is_registered(backing_device, target_sysfs_path) # via the holders we can identify which bcache device we just created # for a given backing device from .clear_holders import get_holders holders = get_holders(backing_device) if len(holders) != 1: err = ('Invalid number {} of holding devices:' ' "{}"'.format(len(holders), holders)) LOG.error(err) raise ValueError(err) [bcache_dev] = holders LOG.debug('The just created bcache device is {}'.format(holders)) if cache_device: # if we specify both then we need to attach backing to cache if cset_uuid: attach_backing_to_cacheset(backing_device, cache_device, cset_uuid) else: msg = "Invalid cset_uuid: {}".format(cset_uuid) LOG.error(msg) raise ValueError(msg) if cache_mode: set_cache_mode(bcache_dev, cache_mode) return dev_path(bcache_dev)
def wipe_volume(path, mode="superblock", exclusive=True, strict=False): """wipe a volume/block device :param path: a path to a block device :param mode: how to wipe it. pvremove: wipe a lvm physical volume zero: write zeros to the entire volume random: write random data (/dev/urandom) to the entire volume superblock: zero the beginning and the end of the volume superblock-recursive: zero the beginning of the volume, the end of the volume and beginning and end of any partitions that are known to be on this device. :param exclusive: boolean to control how path is opened :param strict: boolean to control when to raise errors on write failures """ if mode == "pvremove": # We need to use --force --force in case it's already in a volgroup and # pvremove doesn't want to remove it # If pvremove is run and there is no label on the system, # then it exits with 5. That is also okay, because we might be # wiping something that is already blank util.subp(['pvremove', '--force', '--force', '--yes', path], rcs=[0, 5], capture=True) lvm.lvm_scan() elif mode == "zero": wipe_file(path, exclusive=exclusive) elif mode == "random": with open("/dev/urandom", "rb") as reader: wipe_file(path, reader=reader.read, exclusive=exclusive) elif mode == "superblock": quick_zero(path, partitions=False, exclusive=exclusive, strict=strict) elif mode == "superblock-recursive": quick_zero(path, partitions=True, exclusive=exclusive, strict=strict) else: raise ValueError("wipe mode %s not supported" % mode)
def get_scsi_wwid(device, replace_whitespace=False): """ Issue a call to scsi_id utility to get WWID of the device. """ cmd = ['/lib/udev/scsi_id', '--whitelisted', '--device=%s' % device] if replace_whitespace: cmd.append('--replace-whitespace') try: (out, err) = util.subp(cmd, capture=True) LOG.debug("scsi_id output raw:\n%s\nerror:\n%s", out, err) scsi_wwid = out.rstrip('\n') return scsi_wwid except util.ProcessExecutionError as e: LOG.warn("Failed to get WWID: %s", e) return None
def mdadm_assemble(md_devname=None, devices=[], spares=[], scan=False, ignore_errors=False): # md_devname is a /dev/XXXX # devices is non-empty list of /dev/xxx # if spares is non-empt list append of /dev/xxx cmd = ["mdadm", "--assemble"] if scan: cmd += ['--scan', '-v'] else: valid_mdname(md_devname) cmd += [md_devname, "--run"] + devices if spares: cmd += spares try: # mdadm assemble returns 1 when no arrays are found. this might not be # an error depending on the situation this function was called in, so # accept a return code of 1 # mdadm assemble returns 2 when called on an array that is already # assembled. this is not an error, so accept return code of 2 # all other return codes can be accepted with ignore_error set to true scan, err = util.subp(cmd, capture=True, rcs=[0, 1, 2]) LOG.debug('mdadm assemble scan results:\n%s\n%s', scan, err) scan, err = util.subp(['mdadm', '--detail', '--scan', '-v'], capture=True, rcs=[0, 1]) LOG.debug('mdadm detail scan after assemble:\n%s\n%s', scan, err) except util.ProcessExecutionError: LOG.warning("mdadm_assemble had unexpected return code") if not ignore_errors: raise udev.udevadm_settle()
def _filter_lvm_info(lvtool, match_field, query_field, match_key): """ filter output of pv/vg/lvdisplay tools """ (out, _) = util.subp([ lvtool, '-C', '--separator', _SEP, '--noheadings', '-o', ','.join( [match_field, query_field]) ], capture=True) return [ qf for (mf, qf) in [l.strip().split(_SEP) for l in out.strip().splitlines()] if mf == match_key ]
def mdadm_query_detail(md_devname, export=MDADM_USE_EXPORT): valid_mdname(md_devname) cmd = ["mdadm", "--query", "--detail"] if export: cmd.extend(["--export"]) cmd.extend([md_devname]) (out, _err) = util.subp(cmd, capture=True) if export: data = __mdadm_export_to_dict(out) else: data = __upgrade_detail_dict(__mdadm_detail_to_dict(out)) return data
def superblock_asdict(device=None, data=None): """ Convert output from bcache-super-show into a dictionary""" if not device and not data: raise ValueError('Supply a device name, or data to parse') if not data: data, _err = util.subp(['bcache-super-show', device], capture=True) bcache_super = {} for line in data.splitlines(): if not line: continue values = [val for val in line.split('\t') if val] bcache_super.update({values[0]: values[1]}) return bcache_super
def dasdview(devname, rawoutput=False): ''' Run dasdview on devname and return dictionary of data. dasdview --extended has 3 sections general (2:6), geometry (8:12), extended (14:) ''' if not os.path.exists(devname): raise ValueError("Invalid dasd device name: '%s'" % devname) out, err = util.subp(['dasdview', '--extended', devname], capture=True) if rawoutput: return (out, err) return _parse_dasdview(out)