Example #1
def lookup_disk(serial):
    Search for a disk by its serial number using /dev/disk/by-id/
    # Get all volumes in /dev/disk/by-id/ containing the serial string. The
    # string specified can be either in the short or long serial format
    # hack, some serials have spaces, udev usually converts ' ' -> '_'
    serial_udev = serial.replace(' ', '_')
    LOG.info('Processing serial %s via udev to %s', serial, serial_udev)

    disks = list(
        filter(lambda x: serial_udev in x, os.listdir("/dev/disk/by-id/")))
    if not disks or len(disks) < 1:
        raise ValueError("no disk with serial '%s' found" % serial_udev)

    # Sort by length and take the shortest path name, as the longer path names
    # will be the partitions on the disk. Then use os.path.realpath to
    # determine the path to the block device in /dev/
    disks.sort(key=lambda x: len(x))
    LOG.debug('lookup_disks found: %s', disks)
    path = os.path.realpath("/dev/disk/by-id/%s" % disks[0])
    LOG.debug('lookup_disks realpath(%s)=%s', disks[0], path)
    if multipath.is_mpath_device(path):
        LOG.debug('Detected multipath device, finding a members')
        info = udevadm_info(path)
        mpath_members = sorted(multipath.find_mpath_members(info['DM_NAME']))
        LOG.debug('mpath members: %s', mpath_members)
        if len(mpath_members):
            path = mpath_members[0]

    if not os.path.exists(path):
        raise ValueError("path '%s' to block device for disk with serial '%s' \
            does not exist" % (path, serial_udev))
    return path
Example #2
def parse_sb_version(device=None, sbdict=None):
    """ Parse bcache 'sb_version' field to integer if possible.

    if not device and not sbdict:
        raise ValueError('Supply a device name or bcache superblock dict')

    if not sbdict:
        sbdict = superblock_asdict(device=device)
        if not sbdict:
            LOG.info('Cannot parse sb.version without bcache superblock')
            return None
    if not isinstance(sbdict, dict):
        raise ValueError('Invalid sbdict type, must be dict')

    sb_version = sbdict.get('sb.version')
        # 'sb.version': '1 [backing device]'
        # 'sb.version': '3 [caching device]'
        version = int(sb_version.split()[0])
    except (AttributeError, ValueError):
            "Failed to parse bcache 'sb.version' field"
            " as integer: %s", sb_version)

    return version
Example #3
def apply_debconf_selections(cfg, target=None):
    """apply_debconf_selections - push content to debconf"""
    # debconf_selections:
    #  set1: |
    #   cloud-init cloud-init/datasources multiselect MAAS
    #  set2: pkg pkg/value string bar
    selsets = cfg.get('debconf_selections')
    if not selsets:
        LOG.debug("debconf_selections was not set in config")

    LOG.debug('Applying debconf selections')
    selections = '\n'.join([selsets[key] for key in sorted(selsets.keys())])
    debconf_set_selections(selections.encode() + b"\n", target=target)

    # get a complete list of packages listed in input
    pkgs_cfgd = set()
    for key, content in selsets.items():
        for line in content.splitlines():
            if line.startswith("#"):
            pkg = re.sub(r"[:\s].*", "", line)

    pkgs_installed = distro.get_installed_packages(target)
    need_reconfig = pkgs_cfgd.intersection(pkgs_installed)
    if len(need_reconfig) == 0:

    dpkg_reconfigure(need_reconfig, target=target)
Example #4
    def needs_formatting(self, blksize, layout, volser):
        """ Determine if DasdDevice attributes matches the required parameters.

        Note that devices that indicate they are unformatted will require

        :param blksize: expected blocksize of the device.
        :param layout: expected disk layout.
        :param volser: expected label, if None, label is ignored.
        :returns: boolean, True if formatting is needed, else False.
        LOG.debug('Checking if dasd %s needs formatting', self.device_id)
        if self.is_not_formatted():
            LOG.debug('dasd %s is not formatted', self.device_id)
            return True

        if int(blksize) != int(self.blocksize()):
            LOG.debug('dasd %s block size (%s) does not match (%s)',
                      self.device_id, self.blocksize(), blksize)
            return True

        if layout != self.disk_layout():
            LOG.debug('dasd %s disk layout (%s) does not match %s',
                      self.device_id, self.disk_layout(), layout)
            return True

        if volser and volser != self.label():
            LOG.debug('dasd %s volser (%s) does not match %s',
                      self.device_id, self.label(), volser)
            return True

        return False
Example #5
def attach_backing_to_cacheset(backing_device, cache_device, cset_uuid):
    LOG.info("Attaching backing device to cacheset: "
             "{} -> {} cset.uuid: {}".format(backing_device, cache_device,
    backing_device_sysfs = sys_block_path(backing_device)
    attach = os.path.join(backing_device_sysfs, "bcache", "attach")
    util.write_file(attach, cset_uuid, mode=None)
Example #6
def mdadm_remove(devpath):

    LOG.info("mdadm removing: %s" % devpath)
    out, err = util.subp(["mdadm", "--remove", devpath],
                         rcs=[0], capture=True)
    LOG.debug("mdadm remove:\n%s\n%s", out, err)
Example #7
def remove_device(mddev, arraydev):

    LOG.info("mdadm remove %s from array %s", arraydev, mddev)
    out, err = util.subp(["mdadm", "--remove", mddev, arraydev],
                         rcs=[0], capture=True)
    LOG.debug("mdadm remove:\n%s\n%s", out, err)
Example #8
def disable_overlayroot(cfg, target):
    # cloud images come with overlayroot, but installed systems need disabled
    disable = cfg.get('disable_overlayroot', True)
    local_conf = os.path.sep.join([target, 'etc/overlayroot.local.conf'])
    if disable and os.path.exists(local_conf):
        LOG.debug("renaming %s to %s", local_conf, local_conf + ".old")
        shutil.move(local_conf, local_conf + ".old")
Example #9
def md_present(mdname):
    """Check if mdname is present in /proc/mdstat"""
    if not mdname:
        raise ValueError('md_present requires a valid md name')

        mdstat = util.load_file('/proc/mdstat')
    except IOError as e:
        if util.is_file_not_found_exc(e):
            LOG.warning('Failed to read /proc/mdstat; '
                        'md modules might not be loaded')
            return False
            raise e

    md_kname = dev_short(mdname)
    # Find lines like:
    # md10 : active raid1 vdc1[1] vda2[0]
    present = [
        line for line in mdstat.splitlines()
        if line.split(":")[0].rstrip() == md_kname
    if len(present) > 0:
        return True
    return False
Example #10
def detect_required_packages(cfg):
    detect packages that will be required in-target by custom config items

    mapping = {
        'storage': block.detect_required_packages_mapping(),
        'network': net.detect_required_packages_mapping(),

    needed_packages = []
    for cfg_type, cfg_map in mapping.items():

        # skip missing or invalid config items, configs may
        # only have network or storage, not always both
        if not isinstance(cfg.get(cfg_type), dict):

        cfg_version = cfg[cfg_type].get('version')
        if not isinstance(cfg_version, int) or cfg_version not in cfg_map:
            msg = ('Supplied configuration version "%s", for config type'
                   '"%s" is not present in the known mapping.' %
                   (cfg_version, cfg_type))
            raise ValueError(msg)

        mapped_config = cfg_map[cfg_version]
        found_reqs = mapped_config['handler'](cfg, mapped_config['mapping'])

    LOG.debug('Curtin config dependencies requires additional packages: %s',
    return needed_packages
Example #11
def ubuntu_core_curthooks(cfg, target=None):
    """ Ubuntu-Core 16 images cannot execute standard curthooks
        Instead we copy in any cloud-init configuration to
        the 'LABEL=writable' partition mounted at target.

    ubuntu_core_target = os.path.join(target, "system-data")
    cc_target = os.path.join(ubuntu_core_target, 'etc/cloud/cloud.cfg.d')

    cloudconfig = cfg.get('cloudconfig', None)
    if cloudconfig:
        # remove cloud-init.disabled, if found
        cloudinit_disable = os.path.join(ubuntu_core_target,
        if os.path.exists(cloudinit_disable):

        handle_cloudconfig(cloudconfig, base_dir=cc_target)

    netconfig = cfg.get('network', None)
    if netconfig:
        LOG.info('Writing network configuration')
        ubuntu_core_netconfig = os.path.join(cc_target,
                        content=config.dump_config({'network': netconfig}))
Example #12
def add_swap(cfg, target, fstab):
    # add swap file per cfg to filesystem root at target. update fstab.
    # swap:
    #  filename: 'swap.img',
    #  size: None # (or 1G)
    #  maxsize: 2G
    if 'swap' in cfg and not cfg.get('swap'):
        LOG.debug("disabling 'add_swap' due to config")

    swapcfg = cfg.get('swap', {})
    fname = swapcfg.get('filename', None)
    size = swapcfg.get('size', None)
    maxsize = swapcfg.get('maxsize', None)

    if size:
        size = util.human2bytes(str(size))
    if maxsize:
        maxsize = util.human2bytes(str(maxsize))

Example #13
def copy_mdadm_conf(mdadm_conf, target):
    if not mdadm_conf:
        LOG.warn("mdadm config must be specified, not copying")

    LOG.info("copying mdadm.conf into target")
    shutil.copy(mdadm_conf, os.path.sep.join([target, 'etc/mdadm/mdadm.conf']))
Example #14
def copy_iscsi_conf(nodes_dir, target):
    if not nodes_dir:
        LOG.warn("nodes directory must be specified, not copying")

    LOG.info("copying iscsi nodes database into target")
    shutil.copytree(nodes_dir, os.path.sep.join([target, 'etc/iscsi/nodes']))
Example #15
def dmsetup_info(devname):
    ''' returns dict of info about device mapper dev.

    {'blkdevname': 'dm-0',
     'blkdevs_used': 'sda5',
     'name': 'sda5_crypt',
     'subsystem': 'CRYPT',
     'uuid': 'CRYPT-LUKS1-2b370697149743b0b2407d11f88311f1-sda5_crypt'
    _SEP = '='
    fields = ('name,uuid,blkdevname,blkdevs_used,subsystem'.split(','))
        (out, _err) = util.subp([
            'dmsetup', 'info', devname, '-C', '-o', ','.join(fields),
            '--noheading', '--separator', _SEP
    except util.ProcessExecutionError as e:
        LOG.error('Failed to run dmsetup info: %s', e)
        return {}

    values = out.strip().split(_SEP)
    info = dict(zip(fields, values))
    return info
Example #16
def validate_config(config, sourcefile=None):
    """Validate storage config object."""
    if not sourcefile:
        sourcefile = ''
        import jsonschema
        jsonschema.validate(config, STORAGE_CONFIG_SCHEMA)
    except ImportError:
        LOG.error('Cannot validate storage config, missing jsonschema')
    except jsonschema.exceptions.ValidationError as e:
        if isinstance(e.instance, int):
            msg = 'Unexpected value (%s) for property "%s"' % (e.path[0],
            raise ValueError(msg)
        if 'type' not in e.instance:
            msg = "%s in %s" % (e.message, e.instance)
            raise ValueError(msg)

        instance_type = e.instance['type']
        stype = get_storage_types().get(instance_type)
        if stype:
                jsonschema.validate(e.instance, stype.schema)
            except jsonschema.exceptions.ValidationError as f:
                msg = "%s in %s\n%s" % (f.message, sourcefile,
                raise (ValueError(msg))
            msg = "Unknown storage type: %s in %s" % (instance_type,
            raise ValueError(msg)
Example #17
def lvm_scan(activate=True, multipath=False):
    run full scan for volgroups, logical volumes and physical volumes
    # prior to xenial, lvmetad is not packaged, so even if a tool supports
    # flag --cache it has no effect. In Xenial and newer the --cache flag is
    # used (if lvmetad is running) to ensure that the data cached by
    # lvmetad is updated.

    # before appending the cache flag though, check if lvmetad is running. this
    # ensures that we do the right thing even if lvmetad is supported but is
    # not running
    release = distro.lsb_release().get('codename')
    if release in [None, 'UNAVAILABLE']:
        LOG.warning('unable to find release number, assuming xenial or later')
        release = 'xenial'

    if multipath:
        # only operate on mp devices
        mponly = 'devices{ filter = [ "a|/dev/mapper/mpath.*|", "r|.*|" ] }'

    for cmd in [['pvscan'], ['vgscan']]:
        if release != 'precise' and lvmetad_running():
        if multipath:
            cmd.extend(['--config', mponly])
        util.subp(cmd, capture=True)
Example #18
def start_clear_holders_deps():
    prepare system for clear holders to be able to scan old devices
    # a mdadm scan has to be started in case there is a md device that needs to
    # be detected. if the scan fails, it is either because there are no mdadm
    # devices on the system, or because there is a mdadm device in a damaged
    # state that could not be started. due to the nature of mdadm tools, it is
    # difficult to know which is the case. if any errors did occur, then ignore
    # them, since no action needs to be taken if there were no mdadm devices on
    # the system, and in the case where there is some mdadm metadata on a disk,
    # but there was not enough to start the array, the call to wipe_volume on
    # all disks and partitions should be sufficient to remove the mdadm
    # metadata
    mdadm.mdadm_assemble(scan=True, ignore_errors=True)
    # scan and activate for logical volumes
    # the bcache module needs to be present to properly detect bcache devs
    # on some systems (precise without hwe kernel) it may not be possible to
    # lad the bcache module bcause it is not present in the kernel. if this
    # happens then there is no need to halt installation, as the bcache devices
    # will never appear and will never prevent the disk from being reformatted

    if not zfs.zfs_supported():
        LOG.warning('zfs filesystem is not supported in this environment')
Example #19
def fail_device(mddev, arraydev):

    LOG.info("mdadm mark faulty: %s in array %s", arraydev, mddev)
    out, err = util.subp(["mdadm", "--fail", mddev, arraydev],
                         rcs=[0], capture=True)
    LOG.debug("mdadm mark faulty:\n%s\n%s", out, err)
Example #20
def restart_iscsi_service():
    LOG.info('restarting iscsi service')
    if util.uses_systemd():
        cmd = ['systemctl', 'reload-or-restart', 'open-iscsi']
        cmd = ['service', 'open-iscsi', 'restart']
    util.subp(cmd, capture=True)
Example #21
    def from_fdasd(cls, devname):
        """Use fdasd to construct a DasdPartitionTable.

        % fdasd --table /dev/dasdc
        reading volume label ..: VOL1
        reading vtoc ..........: ok

        Disk /dev/dasdc:
          cylinders ............: 10017
          tracks per cylinder ..: 15
          blocks per track .....: 12
          bytes per block ......: 4096
          volume label .........: VOL1
          volume serial ........: 0X1522
          max partitions .......: 3

         ------------------------------- tracks -------------------------------
                       Device      start      end   length   Id  System
                  /dev/dasdc1          2    43694    43693    1  Linux native
                  /dev/dasdc2      43695    87387    43693    2  Linux native
                  /dev/dasdc3      87388   131080    43693    3  Linux native
                                  131081   150254    19174       unused
        cmd = ['fdasd', '--table', devname]
        out, _err = util.subp(cmd, capture=True)
        LOG.debug("from_fdasd output:\n---\n%s\n---\n", out)
        return cls.from_fdasd_output(devname, out)
Example #22
def get_iscsi_disks_from_config(cfg):
    """Return a list of IscsiDisk objects for each iscsi volume present."""
    # Construct IscsiDisk objects for each iscsi volume present
    iscsi_disks = [IscsiDisk(volume) for volume in
    LOG.debug('Found %s iscsi disks in storage config', len(iscsi_disks))
    return iscsi_disks
Example #23
    def add_partition(self, partnumber, partsize):
        """ Add a partition to this DasdDevice specifying partnumber and size.

        :param partnumber: integer value of partition number (1, 2 or 3)
        :param partsize: partition sizes in bytes.

        :raises: ValueError on invalid devname

        Example fdasd command with defaults:
          fdasd --verbose --config=/tmp/curtin/dasd-part1.fdasd /dev/dasdb
            "add_partition: partnumber: %s partsize: %s",
            partnumber, partsize)

        partitions = self._ptable_for_new_partition(partnumber, partsize)
        LOG.debug("fdasd: partitions to be created: %s", partitions)
        content = "\n".join([
            "[%s,%s]" % (part[0], part[1]) for part in partitions
        LOG.debug("fdasd: content=\n%s", content)
        wfp = tempfile.NamedTemporaryFile(suffix=".fdasd", delete=False)
        util.write_file(wfp.name, content)
        cmd = ['fdasd', '--verbose', '--config=%s' % wfp.name, self.devname]
        LOG.debug('Partitioning %s with %s', self.devname, cmd)
            out, err = util.subp(cmd, capture=True)
        except util.ProcessExecutionError as e:
            LOG.error("Partitioning failed: %s", e)
            if os.path.exists(wfp.name):
Example #24
def sys_block_path(devname, add=None, strict=True):
    get path to device in /sys/class/block
    toks = ['/sys/class/block']
    # insert parent dev if devname is partition
    devname = os.path.normpath(devname)
    if devname.startswith('/dev/') and not os.path.exists(devname):
        LOG.warning('block.sys_block_path: devname %s does not exist', devname)
    (parent, partnum) = get_blockdev_for_partition(devname, strict=strict)
    if partnum:


    if add is not None:
    path = os.sep.join(toks)

    if strict and not os.path.exists(path):
        err = OSError("devname '{}' did not have existing syspath '{}'".format(
            devname, path))
        err.errno = errno.ENOENT
        raise err

    return os.path.normpath(path)
Example #25
def get_backing_device(bcache_kname):
    """ For a given bcacheN kname, return the backing device
        bcache sysfs dir.

        bcache0 -> /sys/.../devices/.../device/bcache
    bcache_deps = '/sys/class/block/%s/slaves' % bcache_kname

        # if the bcache device is deleted, this may fail
        deps = os.listdir(bcache_deps)
    except util.FileMissingError as e:
        LOG.debug('Transient race, bcache slave path not found: %s', e)
        return None

    # a running bcache device has two entries in slaves, the cacheset
    # device, and the backing device. There may only be the backing
    # device (if a bcache device is found but not currently attached
    # to a cacheset.
    if len(deps) == 0:
        raise RuntimeError('%s unexpected empty dir: %s' %
                           (bcache_kname, bcache_deps))

    for dev in (sysfs_path(dep) for dep in deps):
        if is_backing(dev):
            return dev

    return None
Example #26
def wipe_file(path, reader=None, buflen=4 * 1024 * 1024, exclusive=True):
    wipe the existing file at path.
    if reader is provided, it will be called as a 'reader(buflen)'
    to provide data for each write.  Otherwise, zeros are used.
    writes will be done in size of buflen.
    if reader:
        readfunc = reader
        buf = buflen * b'\0'

        def readfunc(size):
            return buf

    size = util.file_size(path)
    LOG.debug("%s is %s bytes. wiping with buflen=%s", path, size, buflen)

    with exclusive_open(path, exclusive=exclusive) as fp:
        while True:
            pbuf = readfunc(buflen)
            pos = fp.tell()
            if len(pbuf) != buflen and len(pbuf) + pos < size:
                raise ValueError(
                    "short read on reader got %d expected %d after %d" %
                    (len(pbuf), buflen, pos))

            if pos + buflen >= size:
                fp.write(pbuf[0:size - pos])
Example #27
def dpkg_reconfigure(packages, target=None):
    # For any packages that are already installed, but have preseed data
    # we populate the debconf database, but the filesystem configuration
    # would be preferred on a subsequent dpkg-reconfigure.
    # so, what we have to do is "know" information about certain packages
    # to unconfigure them.
    unhandled = []
    to_config = []
    for pkg in packages:
        if pkg in CONFIG_CLEANERS:
            LOG.debug("unconfiguring %s", pkg)

    if len(unhandled):
            "The following packages were installed and preseeded, "
            "but cannot be unconfigured: %s", unhandled)

    if len(to_config):
        util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
Example #28
def quick_zero(path, partitions=True, exclusive=True):
    zero 1M at front, 1M at end, and 1M at front
    if this is a block device and partitions is true, then
    zero 1M at front and end of each partition.
    buflen = 1024
    count = 1024
    zero_size = buflen * count
    offsets = [0, -zero_size]
    is_block = is_block_device(path)
    if not (is_block or os.path.isfile(path)):
        raise ValueError("%s: not an existing file or block device", path)

    pt_names = []
    if partitions and is_block:
        ptdata = sysfs_partition_data(path)
        for kname, ptnum, start, size in ptdata:
            pt_names.append((dev_path(kname), kname, ptnum))

    for (pt, kname, ptnum) in pt_names:
        LOG.debug('Wiping path: dev:%s kname:%s partnum:%s', pt, kname, ptnum)
        quick_zero(pt, partitions=False)

    LOG.debug("wiping 1M on %s at offsets %s", path, offsets)
    return zero_file_at_offsets(path,
Example #29
def generate_sources_list(cfg, release, mirrors, target=None):
    """ generate_sources_list
        create a source.list file based on a custom or default template
        by replacing mirrors and release in the template
    default_mirrors = get_default_mirrors(distro.get_architecture(target))
    aptsrc = "/etc/apt/sources.list"
    params = {'RELEASE': release}
    for k in mirrors:
        params[k] = mirrors[k]

    tmpl = cfg.get('sources_list', None)
    if tmpl is None:
            "No custom template provided, fall back to modify"
            "mirrors in %s on the target system", aptsrc)
        tmpl = util.load_file(paths.target_path(target, aptsrc))
        # Strategy if no custom template was provided:
        # - Only replacing mirrors
        # - no reason to replace "release" as it is from target anyway
        # - The less we depend upon, the more stable this is against changes
        # - warn if expected original content wasn't found
        tmpl = mirror_to_placeholder(tmpl, default_mirrors['PRIMARY'],
        tmpl = mirror_to_placeholder(tmpl, default_mirrors['SECURITY'],

    orig = paths.target_path(target, aptsrc)
    if os.path.exists(orig):
        os.rename(orig, orig + ".curtin.old")

    rendered = util.render_string(tmpl, params)
    disabled = disable_suites(cfg.get('disable_suites'), rendered, release)
    util.write_file(paths.target_path(target, aptsrc), disabled, mode=0o644)
Example #30
def lookup_disk(serial):
    Search for a disk by its serial number using /dev/disk/by-id/
    # Get all volumes in /dev/disk/by-id/ containing the serial string. The
    # string specified can be either in the short or long serial format
    # hack, some serials have spaces, udev usually converts ' ' -> '_'
    serial_udev = serial.replace(' ', '_')
    LOG.info('Processing serial %s via udev to %s', serial, serial_udev)

    disks = list(
        filter(lambda x: serial_udev in x, os.listdir("/dev/disk/by-id/")))
    if not disks or len(disks) < 1:
        raise ValueError("no disk with serial '%s' found" % serial_udev)

    # Sort by length and take the shortest path name, as the longer path names
    # will be the partitions on the disk. Then use os.path.realpath to
    # determine the path to the block device in /dev/
    disks.sort(key=lambda x: len(x))
    path = os.path.realpath("/dev/disk/by-id/%s" % disks[0])

    if not os.path.exists(path):
        raise ValueError("path '%s' to block device for disk with serial '%s' \
            does not exist" % (path, serial_udev))
    return path