def add_swap(cfg, target, fstab):
# add swap file per cfg to filesystem root at target. update fstab.
#
# swap:
# filename: 'swap.img',
# size: None # (or 1G)
# maxsize: 2G
if 'swap' in cfg and not cfg.get('swap'):
LOG.debug("disabling 'add_swap' due to config")
return
swapcfg = cfg.get('swap', {})
fname = swapcfg.get('filename', None)
size = swapcfg.get('size', None)
maxsize = swapcfg.get('maxsize', None)
if size:
size = util.human2bytes(str(size))
if maxsize:
maxsize = util.human2bytes(str(maxsize))
swap.setup_swapfile(target=target,
fstab=fstab,
swapfile=fname,
size=size,
maxsize=maxsize)
def detect_required_packages(cfg):
"""
detect packages that will be required in-target by custom config items
"""
mapping = {
'storage': block.detect_required_packages_mapping(),
'network': net.detect_required_packages_mapping(),
}
needed_packages = []
for cfg_type, cfg_map in mapping.items():
# skip missing or invalid config items, configs may
# only have network or storage, not always both
if not isinstance(cfg.get(cfg_type), dict):
continue
cfg_version = cfg[cfg_type].get('version')
if not isinstance(cfg_version, int) or cfg_version not in cfg_map:
msg = ('Supplied configuration version "%s", for config type'
'"%s" is not present in the known mapping.' %
(cfg_version, cfg_type))
raise ValueError(msg)
mapped_config = cfg_map[cfg_version]
found_reqs = mapped_config['handler'](cfg, mapped_config['mapping'])
needed_packages.extend(found_reqs)
LOG.debug('Curtin config dependencies requires additional packages: %s',
needed_packages)
return needed_packages
def extract_root_layered_fsimage_url(uri, target):
''' Build images list to consider from a layered structure
uri: URI of the layer file
target: Target file system to provision
return: None
'''
path = _path_from_file_url(uri)
image_stack = _get_image_stack(path)
LOG.debug("Considering fsimages: '%s'", ",".join(image_stack))
tmp_dir = None
try:
# Download every remote images if remote url
if url_helper.urlparse(path).scheme != "":
tmp_dir = tempfile.mkdtemp()
image_stack = _download_layered_images(image_stack, tmp_dir)
# Check that all images exists on disk and are not empty
for img in image_stack:
if not os.path.isfile(img) or os.path.getsize(img) <= 0:
raise ValueError(
"Failed to use fsimage: '%s' doesn't exist " +
"or is invalid", img)
return _extract_root_layered_fsimage(image_stack, target)
finally:
if tmp_dir and os.path.exists(tmp_dir):
shutil.rmtree(tmp_dir)
def render_network_state(target, network_state):
LOG.debug("rendering eni from netconfig")
eni = 'etc/network/interfaces'
netrules = 'etc/udev/rules.d/70-persistent-net.rules'
cc = 'etc/cloud/cloud.cfg.d/curtin-disable-cloudinit-networking.cfg'
eni = os.path.sep.join((
target,
eni,
))
LOG.info('Writing ' + eni)
util.write_file(eni, content=render_interfaces(network_state))
netrules = os.path.sep.join((
target,
netrules,
))
LOG.info('Writing ' + netrules)
util.write_file(netrules, content=render_persistent_net(network_state))
cc_disable = os.path.sep.join((
target,
cc,
))
LOG.info('Writing ' + cc_disable)
util.write_file(cc_disable, content='network: {config: disabled}\n')
def is_connected(devname):
# is_connected isn't really as simple as that. 2 is
# 'physically connected'. 3 is 'not connected'. but a wlan interface will
# always show 3.
try:
iflink = read_sys_net(devname, "iflink", enoent=False)
if iflink == "2":
return True
if not is_wireless(devname):
return False
LOG.debug("'%s' is wireless, basing 'connected' on carrier", devname)
return read_sys_net(devname,
"carrier",
enoent=False,
keyerror=False,
translate={
'0': False,
'1': True
})
except IOError as e:
if e.errno == errno.EINVAL:
return False
raise
def disable_overlayroot(cfg, target):
# cloud images come with overlayroot, but installed systems need disabled
disable = cfg.get('disable_overlayroot', True)
local_conf = os.path.sep.join([target, 'etc/overlayroot.local.conf'])
if disable and os.path.exists(local_conf):
LOG.debug("renaming %s to %s", local_conf, local_conf + ".old")
shutil.move(local_conf, local_conf + ".old")
def quick_zero(path, partitions=True, exclusive=True):
"""
zero 1M at front, 1M at end, and 1M at front
if this is a block device and partitions is true, then
zero 1M at front and end of each partition.
"""
buflen = 1024
count = 1024
zero_size = buflen * count
offsets = [0, -zero_size]
is_block = is_block_device(path)
if not (is_block or os.path.isfile(path)):
raise ValueError("%s: not an existing file or block device", path)
pt_names = []
if partitions and is_block:
ptdata = sysfs_partition_data(path)
for kname, ptnum, start, size in ptdata:
pt_names.append((dev_path(kname), kname, ptnum))
pt_names.reverse()
for (pt, kname, ptnum) in pt_names:
LOG.debug('Wiping path: dev:%s kname:%s partnum:%s', pt, kname, ptnum)
quick_zero(pt, partitions=False)
LOG.debug("wiping 1M on %s at offsets %s", path, offsets)
return zero_file_at_offsets(path,
offsets,
buflen=buflen,
count=count,
exclusive=exclusive)
def apt_command(args):
""" Main entry point for curtin apt-config standalone command
This does not read the global config as handled by curthooks, but
instead one can specify a different "target" and a new cfg via --config
"""
cfg = config.load_command_config(args, {})
if args.target is not None:
target = args.target
else:
state = util.load_command_environment()
target = state['target']
if target is None:
sys.stderr.write("Unable to find target. "
"Use --target or set TARGET_MOUNT_POINT\n")
sys.exit(2)
apt_cfg = cfg.get("apt")
# if no apt config section is available, do nothing
if apt_cfg is not None:
LOG.debug("Handling apt to target %s with config %s", target, apt_cfg)
try:
with util.ChrootableTarget(target, sys_resolvconf=True):
handle_apt(apt_cfg, target)
except (RuntimeError, TypeError, ValueError, IOError):
LOG.exception("Failed to configure apt features '%s'", apt_cfg)
sys.exit(1)
else:
LOG.info("No apt config provided, skipping")
sys.exit(0)
def disconnect_target_disks(target_root_path=None):
target_nodes_path = paths.target_path(target_root_path, '/etc/iscsi/nodes')
fails = []
if os.path.isdir(target_nodes_path):
for target in os.listdir(target_nodes_path):
if target not in iscsiadm_sessions():
LOG.debug('iscsi target %s not active, skipping', target)
continue
# conn is "host,port,lun"
for conn in os.listdir(
os.path.sep.join([target_nodes_path, target])):
host, port, _ = conn.split(',')
try:
util.subp(['sync'])
iscsiadm_logout(target, '%s:%s' % (host, port))
except util.ProcessExecutionError as e:
fails.append(target)
LOG.warn("Unable to logout of iSCSI target %s: %s",
target, e)
else:
LOG.warning('Skipping disconnect: failed to find iscsi nodes path: %s',
target_nodes_path)
if fails:
raise RuntimeError(
"Unable to logout of iSCSI targets: %s" % ', '.join(fails))
def fail_device(mddev, arraydev):
assert_valid_devpath(mddev)
LOG.info("mdadm mark faulty: %s in array %s", arraydev, mddev)
out, err = util.subp(["mdadm", "--fail", mddev, arraydev],
rcs=[0], capture=True)
LOG.debug("mdadm mark faulty:\n%s\n%s", out, err)
def md_check(md_devname, raidlevel, devices, spares, container):
''' Check passed in variables from storage configuration against
the system we're running upon.
'''
LOG.debug('RAID validation: ' +
'name={} raidlevel={} devices={} spares={} container={}'.format(
md_devname, raidlevel, devices, spares, container))
assert_valid_devpath(md_devname)
detail = mdadm_query_detail(md_devname)
if raidlevel != "container":
md_check_array_state(md_devname)
md_check_raidlevel(md_devname, detail, raidlevel)
md_check_uuid(md_devname)
if container is None:
md_check_devices(md_devname, devices)
md_check_spares(md_devname, spares)
md_check_array_membership(md_devname, devices + spares)
else:
if 'MD_CONTAINER' not in detail:
raise ValueError("%s is not in a container" % (md_devname))
actual_container = os.path.realpath(detail['MD_CONTAINER'])
if actual_container != container:
raise ValueError("%s is in container %r, not %r" %
(md_devname, actual_container, container))
LOG.debug('RAID array OK: ' + md_devname)
def needs_formatting(self, blksize, layout, volser):
""" Determine if DasdDevice attributes matches the required parameters.
Note that devices that indicate they are unformatted will require
formatting.
:param blksize: expected blocksize of the device.
:param layout: expected disk layout.
:param volser: expected label, if None, label is ignored.
:returns: boolean, True if formatting is needed, else False.
"""
LOG.debug('Checking if dasd %s needs formatting', self.device_id)
if self.is_not_formatted():
LOG.debug('dasd %s is not formatted', self.device_id)
return True
if int(blksize) != int(self.blocksize()):
LOG.debug('dasd %s block size (%s) does not match (%s)',
self.device_id, self.blocksize(), blksize)
return True
if layout != self.disk_layout():
LOG.debug('dasd %s disk layout (%s) does not match %s',
self.device_id, self.disk_layout(), layout)
return True
if volser and volser != self.label():
LOG.debug('dasd %s volser (%s) does not match %s',
self.device_id, self.label(), volser)
return True
return False
def shutdown_lvm(device):
"""
Shutdown specified lvm device.
"""
device = block.sys_block_path(device)
# lvm devices have a dm directory that containes a file 'name' containing
# '{volume group}-{logical volume}'. The volume can be freed using lvremove
name_file = os.path.join(device, 'dm', 'name')
lvm_name = util.load_file(name_file).strip()
(vg_name, lv_name) = lvm.split_lvm_name(lvm_name)
vg_lv_name = "%s/%s" % (vg_name, lv_name)
devname = "/dev/" + vg_lv_name
# wipe contents of the logical volume first
LOG.info('Wiping lvm logical volume: %s', devname)
block.quick_zero(devname, partitions=False)
# remove the logical volume
LOG.debug('using "lvremove" on %s', vg_lv_name)
util.subp(['lvremove', '--force', '--force', vg_lv_name])
# if that was the last lvol in the volgroup, get rid of volgroup
if len(lvm.get_lvols_in_volgroup(vg_name)) == 0:
pvols = lvm.get_pvols_in_volgroup(vg_name)
util.subp(['vgremove', '--force', '--force', vg_name], rcs=[0, 5])
# wipe the underlying physical volumes
for pv in pvols:
LOG.info('Wiping lvm physical volume: %s', pv)
block.quick_zero(pv, partitions=False)
# refresh lvmetad
lvm.lvm_scan()
def from_fdasd(cls, devname):
"""Use fdasd to construct a DasdPartitionTable.
% fdasd --table /dev/dasdc
reading volume label ..: VOL1
reading vtoc ..........: ok
Disk /dev/dasdc:
cylinders ............: 10017
tracks per cylinder ..: 15
blocks per track .....: 12
bytes per block ......: 4096
volume label .........: VOL1
volume serial ........: 0X1522
max partitions .......: 3
------------------------------- tracks -------------------------------
Device start end length Id System
/dev/dasdc1 2 43694 43693 1 Linux native
/dev/dasdc2 43695 87387 43693 2 Linux native
/dev/dasdc3 87388 131080 43693 3 Linux native
131081 150254 19174 unused
exiting...
"""
cmd = ['fdasd', '--table', devname]
out, _err = util.subp(cmd, capture=True)
LOG.debug("from_fdasd output:\n---\n%s\n---\n", out)
return cls.from_fdasd_output(devname, out)
def mdadm_remove(devpath):
assert_valid_devpath(devpath)
LOG.info("mdadm removing: %s" % devpath)
out, err = util.subp(["mdadm", "--remove", devpath],
rcs=[0], capture=True)
LOG.debug("mdadm remove:\n%s\n%s", out, err)
def apply_preserve_sources_list(target):
# protect the just generated sources.list from cloud-init
cloudfile = "/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg"
target_ver = distro.get_package_version('cloud-init', target=target)
if not target_ver:
LOG.info(
"Attempt to read cloud-init version from target returned "
"'%s', not writing preserve_sources_list config.", target_ver)
return
cfg = {'apt': {'preserve_sources_list': True}}
if target_ver['major'] < 1:
# anything cloud-init 0.X.X will get the old config key.
cfg = {'apt_preserve_sources_list': True}
try:
util.write_file(paths.target_path(target, cloudfile),
config.dump_config(cfg),
mode=0o644)
LOG.debug("Set preserve_sources_list to True in %s with: %s",
cloudfile, cfg)
except IOError:
LOG.exception(
"Failed to protect /etc/apt/sources.list from cloud-init in '%s'",
cloudfile)
raise
def lookup_disk(serial):
"""
Search for a disk by its serial number using /dev/disk/by-id/
"""
# Get all volumes in /dev/disk/by-id/ containing the serial string. The
# string specified can be either in the short or long serial format
# hack, some serials have spaces, udev usually converts ' ' -> '_'
serial_udev = serial.replace(' ', '_')
LOG.info('Processing serial %s via udev to %s', serial, serial_udev)
disks = list(
filter(lambda x: serial_udev in x, os.listdir("/dev/disk/by-id/")))
if not disks or len(disks) < 1:
raise ValueError("no disk with serial '%s' found" % serial_udev)
# Sort by length and take the shortest path name, as the longer path names
# will be the partitions on the disk. Then use os.path.realpath to
# determine the path to the block device in /dev/
disks.sort(key=lambda x: len(x))
LOG.debug('lookup_disks found: %s', disks)
path = os.path.realpath("/dev/disk/by-id/%s" % disks[0])
# /dev/dm-X
if multipath.is_mpath_device(path):
info = udevadm_info(path)
path = os.path.join('/dev/mapper', info['DM_NAME'])
# /dev/sdX
elif multipath.is_mpath_member(path):
mp_name = multipath.find_mpath_id_by_path(path)
path = os.path.join('/dev/mapper', mp_name)
if not os.path.exists(path):
raise ValueError("path '%s' to block device for disk with serial '%s' \
does not exist" % (path, serial_udev))
LOG.debug('block.lookup_disk() returning path %s', path)
return path
def mdadm_assemble(md_devname=None, devices=[], spares=[], scan=False,
ignore_errors=False):
# md_devname is a /dev/XXXX
# devices is non-empty list of /dev/xxx
# if spares is non-empt list append of /dev/xxx
cmd = ["mdadm", "--assemble"]
if scan:
cmd += ['--scan', '-v']
else:
valid_mdname(md_devname)
cmd += [md_devname, "--run"] + devices
if spares:
cmd += spares
try:
# mdadm assemble returns 1 when no arrays are found. this might not be
# an error depending on the situation this function was called in, so
# accept a return code of 1
# mdadm assemble returns 2 when called on an array that is already
# assembled. this is not an error, so accept return code of 2
# all other return codes can be accepted with ignore_error set to true
scan, err = util.subp(cmd, capture=True, rcs=[0, 1, 2])
LOG.debug('mdadm assemble scan results:\n%s\n%s', scan, err)
scan, err = util.subp(['mdadm', '--detail', '--scan', '-v'],
capture=True, rcs=[0, 1])
LOG.debug('mdadm detail scan after assemble:\n%s\n%s',
scan, err)
except util.ProcessExecutionError:
LOG.warning("mdadm_assemble had unexpected return code")
if not ignore_errors:
raise
udev.udevadm_settle()
def is_mpath_partition(devpath):
if devpath.startswith('/dev/dm-'):
if 'DM_PART' in udev.udevadm_info(devpath):
LOG.debug("%s is multipath device partition", devpath)
return True
return False
def wipe_file(path, reader=None, buflen=4 * 1024 * 1024, exclusive=True):
"""
wipe the existing file at path.
if reader is provided, it will be called as a 'reader(buflen)'
to provide data for each write. Otherwise, zeros are used.
writes will be done in size of buflen.
"""
if reader:
readfunc = reader
else:
buf = buflen * b'\0'
def readfunc(size):
return buf
size = util.file_size(path)
LOG.debug("%s is %s bytes. wiping with buflen=%s", path, size, buflen)
with exclusive_open(path, exclusive=exclusive) as fp:
while True:
pbuf = readfunc(buflen)
pos = fp.tell()
if len(pbuf) != buflen and len(pbuf) + pos < size:
raise ValueError(
"short read on reader got %d expected %d after %d" %
(len(pbuf), buflen, pos))
if pos + buflen >= size:
fp.write(pbuf[0:size - pos])
break
else:
fp.write(pbuf)
def force_devmapper_symlinks():
"""Check if /dev/mapper/mpath* files are symlinks, if not trigger udev."""
LOG.debug('Verifying /dev/mapper/mpath* files are symlinks')
needs_trigger = []
for mp_id, dm_dev in dmname_to_blkdev_mapping().items():
if mp_id.startswith('mpath'):
mapper_path = '/dev/mapper/' + mp_id
if not os.path.islink(mapper_path):
LOG.warning(
'Found invalid device mapper mp path: %s, removing',
mapper_path)
util.del_file(mapper_path)
needs_trigger.append((mapper_path, dm_dev))
if len(needs_trigger):
for (mapper_path, dm_dev) in needs_trigger:
LOG.debug('multipath: regenerating symlink for %s (%s)',
mapper_path, dm_dev)
util.subp([
'udevadm', 'trigger', '--subsystem-match=block',
'--action=add', '/sys/class/block/' + os.path.basename(dm_dev)
])
udev.udevadm_settle(exists=mapper_path)
if not os.path.islink(mapper_path):
LOG.error('Failed to regenerate udev symlink %s', mapper_path)
def get_backing_device(bcache_kname):
""" For a given bcacheN kname, return the backing device
bcache sysfs dir.
bcache0 -> /sys/.../devices/.../device/bcache
"""
bcache_deps = '/sys/class/block/%s/slaves' % bcache_kname
try:
# if the bcache device is deleted, this may fail
deps = os.listdir(bcache_deps)
except util.FileMissingError as e:
LOG.debug('Transient race, bcache slave path not found: %s', e)
return None
# a running bcache device has two entries in slaves, the cacheset
# device, and the backing device. There may only be the backing
# device (if a bcache device is found but not currently attached
# to a cacheset.
if len(deps) == 0:
raise RuntimeError('%s unexpected empty dir: %s' %
(bcache_kname, bcache_deps))
for dev in (sysfs_path(dep) for dep in deps):
if is_backing(dev):
return dev
return None
def get_iscsi_disks_from_config(cfg):
"""Return a list of IscsiDisk objects for each iscsi volume present."""
# Construct IscsiDisk objects for each iscsi volume present
iscsi_disks = [IscsiDisk(volume) for volume in
get_iscsi_volumes_from_config(cfg)]
LOG.debug('Found %s iscsi disks in storage config', len(iscsi_disks))
return iscsi_disks
def remove_device(mddev, arraydev):
assert_valid_devpath(mddev)
LOG.info("mdadm remove %s from array %s", arraydev, mddev)
out, err = util.subp(["mdadm", "--remove", mddev, arraydev],
rcs=[0], capture=True)
LOG.debug("mdadm remove:\n%s\n%s", out, err)
def dpkg_reconfigure(packages, target=None):
# For any packages that are already installed, but have preseed data
# we populate the debconf database, but the filesystem configuration
# would be preferred on a subsequent dpkg-reconfigure.
# so, what we have to do is "know" information about certain packages
# to unconfigure them.
unhandled = []
to_config = []
for pkg in packages:
if pkg in CONFIG_CLEANERS:
LOG.debug("unconfiguring %s", pkg)
CONFIG_CLEANERS[pkg](target)
to_config.append(pkg)
else:
unhandled.append(pkg)
if len(unhandled):
LOG.warn(
"The following packages were installed and preseeded, "
"but cannot be unconfigured: %s", unhandled)
if len(to_config):
util.subp(['dpkg-reconfigure', '--frontend=noninteractive'] +
list(to_config),
data=None,
target=target,
capture=True)
def apply_debconf_selections(cfg, target=None):
"""apply_debconf_selections - push content to debconf"""
# debconf_selections:
# set1: |
# cloud-init cloud-init/datasources multiselect MAAS
# set2: pkg pkg/value string bar
selsets = cfg.get('debconf_selections')
if not selsets:
LOG.debug("debconf_selections was not set in config")
return
LOG.debug('Applying debconf selections')
selections = '\n'.join([selsets[key] for key in sorted(selsets.keys())])
debconf_set_selections(selections.encode() + b"\n", target=target)
# get a complete list of packages listed in input
pkgs_cfgd = set()
for key, content in selsets.items():
for line in content.splitlines():
if line.startswith("#"):
continue
pkg = re.sub(r"[:\s].*", "", line)
pkgs_cfgd.add(pkg)
pkgs_installed = distro.get_installed_packages(target)
need_reconfig = pkgs_cfgd.intersection(pkgs_installed)
if len(need_reconfig) == 0:
return
dpkg_reconfigure(need_reconfig, target=target)
def clean_cloud_init(target):
"""clean out any local cloud-init config"""
flist = glob.glob(
paths.target_path(target, "/etc/cloud/cloud.cfg.d/*dpkg*"))
LOG.debug("cleaning cloud-init config from: %s", flist)
for dpkg_cfg in flist:
os.unlink(dpkg_cfg)
def hook(args):
if not args.target:
raise ValueError("Target must be provided or set in environment")
LOG.debug("Finalizing %s" % args.target)
curtin.util.run_hook_if_exists(args.target, "finalize")
sys.exit(0)
def iscsiadm_logout(target, portal):
LOG.debug('iscsiadm_logout: target=%s portal=%s', target, portal)
cmd = ['iscsiadm', '--mode=node', '--targetname=%s' % target,
'--portal=%s' % portal, '--logout']
util.subp(cmd, capture=True, log_captured=True)
udev.udevadm_settle()
def iscsiadm_set_automatic(target, portal):
LOG.debug('iscsiadm_set_automatic: target=%s portal=%s', target, portal)
cmd = ['iscsiadm', '--mode=node', '--targetname=%s' % target,
'--portal=%s' % portal, '--op=update',
'--name=node.startup', '--value=automatic']
util.subp(cmd, capture=True, log_captured=True)
