def is_online(device):
""" check if device is online """
sys_path = sys_block_path(device)
device_size = util.load_file(
os.path.join(sys_path, 'size'))
# a block device should have non-zero size to be usable
return int(device_size) > 0
def md_present(mdname):
"""Check if mdname is present in /proc/mdstat"""
if not mdname:
raise ValueError('md_present requires a valid md name')
try:
mdstat = util.load_file('/proc/mdstat')
except IOError as e:
if util.is_file_not_found_exc(e):
LOG.warning('Failed to read /proc/mdstat; '
'md modules might not be loaded')
return False
else:
raise e
md_kname = dev_short(mdname)
# Find lines like:
# md10 : active raid1 vdc1[1] vda2[0]
present = [
line for line in mdstat.splitlines()
if line.split(":")[0].rstrip() == md_kname
]
if len(present) > 0:
return True
return False
def shutdown_lvm(device):
"""
Shutdown specified lvm device.
"""
device = block.sys_block_path(device)
# lvm devices have a dm directory that containes a file 'name' containing
# '{volume group}-{logical volume}'. The volume can be freed using lvremove
name_file = os.path.join(device, 'dm', 'name')
lvm_name = util.load_file(name_file).strip()
(vg_name, lv_name) = lvm.split_lvm_name(lvm_name)
vg_lv_name = "%s/%s" % (vg_name, lv_name)
devname = "/dev/" + vg_lv_name
# wipe contents of the logical volume first
LOG.info('Wiping lvm logical volume: %s', devname)
block.quick_zero(devname, partitions=False)
# remove the logical volume
LOG.debug('using "lvremove" on %s', vg_lv_name)
util.subp(['lvremove', '--force', '--force', vg_lv_name])
# if that was the last lvol in the volgroup, get rid of volgroup
if len(lvm.get_lvols_in_volgroup(vg_name)) == 0:
pvols = lvm.get_pvols_in_volgroup(vg_name)
util.subp(['vgremove', '--force', '--force', vg_name], rcs=[0, 5])
# wipe the underlying physical volumes
for pv in pvols:
LOG.info('Wiping lvm physical volume: %s', pv)
block.quick_zero(pv, partitions=False)
# refresh lvmetad
lvm.lvm_scan()
def test_load_file_simple(self):
fname = 'test.cfg'
contents = "#curtin-config"
with simple_mocked_open(content=contents) as m_open:
loaded_contents = util.load_file(fname, decode=False)
self.assertEqual(contents, loaded_contents)
m_open.assert_called_with(fname, 'rb')
def test_udevadm_info_escape_quotes(self, m_subp):
"""verify we escape quotes when we fail to split. """
mypath = '/dev/sdz'
datafile = 'tests/data/udevadm_info_sandisk_cruzer.txt'
m_subp.return_value = (util.load_file(datafile), "")
info = udevadm_info(mypath)
m_subp.assert_called_with(
['udevadm', 'info', '--query=property', '--export', mypath],
capture=True)
"""
Replicate what udevadm_info parsing does and use pdb to examine what's
happening.
(Pdb) original_value
"SanDisk'"
(Pdb) quoted_value
'\'SanDisk\'"\'"\'\''
(Pdb) split_value
["SanDisk'"]
(Pdb) expected_value
"SanDisk'"
"""
original_value = "SanDisk'"
quoted_value = shlex_quote(original_value)
split_value = shlex.split(quoted_value)
expected_value = split_value if ' ' in split_value else split_value[0]
self.assertEqual(expected_value, info['SCSI_VENDOR'])
self.assertEqual(expected_value, info['SCSI_VENDOR_ENC'])
def test_load_file_respects_decode_false(self, mock_decode):
fname = 'test.cfg'
contents = b'start \xc3\xa9 end'
with simple_mocked_open(contents):
loaded_contents = util.load_file(fname, decode=False)
self.assertEqual(type(loaded_contents), bytes)
self.assertEqual(loaded_contents, contents)
def run_install(self, cfg):
# runs an install with a provided config
# return stdout, stderr, exit_code, log_file_contents
# src_url is required by arg parser, but not used here.
src_url = 'file://' + self.tmpd + '/NOT_USED'
mcfg = cfg.copy()
log_file = cfg_file = None
rc = None
try:
log_file = tempfile.mktemp(dir=self.tmpd)
cfg_file = tempfile.mktemp(dir=self.tmpd)
mcfg['install'] = cfg.get('install', {})
mcfg['install']['log_file'] = log_file
mcfg['sources'] = {'testsrc': src_url}
util.write_file(cfg_file, json.dumps(mcfg))
print(json.dumps(mcfg))
try:
out, err = self.run_main(['install', '--config=' + cfg_file])
rc = 0
except util.ProcessExecutionError as e:
out = e.stdout
err = e.stderr
rc = e.exit_code
log_contents = util.load_file(log_file)
finally:
for f in [f for f in (log_file, cfg_file) if f]:
os.unlink(f)
return out, err, rc, log_contents
def _maybe_remove_legacy_eth0(target,
path="etc/network/interfaces.d/eth0.cfg"):
"""Ubuntu cloud images previously included a 'eth0.cfg' that had
hard coded content. That file would interfere with the rendered
configuration if it was present.
if the file does not exist do nothing.
If the file exists:
- with known content, remove it and warn
- with unknown content, leave it and warn
"""
cfg = util.target_path(target, path=path)
if not os.path.exists(cfg):
LOG.warn('Failed to find legacy network conf file %s', cfg)
return
bmsg = "Dynamic networking config may not apply."
try:
contents = util.load_file(cfg)
known_contents = ["auto eth0", "iface eth0 inet dhcp"]
lines = [
f.strip() for f in contents.splitlines() if not f.startswith("#")
]
if lines == known_contents:
util.del_file(cfg)
msg = "removed %s with known contents" % cfg
else:
msg = (bmsg + " '%s' exists with user configured content." % cfg)
except Exception:
msg = bmsg + " %s exists, but could not be read." % cfg
LOG.exception(msg)
raise
LOG.warn(msg)
def test_default_is_zero(self):
flen = 1024
myfile = self.tmp_path("def_zero")
util.write_file(myfile, flen * b'\1', omode="wb")
block.wipe_file(myfile)
found = util.load_file(myfile, decode=False)
self.assertEqual(found, flen * b'\0')
def generate_sources_list(cfg, release, mirrors, target=None):
""" generate_sources_list
create a source.list file based on a custom or default template
by replacing mirrors and release in the template
"""
default_mirrors = get_default_mirrors(distro.get_architecture(target))
aptsrc = "/etc/apt/sources.list"
params = {'RELEASE': release}
for k in mirrors:
params[k] = mirrors[k]
tmpl = cfg.get('sources_list', None)
if tmpl is None:
LOG.info(
"No custom template provided, fall back to modify"
"mirrors in %s on the target system", aptsrc)
tmpl = util.load_file(paths.target_path(target, aptsrc))
# Strategy if no custom template was provided:
# - Only replacing mirrors
# - no reason to replace "release" as it is from target anyway
# - The less we depend upon, the more stable this is against changes
# - warn if expected original content wasn't found
tmpl = mirror_to_placeholder(tmpl, default_mirrors['PRIMARY'],
"$MIRROR")
tmpl = mirror_to_placeholder(tmpl, default_mirrors['SECURITY'],
"$SECURITY")
orig = paths.target_path(target, aptsrc)
if os.path.exists(orig):
os.rename(orig, orig + ".curtin.old")
rendered = util.render_string(tmpl, params)
disabled = disable_suites(cfg.get('disable_suites'), rendered, release)
util.write_file(paths.target_path(target, aptsrc), disabled, mode=0o644)
def _md_get_members_list(devpath, state_check):
md_dev, _partno = get_blockdev_for_partition(devpath)
sysfs_md = sys_block_path(md_dev, "md")
return [
dev_path(dev[4:]) for dev in os.listdir(sysfs_md)
if (dev.startswith('dev-') and state_check(
util.load_file(os.path.join(sysfs_md, dev, 'state')).strip()))
]
def md_get_devices_list(devpath):
sysfs_md = sys_block_path(devpath, "md")
devices = [
dev_path(dev[4:]) for dev in os.listdir(sysfs_md)
if (dev.startswith('dev-') and util.load_file(
os.path.join(sysfs_md, dev, 'state')).strip() != 'spare')
]
return devices
def get_blockdev_for_partition(devpath, strict=True):
"""
find the parent device for a partition.
returns a tuple of the parent block device and the partition number
if device is not a partition, None will be returned for partition number
"""
# normalize path
rpath = os.path.realpath(devpath)
# convert an entry in /dev/ to parent disk and partition number
# if devpath is a block device and not a partition, return (devpath, None)
base = '/sys/class/block'
# input of /dev/vdb, /dev/disk/by-label/foo, /sys/block/foo,
# /sys/block/class/foo, or just foo
syspath = os.path.join(base, path_to_kname(devpath))
# don't need to try out multiple sysfs paths as path_to_kname handles cciss
if strict and not os.path.exists(syspath):
raise OSError("%s had no syspath (%s)" % (devpath, syspath))
if rpath.startswith('/dev/dm-'):
parent_info = multipath.mpath_partition_to_mpath_id_and_partnumber(
rpath)
if parent_info is not None:
mpath_id, ptnum = parent_info
return os.path.realpath('/dev/mapper/' + mpath_id), ptnum
ptpath = os.path.join(syspath, "partition")
if not os.path.exists(ptpath):
return (rpath, None)
ptnum = util.load_file(ptpath).rstrip()
# for a partition, real syspath is something like:
# /sys/devices/pci0000:00/0000:00:04.0/virtio1/block/vda/vda1
rsyspath = os.path.realpath(syspath)
disksyspath = os.path.dirname(rsyspath)
diskmajmin = util.load_file(os.path.join(disksyspath, "dev")).rstrip()
diskdevpath = os.path.realpath("/dev/block/%s" % diskmajmin)
# diskdevpath has something like 253:0
# and udev has put links in /dev/block/253:0 to the device name in /dev/
return (diskdevpath, ptnum)
def test_skip_rename_resolvconf_gone(self, m_rename):
self.m_shutil.copy.side_effect = self.mycopy
self.m_shutil.rmtree.side_effect = self.mydel
with util.ChrootableTarget(self.target):
tp = paths.target_path(self.target, path='/etc/resolv.conf')
target_conf = util.load_file(tp)
self.assertEqual(self.host_content, target_conf)
self.assertEqual(0, m_rename.call_count)
def mdadm_stop(devpath, retries=None):
assert_valid_devpath(devpath)
if not retries:
retries = [0.2] * 60
sync_action = md_sysfs_attr_path(devpath, 'sync_action')
sync_max = md_sysfs_attr_path(devpath, 'sync_max')
sync_min = md_sysfs_attr_path(devpath, 'sync_min')
LOG.info("mdadm stopping: %s" % devpath)
for (attempt, wait) in enumerate(retries):
try:
LOG.debug('mdadm: stop on %s attempt %s', devpath, attempt)
# An array in 'resync' state may not be stoppable, attempt to
# cancel an ongoing resync
val = md_sysfs_attr(devpath, 'sync_action')
LOG.debug('%s/sync_max = %s', sync_action, val)
if val != "idle":
LOG.debug("mdadm: setting array sync_action=idle")
try:
util.write_file(sync_action, content="idle")
except (IOError, OSError) as e:
LOG.debug("mdadm: (non-fatal) write to %s failed %s",
sync_action, e)
# Setting the sync_{max,min} may can help prevent the array from
# changing back to 'resync' which may prevent the array from being
# stopped
val = md_sysfs_attr(devpath, 'sync_max')
LOG.debug('%s/sync_max = %s', sync_max, val)
if val != "0":
LOG.debug("mdadm: setting array sync_{min,max}=0")
try:
for sync_file in [sync_max, sync_min]:
util.write_file(sync_file, content="0")
except (IOError, OSError) as e:
LOG.debug('mdadm: (non-fatal) write to %s failed %s',
sync_file, e)
# one wonders why this command doesn't do any of the above itself?
out, err = util.subp(["mdadm", "--manage", "--stop", devpath],
capture=True)
LOG.debug("mdadm stop command output:\n%s\n%s", out, err)
LOG.info("mdadm: successfully stopped %s after %s attempt(s)",
devpath, attempt+1)
return
except util.ProcessExecutionError:
LOG.warning("mdadm stop failed, retrying ")
if os.path.isfile('/proc/mdstat'):
LOG.critical("/proc/mdstat:\n%s",
util.load_file('/proc/mdstat'))
LOG.debug("mdadm: stop failed, retrying in %s seconds", wait)
time.sleep(wait)
pass
raise OSError('Failed to stop mdadm device %s', devpath)
def test_chrootable_target_renames_and_copies_resolvconf_if_symlink(self):
target_rconf = os.path.join(self.target, 'etc/resolv.conf')
os.symlink('../run/foobar/wark.conf', target_rconf)
self.m_shutil.copy.side_effect = self.mycopy
self.m_shutil.rmtree.side_effect = self.mydel
with util.ChrootableTarget(self.target):
target_conf = util.load_file(
paths.target_path(self.target, path='/etc/resolv.conf'))
self.assertEqual(self.host_content, target_conf)
def shutdown_swap(path):
"""release swap device from kernel swap pool if present"""
procswaps = util.load_file('/proc/swaps')
for swapline in procswaps.splitlines():
if swapline.startswith(path):
msg = ('Removing %s from active use as swap device, '
'needed for storage config' % path)
LOG.warning(msg)
util.subp(['swapoff', path])
return
def test_reader_fhandle(self):
srcfile = self.tmp_path("fhandle_src")
trgfile = self.tmp_path("fhandle_trg")
data = '\n'.join(["this is source file." for f in range(0, 10)] + [])
util.write_file(srcfile, data)
util.write_file(trgfile, 'a' * len(data))
with open(srcfile, "rb") as fp:
block.wipe_file(trgfile, reader=fp.read)
found = util.load_file(trgfile)
self.assertEqual(data, found)
def sysfs_partition_data(blockdev=None, sysfs_path=None):
# given block device or sysfs_path, return a list of tuples
# of (kernel_name, number, offset, size)
if blockdev:
blockdev = os.path.normpath(blockdev)
sysfs_path = sys_block_path(blockdev)
elif sysfs_path:
# use normpath to ensure that paths with trailing slash work
sysfs_path = os.path.normpath(sysfs_path)
blockdev = os.path.join('/dev', os.path.basename(sysfs_path))
else:
raise ValueError("Blockdev and sysfs_path cannot both be None")
# queue property is only on parent devices, ie, we can't read
# /sys/class/block/vda/vda1/queue/* as queue is only on the
# parent device
sysfs_prefix = sysfs_path
(parent, partnum) = get_blockdev_for_partition(blockdev)
if partnum:
sysfs_prefix = sys_block_path(parent)
partnum = int(partnum)
block_size = int(
util.load_file(os.path.join(sysfs_prefix, 'queue/logical_block_size')))
unit = block_size
ptdata = []
for part_sysfs in get_sysfs_partitions(sysfs_prefix):
data = {}
for sfile in ('partition', 'start', 'size'):
dfile = os.path.join(part_sysfs, sfile)
if not os.path.isfile(dfile):
continue
data[sfile] = int(util.load_file(dfile))
if partnum is None or data['partition'] == partnum:
ptdata.append((
path_to_kname(part_sysfs),
data['partition'],
data['start'] * unit,
data['size'] * unit,
))
return ptdata
def sysfs_to_dict(input):
""" Simple converter for loading key: value lines from a recursive
grep -r . <sysfs path> > sysfs_data
"""
data = util.load_file(input)
return {
y[0]: ":".join(y[1:])
for y in [x.split(":") for x in data.split("\n") if len(x) > 0]
}
def test_mount_apply_skips_mounting_swap(self, m_mount_fstab_data):
"""mount_apply does not mount swap fs, but should write fstab."""
fdata = block_meta.FstabData(spec="/dev/xxxx1",
path="none",
fstype='swap')
fstab = self.tmp_path("fstab")
block_meta.mount_apply(fdata, fstab=fstab)
contents = util.load_file(fstab)
self.assertEqual(0, m_mount_fstab_data.call_count)
self.assertIn("/dev/xxxx1", contents)
self.assertIn("swap", contents)
def test_apt_srcl_custom(self):
"""test_apt_srcl_custom - Test rendering a custom source template"""
cfg = yaml.safe_load(YAML_TEXT_CUSTOM_SL)
target = self.new_root
arch = util.get_architecture()
# would fail inside the unittest context
with mock.patch.object(util, 'get_architecture', return_value=arch):
with mock.patch.object(util,
'lsb_release',
return_value={'codename': 'fakerel'}):
apt_config.handle_apt(cfg, target)
self.assertEqual(
EXPECTED_CONVERTED_CONTENT,
util.load_file(util.target_path(target, "/etc/apt/sources.list")))
cloudfile = util.target_path(
target, '/etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg')
self.assertEqual({'apt_preserve_sources_list': True},
yaml.load(util.load_file(cloudfile)))
def md_sysfs_attr(md_devname, attrname):
""" Return the attribute str of an md device found under the 'md' dir """
attrdata = ''
if not valid_mdname(md_devname):
raise ValueError('Invalid md devicename: [{}]'.format(md_devname))
sysfs_attr_path = md_sysfs_attr_path(md_devname, attrname)
if os.path.isfile(sysfs_attr_path):
attrdata = util.load_file(sysfs_attr_path).strip()
return attrdata
def test_chrootable_target_renames_and_copies_resolvconf(self):
content = "target_resolvconf"
util.write_file(os.path.join(self.target, 'etc/resolv.conf'), content)
self.m_shutil.copy.side_effect = self.mycopy
self.m_shutil.rmtree.side_effect = self.mydel
with util.ChrootableTarget(self.target):
target_conf = util.load_file(
paths.target_path(self.target, path='/etc/resolv.conf'))
self.assertEqual(self.host_content, target_conf)
def load_tfile(filename):
""" load_tfile
load file and return content after decoding
"""
try:
content = util.load_file(filename, decode=True)
except Exception as error:
print('failed to load file content for test: %s' % error)
raise
return content
def add_size_to_holders_tree(tree):
"""add size information to generated holders trees"""
size_file = os.path.join(tree['device'], 'size')
# size file is always represented in 512 byte sectors even if
# underlying disk uses a larger logical_block_size
size = ((512 * int(util.load_file(size_file)))
if os.path.exists(size_file) else None)
tree['size'] = util.bytes2human(size) if args.human else str(size)
for holder in tree['holders']:
add_size_to_holders_tree(holder)
return tree
def test_reader_used(self):
flen = 17
def reader(size):
return size * b'\1'
myfile = self.tmp_path("reader_used")
# populate with nulls
util.write_file(myfile, flen * b'\0', omode="wb")
block.wipe_file(myfile, reader=reader, buflen=flen)
found = util.load_file(myfile, decode=False)
self.assertEqual(found, flen * b'\1')
def get_supported_filesystems():
""" Return a list of filesystems that the kernel currently supports
as read from /proc/filesystems.
Raises RuntimeError if /proc/filesystems does not exist.
"""
proc_fs = "/proc/filesystems"
if not os.path.exists(proc_fs):
raise RuntimeError("Unable to read 'filesystems' from %s" % proc_fs)
return [l.split('\t')[1].strip()
for l in util.load_file(proc_fs).splitlines()]
def check_dos_signature(device):
"""
check if there is a dos partition table signature present on device
"""
# the last 2 bytes of a dos partition table have the signature with the
# value 0xAA55. the dos partition table is always 0x200 bytes long, even if
# the underlying disk uses a larger logical block size, so the start of
# this signature must be at 0x1fe
# https://en.wikipedia.org/wiki/Master_boot_record#Sector_layout
devname = dev_path(path_to_kname(device))
return (is_block_device(devname) and util.file_size(devname) >= 0x200 and
(util.load_file(devname, decode=False, read_len=2, offset=0x1fe)
== b'\x55\xAA'))
def blocksize(self):
""" Read and return device_id's 'blocksize' value.
:param: device_id: string of device ccw bus_id.
:returns: string: the device's current blocksize.
"""
blkattr = 'block/*/queue/hw_sector_size'
# In practice there will only be one entry in the directory
# /sys/bus/ccw/devices/{device_id}/block/, but in case
# something strange happens and there are more, this assumes
# all block devices connected to the dasd have the same block
# size...
path = glob.glob(self.ccw_device_attr_path(blkattr))[0]
return util.load_file(path)