def test_exclusive_open_non_exclusive_exception(self, mock_os_open,
mock_holders,
mock_list_mounts,
mock_os_close,
mock_util_fuser):
flen = 1024
myfile = self.tmp_path("my_exclusive_file")
util.write_file(myfile, flen * b'\1', omode="wb")
mock_os_open.side_effect = OSError("NO_O_EXCL")
mock_holders.return_value = ['md1']
mock_list_mounts.return_value = []
mock_util_fuser.return_value = {}
with self.assertRaises(OSError):
with block.exclusive_open(myfile) as fp:
fp.close()
mock_os_open.assert_called_with(myfile, os.O_RDWR | os.O_EXCL)
mock_holders.assert_called_with(myfile)
mock_list_mounts.assert_called_with(myfile)
self.assertEqual([], mock_os_close.call_args_list)
def replace_grub_cmdline_linux_default(target, new_args):
# we always update /etc/default/grub to avoid "hiding" the override in
# a grub.d directory.
newcontent = 'GRUB_CMDLINE_LINUX_DEFAULT="%s"' % " ".join(new_args)
target_grubconf = target_path(target, '/etc/default/grub')
content = ""
if os.path.exists(target_grubconf):
content = util.load_file(target_grubconf)
existing = re.search(r'GRUB_CMDLINE_LINUX_DEFAULT=.*', content,
re.MULTILINE)
if existing:
omode = 'w+'
updated_content = content[:existing.start()]
updated_content += newcontent
updated_content += content[existing.end():]
else:
omode = 'a+'
updated_content = newcontent + '\n'
util.write_file(target_grubconf, updated_content, omode=omode)
LOG.debug('updated %s to set: %s', target_grubconf, newcontent)
def test_curthooks_cloud_config_remove_disabled(self, mock_handle_cc):
self.target = self.tmp_dir()
uc_cloud = os.path.join(self.target, 'system-data', 'etc/cloud')
cc_disabled = os.path.join(uc_cloud, 'cloud-init.disabled')
cc_path = os.path.join(uc_cloud, 'cloud.cfg.d')
util.ensure_dir(uc_cloud)
util.write_file(cc_disabled, content="# disable cloud-init\n")
cfg = {
'cloudconfig': {
'file1': {
'content': "Hello World!\n",
}
}
}
self.assertTrue(os.path.exists(cc_disabled))
curthooks.ubuntu_core_curthooks(cfg, target=self.target)
mock_handle_cc.assert_called_with(cfg.get('cloudconfig'),
base_dir=cc_path)
self.assertFalse(os.path.exists(cc_disabled))
def render_network_state(target, network_state):
LOG.debug("rendering eni from netconfig")
eni = 'etc/network/interfaces'
netrules = 'etc/udev/rules.d/70-persistent-net.rules'
cc = 'etc/cloud/cloud.cfg.d/curtin-disable-cloudinit-networking.cfg'
eni = os.path.sep.join((
target,
eni,
))
LOG.info('Writing ' + eni)
util.write_file(eni, content=render_interfaces(network_state))
netrules = os.path.sep.join((
target,
netrules,
))
LOG.info('Writing ' + netrules)
util.write_file(netrules, content=render_persistent_net(network_state))
cc_disable = os.path.sep.join((
target,
cc,
))
LOG.info('Writing ' + cc_disable)
util.write_file(cc_disable, content='network: {config: disabled}\n')
def set_sync_action(devpath, action=None, retries=None):
assert_valid_devpath(devpath)
if not action:
return
if not retries:
retries = [0.2] * 60
sync_action = md_sysfs_attr_path(devpath, 'sync_action')
if not os.path.exists(sync_action):
# arrays without sync_action can't set values
return
LOG.info("mdadm set sync_action=%s on array %s", action, devpath)
for (attempt, wait) in enumerate(retries):
try:
LOG.debug('mdadm: set sync_action %s attempt %s',
devpath, attempt)
val = md_sysfs_attr(devpath, 'sync_action').strip()
LOG.debug('sync_action = "%s" ? "%s"', val, action)
if val != action:
LOG.debug("mdadm: setting array sync_action=%s", action)
try:
util.write_file(sync_action, content=action)
except (IOError, OSError) as e:
LOG.debug("mdadm: (non-fatal) write to %s failed %s",
sync_action, e)
else:
LOG.debug("mdadm: set array sync_action=%s SUCCESS", action)
return
except util.ProcessExecutionError:
LOG.debug(
"mdadm: set sync_action failed, retrying in %s seconds", wait)
time.sleep(wait)
pass
def apply_apt_proxy_config(cfg, proxy_fname, config_fname):
"""apply_apt_proxy_config
Applies any apt*proxy config from if specified
"""
# Set up any apt proxy
cfgs = (('proxy', 'Acquire::http::Proxy "%s";'),
('http_proxy', 'Acquire::http::Proxy "%s";'),
('ftp_proxy', 'Acquire::ftp::Proxy "%s";'),
('https_proxy', 'Acquire::https::Proxy "%s";'))
proxies = [fmt % cfg.get(name) for (name, fmt) in cfgs if cfg.get(name)]
if len(proxies):
LOG.debug("write apt proxy info to %s", proxy_fname)
util.write_file(proxy_fname, '\n'.join(proxies) + '\n')
elif os.path.isfile(proxy_fname):
util.del_file(proxy_fname)
LOG.debug("no apt proxy configured, removed %s", proxy_fname)
if cfg.get('conf', None):
LOG.debug("write apt config info to %s", config_fname)
util.write_file(config_fname, cfg.get('conf'))
elif os.path.isfile(config_fname):
util.del_file(config_fname)
LOG.debug("no apt config configured, removed %s", config_fname)
def register_bcache(bcache_device):
LOG.debug('register_bcache: %s > /sys/fs/bcache/register', bcache_device)
util.write_file('/sys/fs/bcache/register', bcache_device, mode=None)
def write_label(label, device):
""" write label to bcache device """
bcache_sys_attr = os.path.join(sysfs_path(device), 'label')
util.write_file(bcache_sys_attr, content=label, mode=None)
def add_apt_sources(srcdict,
target=None,
template_params=None,
aa_repo_match=None):
"""
add entries in /etc/apt/sources.list.d for each abbreviated
sources.list entry in 'srcdict'. When rendering template, also
include the values in dictionary searchList
"""
if template_params is None:
template_params = {}
if aa_repo_match is None:
raise ValueError('did not get a valid repo matcher')
if not isinstance(srcdict, dict):
raise TypeError('unknown apt format: %s' % (srcdict))
for filename in srcdict:
ent = srcdict[filename]
if 'filename' not in ent:
ent['filename'] = filename
add_apt_key(ent['filename'], ent, target)
if 'source' not in ent:
continue
source = ent['source']
if source == 'proposed':
source = APT_SOURCES_PROPOSED
source = util.render_string(source, template_params)
if not ent['filename'].startswith("/"):
ent['filename'] = os.path.join("/etc/apt/sources.list.d/",
ent['filename'])
if not ent['filename'].endswith(".list"):
ent['filename'] += ".list"
if aa_repo_match(source):
with util.ChrootableTarget(target,
sys_resolvconf=True) as in_chroot:
try:
in_chroot.subp(["add-apt-repository", source],
retries=(1, 2, 5, 10))
except util.ProcessExecutionError:
LOG.exception("add-apt-repository failed.")
raise
continue
sourcefn = paths.target_path(target, ent['filename'])
try:
contents = "%s\n" % (source)
util.write_file(sourcefn, contents, omode="a")
except IOError as detail:
LOG.exception("failed write to file %s: %s", sourcefn, detail)
raise
distro.apt_update(target=target,
force=True,
comment="apt-source changed config")
return
def centos_apply_network_config(netcfg, target=None):
""" CentOS images execute built-in curthooks which only supports
simple networking configuration. This hook enables advanced
network configuration via config passthrough to the target.
"""
def cloud_init_repo(version):
if not version:
raise ValueError('Missing required version parameter')
return CLOUD_INIT_YUM_REPO_TEMPLATE % version
if netcfg:
LOG.info('Removing embedded network configuration (if present)')
ifcfgs = glob.glob(
util.target_path(target, 'etc/sysconfig/network-scripts') +
'/ifcfg-*')
# remove ifcfg-* (except ifcfg-lo)
for ifcfg in ifcfgs:
if os.path.basename(ifcfg) != "ifcfg-lo":
util.del_file(ifcfg)
LOG.info(
'Checking cloud-init in target [%s] for network '
'configuration passthrough support.', target)
passthrough = net.netconfig_passthrough_available(target)
LOG.debug('passthrough available via in-target: %s', passthrough)
# if in-target cloud-init is not updated, upgrade via cloud-init repo
if not passthrough:
cloud_init_yum_repo = (util.target_path(
target, 'etc/yum.repos.d/curtin-cloud-init.repo'))
# Inject cloud-init daily yum repo
util.write_file(cloud_init_yum_repo,
content=cloud_init_repo(rpm_get_dist_id(target)))
# we separate the installation of repository packages (epel,
# cloud-init-el-release) as we need a new invocation of yum
# to read the newly installed repo files.
YUM_CMD = ['yum', '-y', '--noplugins', 'install']
retries = [1] * 30
with util.ChrootableTarget(target) as in_chroot:
# ensure up-to-date ca-certificates to handle https mirror
# connections
in_chroot.subp(YUM_CMD + ['ca-certificates'],
capture=True,
log_captured=True,
retries=retries)
in_chroot.subp(YUM_CMD + ['epel-release'],
capture=True,
log_captured=True,
retries=retries)
in_chroot.subp(YUM_CMD + ['cloud-init-el-release'],
log_captured=True,
capture=True,
retries=retries)
in_chroot.subp(YUM_CMD + ['cloud-init'],
capture=True,
log_captured=True,
retries=retries)
# remove cloud-init el-stable bootstrap repo config as the
# cloud-init-el-release package points to the correct repo
util.del_file(cloud_init_yum_repo)
# install bridge-utils if needed
with util.ChrootableTarget(target) as in_chroot:
try:
in_chroot.subp(['rpm', '-q', 'bridge-utils'],
capture=False,
rcs=[0])
except util.ProcessExecutionError:
LOG.debug('Image missing bridge-utils package, installing')
in_chroot.subp(YUM_CMD + ['bridge-utils'],
capture=True,
log_captured=True,
retries=retries)
LOG.info('Passing network configuration through to target')
net.render_netconfig_passthrough(target, netconfig={'network': netcfg})
def detect_and_handle_multipath(cfg, target):
DEFAULT_MULTIPATH_PACKAGES = ['multipath-tools-boot']
mpcfg = cfg.get('multipath', {})
mpmode = mpcfg.get('mode', 'auto')
mppkgs = mpcfg.get('packages', DEFAULT_MULTIPATH_PACKAGES)
mpbindings = mpcfg.get('overwrite_bindings', True)
if isinstance(mppkgs, str):
mppkgs = [mppkgs]
if mpmode == 'disabled':
return
if mpmode == 'auto' and not block.detect_multipath(target):
return
LOG.info("Detected multipath devices. Installing support via %s", mppkgs)
util.install_packages(mppkgs, target=target)
replace_spaces = True
try:
# check in-target version
pkg_ver = util.get_package_version('multipath-tools', target=target)
LOG.debug("get_package_version:\n%s", pkg_ver)
LOG.debug("multipath version is %s (major=%s minor=%s micro=%s)",
pkg_ver['semantic_version'], pkg_ver['major'],
pkg_ver['minor'], pkg_ver['micro'])
# multipath-tools versions < 0.5.0 do _NOT_ want whitespace replaced
# i.e. 0.4.X in Trusty.
if pkg_ver['semantic_version'] < 500:
replace_spaces = False
except Exception as e:
LOG.warn(
"failed reading multipath-tools version, "
"assuming it wants no spaces in wwids: %s", e)
multipath_cfg_path = os.path.sep.join([target, '/etc/multipath.conf'])
multipath_bind_path = os.path.sep.join([target, '/etc/multipath/bindings'])
# We don't want to overwrite multipath.conf file provided by the image.
if not os.path.isfile(multipath_cfg_path):
# Without user_friendly_names option enabled system fails to boot
# if any of the disks has spaces in its name. Package multipath-tools
# has bug opened for this issue (LP: 1432062) but it was not fixed yet.
multipath_cfg_content = '\n'.join([
'# This file was created by curtin while installing the system.',
'defaults {', ' user_friendly_names yes', '}', ''
])
util.write_file(multipath_cfg_path, content=multipath_cfg_content)
if mpbindings or not os.path.isfile(multipath_bind_path):
# we do assume that get_devices_for_mp()[0] is /
target_dev = block.get_devices_for_mp(target)[0]
wwid = block.get_scsi_wwid(target_dev,
replace_whitespace=replace_spaces)
blockdev, partno = block.get_blockdev_for_partition(target_dev)
mpname = "mpath0"
grub_dev = "/dev/mapper/" + mpname
if partno is not None:
grub_dev += "-part%s" % partno
LOG.debug("configuring multipath install for root=%s wwid=%s",
grub_dev, wwid)
multipath_bind_content = '\n'.join([
'# This file was created by curtin while installing the system.',
"%s %s" % (mpname, wwid), '# End of content generated by curtin.',
'# Everything below is maintained by multipath subsystem.', ''
])
util.write_file(multipath_bind_path, content=multipath_bind_content)
grub_cfg = os.path.sep.join(
[target, '/etc/default/grub.d/50-curtin-multipath.cfg'])
msg = '\n'.join([
'# Written by curtin for multipath device wwid "%s"' % wwid,
'GRUB_DEVICE=%s' % grub_dev, 'GRUB_DISABLE_LINUX_UUID=true', ''
])
util.write_file(grub_cfg, content=msg)
else:
LOG.warn("Not sure how this will boot")
# Initrams needs to be updated to include /etc/multipath.cfg
# and /etc/multipath/bindings files.
update_initramfs(target, all_kernels=True)
def mycopy(self, src, dst):
print('mycopy(src=%s dst=%s)' % (src, dst))
util.write_file(dst, self.host_content)
def test_no_change_needed_returns_none(self):
tmpf = self.tmp_path('testfile')
util.write_file(tmpf, '')
os.chmod(tmpf, 0o600)
ret = util.set_unexecutable(tmpf)
self.assertEqual(ret, None)
def test_change_needed_returns_original_mode(self):
tmpf = self.tmp_path('testfile')
util.write_file(tmpf, '')
os.chmod(tmpf, 0o755)
ret = util.set_unexecutable(tmpf)
self.assertEqual(ret, 0o0755)
def set_cache_mode(bcache_dev, cache_mode):
LOG.info("Setting cache_mode on {} to {}".format(bcache_dev, cache_mode))
cache_mode_file = '/sys/block/{}/bcache/cache_mode'.format(bcache_dev)
util.write_file(cache_mode_file, cache_mode, mode=None)
def test_trusty_source_lists(self, m_get_arch, m_lsb_release):
"""Support mirror equivalency with and without trailing /.
Trusty official images do not have a trailing slash on
http://archive.ubuntu.com/ubuntu ."""
orig_primary = apt_config.PRIMARY_ARCH_MIRRORS['PRIMARY']
orig_security = apt_config.PRIMARY_ARCH_MIRRORS['SECURITY']
msg = "Test is invalid. %s mirror does not end in a /."
self.assertEqual(orig_primary[-1], "/", msg % "primary")
self.assertEqual(orig_security[-1], "/", msg % "security")
orig_primary = orig_primary[:-1]
orig_security = orig_security[:-1]
m_lsb_release.return_value = {
'codename': 'trusty',
'description': 'Ubuntu 14.04.5 LTS',
'id': 'Ubuntu',
'release': '14.04'
}
target = self.new_root
my_primary = 'http://fixed-primary.ubuntu.com/ubuntu'
my_security = 'http://fixed-security.ubuntu.com/ubuntu'
cfg = {
'preserve_sources_list': False,
'primary': [{
'arches': ['amd64'],
'uri': my_primary
}],
'security': [{
'arches': ['amd64'],
'uri': my_security
}]
}
# this is taken from a trusty image /etc/apt/sources.list
tmpl = textwrap.dedent("""\
deb {mirror} {release} {comps}
deb {mirror} {release}-updates {comps}
deb {mirror} {release}-backports {comps}
deb {security} {release}-security {comps}
# not modified
deb http://my.example.com/updates testing main
""")
release = 'trusty'
comps = 'main universe multiverse restricted'
easl = paths.target_path(target, 'etc/apt/sources.list')
orig_content = tmpl.format(mirror=orig_primary,
security=orig_security,
release=release,
comps=comps)
orig_content_slash = tmpl.format(mirror=orig_primary + "/",
security=orig_security + "/",
release=release,
comps=comps)
expected = tmpl.format(mirror=my_primary,
security=my_security,
release=release,
comps=comps)
# Avoid useless test. Make sure the strings don't start out equal.
self.assertNotEqual(expected, orig_content)
util.write_file(easl, orig_content)
apt_config.handle_apt(cfg, target)
self.assertEqual(expected, util.load_file(easl))
util.write_file(easl, orig_content_slash)
apt_config.handle_apt(cfg, target)
self.assertEqual(expected, util.load_file(easl))
def test_identify_swap_false_if_tiny(self, mock_getpagesize):
"""small files do not trip up is_swap_device()."""
path = self.tmp_path("tiny")
util.write_file(path, b'tinystuff', omode='wb')
self.assertFalse(swap.is_swap_device(path))
def test_identify_zeros_are_swap(self, mock_getpagesize):
"""swap.is_swap_device() returns false on all zeros"""
pagesize = mock_getpagesize()
path = self.tmp_path("notswap0")
util.write_file(path, pagesize * 2 * b'\0', omode="wb")
self.assertFalse(swap.is_swap_device(path))
def write_label(label, device):
""" write label to bcache device """
sys_block = sys_block_path(device)
bcache_sys_attr = os.path.join(sys_block, 'bcache', 'label')
util.write_file(bcache_sys_attr, content=label)
