def _apt_source_list(self, cfg, expected):
"_apt_source_list - Test rendering from template (generic)"
arch = distro.get_architecture()
# would fail inside the unittest context
bpath = "curtin.commands.apt_config."
upath = bpath + "util."
dpath = bpath + 'distro.'
self.add_patch(dpath + "get_architecture", "mockga", return_value=arch)
self.add_patch(upath + "write_file", "mockwrite")
self.add_patch(bpath + "os.rename", "mockrename")
self.add_patch(upath + "load_file",
"mockload_file",
return_value=MOCKED_APT_SRC_LIST)
self.add_patch(bpath + "distro.lsb_release",
"mock_lsb_release",
return_value={'codename': 'fakerel'})
self.add_patch(bpath + "apply_preserve_sources_list",
"mock_apply_preserve_sources_list")
apt_config.handle_apt(cfg, TARGET)
self.mockga.assert_called_with(TARGET)
self.mock_apply_preserve_sources_list.assert_called_with(TARGET)
calls = [
call(paths.target_path(TARGET, '/etc/apt/sources.list'),
expected,
mode=0o644)
]
self.mockwrite.assert_has_calls(calls)
def generate_sources_list(cfg, release, mirrors, target=None):
""" generate_sources_list
create a source.list file based on a custom or default template
by replacing mirrors and release in the template
"""
default_mirrors = get_default_mirrors(distro.get_architecture(target))
aptsrc = "/etc/apt/sources.list"
params = {'RELEASE': release}
for k in mirrors:
params[k] = mirrors[k]
tmpl = cfg.get('sources_list', None)
if tmpl is None:
LOG.info(
"No custom template provided, fall back to modify"
"mirrors in %s on the target system", aptsrc)
tmpl = util.load_file(paths.target_path(target, aptsrc))
# Strategy if no custom template was provided:
# - Only replacing mirrors
# - no reason to replace "release" as it is from target anyway
# - The less we depend upon, the more stable this is against changes
# - warn if expected original content wasn't found
tmpl = mirror_to_placeholder(tmpl, default_mirrors['PRIMARY'],
"$MIRROR")
tmpl = mirror_to_placeholder(tmpl, default_mirrors['SECURITY'],
"$SECURITY")
orig = paths.target_path(target, aptsrc)
if os.path.exists(orig):
os.rename(orig, orig + ".curtin.old")
rendered = util.render_string(tmpl, params)
disabled = disable_suites(cfg.get('disable_suites'), rendered, release)
util.write_file(paths.target_path(target, aptsrc), disabled, mode=0o644)
def test_redhat_osfamily_calls_rpm_get_arch(self):
osfamily = distro.DISTROS.redhat
expected_result = self.m_rpm_get_arch.return_value
result = distro.get_architecture(target=self.target, osfamily=osfamily)
self.assertEqual(expected_result, result)
self.assertEqual([mock.call(target=self.target)],
self.m_rpm_get_arch.call_args_list)
self.assertEqual(0, self.m_dpkg_get_arch.call_count)
def _get_default_params():
""" get_default_params
Get the most basic default mrror and release info to be used in tests
"""
params = {}
params['RELEASE'] = distro.lsb_release()['codename']
arch = distro.get_architecture()
params['MIRROR'] = apt_config.get_default_mirrors(arch)["PRIMARY"]
return params
def get_default_mirrors(arch=None):
"""returns the default mirrors for the target. These depend on the
architecture, for more see:
https://wiki.ubuntu.com/UbuntuDevelopment/PackageArchive#Ports"""
if arch is None:
arch = distro.get_architecture()
if arch in PRIMARY_ARCHES:
return PRIMARY_ARCH_MIRRORS.copy()
if arch in PORTS_ARCHES:
return PORTS_MIRRORS.copy()
raise ValueError("No default mirror known for arch %s" % arch)
def test_mirror(self):
model = SubiquityModel('test')
mirror_val = 'http://my-mirror'
model.mirror.set_mirror(mirror_val)
config = model.render('ident')
from curtin.commands.apt_config import get_mirror
try:
from curtin.distro import get_architecture
except ImportError:
from curtin.util import get_architecture
self.assertEqual(
get_mirror(config["apt"], "primary", get_architecture()),
mirror_val)
def test_mirror_default(self):
"""test_mirror_default - Test without defining a mirror"""
arch = distro.get_architecture()
default_mirrors = apt_config.get_default_mirrors(arch)
pmir = default_mirrors["PRIMARY"]
smir = default_mirrors["SECURITY"]
mirrors = apt_config.find_apt_mirror_info({}, arch)
self.assertEqual(mirrors['MIRROR'],
pmir)
self.assertEqual(mirrors['PRIMARY'],
pmir)
self.assertEqual(mirrors['SECURITY'],
smir)
def test_apt_srcl_custom(self):
"""test_apt_srcl_custom - Test rendering a custom source template"""
cfg = yaml.safe_load(YAML_TEXT_CUSTOM_SL)
target = self.new_root
arch = distro.get_architecture()
# would fail inside the unittest context
with mock.patch.object(distro, 'get_architecture', return_value=arch):
with mock.patch.object(distro,
'lsb_release',
return_value={'codename': 'fakerel'}):
apt_config.handle_apt(cfg, target)
self.assertEqual(
EXPECTED_CONVERTED_CONTENT,
util.load_file(paths.target_path(target, "/etc/apt/sources.list")))
def install_grub(devices, target, uefi=None, grubcfg=None):
"""Install grub to devices inside target chroot.
:param: devices: List of block device paths to install grub upon.
:param: target: A string specifying the path to the chroot mountpoint.
:param: uefi: A boolean set to True if system is UEFI bootable otherwise
False.
:param: grubcfg: An config dict with grub config options.
"""
if not devices:
raise ValueError("Invalid parameter 'devices': %s" % devices)
if not target:
raise ValueError("Invalid parameter 'target': %s" % target)
LOG.debug("installing grub to target=%s devices=%s [replace_defaults=%s]",
target, devices, grubcfg.get('replace_default'))
update_nvram = config.value_as_boolean(grubcfg.get('update_nvram', True))
distroinfo = distro.get_distroinfo(target=target)
target_arch = distro.get_architecture(target=target)
rhel_ver = (distro.rpm_get_dist_id(target)
if distroinfo.family == distro.DISTROS.redhat else None)
check_target_arch_machine(target, arch=target_arch, uefi=uefi)
grub_name, grub_target = get_grub_package_name(target_arch, uefi, rhel_ver)
grub_conf = get_grub_config_file(target, distroinfo.family)
new_params = get_carryover_params(distroinfo)
prepare_grub_dir(target, grub_conf)
write_grub_config(target, grubcfg, grub_conf, new_params)
grub_cmd = get_grub_install_command(uefi, distroinfo, target)
if uefi:
install_cmds, post_cmds = gen_uefi_install_commands(
grub_name, grub_target, grub_cmd, update_nvram, distroinfo,
devices, target)
else:
install_cmds, post_cmds = gen_install_commands(grub_name, grub_cmd,
distroinfo, devices,
rhel_ver)
env = os.environ.copy()
env['DEBIAN_FRONTEND'] = 'noninteractive'
LOG.debug('Grub install cmds:\n%s', str(install_cmds + post_cmds))
with util.ChrootableTarget(target) as in_chroot:
for cmd in install_cmds + post_cmds:
in_chroot.subp(cmd, env=env, capture=True)
def check_target_arch_machine(target, arch=None, machine=None, uefi=None):
""" Check target arch and machine type are grub supported. """
if not arch:
arch = distro.get_architecture(target=target)
if not machine:
machine = platform.machine()
errmsg = "Grub is not supported on arch=%s machine=%s" % (arch, machine)
# s390x uses zipl
if arch == "s390x":
raise RuntimeError(errmsg)
# As a rule, ARMv7 systems don't use grub. This may change some
# day, but for now, assume no. They do require the initramfs
# to be updated, and this also triggers boot loader setup via
# flash-kernel.
if (machine.startswith('armv7') or machine.startswith('s390x')
or machine.startswith('aarch64') and not uefi):
raise RuntimeError(errmsg)
def test_mirror_arches_sysdefault(self):
"""test_mirror_arches - Test arches falling back to sys default"""
arch = distro.get_architecture()
default_mirrors = apt_config.get_default_mirrors(arch)
pmir = default_mirrors["PRIMARY"]
smir = default_mirrors["SECURITY"]
cfg = {"primary": [{'arches': ["thisarchdoesntexist_64"],
"uri": "notthis"},
{'arches': ["thisarchdoesntexist"],
"uri": "notthiseither"}],
"security": [{'arches': ["thisarchdoesntexist"],
"uri": "nothat"},
{'arches': ["thisarchdoesntexist_64"],
"uri": "nothateither"}]}
mirrors = apt_config.find_apt_mirror_info(cfg, arch)
self.assertEqual(mirrors['MIRROR'], pmir)
self.assertEqual(mirrors['PRIMARY'], pmir)
self.assertEqual(mirrors['SECURITY'], smir)
def handle_apt(cfg, target=None):
""" handle_apt
process the config for apt_config. This can be called from
curthooks if a global apt config was provided or via the "apt"
standalone command.
"""
release = distro.lsb_release(target=target)['codename']
arch = distro.get_architecture(target)
mirrors = find_apt_mirror_info(cfg, arch)
LOG.debug("Apt Mirror info: %s", mirrors)
apply_debconf_selections(cfg, target)
if not config.value_as_boolean(cfg.get('preserve_sources_list', True)):
generate_sources_list(cfg, release, mirrors, target)
apply_preserve_sources_list(target)
rename_apt_lists(mirrors, target)
try:
apply_apt_proxy_config(cfg, target + APT_PROXY_FN,
target + APT_CONFIG_FN)
except (IOError, OSError):
LOG.exception("Failed to apply proxy or apt config info:")
# Process 'apt_source -> sources {dict}'
if 'sources' in cfg:
params = mirrors
params['RELEASE'] = release
params['MIRROR'] = mirrors["MIRROR"]
matcher = None
matchcfg = cfg.get('add_apt_repo_match', ADD_APT_REPO_MATCH)
if matchcfg:
matcher = re.compile(matchcfg).search
add_apt_sources(cfg['sources'],
target,
template_params=params,
aa_repo_match=matcher)
def rename_apt_lists(new_mirrors, target=None):
"""rename_apt_lists - rename apt lists to preserve old cache data"""
default_mirrors = get_default_mirrors(distro.get_architecture(target))
pre = paths.target_path(target, APT_LISTS)
for (name, omirror) in default_mirrors.items():
nmirror = new_mirrors.get(name)
if not nmirror:
continue
oprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(omirror)
nprefix = pre + os.path.sep + mirrorurl_to_apt_fileprefix(nmirror)
if oprefix == nprefix:
continue
olen = len(oprefix)
for filename in glob.glob("%s_*" % oprefix):
newname = "%s%s" % (nprefix, filename[olen:])
LOG.debug("Renaming apt list %s to %s", filename, newname)
try:
os.rename(filename, newname)
except OSError:
# since this is a best effort task, warn with but don't fail
LOG.warn("Failed to rename apt list:", exc_info=True)
def find_apt_mirror_info(cfg, arch=None):
"""find_apt_mirror_info
find an apt_mirror given the cfg provided.
It can check for separate config of primary and security mirrors
If only primary is given security is assumed to be equal to primary
If the generic apt_mirror is given that is defining for both
"""
if arch is None:
arch = distro.get_architecture()
LOG.debug("got arch for mirror selection: %s", arch)
pmirror = get_mirror(cfg, "primary", arch)
LOG.debug("got primary mirror: %s", pmirror)
smirror = get_mirror(cfg, "security", arch)
LOG.debug("got security mirror: %s", smirror)
# Note: curtin has no cloud-datasource fallback
mirror_info = update_mirror_info(pmirror, smirror, arch)
# less complex replacements use only MIRROR, derive from primary
mirror_info["MIRROR"] = mirror_info["PRIMARY"]
return mirror_info
def test_osfamily_none_calls_get_osfamily(self):
distro.get_architecture(target=self.target, osfamily=None)
self.assertEqual([mock.call(target=self.target)],
self.m_get_osfamily.call_args_list)
class TestOldAptAbs(VMBaseClass):
"""TestOldAptAbs - Basic tests for old apt features of curtin"""
interactive = False
test_type = 'config'
extra_disks = []
fstab_expected = {}
disk_to_check = []
extra_collect_scripts = [textwrap.dedent("""
cd OUTPUT_COLLECT_D
grep -A 3 "Name: debconf/priority" /var/cache/debconf/config.dat > debc
apt-config dump > aptconf
cp /etc/apt/apt.conf.d/90curtin-aptproxy .
cp /etc/apt/sources.list .
cp /etc/cloud/cloud.cfg.d/curtin-preserve-sources.cfg .
cp /etc/cloud/cloud.cfg.d/90_dpkg.cfg .
exit 0
""")]
arch = distro.get_architecture()
target_arch = arch
if target_arch in ['amd64', 'i386']:
conf_file = "examples/tests/test_old_apt_features.yaml"
exp_mirror = "http://us.archive.ubuntu.com/ubuntu"
exp_secmirror = "http://archive.ubuntu.com/ubuntu"
if target_arch in ['s390x', 'arm64', 'armhf', 'powerpc', 'ppc64el']:
conf_file = "examples/tests/test_old_apt_features_ports.yaml"
exp_mirror = "http://ports.ubuntu.com/ubuntu-ports"
exp_secmirror = "http://ports.ubuntu.com/ubuntu-ports"
def test_output_files_exist(self):
"""test_output_files_exist - Check if all output files exist"""
self.output_files_exist(
["debc", "aptconf", "sources.list", "curtin-preserve-sources.cfg",
"90_dpkg.cfg"])
def test_preserve_source(self):
"""test_preserve_source - no clobbering sources.list by cloud-init"""
# For earlier than xenial 'apt_preserve_sources_list' is expected
self.assertEqual(
{'apt': {'preserve_sources_list': True}},
load_config(self.collect_path("curtin-preserve-sources.cfg")))
def test_debconf(self):
"""test_debconf - Check if debconf is in place"""
self.check_file_strippedline("debc", "Value: low")
def test_aptconf(self):
"""test_aptconf - Check if apt conf for proxy is in place"""
# this gets configured by tools/launch and get_apt_proxy in
# tests/vmtests/__init__.py, so compare with those, if set
if not self.proxy:
self.skipTest('Host apt-proxy not set')
self.output_files_exist(["90curtin-aptproxy"])
rproxy = r"Acquire::http::Proxy \"" + re.escape(self.proxy) + r"\";"
self.check_file_regex("aptconf", rproxy)
self.check_file_regex("90curtin-aptproxy", rproxy)
def test_mirrors(self):
"""test_mirrors - Check for mirrors placed in source.list"""
lines = self.load_collect_file('sources.list').splitlines()
data = sources_to_dict(lines)
self.assertIn(self.exp_secmirror, data)
self.assertIn(self.exp_mirror, data)
components = sorted(["main", "restricted", "universe", "multiverse"])
self.assertEqual(
components,
sorted(data[self.exp_secmirror]['%s-security' % self.release]))
self.assertEqual(components,
sorted(data[self.exp_mirror][self.release]))
def test_cloudinit_seeded(self):
content = self.load_collect_file("90_dpkg.cfg")
# not the greatest test, but we seeded NoCloud as the only datasource
# in examples/tests/test_old_apt_features.yaml. Just verify that
# there are no others there.
self.assertIn("nocloud", content.lower())
self.assertNotIn("maas", content.lower())
REQUIRED_KERNEL_MODULES = [
# kmod name
]
if lsb_release()['codename'] == "precise":
REQUIRED_IMPORTS.append(('import oauth.oauth', 'python-oauth', None), )
else:
REQUIRED_IMPORTS.append(
('import oauthlib.oauth1', 'python-oauthlib', 'python3-oauthlib'), )
# zfs is > trusty only
if not lsb_release()['codename'] in ["precise", "trusty"]:
REQUIRED_EXECUTABLES.append(('zfs', 'zfsutils-linux'))
REQUIRED_KERNEL_MODULES.append('zfs')
if not is_uefi_bootable() and 'arm' in get_architecture():
REQUIRED_EXECUTABLES.append(('flash-kernel', 'flash-kernel'))
class MissingDeps(Exception):
def __init__(self, message, deps):
self.message = message
if isinstance(deps, str) or deps is None:
deps = [deps]
self.deps = [d for d in deps if d is not None]
self.fatal = None in deps
def __str__(self):
if self.fatal:
if not len(self.deps):
return self.message + " Unresolvable."
def test_unhandled_osfamily_raises_value_error(self):
osfamily = distro.DISTROS.arch
with self.assertRaises(ValueError):
distro.get_architecture(target=self.target, osfamily=osfamily)
self.assertEqual(0, self.m_dpkg_get_arch.call_count)
self.assertEqual(0, self.m_rpm_get_arch.call_count)
def __init__(self):
self.config = copy.deepcopy(DEFAULT)
self.architecture = get_architecture()
self.default_mirror = self.get_mirror()
