def test_build_nodbus(self, request, prepared_test_build, bitbake_path):
"""Test that we can remove dbus from PACKAGECONFIG, and that this causes the
library dependency to be gone. The opposite is not tested, since we
assume failure to link to the library will be caught in other tests that
test DBus functionality."""
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
"mender-client",
['PACKAGECONFIG_remove = "dbus"'],
)
env = get_bitbake_variables(request,
"mender-client",
prepared_test_build=prepared_test_build)
# Get dynamic section info from binary.
output = subprocess.check_output(
[env["READELF"], "-d",
os.path.join(env["D"], "usr/bin/mender")]).decode()
# Verify the output is sane.
assert "libc" in output
# Actual test.
assert "libglib" not in output
# Make sure busconfig files are also gone.
assert not os.path.exists(
os.path.join(env["D"], "usr/share/dbus-1/system.d/io.mender.conf"))
assert not os.path.exists(
os.path.join(env["D"], "etc/dbus-1/system.d/io.mender.conf"))
def test_module_install(
self, request, prepared_test_build, bitbake_path, latest_rootfs, bitbake_image
):
# List of expected update modules
default_update_modules = [
"deb",
"directory",
"docker",
"rootfs-image-v2",
"rpm",
"script",
"single-file",
]
mender_vars = get_bitbake_variables(request, "mender-client")
if "modules" in mender_vars["PACKAGECONFIG"].split():
originally_on = True
else:
originally_on = False
output = subprocess.check_output(
["debugfs", "-R", "ls -p /usr/share/mender/modules/v3", latest_rootfs]
).decode()
entries = [
elem.split("/")[5] for elem in output.split("\n") if elem.startswith("/")
]
if originally_on:
assert all([e in entries for e in default_update_modules])
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
['PACKAGECONFIG_remove = "modules"'],
)
else:
assert not any([e in entries for e in default_update_modules])
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
['PACKAGECONFIG_append = " modules"'],
)
new_rootfs = latest_build_artifact(
request, prepared_test_build["build_dir"], "core-image*.ext4"
)
output = subprocess.check_output(
["debugfs", "-R", "ls -p /usr/share/mender/modules/v3", new_rootfs]
).decode()
entries = [
elem.split("/")[5] for elem in output.split("\n") if elem.startswith("/")
]
if originally_on:
assert not any([e in entries for e in default_update_modules])
else:
assert all([e in entries for e in default_update_modules])
def test_older_uboot_build(self, request, prepared_test_build,
bitbake_image):
"""Test that we can provide our own custom U-Boot provider."""
# Get rid of build outputs in deploy directory that may get in the way.
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c clean u-boot",
)
try:
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
[
'PREFERRED_PROVIDER_u-boot = "u-boot-testing"',
'PREFERRED_RPROVIDER_u-boot = "u-boot-testing"',
],
)
finally:
# Get rid of build outputs in deploy directory that may get in the
# way.
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c clean u-boot-testing",
)
# Reset local.conf.
reset_build_conf(prepared_test_build["build_dir"])
bitbake_vars = get_bitbake_variables(
request, "u-boot", prepared_test_build=prepared_test_build)
if bitbake_vars["MENDER_UBOOT_AUTO_CONFIGURE"] == "0":
# The rest of the test is irrelevant if MENDER_UBOOT_AUTO_CONFIGURE
# is already off.
return
try:
# Capture and discard output, it looks very ugly in the log.
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
['MENDER_UBOOT_AUTO_CONFIGURE_pn-u-boot = "0"'],
capture=True,
)
pytest.fail(
"Build should not succeed when MENDER_UBOOT_AUTO_CONFIGURE is turned off"
)
except subprocess.CalledProcessError:
pass
def test_bootloader_embed(self, request, prepared_test_build,
bitbake_image):
"""Test that MENDER_IMAGE_BOOTLOADER_FILE causes the bootloader to be embedded
correctly in the resulting sdimg."""
loader_file = "bootloader.bin"
loader_offset = 4
new_bb_vars = get_bitbake_variables(
request,
"core-image-minimal",
prepared_test_build=prepared_test_build)
loader_dir = new_bb_vars["DEPLOY_DIR_IMAGE"]
loader_path = os.path.join(loader_dir, loader_file)
run_verbose("mkdir -p %s" % os.path.dirname(loader_path))
run_verbose("cp /etc/os-release %s" % loader_path)
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
[
'MENDER_IMAGE_BOOTLOADER_FILE = "%s"' % loader_file,
'MENDER_IMAGE_BOOTLOADER_BOOTSECTOR_OFFSET = "%d"' %
loader_offset,
],
)
built_sdimg = latest_build_artifact(request,
prepared_test_build["build_dir"],
"core-image*.sdimg")
original = os.open(loader_path, os.O_RDONLY)
embedded = os.open(built_sdimg, os.O_RDONLY)
os.lseek(embedded, loader_offset * 512, 0)
block_size = 4096
while True:
org_read = os.read(original, block_size)
org_read_size = len(org_read)
emb_read = os.read(embedded, org_read_size)
assert (
org_read == emb_read
), "Embedded bootloader is not identical to the file specified in MENDER_IMAGE_BOOTLOADER_FILE"
if org_read_size < block_size:
break
os.close(original)
os.close(embedded)
def test_module_install(self, request, prepared_test_build, bitbake_path,
latest_rootfs, bitbake_image):
mender_vars = get_bitbake_variables(request, "mender-client")
if "modules" in mender_vars["PACKAGECONFIG"].split():
originally_on = True
else:
originally_on = False
output = subprocess.check_output(
["debugfs", "-R", "ls -p /usr/share/mender",
latest_rootfs]).decode()
entries = [
elem.split("/")[5] for elem in output.split("\n")
if elem.startswith("/")
]
if originally_on:
assert "modules" in entries
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
['PACKAGECONFIG_remove = "modules"'],
)
else:
assert "modules" not in entries
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
['PACKAGECONFIG_append = " modules"'],
)
new_rootfs = latest_build_artifact(request,
prepared_test_build["build_dir"],
"core-image*.ext4")
output = subprocess.check_output(
["debugfs", "-R", "ls -p /usr/share/mender", new_rootfs]).decode()
entries = [
elem.split("/")[5] for elem in output.split("\n")
if elem.startswith("/")
]
if originally_on:
assert "modules" not in entries
else:
assert "modules" in entries
def test_certificate_split(self, request, bitbake_image):
"""Test that the certificate added in the mender-server-certificate
recipe is split correctly."""
# Currently this is hardcoded to the md5 sums of the split demo
# certificate as of 2021-04-07. Please update if it is replaced.
md5sums = """02d20627f63664f9495cea2e54b28e1b ./usr/local/share/ca-certificates/mender/server-1.crt
b524b8b3f13902ef8014c0af7aa408bc ./usr/local/share/ca-certificates/mender/server-2.crt"""
rootfs = get_bitbake_variables(request, bitbake_image)["IMAGE_ROOTFS"]
output = (subprocess.check_output(
"md5sum ./usr/local/share/ca-certificates/mender/*",
shell=True,
cwd=rootfs,
).decode().strip())
assert md5sums == output
def test_various_mtd_combinations(self, request, test_case_name, test_case,
prepared_test_build, bitbake_image):
"""Tests that we can build with various combinations of MTD variables,
and that they receive the correct values."""
try:
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
["\n".join(test_case["vars"])],
)
assert test_case["success"], "Build succeeded, but should fail"
except subprocess.CalledProcessError:
assert not test_case["success"], "Build failed"
variables = get_bitbake_variables(
request, bitbake_image, prepared_test_build=prepared_test_build)
for key in test_case["expected"]:
assert test_case["expected"][key] == variables[key]
def test_mender_dbus_interface_file(
self, request, prepared_test_build, bitbake_image
):
"""
Test the D-Bus interface file is provided by the mender-client-dev package,
but not installed by default.
"""
EXPECTED_FILES = [
"io.mender.Authentication1.xml",
]
# clean up the mender-client
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c clean mender-client",
)
# build mender-client
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
"mender-client",
)
# verify the files
deploy_dir_rpm = get_bitbake_variables(request, bitbake_image)["DEPLOY_DIR_RPM"]
output = subprocess.check_output(
["rpm", "-qlp", f"{deploy_dir_rpm}/*/mender-client-dev-*.rpm"],
)
for file in EXPECTED_FILES:
assert (
bytes(file, "utf-8") in output
), f"{file} seems not to be a part of the mender-client-dev package, like it should"
def test_incorrect_Kconfig_setting(self, request, prepared_test_build,
bitbake_image):
"""First produce a patch using the auto-patcher, then disable
auto-patching and apply the patch with a slight modification that makes
its settings incompatible, and check that this is detected."""
bitbake_variables = get_bitbake_variables(
request, "u-boot", prepared_test_build=prepared_test_build)
# Only run if auto-configuration is on.
if ("MENDER_UBOOT_AUTO_CONFIGURE" in bitbake_variables
and bitbake_variables["MENDER_UBOOT_AUTO_CONFIGURE"] == "0"):
pytest.skip(
"Test is not applicable when MENDER_UBOOT_AUTO_CONFIGURE is off"
)
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c save_mender_auto_configured_patch u-boot",
)
try:
patch_name = os.path.join(bitbake_variables["WORKDIR"],
"mender_auto_configured.patch")
new_patch_name = "../../meta-mender-core/recipes-bsp/u-boot/patches/mender_broken_definition.patch"
with open(patch_name) as patch, open(new_patch_name,
"w") as new_patch:
for line in patch.readlines():
if line.startswith("+CONFIG_MTDIDS_DEFAULT="):
# Change to a wrong value:
new_patch.write(
'+CONFIG_MTDIDS_DEFAULT="nand0-wrongvalue=00000000.flash"\n'
)
else:
new_patch.write(line)
# We need to add the code using TEST_SRC_URI_APPEND make sure it is
# absolutely last, otherwise platform specific layers may add
# patches after us.
# Normally changes to SRC_URI are picked up automatically, but since
# we are sneaking it in via the TEST_SRC_URI_APPEND and its
# associated python snippet, we need to clean the build manually.
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
[
'MENDER_UBOOT_AUTO_CONFIGURE_pn-u-boot = "0"',
'TEST_SRC_URI_APPEND_pn-u-boot = " file://%s"' %
os.path.basename(new_patch_name),
],
target="-c clean u-boot",
)
try:
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c compile u-boot",
)
# Should never get here.
pytest.fail(
"Bitbake succeeded even though we intentionally broke the patch!"
)
except subprocess.CalledProcessError as e:
# A bit risky change after upgrading tests from python2.7 to python3.
# It seems that underneath subprocess.check_output() call in not
# capturing the output as `capture_output` flag is not set.
if e.output:
assert e.output.find(
"Please fix U-Boot's configuration file") >= 0
finally:
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c clean u-boot",
)
os.unlink(new_patch_name)
def test_save_mender_auto_configured_patch(self, request,
prepared_test_build,
bitbake_image):
"""Test that we can invoke the save_mender_auto_configured_patch task,
that it produces a patch file, and that this patch file can be used
instead of MENDER_UBOOT_AUTO_CONFIGURE."""
bitbake_variables = get_bitbake_variables(
request, "u-boot", prepared_test_build=prepared_test_build)
# Only run if auto-configuration is on.
if ("MENDER_UBOOT_AUTO_CONFIGURE" in bitbake_variables
and bitbake_variables["MENDER_UBOOT_AUTO_CONFIGURE"] == "0"):
pytest.skip(
"Test is not applicable when MENDER_UBOOT_AUTO_CONFIGURE is off"
)
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c save_mender_auto_configured_patch u-boot",
)
patch_name = os.path.join(bitbake_variables["WORKDIR"],
"mender_auto_configured.patch")
with open(patch_name) as fd:
content = fd.read()
# Make sure it looks like a patch.
assert "---" in content
assert "+++" in content
# Now check if we can use the patch.
new_patch_name = "../../meta-mender-core/recipes-bsp/u-boot/patches/mender_auto_configured.patch"
shutil.copyfile(patch_name, new_patch_name)
try:
# We need to add the code using TEST_SRC_URI_APPEND make sure it is
# absolutely last, otherwise platform specific layers may add
# patches after us.
# Normally changes to SRC_URI are picked up automatically, but since
# we are sneaking it in via the TEST_SRC_URI_APPEND and its
# associated python snippet, we need to clean the build manually.
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
[
'MENDER_UBOOT_AUTO_CONFIGURE_pn-u-boot = "0"',
'TEST_SRC_URI_APPEND_pn-u-boot = " file://%s"' %
os.path.basename(new_patch_name),
],
target="-c clean u-boot",
)
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="u-boot",
)
finally:
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c clean u-boot",
)
os.unlink(new_patch_name)
def run_test_uboot_compile(self, request, bitbake_variables):
# No need to test this on non-vexpress-qemu. It is a very resource
# consuming test, and it is identical on all boards, since it internally
# tests all boards.
machine = bitbake_variables["MACHINE"]
if not machine.startswith("vexpress-qemu"):
pytest.skip("Skipping test on non-vexpress-qemu platforms")
# This is a slow running test. Skip if appropriate.
self.check_if_should_run()
for task in ["do_provide_mender_defines", "prepare_recipe_sysroot"]:
subprocess.check_call(
"cd %s && bitbake -c %s u-boot" %
(os.environ["BUILDDIR"], task),
shell=True,
)
bitbake_variables = get_bitbake_variables(request, "u-boot")
shutil.rmtree("/dev/shm/test_uboot_compile", ignore_errors=True)
env = copy.copy(os.environ)
env["UBOOT_SRC"] = bitbake_variables["S"]
env["TESTS_DIR"] = os.getcwd()
env["LOGS"] = os.path.join(os.getcwd(), "test_uboot_compile-logs")
if os.path.exists(env["LOGS"]):
print(
"WARNING: %s already exists. Will use cached logs from there. Recreate to reset."
% env["LOGS"])
else:
os.mkdir(env["LOGS"])
configs_to_test = self.collect_and_prepare_boards_to_test(
bitbake_variables, env)
env["BOARD_LOGS"] = " ".join(configs_to_test)
try:
sanitized_makeflags = bitbake_variables["EXTRA_OEMAKE"]
sanitized_makeflags = sanitized_makeflags.replace('\\"', '"')
sanitized_makeflags = re.sub(" +", " ", sanitized_makeflags)
env["MAYBE_UBI"] = "--ubi" if machine == "vexpress-qemu-flash" else ""
# Compile all boards. The reason for using a makefile is to get easy
# parallelization.
subprocess.check_call(
"make -j %d -f %s SUBJOBCOUNT=-j%d TMP=/dev/shm/test_uboot_compile %s"
% (
self.parallel_job_count(),
os.path.join(env["TESTS_DIR"],
"files/Makefile.test_uboot_automation"),
self.parallel_subjob_count(),
sanitized_makeflags,
),
shell=True,
env=env,
stderr=subprocess.STDOUT,
)
# Now check that the ratio of compiled boards is as expected. This
# number may change over time as U-Boot changes, but big discrepancies
# should be checked out.
failed = 0.0
total = 0.0
for file in os.listdir(env["LOGS"]):
if not file.endswith("_defconfig"):
continue
total += 1
with open(os.path.join(env["LOGS"], file)) as fd:
if "AutoPatchFailed\n" in fd.readlines():
failed += 1
assert total == len(
configs_to_test
), "Number of logs do not match the number of boards we tested? Should not happen"
if machine == "vexpress-qemu":
# PLEASE UPDATE the version you used to find this number if you update it.
# From version: v2020.01
measured_failed_ratio = 57.0 / 561.0
elif machine == "vexpress-qemu-flash":
# PLEASE UPDATE the version you used to find this number if you update it.
# From version: v2021.07
measured_failed_ratio = 13.0 / 129.0
# We tolerate a certain percentage discrepancy in either direction.
tolerated_discrepancy = 0.1
lower_bound = measured_failed_ratio * (1.0 - tolerated_discrepancy)
upper_bound = measured_failed_ratio * (1.0 + tolerated_discrepancy)
try:
assert failed / total >= lower_bound, (
"Less boards failed than expected. Good? Or a mistake somewhere? Failed: %d, Total: %d"
% (failed, total))
assert failed / total <= upper_bound, (
"More boards failed than expected. Failed: %d, Total: %d" %
(failed, total))
except AssertionError:
for file in os.listdir(env["LOGS"]):
with open(os.path.join(env["LOGS"], file)) as fd:
log = fd.readlines()
if "AutoPatchFailed\n" in log:
print(
"Last 50 lines of output from failed board: " +
file)
print("".join(log[-50:]))
raise
shutil.rmtree(env["LOGS"])
finally:
shutil.rmtree("/dev/shm/test_uboot_compile", ignore_errors=True)
def test_perform_update(
self,
request,
setup_board,
prepared_test_build,
bitbake_variables,
bitbake_image,
bitbake_path,
connection,
http_server,
board_type,
use_s3,
s3_address,
):
"""Perform a delta update.
"""
if ("read-only-rootfs"
not in bitbake_variables["IMAGE_FEATURES"].strip().split()):
pytest.skip("Only works when using read-only-rootfs IMAGE_FEATURE")
if distutils.spawn.find_executable(
"mender-binary-delta-generator") is None:
pytest.fail("mender-binary-delta-generator not found in PATH")
built_artifact = self.do_install_mender_binary_delta(
request,
prepared_test_build,
bitbake_variables,
bitbake_image,
connection,
http_server,
board_type,
use_s3,
s3_address,
)
binary_delta_vars = get_bitbake_variables(request,
"mender-binary-delta",
prepared_test_build)
if LooseVersion(binary_delta_vars["PV"]) < LooseVersion("1.3.0"):
pytest.skip(
"This version of mender-binary-delta does not support grub-mender-grubenv-print and friends."
)
with make_tempdir() as tmpdir:
# Copy previous build
artifact_from = os.path.join(tmpdir, "artifact_from.mender")
shutil.copyfile(built_artifact, artifact_from)
# Create new image installing some extra software
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
['IMAGE_INSTALL_append = " nano"'],
)
built_artifact = latest_build_artifact(
request, prepared_test_build["build_dir"],
"core-image*.mender")
artifact_to = os.path.join(tmpdir, "artifact_to.mender")
shutil.copyfile(built_artifact, artifact_to)
# Create delta Artifact using mender-binary-delta-generator
artifact_delta = os.path.join(tmpdir, "artifact_delta.mender")
subprocess.check_call(
f"mender-binary-delta-generator -n v2.0-deltafrom-v1.0 {artifact_from} {artifact_to} -o {artifact_delta}",
shell=True,
)
# Verbose provides/depends of the different Artifacts and the client (when supported)
connection.run("mender show-provides", warn=True)
subprocess.check_call(
"mender-artifact read %s" % artifact_from,
shell=True,
)
subprocess.check_call(
"mender-artifact read %s" % artifact_to,
shell=True,
)
subprocess.check_call(
"mender-artifact read %s" % artifact_delta,
shell=True,
)
# Install Artifact, verify partitions and commit
(active,
passive) = determine_active_passive_part(bitbake_variables,
connection)
Helpers.install_update(artifact_delta, connection, http_server,
board_type, use_s3, s3_address)
reboot(connection)
run_after_connect("true", connection)
(new_active, new_passive) = determine_active_passive_part(
bitbake_variables, connection)
assert new_active == passive
assert new_passive == active
connection.run("mender commit")
