def test_bootloader_embed(self, prepared_test_build, bitbake_image):
"""Test that MENDER_IMAGE_BOOTLOADER_FILE causes the bootloader to be embedded
correctly in the resulting sdimg."""
loader_file = "bootloader.bin"
loader_offset = 4
init_env_cmd = "cd %s && . oe-init-build-env %s" % (
prepared_test_build["bitbake_corebase"],
prepared_test_build["build_dir"],
)
new_bb_vars = get_bitbake_variables("core-image-minimal",
env_setup=init_env_cmd)
loader_dir = new_bb_vars["DEPLOY_DIR_IMAGE"]
loader_path = os.path.join(loader_dir, loader_file)
run_verbose("mkdir -p %s" % os.path.dirname(loader_path))
run_verbose("cp /etc/os-release %s" % loader_path)
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
[
'MENDER_IMAGE_BOOTLOADER_FILE = "%s"' % loader_file,
'MENDER_IMAGE_BOOTLOADER_BOOTSECTOR_OFFSET = "%d"' %
loader_offset,
],
)
built_sdimg = latest_build_artifact(prepared_test_build["build_dir"],
"core-image*.sdimg")
original = os.open(loader_path, os.O_RDONLY)
embedded = os.open(built_sdimg, os.O_RDONLY)
os.lseek(embedded, loader_offset * 512, 0)
block_size = 4096
while True:
org_read = os.read(original, block_size)
org_read_size = len(org_read)
emb_read = os.read(embedded, org_read_size)
assert (
org_read == emb_read
), "Embedded bootloader is not identical to the file specified in MENDER_IMAGE_BOOTLOADER_FILE"
if org_read_size < block_size:
break
os.close(original)
os.close(embedded)
def test_module_install(
self, prepared_test_build, bitbake_path, latest_rootfs, bitbake_image
):
mender_vars = get_bitbake_variables("mender-client")
if "modules" in mender_vars["PACKAGECONFIG"].split():
originally_on = True
else:
originally_on = False
output = subprocess.check_output(
["debugfs", "-R", "ls -p /usr/share/mender", latest_rootfs]
).decode()
entries = [
elem.split("/")[5] for elem in output.split("\n") if elem.startswith("/")
]
if originally_on:
assert "modules" in entries
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
['PACKAGECONFIG_remove = "modules"'],
)
else:
assert "modules" not in entries
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
['PACKAGECONFIG_append = " modules"'],
)
new_rootfs = latest_build_artifact(
prepared_test_build["build_dir"], "core-image*.ext4"
)
output = subprocess.check_output(
["debugfs", "-R", "ls -p /usr/share/mender", new_rootfs]
).decode()
entries = [
elem.split("/")[5] for elem in output.split("\n") if elem.startswith("/")
]
if originally_on:
assert "modules" not in entries
else:
assert "modules" in entries
def bitbake_path(request, conversion):
"""Fixture that enables the PATH we need for our testing tools."""
old_path = os.environ["PATH"]
if not conversion:
run_verbose(
"bitbake -c prepare_recipe_sysroot mender-test-dependencies")
bb_testing_variables = get_bitbake_variables(
"mender-test-dependencies")
os.environ[
"PATH"] = bb_testing_variables["PATH"] + ":" + os.environ["PATH"]
def path_restore():
os.environ["PATH"] = old_path
request.addfinalizer(path_restore)
return os.environ["PATH"]
def test_incorrect_Kconfig_setting(self, prepared_test_build,
bitbake_image):
"""First produce a patch using the auto-patcher, then disable
auto-patching and apply the patch with a slight modification that makes
its settings incompatible, and check that this is detected."""
env_setup = "cd %s && . oe-init-build-env %s" % (
prepared_test_build["bitbake_corebase"],
prepared_test_build["build_dir"],
)
bitbake_variables = get_bitbake_variables("u-boot",
env_setup=env_setup)
# Only run if auto-configuration is on.
if ("MENDER_UBOOT_AUTO_CONFIGURE" in bitbake_variables
and bitbake_variables["MENDER_UBOOT_AUTO_CONFIGURE"] == "0"):
pytest.skip(
"Test is not applicable when MENDER_UBOOT_AUTO_CONFIGURE is off"
)
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c save_mender_auto_configured_patch u-boot",
)
try:
patch_name = os.path.join(bitbake_variables["WORKDIR"],
"mender_auto_configured.patch")
new_patch_name = "../../meta-mender-core/recipes-bsp/u-boot/patches/mender_broken_definition.patch"
with open(patch_name) as patch, open(new_patch_name,
"w") as new_patch:
for line in patch.readlines():
if line.startswith("+CONFIG_MTDIDS_DEFAULT="):
# Change to a wrong value:
new_patch.write(
'+CONFIG_MTDIDS_DEFAULT="nand0-wrongvalue=00000000.flash"\n'
)
else:
new_patch.write(line)
# We need to add the code using TEST_SRC_URI_APPEND make sure it is
# absolutely last, otherwise platform specific layers may add
# patches after us.
# Normally changes to SRC_URI are picked up automatically, but since
# we are sneaking it in via the TEST_SRC_URI_APPEND and its
# associated python snippet, we need to clean the build manually.
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
[
'MENDER_UBOOT_AUTO_CONFIGURE_pn-u-boot = "0"',
'TEST_SRC_URI_APPEND_pn-u-boot = " file://%s"' %
os.path.basename(new_patch_name),
],
target="-c clean u-boot",
)
try:
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c compile u-boot",
)
# Should never get here.
pytest.fail(
"Bitbake succeeded even though we intentionally broke the patch!"
)
except subprocess.CalledProcessError as e:
# A bit risky change after upgrading tests from python2.7 to python3.
# It seems that underneath subprocess.check_output() call in not
# capturing the output as `capture_output` flag is not set.
if e.output:
assert e.output.find(
"Please fix U-Boot's configuration file") >= 0
finally:
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c clean u-boot",
)
os.unlink(new_patch_name)
def test_save_mender_auto_configured_patch(self, prepared_test_build,
bitbake_image):
"""Test that we can invoke the save_mender_auto_configured_patch task,
that it produces a patch file, and that this patch file can be used
instead of MENDER_UBOOT_AUTO_CONFIGURE."""
env_setup = "cd %s && . oe-init-build-env %s" % (
prepared_test_build["bitbake_corebase"],
prepared_test_build["build_dir"],
)
bitbake_variables = get_bitbake_variables("u-boot",
env_setup=env_setup)
# Only run if auto-configuration is on.
if ("MENDER_UBOOT_AUTO_CONFIGURE" in bitbake_variables
and bitbake_variables["MENDER_UBOOT_AUTO_CONFIGURE"] == "0"):
pytest.skip(
"Test is not applicable when MENDER_UBOOT_AUTO_CONFIGURE is off"
)
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c save_mender_auto_configured_patch u-boot",
)
patch_name = os.path.join(bitbake_variables["WORKDIR"],
"mender_auto_configured.patch")
with open(patch_name) as fd:
content = fd.read()
# Make sure it looks like a patch.
assert "---" in content
assert "+++" in content
# Now check if we can use the patch.
new_patch_name = "../../meta-mender-core/recipes-bsp/u-boot/patches/mender_auto_configured.patch"
shutil.copyfile(patch_name, new_patch_name)
try:
# We need to add the code using TEST_SRC_URI_APPEND make sure it is
# absolutely last, otherwise platform specific layers may add
# patches after us.
# Normally changes to SRC_URI are picked up automatically, but since
# we are sneaking it in via the TEST_SRC_URI_APPEND and its
# associated python snippet, we need to clean the build manually.
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
[
'MENDER_UBOOT_AUTO_CONFIGURE_pn-u-boot = "0"',
'TEST_SRC_URI_APPEND_pn-u-boot = " file://%s"' %
os.path.basename(new_patch_name),
],
target="-c clean u-boot",
)
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="u-boot",
)
finally:
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
target="-c clean u-boot",
)
os.unlink(new_patch_name)
def run_test_uboot_compile(self, bitbake_variables):
# No need to test this on non-vexpress-qemu. It is a very resource
# consuming test, and it is identical on all boards, since it internally
# tests all boards.
machine = bitbake_variables["MACHINE"]
if not machine.startswith("vexpress-qemu"):
pytest.skip("Skipping test on non-vexpress-qemu platforms")
# This is a slow running test. Skip if appropriate.
self.check_if_should_run()
for task in ["do_provide_mender_defines", "prepare_recipe_sysroot"]:
subprocess.check_call(
"cd %s && bitbake -c %s u-boot" %
(os.environ["BUILDDIR"], task),
shell=True,
)
bitbake_variables = get_bitbake_variables("u-boot")
shutil.rmtree("/dev/shm/test_uboot_compile", ignore_errors=True)
env = copy.copy(os.environ)
env["UBOOT_SRC"] = bitbake_variables["S"]
env["TESTS_DIR"] = os.getcwd()
env["LOGS"] = os.path.join(os.getcwd(), "test_uboot_compile-logs")
if os.path.exists(env["LOGS"]):
print(
"WARNING: %s already exists. Will use cached logs from there. Recreate to reset."
% env["LOGS"])
else:
os.mkdir(env["LOGS"])
configs_to_test = self.collect_and_prepare_boards_to_test(
bitbake_variables, env)
env["BOARD_LOGS"] = " ".join(configs_to_test)
try:
sanitized_makeflags = bitbake_variables["EXTRA_OEMAKE"]
sanitized_makeflags = sanitized_makeflags.replace('\\"', '"')
sanitized_makeflags = re.sub(" +", " ", sanitized_makeflags)
env["MAYBE_UBI"] = "--ubi" if machine == "vexpress-qemu-flash" else ""
# Compile all boards. The reason for using a makefile is to get easy
# parallelization.
subprocess.check_call(
"make -j %d -f %s SUBJOBCOUNT=-j%d TMP=/dev/shm/test_uboot_compile %s"
% (
self.parallel_job_count(),
os.path.join(env["TESTS_DIR"],
"files/Makefile.test_uboot_automation"),
self.parallel_subjob_count(),
sanitized_makeflags,
),
shell=True,
env=env,
stderr=subprocess.STDOUT,
)
# Now check that the ratio of compiled boards is as expected. This
# number may change over time as U-Boot changes, but big discrepancies
# should be checked out.
failed = 0.0
total = 0.0
for file in os.listdir(env["LOGS"]):
if not file.endswith("_defconfig"):
continue
total += 1
with open(os.path.join(env["LOGS"], file)) as fd:
if "AutoPatchFailed\n" in fd.readlines():
failed += 1
assert total == len(
configs_to_test
), "Number of logs do not match the number of boards we tested? Should not happen"
if machine == "vexpress-qemu":
# PLEASE UPDATE the version you used to find this number if you update it.
# From version: v2020.01
measured_failed_ratio = 57.0 / 561.0
elif machine == "vexpress-qemu-flash":
# PLEASE UPDATE the version you used to find this number if you update it.
# From version: v2020.01
measured_failed_ratio = 13.0 / 143.0
# We tolerate a certain percentage discrepancy in either direction.
tolerated_discrepancy = 0.1
lower_bound = measured_failed_ratio * (1.0 - tolerated_discrepancy)
upper_bound = measured_failed_ratio * (1.0 + tolerated_discrepancy)
try:
assert failed / total >= lower_bound, (
"Less boards failed than expected. Good? Or a mistake somewhere? Failed: %d, Total: %d"
% (failed, total))
assert failed / total <= upper_bound, (
"More boards failed than expected. Failed: %d, Total: %d" %
(failed, total))
except AssertionError:
for file in os.listdir(env["LOGS"]):
with open(os.path.join(env["LOGS"], file)) as fd:
log = fd.readlines()
if "AutoPatchFailed\n" in log:
print(
"Last 50 lines of output from failed board: " +
file)
print("".join(log[-50:]))
raise
shutil.rmtree(env["LOGS"])
finally:
shutil.rmtree("/dev/shm/test_uboot_compile", ignore_errors=True)
