def test_image_update_broken_kernel(
self,
bitbake_variables,
connection,
latest_mender_image,
http_server,
board_type,
use_s3,
s3_address,
):
"""Test that an update with a broken kernel rolls back correctly. This is
distinct from the test_broken_image_update test, which corrupts the
filesystem. When grub.d integration is enabled, these two scenarios
trigger very different code paths."""
file_flag = Helpers.get_file_flag(bitbake_variables)
(active_before, passive_before) = determine_active_passive_part(
bitbake_variables, connection)
image_type = bitbake_variables["MENDER_DEVICE_TYPE"]
temp_artifact = "temporary_artifact.mender"
try:
shutil.copyfile(latest_mender_image, temp_artifact)
# Assume that artifact has the same kernel names as the currently
# running image.
kernels = connection.run(
"find /boot/ -maxdepth 1 -name '*linu[xz]*' -o -name '*Image'"
).stdout.split()
for kernel in kernels:
# Inefficient, but there shouldn't be too many kernels.
subprocess.check_call(
["mender-artifact", "rm", f"{temp_artifact}:{kernel}"])
Helpers.install_update(
temp_artifact,
connection,
http_server,
board_type,
use_s3,
s3_address,
)
reboot(connection)
# Now qemu is auto-rebooted twice; once to boot the dummy image,
# where it fails, and the boot loader auto-reboots a second time
# into the original partition.
output = run_after_connect("mount", connection)
# The update should have reverted to the original active partition,
# since the kernel was missing.
assert output.find(active_before) >= 0
assert output.find(passive_before) < 0
finally:
os.remove(temp_artifact)
def test_broken_image_update(self, bitbake_variables, connection):
"""Test that an update with a broken filesystem rolls back correctly."""
file_flag = Helpers.get_file_flag(bitbake_variables)
(active_before, passive_before) = determine_active_passive_part(
bitbake_variables, connection)
image_type = bitbake_variables["MENDER_DEVICE_TYPE"]
try:
# Make a dummy/broken update
retcode = subprocess.call(
"dd if=/dev/zero of=image.dat bs=1M count=0 seek=16",
shell=True)
if retcode != 0:
raise Exception("error creating dummy image")
retcode = subprocess.call(
"mender-artifact write rootfs-image -t %s -n test-update %s image.dat -o image.mender"
% (image_type, file_flag),
shell=True,
)
if retcode != 0:
raise Exception(
"error writing mender artifact using command: mender-artifact write rootfs-image -t %s -n test-update %s image.dat -o image.mender"
% (image_type, file_flag))
put_no_sftp("image.mender",
connection,
remote="/var/tmp/image.mender")
connection.run("mender install /var/tmp/image.mender")
reboot(connection)
# Now qemu is auto-rebooted twice; once to boot the dummy image,
# where it fails, and the boot loader auto-reboots a second time
# into the original partition.
output = run_after_connect("mount", connection)
# The update should have reverted to the original active partition,
# since the image was bogus.
assert output.find(active_before) >= 0
assert output.find(passive_before) < 0
finally:
# Cleanup.
if os.path.exists("image.mender"):
os.remove("image.mender")
if os.path.exists("image.dat"):
os.remove("image.dat")
def test_runtime_checksum(
self,
request,
setup_board,
prepared_test_build,
bitbake_variables,
bitbake_image,
connection,
http_server,
board_type,
use_s3,
s3_address,
):
"""Check that the checksum of the running root filesystem is what we
expect. This is important in order for it to match when applying a delta
update.
"""
if ("read-only-rootfs"
not in bitbake_variables["IMAGE_FEATURES"].strip().split()):
pytest.skip("Only works when using read-only-rootfs IMAGE_FEATURE")
image = self.do_install_mender_binary_delta(
request,
prepared_test_build,
bitbake_variables,
bitbake_image,
connection,
http_server,
board_type,
use_s3,
s3_address,
)
# Check that checksum of the currently mounted rootfs matches that
# of the artifact which we just updated to.
(active, _) = determine_active_passive_part(bitbake_variables,
connection)
output = connection.run("sha256sum %s" % active)
rootfs_sum = output.stdout.split()[0]
output = subprocess.check_output("mender-artifact read %s" % image,
shell=True).decode()
match = re.search("checksum: *([0-9a-f]+)", output)
assert match is not None, (
"Could not find checksum in mender-artifact output: %s" % output)
artifact_sum = match.group(1)
assert rootfs_sum == artifact_sum
def test_snapshot_using_mender_artifact(self, terminal, bitbake_path,
bitbake_variables, connection):
try:
(active,
_) = determine_active_passive_part(bitbake_variables, connection)
common_args = get_ssh_common_args(connection)
# mender-artifact prefixes each ssh argument with "-S"
common_args = common_args.replace(" ", " -S ")
try:
subprocess.check_call(
"%s mender-artifact write rootfs-image -S %s -n test -t test -o test_snapshot_using_mender_artifact.mender -f ssh://%s@%s:%s"
% (
terminal,
common_args,
connection.user,
connection.host,
connection.port,
),
shell=True,
)
finally:
subprocess.call("cat screen.log.tmp ; rm -f screen.log.tmp",
shell=True)
output = subprocess.check_output(
"mender-artifact read test_snapshot_using_mender_artifact.mender",
shell=True,
).decode()
partsize = connection.run("blockdev --getsize64 %s" %
active).stdout.strip()
# Ensure that the payload size of the produced artifact matches the
# partition.
assert re.search("size: *%s" % partsize, output) is not None
finally:
try:
os.remove("test_snapshot_using_mender_artifact.mender")
except:
pass
def test_basic_snapshot(self, compression, bitbake_variables, connection):
try:
(_,
passive) = determine_active_passive_part(bitbake_variables,
connection)
# Wipe the inactive partition first.
connection.run("dd if=/dev/zero of=%s bs=1M count=100" % passive)
# Dump what we currently have to the inactive partition.
connection.run("mender snapshot dump %s %s %s" %
(compression[0], compression[1], passive))
# Make sure this looks like a sane filesystem.
connection.run("fsck.ext4 -p %s" % passive)
# And that it can be mounted with actual content.
connection.run("mount %s /mnt" % passive)
connection.run("test -f /mnt/etc/passwd")
finally:
connection.run("umount /mnt || true")
def test_snapshot_inactive(self, bitbake_variables, connection):
try:
(_,
passive) = determine_active_passive_part(bitbake_variables,
connection)
test_str = "TeSt StrIng!#"
# Seed the initial part of the inactive partition with a test string
connection.run("echo '%s' | dd of=%s" % (test_str, passive))
# Try to snapshot inactive partition, keeping the initial part, and
# dumping the rest.
connection.run(
"mender snapshot dump --source %s | ( dd of=/data/snapshot-test bs=%d count=1; cat > /dev/null )"
% (passive, len(test_str)))
output = connection.run("cat /data/snapshot-test").stdout.strip()
assert output == test_str
finally:
connection.run("rm -f /data/snapshot-test")
def test_snapshot_device_file(self, bitbake_variables, connection):
try:
(active,
passive) = determine_active_passive_part(bitbake_variables,
connection)
# Wipe the inactive partition first.
connection.run("dd if=/dev/zero of=%s bs=1M count=100" % passive)
# Dump what we currently have to the inactive partition, using
# device file reference.
connection.run("mender snapshot dump --source %s > %s" %
(active, passive))
# Make sure this looks like a sane filesystem.
connection.run("fsck.ext4 -p %s" % passive)
# And that it can be mounted with actual content.
connection.run("mount %s /mnt" % passive)
connection.run("test -f /mnt/etc/passwd")
finally:
connection.run("umount /mnt || true")
def test_perform_update(
self,
request,
setup_board,
prepared_test_build,
bitbake_variables,
bitbake_image,
connection,
http_server,
board_type,
use_s3,
s3_address,
):
"""Perform a delta update.
"""
if ("read-only-rootfs"
not in bitbake_variables["IMAGE_FEATURES"].strip().split()):
pytest.skip("Only works when using read-only-rootfs IMAGE_FEATURE")
if distutils.spawn.find_executable(
"mender-binary-delta-generator") is None:
pytest.fail("mender-binary-delta-generator not found in PATH")
built_artifact = self.do_install_mender_binary_delta(
request,
prepared_test_build,
bitbake_variables,
bitbake_image,
connection,
http_server,
board_type,
use_s3,
s3_address,
)
with make_tempdir() as tmpdir:
# Copy previous build
artifact_from = os.path.join(tmpdir, "artifact_from.mender")
shutil.copyfile(built_artifact, artifact_from)
# Create new image installing some extra software
build_image(
prepared_test_build["build_dir"],
prepared_test_build["bitbake_corebase"],
bitbake_image,
['IMAGE_INSTALL_append = " nano"'],
)
built_artifact = latest_build_artifact(
request, prepared_test_build["build_dir"],
"core-image*.mender")
artifact_to = os.path.join(tmpdir, "artifact_to.mender")
shutil.copyfile(built_artifact, artifact_to)
# Create delta Artifact using mender-binary-delta-generator
artifact_delta = os.path.join(tmpdir, "artifact_delta.mender")
subprocess.check_call(
f"mender-binary-delta-generator -n v2.0-deltafrom-v1.0 {artifact_from} {artifact_to} -o {artifact_delta}",
shell=True,
)
# Verbose provides/depends of the different Artifacts and the client (when supported)
connection.run("mender show-provides", warn=True)
subprocess.check_call(
"mender-artifact read %s" % artifact_from,
shell=True,
)
subprocess.check_call(
"mender-artifact read %s" % artifact_to,
shell=True,
)
subprocess.check_call(
"mender-artifact read %s" % artifact_delta,
shell=True,
)
# Install Artifact, verify partitions and commit
(active,
passive) = determine_active_passive_part(bitbake_variables,
connection)
Helpers.install_update(artifact_delta, connection, http_server,
board_type, use_s3, s3_address)
reboot(connection)
run_after_connect("true", connection)
(new_active, new_passive) = determine_active_passive_part(
bitbake_variables, connection)
assert new_active == passive
assert new_passive == active
connection.run("mender -commit")
def test_redundant_grub_env(self, successful_image_update_mender,
bitbake_variables, connection):
"""This tests pretty much the same thing as the test_redundant_uboot_env
above, but the details differ. U-Boot maintains a counter in each
environment, and then only updates one of them. However, the GRUB
variant we have implemented in the GRUB scripting language, where we
cannot do this, so instead we update both, and use the validity of the
variables instead as a crude checksum."""
(active,
passive) = determine_active_passive_part(bitbake_variables,
connection)
# Corrupt the passive partition.
connection.run("dd if=/dev/zero of=%s bs=1024 count=1024" % passive)
if ("mender-bios" in bitbake_variables.get("MENDER_FEATURES",
"").split()
or "mender-bios" in bitbake_variables.get(
"DISTRO_FEATURES", "").split()):
env_dir = "/boot/grub/grub-mender-grubenv"
else:
env_dir = "/boot/efi/grub-mender-grubenv"
# Now try to corrupt the environment, and make sure it doesn't get booted into.
for env_num in [1, 2]:
# Make a copy of the two environments.
connection.run(
"cp %s/{mender_grubenv1/env,mender_grubenv1/env.backup}" %
env_dir)
connection.run(
"cp %s/{mender_grubenv1/lock,mender_grubenv1/lock.backup}" %
env_dir)
connection.run(
"cp %s/{mender_grubenv2/env,mender_grubenv2/env.backup}" %
env_dir)
connection.run(
"cp %s/{mender_grubenv2/lock,mender_grubenv2/lock.backup}" %
env_dir)
try:
env_file = "%s/mender_grubenv%d/env" % (env_dir, env_num)
lock_file = "%s/mender_grubenv%d/lock" % (env_dir, env_num)
connection.run('sed -e "s/editing=.*/editing=1/" %s' %
lock_file)
connection.run(
'sed -e "s/mender_boot_part=.*/mender_boot_part=%s/" %s' %
(passive[-1], lock_file))
reboot(connection)
run_after_connect("true", connection)
(new_active, new_passive) = determine_active_passive_part(
bitbake_variables, connection)
assert new_active == active
assert new_passive == passive
finally:
# Restore the two environments.
connection.run(
"mv %s/{mender_grubenv1/env.backup,mender_grubenv1/env}" %
env_dir)
connection.run(
"mv %s/{mender_grubenv1/lock.backup,mender_grubenv1/lock}"
% env_dir)
connection.run(
"mv %s/{mender_grubenv2/env.backup,mender_grubenv2/env}" %
env_dir)
connection.run(
"mv %s/{mender_grubenv2/lock.backup,mender_grubenv2/lock}"
% env_dir)
def test_signed_updates(self, sig_case, bitbake_variables, connection):
"""Test various combinations of signed and unsigned, present and non-
present verification keys."""
file_flag = Helpers.get_file_flag(bitbake_variables)
# mmc mount points are named: /dev/mmcblk0p1
# ubi volumes are named: ubi0_1
(active,
passive) = determine_active_passive_part(bitbake_variables,
connection)
if passive.startswith("ubi"):
passive = "/dev/" + passive
# Generate "update" appropriate for this test case.
# Cheat a little. Instead of spending a lot of time on a lot of reboots,
# just verify that the contents of the update are correct.
new_content = sig_case.label
with open("image.dat", "w") as fd:
fd.write(new_content)
# Write some extra data just to make sure the update is big enough
# to be written even if the checksum is wrong. If it's too small it
# may fail before it has a chance to be written.
fd.write("\x00" * (1048576 * 8))
artifact_args = ""
# Generate artifact with or without signature.
if sig_case.signature:
artifact_args += " -k %s" % signing_key(sig_case.key_type).private
# Generate artifact with specific version. None means default.
if sig_case.artifact_version is not None:
artifact_args += " -v %d" % sig_case.artifact_version
if sig_case.key_type:
sig_key = signing_key(sig_case.key_type)
else:
sig_key = None
image_type = bitbake_variables["MENDER_DEVICE_TYPE"]
subprocess.check_call(
"mender-artifact write rootfs-image %s -t %s -n test-update %s image.dat -o image.mender"
% (artifact_args, image_type, file_flag),
shell=True,
)
# If instructed to, corrupt the signature and/or checksum.
if ((sig_case.signature and not sig_case.signature_ok)
or not sig_case.checksum_ok
or not sig_case.header_checksum_ok):
tar = subprocess.check_output(["tar", "tf", "image.mender"])
tar_list = tar.split()
tmpdir = tempfile.mkdtemp()
try:
shutil.copy("image.mender",
os.path.join(tmpdir, "image.mender"))
cwd = os.open(".", os.O_RDONLY)
os.chdir(tmpdir)
try:
tar = subprocess.check_output(
["tar", "xf", "image.mender"])
if not sig_case.signature_ok:
# Corrupt signature.
with open("manifest.sig", "r+") as fd:
Helpers.corrupt_middle_byte(fd)
if not sig_case.checksum_ok:
os.chdir("data")
try:
data_list = subprocess.check_output(
["tar", "tzf", "0000.tar.gz"])
data_list = data_list.split()
subprocess.check_call(
["tar", "xzf", "0000.tar.gz"])
# Corrupt checksum by changing file slightly.
with open("image.dat", "r+") as fd:
Helpers.corrupt_middle_byte(fd)
# Pack it up again in same order.
os.remove("0000.tar.gz")
subprocess.check_call(
["tar", "czf", "0000.tar.gz"] + data_list)
for data_file in data_list:
os.remove(data_file)
finally:
os.chdir("..")
if not sig_case.header_checksum_ok:
data_list = subprocess.check_output(
["tar", "tzf", "header.tar.gz"])
data_list = data_list.split()
subprocess.check_call(["tar", "xzf", "header.tar.gz"])
# Corrupt checksum by changing file slightly.
with open("headers/0000/files", "a") as fd:
# Some extra data to corrupt the header checksum,
# but still valid JSON.
fd.write(" ")
# Pack it up again in same order.
os.remove("header.tar.gz")
subprocess.check_call(["tar", "czf", "header.tar.gz"] +
data_list)
for data_file in data_list:
os.remove(data_file)
# Make sure we put it back in the same order.
os.remove("image.mender")
subprocess.check_call(["tar", "cf", "image.mender"] +
tar_list)
finally:
os.fchdir(cwd)
os.close(cwd)
shutil.move(os.path.join(tmpdir, "image.mender"),
"image.mender")
finally:
shutil.rmtree(tmpdir, ignore_errors=True)
put_no_sftp("image.mender", connection, remote="/data/image.mender")
# mender-convert'ed images don't have transient mender.conf
device_has_mender_conf = (connection.run(
"test -f /etc/mender/mender.conf", warn=True).return_code == 0)
# mender-convert'ed images don't have this directory, but the test uses
# it to save certificates
connection.run("mkdir -p /data/etc/mender")
try:
# Get configuration from device or create an empty one
if device_has_mender_conf:
connection.run(
"cp /etc/mender/mender.conf /data/etc/mender/mender.conf.bak"
)
get_no_sftp("/etc/mender/mender.conf", connection)
else:
with open("mender.conf", "w") as fd:
json.dump({}, fd)
# Update key in configuration.
with open("mender.conf") as fd:
config = json.load(fd)
if sig_case.key:
config[
"ArtifactVerifyKey"] = "/data/etc/mender/%s" % os.path.basename(
sig_key.public)
put_no_sftp(
sig_key.public,
connection,
remote="/data/etc/mender/%s" %
os.path.basename(sig_key.public),
)
else:
if config.get("ArtifactVerifyKey"):
del config["ArtifactVerifyKey"]
# Send new configuration to device
with open("mender.conf", "w") as fd:
json.dump(config, fd)
put_no_sftp("mender.conf",
connection,
remote="/etc/mender/mender.conf")
os.remove("mender.conf")
# Start by writing known "old" content in the partition.
old_content = "Preexisting partition content"
if "ubi" in passive:
# ubi volumes cannot be directly written to, we have to use
# ubiupdatevol
connection.run('echo "%s" | dd of=/tmp/update.tmp && '
"ubiupdatevol %s /tmp/update.tmp; "
"rm -f /tmp/update.tmp" %
(old_content, passive))
else:
connection.run('echo "%s" | dd of=%s' % (old_content, passive))
result = connection.run("mender install /data/image.mender",
warn=True)
if sig_case.success:
if result.return_code != 0:
pytest.fail(
"Update failed when it should have succeeded: %s, Output: %s"
% (sig_case.label, result))
else:
if result.return_code == 0:
pytest.fail(
"Update succeeded when it should not have: %s, Output: %s"
% (sig_case.label, result))
if sig_case.update_written:
expected_content = new_content
else:
expected_content = old_content
try:
content = connection.run(
"dd if=%s bs=%d count=1" %
(passive, len(expected_content))).stdout
assert content == expected_content, "Case: %s" % sig_case.label
# In Fabric context, SystemExit means CalledProcessError. We should
# not catch all exceptions, because we want to leave assertions
# alone.
# In Fabric2 there might be different exception thrown in that case
# which is UnexpectedExit.
except (SystemExit, UnexpectedExit):
if ("mender-ubi" in bitbake_variables.get(
"MENDER_FEATURES", "").split()
or "mender-ubi" in bitbake_variables.get(
"DISTRO_FEATURES", "").split()):
# For UBI volumes specifically: The UBI_IOCVOLUP call which
# Mender uses prior to writing the data, takes a size
# argument, and if you don't write that amount of bytes, the
# volume is marked corrupted as a security measure. This
# sometimes triggers in our checksum mismatch tests, so
# accept the volume being unreadable in that case.
pass
else:
raise
finally:
# Reset environment to what it was.
_, bootenv_set = bootenv_tools(connection)
connection.run(f"{bootenv_set} mender_boot_part %s" % active[-1:])
connection.run(f"{bootenv_set} mender_boot_part_hex %x" %
int(active[-1:]))
connection.run(f"{bootenv_set} upgrade_available 0")
if device_has_mender_conf:
connection.run(
"cp -L /data/etc/mender/mender.conf.bak $(realpath /etc/mender/mender.conf)"
)
if sig_key:
connection.run("rm -f /etc/mender/%s" %
os.path.basename(sig_key.public))
def test_network_based_image_update(
self,
successful_image_update_mender,
bitbake_variables,
connection,
http_server,
board_type,
use_s3,
s3_address,
):
(active_before, passive_before) = determine_active_passive_part(
bitbake_variables, connection)
Helpers.install_update(
successful_image_update_mender,
connection,
http_server,
board_type,
use_s3,
s3_address,
)
bootenv_print, _ = bootenv_tools(connection)
output = connection.run(f"{bootenv_print} bootcount").stdout
assert output.rstrip("\n") == "bootcount=0"
output = connection.run(f"{bootenv_print} upgrade_available").stdout
assert output.rstrip("\n") == "upgrade_available=1"
output = connection.run(f"{bootenv_print} mender_boot_part").stdout
assert output.rstrip("\n") == "mender_boot_part=" + passive_before[-1:]
# Delete kernel and associated files from currently running partition,
# so that the boot will fail if U-Boot for any reason tries to grab the
# kernel from the wrong place.
connection.run("rm -f /boot/* || true")
reboot(connection)
run_after_connect("true", connection)
(active_after, passive_after) = determine_active_passive_part(
bitbake_variables, connection)
# The OS should have moved to a new partition, since the image was fine.
assert active_after == passive_before
assert passive_after == active_before
output = connection.run(f"{bootenv_print} bootcount").stdout
assert output.rstrip("\n") == "bootcount=1"
output = connection.run(f"{bootenv_print} upgrade_available").stdout
assert output.rstrip("\n") == "upgrade_available=1"
output = connection.run(f"{bootenv_print} mender_boot_part").stdout
assert output.rstrip("\n") == "mender_boot_part=" + active_after[-1:]
connection.run("mender commit")
output = connection.run(f"{bootenv_print} upgrade_available").stdout
assert output.rstrip("\n") == "upgrade_available=0"
output = connection.run(f"{bootenv_print} mender_boot_part").stdout
assert output.rstrip("\n") == "mender_boot_part=" + active_after[-1:]
active_before = active_after
passive_before = passive_after
reboot(connection)
run_after_connect("true", connection)
(active_after, passive_after) = determine_active_passive_part(
bitbake_variables, connection)
# The OS should have stayed on the same partition, since we committed.
assert active_after == active_before
assert passive_after == passive_before
def test_snapshot_using_mender_artifact_no_sudo( # see MEN-3987
self, terminal, bitbake_path, bitbake_variables, connection):
try:
(active,
passive) = determine_active_passive_part(bitbake_variables,
connection)
common_args = get_ssh_common_args(connection)
# mender-artifact prefixes each ssh argument with "-S"
common_args = common_args.replace(" ", " -S ")
# /usr/bin/sudo is a link to /data/usr/bin/sudo see
# meta-mender-qemu/recipes-extended/sudo/sudo_%.bbappend
sudo_path = "/data/usr/bin/sudo"
# let's move the sudo away
result = connection.run("mv %s %s-off" % (sudo_path, sudo_path),
warn=True,
echo=True)
assert result.return_code == 0
# lets be sure it does not work
result = connection.run("sudo --help", echo=True, warn=True)
assert result.return_code != 0
try:
# mender-artifact as of mender-artifact/pull/305 does not use sudo
# when user is root or when uid is 0
subprocess.check_call(
"%s mender-artifact write rootfs-image -S %s -n test -t test -o test_snapshot_using_mender_artifact.mender -f ssh://%s@%s:%s"
% (
terminal,
common_args,
connection.user,
connection.host,
connection.port,
),
shell=True,
)
finally:
subprocess.call("cat screen.log.tmp ; rm -f screen.log.tmp",
shell=True)
# let's put the sudo back in place and verify that it works
result = connection.run("mv %s-off %s" %
(sudo_path, sudo_path),
warn=True,
echo=True)
result = connection.run("sudo --help", echo=True, warn=True)
assert result.return_code == 0
output = subprocess.check_output(
"mender-artifact read test_snapshot_using_mender_artifact.mender",
shell=True,
).decode()
partsize = connection.run("blockdev --getsize64 %s" %
active).stdout.strip()
# Ensure that the payload size of the produced artifact matches the
# partition.
assert re.search("size: *%s" % partsize, output) is not None
finally:
try:
os.remove("test_snapshot_using_mender_artifact.mender")
except:
pass
