def update_image_successful(install_image,
regenerate_image_id=True,
signed=False,
skip_reboot_verification=False,
expected_mender_clients=1,
pre_upload_callback=lambda: None,
pre_deployment_callback=lambda: None,
deployment_triggered_callback=lambda: None,
compression_type="gzip"):
"""
Perform a successful upgrade, and assert that deployment status/logs are correct.
A reboot is performed, and running partitions have been swapped.
Deployment status will be set as successful for device.
Logs will not be retrieved, and result in 404.
"""
previous_inactive_part = Helpers.get_passive_partition()
with Helpers.RebootDetector() as reboot:
deployment_id, expected_image_id = common_update_procedure(
install_image,
regenerate_image_id,
signed=signed,
pre_deployment_callback=pre_deployment_callback,
deployment_triggered_callback=deployment_triggered_callback,
compression_type=compression_type)
reboot.verify_reboot_performed()
with Helpers.RebootDetector() as reboot:
try:
assert Helpers.get_active_partition() == previous_inactive_part
except AssertionError:
logs = []
for d in adm.get_devices():
logs.append(deploy.get_logs(d["device_id"], deployment_id))
pytest.fail(
"device did not flip partitions during update, here are the device logs:\n\n %s"
% (logs))
deploy.check_expected_statistics(deployment_id, "success",
expected_mender_clients)
for d in adm.get_devices():
deploy.get_logs(d["device_id"], deployment_id, expected_status=404)
if not skip_reboot_verification:
reboot.verify_reboot_not_performed()
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
deploy.check_expected_status("finished", deployment_id)
# make sure backend recognizes signed and unsigned images
artifact_id = deploy.get_deployment(deployment_id)["artifacts"][0]
artifact_info = deploy.get_artifact_details(artifact_id)
assert artifact_info[
"signed"] is signed, "image was not correct recognized as signed/unsigned"
return deployment_id
def test_deployed_during_network_outage(self, install_image=conftest.get_valid_image()):
"""
Install a valid upgrade image while there is no network availability on the device
Re-establishing the network connectivity results in the upgrade to be triggered.
Emulate a flaky network connection, and ensure that the deployment still succeeds.
"""
if not env.host_string:
execute(self.test_deployed_during_network_outage,
hosts=get_mender_clients(),
install_image=install_image)
return
Helpers.gateway_connectivity(False)
deployment_id, expected_yocto_id = common_update_proceduce(install_image)
time.sleep(60)
for i in range(5):
time.sleep(5)
Helpers.gateway_connectivity(i % 2 == 0)
Helpers.gateway_connectivity(True)
logging.info("Network stabilized")
Helpers.verify_reboot_performed()
deploy.check_expected_status(deployment_id, "success", len(get_mender_clients()))
assert Helpers.yocto_id_installed_on_machine() == expected_yocto_id
def test_update_image_id_already_installed(
self, install_image=conftest.get_valid_image()):
"""Uploading an image with an incorrect name set results in failure and rollback."""
if not env.host_string:
execute(self.test_update_image_id_already_installed,
hosts=get_mender_clients(),
install_image=install_image)
return
previous_inactive_part = Helpers.get_passive_partition()
deployment_id, expected_image_id = common_update_proceduce(
install_image, True)
Helpers.verify_reboot_performed()
devices_accepted_id = [
device["id"] for device in adm.get_devices_status("accepted")
]
deployment_id = deploy.trigger_deployment(
name="New valid update",
artifact_name=expected_image_id,
devices=devices_accepted_id)
deploy.check_expected_status(deployment_id, "already-installed",
len(get_mender_clients()))
def update_image_failed(install_image="broken_update.ext4"):
"""
Perform a upgrade using a broken image (random data)
The device will reboot, uboot will detect this is not a bootable image, and revert to the previous partition.
The resulting upgrade will be considered a failure.
"""
devices_accepted = get_mender_clients()
original_image_id = Helpers.yocto_id_installed_on_machine()
previous_active_part = Helpers.get_active_partition()
deployment_id, _ = common_update_procedure(install_image,
broken_image=True)
Helpers.verify_reboot_performed()
assert Helpers.get_active_partition() == previous_active_part
deploy.check_expected_statistics(deployment_id, "failure",
len(devices_accepted))
for d in adm.get_devices():
assert "running rollback image" in deploy.get_logs(
d["device_id"], deployment_id)
assert Helpers.yocto_id_installed_on_machine() == original_image_id
Helpers.verify_reboot_not_performed()
deploy.check_expected_status("finished", deployment_id)
def test_update_image_successful(self,
install_image=conftest.get_valid_image(),
regnerate_image_id=True):
"""
Perform a successful upgrade, and assert that deployment status/logs are correct.
A reboot is performed, and running partitions have been swapped.
Deployment status will be set as successful for device.
Logs will not be retrieved, and result in 404.
"""
if not env.host_string:
execute(self.test_update_image_successful,
hosts=get_mender_clients(),
install_image=install_image,
regnerate_image_id=regnerate_image_id)
return
previous_inactive_part = Helpers.get_passive_partition()
deployment_id, expected_image_id = common_update_proceduce(
install_image, regnerate_image_id)
Helpers.verify_reboot_performed()
assert Helpers.get_active_partition() == previous_inactive_part
deploy.check_expected_status(deployment_id, "success",
len(get_mender_clients()))
for d in adm.get_devices():
deploy.get_logs(d["id"], deployment_id, expected_status=404)
Helpers.verify_reboot_not_performed()
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
def update_image_failed(install_image="broken_update.ext4",
expected_mender_clients=1):
"""
Perform a upgrade using a broken image (random data)
The device will reboot, uboot will detect this is not a bootable image, and revert to the previous partition.
The resulting upgrade will be considered a failure.
"""
devices_accepted = get_mender_clients()
original_image_id = Helpers.yocto_id_installed_on_machine()
previous_active_part = Helpers.get_active_partition()
with Helpers.RebootDetector() as reboot:
deployment_id, _ = common_update_procedure(install_image,
broken_image=True)
reboot.verify_reboot_performed()
with Helpers.RebootDetector() as reboot:
assert Helpers.get_active_partition() == previous_active_part
deploy.check_expected_statistics(deployment_id, "failure",
expected_mender_clients)
for d in auth_v2.get_devices():
assert "got invalid entrypoint into the state machine" in deploy.get_logs(
d["id"], deployment_id)
assert Helpers.yocto_id_installed_on_machine() == original_image_id
reboot.verify_reboot_not_performed()
deploy.check_expected_status("finished", deployment_id)
def common_update_procedure(install_image,
regenerate_image_id=True,
device_type=conftest.machine_name,
broken_image=False,
verify_status=True,
signed=False,
devices=None,
scripts=[],
pre_upload_callback=lambda: None,
pre_deployment_callback=lambda: None,
deployment_triggered_callback=lambda: None,
compression_type="gzip"):
with artifact_lock:
if broken_image:
artifact_id = "broken_image_" + str(random.randint(0, 999999))
elif regenerate_image_id:
artifact_id = Helpers.artifact_id_randomize(install_image)
logger.debug("randomized image id: " + artifact_id)
else:
artifact_id = Helpers.yocto_id_from_ext4(install_image)
compression_arg = "--compression " + compression_type
# create atrifact
with tempfile.NamedTemporaryFile() as artifact_file:
created_artifact = image.make_artifact(
install_image,
device_type,
artifact_id,
artifact_file,
signed=signed,
scripts=scripts,
global_flags=compression_arg)
if created_artifact:
pre_upload_callback()
deploy.upload_image(created_artifact)
if devices is None:
devices = list(
set([
device["device_id"]
for device in adm.get_devices_status("accepted")
]))
pre_deployment_callback()
deployment_id = deploy.trigger_deployment(
name="New valid update",
artifact_name=artifact_id,
devices=devices)
else:
logger.warn("failed to create artifact")
pytest.fail("error creating artifact")
deployment_triggered_callback()
# wait until deployment is in correct state
if verify_status:
deploy.check_expected_status("inprogress", deployment_id)
return deployment_id, artifact_id
def test_large_update_image(self):
"""Installing an image larger than the passive/active parition size should result in a failure."""
if not env.host_string:
execute(self.test_large_update_image, hosts=get_mender_clients())
return
with Helpers.RebootDetector() as reboot:
deployment_id, _ = common_update_procedure(install_image="large_image.dat", regenerate_image_id=False, broken_image=True)
deploy.check_expected_statistics(deployment_id, "failure", len(get_mender_clients()))
reboot.verify_reboot_not_performed()
deploy.check_expected_status("finished", deployment_id)
def test_artifacts_persisted(self):
devices_to_update = list(
set([
device["device_id"]
for device in adm.get_devices_status("accepted",
expected_devices=10)
]))
deployment_id = deploy.trigger_deployment(
name="artifact survived backed upgrade",
artifact_name=self.provisioned_artifact_id,
devices=devices_to_update)
deploy.check_expected_status("finished", deployment_id)
def test_deployments_post_upgrade(self):
adm.get_devices_status("accepted", 10)
# perform upgrade
devices_to_update = list(set([device["device_id"] for device in adm.get_devices_status("accepted", expected_devices=10)]))
deployment_id, artifact_id = common_update_procedure("core-image-full-cmdline-%s.ext4" % conftest.machine_name,
device_type="test",
devices=devices_to_update)
deploy.check_expected_status("finished", deployment_id)
assert deploy.get_statistics(deployment_id)["success"] == 7
assert deploy.get_statistics(deployment_id)["failure"] == 3
deploy.get_status("finished")
def abort_deployment(self, abort_step=None, mender_performs_reboot=False):
"""
Trigger a deployment, and cancel it within 15 seconds, make sure no deployment is performed.
Args:
mender_performs_reboot: if set to False, a manual reboot is performed and
checks are performed.
if set to True, wait until device is rebooted.
"""
if not env.host_string:
execute(self.abort_deployment,
abort_step=abort_step,
mender_performs_reboot=mender_performs_reboot,
hosts=get_mender_clients())
return
install_image = conftest.get_valid_image()
expected_partition = Helpers.get_active_partition()
expected_image_id = Helpers.yocto_id_installed_on_machine()
with Helpers.RebootDetector() as reboot:
deployment_id, _ = common_update_procedure(install_image,
verify_status=False)
if abort_step is not None:
deploy.check_expected_statistics(deployment_id, abort_step,
len(get_mender_clients()))
deploy.abort(deployment_id)
deploy.check_expected_statistics(deployment_id, "aborted",
len(get_mender_clients()))
# no deployment logs are sent by the client, is this expected?
for d in auth_v2.get_devices():
deploy.get_logs(d["id"], deployment_id, expected_status=404)
if mender_performs_reboot:
# If Mender performs reboot, we need to wait for it to reboot
# back into the original filesystem.
reboot.verify_reboot_performed(number_of_reboots=2)
else:
# Else we reboot ourselves, just to make sure that we have not
# unintentionally switched to the new partition.
reboot.verify_reboot_not_performed()
run("( sleep 10 ; reboot ) 2>/dev/null >/dev/null &")
reboot.verify_reboot_performed()
assert Helpers.get_active_partition() == expected_partition
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
deploy.check_expected_status("finished", deployment_id)
def test_update_image_breaks_networking(self, install_image="core-image-full-cmdline-vexpress-qemu-broken-network.ext4"):
"""
Install an image without systemd-networkd binary existing.
The network will not function, mender will not be able to send any logs.
The expected status is the update will rollback, and be considered a failure
"""
if not env.host_string:
execute(self.test_update_image_breaks_networking,
hosts=get_mender_clients(),
install_image=install_image)
return
deployment_id, _ = common_update_proceduce(install_image)
Helpers.verify_reboot_performed() # since the network is broken, two reboots will be performed, and the last one will be detected
deploy.check_expected_status(deployment_id, "failure", len(get_mender_clients()))
def test_deployment_abortion_success(self):
# maybe an acceptance test is enough for this check?
if not env.host_string:
execute(self.test_deployment_abortion_success,
hosts=get_mender_clients())
return
install_image = conftest.get_valid_image()
deployment_id, _ = common_update_procedure(install_image)
Helpers.verify_reboot_performed()
deploy.check_expected_statistics(deployment_id, "success", len(get_mender_clients()))
deploy.abort_finished_deployment(deployment_id)
deploy.check_expected_statistics(deployment_id, "success", len(get_mender_clients()))
deploy.check_expected_status("finished", deployment_id)
def test_unsigned_artifact_fails_deployment(self, standard_setup_with_signed_artifact_client):
"""
Make sure that an unsigned image fails, and is handled by the backend.
Notice that this test needs a fresh new version of the backend, since
we installed a signed image earlier without a verification key in mender.conf
"""
if not env.host_string:
execute(self.test_unsigned_artifact_fails_deployment,
standard_setup_with_signed_artifact_client,
hosts=get_mender_clients())
return
deployment_id, _ = common_update_procedure(install_image=conftest.get_valid_image())
deploy.check_expected_status("finished", deployment_id)
deploy.check_expected_statistics(deployment_id, "failure", 1)
for d in adm.get_devices():
assert "expecting signed artifact, but no signature file found" in \
deploy.get_logs(d["device_id"], deployment_id)
def abort_deployment(self, abort_step=None, mender_performs_reboot=False):
"""
Trigger a deployment, and cancel it within 15 seconds, make sure no deployment is performed.
Args:
mender_performs_reboot: if set to False, a manual reboot is performed and
checks are performed.
if set to True, wait until device is rebooted.
"""
if not env.host_string:
execute(self.abort_deployment,
abort_step=abort_step,
mender_performs_reboot=mender_performs_reboot,
hosts=get_mender_clients())
return
install_image = conftest.get_valid_image()
expected_partition = Helpers.get_active_partition()
expected_image_id = Helpers.yocto_id_installed_on_machine()
token = Helpers.place_reboot_token()
deployment_id, _ = common_update_procedure(install_image,
verify_status=False)
if abort_step is not None:
deploy.check_expected_statistics(deployment_id, abort_step,
len(get_mender_clients()))
deploy.abort(deployment_id)
deploy.check_expected_statistics(deployment_id, "aborted",
len(get_mender_clients()))
# no deployment logs are sent by the client, is this expected?
for d in adm.get_devices():
deploy.get_logs(d["device_id"], deployment_id, expected_status=404)
if not mender_performs_reboot:
token.verify_reboot_not_performed()
run("( sleep 10 ; reboot ) 2>/dev/null >/dev/null &")
token.verify_reboot_performed()
assert Helpers.get_active_partition() == expected_partition
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
deploy.check_expected_status("finished", deployment_id)
def common_update_procedure(install_image,
regenerate_image_id=True,
device_type="vexpress-qemu",
broken_image=False,
verify_status=True,
devices=None):
with artifact_lock:
if broken_image:
artifact_id = "broken_image_" + str(random.randint(0, 999999))
elif regenerate_image_id:
artifact_id = Helpers.artifact_id_randomize(install_image)
logger.debug("Randomized image id: " + artifact_id)
else:
artifact_id = Helpers.yocto_id_from_ext4(install_image)
# create atrifact
with tempfile.NamedTemporaryFile() as artifact_file:
created_artifact = image.make_artifact(install_image, device_type,
artifact_id, artifact_file)
if created_artifact:
deploy.upload_image(created_artifact)
if devices is None:
devices = list(
set([
device["device_id"]
for device in adm.get_devices_status("accepted")
]))
deployment_id = deploy.trigger_deployment(
name="New valid update",
artifact_name=artifact_id,
devices=devices)
else:
pytest.fail("error creating artifact")
# wait until deployment is in correct state
if verify_status:
deploy.check_expected_status("inprogress", deployment_id)
return deployment_id, artifact_id
def update_image_successful(
install_image=conftest.get_valid_image(), regenerate_image_id=True):
"""
Perform a successful upgrade, and assert that deployment status/logs are correct.
A reboot is performed, and running partitions have been swapped.
Deployment status will be set as successful for device.
Logs will not be retrieved, and result in 404.
"""
previous_inactive_part = Helpers.get_passive_partition()
deployment_id, expected_image_id = common_update_procedure(
install_image, regenerate_image_id)
Helpers.verify_reboot_performed()
try:
assert Helpers.get_active_partition() == previous_inactive_part
except AssertionError:
logs = []
for d in adm.get_devices():
logs.append(deploy.get_logs(d["device_id"], deployment_id))
pytest.fail(
"device did not flip partitions during update, here are the device logs:\n\n %s"
% (logs))
deploy.check_expected_statistics(deployment_id, "success",
len(get_mender_clients()))
for d in adm.get_devices():
deploy.get_logs(d["device_id"], deployment_id, expected_status=404)
Helpers.verify_reboot_not_performed()
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
deploy.check_expected_status("finished", deployment_id)
def test_update_image_recovery(self, install_image=conftest.get_valid_image()):
"""
Install an update, and reboot the system when we detect it's being copied over to the inactive parition.
The test should result in a failure.
"""
if not env.host_string:
execute(self.test_update_image_recovery,
hosts=get_mender_clients(),
install_image=install_image)
return
installed_yocto_id = Helpers.yocto_id_installed_on_machine()
inactive_part = Helpers.get_passive_partition()
deployment_id, _ = common_update_proceduce(install_image)
active_part = Helpers.get_active_partition()
for i in range(60):
time.sleep(0.5)
with quiet():
# make sure we are writing to the inactive partition
output = run("fuser -mv %s" % (inactive_part))
if output.return_code == 0:
run("killall -s 9 mender")
with settings(warn_only=True):
run("( sleep 3 ; reboot ) 2>/dev/null >/dev/null &")
break
logging.info("Waiting for system to finish reboot")
Helpers.verify_reboot_performed()
assert Helpers.get_active_partition() == active_part
deploy.check_expected_status(deployment_id, "failure", len(get_mender_clients()))
Helpers.verify_reboot_not_performed()
assert Helpers.yocto_id_installed_on_machine() == installed_yocto_id