def test_reject_bootstrap(self):
"""Make sure a rejected device does not perform an upgrade, and that it gets it's auth token removed"""
if not env.host_string:
execute(self.test_reject_bootstrap, hosts=get_mender_clients())
return
# iterate over devices and reject them
for device in adm.get_devices():
adm.set_device_status(device["id"], "rejected")
logging.info("Rejecting DeviceID: %s" % device["id"])
adm.check_expected_status("rejected", len(get_mender_clients()))
try:
deployment_id, _ = common_update_proceduce(
install_image=conftest.get_valid_image())
except AssertionError:
logging.info("Failed to deploy upgrade to rejected device.")
Helpers.verify_reboot_not_performed()
# authtoken has been removed
assert not exists("/data/mender/authtoken")
else:
raise ("No error while trying to deploy to rejected device")
# re-accept device after test is done
adm.accept_devices(1)
def test_update_image_id_already_installed(
self, install_image=conftest.get_valid_image()):
"""Uploading an image with an incorrect name set results in failure and rollback."""
if not env.host_string:
execute(self.test_update_image_id_already_installed,
hosts=get_mender_clients(),
install_image=install_image)
return
previous_inactive_part = Helpers.get_passive_partition()
deployment_id, expected_image_id = common_update_proceduce(
install_image, True)
Helpers.verify_reboot_performed()
devices_accepted_id = [
device["id"] for device in adm.get_devices_status("accepted")
]
deployment_id = deploy.trigger_deployment(
name="New valid update",
artifact_name=expected_image_id,
devices=devices_accepted_id)
deploy.check_expected_status(deployment_id, "already-installed",
len(get_mender_clients()))
def test_image_download_retry_2(self,
install_image=conftest.get_valid_image()):
"""
Block storage host (minio) by modifying the hosts file.
"""
if not env.host_string:
execute(self.test_image_download_retry_2,
hosts=get_mender_clients(),
install_image=install_image)
return
inactive_part = Helpers.get_passive_partition()
run("echo '1.1.1.1 s3.docker.mender.io' >> /etc/hosts"
) # break s3 connectivity before triggering deployment
deployment_id, new_yocto_id = common_update_proceduce(install_image)
self.wait_for_download_retry_attempts()
run("sed -i.bak '/1.1.1.1/d' /etc/hosts")
Helpers.verify_reboot_performed()
assert Helpers.get_active_partition() == inactive_part
assert Helpers.yocto_id_installed_on_machine() == new_yocto_id
Helpers.verify_reboot_not_performed()
def test_deployed_during_network_outage(
self, install_image=conftest.get_valid_image()):
"""
Install a valid upgrade image while there is no network availability on the device
Re-establishing the network connectivity results in the upgrade to be triggered.
Emulate a flaky network connection, and ensure that the deployment still succeeds.
"""
if not env.host_string:
execute(self.test_deployed_during_network_outage,
hosts=get_mender_clients(),
install_image=install_image)
return
Helpers.gateway_connectivity(False)
deployment_id, expected_yocto_id = common_update_proceduce(
install_image, verify_status=False)
time.sleep(60)
for i in range(5):
time.sleep(5)
Helpers.gateway_connectivity(i % 2 == 0)
Helpers.gateway_connectivity(True)
logging.info("Network stabilized")
Helpers.verify_reboot_performed()
deploy.check_expected_statistics(deployment_id, "success",
len(get_mender_clients()))
assert Helpers.yocto_id_installed_on_machine() == expected_yocto_id
def test_update_image_failed(self, install_image="broken_update.ext4"):
"""
Perform a upgrade using a broken image (random data)
The device will reboot, uboot will detect this is not a bootable image, and revert to the previous partition.
The resulting upgrade will be considered a failure.
"""
if not env.host_string:
execute(self.test_update_image_failed,
hosts=get_mender_clients(),
install_image=install_image)
return
devices_accepted = get_mender_clients()
original_image_id = Helpers.yocto_id_installed_on_machine()
previous_active_part = Helpers.get_active_partition()
deployment_id, _ = common_update_proceduce(install_image,
broken_image=True)
Helpers.verify_reboot_performed()
assert Helpers.get_active_partition() == previous_active_part
deploy.check_expected_status(deployment_id, "failure",
len(devices_accepted))
for d in adm.get_devices():
assert "running rollback image" in deploy.get_logs(
d["id"], deployment_id)
assert Helpers.yocto_id_installed_on_machine() == original_image_id
Helpers.verify_reboot_not_performed()
def test_update_image_successful(self,
install_image=conftest.get_valid_image(),
regnerate_image_id=True):
"""
Perform a successful upgrade, and assert that deployment status/logs are correct.
A reboot is performed, and running partitions have been swapped.
Deployment status will be set as successful for device.
Logs will not be retrieved, and result in 404.
"""
if not env.host_string:
execute(self.test_update_image_successful,
hosts=get_mender_clients(),
install_image=install_image,
regnerate_image_id=regnerate_image_id)
return
previous_inactive_part = Helpers.get_passive_partition()
deployment_id, expected_image_id = common_update_proceduce(
install_image, regnerate_image_id)
Helpers.verify_reboot_performed()
assert Helpers.get_active_partition() == previous_inactive_part
deploy.check_expected_status(deployment_id, "success",
len(get_mender_clients()))
for d in adm.get_devices():
deploy.get_logs(d["id"], deployment_id, expected_status=404)
Helpers.verify_reboot_not_performed()
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
def test_large_update_image(self):
"""Installing an image larger than the passive/active parition size should result in a failure."""
if not env.host_string:
execute(self.test_large_update_image, hosts=get_mender_clients())
return
deployment_id, _ = common_update_proceduce(
install_image="large_image.dat",
regnerate_image_id=False,
broken_image=True)
deploy.check_expected_status(deployment_id, "failure",
len(get_mender_clients()))
Helpers.verify_reboot_not_performed()
def test_update_image_breaks_networking(self, install_image="core-image-full-cmdline-vexpress-qemu-broken-network.ext4"):
"""
Install an image without systemd-networkd binary existing.
The network will not function, mender will not be able to send any logs.
The expected status is the update will rollback, and be considered a failure
"""
if not env.host_string:
execute(self.test_update_image_breaks_networking,
hosts=get_mender_clients(),
install_image=install_image)
return
deployment_id, _ = common_update_proceduce(install_image)
Helpers.verify_reboot_performed() # since the network is broken, two reboots will be performed, and the last one will be detected
deploy.check_expected_status(deployment_id, "failure", len(get_mender_clients()))
def test_image_download_retry_1(self,
install_image=conftest.get_valid_image()):
"""
Install an update, and block storage connection when we detect it's
being copied over to the inactive parition.
The test should result in a successful download retry.
"""
if not env.host_string:
execute(self.test_image_download_retry_1,
hosts=get_mender_clients(),
install_image=install_image)
return
# make tcp timeout quicker, none persistent changes
run("echo 2 > /proc/sys/net/ipv4/tcp_keepalive_time")
run("echo 2 > /proc/sys/net/ipv4/tcp_keepalive_intvl")
run("echo 3 > /proc/sys/net/ipv4/tcp_syn_retries")
# to speed up timeouting client connection
run("echo 1 > /proc/sys/net/ipv4/tcp_keepalive_probes")
inactive_part = Helpers.get_passive_partition()
deployment_id, new_yocto_id = common_update_proceduce(install_image)
# use iptables to block traffic to storage when installing starts
for _ in range(60):
time.sleep(0.5)
with quiet():
# make sure we are writing to the inactive partition
output = run("fuser -mv %s" % (inactive_part))
if output.return_code == 0:
Helpers.gateway_connectivity(False,
hosts=["s3.docker.mender.io"
]) # disable connectivity
break
# re-enable connectivity after 2 retries
self.wait_for_download_retry_attempts()
Helpers.gateway_connectivity(True, hosts=["s3.docker.mender.io"
]) # re-enable connectivity
Helpers.verify_reboot_performed()
assert Helpers.get_active_partition() == inactive_part
assert Helpers.yocto_id_installed_on_machine() == new_yocto_id
Helpers.verify_reboot_not_performed()
def test_deployment_abortion_success(self):
# maybe an acceptance test is enough for this check?
if not env.host_string:
execute(self.test_deployment_abortion_success,
hosts=get_mender_clients())
return
install_image = conftest.get_valid_image()
deployment_id, _ = common_update_proceduce(install_image)
Helpers.verify_reboot_performed()
deploy.check_expected_statistics(deployment_id, "success", len(get_mender_clients()))
deploy.abort(deployment_id)
deploy.check_expected_statistics(deployment_id, "success", len(get_mender_clients()))
deploy.check_expected_status("finished", deployment_id)
def test_update_image_recovery(self,
install_image=conftest.get_valid_image()):
"""
Install an update, and reboot the system when we detect it's being copied over to the inactive parition.
The test should result in a failure.
"""
if not env.host_string:
execute(self.test_update_image_recovery,
hosts=get_mender_clients(),
install_image=install_image)
return
installed_yocto_id = Helpers.yocto_id_installed_on_machine()
inactive_part = Helpers.get_passive_partition()
deployment_id, _ = common_update_proceduce(install_image)
active_part = Helpers.get_active_partition()
for i in range(60):
time.sleep(0.5)
with quiet():
# make sure we are writing to the inactive partition
output = run("fuser -mv %s" % (inactive_part))
if output.return_code == 0:
run("killall -s 9 mender")
with settings(warn_only=True):
run("( sleep 3 ; reboot ) 2>/dev/null >/dev/null &")
break
logging.info("Waiting for system to finish reboot")
Helpers.verify_reboot_performed()
assert Helpers.get_active_partition() == active_part
deploy.check_expected_statistics(deployment_id, "failure",
len(get_mender_clients()))
Helpers.verify_reboot_not_performed()
assert Helpers.yocto_id_installed_on_machine() == installed_yocto_id
def abort_deployment(self, abort_step, mender_performs_reboot=False):
"""
Trigger a deployment, and cancel it within 15 seconds, make sure no deployment is performed.
Args:
mender_performs_reboot: if set to False, a manual reboot is performed and
checks are performed.
if set to True, wait until device is rebooted.
"""
if not env.host_string:
execute(self.abort_deployment,
abort_step=abort_step,
mender_performs_reboot=mender_performs_reboot,
hosts=get_mender_clients())
return
install_image=conftest.get_valid_image()
expected_partition = Helpers.get_active_partition()
expected_image_id = Helpers.yocto_id_installed_on_machine()
deployment_id, _ = common_update_proceduce(install_image, verify_status=False)
deploy.check_expected_statistics(deployment_id, abort_step, len(get_mender_clients()))
deploy.abort(deployment_id)
deploy.check_expected_statistics(deployment_id, "aborted", len(get_mender_clients()))
# no deployment logs are sent by the client, is this expected?
for d in adm.get_devices():
deploy.get_logs(d["device_id"], deployment_id, expected_status=404)
if not mender_performs_reboot:
Helpers.verify_reboot_not_performed()
run("( sleep 3 ; reboot ) 2>/dev/null >/dev/null &")
Helpers.verify_reboot_performed()
assert Helpers.get_active_partition() == expected_partition
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
deploy.check_expected_status("finished", deployment_id)