def test_multi_tenancy_deployment_aborting(self):
""" Simply make sure we are able to run the multi tenancy setup and
bootstrap 2 different devices to different tenants """
auth.reset_auth_token()
users = [
{
"email": "*****@*****.**",
"password": "******",
"username": "******",
"container": "mender-client-deployment-aborting-1",
}
]
for user in users:
auth.new_tenant(user["username"], user["email"], user["password"])
t = auth.current_tenant["tenant_token"]
new_tenant_client(user["container"], t)
adm.accept_devices(1)
for user in users:
deployment_id, _ = common_update_procedure(install_image=conftest.get_valid_image())
deploy.abort(deployment_id)
deploy.check_expected_statistics(deployment_id, "aborted", 1)
execute(self.mender_log_contains_aborted_string,
hosts=get_mender_client_by_container_name(user["container"]))
def abort_deployment(self, abort_step=None, mender_performs_reboot=False):
"""
Trigger a deployment, and cancel it within 15 seconds, make sure no deployment is performed.
Args:
mender_performs_reboot: if set to False, a manual reboot is performed and
checks are performed.
if set to True, wait until device is rebooted.
"""
if not env.host_string:
execute(self.abort_deployment,
abort_step=abort_step,
mender_performs_reboot=mender_performs_reboot,
hosts=get_mender_clients())
return
install_image = conftest.get_valid_image()
expected_partition = Helpers.get_active_partition()
expected_image_id = Helpers.yocto_id_installed_on_machine()
with Helpers.RebootDetector() as reboot:
deployment_id, _ = common_update_procedure(install_image,
verify_status=False)
if abort_step is not None:
deploy.check_expected_statistics(deployment_id, abort_step,
len(get_mender_clients()))
deploy.abort(deployment_id)
deploy.check_expected_statistics(deployment_id, "aborted",
len(get_mender_clients()))
# no deployment logs are sent by the client, is this expected?
for d in auth_v2.get_devices():
deploy.get_logs(d["id"], deployment_id, expected_status=404)
if mender_performs_reboot:
# If Mender performs reboot, we need to wait for it to reboot
# back into the original filesystem.
reboot.verify_reboot_performed(number_of_reboots=2)
else:
# Else we reboot ourselves, just to make sure that we have not
# unintentionally switched to the new partition.
reboot.verify_reboot_not_performed()
run("( sleep 10 ; reboot ) 2>/dev/null >/dev/null &")
reboot.verify_reboot_performed()
assert Helpers.get_active_partition() == expected_partition
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
deploy.check_expected_status("finished", deployment_id)
def test_deployment_abortion_success(self):
# maybe an acceptance test is enough for this check?
if not env.host_string:
execute(self.test_deployment_abortion_success,
hosts=get_mender_clients())
return
install_image = conftest.get_valid_image()
deployment_id, _ = common_update_proceduce(install_image)
Helpers.verify_reboot_performed()
deploy.check_expected_statistics(deployment_id, "success", len(get_mender_clients()))
deploy.abort(deployment_id)
deploy.check_expected_statistics(deployment_id, "success", len(get_mender_clients()))
deploy.check_expected_status("finished", deployment_id)
def abort_deployment(self, abort_step=None, mender_performs_reboot=False):
"""
Trigger a deployment, and cancel it within 15 seconds, make sure no deployment is performed.
Args:
mender_performs_reboot: if set to False, a manual reboot is performed and
checks are performed.
if set to True, wait until device is rebooted.
"""
if not env.host_string:
execute(self.abort_deployment,
abort_step=abort_step,
mender_performs_reboot=mender_performs_reboot,
hosts=get_mender_clients())
return
install_image = conftest.get_valid_image()
expected_partition = Helpers.get_active_partition()
expected_image_id = Helpers.yocto_id_installed_on_machine()
token = Helpers.place_reboot_token()
deployment_id, _ = common_update_procedure(install_image,
verify_status=False)
if abort_step is not None:
deploy.check_expected_statistics(deployment_id, abort_step,
len(get_mender_clients()))
deploy.abort(deployment_id)
deploy.check_expected_statistics(deployment_id, "aborted",
len(get_mender_clients()))
# no deployment logs are sent by the client, is this expected?
for d in adm.get_devices():
deploy.get_logs(d["device_id"], deployment_id, expected_status=404)
if not mender_performs_reboot:
token.verify_reboot_not_performed()
run("( sleep 10 ; reboot ) 2>/dev/null >/dev/null &")
token.verify_reboot_performed()
assert Helpers.get_active_partition() == expected_partition
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
deploy.check_expected_status("finished", deployment_id)
def test_state_scripts(self, description, test_set):
"""Test that state scripts are executed in right order, and that errors
are treated like they should."""
if not env.host_string:
execute(self.test_state_scripts,
description,
test_set,
hosts=get_mender_clients())
return
client = env.host_string
work_dir = "test_state_scripts.%s" % client
deployment_id = None
try:
script_content = '#!/bin/sh\n\necho "$(basename $0)" >> /data/test_state_scripts.log\n'
script_failure_content = script_content + "exit 1\n"
old_active = Helpers.get_active_partition()
# Make rootfs-scripts and put them in rootfs image.
rootfs_script_dir = os.path.join(work_dir, "rootfs-scripts")
shutil.rmtree(work_dir, ignore_errors=True)
os.mkdir(work_dir)
os.mkdir(rootfs_script_dir)
new_rootfs = os.path.join(work_dir, "rootfs.ext4")
shutil.copy(conftest.get_valid_image(), new_rootfs)
ps = subprocess.Popen(["debugfs", "-w", new_rootfs],
stdin=subprocess.PIPE)
ps.stdin.write("cd /etc/mender\n" "mkdir scripts\n" "cd scripts\n")
with open(os.path.join(rootfs_script_dir, "version"), "w") as fd:
if test_set.get('CorruptEtcScriptVersionInUpdate'):
fd.write("1000")
else:
fd.write("2")
ps.stdin.write("rm version\n")
ps.stdin.write("write %s version\n" %
os.path.join(rootfs_script_dir, "version"))
for script in self.scripts:
if script.startswith("Artifact"):
# This is a script for the artifact, skip this one.
continue
with open(os.path.join(rootfs_script_dir, script), "w") as fd:
if script in test_set['FailureScript']:
fd.write(script_failure_content)
else:
fd.write(script_content)
os.fchmod(fd.fileno(), 0755)
ps.stdin.write(
"write %s %s\n" %
(os.path.join(rootfs_script_dir, script), script))
ps.stdin.close()
ps.wait()
# Write this again in case it was corrupted above.
with open(os.path.join(rootfs_script_dir, "version"), "w") as fd:
fd.write("2")
# Then copy them to QEMU host.
# Zip them all up to avoid having to copy each and every file, which is
# quite slow.
subprocess.check_call(
["tar", "czf", "../rootfs-scripts.tar.gz", "."],
cwd=rootfs_script_dir)
# Stop client first to avoid race conditions.
run("systemctl stop mender")
try:
put(os.path.join(work_dir, "rootfs-scripts.tar.gz"),
remote_path="/")
run("mkdir -p cd /etc/mender/scripts && " +
"cd /etc/mender/scripts && " +
"tar xzf /rootfs-scripts.tar.gz && " +
"rm -f /rootfs-scripts.tar.gz")
finally:
run("systemctl start mender")
# Put artifact-scripts in the artifact.
artifact_script_dir = os.path.join(work_dir, "artifact-scripts")
os.mkdir(artifact_script_dir)
for script in self.scripts:
if not script.startswith("Artifact"):
# Not an artifact script, skip this one.
continue
with open(os.path.join(artifact_script_dir, script),
"w") as fd:
if script in test_set['FailureScript']:
fd.write(script_failure_content)
else:
fd.write(script_content)
if test_set.get("SimulateBootFailureIn") == script:
# Simulate that boot failed by immediately forcing a
# rollback with U-Boot.
fd.write("fw_setenv bootcount 1\n")
if test_set.get("CorruptDataScriptVersionIn") == script:
fd.write(
"printf '1000' > /data/mender/scripts/version\n")
# Now create the artifact, and make the deployment.
device_id = Helpers.ip_to_device_id_map([client])[client]
broken_artifact_id = test_set.get('BrokenArtifactId')
if broken_artifact_id is None:
broken_artifact_id = False
deployment_id = common_update_procedure(
install_image=new_rootfs,
broken_image=broken_artifact_id,
verify_status=False,
devices=[device_id],
scripts=[artifact_script_dir])[0]
if test_set['ExpectedStatus'] is None:
# In this case we don't expect the deployment to even be
# attempted, presumably due to failing Idle/Sync/Download
# scripts on the client. So no deployment checking. Just wait
# until there is at least one Error script in the log, which
# will always be the case if ExpectedStatus is none (since one
# of them is preventing the update from being attempted).
def fetch_info(cmd_list):
all_output = ""
for cmd in cmd_list:
with settings(warn_only=True):
output = run(cmd)
logger.error("%s:\n%s" % (cmd, output))
all_output += "%s\n" % output
return all_output
info_query = [
"cat /data/test_state_scripts.log 1>&2",
"journalctl -u mender",
"top -n5 -b",
"ls -l /proc/`pgrep mender`/fd",
"for fd in /proc/`pgrep mender`/fdinfo/*; do echo $fd:; cat $fd; done",
]
starttime = time.time()
while starttime + 60 * 60 >= time.time():
with settings(warn_only=True):
result = run("grep Error /data/test_state_scripts.log")
if result.succeeded:
# If it succeeds, stop.
break
else:
fetch_info(info_query)
time.sleep(10)
continue
else:
info = fetch_info(info_query)
pytest.fail(
'Waited too long for "Error" to appear in log:\n%s' %
info)
else:
deploy.check_expected_statistics(deployment_id,
test_set['ExpectedStatus'], 1)
# Always give the client a little bit of time to settle in the base
# state after an update.
time.sleep(10)
output = run("cat /data/test_state_scripts.log")
self.verify_script_log_correct(test_set, output.split('\n'))
new_active = Helpers.get_active_partition()
should_switch_partition = (test_set['ExpectedStatus'] == "success")
# TODO
if test_set.get('SwapPartitionExpectation') is not None:
should_switch_partition = not should_switch_partition
if should_switch_partition:
assert old_active != new_active, "Device did not switch partition as expected!"
else:
assert old_active == new_active, "Device switched partition which was not expected!"
finally:
shutil.rmtree(work_dir, ignore_errors=True)
if deployment_id:
try:
deploy.abort(deployment_id)
except:
pass
run("systemctl stop mender && " +
"rm -f /data/test_state_scripts.log && " +
"rm -rf /etc/mender/scripts && " +
"rm -rf /data/mender/scripts && " + "systemctl start mender")