def update_image_successful(install_image,
regenerate_image_id=True,
signed=False,
skip_reboot_verification=False,
expected_mender_clients=1,
pre_upload_callback=lambda: None,
pre_deployment_callback=lambda: None,
deployment_triggered_callback=lambda: None,
compression_type="gzip"):
"""
Perform a successful upgrade, and assert that deployment status/logs are correct.
A reboot is performed, and running partitions have been swapped.
Deployment status will be set as successful for device.
Logs will not be retrieved, and result in 404.
"""
previous_inactive_part = Helpers.get_passive_partition()
with Helpers.RebootDetector() as reboot:
deployment_id, expected_image_id = common_update_procedure(
install_image,
regenerate_image_id,
signed=signed,
pre_deployment_callback=pre_deployment_callback,
deployment_triggered_callback=deployment_triggered_callback,
compression_type=compression_type)
reboot.verify_reboot_performed()
with Helpers.RebootDetector() as reboot:
try:
assert Helpers.get_active_partition() == previous_inactive_part
except AssertionError:
logs = []
for d in auth_v2.get_devices():
logs.append(deploy.get_logs(d["id"], deployment_id))
pytest.fail(
"device did not flip partitions during update, here are the device logs:\n\n %s"
% (logs))
deploy.check_expected_statistics(deployment_id, "success",
expected_mender_clients)
for d in auth_v2.get_devices():
deploy.get_logs(d["id"], deployment_id, expected_status=404)
if not skip_reboot_verification:
reboot.verify_reboot_not_performed()
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
deploy.check_expected_status("finished", deployment_id)
# make sure backend recognizes signed and unsigned images
artifact_id = deploy.get_deployment(deployment_id)["artifacts"][0]
artifact_info = deploy.get_artifact_details(artifact_id)
assert artifact_info[
"signed"] is signed, "image was not correct recognized as signed/unsigned"
return deployment_id
def test_clients_exclusive_to_user(self):
users = [
{
"email": "*****@*****.**",
"password": "******",
"username": "******",
"container": "mender-client-exclusive-1",
"client_id": "",
"device_id": ""
},
{
"email": "*****@*****.**",
"password": "******",
"username": "******",
"container": "mender-client-exclusive-2",
"client_id": "",
"device_id": ""
}
]
for user in users:
auth.set_tenant(user["username"], user["email"], user["password"])
t = auth.current_tenant["tenant_token"]
new_tenant_client(user["container"], t)
auth_v2.accept_devices(1)
# get the new devices client_id and setting it to the our test parameter
assert len(inv.get_devices()) == 1
user["client_id"] = inv.get_devices()[0]["id"]
user["device_id"] = auth_v2.get_devices()[0]["id"]
for user in users:
# make sure that the correct number of clients appear for the given tenant
auth.set_tenant(user["username"], user["email"], user["password"])
assert len(inv.get_devices()) == 1
assert inv.get_devices()[0]["id"] == user["client_id"]
for user in users:
# wait until inventory is populated
auth.set_tenant(user["username"], user["email"], user["password"])
auth_v2.decommission(user["client_id"])
timeout = time.time() + (60 * 5)
device_id = user["device_id"]
while time.time() < timeout:
newAdmissions = auth_v2.get_devices()[0]
if device_id != newAdmissions["id"] \
and user["client_id"] != newAdmissions["id"]:
logger.info("device [%s] not found in inventory [%s]" % (device_id, str(newAdmissions)))
break
else:
logger.info("device [%s] found in inventory..." % (device_id))
time.sleep(.5)
else:
assert False, "decommissioned device still available in inventory"
def update_image_failed(install_image="broken_update.ext4",
expected_mender_clients=1):
"""
Perform a upgrade using a broken image (random data)
The device will reboot, uboot will detect this is not a bootable image, and revert to the previous partition.
The resulting upgrade will be considered a failure.
"""
devices_accepted = get_mender_clients()
original_image_id = Helpers.yocto_id_installed_on_machine()
previous_active_part = Helpers.get_active_partition()
with Helpers.RebootDetector() as reboot:
deployment_id, _ = common_update_procedure(install_image,
broken_image=True)
reboot.verify_reboot_performed()
with Helpers.RebootDetector() as reboot:
assert Helpers.get_active_partition() == previous_active_part
deploy.check_expected_statistics(deployment_id, "failure",
expected_mender_clients)
for d in auth_v2.get_devices():
assert "got invalid entrypoint into the state machine" in deploy.get_logs(
d["id"], deployment_id)
assert Helpers.yocto_id_installed_on_machine() == original_image_id
reboot.verify_reboot_not_performed()
deploy.check_expected_status("finished", deployment_id)
def test_token_validity(self):
""" verify that only devices with valid tokens can bootstrap
successfully to a multitenancy setup """
wrong_token = "wrong-token"
def wait_until_bootstrap_attempt():
if not env.host_string:
return execute(wait_until_bootstrap_attempt,
hosts=get_mender_clients())
ssh_is_opened()
for i in range(1, 20):
with settings(hide('everything'), warn_only=True):
out = run('journalctl -u mender | grep "bootstrapped -> authorize-wait"')
if out.succeeded:
return True
time.sleep(20/i)
return False
def set_correct_tenant_token(token):
if not env.host_string:
return execute(set_correct_tenant_token,
token,
hosts=get_mender_clients())
run("sed -i 's/%s/%s/g' /etc/mender/mender.conf" % (wrong_token, token))
run("systemctl restart mender")
auth.reset_auth_token()
auth.new_tenant("admin", "*****@*****.**", "hunter2hunter2")
token = auth.current_tenant["tenant_token"]
# create a new client with an incorrect token set
new_tenant_client("mender-client", wrong_token)
if wait_until_bootstrap_attempt():
for _ in range(5):
time.sleep(5)
auth_v2.get_devices(expected_devices=0) # make sure device not seen
else:
pytest.fail("failed to bootstrap device")
# setting the correct token makes the client visible to the backend
set_correct_tenant_token(token)
auth_v2.get_devices(expected_devices=1)
def test_device_decommissioning(self):
""" Decommission a device successfully """
if not env.host_string:
execute(self.test_device_decommissioning,
hosts=get_mender_clients())
return
auth_v2.check_expected_status("pending", len(get_mender_clients()))
device = auth_v2.get_devices()[0]
device_id = device["id"]
auth_v2.set_device_auth_set_status(device_id,
device["auth_sets"][0]["id"],
"accepted")
# wait until inventory is populated
timeout = time.time() + (60 * 5)
while time.time() < timeout:
inventoryJSON = inv.get_devices()
# we haven't gotten an inventory data yet.
if len(inventoryJSON) == 0:
continue
if "attributes" in inventoryJSON[0]:
break
time.sleep(.5)
else:
assert False, "never got inventory"
# get all completed decommission_device WFs for reference
c = Conductor(get_mender_conductor())
initial_wfs = c.get_decommission_device_wfs(device_id)
# decommission actual device
auth_v2.decommission(device_id)
# check that the workflow completed successfully
timeout = time.time() + (60 * 5)
while time.time() < timeout:
wfs = c.get_decommission_device_wfs(device_id)
if wfs['totalHits'] == initial_wfs['totalHits'] + 1:
break
else:
logger.info("waiting for decommission_device workflow...")
time.sleep(.5)
else:
assert False, "decommission_device workflow didn't complete for [%s]" % (
device_id, )
# check device gone from inventory
self.check_gone_from_inventory(device_id)
# check device gone from deviceauth
self.check_gone_from_deviceauth(device_id)
def do_test_ok_preauth_and_remove(self):
"""
Test the removal of a preauthorized auth set, verify it's gone from all API results.
"""
# preauthorize
preauth_iddata = json.loads("{\"mac\":\"preauth-mac\"}")
preauth_key = '''-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzogVU7RGDilbsoUt/DdH
VJvcepl0A5+xzGQ50cq1VE/Dyyy8Zp0jzRXCnnu9nu395mAFSZGotZVr+sWEpO3c
yC3VmXdBZmXmQdZqbdD/GuixJOYfqta2ytbIUPRXFN7/I7sgzxnXWBYXYmObYvdP
okP0mQanY+WKxp7Q16pt1RoqoAd0kmV39g13rFl35muSHbSBoAW3GBF3gO+mF5Ty
1ddp/XcgLOsmvNNjY+2HOD5F/RX0fs07mWnbD7x+xz7KEKjF+H7ZpkqCwmwCXaf0
iyYyh1852rti3Afw4mDxuVSD7sd9ggvYMc0QHIpQNkD4YWOhNiE1AB0zH57VbUYG
UwIDAQAB
-----END PUBLIC KEY-----
'''
r = auth_v2.preauth(preauth_iddata, preauth_key)
assert r.status_code == 201
devs = auth_v2.get_devices(2)
dev_preauth = [d for d in devs if d['identity_data'] == preauth_iddata]
assert len(dev_preauth) == 1
dev_preauth = dev_preauth[0]
# remove from deviceauth
r = auth_v2.delete_auth_set(dev_preauth['id'],
dev_preauth["auth_sets"][0]["id"])
assert r.status_code == 204
# verify removed from deviceauth
devs = auth_v2.get_devices(1)
dev_removed = [d for d in devs if d['identity_data'] == preauth_iddata]
assert len(dev_removed) == 0
# verify removed from deviceauth
r = auth_v2.get_device(dev_preauth['id'])
assert r.status_code == 404
# verify removed from inventory
r = inv.get_device(dev_preauth['id'])
assert r.status_code == 404
def abort_deployment(self, abort_step=None, mender_performs_reboot=False):
"""
Trigger a deployment, and cancel it within 15 seconds, make sure no deployment is performed.
Args:
mender_performs_reboot: if set to False, a manual reboot is performed and
checks are performed.
if set to True, wait until device is rebooted.
"""
if not env.host_string:
execute(self.abort_deployment,
abort_step=abort_step,
mender_performs_reboot=mender_performs_reboot,
hosts=get_mender_clients())
return
install_image = conftest.get_valid_image()
expected_partition = Helpers.get_active_partition()
expected_image_id = Helpers.yocto_id_installed_on_machine()
with Helpers.RebootDetector() as reboot:
deployment_id, _ = common_update_procedure(install_image,
verify_status=False)
if abort_step is not None:
deploy.check_expected_statistics(deployment_id, abort_step,
len(get_mender_clients()))
deploy.abort(deployment_id)
deploy.check_expected_statistics(deployment_id, "aborted",
len(get_mender_clients()))
# no deployment logs are sent by the client, is this expected?
for d in auth_v2.get_devices():
deploy.get_logs(d["id"], deployment_id, expected_status=404)
if mender_performs_reboot:
# If Mender performs reboot, we need to wait for it to reboot
# back into the original filesystem.
reboot.verify_reboot_performed(number_of_reboots=2)
else:
# Else we reboot ourselves, just to make sure that we have not
# unintentionally switched to the new partition.
reboot.verify_reboot_not_performed()
run("( sleep 10 ; reboot ) 2>/dev/null >/dev/null &")
reboot.verify_reboot_performed()
assert Helpers.get_active_partition() == expected_partition
assert Helpers.yocto_id_installed_on_machine() == expected_image_id
deploy.check_expected_status("finished", deployment_id)
def accept_devices(self):
auth_v2.check_expected_status("pending", len(get_mender_clients()))
# iterate over devices and accept them
for d in auth_v2.get_devices():
auth_v2.set_device_auth_set_status(d["id"], d["auth_sets"][0]["id"], "accepted")
logging.info("Accepting DeviceID: %s" % d["id"])
# make sure all devices are accepted
auth_v2.check_expected_status("accepted", len(get_mender_clients()))
# make sure mender-store contains authtoken
have_token()
# print all device ids
for device in auth_v2.get_devices_status("accepted"):
logging.info("Accepted DeviceID: %s" % device["id"])
def ip_to_device_id_map(clients):
# Get deviceauth data, which includes device identity.
devauth_devices = auth_v2.get_devices(expected_devices=len(clients))
# Collect identity of each client.
ret = execute(run, "/usr/share/mender/identity/mender-device-identity", hosts=clients)
# Calculate device identities.
identity_to_ip = {}
for client in clients:
identity_to_ip[Helpers.identity_script_to_identity_string(ret[client])] = client
# Match them.
ip_to_device_id = {}
for device in devauth_devices:
ip_to_device_id[identity_to_ip[json.dumps(device['identity_data'], separators=(",", ":"))]] = device['id']
return ip_to_device_id
def test_reject_bootstrap(self):
"""Make sure a rejected device does not perform an upgrade, and that it gets it's auth token removed"""
if not env.host_string:
execute(self.test_reject_bootstrap, hosts=get_mender_clients())
return
# iterate over devices and reject them
for device in auth_v2.get_devices():
auth_v2.set_device_auth_set_status(device["id"], device["auth_sets"][0]["id"], "rejected")
logging.info("Rejecting DeviceID: %s" % device["id"])
auth_v2.check_expected_status("rejected", len(get_mender_clients()))
with Helpers.RebootDetector() as reboot:
try:
deployment_id, _ = common_update_procedure(install_image=conftest.get_valid_image())
except AssertionError:
logging.info("Failed to deploy upgrade to rejected device.")
reboot.verify_reboot_not_performed()
else:
# use assert to fail, so we can get backend logs
pytest.fail("no error while trying to deploy to rejected device")
return
finished = False
# wait until auththoken is removed from file
for _ in range(10):
with settings(abort_exception=Exception):
try:
run("journalctl -u mender -l -n 3 | grep -q 'authentication request rejected'")
except:
time.sleep(30)
else:
finished = True
break
auth_v2.accept_devices(1)
if not finished:
pytest.fail("failed to remove authtoken from mender-store file")
def do_test_fail_preauth_existing(self):
"""
Test 'conflict' response when an identity data set already exists.
"""
# wait for the device to appear
devs = auth_v2.get_devices(1)
dev = devs[0]
# try to preauthorize the same id data, new key
preauth_key = '''-----BEGIN PUBLIC KEY-----
MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzogVU7RGDilbsoUt/DdH
VJvcepl0A5+xzGQ50cq1VE/Dyyy8Zp0jzRXCnnu9nu395mAFSZGotZVr+sWEpO3c
yC3VmXdBZmXmQdZqbdD/GuixJOYfqta2ytbIUPRXFN7/I7sgzxnXWBYXYmObYvdP
okP0mQanY+WKxp7Q16pt1RoqoAd0kmV39g13rFl35muSHbSBoAW3GBF3gO+mF5Ty
1ddp/XcgLOsmvNNjY+2HOD5F/RX0fs07mWnbD7x+xz7KEKjF+H7ZpkqCwmwCXaf0
iyYyh1852rti3Afw4mDxuVSD7sd9ggvYMc0QHIpQNkD4YWOhNiE1AB0zH57VbUYG
UwIDAQAB
-----END PUBLIC KEY-----
'''
r = auth_v2.preauth(dev['identity_data'], preauth_key)
assert r.status_code == 409
def test_unsigned_artifact_fails_deployment(
self, standard_setup_with_signed_artifact_client):
"""
Make sure that an unsigned image fails, and is handled by the backend.
Notice that this test needs a fresh new version of the backend, since
we installed a signed image earlier without a verification key in mender.conf
"""
if not env.host_string:
execute(self.test_unsigned_artifact_fails_deployment,
standard_setup_with_signed_artifact_client,
hosts=get_mender_clients())
return
deployment_id, _ = common_update_procedure(
install_image=conftest.get_valid_image())
deploy.check_expected_status("finished", deployment_id)
deploy.check_expected_statistics(deployment_id, "failure", 1)
for d in auth_v2.get_devices():
assert "expecting signed artifact, but no signature file found" in \
deploy.get_logs(d["id"], deployment_id)
def do_test_ok_preauth_and_bootstrap(self):
"""
Test the happy path from preauthorizing a device to a successful bootstrap.
Verify that the device/auth set appear correctly in devauth API results.
"""
client = get_mender_clients()[0]
# we'll use the same pub key for the preauth'd device, so get it
res = execute(Client.get_pub_key, hosts=client)
preauth_key = res[client].exportKey()
# stick an extra newline on the key - this is how a device would send it
preauth_key += '\n'
# preauthorize a new device
preauth_iddata = {"mac": "mac-preauth"}
# serialize manually to avoid an extra space (id data helper doesn't insert one)
preauth_iddata_str = "{\"mac\":\"mac-preauth\"}"
r = auth_v2.preauth(json.loads(preauth_iddata_str), preauth_key)
assert r.status_code == 201
# verify the device appears correctly in api results
devs = auth_v2.get_devices(2)
dev_preauth = [d for d in devs if d['status'] == 'preauthorized']
assert len(dev_preauth) == 1
dev_preauth = dev_preauth[0]
assert dev_preauth['identity_data'] == preauth_iddata_str
assert dev_preauth['pubkey'] == preauth_key
# make one of the existing devices the preauthorized device
# by substituting id data and restarting
res = execute(Client.substitute_id_data, preauth_iddata, hosts=client)
res = execute(Client.restart, hosts=client)
# verify api results - after some time the device should be 'accepted'
for _ in range(120):
time.sleep(15)
dev_accepted = auth_v2.get_devices_status(status="accepted",
expected_devices=2)
if len([d for d in dev_accepted
if d['status'] == 'accepted']) == 1:
break
logging.info("devices: " + str(dev_accepted))
dev_accepted = [d for d in dev_accepted if d['status'] == 'accepted']
logging.info("accepted devices: " + str(dev_accepted))
execute(Client.get_logs, hosts=client)
assert len(
dev_accepted) == 1, "looks like the device was never accepted"
dev_accepted = dev_accepted[0]
logging.info("accepted device: " + str(dev_accepted))
assert dev_accepted['identity_data'] == preauth_iddata_str
assert dev_accepted['pubkey'] == preauth_key
# verify device was issued a token
res = execute(Client.have_authtoken, hosts=client)
assert res[client]
# start docker-compose
ret = subprocess.call([
"docker-compose", "-p", conftest.docker_compose_instance, "-f",
"docker-compose.yml", "-f", "docker-compose.storage.minio.yml", "-f",
"./production-testing-env.yml", "up", "-d"
],
cwd="../")
assert ret == 0, "failed to start docker-compose"
if args.deploy:
# create account for management api
auth.get_auth_token()
# wait for 10 devices to be available
devices = auth_v2.get_devices(10)
assert len(devices) == 10
# accept all devices
for d in devices:
auth_v2.set_device_auth_set_status(d["id"], d["auth_sets"][0]["id"],
"accepted")
# make sure artifact tool in current workdir is being used
os.environ["PATH"] = os.path.dirname(os.path.realpath(
__file__)) + "/downloaded-tools" + os.pathsep + os.environ["PATH"]
# perform upgrade
devices_to_update = list(
set([
device["id"]
def test_inventory(self):
"""Test that device reports inventory after having bootstrapped."""
attempts = 10
while True:
attempts = attempts - 1
try:
inv_json = inv.get_devices()
auth_json = auth_v2.get_devices()
auth_ids = [device['id'] for device in auth_json]
assert (len(inv_json) > 0)
for device in inv_json:
try:
# Check that authentication and inventory agree.
assert (device['id'] in auth_ids)
attrs = device['attributes']
# Check individual attributes.
network_interfaces = [
elem for elem in attrs
if elem['name'] == "network_interfaces"
]
assert len(network_interfaces) == 1
network_interfaces = network_interfaces[0]
if type(network_interfaces['value']) is str:
assert any(network_interfaces['value'] == iface
for iface in ["eth0", "enp0s3"])
else:
assert any(iface in network_interfaces['value']
for iface in ["eth0", "enp0s3"])
assert (json.loads(
'{"name": "hostname", "value": "%s"}' %
conftest.machine_name) in attrs)
assert (json.loads(
'{"name": "device_type", "value": "%s"}' %
conftest.machine_name) in attrs)
if conftest.machine_name == "qemux86-64":
bootloader_integration = "uefi_grub"
elif conftest.machine_name == "vexpress-qemu":
bootloader_integration = "uboot"
else:
pytest.fail(
"Unknown machine_name. Please add an expected bootloader_integration for this machine_name"
)
assert (json.loads(
'{"name": "mender_bootloader_integration", "value": "%s"}'
% bootloader_integration) in attrs)
# Check that all known keys are present.
keys = [str(attr['name']) for attr in attrs]
expected_keys = [
"hostname",
"network_interfaces",
"cpu_model",
"mem_total_kB",
"device_type",
["ipv4_enp0s3", "ipv6_enp0s3",
"ipv4_eth0"], # Multiple possibilities
["mac_enp0s3", "mac_eth0"],
"mender_client_version",
"artifact_name",
"kernel",
"os",
]
for key in expected_keys:
if type(key) is list:
assert any([subkey in keys for subkey in key])
else:
assert key in keys
except:
print("Exception caught, 'device' json: ", device)
raise
break
except:
# This may pass only after the client has had some time to
# report.
if attempts > 0:
time.sleep(5)
continue
else:
raise
def test_decommissioning_post_upgrade(self):
# assertion error occurs here on decommissioning fail
for device in auth_v2.get_devices(10):
auth_v2.decommission(device["id"])