def update_image_successful(install_image, regenerate_image_id=True, signed=False, skip_reboot_verification=False, expected_mender_clients=1, pre_upload_callback=lambda: None, pre_deployment_callback=lambda: None, deployment_triggered_callback=lambda: None, compression_type="gzip"): """ Perform a successful upgrade, and assert that deployment status/logs are correct. A reboot is performed, and running partitions have been swapped. Deployment status will be set as successful for device. Logs will not be retrieved, and result in 404. """ previous_inactive_part = Helpers.get_passive_partition() with Helpers.RebootDetector() as reboot: deployment_id, expected_image_id = common_update_procedure( install_image, regenerate_image_id, signed=signed, pre_deployment_callback=pre_deployment_callback, deployment_triggered_callback=deployment_triggered_callback, compression_type=compression_type) reboot.verify_reboot_performed() with Helpers.RebootDetector() as reboot: try: assert Helpers.get_active_partition() == previous_inactive_part except AssertionError: logs = [] for d in auth_v2.get_devices(): logs.append(deploy.get_logs(d["id"], deployment_id)) pytest.fail( "device did not flip partitions during update, here are the device logs:\n\n %s" % (logs)) deploy.check_expected_statistics(deployment_id, "success", expected_mender_clients) for d in auth_v2.get_devices(): deploy.get_logs(d["id"], deployment_id, expected_status=404) if not skip_reboot_verification: reboot.verify_reboot_not_performed() assert Helpers.yocto_id_installed_on_machine() == expected_image_id deploy.check_expected_status("finished", deployment_id) # make sure backend recognizes signed and unsigned images artifact_id = deploy.get_deployment(deployment_id)["artifacts"][0] artifact_info = deploy.get_artifact_details(artifact_id) assert artifact_info[ "signed"] is signed, "image was not correct recognized as signed/unsigned" return deployment_id
def test_clients_exclusive_to_user(self): users = [ { "email": "*****@*****.**", "password": "******", "username": "******", "container": "mender-client-exclusive-1", "client_id": "", "device_id": "" }, { "email": "*****@*****.**", "password": "******", "username": "******", "container": "mender-client-exclusive-2", "client_id": "", "device_id": "" } ] for user in users: auth.set_tenant(user["username"], user["email"], user["password"]) t = auth.current_tenant["tenant_token"] new_tenant_client(user["container"], t) auth_v2.accept_devices(1) # get the new devices client_id and setting it to the our test parameter assert len(inv.get_devices()) == 1 user["client_id"] = inv.get_devices()[0]["id"] user["device_id"] = auth_v2.get_devices()[0]["id"] for user in users: # make sure that the correct number of clients appear for the given tenant auth.set_tenant(user["username"], user["email"], user["password"]) assert len(inv.get_devices()) == 1 assert inv.get_devices()[0]["id"] == user["client_id"] for user in users: # wait until inventory is populated auth.set_tenant(user["username"], user["email"], user["password"]) auth_v2.decommission(user["client_id"]) timeout = time.time() + (60 * 5) device_id = user["device_id"] while time.time() < timeout: newAdmissions = auth_v2.get_devices()[0] if device_id != newAdmissions["id"] \ and user["client_id"] != newAdmissions["id"]: logger.info("device [%s] not found in inventory [%s]" % (device_id, str(newAdmissions))) break else: logger.info("device [%s] found in inventory..." % (device_id)) time.sleep(.5) else: assert False, "decommissioned device still available in inventory"
def update_image_failed(install_image="broken_update.ext4", expected_mender_clients=1): """ Perform a upgrade using a broken image (random data) The device will reboot, uboot will detect this is not a bootable image, and revert to the previous partition. The resulting upgrade will be considered a failure. """ devices_accepted = get_mender_clients() original_image_id = Helpers.yocto_id_installed_on_machine() previous_active_part = Helpers.get_active_partition() with Helpers.RebootDetector() as reboot: deployment_id, _ = common_update_procedure(install_image, broken_image=True) reboot.verify_reboot_performed() with Helpers.RebootDetector() as reboot: assert Helpers.get_active_partition() == previous_active_part deploy.check_expected_statistics(deployment_id, "failure", expected_mender_clients) for d in auth_v2.get_devices(): assert "got invalid entrypoint into the state machine" in deploy.get_logs( d["id"], deployment_id) assert Helpers.yocto_id_installed_on_machine() == original_image_id reboot.verify_reboot_not_performed() deploy.check_expected_status("finished", deployment_id)
def test_token_validity(self): """ verify that only devices with valid tokens can bootstrap successfully to a multitenancy setup """ wrong_token = "wrong-token" def wait_until_bootstrap_attempt(): if not env.host_string: return execute(wait_until_bootstrap_attempt, hosts=get_mender_clients()) ssh_is_opened() for i in range(1, 20): with settings(hide('everything'), warn_only=True): out = run('journalctl -u mender | grep "bootstrapped -> authorize-wait"') if out.succeeded: return True time.sleep(20/i) return False def set_correct_tenant_token(token): if not env.host_string: return execute(set_correct_tenant_token, token, hosts=get_mender_clients()) run("sed -i 's/%s/%s/g' /etc/mender/mender.conf" % (wrong_token, token)) run("systemctl restart mender") auth.reset_auth_token() auth.new_tenant("admin", "*****@*****.**", "hunter2hunter2") token = auth.current_tenant["tenant_token"] # create a new client with an incorrect token set new_tenant_client("mender-client", wrong_token) if wait_until_bootstrap_attempt(): for _ in range(5): time.sleep(5) auth_v2.get_devices(expected_devices=0) # make sure device not seen else: pytest.fail("failed to bootstrap device") # setting the correct token makes the client visible to the backend set_correct_tenant_token(token) auth_v2.get_devices(expected_devices=1)
def test_device_decommissioning(self): """ Decommission a device successfully """ if not env.host_string: execute(self.test_device_decommissioning, hosts=get_mender_clients()) return auth_v2.check_expected_status("pending", len(get_mender_clients())) device = auth_v2.get_devices()[0] device_id = device["id"] auth_v2.set_device_auth_set_status(device_id, device["auth_sets"][0]["id"], "accepted") # wait until inventory is populated timeout = time.time() + (60 * 5) while time.time() < timeout: inventoryJSON = inv.get_devices() # we haven't gotten an inventory data yet. if len(inventoryJSON) == 0: continue if "attributes" in inventoryJSON[0]: break time.sleep(.5) else: assert False, "never got inventory" # get all completed decommission_device WFs for reference c = Conductor(get_mender_conductor()) initial_wfs = c.get_decommission_device_wfs(device_id) # decommission actual device auth_v2.decommission(device_id) # check that the workflow completed successfully timeout = time.time() + (60 * 5) while time.time() < timeout: wfs = c.get_decommission_device_wfs(device_id) if wfs['totalHits'] == initial_wfs['totalHits'] + 1: break else: logger.info("waiting for decommission_device workflow...") time.sleep(.5) else: assert False, "decommission_device workflow didn't complete for [%s]" % ( device_id, ) # check device gone from inventory self.check_gone_from_inventory(device_id) # check device gone from deviceauth self.check_gone_from_deviceauth(device_id)
def do_test_ok_preauth_and_remove(self): """ Test the removal of a preauthorized auth set, verify it's gone from all API results. """ # preauthorize preauth_iddata = json.loads("{\"mac\":\"preauth-mac\"}") preauth_key = '''-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzogVU7RGDilbsoUt/DdH VJvcepl0A5+xzGQ50cq1VE/Dyyy8Zp0jzRXCnnu9nu395mAFSZGotZVr+sWEpO3c yC3VmXdBZmXmQdZqbdD/GuixJOYfqta2ytbIUPRXFN7/I7sgzxnXWBYXYmObYvdP okP0mQanY+WKxp7Q16pt1RoqoAd0kmV39g13rFl35muSHbSBoAW3GBF3gO+mF5Ty 1ddp/XcgLOsmvNNjY+2HOD5F/RX0fs07mWnbD7x+xz7KEKjF+H7ZpkqCwmwCXaf0 iyYyh1852rti3Afw4mDxuVSD7sd9ggvYMc0QHIpQNkD4YWOhNiE1AB0zH57VbUYG UwIDAQAB -----END PUBLIC KEY----- ''' r = auth_v2.preauth(preauth_iddata, preauth_key) assert r.status_code == 201 devs = auth_v2.get_devices(2) dev_preauth = [d for d in devs if d['identity_data'] == preauth_iddata] assert len(dev_preauth) == 1 dev_preauth = dev_preauth[0] # remove from deviceauth r = auth_v2.delete_auth_set(dev_preauth['id'], dev_preauth["auth_sets"][0]["id"]) assert r.status_code == 204 # verify removed from deviceauth devs = auth_v2.get_devices(1) dev_removed = [d for d in devs if d['identity_data'] == preauth_iddata] assert len(dev_removed) == 0 # verify removed from deviceauth r = auth_v2.get_device(dev_preauth['id']) assert r.status_code == 404 # verify removed from inventory r = inv.get_device(dev_preauth['id']) assert r.status_code == 404
def abort_deployment(self, abort_step=None, mender_performs_reboot=False): """ Trigger a deployment, and cancel it within 15 seconds, make sure no deployment is performed. Args: mender_performs_reboot: if set to False, a manual reboot is performed and checks are performed. if set to True, wait until device is rebooted. """ if not env.host_string: execute(self.abort_deployment, abort_step=abort_step, mender_performs_reboot=mender_performs_reboot, hosts=get_mender_clients()) return install_image = conftest.get_valid_image() expected_partition = Helpers.get_active_partition() expected_image_id = Helpers.yocto_id_installed_on_machine() with Helpers.RebootDetector() as reboot: deployment_id, _ = common_update_procedure(install_image, verify_status=False) if abort_step is not None: deploy.check_expected_statistics(deployment_id, abort_step, len(get_mender_clients())) deploy.abort(deployment_id) deploy.check_expected_statistics(deployment_id, "aborted", len(get_mender_clients())) # no deployment logs are sent by the client, is this expected? for d in auth_v2.get_devices(): deploy.get_logs(d["id"], deployment_id, expected_status=404) if mender_performs_reboot: # If Mender performs reboot, we need to wait for it to reboot # back into the original filesystem. reboot.verify_reboot_performed(number_of_reboots=2) else: # Else we reboot ourselves, just to make sure that we have not # unintentionally switched to the new partition. reboot.verify_reboot_not_performed() run("( sleep 10 ; reboot ) 2>/dev/null >/dev/null &") reboot.verify_reboot_performed() assert Helpers.get_active_partition() == expected_partition assert Helpers.yocto_id_installed_on_machine() == expected_image_id deploy.check_expected_status("finished", deployment_id)
def accept_devices(self): auth_v2.check_expected_status("pending", len(get_mender_clients())) # iterate over devices and accept them for d in auth_v2.get_devices(): auth_v2.set_device_auth_set_status(d["id"], d["auth_sets"][0]["id"], "accepted") logging.info("Accepting DeviceID: %s" % d["id"]) # make sure all devices are accepted auth_v2.check_expected_status("accepted", len(get_mender_clients())) # make sure mender-store contains authtoken have_token() # print all device ids for device in auth_v2.get_devices_status("accepted"): logging.info("Accepted DeviceID: %s" % device["id"])
def ip_to_device_id_map(clients): # Get deviceauth data, which includes device identity. devauth_devices = auth_v2.get_devices(expected_devices=len(clients)) # Collect identity of each client. ret = execute(run, "/usr/share/mender/identity/mender-device-identity", hosts=clients) # Calculate device identities. identity_to_ip = {} for client in clients: identity_to_ip[Helpers.identity_script_to_identity_string(ret[client])] = client # Match them. ip_to_device_id = {} for device in devauth_devices: ip_to_device_id[identity_to_ip[json.dumps(device['identity_data'], separators=(",", ":"))]] = device['id'] return ip_to_device_id
def test_reject_bootstrap(self): """Make sure a rejected device does not perform an upgrade, and that it gets it's auth token removed""" if not env.host_string: execute(self.test_reject_bootstrap, hosts=get_mender_clients()) return # iterate over devices and reject them for device in auth_v2.get_devices(): auth_v2.set_device_auth_set_status(device["id"], device["auth_sets"][0]["id"], "rejected") logging.info("Rejecting DeviceID: %s" % device["id"]) auth_v2.check_expected_status("rejected", len(get_mender_clients())) with Helpers.RebootDetector() as reboot: try: deployment_id, _ = common_update_procedure(install_image=conftest.get_valid_image()) except AssertionError: logging.info("Failed to deploy upgrade to rejected device.") reboot.verify_reboot_not_performed() else: # use assert to fail, so we can get backend logs pytest.fail("no error while trying to deploy to rejected device") return finished = False # wait until auththoken is removed from file for _ in range(10): with settings(abort_exception=Exception): try: run("journalctl -u mender -l -n 3 | grep -q 'authentication request rejected'") except: time.sleep(30) else: finished = True break auth_v2.accept_devices(1) if not finished: pytest.fail("failed to remove authtoken from mender-store file")
def do_test_fail_preauth_existing(self): """ Test 'conflict' response when an identity data set already exists. """ # wait for the device to appear devs = auth_v2.get_devices(1) dev = devs[0] # try to preauthorize the same id data, new key preauth_key = '''-----BEGIN PUBLIC KEY----- MIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8AMIIBCgKCAQEAzogVU7RGDilbsoUt/DdH VJvcepl0A5+xzGQ50cq1VE/Dyyy8Zp0jzRXCnnu9nu395mAFSZGotZVr+sWEpO3c yC3VmXdBZmXmQdZqbdD/GuixJOYfqta2ytbIUPRXFN7/I7sgzxnXWBYXYmObYvdP okP0mQanY+WKxp7Q16pt1RoqoAd0kmV39g13rFl35muSHbSBoAW3GBF3gO+mF5Ty 1ddp/XcgLOsmvNNjY+2HOD5F/RX0fs07mWnbD7x+xz7KEKjF+H7ZpkqCwmwCXaf0 iyYyh1852rti3Afw4mDxuVSD7sd9ggvYMc0QHIpQNkD4YWOhNiE1AB0zH57VbUYG UwIDAQAB -----END PUBLIC KEY----- ''' r = auth_v2.preauth(dev['identity_data'], preauth_key) assert r.status_code == 409
def test_unsigned_artifact_fails_deployment( self, standard_setup_with_signed_artifact_client): """ Make sure that an unsigned image fails, and is handled by the backend. Notice that this test needs a fresh new version of the backend, since we installed a signed image earlier without a verification key in mender.conf """ if not env.host_string: execute(self.test_unsigned_artifact_fails_deployment, standard_setup_with_signed_artifact_client, hosts=get_mender_clients()) return deployment_id, _ = common_update_procedure( install_image=conftest.get_valid_image()) deploy.check_expected_status("finished", deployment_id) deploy.check_expected_statistics(deployment_id, "failure", 1) for d in auth_v2.get_devices(): assert "expecting signed artifact, but no signature file found" in \ deploy.get_logs(d["id"], deployment_id)
def do_test_ok_preauth_and_bootstrap(self): """ Test the happy path from preauthorizing a device to a successful bootstrap. Verify that the device/auth set appear correctly in devauth API results. """ client = get_mender_clients()[0] # we'll use the same pub key for the preauth'd device, so get it res = execute(Client.get_pub_key, hosts=client) preauth_key = res[client].exportKey() # stick an extra newline on the key - this is how a device would send it preauth_key += '\n' # preauthorize a new device preauth_iddata = {"mac": "mac-preauth"} # serialize manually to avoid an extra space (id data helper doesn't insert one) preauth_iddata_str = "{\"mac\":\"mac-preauth\"}" r = auth_v2.preauth(json.loads(preauth_iddata_str), preauth_key) assert r.status_code == 201 # verify the device appears correctly in api results devs = auth_v2.get_devices(2) dev_preauth = [d for d in devs if d['status'] == 'preauthorized'] assert len(dev_preauth) == 1 dev_preauth = dev_preauth[0] assert dev_preauth['identity_data'] == preauth_iddata_str assert dev_preauth['pubkey'] == preauth_key # make one of the existing devices the preauthorized device # by substituting id data and restarting res = execute(Client.substitute_id_data, preauth_iddata, hosts=client) res = execute(Client.restart, hosts=client) # verify api results - after some time the device should be 'accepted' for _ in range(120): time.sleep(15) dev_accepted = auth_v2.get_devices_status(status="accepted", expected_devices=2) if len([d for d in dev_accepted if d['status'] == 'accepted']) == 1: break logging.info("devices: " + str(dev_accepted)) dev_accepted = [d for d in dev_accepted if d['status'] == 'accepted'] logging.info("accepted devices: " + str(dev_accepted)) execute(Client.get_logs, hosts=client) assert len( dev_accepted) == 1, "looks like the device was never accepted" dev_accepted = dev_accepted[0] logging.info("accepted device: " + str(dev_accepted)) assert dev_accepted['identity_data'] == preauth_iddata_str assert dev_accepted['pubkey'] == preauth_key # verify device was issued a token res = execute(Client.have_authtoken, hosts=client) assert res[client]
# start docker-compose ret = subprocess.call([ "docker-compose", "-p", conftest.docker_compose_instance, "-f", "docker-compose.yml", "-f", "docker-compose.storage.minio.yml", "-f", "./production-testing-env.yml", "up", "-d" ], cwd="../") assert ret == 0, "failed to start docker-compose" if args.deploy: # create account for management api auth.get_auth_token() # wait for 10 devices to be available devices = auth_v2.get_devices(10) assert len(devices) == 10 # accept all devices for d in devices: auth_v2.set_device_auth_set_status(d["id"], d["auth_sets"][0]["id"], "accepted") # make sure artifact tool in current workdir is being used os.environ["PATH"] = os.path.dirname(os.path.realpath( __file__)) + "/downloaded-tools" + os.pathsep + os.environ["PATH"] # perform upgrade devices_to_update = list( set([ device["id"]
def test_inventory(self): """Test that device reports inventory after having bootstrapped.""" attempts = 10 while True: attempts = attempts - 1 try: inv_json = inv.get_devices() auth_json = auth_v2.get_devices() auth_ids = [device['id'] for device in auth_json] assert (len(inv_json) > 0) for device in inv_json: try: # Check that authentication and inventory agree. assert (device['id'] in auth_ids) attrs = device['attributes'] # Check individual attributes. network_interfaces = [ elem for elem in attrs if elem['name'] == "network_interfaces" ] assert len(network_interfaces) == 1 network_interfaces = network_interfaces[0] if type(network_interfaces['value']) is str: assert any(network_interfaces['value'] == iface for iface in ["eth0", "enp0s3"]) else: assert any(iface in network_interfaces['value'] for iface in ["eth0", "enp0s3"]) assert (json.loads( '{"name": "hostname", "value": "%s"}' % conftest.machine_name) in attrs) assert (json.loads( '{"name": "device_type", "value": "%s"}' % conftest.machine_name) in attrs) if conftest.machine_name == "qemux86-64": bootloader_integration = "uefi_grub" elif conftest.machine_name == "vexpress-qemu": bootloader_integration = "uboot" else: pytest.fail( "Unknown machine_name. Please add an expected bootloader_integration for this machine_name" ) assert (json.loads( '{"name": "mender_bootloader_integration", "value": "%s"}' % bootloader_integration) in attrs) # Check that all known keys are present. keys = [str(attr['name']) for attr in attrs] expected_keys = [ "hostname", "network_interfaces", "cpu_model", "mem_total_kB", "device_type", ["ipv4_enp0s3", "ipv6_enp0s3", "ipv4_eth0"], # Multiple possibilities ["mac_enp0s3", "mac_eth0"], "mender_client_version", "artifact_name", "kernel", "os", ] for key in expected_keys: if type(key) is list: assert any([subkey in keys for subkey in key]) else: assert key in keys except: print("Exception caught, 'device' json: ", device) raise break except: # This may pass only after the client has had some time to # report. if attempts > 0: time.sleep(5) continue else: raise
def test_decommissioning_post_upgrade(self): # assertion error occurs here on decommissioning fail for device in auth_v2.get_devices(10): auth_v2.decommission(device["id"])