def test_remove_worker(deployment, conf, platform, skuba, kubectl): initial_workers = skuba.num_of_nodes("worker") # Remove the worker skuba.node_remove(role="worker", nr=initial_workers - 1) assert skuba.num_of_nodes("worker") == initial_workers - 1 wait( kubectl.run_kubectl, 'wait --timeout=5m --for=condition=Ready pods --all --namespace=kube-system', wait_delay=60, wait_timeout=300, wait_backoff=30, wait_retries=5, wait_allow=(RuntimeError))
def test_reboot(deployment, platform): try: platform.ssh_run("worker", 0, "sudo reboot &") except Exception: pass wait(platform.ssh_run, "worker", 0, "/bin/true", wait_delay=30, wait_timeout=10, wait_backoff=30, wait_retries=5, wait_allow=(RuntimeError))
def test_upgrade_apply_user_lock(provision, platform, kubectl, skuba): """ Starting from an outdated cluster, check what node upgrade apply reports. """ # lock kured kubectl_cmd = ( "-n kube-system annotate ds kured weave.works/kured-node-lock=" "'{\"nodeID\":\"manual\"}'") kubectl.run_kubectl(kubectl_cmd) for role in ("master", "worker"): num_nodes = platform.get_num_nodes(role) for n in range(0, num_nodes): # disable skuba-update.timer platform.ssh_run( role, n, "sudo systemctl disable --now skuba-update.timer") assert node_is_ready(platform, kubectl, role, n) result = skuba.node_upgrade("apply", role, n) assert result.find("successfully upgraded") != -1 assert node_is_upgraded(kubectl, platform, role, n) ssh_cmd = "sudo systemctl is-enabled skuba-update.timer || :" assert platform.ssh_run(role, n, ssh_cmd).find("disabled") != -1 kubectl_cmd = (r"-n kube-system get ds/kured -o jsonpath=" r"'{.metadata.annotations.weave\.works/kured-node-lock}'") result = wait(kubectl.run_kubectl, kubectl_cmd, wait_backoff=30, wait_retries=3, wait_allow=(RuntimeError)) assert result.find("manual") != -1
def test_remove_worker(deployment, conf, platform, skuba, kubectl): workers = kubectl.get_node_names_by_role("worker") workers_count = len(workers) # Remove the worker skuba.node_remove(role="worker", nr=workers_count - 1) assert len(kubectl.get_node_names_by_role("worker")) == workers_count - 1 wait( kubectl.run_kubectl, 'wait --timeout=5m --for=condition=Ready pods --all --namespace=kube-system', wait_delay=60, wait_timeout=300, wait_backoff=30, wait_retries=5, wait_allow=(RuntimeError))
def test_node_join_cpi_openstack_cluster(skuba, kubectl): """ Join nodes to the cluster with the openstack cpi enabled """ try: skuba.join_nodes(masters=1, workers=3) except: pytest.fail( "Failure on joinning nodes to the cluster with cpi optionstack ..." ) wait(check_system_pods_ready, kubectl, wait_delay=60, wait_timeout=10, wait_backoff=30, wait_elapsed=300, wait_allow=(AssertionError))
def setup_kubernetes_version(platform, skuba, kubectl, kubernetes_version=None): """ Initialize the cluster with the given kubernetes_version, bootstrap it and join nodes. """ skuba.cluster_init(kubernetes_version) skuba.node_bootstrap() skuba.join_nodes() wait(check_nodes_ready, kubectl, wait_delay=60, wait_backoff=30, wait_elapsed=60 * 10, wait_allow=(AssertionError))
def migrate_node(platform, kubectl, role, node, regcode, option=1): platform.ssh_run(role, node, f'sudo SUSEConnect -r {regcode}') platform.ssh_run(role, node, "sudo SUSEConnect -p sle-module-containers/15.1/x86_64") platform.ssh_run(role, node, f'sudo SUSEConnect -p caasp/4.0/x86_64 -r {regcode}') platform.ssh_run(role, node, "sudo sudo zypper in -y --no-recommends zypper-migration-plugin") platform.ssh_run(role, node, (f'sudo zypper migration --migration {option}' ' --no-recommends --non-interactive' ' --auto-agree-with-licenses --allow-vendor-change')) #:FIXME use kured for controlled reboot. platform.ssh_run(role, node, "sudo reboot &") # wait the node become live wait(platform.ssh_run, role, node, "true", wait_delay=60, wait_timeout=10, wait_backoff=30, wait_elapsed=180, wait_allow=(RuntimeError))
def migrate_node(platform, checker, kubectl, role, node, regcode, option=1): platform.ssh_run(role, node, f'sudo SUSEConnect -r {regcode}') platform.ssh_run(role, node, "sudo SUSEConnect -p sle-module-containers/15.1/x86_64") platform.ssh_run(role, node, f'sudo SUSEConnect -p caasp/4.0/x86_64 -r {regcode}') platform.ssh_run( role, node, "sudo sudo zypper in -y --no-recommends zypper-migration-plugin") platform.ssh_run(role, node, (f'sudo zypper migration --migration {option}' ' --no-recommends --non-interactive' ' --auto-agree-with-licenses --allow-vendor-change')) # update to latest kubernetes package version, if any k8s_major, k8s_minor, _ = CURRENT_VERSION.split('.') platform.ssh_run( role, node, 'sudo zypper ar http://download.suse.de/ibs/Devel:/CaaSP:/4.5/SLE_15_SP2/ CaasP_Devel' ) platform.ssh_run(role, node, ('sudo zypper update --allow-vendor-change -y' f' kubernetes-{k8s_major}.{k8s_minor}-kubeadm')) #:FIXME use kured for controlled reboot. platform.ssh_run(role, node, "sudo reboot &") # wait the node become live wait(platform.ssh_run, role, node, "true", wait_delay=60, wait_timeout=10, wait_backoff=30, wait_elapsed=180, wait_allow=(RuntimeError)) checker.check_node(role, node, stage="joined", timeout=60)
def test_hard_reboot(deployment, platform, skuba, kubectl, node_type, node_number): assert skuba.num_of_nodes(node_type) > node_number logger.info('Wait for all the nodes to be ready') wait(kubectl.run_kubectl, 'wait --for=condition=Ready node --all --timeout=0', wait_timeout=10, wait_backoff=30, wait_elapsed=300, wait_allow=(RuntimeError)) logger.info(f'Rebooting {node_type} {node_number}') platform.ssh_run(node_type, node_number, 'sudo reboot &') # wait the node to reboot wait(platform.ssh_run, node_type, node_number, 'echo hello', wait_backoff=30, wait_timeout=30, wait_elapsed=300, wait_allow=(RuntimeError)) # wait the node to become ready wait(kubectl.run_kubectl, 'wait --for=condition=Ready node --all --timeout=0', wait_delay=30, wait_timeout=10, wait_backoff=30, wait_elapsed=180, wait_allow=(RuntimeError)) # wait pods to get become ready wait(check_pods_ready, kubectl, wait_timeout=10, wait_backoff=30, wait_elapsed=180, wait_allow=(AssertionError))
def deployment(request, bootstrap, skuba, kubectl): if request.config.getoption("skip_setup") != 'deployed': skuba.join_nodes() wait(kubectl.run_kubectl, 'wait --timeout=5m --for=condition=Ready pods --all --namespace=kube-system', wait_delay=60, wait_timeout=300, wait_backoff=30, wait_retries=5, wait_allow=(RuntimeError))
def test_cillium(deployment, kubectl): landing_req = 'curl -sm10 -XPOST deathstar.default.svc.cluster.local/v1/request-landing' logger.info("Deploy deathstar") kubectl.run_kubectl("create -f https://raw.githubusercontent.com/cilium/cilium/v1.6/examples/minikube/http-sw-app.yaml") # FIXME: This check should be only get the star wars application's pods wait(kubectl.run_kubectl, "wait --for=condition=ready pods --all --timeout=0", wait_delay=30, wait_timeout=10, wait_backoff=30, wait_elapsed=180) # FIXME: this hardcoded wait should be replaces with a (cilum?) condition time.sleep(100) logger.info("Check with L3/L4 policy") kubectl.run_kubectl("create -f https://raw.githubusercontent.com/cilium/cilium/v1.6/examples/minikube/sw_l3_l4_policy.yaml") tie_out = kubectl.run_kubectl("exec tiefighter -- {}".format(landing_req)) assert 'Ship landed' in tie_out xwing_out = kubectl.run_kubectl("exec xwing -- {} 2>&1 || :".format(landing_req)) assert 'terminated with exit code 28' in xwing_out logger.info("Check status (N/N)") node_list = kubectl.run_kubectl("get nodes -o jsonpath='{ .items[*].metadata.name }'") node_count = len(node_list.split(" ")) cilium_podlist = kubectl.run_kubectl("get pods -n kube-system -l k8s-app=cilium -o jsonpath='{ .items[0].metadata.name }'").split(" ") cilium_podid = cilium_podlist[0] cilium_status_cmd = "-n kube-system exec {} -- cilium status".format(cilium_podid) cilium_status = kubectl.run_kubectl(cilium_status_cmd) assert re.search(r'Controller Status:\s+([0-9]+)/\1 healthy', cilium_status) is not None for i in range(1, 10): cilium_status = kubectl.run_kubectl(cilium_status_cmd) all_reachable = re.search(r"Cluster health:\s+({})/\1 reachable".format(node_count), cilium_status) if all_reachable: break time.sleep(30) assert all_reachable logger.info("Change Cilium ConfigMap and set MTU 1450 bytes") # Change Cilium ConfigMap and set MTU 1450 bytes kubectl.run_kubectl("get configmap -n kube-system cilium-config -o json | jq '.data.mtu = \"1450\"'") # Apply changes and restart cilium pods kubectl.run_kubectl("delete pod -n kube-system -l name=cilium-operator") kubectl.run_kubectl("delete pods -n kube-system -l k8s-app=cilium") logger.info("Check status (N/N) with cilium MTU 1450") cilium_podlist = kubectl.run_kubectl("get pods -n kube-system -l k8s-app=cilium -o jsonpath='{ .items[0].metadata.name }'").split(" ") cilium_podid = cilium_podlist[0] cilium_status_cmd = "-n kube-system exec {} -- cilium status".format(cilium_podid) cilium_status = kubectl.run_kubectl(cilium_status_cmd) assert re.search(r'Controller Status:\s+([0-9]+)/\1 healthy', cilium_status) is not None for i in range(1, 10): cilium_status = kubectl.run_kubectl(cilium_status_cmd) all_reachable = re.search(r"Cluster health:\s+({})/\1 reachable".format(node_count), cilium_status) if all_reachable: break time.sleep(30) assert all_reachable