def test_remove_worker(deployment, conf, platform, skuba, kubectl):
initial_workers = skuba.num_of_nodes("worker")
# Remove the worker
skuba.node_remove(role="worker", nr=initial_workers - 1)
assert skuba.num_of_nodes("worker") == initial_workers - 1
wait(
kubectl.run_kubectl,
'wait --timeout=5m --for=condition=Ready pods --all --namespace=kube-system',
wait_delay=60,
wait_timeout=300,
wait_backoff=30,
wait_retries=5,
wait_allow=(RuntimeError))
def test_reboot(deployment, platform):
try:
platform.ssh_run("worker", 0, "sudo reboot &")
except Exception:
pass
wait(platform.ssh_run,
"worker",
0,
"/bin/true",
wait_delay=30,
wait_timeout=10,
wait_backoff=30,
wait_retries=5,
wait_allow=(RuntimeError))
def test_upgrade_apply_user_lock(provision, platform, kubectl, skuba):
"""
Starting from an outdated cluster, check what node upgrade apply reports.
"""
# lock kured
kubectl_cmd = (
"-n kube-system annotate ds kured weave.works/kured-node-lock="
"'{\"nodeID\":\"manual\"}'")
kubectl.run_kubectl(kubectl_cmd)
for role in ("master", "worker"):
num_nodes = platform.get_num_nodes(role)
for n in range(0, num_nodes):
# disable skuba-update.timer
platform.ssh_run(
role, n, "sudo systemctl disable --now skuba-update.timer")
assert node_is_ready(platform, kubectl, role, n)
result = skuba.node_upgrade("apply", role, n)
assert result.find("successfully upgraded") != -1
assert node_is_upgraded(kubectl, platform, role, n)
ssh_cmd = "sudo systemctl is-enabled skuba-update.timer || :"
assert platform.ssh_run(role, n, ssh_cmd).find("disabled") != -1
kubectl_cmd = (r"-n kube-system get ds/kured -o jsonpath="
r"'{.metadata.annotations.weave\.works/kured-node-lock}'")
result = wait(kubectl.run_kubectl,
kubectl_cmd,
wait_backoff=30,
wait_retries=3,
wait_allow=(RuntimeError))
assert result.find("manual") != -1
def test_remove_worker(deployment, conf, platform, skuba, kubectl):
workers = kubectl.get_node_names_by_role("worker")
workers_count = len(workers)
# Remove the worker
skuba.node_remove(role="worker", nr=workers_count - 1)
assert len(kubectl.get_node_names_by_role("worker")) == workers_count - 1
wait(
kubectl.run_kubectl,
'wait --timeout=5m --for=condition=Ready pods --all --namespace=kube-system',
wait_delay=60,
wait_timeout=300,
wait_backoff=30,
wait_retries=5,
wait_allow=(RuntimeError))
def test_node_join_cpi_openstack_cluster(skuba, kubectl):
"""
Join nodes to the cluster with the openstack cpi enabled
"""
try:
skuba.join_nodes(masters=1, workers=3)
except:
pytest.fail(
"Failure on joinning nodes to the cluster with cpi optionstack ..."
)
wait(check_system_pods_ready,
kubectl,
wait_delay=60,
wait_timeout=10,
wait_backoff=30,
wait_elapsed=300,
wait_allow=(AssertionError))
def setup_kubernetes_version(platform,
skuba,
kubectl,
kubernetes_version=None):
"""
Initialize the cluster with the given kubernetes_version, bootstrap it and
join nodes.
"""
skuba.cluster_init(kubernetes_version)
skuba.node_bootstrap()
skuba.join_nodes()
wait(check_nodes_ready,
kubectl,
wait_delay=60,
wait_backoff=30,
wait_elapsed=60 * 10,
wait_allow=(AssertionError))
def migrate_node(platform, kubectl, role, node, regcode, option=1):
platform.ssh_run(role, node, f'sudo SUSEConnect -r {regcode}')
platform.ssh_run(role, node, "sudo SUSEConnect -p sle-module-containers/15.1/x86_64")
platform.ssh_run(role, node, f'sudo SUSEConnect -p caasp/4.0/x86_64 -r {regcode}')
platform.ssh_run(role, node, "sudo sudo zypper in -y --no-recommends zypper-migration-plugin")
platform.ssh_run(role, node, (f'sudo zypper migration --migration {option}'
' --no-recommends --non-interactive'
' --auto-agree-with-licenses --allow-vendor-change'))
#:FIXME use kured for controlled reboot.
platform.ssh_run(role, node, "sudo reboot &")
# wait the node become live
wait(platform.ssh_run,
role,
node,
"true",
wait_delay=60,
wait_timeout=10,
wait_backoff=30,
wait_elapsed=180,
wait_allow=(RuntimeError))
def migrate_node(platform, checker, kubectl, role, node, regcode, option=1):
platform.ssh_run(role, node, f'sudo SUSEConnect -r {regcode}')
platform.ssh_run(role, node,
"sudo SUSEConnect -p sle-module-containers/15.1/x86_64")
platform.ssh_run(role, node,
f'sudo SUSEConnect -p caasp/4.0/x86_64 -r {regcode}')
platform.ssh_run(
role, node,
"sudo sudo zypper in -y --no-recommends zypper-migration-plugin")
platform.ssh_run(role, node,
(f'sudo zypper migration --migration {option}'
' --no-recommends --non-interactive'
' --auto-agree-with-licenses --allow-vendor-change'))
# update to latest kubernetes package version, if any
k8s_major, k8s_minor, _ = CURRENT_VERSION.split('.')
platform.ssh_run(
role, node,
'sudo zypper ar http://download.suse.de/ibs/Devel:/CaaSP:/4.5/SLE_15_SP2/ CaasP_Devel'
)
platform.ssh_run(role, node,
('sudo zypper update --allow-vendor-change -y'
f' kubernetes-{k8s_major}.{k8s_minor}-kubeadm'))
#:FIXME use kured for controlled reboot.
platform.ssh_run(role, node, "sudo reboot &")
# wait the node become live
wait(platform.ssh_run,
role,
node,
"true",
wait_delay=60,
wait_timeout=10,
wait_backoff=30,
wait_elapsed=180,
wait_allow=(RuntimeError))
checker.check_node(role, node, stage="joined", timeout=60)
def test_hard_reboot(deployment, platform, skuba, kubectl, node_type,
node_number):
assert skuba.num_of_nodes(node_type) > node_number
logger.info('Wait for all the nodes to be ready')
wait(kubectl.run_kubectl,
'wait --for=condition=Ready node --all --timeout=0',
wait_timeout=10,
wait_backoff=30,
wait_elapsed=300,
wait_allow=(RuntimeError))
logger.info(f'Rebooting {node_type} {node_number}')
platform.ssh_run(node_type, node_number, 'sudo reboot &')
# wait the node to reboot
wait(platform.ssh_run,
node_type,
node_number,
'echo hello',
wait_backoff=30,
wait_timeout=30,
wait_elapsed=300,
wait_allow=(RuntimeError))
# wait the node to become ready
wait(kubectl.run_kubectl,
'wait --for=condition=Ready node --all --timeout=0',
wait_delay=30,
wait_timeout=10,
wait_backoff=30,
wait_elapsed=180,
wait_allow=(RuntimeError))
# wait pods to get become ready
wait(check_pods_ready,
kubectl,
wait_timeout=10,
wait_backoff=30,
wait_elapsed=180,
wait_allow=(AssertionError))
def deployment(request, bootstrap, skuba, kubectl):
if request.config.getoption("skip_setup") != 'deployed':
skuba.join_nodes()
wait(kubectl.run_kubectl, 'wait --timeout=5m --for=condition=Ready pods --all --namespace=kube-system', wait_delay=60, wait_timeout=300, wait_backoff=30, wait_retries=5, wait_allow=(RuntimeError))
def test_cillium(deployment, kubectl):
landing_req = 'curl -sm10 -XPOST deathstar.default.svc.cluster.local/v1/request-landing'
logger.info("Deploy deathstar")
kubectl.run_kubectl("create -f https://raw.githubusercontent.com/cilium/cilium/v1.6/examples/minikube/http-sw-app.yaml")
# FIXME: This check should be only get the star wars application's pods
wait(kubectl.run_kubectl,
"wait --for=condition=ready pods --all --timeout=0",
wait_delay=30,
wait_timeout=10,
wait_backoff=30,
wait_elapsed=180)
# FIXME: this hardcoded wait should be replaces with a (cilum?) condition
time.sleep(100)
logger.info("Check with L3/L4 policy")
kubectl.run_kubectl("create -f https://raw.githubusercontent.com/cilium/cilium/v1.6/examples/minikube/sw_l3_l4_policy.yaml")
tie_out = kubectl.run_kubectl("exec tiefighter -- {}".format(landing_req))
assert 'Ship landed' in tie_out
xwing_out = kubectl.run_kubectl("exec xwing -- {} 2>&1 || :".format(landing_req))
assert 'terminated with exit code 28' in xwing_out
logger.info("Check status (N/N)")
node_list = kubectl.run_kubectl("get nodes -o jsonpath='{ .items[*].metadata.name }'")
node_count = len(node_list.split(" "))
cilium_podlist = kubectl.run_kubectl("get pods -n kube-system -l k8s-app=cilium -o jsonpath='{ .items[0].metadata.name }'").split(" ")
cilium_podid = cilium_podlist[0]
cilium_status_cmd = "-n kube-system exec {} -- cilium status".format(cilium_podid)
cilium_status = kubectl.run_kubectl(cilium_status_cmd)
assert re.search(r'Controller Status:\s+([0-9]+)/\1 healthy', cilium_status) is not None
for i in range(1, 10):
cilium_status = kubectl.run_kubectl(cilium_status_cmd)
all_reachable = re.search(r"Cluster health:\s+({})/\1 reachable".format(node_count), cilium_status)
if all_reachable:
break
time.sleep(30)
assert all_reachable
logger.info("Change Cilium ConfigMap and set MTU 1450 bytes")
# Change Cilium ConfigMap and set MTU 1450 bytes
kubectl.run_kubectl("get configmap -n kube-system cilium-config -o json | jq '.data.mtu = \"1450\"'")
# Apply changes and restart cilium pods
kubectl.run_kubectl("delete pod -n kube-system -l name=cilium-operator")
kubectl.run_kubectl("delete pods -n kube-system -l k8s-app=cilium")
logger.info("Check status (N/N) with cilium MTU 1450")
cilium_podlist = kubectl.run_kubectl("get pods -n kube-system -l k8s-app=cilium -o jsonpath='{ .items[0].metadata.name }'").split(" ")
cilium_podid = cilium_podlist[0]
cilium_status_cmd = "-n kube-system exec {} -- cilium status".format(cilium_podid)
cilium_status = kubectl.run_kubectl(cilium_status_cmd)
assert re.search(r'Controller Status:\s+([0-9]+)/\1 healthy', cilium_status) is not None
for i in range(1, 10):
cilium_status = kubectl.run_kubectl(cilium_status_cmd)
all_reachable = re.search(r"Cluster health:\s+({})/\1 reachable".format(node_count), cilium_status)
if all_reachable:
break
time.sleep(30)
assert all_reachable