def validate_knative():
"""
Validate Knative by deploying the helloworld-go app.
"""
if platform.machine() != "x86_64":
print("Knative tests are only relevant in x86 architectures")
return
wait_for_installation()
knative_services = [
"activator",
"autoscaler",
"controller",
]
for service in knative_services:
wait_for_pod_state("",
"knative-serving",
"running",
label="app={}".format(service))
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "knative-helloworld.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("",
"default",
"running",
label="serving.knative.dev/service=helloworld-go")
kubectl("delete -f {}".format(manifest))
def validate_istio():
"""
Validate istio by deploying the bookinfo app.
"""
if platform.machine() != "x86_64":
print("Istio tests are only relevant in x86 architectures")
return
wait_for_installation()
istio_services = [
"citadel",
"egressgateway",
"galley",
"ingressgateway",
"sidecar-injector",
]
for service in istio_services:
wait_for_pod_state("",
"istio-system",
"running",
label="istio={}".format(service))
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "bookinfo.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("", "default", "running", label="app=details")
kubectl("delete -f {}".format(manifest))
def validate_linkerd():
"""
Validate Linkerd by deploying emojivoto.
"""
if platform.machine() != "x86_64":
print("Linkerd tests are only relevant in x86 architectures")
return
wait_for_installation()
wait_for_pod_state(
"",
"linkerd",
"running",
label="linkerd.io/control-plane-component=controller",
timeout_insec=300,
)
print("Linkerd controller up and running.")
wait_for_pod_state(
"",
"linkerd",
"running",
label="linkerd.io/control-plane-component=proxy-injector",
timeout_insec=300,
)
print("Linkerd proxy injector up and running.")
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "emojivoto.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("",
"emojivoto",
"running",
label="app=emoji-svc",
timeout_insec=600)
kubectl("delete -f {}".format(manifest))
def validate_cilium():
"""
Validate cilium by deploying the bookinfo app.
"""
if platform.machine() != "x86_64":
print("Cilium tests are only relevant in x86 architectures")
return
wait_for_installation()
wait_for_pod_state("", "kube-system", "running", label="k8s-app=cilium")
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "nginx-pod.yaml")
# Try up to three times to get nginx under cilium
for attempt in range(0, 10):
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("", "default", "running", label="app=nginx")
output = cilium("endpoint list -o json", timeout_insec=20)
if "nginx" in output:
kubectl("delete -f {}".format(manifest))
break
else:
print("Cilium not ready will retry testing.")
kubectl("delete -f {}".format(manifest))
time.sleep(20)
else:
print("Cilium testing failed.")
assert False
def validate_multus():
"""
Validate multus by making sure the multus pod is running.
"""
wait_for_installation()
wait_for_pod_state("", "kube-system", "running", label="app=multus")
def validate_multus():
"""
Validate multus by deploying alpine pod with 3 interfaces.
"""
wait_for_installation()
here = os.path.dirname(os.path.abspath(__file__))
shutil.rmtree("/tmp/microk8s-multus-test-nets", ignore_errors=True)
networks = os.path.join(here, "templates", "multus-networks.yaml")
kubectl("create -f {}".format(networks))
manifest = os.path.join(here, "templates", "multus-alpine.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("", "default", "running", label="app=multus-alpine")
output = kubectl("exec multus-alpine -- ifconfig eth1",
timeout_insec=900,
err_out="no")
assert "10.111.111.111" in output
output = kubectl("exec multus-alpine -- ifconfig eth2",
timeout_insec=900,
err_out="no")
assert "10.222.222.222" in output
kubectl("delete -f {}".format(manifest))
kubectl("delete -f {}".format(networks))
shutil.rmtree("/tmp/microk8s-multus-test-nets", ignore_errors=True)
def validate_openebs():
"""
Validate OpenEBS
"""
wait_for_installation()
wait_for_pod_state(
"",
"openebs",
"running",
label="openebs.io/component-name=maya-apiserver",
timeout_insec=900,
)
print("OpenEBS is up and running.")
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "openebs-test.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("",
"default",
"running",
label="app=openebs-test-busybox",
timeout_insec=900)
output = kubectl("exec openebs-test-busybox -- ls /",
timeout_insec=900,
err_out="no")
assert "my-data" in output
kubectl("delete -f {}".format(manifest))
def validate_kata():
"""
Validate Kata
"""
wait_for_installation()
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "nginx-kata.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("", "default", "running", label="app=kata")
kubectl("delete -f {}".format(manifest))
def validate_keda():
"""
Validate keda
"""
wait_for_installation()
wait_for_pod_state("", "keda", "running", label="app=keda-operator")
print("KEDA operator up and running.")
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "keda-scaledobject.yaml")
kubectl("apply -f {}".format(manifest))
scaledObject = kubectl("-n gonuts get scaledobject.keda.sh")
assert "stan-scaledobject" in scaledObject
kubectl("delete -f {}".format(manifest))
def validate_cilium():
"""
Validate cilium by deploying the bookinfo app.
"""
if platform.machine() != 'x86_64':
print("Cilium tests are only relevant in x86 architectures")
return
wait_for_installation()
wait_for_pod_state("", "kube-system", "running", label="k8s-app=cilium")
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "nginx-pod.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("", "default", "running", label="app=nginx")
output = cilium('endpoint list -o json')
assert "nginx" in output
kubectl("delete -f {}".format(manifest))
def validate_multus():
"""
Validate multus by deploying alpine pod with 3 interfaces.
"""
wait_for_installation()
here = os.path.dirname(os.path.abspath(__file__))
networks = os.path.join(here, "templates", "multus-networks.yaml")
kubectl("create -f {}".format(networks))
manifest = os.path.join(here, "templates", "multus-alpine.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("", "default", "running", label="app=multus-alpine")
output = kubectl("exec multus-alpine -- ifconfig eth1", err_out='no')
assert "10.111.111.111" in output
output = kubectl("exec multus-alpine -- ifconfig eth2", err_out='no')
assert "10.222.222.222" in output
kubectl("delete -f {}".format(manifest))
kubectl("delete -f {}".format(networks))
def validate_istio():
"""
Validate istio by deploying the bookinfo app.
"""
if platform.machine() != "x86_64":
print("Istio tests are only relevant in x86 architectures")
return
wait_for_installation()
istio_services = [
"pilot",
"egressgateway",
"ingressgateway",
]
for service in istio_services:
wait_for_pod_state("", "istio-system", "running", label="istio={}".format(service))
cmd = "/snap/bin/microk8s.istioctl verify-install"
return run_until_success(cmd, timeout_insec=900, err_out="no")
def validate_linkerd():
"""
Validate Linkerd by deploying emojivoto.
"""
if platform.machine() != 'x86_64':
print("Linkerd tests are only relevant in x86 architectures")
return
wait_for_installation()
wait_for_pod_state("",
"linkerd",
"running",
label="linkerd.io/control-plane-ns")
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "emojivoto.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("", "emojivoto", "running", label="app=emoji-svc")
kubectl("delete -f {}".format(manifest))
def validate_istio():
"""
Validate istio by deploying the bookinfo app.
"""
wait_for_installation()
istio_services = [
"citadel",
"egressgateway",
"galley",
"ingressgateway",
"sidecar-injector",
"statsd-prom-bridge",
]
for service in istio_services:
wait_for_pod_state("", "istio-system", "running", label="istio={}".format(service))
here = os.path.dirname(os.path.abspath(__file__))
manifest = os.path.join(here, "templates", "bookinfo.yaml")
kubectl("apply -f {}".format(manifest))
wait_for_pod_state("", "default", "running", label="app=details")
kubectl("delete -f {}".format(manifest))
def test_refresh_path(self):
"""
Deploy an old snap and try to refresh until the current one.
"""
start_channel = 14
last_stable_minor = None
if upgrade_from.startswith('latest') or '/' not in upgrade_from:
attempt = 0
release_url = "https://dl.k8s.io/release/stable.txt"
while attempt < 10 and not last_stable_minor:
r = requests.get(release_url)
if r.status_code == 200:
last_stable_str = r.content.decode().strip()
# We have "v1.18.4" and we need the "18"
last_stable_parts = last_stable_str.split('.')
last_stable_minor = int(last_stable_parts[1])
else:
time.sleep(3)
attempt += 1
else:
channel_parts = upgrade_from.split('.')
channel_parts = channel_parts[1].split('/')
print(channel_parts)
last_stable_minor = int(channel_parts[0])
print("")
print(
"Testing refresh path from 1.{} to 1.{} and finally refresh to {}".
format(start_channel, last_stable_minor, upgrade_to))
assert last_stable_minor is not None
channel = "1.{}/stable".format(start_channel)
print("Installing {}".format(channel))
cmd = "sudo snap install microk8s --classic --channel={}".format(
channel)
run_until_success(cmd)
wait_for_installation()
channel_minor = start_channel
channel_minor += 1
while channel_minor <= last_stable_minor:
channel = "1.{}/stable".format(channel_minor)
print("Refreshing to {}".format(channel))
cmd = "sudo snap refresh microk8s --classic --channel={}".format(
channel)
run_until_success(cmd)
wait_for_installation()
channel_minor += 1
print("Installing {}".format(upgrade_to))
if upgrade_to.endswith('.snap'):
cmd = "sudo snap install {} --classic --dangerous".format(
upgrade_to)
else:
cmd = "sudo snap refresh microk8s --channel={}".format(upgrade_to)
run_until_success(cmd)
# Allow for the refresh to be processed
time.sleep(10)
wait_for_installation()
def validate_linkerd():
"""
Validate Linkerd by deploying emojivoto.
"""
if platform.machine() != 'x86_64':
print("Linkerd tests are only relevant in x86 architectures")
return
wait_for_installation()
wait_for_pod_state("",
"linkerd",
"running",
label="linkerd.io/control-plane-component=controller",
timeout_insec=300)
print("Linkerd controller up and running.")
wait_for_pod_state(
"",
"linkerd",
"running",
label="linkerd.io/control-plane-component=proxy-injector",
timeout_insec=300)
print("Linkerd proxy injector up and running.")
def test_upgrade(self):
"""
Deploy, probe, upgrade, validate nothing broke.
"""
print("Testing upgrade from {} to {}".format(upgrade_from, upgrade_to))
cmd = "sudo snap install microk8s --classic --channel={}".format(upgrade_from)
run_until_success(cmd)
wait_for_installation()
if is_container():
# In some setups (eg LXC on GCE) the hashsize nf_conntrack file under
# sys is marked as rw but any update on it is failing causing kube-proxy
# to fail.
here = os.path.dirname(os.path.abspath(__file__))
apply_patch = os.path.join(here, "patch-kube-proxy.sh")
check_call("sudo {}".format(apply_patch).split())
# Run through the validators and
# select those that were valid for the original snap
test_matrix = {}
try:
enable = microk8s_enable("dns")
wait_for_pod_state("", "kube-system", "running", label="k8s-app=kube-dns")
assert "Nothing to do for" not in enable
enable = microk8s_enable("dashboard")
assert "Nothing to do for" not in enable
validate_dns_dashboard()
test_matrix['dns_dashboard'] = validate_dns_dashboard
except CalledProcessError:
print('Will not test dns-dashboard')
try:
enable = microk8s_enable("storage")
assert "Nothing to do for" not in enable
validate_storage()
test_matrix['storage'] = validate_storage
except CalledProcessError:
print('Will not test storage')
try:
enable = microk8s_enable("ingress")
assert "Nothing to do for" not in enable
validate_ingress()
test_matrix['ingress'] = validate_ingress
except CalledProcessError:
print('Will not test ingress')
try:
enable = microk8s_enable("gpu")
assert "Nothing to do for" not in enable
validate_gpu()
test_matrix['gpu'] = validate_gpu
except CalledProcessError:
print('Will not test gpu')
try:
enable = microk8s_enable("registry")
assert "Nothing to do for" not in enable
validate_registry()
test_matrix['registry'] = validate_registry
except CalledProcessError:
print('Will not test registry')
try:
validate_forward()
test_matrix['forward'] = validate_forward
except CalledProcessError:
print('Will not test port forward')
try:
enable = microk8s_enable("metrics-server")
assert "Nothing to do for" not in enable
validate_metrics_server()
test_matrix['metrics_server'] = validate_metrics_server
except CalledProcessError:
print('Will not test the metrics server')
# AMD64 only tests
if platform.machine() == 'x86_64' and under_time_pressure == 'False':
"""
# Prometheus operator on our lxc is chashlooping disabling the test for now.
try:
enable = microk8s_enable("prometheus", timeout_insec=30)
assert "Nothing to do for" not in enable
validate_prometheus()
test_matrix['prometheus'] = validate_prometheus
except:
print('Will not test the prometheus')
# The kubeflow deployment is huge. It will not fit comfortably
# with the rest of the addons on the same machine during an upgrade
# we will need to find another way to test it.
try:
enable = microk8s_enable("kubeflow", timeout_insec=30)
assert "Nothing to do for" not in enable
validate_kubeflow()
test_matrix['kubeflow'] = validate_kubeflow
except:
print('Will not test kubeflow')
"""
try:
enable = microk8s_enable("fluentd", timeout_insec=30)
assert "Nothing to do for" not in enable
validate_fluentd()
test_matrix['fluentd'] = validate_fluentd
except CalledProcessError:
print('Will not test the fluentd')
try:
enable = microk8s_enable("jaeger", timeout_insec=30)
assert "Nothing to do for" not in enable
validate_jaeger()
test_matrix['jaeger'] = validate_jaeger
except CalledProcessError:
print('Will not test the jaeger addon')
# We are not testing cilium because we want to test the upgrade of the default CNI
"""
try:
enable = microk8s_enable("cilium", timeout_insec=300)
assert "Nothing to do for" not in enable
validate_cilium()
test_matrix['cilium'] = validate_cilium
except CalledProcessError:
print('Will not test the cilium addon')
"""
try:
ip_ranges = (
"192.168.0.105-192.168.0.105,192.168.0.110-192.168.0.111,192.168.1.240/28"
)
enable = microk8s_enable("{}:{}".format("metallb", ip_ranges), timeout_insec=500)
assert "MetalLB is enabled" in enable and "Nothing to do for" not in enable
validate_metallb_config(ip_ranges)
test_matrix['metallb'] = validate_metallb_config
except CalledProcessError:
print("Will not test the metallb addon")
# We will not be testing multus because it takes too long for cilium and multus
# to settle after the update and the multus test needs to be refactored so we do
# delete and recreate the networks configured.
"""
try:
enable = microk8s_enable("multus", timeout_insec=150)
assert "Nothing to do for" not in enable
validate_multus()
test_matrix['multus'] = validate_multus
except CalledProcessError:
print('Will not test the multus addon')
"""
# Refresh the snap to the target
if upgrade_to.endswith('.snap'):
cmd = "sudo snap install {} --classic --dangerous".format(upgrade_to)
else:
cmd = "sudo snap refresh microk8s --channel={}".format(upgrade_to)
run_until_success(cmd)
# Allow for the refresh to be processed
time.sleep(10)
wait_for_installation()
# Test any validations that were valid for the original snap
for test, validation in test_matrix.items():
print("Testing {}".format(test))
validation()
if not is_container():
# On lxc umount docker overlay is not permitted.
check_call("sudo snap remove microk8s".split())
def test_upgrade(self):
"""
Deploy, probe, upgrade, validate nothing broke.
"""
print("Testing upgrade from {} to {}".format(upgrade_from, upgrade_to))
cmd = "sudo snap install microk8s --classic --channel={}".format(upgrade_from)
run_until_success(cmd)
wait_for_installation()
if is_container():
# In some setups (eg LXC on GCE) the hashsize nf_conntrack file under
# sys is marked as rw but any update on it is failing causing kube-proxy
# to fail.
here = os.path.dirname(os.path.abspath(__file__))
apply_patch = os.path.join(here, "patch-kube-proxy.sh")
check_call("sudo {}".format(apply_patch).split())
# Run through the validators and
# select those that were valid for the original snap
test_matrix = {}
try:
enable = microk8s_enable("dns")
wait_for_pod_state("", "kube-system", "running", label="k8s-app=kube-dns")
assert "Nothing to do for" not in enable
enable = microk8s_enable("dashboard")
assert "Nothing to do for" not in enable
validate_dns_dashboard()
test_matrix['dns_dashboard'] = validate_dns_dashboard
except:
print('Will not test dns-dashboard')
try:
enable = microk8s_enable("storage")
assert "Nothing to do for" not in enable
validate_storage()
test_matrix['storage'] = validate_storage
except:
print('Will not test storage')
try:
enable = microk8s_enable("ingress")
assert "Nothing to do for" not in enable
validate_ingress()
test_matrix['ingress'] = validate_ingress
except:
print('Will not test ingress')
try:
enable = microk8s_enable("gpu")
assert "Nothing to do for" not in enable
validate_gpu()
test_matrix['gpu'] = validate_gpu
except:
print('Will not test gpu')
try:
enable = microk8s_enable("registry")
assert "Nothing to do for" not in enable
validate_registry()
test_matrix['registry'] = validate_registry
except:
print('Will not test registry')
try:
validate_forward()
test_matrix['forward'] = validate_forward
except:
print('Will not test port forward')
try:
enable = microk8s_enable("metrics-server")
assert "Nothing to do for" not in enable
validate_metrics_server()
test_matrix['metrics_server'] = validate_metrics_server
except:
print('Will not test the metrics server')
# AMD64 only tests
if platform.machine() == 'x86_64' and under_time_pressure == 'False':
'''
Prometheus operator on our lxc is chashlooping disabling the test for now.
try:
enable = microk8s_enable("prometheus", timeout_insec=30)
assert "Nothing to do for" not in enable
validate_prometheus()
test_matrix['prometheus'] = validate_prometheus
except:
print('Will not test the prometheus')
'''
try:
enable = microk8s_enable("fluentd", timeout_insec=30)
assert "Nothing to do for" not in enable
validate_fluentd()
test_matrix['fluentd'] = validate_fluentd
except:
print('Will not test the fluentd')
try:
enable = microk8s_enable("jaeger", timeout_insec=30)
assert "Nothing to do for" not in enable
validate_jaeger()
test_matrix['jaeger'] = validate_jaeger
except:
print('Will not test the jaeger addon')
# Refresh the snap to the target
if upgrade_to.endswith('.snap'):
cmd = "sudo snap install {} --classic --dangerous".format(upgrade_to)
else:
cmd = "sudo snap refresh microk8s --channel={}".format(upgrade_to)
run_until_success(cmd)
# Allow for the refresh to be processed
time.sleep(10)
wait_for_installation()
# Test any validations that were valid for the original snap
for test, validation in test_matrix.items():
print("Testing {}".format(test))
validation()
if not is_container():
# On lxc umount docker overlay is not permitted.
check_call("sudo snap remove microk8s".split())
def test_upgrade(self):
"""
Deploy, probe, upgrade, validate nothing broke.
"""
print("Testing upgrade from {} to {}".format(upgrade_from, upgrade_to))
cmd = "sudo snap install microk8s --classic --channel={}".format(
upgrade_from).split()
check_call(cmd)
wait_for_installation()
# Run through the validators and
# select those that were valid for the original snap
test_matrix = {}
try:
enable = microk8s_enable("dns")
wait_for_pod_state("",
"kube-system",
"running",
label="k8s-app=kube-dns")
assert "Nothing to do for" not in enable
validate_dns()
test_matrix['dns'] = validate_dns
except:
print('Will not test dns')
try:
enable = microk8s_enable("dashboard")
assert "Nothing to do for" not in enable
validate_dashboard()
test_matrix['dashboard'] = validate_dashboard
except:
print('Will not test dashboard')
try:
enable = microk8s_enable("storage")
assert "Nothing to do for" not in enable
validate_storage()
test_matrix['storage'] = validate_storage
except:
print('Will not test storage')
try:
enable = microk8s_enable("ingress")
assert "Nothing to do for" not in enable
validate_ingress()
test_matrix['ingress'] = validate_ingress
except:
print('Will not test ingress')
try:
enable = microk8s_enable("gpu")
assert "Nothing to do for" not in enable
validate_gpu()
test_matrix['gpu'] = validate_gpu
except:
print('Will not test gpu')
try:
enable = microk8s_enable("registry")
assert "Nothing to do for" not in enable
validate_registry()
test_matrix['registry'] = validate_registry
except:
print('Will not test registry')
try:
validate_forward()
test_matrix['forward'] = validate_forward
except:
print('Will not test port forward')
try:
enable = microk8s_enable("metrics-server")
assert "Nothing to do for" not in enable
validate_metrics_server()
test_matrix['metrics_server'] = validate_metrics_server
except:
print('Will not test the metrics server')
# Refresh the snap to the target
if upgrade_to.endswith('.snap'):
cmd = "sudo snap install {} --classic --dangerous".format(
upgrade_to).split()
else:
cmd = "sudo snap refresh microk8s --channel={}".format(
upgrade_to).split()
check_call(cmd)
# Allow for the refresh to be processed
time.sleep(10)
wait_for_installation()
# Test any validations that were valid for the original snap
for test, validation in test_matrix.items():
print("Testing {}".format(test))
validation()
# On lxc umount docker overlay is not permitted.
try:
check_call(
"sudo grep -E (lxc|hypervisor) /proc/1/environ /proc/cpuinfo".
split())
except CalledProcessError:
check_call("sudo snap remove microk8s".split())
def test_upgrade(self):
"""
Deploy, probe, upgrade, validate nothing broke.
"""
print("Testing upgrade from {} to {}".format(upgrade_from, upgrade_to))
cmd = "sudo snap install microk8s --classic --{}".format(
upgrade_from).split()
check_call(cmd)
wait_for_installation()
# Run through the validators and
# select those that were valid for the original snap
test_matrix = {}
try:
microk8s_enable("dns")
wait_for_pod_state("",
"kube-system",
"running",
label="k8s-app=kube-dns")
validate_dns()
test_matrix['dns'] = validate_dns
except:
print('Will not test dns')
try:
microk8s_enable("dashboard")
validate_dashboard()
test_matrix['dashboard'] = validate_dashboard
except:
print('Will not test dashboard')
try:
microk8s_enable("storage")
validate_storage()
test_matrix['storage'] = validate_storage
except:
print('Will not test storage')
try:
microk8s_enable("ingress")
validate_ingress()
test_matrix['ingress'] = validate_ingress
except:
print('Will not test ingress')
# Refresh the snap to the target
if upgrade_to.endswith('.snap'):
cmd = "sudo snap install {} --classic --dangerous".format(
upgrade_to).split()
else:
cmd = "sudo snap refresh microk8s --{}".format(upgrade_to).split()
check_call(cmd)
# Allow for the refresh to be processed
time.sleep(10)
wait_for_installation()
# Test any validations that were valid for the original snap
for test, validation in test_matrix.items():
print("Testing {}".format(test))
validation()
cmd = "sudo snap remove microk8s".split()
check_call(cmd)