def verifyULConnectivity(object, tepList):
tutils.tcLog("Create pods in host ns")
pods = tutils.createDs("ul-tester")
for pod in pods:
for tep in tepList:
tutils.verifyPing(pod, "default", tep)
tutils.tcLog("Delete pods in host ns")
tutils.deleteDs("ul-tester")
def countCompHv(object):
vmm_name = object.config["aci_config"]["system_id"]
tutils.tcLog("Count compHv objects for {}".format(vmm_name))
qry = '/api/mo/comp/prov-Kubernetes/ctrlr-[{}]-{}.json?query-target=children&target-subtree-class=compHv'.format(
vmm_name, vmm_name)
resp = object.apic.get(path=qry)
resJson = json.loads(resp.content)
return len(resJson["imdata"])
def verifyOLConnectivity(object, vmIP):
tutils.tcLog("Create pods in epg-a")
pods = tutils.createDs("ol-tester")
object.verifyCapicEPs(pods, "epg-a")
for pod in pods:
tutils.verifyPing(pod, "default", vmIP)
tutils.tcLog("Delete pods in epg-a")
tutils.deleteDs("ol-tester")
def getCniTep(object):
tutils.tcLog("Get CniTep objects")
qry = '/api/class/hcl3CniTep.json'
resp = object.apic.get(path=qry)
resJson = json.loads(resp.content)
tepList = []
for mo in resJson['imdata']:
addr = mo['hcl3CniTep']['attributes']['addr']
tep = addr.split("/")[0]
tepList.append(tep)
return tepList
def verifyClusterInfo(object):
vmm_name = object.config["aci_config"]["system_id"]
tn_name = object.config["aci_config"]["tenant"]["name"]
tutils.tcLog("Verify cluster info objects for {}/{}".format(
tn_name, vmm_name))
moClasses = ["compClusterInfo", "hcloudClusterInfo"]
for moCl in moClasses:
qry = '/api/class/{}.json?query-target-filter=and(eq({}.name,"{}"),eq({}.accountName,"{}"))'.format(
moCl, moCl, moCl, vmm_name, tn_name)
resp = object.apic.get(path=qry)
resJson = json.loads(resp.content)
assert len(resJson["imdata"]) == 1
def getVmIP(object, epg):
tutils.tcLog("Get vm ip for epg {}".format(epg))
qry = '/api/class/hcloudRtEpToSecurityGroup.json?query-target-filter=and(wcard(hcloudRtEpToSecurityGroup.dn,"{}"))'.format(
epg)
resp = object.apic.get(path=qry)
epDns = object.getFromResp(resp, 'hcloudRtEpToSecurityGroup', 'tDn')
assert len(epDns) > 0
print("epDn: {}".format(epDns[0]))
qry = '/api/mo/{}.json'.format(epDns[0])
resp = object.apic.get(path=qry)
epIPs = object.getFromResp(resp, 'hcloudEndPoint', 'primaryIpV4Addr')
assert len(epIPs) > 0
return epIPs[0]
def getAllEpIPs(object, epg):
tutils.tcLog("Get all hcloudEP ip for epg {}".format(epg))
ip_dict = {}
qry = '/api/class/hcloudRtEpToSecurityGroup.json?query-target-filter=and(wcard(hcloudRtEpToSecurityGroup.dn,"{}]"))'.format(
epg)
resp = object.apic.get(path=qry)
epDns = object.getFromResp(resp, 'hcloudRtEpToSecurityGroup', 'tDn')
for epDn in epDns:
qry = '/api/mo/{}.json'.format(epDn)
resp = object.apic.get(path=qry)
epIPs = object.getFromResp(resp, 'hcloudEndPoint',
'primaryIpV4Addr')
if len(epIPs) > 0:
ep_ip = epIPs[0].split("/")[0]
ip_dict[ep_ip] = True
return ip_dict
def verifyCsr(object, csr, tunnelInfo, vrf_encap_id):
dn = csr["hcloudCsr"]["attributes"]["dn"]
qry = '/api/mo/{}.json?query-target=subtree&target-subtree-class=hcloudEndPointOper&query-target-filter=and(ne(hcloudEndPointOper.publicIpv4Addr,"0.0.0.0"))'.format(
dn)
resp = object.apic.get(path=qry)
resJson = json.loads(resp.content)
def getCsrIP():
for epOper in resJson["imdata"]:
if "hcloudEndPointOper" in epOper:
att = epOper["hcloudEndPointOper"]["attributes"]
operDn = att["dn"]
if "nwif-0" in operDn:
print("CSR IP: {}".format(att["publicIpv4Addr"]))
return att["publicIpv4Addr"]
return None
csrIP = getCsrIP()
assert csrIP is not None
print(csrIP)
csr_c = paramiko.SSHClient()
csr_c.load_system_host_keys()
csr_c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
pw = object.config["aci_config"]["apic_login"]["password"]
csr_c.connect(csrIP, username='******', password=pw)
stdin, stdout, stderr = csr_c.exec_command(
'show run | sec interface Tunn')
# format output
out_dict = {}
key = ""
for line in stdout:
ll = line.strip('\n')
#print(ll)
if "interface Tunnel" in ll:
key = ll.split()[1]
out_dict[key] = {}
else:
if key:
out_dict[key][ll.strip()] = True
csr_c.close()
tutils.tcLog("Verify tunnels {} is in CSR".format(tunnelInfo))
object.verifyTunnels(out_dict, tunnelInfo, vrf_encap_id)
tutils.tcLog("Verify routes in CSR({})".format(csrIP))
object.verifyCsrRoutes(csr_c, csrIP)
def test_concrete(object):
object.setup()
addr = object.config["aci_config"]["apic_hosts"][0]
tutils.tcLog("Verify capic {} access".format(addr))
assert object.apic is not None
object.verifyClusterInfo()
nodeCount = object.countCompHv()
print("nodeCount: {}".format(nodeCount))
csrList = object.getCSRs()
csrCount = len(csrList)
print("csrCount: {}".format(csrCount))
if nodeCount == 0 or csrCount == 0:
print("No further tests possible with these csr/compHv count")
return
tepList = object.getCniTep()
assert len(tepList) > 0
tutils.tcLog(
"Verify connectivity from host ns to teps {}".format(tepList))
object.verifyULConnectivity(tepList)
tunnelInfo = object.getTunnelInfo()
print(tunnelInfo)
vrf_encap_id = tutils.read_vrf_encap_id()
print("vrf-encap-id is {}".format(vrf_encap_id))
for csr in csrList:
object.verifyCsr(csr, tunnelInfo, vrf_encap_id)
vmIP = object.getVmIP("test-vm-epg")
print("vmIP: {}".format(vmIP))
assert vmIP
tutils.tcLog("Verify connectivity from epg-a to {}".format(vmIP))
object.verifyOLConnectivity(vmIP)
def test_nslookup(object):
v1 = client.CoreV1Api()
# use alpine because busybox nslookup seems broken
tutils.tcLog("Create a dns client pod")
createPod("alpine-pod")
s = v1.read_namespaced_service("kubernetes", "default")
svcIP = s.spec.cluster_ip
# verify ping fails across epgs
print("\nVerify nslookup")
tutils.tcLog("Verify dns lookup of kubernetes service")
ns_lookup_cmd = ['nslookup', 'kubernetes']
def respChecker():
logging.debug("===svcIP is {}".format(svcIP))
resp = stream(v1.connect_get_namespaced_pod_exec, "alpine-pod", 'default',
command=ns_lookup_cmd, stderr=True, stdin=False, stdout=True, tty=False)
if svcIP in resp:
logging.debug("=>command {}".format(ns_lookup_cmd))
logging.debug("=>Resp is {}".format(resp))
return ""
else:
logging.debug("NR=>Resp is {}".format(resp))
#return ""
#FIXME this is not reliable yet...
return "svc not resolved {}".format(ns_lookup_cmd)
# Basic inspection in case of a dns test failer
def dnsInspector():
tutils.inspectLog("Checking coredns status on failure")
av1 = client.AppsV1Api()
sr = av1.read_namespaced_deployment_status("coredns", "kube-system")
tutils.inspectLog("CoreDNS available: {}, ready: {}".format(sr.status.available_replicas, sr.status.ready_replicas))
tutils.assertEventually(respChecker, 1, 5, dnsInspector)
tutils.tcLog("Create a service and check dns resolution")
ns_lookup_cmd = ['nslookup', 'dns-test-svc']
svcIP = createNsSvc("default", "dns-test-svc")
logging.info("***svcIP is {}".format(svcIP))
tutils.assertEventually(respChecker, 1, 60)
v1.delete_namespaced_pod("alpine-pod", "default", client.V1DeleteOptions())
v1.delete_namespaced_service("dns-test-svc", "default", client.V1DeleteOptions())
def test_ovs_restart(object):
tutils.tcLog("Getting list of ovs pods")
node_list = v1.list_node()
ns = tutils.getSysNs()
pod_list = v1.list_namespaced_pod(ns, label_selector="name=aci-containers-openvswitch")
assert len(node_list.items) > 0
assert len(node_list.items) == len(pod_list.items)
tutils.tcLog("Delete current ovs pods")
for pod in pod_list.items:
tutils.deletePod(ns, pod.metadata.name)
tutils.tcLog("Check new ovs pods")
def checker():
new_pod_list = v1.list_namespaced_pod(ns, label_selector="name=aci-containers-openvswitch")
if len(pod_list.items) == len(new_pod_list.items):
return ""
return "{} ne {}".format(len(new_pod_list.items), len(pod_list.items))
tutils.assertEventually(checker, 2, 30)
def test_policy(object):
v1 = client.CoreV1Api()
createCRD("contracts", "tcp-6020")
createCRD("contracts", "tcp-6021")
sleep(1)
createCRD("epgs", "epg-a")
createCRD("epgs", "epg-b")
# pod in epg-a
createPod("pod-a")
# pods in epg-b
ip_6020 = createPod("pod-b6020")
ip_6021 = createPod("pod-b6021")
tutils.checkAgentLog()
# verify ping fails across epgs
tutils.tcLog("Verify ping failure across epgs")
ping_cmd = ['ping', '-c', '6', '-t', '1', ip_6020]
resp = stream(v1.connect_get_namespaced_pod_exec,
"pod-a",
'default',
command=ping_cmd,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("=>Resp is {}".format(resp))
assert "0 packets received" in resp
tutils.tcLog("Verify tcp contract")
# verify port 6020 access from pod-a to epg-b
tutils.tcLog("port 6020 access from pod-a to epg-b")
cmd1 = ['nc', '-zvnw', '1', ip_6020, '6020']
resp = stream(v1.connect_get_namespaced_pod_exec,
"pod-a",
'default',
command=cmd1,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("=>pod-a to epg-b[6020] Resp is {}".format(resp))
assert "open" in resp
sleep(5)
# verify port 6021 is inaccessible from pod-a to epg-b
tutils.tcLog("port 6021 deny from pod-a to epg-b")
cmd2 = ['nc', '-zvnw', '1', ip_6021, '6021']
resp = stream(v1.connect_get_namespaced_pod_exec,
"pod-a",
'default',
command=cmd2,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("=>pod-a to epg-b[6021] Resp is {}".format(resp))
assert "timed out" in resp
sleep(5)
# verify port 6021 is accessible within epg-b
tutils.tcLog("port 6021 accessible within epg-b")
resp = stream(v1.connect_get_namespaced_pod_exec,
"pod-b6020",
'default',
command=cmd2,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("=>pod-b6020 to pod-b[6021] Resp is {}".format(resp))
assert "open" in resp
tutils.tcLog("Change allowed port to 6021")
replaceCRD("epgs", "epg-a", "epg-a-upd")
replaceCRD("epgs", "epg-b", "epg-b-upd")
tutils.tcLog("Verify new contract is on agents")
def contractChecker():
resp = tutils.verifyAgentContracts(["GbpeL24Classifier/tcp-6021"],
True)
if resp == "":
print("GbpeL24Classifier/tcp-6021 present")
else:
print(resp)
return resp
tutils.assertEventually(contractChecker, 1, 10)
sleep(5)
tutils.tcLog("port 6021 now allowed from pod-a to epg-b")
cmd2 = ['nc', '-zvnw', '1', ip_6021, '6021']
resp = stream(v1.connect_get_namespaced_pod_exec,
"pod-a",
'default',
command=cmd2,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("=>pod-a to epg-b[6021] Resp is {}".format(resp))
assert "open" in resp
def contractRemChecker():
resp = tutils.verifyAgentContracts([
"GbpEpGroupToConsContractRSrc/288/tcp-6020",
"GbpEpGroupToProvContractRSrc/288/tcp-6020"
], False)
if resp == "":
print(
"GbpEpGroupToConsContractRSrc/288/tcp-6020, GbpEpGroupToProvContractRSrc/288/tcp-6020 removed"
)
else:
print(resp)
return resp
tutils.assertEventually(contractRemChecker, 1, 10)
tutils.tcLog("port 6020 now denied from pod-a to epg-b")
cmd1 = ['nc', '-zvnw', '1', ip_6020, '6020']
resp = stream(v1.connect_get_namespaced_pod_exec,
"pod-a",
'default',
command=cmd1,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("=>pod-a to epg-b[6020] Resp is {}".format(resp))
assert "timed out" in resp
toDelete = ["pod-a", "pod-b6020", "pod-b6021"]
logging.info("Deleting {}\n".format(toDelete))
for pod in toDelete:
v1.delete_namespaced_pod(pod, "default", client.V1DeleteOptions())
for pod in toDelete:
tutils.checkPodDeleted(v1, "default", pod, 120)
deleteCRD("contracts", "tcp-6020")
deleteCRD("contracts", "tcp-6021")
deleteCRD("epgs", "epg-a")
deleteCRD("epgs", "epg-b")
tutils.checkAgentLog()
def test_np(object):
tutils.tcLog("Verify GW flows are present")
tutils.checkGwFlows("11.3.0.1")
tutils.tcLog("Create a namespace, with simple service")
createNs("e2e")
createNsPod("e2e", "srvr-80")
createNsPod("e2e", "srvr-81")
svcIP = createNsSvc("e2e", "simple-svc")
tutils.tcLog("Create two client pods")
client_pods = ["client-a", "client-b"]
for pod in client_pods:
createNsPod("e2e", pod)
tutils.tcLog("Verify access without netpol")
for pod in client_pods:
verifyAccess("e2e", pod, svcIP, "80", "open")
verifyAccess("e2e", pod, svcIP, "81", "open")
tutils.tcLog("Delete clients")
for pod in client_pods:
tutils.deletePod("e2e", pod)
tutils.tcLog("Apply network policy client-a->port80")
createNsNetPol("e2e", "upd_policy1")
tutils.tcLog("Create two client pods")
for pod in client_pods:
createNsPod("e2e", pod)
tutils.tcLog("Verify client-a can access port 80")
verifyAccess("e2e", "client-a", svcIP, "80", "open")
tutils.tcLog("Verify failures for a/81, b/[80,81]")
verifyAccess("e2e", "client-a", svcIP, "81", "timed out")
verifyAccess("e2e", "client-b", svcIP, "80", "timed out")
verifyAccess("e2e", "client-b", svcIP, "81", "timed out")
tutils.tcLog("Delete clients")
for pod in client_pods:
tutils.deletePod("e2e", pod)
tutils.tcLog("Update network policy client-b->port81")
replaceNsNetPol("e2e", "upd_policy2")
tutils.tcLog("Create two client pods")
for pod in client_pods:
createNsPod("e2e", pod)
tutils.tcLog("Verify client-b can access port 81")
verifyAccess("e2e", "client-b", svcIP, "81", "open")
tutils.tcLog("Verify failures for b/81, a/[80,81]")
verifyAccess("e2e", "client-b", svcIP, "80", "timed out")
verifyAccess("e2e", "client-a", svcIP, "80", "timed out")
verifyAccess("e2e", "client-a", svcIP, "81", "timed out")
v1 = client.CoreV1Api()
v1.delete_namespace("e2e", client.V1DeleteOptions())
def nsDelChecker():
if tutils.namespaceExists("e2e"):
return "e2e exists"
return ""
tutils.tcLog("Verify ns is deleted")
tutils.assertEventually(nsDelChecker, 2, 40)
def test_default(object):
tutils.tcLog("Verify GW flows are present")
tutils.checkGwFlows("11.3.0.1")
delete_rc = True
k8s_api = client.CoreV1Api()
if not checkRCStatus(k8s_api, 3):
logging.debug("Using existing rc")
delete_rc = False
else:
tutils.tcLog("Create 3 pods in default epg")
k8s_api = utils.create_from_yaml(k8s_client, "yamls/busybox.yaml")
# check rc is ready
def rcChecker():
return checkRCStatus(k8s_api, 3)
tutils.assertEventually(rcChecker, 1, 60)
tutils.checkAgentLog()
# check pods are ready
def podChecker():
pod_list = k8s_api.list_namespaced_pod(
"default", label_selector="app=busybox")
if len(pod_list.items) < 3:
return "Expected 3 pods, got {}".format(len(pod_list.items))
for pod in pod_list.items:
if pod.status.phase != "Running":
return "pod {} status is {}".format(
pod.metadata.name, pod.status.phase)
return ""
tutils.assertEventually(podChecker, 1, 60)
# verify connectivity
pod_ips = tutils.getPodIPs("default", "app=busybox")
tutils.tcLog("Verify EPs are populated")
def remEPChecker():
res = tutils.verifyAgentEPs(pod_ips)
if res == "":
print("{} present".format(pod_ips))
else:
print(res)
return res
tutils.assertEventually(remEPChecker, 1, 10)
node_ips = getNodeIPs(k8s_api)
ips = pod_ips + node_ips
print("ip's are: {}".format(ips))
pod_list = k8s_api.list_namespaced_pod("default",
label_selector="app=busybox")
pod = next(iter(pod_list.items), None)
assert pod != None
tutils.tcLog("Verify default connectivity")
def pingChecker():
ping_cmd = ['ping', '-c', '3']
for ip in ips:
v1 = client.CoreV1Api()
cmd = list(ping_cmd)
cmd.append(ip)
resp = stream(v1.connect_get_namespaced_pod_exec,
pod.metadata.name,
'default',
command=cmd,
stderr=True,
stdin=False,
stdout=True,
tty=False)
print("=>Resp is {}".format(resp))
if "3 packets received" not in resp:
return "3 packets not received"
return ""
tutils.assertEventually(pingChecker, 1, 60)
tutils.tcLog("Check pods access to API server")
v1 = client.CoreV1Api()
s = v1.read_namespaced_service("kubernetes", "default")
svcIP = s.spec.cluster_ip
logging.debug("=>K8s svc IP is {}".format(svcIP))
pod_list = k8s_api.list_namespaced_pod("default",
label_selector="app=busybox")
cmd2 = ['nc', '-zvnw', '1', svcIP, '443']
def k8sChecker():
for pod in pod_list.items:
resp = stream(v1.connect_get_namespaced_pod_exec,
pod.metadata.name,
'default',
command=cmd2,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("=>Resp is {}".format(resp))
if "open" not in resp:
return resp
return ""
tutils.assertEventually(k8sChecker, 1, 10)
if delete_rc:
tutils.tcLog("Delete pods")
tutils.scaleRc("busybox", 0)
k8s_api.delete_namespaced_replication_controller(
"busybox", "default", client.V1DeleteOptions())
else:
logging.debug("Skipping rc delete")
def test_np(object):
tutils.tcLog("Verify GW flows are present")
tutils.checkGwFlows("11.3.0.1")
# create a service/deployment in default namespace
svcIP = createNsSvc("default", "hostnames-svc")
k8s_api = utils.create_from_yaml(k8s_client,
"yamls/hostnames-dep.yaml")
v1 = client.CoreV1Api()
# check deployment is ready
svc_hosts = []
def depChecker():
svc_hosts = getPodNames(v1, "default", "app=hostnames")
if len(svc_hosts) >= 3:
return ""
return "Need 3 hosts, have {}".format(len(svc_hosts))
tutils.assertEventually(depChecker, 1, 180)
# create two namespaces, with a client pod in each
nsList = ["prod", "dev"]
for ns in nsList:
createNs(ns)
logging.info("\nWaiting for namespaces to be available")
sleep(3)
for ns in nsList:
createNsPod(ns, "client-pod")
tutils.tcLog("Verify loadbalancing from dev and prod")
# verify both clients can access the service, and the service load balances.
for ns in nsList:
cmd = ['curl', '--connect-timeout', '1', '-s', svcIP]
backends = dict()
for count in range(0, 12):
resp = stream(v1.connect_get_namespaced_pod_exec,
"client-pod",
ns,
command=cmd,
stderr=True,
stdin=False,
stdout=True,
tty=False)
if resp == "":
sleep(1)
continue
backends[resp] = True
if len(backends) == 2:
break
lbWorkaround()
#assert len(backends) == 2
logging.info("backends: {}".format(backends.keys()))
tutils.tcLog("Verify there are no timeouts")
nc_cmd = ['nc', '-zvnw', '1', svcIP, '80']
for ns in nsList:
for count in range(0, 6):
resp = stream(v1.connect_get_namespaced_pod_exec,
"client-pod",
ns,
command=nc_cmd,
stderr=True,
stdin=False,
stdout=True,
tty=False)
if 'open' not in resp:
print("Error - {}, expected open".format(resp))
assert (False)
lbWorkaround()
# apply networkpolicy allowing access only to prod
tutils.tcLog("Create k8s network policy")
createNsNetPol("default", "hostnames-allow-prod")
sleep(10)
cmd = ['curl', '--connect-timeout', '1', svcIP]
# wait for netpol to take effect
def waiter():
count = 0
for ix in range(0, 10):
resp1 = stream(v1.connect_get_namespaced_pod_exec,
"client-pod",
"dev",
command=cmd,
stderr=True,
stdin=False,
stdout=True,
tty=False)
if "timed out" in resp1:
count += 1
else:
sleep(1)
if count == 10:
return ""
return "still accessible"
tutils.tcLog("Verify dev cannot access the svc")
tutils.assertEventually(waiter, 1, 120)
tutils.tcLog("Verify prod can access the svc")
# verify prod can access the svc
cmd2 = ['nc', '-zvnw', '1', svcIP, '80']
def prodChecker():
for ix in range(0, 3):
resp2 = stream(v1.connect_get_namespaced_pod_exec,
"client-pod",
"prod",
command=cmd2,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("prod: {}".format(resp2))
if "open" not in resp2:
return resp2
lbWorkaround()
return ""
clientIP = tutils.getPodIP("client-pod", "prod")
ovs_pods = getPodNames(v1, "kube-system",
"name=aci-containers-openvswitch")
ovs_cmd = ['ovs-ofctl', 'dump-flows', 'br-access', '-OOpenFlow13']
def npInspector():
tutils.inspectLog(
"Look for flows on br-access for {}".format(clientIP))
for ovs_pod in ovs_pods:
ovs_resp = stream(v1.connect_get_namespaced_pod_exec,
ovs_pod,
"kube-system",
command=ovs_cmd,
stderr=True,
stdin=False,
stdout=True,
tty=False)
print("ovs_Pod: {}\n".format(ovs_pod))
for line in ovs_resp.splitlines():
if clientIP in line:
print(line)
tutils.assertEventually(prodChecker, 1, 5, npInspector)
tutils.tcLog("Verify dev still can't access the svc")
# and dev can't
for ix in range(0, 3):
resp3 = stream(v1.connect_get_namespaced_pod_exec,
"client-pod",
"dev",
command=cmd2,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("dev: {}".format(resp3))
assert "timed out" in resp3
lbWorkaround()
# delete everything
nv1 = client.NetworkingV1Api()
nv1.delete_namespaced_network_policy("hostnames-allow-prod", "default",
client.V1DeleteOptions())
# verify access-br flows are deleted
def flowChecker():
for ovs_pod in ovs_pods:
ovs_resp = stream(v1.connect_get_namespaced_pod_exec,
ovs_pod,
"kube-system",
command=ovs_cmd,
stderr=True,
stdin=False,
stdout=True,
tty=False)
for line in ovs_resp.splitlines():
if clientIP in line:
return line
return ""
tutils.tcLog("Verify network policy flows are deleted")
tutils.assertEventually(flowChecker, 1, 120)
tutils.scaleDep("default", "hostnames-dep", 0)
for ns in nsList:
v1.delete_namespace(
ns, client.V1DeleteOptions()) # deletes the client-pod too
v1.delete_namespaced_service("hostnames-svc", "default",
client.V1DeleteOptions())
av1 = client.AppsV1Api()
av1.delete_namespaced_deployment("hostnames-dep", "default",
client.V1DeleteOptions())
def delChecker():
dList = av1.list_namespaced_deployment("default")
for dep in dList.items:
if dep.metadata.name == "hostnames-dep":
return "hostnames-dep still present"
sList = v1.list_namespaced_service("default")
for svc in sList.items:
if svc.metadata.name == "hostnames-svc":
return "hostnames-svc still present"
for ns in nsList:
if tutils.namespaceExists(ns):
return "{} exists".format(ns)
return ""
tutils.tcLog("Verify svc/dep is deleted")
tutils.assertEventually(delChecker, 2, 40)
def getCSRs(object):
tutils.tcLog("Get csr objects")
qry = '/api/class/hcloudCsr.json'
resp = object.apic.get(path=qry)
resJson = json.loads(resp.content)
return resJson["imdata"]
def test_cni_status(object):
tutils.tcLog("Verify status of system pods")
assertEventually(checkPodStatus, 1, 90)
tutils.checkAgentLog()
def test_default(object):
tutils.tcLog("Create 3 pods in default epg")
k8s_api = utils.create_from_yaml(k8s_client, "yamls/busybox.yaml")
# check rc is ready
def rcChecker():
return checkRCStatus(k8s_api, 3)
tutils.assertEventually(rcChecker, 1, 60)
tutils.checkAgentLog()
# check pods are ready
def podChecker():
pod_list = k8s_api.list_namespaced_pod(
"default", label_selector="app=busybox")
if len(pod_list.items) != 3:
return "Expected 3 pods, got {}".format(len(pod_list.items))
for pod in pod_list.items:
if pod.status.phase != "Running":
return "pod {} status is {}".format(
pod.metadata.name, pod.status.phase)
return ""
tutils.assertEventually(podChecker, 1, 60)
# verify connectivity
ips = tutils.getPodIPs("default", "app=busybox")
pod_list = k8s_api.list_namespaced_pod("default",
label_selector="app=busybox")
pod = next(iter(pod_list.items), None)
assert pod != None
tutils.tcLog("Verify default connectivity")
def pingChecker():
ping_cmd = ['ping', '-c', '3']
for ip in ips:
v1 = client.CoreV1Api()
cmd = list(ping_cmd)
cmd.append(ip)
resp = stream(v1.connect_get_namespaced_pod_exec,
pod.metadata.name,
'default',
command=cmd,
stderr=True,
stdin=False,
stdout=True,
tty=False)
logging.debug("=>Resp is {}".format(resp))
if "3 packets received" not in resp:
return "3 packets not received"
return ""
tutils.assertEventually(pingChecker, 1, 60)
tutils.tcLog("Delete pods")
tutils.scaleRc("busybox", 0)
k8s_api.delete_namespaced_replication_controller(
"busybox", "default", client.V1DeleteOptions())
def test_np(object):
tutils.tcLog("Verify GW flows are present")
tutils.checkGwFlows("11.3.0.1")
createNs("prod")
# create a deployment in default namespace
k8s_api = utils.create_from_yaml(k8s_client,
"yamls/hostnames-dep.yaml")
v1 = client.CoreV1Api()
# check deployment is ready
svc_hosts = []
def depChecker():
svc_hosts = getPodNames(v1, "default", "app=hostnames")
if len(svc_hosts) >= 3:
return ""
return "Need 3 hosts, have {}".format(len(svc_hosts))
tutils.assertEventually(depChecker, 1, 180)
createNsPod("default", "client-pod")
tutils.tcLog("Verify access without netpol")
podIPs = tutils.getPodIPs("default", "app=hostnames")
assert len(podIPs) == 3
verifyAccess('default', 'open', podIPs)
# apply networkpolicy allowing access only to prod
tutils.tcLog("Create k8s network policy")
createNsNetPol("default", "np/hostnames-deny-all")
tutils.tcLog("Verify access denied")
verifyAccess('default', 'timed out', podIPs)
tutils.tcLog("Setup a netpol to allow access from prod")
createNsNetPol("default", "hostnames-allow-prod")
createNsPod("prod", "client-pod")
tutils.tcLog("Verify allow from prod")
verifyAccess('prod', 'open', podIPs)
tutils.tcLog("Verify deny from default")
verifyAccess('default', 'timed out', podIPs)
tutils.tcLog("Delete deny-all")
deleteNsNetPol('default', 'hostnames-deny-all')
tutils.tcLog("Verify allow from prod")
verifyAccess('prod', 'open', podIPs)
tutils.tcLog("Verify deny from default")
verifyAccess('default', 'timed out', podIPs)
tutils.tcLog("Delete allow-prod")
deleteNsNetPol('default', 'hostnames-allow-prod')
tutils.tcLog("Verify allow from default")
verifyAccess('default', 'open', podIPs)
tutils.tcLog("Verify allow from prod")
verifyAccess('prod', 'open', podIPs)
tutils.tcLog("Allow port 9000 from prod")
createNsNetPol("default", "np/hostnames-allow-prod9000")
tutils.tcLog("Verify deny from default")
verifyAccess('default', 'timed out', podIPs)
tutils.tcLog("Verify deny 9376 from prod")
verifyAccess('prod', 'timed out', podIPs)
tutils.tcLog("Delete allow-prod9000")
deleteNsNetPol('default', 'hostnames-allow-prod9000')
tutils.tcLog("Allow port 9376 from prod")
createNsNetPol("default", "np/hostnames-allow-prod9376")
tutils.tcLog("Verify allow from prod")
verifyAccess('prod', 'open', podIPs)
tutils.tcLog("Verify deny from default")
verifyAccess('default', 'timed out', podIPs)
tutils.tcLog("Delete allow-prod9376")
deleteNsNetPol('default', 'hostnames-allow-prod9376')
tutils.tcLog("Verify allow from default")
verifyAccess('default', 'open', podIPs)
tutils.tcLog("Verify allow from prod")
verifyAccess('prod', 'open', podIPs)
tutils.scaleDep("default", "hostnames-dep", 0)
v1.delete_namespace(
'prod', client.V1DeleteOptions()) # deletes the client-pod too
av1 = client.AppsV1Api()
av1.delete_namespaced_deployment("hostnames-dep", "default",
client.V1DeleteOptions())
tutils.deletePod("default", "client-pod")
def delChecker():
dList = av1.list_namespaced_deployment("default")
for dep in dList.items:
if dep.metadata.name == "hostnames-dep":
return "hostnames-dep still present"
if tutils.namespaceExists('prod'):
return "prod exists"
return ""
tutils.tcLog("Verify cleanup")
tutils.assertEventually(delChecker, 2, 40)
def test_kafka(object):
# setup kafka services
tutils.tcLog("Setup kafka services")
for ky in kafkaYamls:
try:
utils.create_from_yaml(k8s_client, "yamls/"+ky)
except ApiException as e:
logging.debug("{} - ignored".format(e.reason))
k8s_api = client.CoreV1Api()
# check kafka server is ready
assertPodReady("default", "ut-kafka-0", 120)
assertPodReady("default", "kafkakv", 120)
sleep(5)
# collect the current list of ep's from k8s
tutils.tcLog("Collect ep's from k8s")
initialEPList = readCniEPList()
logging.info("EPList is {}".format(initialEPList))
tutils.tcLog("Verifying initial EPList with kafka")
kafkaSyncChecker(initialEPList)
tutils.tcLog("Adding a pod and checking it in kafka")
utils.create_from_yaml(k8s_client, "yamls/alpine-pod.yaml")
assertPodReady("default", "alpine-pod", 45)
vtep = tutils.getPodNodeIP("alpine-pod", "default")
tutils.tcLog("Check for podif")
crdApi = client.CustomObjectsApi()
group = "aci.aw"
ns = "kube-system"
podifName = "default.alpine-pod.{}".format(vtep)
p = crdApi.get_namespaced_custom_object(group, "v1", ns, "podifs", podifName)
logging.debug("podif: {}".format(p))
epStatus = p['status']
epID = "{}.{}.{}".format(epStatus['podns'], epStatus['podname'], epStatus['ifname'])
toCheck = []
toCheck.append(epID)
kafkaChecker(toCheck, "found")
tutils.tcLog("Delete a pod and check removal from kafka")
k8s_api.delete_namespaced_pod("alpine-pod", "default", client.V1DeleteOptions())
kafkaChecker(toCheck, "missing", 8)
tutils.tcLog("Recheck ep sync between k8s and kafka")
kafkaSyncChecker(initialEPList)
tutils.tcLog("Add more endpoints")
utils.create_from_yaml(k8s_client, "yamls/busybox.yaml")
tutils.rcCheckScale("busybox", 3)
tutils.tcLog("Get new eplist from k8s")
EPList1 = readCniEPList()
assert EPList1 != initialEPList
tutils.tcLog("Recheck ep sync between k8s and kafka")
kafkaSyncChecker(EPList1)
tutils.tcLog("Bring controller down")
scaleDep("kube-system", "aci-containers-controller", 0)
sleep(10)
tutils.tcLog("Change some endpoints")
tutils.scaleRc("busybox", 0)
tutils.checkPodsRemoved("app=busybox")
EPList2 = readCniEPList()
assert EPList2 != EPList1
tutils.scaleRc("busybox", 2)
sleep(5)
logging.debug("previous ep list:{}".format(EPList1))
EPList3 = readCniEPList()
logging.debug("new ep list:{}".format(EPList3))
assert EPList3 != EPList1
tutils.tcLog("Bring controller up")
scaleDep("kube-system", "aci-containers-controller", 1)
sleep(10)
tutils.tcLog("Verify controller is up")
tutils.assertEventually(checkSystemStatus, 1, 60)
tutils.tcLog("Check ep sync again")
kafkaSyncChecker(EPList3, 6)
tutils.tcLog("Clean up added endpoints")
tutils.scaleRc("busybox", 0)
tutils.checkPodsRemoved("app=busybox")
k8s_api.delete_namespaced_replication_controller("busybox", "default", client.V1DeleteOptions())
