def test_np(object): tutils.tcLog("Verify GW flows are present") tutils.checkGwFlows("11.3.0.1") tutils.tcLog("Create a namespace, with simple service") createNs("e2e") createNsPod("e2e", "srvr-80") createNsPod("e2e", "srvr-81") svcIP = createNsSvc("e2e", "simple-svc") tutils.tcLog("Create two client pods") client_pods = ["client-a", "client-b"] for pod in client_pods: createNsPod("e2e", pod) tutils.tcLog("Verify access without netpol") for pod in client_pods: verifyAccess("e2e", pod, svcIP, "80", "open") verifyAccess("e2e", pod, svcIP, "81", "open") tutils.tcLog("Delete clients") for pod in client_pods: tutils.deletePod("e2e", pod) tutils.tcLog("Apply network policy client-a->port80") createNsNetPol("e2e", "upd_policy1") tutils.tcLog("Create two client pods") for pod in client_pods: createNsPod("e2e", pod) tutils.tcLog("Verify client-a can access port 80") verifyAccess("e2e", "client-a", svcIP, "80", "open") tutils.tcLog("Verify failures for a/81, b/[80,81]") verifyAccess("e2e", "client-a", svcIP, "81", "timed out") verifyAccess("e2e", "client-b", svcIP, "80", "timed out") verifyAccess("e2e", "client-b", svcIP, "81", "timed out") tutils.tcLog("Delete clients") for pod in client_pods: tutils.deletePod("e2e", pod) tutils.tcLog("Update network policy client-b->port81") replaceNsNetPol("e2e", "upd_policy2") tutils.tcLog("Create two client pods") for pod in client_pods: createNsPod("e2e", pod) tutils.tcLog("Verify client-b can access port 81") verifyAccess("e2e", "client-b", svcIP, "81", "open") tutils.tcLog("Verify failures for b/81, a/[80,81]") verifyAccess("e2e", "client-b", svcIP, "80", "timed out") verifyAccess("e2e", "client-a", svcIP, "80", "timed out") verifyAccess("e2e", "client-a", svcIP, "81", "timed out") v1 = client.CoreV1Api() v1.delete_namespace("e2e", client.V1DeleteOptions()) def nsDelChecker(): if tutils.namespaceExists("e2e"): return "e2e exists" return "" tutils.tcLog("Verify ns is deleted") tutils.assertEventually(nsDelChecker, 2, 40)
def verifyAccess(ns, pod, ip, port, exp_str): v1 = client.CoreV1Api() def checker(): cmd = ['nc', '-zvnw', '1', ip, port] resp = stream(v1.connect_get_namespaced_pod_exec, pod, ns, command=cmd, stderr=True, stdin=False, stdout=True, tty=False) if exp_str not in resp: return "{} gave {}".format(ip, resp) return "" tutils.assertEventually(checker, 1, 10)
def deleteNsNetPol(ns, npName): nv1 = client.NetworkingV1Api() nv1.delete_namespaced_network_policy(npName, ns, client.V1DeleteOptions()) def delChecker(): npList = nv1.list_namespaced_network_policy(ns) for np in npList.items: if np.metadata.name is npName: return "exists" return "" tutils.assertEventually(delChecker, 1, 30)
def kafkaSyncChecker(epKeys, attempts=2): v1 = client.CoreV1Api() def checkIt(): cmd = ['./kafkakv', '-time-out', '30', '-key-list', ",".join(epKeys)] logging.debug("Command: {}".format(cmd)) resp = stream(v1.connect_get_namespaced_pod_exec, "kafkakv", 'default', command=cmd, stderr=True, stdin=False, stdout=True, tty=False) #print("cmd:{} resp: {}".format(cmd, resp)) if "exact match" in resp: return "" return "Got: {}".format(resp) tutils.assertEventually(checkIt, 1, attempts)
def createNsPod(ns, name): v1 = client.CoreV1Api() with open(path.abspath(nameToYaml(name))) as f: pod_obj = yaml.load(f) v1.create_namespaced_pod(ns, pod_obj) # check pod is ready def podChecker(): s = v1.read_namespaced_pod_status(name, ns) if s.status.phase == "Running": return "" return "Pod not ready" tutils.assertEventually(podChecker, 1, 180)
def createPod(name): k8s_api = utils.create_from_yaml(k8s_client, nameToYaml(name)) # check pod is ready def podChecker(): s = k8s_api.read_namespaced_pod_status(name, "default") if s.status.phase == "Running": return "" return "Pod not ready" tutils.assertEventually(podChecker, 1, 60) # return IP s = k8s_api.read_namespaced_pod_status(name, "default") return s.status.pod_ip
def kafkaChecker(epKeys, respWord, attempts=2): v1 = client.CoreV1Api() def checkIt(): for epid in epKeys: cmd = ['./kafkakv', '-key'] cmd.append(epid) resp = stream(v1.connect_get_namespaced_pod_exec, "kafkakv", 'default', command=cmd, stderr=True, stdin=False, stdout=True, tty=False) #print("cmd:{} resp: {}".format(cmd, resp)) if respWord in resp and epid in resp: continue return "{} -- not yet {}".format(epid, respWord) return "" tutils.assertEventually(checkIt, 1, attempts)
def assertPodReady(ns, name, timeout): v1 = client.CoreV1Api() # check pod is ready def podChecker(): s = v1.read_namespaced_pod_status(name, ns) if s.status.phase != "Running": return "Pod not running" for cond in s.status.conditions: if cond.type == "ContainersReady" and cond.status == "True": return "" return "Containers not ready" tutils.assertEventually(podChecker, 1, timeout)
def scaleDep(ns, name, replicas): v1 = client.AppsV1Api() scale = v1.read_namespaced_deployment_scale(name, ns) scale.spec.replicas = replicas resp = v1.replace_namespaced_deployment_scale(name, ns, scale) def scaleChecker(): curr = v1.read_namespaced_deployment_status(name, ns) if curr.status.ready_replicas is None and replicas == 0: return "" if curr.status.ready_replicas == replicas: return "" return "expected {} replicas, got {}".format(replicas, curr.status.ready_replicas) tutils.assertEventually(scaleChecker, 1, 30)
def createCRD(plural, name): crd_api = client.CustomObjectsApi(k8s_client) with open(path.abspath(nameToYaml(name))) as f: crd_obj = yaml.load(f) crd_api.create_namespaced_custom_object("aci.aw", "v1", "kube-system", plural, crd_obj) # check contract is ready def crdChecker(): resp = crd_api.get_namespaced_custom_object("aci.aw", "v1", "kube-system", plural, name) if 'spec' in resp: return "" return "CRD {}/{} not created".format(plural, name) tutils.assertEventually(crdChecker, 1, 60)
def test_nslookup(object): v1 = client.CoreV1Api() # use alpine because busybox nslookup seems broken tutils.tcLog("Create a dns client pod") createPod("alpine-pod") s = v1.read_namespaced_service("kubernetes", "default") svcIP = s.spec.cluster_ip # verify ping fails across epgs print("\nVerify nslookup") tutils.tcLog("Verify dns lookup of kubernetes service") ns_lookup_cmd = ['nslookup', 'kubernetes'] def respChecker(): logging.debug("===svcIP is {}".format(svcIP)) resp = stream(v1.connect_get_namespaced_pod_exec, "alpine-pod", 'default', command=ns_lookup_cmd, stderr=True, stdin=False, stdout=True, tty=False) if svcIP in resp: logging.debug("=>command {}".format(ns_lookup_cmd)) logging.debug("=>Resp is {}".format(resp)) return "" else: logging.debug("NR=>Resp is {}".format(resp)) #return "" #FIXME this is not reliable yet... return "svc not resolved {}".format(ns_lookup_cmd) # Basic inspection in case of a dns test failer def dnsInspector(): tutils.inspectLog("Checking coredns status on failure") av1 = client.AppsV1Api() sr = av1.read_namespaced_deployment_status("coredns", "kube-system") tutils.inspectLog("CoreDNS available: {}, ready: {}".format(sr.status.available_replicas, sr.status.ready_replicas)) tutils.assertEventually(respChecker, 1, 5, dnsInspector) tutils.tcLog("Create a service and check dns resolution") ns_lookup_cmd = ['nslookup', 'dns-test-svc'] svcIP = createNsSvc("default", "dns-test-svc") logging.info("***svcIP is {}".format(svcIP)) tutils.assertEventually(respChecker, 1, 60) v1.delete_namespaced_pod("alpine-pod", "default", client.V1DeleteOptions()) v1.delete_namespaced_service("dns-test-svc", "default", client.V1DeleteOptions())
def test_ovs_restart(object): tutils.tcLog("Getting list of ovs pods") node_list = v1.list_node() ns = tutils.getSysNs() pod_list = v1.list_namespaced_pod(ns, label_selector="name=aci-containers-openvswitch") assert len(node_list.items) > 0 assert len(node_list.items) == len(pod_list.items) tutils.tcLog("Delete current ovs pods") for pod in pod_list.items: tutils.deletePod(ns, pod.metadata.name) tutils.tcLog("Check new ovs pods") def checker(): new_pod_list = v1.list_namespaced_pod(ns, label_selector="name=aci-containers-openvswitch") if len(pod_list.items) == len(new_pod_list.items): return "" return "{} ne {}".format(len(new_pod_list.items), len(pod_list.items)) tutils.assertEventually(checker, 2, 30)
def test_kafka(object): # setup kafka services tutils.tcLog("Setup kafka services") for ky in kafkaYamls: try: utils.create_from_yaml(k8s_client, "yamls/"+ky) except ApiException as e: logging.debug("{} - ignored".format(e.reason)) k8s_api = client.CoreV1Api() # check kafka server is ready assertPodReady("default", "ut-kafka-0", 120) assertPodReady("default", "kafkakv", 120) sleep(5) # collect the current list of ep's from k8s tutils.tcLog("Collect ep's from k8s") initialEPList = readCniEPList() logging.info("EPList is {}".format(initialEPList)) tutils.tcLog("Verifying initial EPList with kafka") kafkaSyncChecker(initialEPList) tutils.tcLog("Adding a pod and checking it in kafka") utils.create_from_yaml(k8s_client, "yamls/alpine-pod.yaml") assertPodReady("default", "alpine-pod", 45) vtep = tutils.getPodNodeIP("alpine-pod", "default") tutils.tcLog("Check for podif") crdApi = client.CustomObjectsApi() group = "aci.aw" ns = "kube-system" podifName = "default.alpine-pod.{}".format(vtep) p = crdApi.get_namespaced_custom_object(group, "v1", ns, "podifs", podifName) logging.debug("podif: {}".format(p)) epStatus = p['status'] epID = "{}.{}.{}".format(epStatus['podns'], epStatus['podname'], epStatus['ifname']) toCheck = [] toCheck.append(epID) kafkaChecker(toCheck, "found") tutils.tcLog("Delete a pod and check removal from kafka") k8s_api.delete_namespaced_pod("alpine-pod", "default", client.V1DeleteOptions()) kafkaChecker(toCheck, "missing", 8) tutils.tcLog("Recheck ep sync between k8s and kafka") kafkaSyncChecker(initialEPList) tutils.tcLog("Add more endpoints") utils.create_from_yaml(k8s_client, "yamls/busybox.yaml") tutils.rcCheckScale("busybox", 3) tutils.tcLog("Get new eplist from k8s") EPList1 = readCniEPList() assert EPList1 != initialEPList tutils.tcLog("Recheck ep sync between k8s and kafka") kafkaSyncChecker(EPList1) tutils.tcLog("Bring controller down") scaleDep("kube-system", "aci-containers-controller", 0) sleep(10) tutils.tcLog("Change some endpoints") tutils.scaleRc("busybox", 0) tutils.checkPodsRemoved("app=busybox") EPList2 = readCniEPList() assert EPList2 != EPList1 tutils.scaleRc("busybox", 2) sleep(5) logging.debug("previous ep list:{}".format(EPList1)) EPList3 = readCniEPList() logging.debug("new ep list:{}".format(EPList3)) assert EPList3 != EPList1 tutils.tcLog("Bring controller up") scaleDep("kube-system", "aci-containers-controller", 1) sleep(10) tutils.tcLog("Verify controller is up") tutils.assertEventually(checkSystemStatus, 1, 60) tutils.tcLog("Check ep sync again") kafkaSyncChecker(EPList3, 6) tutils.tcLog("Clean up added endpoints") tutils.scaleRc("busybox", 0) tutils.checkPodsRemoved("app=busybox") k8s_api.delete_namespaced_replication_controller("busybox", "default", client.V1DeleteOptions())
def test_np(object): tutils.tcLog("Verify GW flows are present") tutils.checkGwFlows("11.3.0.1") createNs("prod") # create a deployment in default namespace k8s_api = utils.create_from_yaml(k8s_client, "yamls/hostnames-dep.yaml") v1 = client.CoreV1Api() # check deployment is ready svc_hosts = [] def depChecker(): svc_hosts = getPodNames(v1, "default", "app=hostnames") if len(svc_hosts) >= 3: return "" return "Need 3 hosts, have {}".format(len(svc_hosts)) tutils.assertEventually(depChecker, 1, 180) createNsPod("default", "client-pod") tutils.tcLog("Verify access without netpol") podIPs = tutils.getPodIPs("default", "app=hostnames") assert len(podIPs) == 3 verifyAccess('default', 'open', podIPs) # apply networkpolicy allowing access only to prod tutils.tcLog("Create k8s network policy") createNsNetPol("default", "np/hostnames-deny-all") tutils.tcLog("Verify access denied") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Setup a netpol to allow access from prod") createNsNetPol("default", "hostnames-allow-prod") createNsPod("prod", "client-pod") tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.tcLog("Verify deny from default") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Delete deny-all") deleteNsNetPol('default', 'hostnames-deny-all') tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.tcLog("Verify deny from default") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Delete allow-prod") deleteNsNetPol('default', 'hostnames-allow-prod') tutils.tcLog("Verify allow from default") verifyAccess('default', 'open', podIPs) tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.tcLog("Allow port 9000 from prod") createNsNetPol("default", "np/hostnames-allow-prod9000") tutils.tcLog("Verify deny from default") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Verify deny 9376 from prod") verifyAccess('prod', 'timed out', podIPs) tutils.tcLog("Delete allow-prod9000") deleteNsNetPol('default', 'hostnames-allow-prod9000') tutils.tcLog("Allow port 9376 from prod") createNsNetPol("default", "np/hostnames-allow-prod9376") tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.tcLog("Verify deny from default") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Delete allow-prod9376") deleteNsNetPol('default', 'hostnames-allow-prod9376') tutils.tcLog("Verify allow from default") verifyAccess('default', 'open', podIPs) tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.scaleDep("default", "hostnames-dep", 0) v1.delete_namespace( 'prod', client.V1DeleteOptions()) # deletes the client-pod too av1 = client.AppsV1Api() av1.delete_namespaced_deployment("hostnames-dep", "default", client.V1DeleteOptions()) tutils.deletePod("default", "client-pod") def delChecker(): dList = av1.list_namespaced_deployment("default") for dep in dList.items: if dep.metadata.name == "hostnames-dep": return "hostnames-dep still present" if tutils.namespaceExists('prod'): return "prod exists" return "" tutils.tcLog("Verify cleanup") tutils.assertEventually(delChecker, 2, 40)
def test_default(object): tutils.tcLog("Create 3 pods in default epg") k8s_api = utils.create_from_yaml(k8s_client, "yamls/busybox.yaml") # check rc is ready def rcChecker(): return checkRCStatus(k8s_api, 3) tutils.assertEventually(rcChecker, 1, 60) tutils.checkAgentLog() # check pods are ready def podChecker(): pod_list = k8s_api.list_namespaced_pod( "default", label_selector="app=busybox") if len(pod_list.items) != 3: return "Expected 3 pods, got {}".format(len(pod_list.items)) for pod in pod_list.items: if pod.status.phase != "Running": return "pod {} status is {}".format( pod.metadata.name, pod.status.phase) return "" tutils.assertEventually(podChecker, 1, 60) # verify connectivity ips = tutils.getPodIPs("default", "app=busybox") pod_list = k8s_api.list_namespaced_pod("default", label_selector="app=busybox") pod = next(iter(pod_list.items), None) assert pod != None tutils.tcLog("Verify default connectivity") def pingChecker(): ping_cmd = ['ping', '-c', '3'] for ip in ips: v1 = client.CoreV1Api() cmd = list(ping_cmd) cmd.append(ip) resp = stream(v1.connect_get_namespaced_pod_exec, pod.metadata.name, 'default', command=cmd, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>Resp is {}".format(resp)) if "3 packets received" not in resp: return "3 packets not received" return "" tutils.assertEventually(pingChecker, 1, 60) tutils.tcLog("Delete pods") tutils.scaleRc("busybox", 0) k8s_api.delete_namespaced_replication_controller( "busybox", "default", client.V1DeleteOptions())
def test_policy(object): v1 = client.CoreV1Api() createCRD("contracts", "tcp-6020") createCRD("contracts", "tcp-6021") sleep(1) createCRD("epgs", "epg-a") createCRD("epgs", "epg-b") # pod in epg-a createPod("pod-a") # pods in epg-b ip_6020 = createPod("pod-b6020") ip_6021 = createPod("pod-b6021") tutils.checkAgentLog() # verify ping fails across epgs tutils.tcLog("Verify ping failure across epgs") ping_cmd = ['ping', '-c', '6', '-t', '1', ip_6020] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=ping_cmd, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>Resp is {}".format(resp)) assert "0 packets received" in resp tutils.tcLog("Verify tcp contract") # verify port 6020 access from pod-a to epg-b tutils.tcLog("port 6020 access from pod-a to epg-b") cmd1 = ['nc', '-zvnw', '1', ip_6020, '6020'] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=cmd1, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-a to epg-b[6020] Resp is {}".format(resp)) assert "open" in resp sleep(5) # verify port 6021 is inaccessible from pod-a to epg-b tutils.tcLog("port 6021 deny from pod-a to epg-b") cmd2 = ['nc', '-zvnw', '1', ip_6021, '6021'] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-a to epg-b[6021] Resp is {}".format(resp)) assert "timed out" in resp sleep(5) # verify port 6021 is accessible within epg-b tutils.tcLog("port 6021 accessible within epg-b") resp = stream(v1.connect_get_namespaced_pod_exec, "pod-b6020", 'default', command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-b6020 to pod-b[6021] Resp is {}".format(resp)) assert "open" in resp tutils.tcLog("Change allowed port to 6021") replaceCRD("epgs", "epg-a", "epg-a-upd") replaceCRD("epgs", "epg-b", "epg-b-upd") tutils.tcLog("Verify new contract is on agents") def contractChecker(): resp = tutils.verifyAgentContracts(["GbpeL24Classifier/tcp-6021"], True) if resp == "": print("GbpeL24Classifier/tcp-6021 present") else: print(resp) return resp tutils.assertEventually(contractChecker, 1, 10) sleep(5) tutils.tcLog("port 6021 now allowed from pod-a to epg-b") cmd2 = ['nc', '-zvnw', '1', ip_6021, '6021'] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-a to epg-b[6021] Resp is {}".format(resp)) assert "open" in resp def contractRemChecker(): resp = tutils.verifyAgentContracts([ "GbpEpGroupToConsContractRSrc/288/tcp-6020", "GbpEpGroupToProvContractRSrc/288/tcp-6020" ], False) if resp == "": print( "GbpEpGroupToConsContractRSrc/288/tcp-6020, GbpEpGroupToProvContractRSrc/288/tcp-6020 removed" ) else: print(resp) return resp tutils.assertEventually(contractRemChecker, 1, 10) tutils.tcLog("port 6020 now denied from pod-a to epg-b") cmd1 = ['nc', '-zvnw', '1', ip_6020, '6020'] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=cmd1, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-a to epg-b[6020] Resp is {}".format(resp)) assert "timed out" in resp toDelete = ["pod-a", "pod-b6020", "pod-b6021"] logging.info("Deleting {}\n".format(toDelete)) for pod in toDelete: v1.delete_namespaced_pod(pod, "default", client.V1DeleteOptions()) for pod in toDelete: tutils.checkPodDeleted(v1, "default", pod, 120) deleteCRD("contracts", "tcp-6020") deleteCRD("contracts", "tcp-6021") deleteCRD("epgs", "epg-a") deleteCRD("epgs", "epg-b") tutils.checkAgentLog()
def test_np(object): tutils.tcLog("Verify GW flows are present") tutils.checkGwFlows("11.3.0.1") # create a service/deployment in default namespace svcIP = createNsSvc("default", "hostnames-svc") k8s_api = utils.create_from_yaml(k8s_client, "yamls/hostnames-dep.yaml") v1 = client.CoreV1Api() # check deployment is ready svc_hosts = [] def depChecker(): svc_hosts = getPodNames(v1, "default", "app=hostnames") if len(svc_hosts) >= 3: return "" return "Need 3 hosts, have {}".format(len(svc_hosts)) tutils.assertEventually(depChecker, 1, 180) # create two namespaces, with a client pod in each nsList = ["prod", "dev"] for ns in nsList: createNs(ns) logging.info("\nWaiting for namespaces to be available") sleep(3) for ns in nsList: createNsPod(ns, "client-pod") tutils.tcLog("Verify loadbalancing from dev and prod") # verify both clients can access the service, and the service load balances. for ns in nsList: cmd = ['curl', '--connect-timeout', '1', '-s', svcIP] backends = dict() for count in range(0, 12): resp = stream(v1.connect_get_namespaced_pod_exec, "client-pod", ns, command=cmd, stderr=True, stdin=False, stdout=True, tty=False) if resp == "": sleep(1) continue backends[resp] = True if len(backends) == 2: break lbWorkaround() #assert len(backends) == 2 logging.info("backends: {}".format(backends.keys())) tutils.tcLog("Verify there are no timeouts") nc_cmd = ['nc', '-zvnw', '1', svcIP, '80'] for ns in nsList: for count in range(0, 6): resp = stream(v1.connect_get_namespaced_pod_exec, "client-pod", ns, command=nc_cmd, stderr=True, stdin=False, stdout=True, tty=False) if 'open' not in resp: print("Error - {}, expected open".format(resp)) assert (False) lbWorkaround() # apply networkpolicy allowing access only to prod tutils.tcLog("Create k8s network policy") createNsNetPol("default", "hostnames-allow-prod") sleep(10) cmd = ['curl', '--connect-timeout', '1', svcIP] # wait for netpol to take effect def waiter(): count = 0 for ix in range(0, 10): resp1 = stream(v1.connect_get_namespaced_pod_exec, "client-pod", "dev", command=cmd, stderr=True, stdin=False, stdout=True, tty=False) if "timed out" in resp1: count += 1 else: sleep(1) if count == 10: return "" return "still accessible" tutils.tcLog("Verify dev cannot access the svc") tutils.assertEventually(waiter, 1, 120) tutils.tcLog("Verify prod can access the svc") # verify prod can access the svc cmd2 = ['nc', '-zvnw', '1', svcIP, '80'] def prodChecker(): for ix in range(0, 3): resp2 = stream(v1.connect_get_namespaced_pod_exec, "client-pod", "prod", command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("prod: {}".format(resp2)) if "open" not in resp2: return resp2 lbWorkaround() return "" clientIP = tutils.getPodIP("client-pod", "prod") ovs_pods = getPodNames(v1, "kube-system", "name=aci-containers-openvswitch") ovs_cmd = ['ovs-ofctl', 'dump-flows', 'br-access', '-OOpenFlow13'] def npInspector(): tutils.inspectLog( "Look for flows on br-access for {}".format(clientIP)) for ovs_pod in ovs_pods: ovs_resp = stream(v1.connect_get_namespaced_pod_exec, ovs_pod, "kube-system", command=ovs_cmd, stderr=True, stdin=False, stdout=True, tty=False) print("ovs_Pod: {}\n".format(ovs_pod)) for line in ovs_resp.splitlines(): if clientIP in line: print(line) tutils.assertEventually(prodChecker, 1, 5, npInspector) tutils.tcLog("Verify dev still can't access the svc") # and dev can't for ix in range(0, 3): resp3 = stream(v1.connect_get_namespaced_pod_exec, "client-pod", "dev", command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("dev: {}".format(resp3)) assert "timed out" in resp3 lbWorkaround() # delete everything nv1 = client.NetworkingV1Api() nv1.delete_namespaced_network_policy("hostnames-allow-prod", "default", client.V1DeleteOptions()) # verify access-br flows are deleted def flowChecker(): for ovs_pod in ovs_pods: ovs_resp = stream(v1.connect_get_namespaced_pod_exec, ovs_pod, "kube-system", command=ovs_cmd, stderr=True, stdin=False, stdout=True, tty=False) for line in ovs_resp.splitlines(): if clientIP in line: return line return "" tutils.tcLog("Verify network policy flows are deleted") tutils.assertEventually(flowChecker, 1, 120) tutils.scaleDep("default", "hostnames-dep", 0) for ns in nsList: v1.delete_namespace( ns, client.V1DeleteOptions()) # deletes the client-pod too v1.delete_namespaced_service("hostnames-svc", "default", client.V1DeleteOptions()) av1 = client.AppsV1Api() av1.delete_namespaced_deployment("hostnames-dep", "default", client.V1DeleteOptions()) def delChecker(): dList = av1.list_namespaced_deployment("default") for dep in dList.items: if dep.metadata.name == "hostnames-dep": return "hostnames-dep still present" sList = v1.list_namespaced_service("default") for svc in sList.items: if svc.metadata.name == "hostnames-svc": return "hostnames-svc still present" for ns in nsList: if tutils.namespaceExists(ns): return "{} exists".format(ns) return "" tutils.tcLog("Verify svc/dep is deleted") tutils.assertEventually(delChecker, 2, 40)
def test_default(object): tutils.tcLog("Verify GW flows are present") tutils.checkGwFlows("11.3.0.1") delete_rc = True k8s_api = client.CoreV1Api() if not checkRCStatus(k8s_api, 3): logging.debug("Using existing rc") delete_rc = False else: tutils.tcLog("Create 3 pods in default epg") k8s_api = utils.create_from_yaml(k8s_client, "yamls/busybox.yaml") # check rc is ready def rcChecker(): return checkRCStatus(k8s_api, 3) tutils.assertEventually(rcChecker, 1, 60) tutils.checkAgentLog() # check pods are ready def podChecker(): pod_list = k8s_api.list_namespaced_pod( "default", label_selector="app=busybox") if len(pod_list.items) < 3: return "Expected 3 pods, got {}".format(len(pod_list.items)) for pod in pod_list.items: if pod.status.phase != "Running": return "pod {} status is {}".format( pod.metadata.name, pod.status.phase) return "" tutils.assertEventually(podChecker, 1, 60) # verify connectivity pod_ips = tutils.getPodIPs("default", "app=busybox") tutils.tcLog("Verify EPs are populated") def remEPChecker(): res = tutils.verifyAgentEPs(pod_ips) if res == "": print("{} present".format(pod_ips)) else: print(res) return res tutils.assertEventually(remEPChecker, 1, 10) node_ips = getNodeIPs(k8s_api) ips = pod_ips + node_ips print("ip's are: {}".format(ips)) pod_list = k8s_api.list_namespaced_pod("default", label_selector="app=busybox") pod = next(iter(pod_list.items), None) assert pod != None tutils.tcLog("Verify default connectivity") def pingChecker(): ping_cmd = ['ping', '-c', '3'] for ip in ips: v1 = client.CoreV1Api() cmd = list(ping_cmd) cmd.append(ip) resp = stream(v1.connect_get_namespaced_pod_exec, pod.metadata.name, 'default', command=cmd, stderr=True, stdin=False, stdout=True, tty=False) print("=>Resp is {}".format(resp)) if "3 packets received" not in resp: return "3 packets not received" return "" tutils.assertEventually(pingChecker, 1, 60) tutils.tcLog("Check pods access to API server") v1 = client.CoreV1Api() s = v1.read_namespaced_service("kubernetes", "default") svcIP = s.spec.cluster_ip logging.debug("=>K8s svc IP is {}".format(svcIP)) pod_list = k8s_api.list_namespaced_pod("default", label_selector="app=busybox") cmd2 = ['nc', '-zvnw', '1', svcIP, '443'] def k8sChecker(): for pod in pod_list.items: resp = stream(v1.connect_get_namespaced_pod_exec, pod.metadata.name, 'default', command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>Resp is {}".format(resp)) if "open" not in resp: return resp return "" tutils.assertEventually(k8sChecker, 1, 10) if delete_rc: tutils.tcLog("Delete pods") tutils.scaleRc("busybox", 0) k8s_api.delete_namespaced_replication_controller( "busybox", "default", client.V1DeleteOptions()) else: logging.debug("Skipping rc delete")