def verifyULConnectivity(object, tepList): tutils.tcLog("Create pods in host ns") pods = tutils.createDs("ul-tester") for pod in pods: for tep in tepList: tutils.verifyPing(pod, "default", tep) tutils.tcLog("Delete pods in host ns") tutils.deleteDs("ul-tester")
def countCompHv(object): vmm_name = object.config["aci_config"]["system_id"] tutils.tcLog("Count compHv objects for {}".format(vmm_name)) qry = '/api/mo/comp/prov-Kubernetes/ctrlr-[{}]-{}.json?query-target=children&target-subtree-class=compHv'.format( vmm_name, vmm_name) resp = object.apic.get(path=qry) resJson = json.loads(resp.content) return len(resJson["imdata"])
def verifyOLConnectivity(object, vmIP): tutils.tcLog("Create pods in epg-a") pods = tutils.createDs("ol-tester") object.verifyCapicEPs(pods, "epg-a") for pod in pods: tutils.verifyPing(pod, "default", vmIP) tutils.tcLog("Delete pods in epg-a") tutils.deleteDs("ol-tester")
def getCniTep(object): tutils.tcLog("Get CniTep objects") qry = '/api/class/hcl3CniTep.json' resp = object.apic.get(path=qry) resJson = json.loads(resp.content) tepList = [] for mo in resJson['imdata']: addr = mo['hcl3CniTep']['attributes']['addr'] tep = addr.split("/")[0] tepList.append(tep) return tepList
def verifyClusterInfo(object): vmm_name = object.config["aci_config"]["system_id"] tn_name = object.config["aci_config"]["tenant"]["name"] tutils.tcLog("Verify cluster info objects for {}/{}".format( tn_name, vmm_name)) moClasses = ["compClusterInfo", "hcloudClusterInfo"] for moCl in moClasses: qry = '/api/class/{}.json?query-target-filter=and(eq({}.name,"{}"),eq({}.accountName,"{}"))'.format( moCl, moCl, moCl, vmm_name, tn_name) resp = object.apic.get(path=qry) resJson = json.loads(resp.content) assert len(resJson["imdata"]) == 1
def getVmIP(object, epg): tutils.tcLog("Get vm ip for epg {}".format(epg)) qry = '/api/class/hcloudRtEpToSecurityGroup.json?query-target-filter=and(wcard(hcloudRtEpToSecurityGroup.dn,"{}"))'.format( epg) resp = object.apic.get(path=qry) epDns = object.getFromResp(resp, 'hcloudRtEpToSecurityGroup', 'tDn') assert len(epDns) > 0 print("epDn: {}".format(epDns[0])) qry = '/api/mo/{}.json'.format(epDns[0]) resp = object.apic.get(path=qry) epIPs = object.getFromResp(resp, 'hcloudEndPoint', 'primaryIpV4Addr') assert len(epIPs) > 0 return epIPs[0]
def getAllEpIPs(object, epg): tutils.tcLog("Get all hcloudEP ip for epg {}".format(epg)) ip_dict = {} qry = '/api/class/hcloudRtEpToSecurityGroup.json?query-target-filter=and(wcard(hcloudRtEpToSecurityGroup.dn,"{}]"))'.format( epg) resp = object.apic.get(path=qry) epDns = object.getFromResp(resp, 'hcloudRtEpToSecurityGroup', 'tDn') for epDn in epDns: qry = '/api/mo/{}.json'.format(epDn) resp = object.apic.get(path=qry) epIPs = object.getFromResp(resp, 'hcloudEndPoint', 'primaryIpV4Addr') if len(epIPs) > 0: ep_ip = epIPs[0].split("/")[0] ip_dict[ep_ip] = True return ip_dict
def verifyCsr(object, csr, tunnelInfo, vrf_encap_id): dn = csr["hcloudCsr"]["attributes"]["dn"] qry = '/api/mo/{}.json?query-target=subtree&target-subtree-class=hcloudEndPointOper&query-target-filter=and(ne(hcloudEndPointOper.publicIpv4Addr,"0.0.0.0"))'.format( dn) resp = object.apic.get(path=qry) resJson = json.loads(resp.content) def getCsrIP(): for epOper in resJson["imdata"]: if "hcloudEndPointOper" in epOper: att = epOper["hcloudEndPointOper"]["attributes"] operDn = att["dn"] if "nwif-0" in operDn: print("CSR IP: {}".format(att["publicIpv4Addr"])) return att["publicIpv4Addr"] return None csrIP = getCsrIP() assert csrIP is not None print(csrIP) csr_c = paramiko.SSHClient() csr_c.load_system_host_keys() csr_c.set_missing_host_key_policy(paramiko.AutoAddPolicy()) pw = object.config["aci_config"]["apic_login"]["password"] csr_c.connect(csrIP, username='******', password=pw) stdin, stdout, stderr = csr_c.exec_command( 'show run | sec interface Tunn') # format output out_dict = {} key = "" for line in stdout: ll = line.strip('\n') #print(ll) if "interface Tunnel" in ll: key = ll.split()[1] out_dict[key] = {} else: if key: out_dict[key][ll.strip()] = True csr_c.close() tutils.tcLog("Verify tunnels {} is in CSR".format(tunnelInfo)) object.verifyTunnels(out_dict, tunnelInfo, vrf_encap_id) tutils.tcLog("Verify routes in CSR({})".format(csrIP)) object.verifyCsrRoutes(csr_c, csrIP)
def test_concrete(object): object.setup() addr = object.config["aci_config"]["apic_hosts"][0] tutils.tcLog("Verify capic {} access".format(addr)) assert object.apic is not None object.verifyClusterInfo() nodeCount = object.countCompHv() print("nodeCount: {}".format(nodeCount)) csrList = object.getCSRs() csrCount = len(csrList) print("csrCount: {}".format(csrCount)) if nodeCount == 0 or csrCount == 0: print("No further tests possible with these csr/compHv count") return tepList = object.getCniTep() assert len(tepList) > 0 tutils.tcLog( "Verify connectivity from host ns to teps {}".format(tepList)) object.verifyULConnectivity(tepList) tunnelInfo = object.getTunnelInfo() print(tunnelInfo) vrf_encap_id = tutils.read_vrf_encap_id() print("vrf-encap-id is {}".format(vrf_encap_id)) for csr in csrList: object.verifyCsr(csr, tunnelInfo, vrf_encap_id) vmIP = object.getVmIP("test-vm-epg") print("vmIP: {}".format(vmIP)) assert vmIP tutils.tcLog("Verify connectivity from epg-a to {}".format(vmIP)) object.verifyOLConnectivity(vmIP)
def test_nslookup(object): v1 = client.CoreV1Api() # use alpine because busybox nslookup seems broken tutils.tcLog("Create a dns client pod") createPod("alpine-pod") s = v1.read_namespaced_service("kubernetes", "default") svcIP = s.spec.cluster_ip # verify ping fails across epgs print("\nVerify nslookup") tutils.tcLog("Verify dns lookup of kubernetes service") ns_lookup_cmd = ['nslookup', 'kubernetes'] def respChecker(): logging.debug("===svcIP is {}".format(svcIP)) resp = stream(v1.connect_get_namespaced_pod_exec, "alpine-pod", 'default', command=ns_lookup_cmd, stderr=True, stdin=False, stdout=True, tty=False) if svcIP in resp: logging.debug("=>command {}".format(ns_lookup_cmd)) logging.debug("=>Resp is {}".format(resp)) return "" else: logging.debug("NR=>Resp is {}".format(resp)) #return "" #FIXME this is not reliable yet... return "svc not resolved {}".format(ns_lookup_cmd) # Basic inspection in case of a dns test failer def dnsInspector(): tutils.inspectLog("Checking coredns status on failure") av1 = client.AppsV1Api() sr = av1.read_namespaced_deployment_status("coredns", "kube-system") tutils.inspectLog("CoreDNS available: {}, ready: {}".format(sr.status.available_replicas, sr.status.ready_replicas)) tutils.assertEventually(respChecker, 1, 5, dnsInspector) tutils.tcLog("Create a service and check dns resolution") ns_lookup_cmd = ['nslookup', 'dns-test-svc'] svcIP = createNsSvc("default", "dns-test-svc") logging.info("***svcIP is {}".format(svcIP)) tutils.assertEventually(respChecker, 1, 60) v1.delete_namespaced_pod("alpine-pod", "default", client.V1DeleteOptions()) v1.delete_namespaced_service("dns-test-svc", "default", client.V1DeleteOptions())
def test_ovs_restart(object): tutils.tcLog("Getting list of ovs pods") node_list = v1.list_node() ns = tutils.getSysNs() pod_list = v1.list_namespaced_pod(ns, label_selector="name=aci-containers-openvswitch") assert len(node_list.items) > 0 assert len(node_list.items) == len(pod_list.items) tutils.tcLog("Delete current ovs pods") for pod in pod_list.items: tutils.deletePod(ns, pod.metadata.name) tutils.tcLog("Check new ovs pods") def checker(): new_pod_list = v1.list_namespaced_pod(ns, label_selector="name=aci-containers-openvswitch") if len(pod_list.items) == len(new_pod_list.items): return "" return "{} ne {}".format(len(new_pod_list.items), len(pod_list.items)) tutils.assertEventually(checker, 2, 30)
def test_policy(object): v1 = client.CoreV1Api() createCRD("contracts", "tcp-6020") createCRD("contracts", "tcp-6021") sleep(1) createCRD("epgs", "epg-a") createCRD("epgs", "epg-b") # pod in epg-a createPod("pod-a") # pods in epg-b ip_6020 = createPod("pod-b6020") ip_6021 = createPod("pod-b6021") tutils.checkAgentLog() # verify ping fails across epgs tutils.tcLog("Verify ping failure across epgs") ping_cmd = ['ping', '-c', '6', '-t', '1', ip_6020] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=ping_cmd, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>Resp is {}".format(resp)) assert "0 packets received" in resp tutils.tcLog("Verify tcp contract") # verify port 6020 access from pod-a to epg-b tutils.tcLog("port 6020 access from pod-a to epg-b") cmd1 = ['nc', '-zvnw', '1', ip_6020, '6020'] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=cmd1, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-a to epg-b[6020] Resp is {}".format(resp)) assert "open" in resp sleep(5) # verify port 6021 is inaccessible from pod-a to epg-b tutils.tcLog("port 6021 deny from pod-a to epg-b") cmd2 = ['nc', '-zvnw', '1', ip_6021, '6021'] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-a to epg-b[6021] Resp is {}".format(resp)) assert "timed out" in resp sleep(5) # verify port 6021 is accessible within epg-b tutils.tcLog("port 6021 accessible within epg-b") resp = stream(v1.connect_get_namespaced_pod_exec, "pod-b6020", 'default', command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-b6020 to pod-b[6021] Resp is {}".format(resp)) assert "open" in resp tutils.tcLog("Change allowed port to 6021") replaceCRD("epgs", "epg-a", "epg-a-upd") replaceCRD("epgs", "epg-b", "epg-b-upd") tutils.tcLog("Verify new contract is on agents") def contractChecker(): resp = tutils.verifyAgentContracts(["GbpeL24Classifier/tcp-6021"], True) if resp == "": print("GbpeL24Classifier/tcp-6021 present") else: print(resp) return resp tutils.assertEventually(contractChecker, 1, 10) sleep(5) tutils.tcLog("port 6021 now allowed from pod-a to epg-b") cmd2 = ['nc', '-zvnw', '1', ip_6021, '6021'] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-a to epg-b[6021] Resp is {}".format(resp)) assert "open" in resp def contractRemChecker(): resp = tutils.verifyAgentContracts([ "GbpEpGroupToConsContractRSrc/288/tcp-6020", "GbpEpGroupToProvContractRSrc/288/tcp-6020" ], False) if resp == "": print( "GbpEpGroupToConsContractRSrc/288/tcp-6020, GbpEpGroupToProvContractRSrc/288/tcp-6020 removed" ) else: print(resp) return resp tutils.assertEventually(contractRemChecker, 1, 10) tutils.tcLog("port 6020 now denied from pod-a to epg-b") cmd1 = ['nc', '-zvnw', '1', ip_6020, '6020'] resp = stream(v1.connect_get_namespaced_pod_exec, "pod-a", 'default', command=cmd1, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>pod-a to epg-b[6020] Resp is {}".format(resp)) assert "timed out" in resp toDelete = ["pod-a", "pod-b6020", "pod-b6021"] logging.info("Deleting {}\n".format(toDelete)) for pod in toDelete: v1.delete_namespaced_pod(pod, "default", client.V1DeleteOptions()) for pod in toDelete: tutils.checkPodDeleted(v1, "default", pod, 120) deleteCRD("contracts", "tcp-6020") deleteCRD("contracts", "tcp-6021") deleteCRD("epgs", "epg-a") deleteCRD("epgs", "epg-b") tutils.checkAgentLog()
def test_np(object): tutils.tcLog("Verify GW flows are present") tutils.checkGwFlows("11.3.0.1") tutils.tcLog("Create a namespace, with simple service") createNs("e2e") createNsPod("e2e", "srvr-80") createNsPod("e2e", "srvr-81") svcIP = createNsSvc("e2e", "simple-svc") tutils.tcLog("Create two client pods") client_pods = ["client-a", "client-b"] for pod in client_pods: createNsPod("e2e", pod) tutils.tcLog("Verify access without netpol") for pod in client_pods: verifyAccess("e2e", pod, svcIP, "80", "open") verifyAccess("e2e", pod, svcIP, "81", "open") tutils.tcLog("Delete clients") for pod in client_pods: tutils.deletePod("e2e", pod) tutils.tcLog("Apply network policy client-a->port80") createNsNetPol("e2e", "upd_policy1") tutils.tcLog("Create two client pods") for pod in client_pods: createNsPod("e2e", pod) tutils.tcLog("Verify client-a can access port 80") verifyAccess("e2e", "client-a", svcIP, "80", "open") tutils.tcLog("Verify failures for a/81, b/[80,81]") verifyAccess("e2e", "client-a", svcIP, "81", "timed out") verifyAccess("e2e", "client-b", svcIP, "80", "timed out") verifyAccess("e2e", "client-b", svcIP, "81", "timed out") tutils.tcLog("Delete clients") for pod in client_pods: tutils.deletePod("e2e", pod) tutils.tcLog("Update network policy client-b->port81") replaceNsNetPol("e2e", "upd_policy2") tutils.tcLog("Create two client pods") for pod in client_pods: createNsPod("e2e", pod) tutils.tcLog("Verify client-b can access port 81") verifyAccess("e2e", "client-b", svcIP, "81", "open") tutils.tcLog("Verify failures for b/81, a/[80,81]") verifyAccess("e2e", "client-b", svcIP, "80", "timed out") verifyAccess("e2e", "client-a", svcIP, "80", "timed out") verifyAccess("e2e", "client-a", svcIP, "81", "timed out") v1 = client.CoreV1Api() v1.delete_namespace("e2e", client.V1DeleteOptions()) def nsDelChecker(): if tutils.namespaceExists("e2e"): return "e2e exists" return "" tutils.tcLog("Verify ns is deleted") tutils.assertEventually(nsDelChecker, 2, 40)
def test_default(object): tutils.tcLog("Verify GW flows are present") tutils.checkGwFlows("11.3.0.1") delete_rc = True k8s_api = client.CoreV1Api() if not checkRCStatus(k8s_api, 3): logging.debug("Using existing rc") delete_rc = False else: tutils.tcLog("Create 3 pods in default epg") k8s_api = utils.create_from_yaml(k8s_client, "yamls/busybox.yaml") # check rc is ready def rcChecker(): return checkRCStatus(k8s_api, 3) tutils.assertEventually(rcChecker, 1, 60) tutils.checkAgentLog() # check pods are ready def podChecker(): pod_list = k8s_api.list_namespaced_pod( "default", label_selector="app=busybox") if len(pod_list.items) < 3: return "Expected 3 pods, got {}".format(len(pod_list.items)) for pod in pod_list.items: if pod.status.phase != "Running": return "pod {} status is {}".format( pod.metadata.name, pod.status.phase) return "" tutils.assertEventually(podChecker, 1, 60) # verify connectivity pod_ips = tutils.getPodIPs("default", "app=busybox") tutils.tcLog("Verify EPs are populated") def remEPChecker(): res = tutils.verifyAgentEPs(pod_ips) if res == "": print("{} present".format(pod_ips)) else: print(res) return res tutils.assertEventually(remEPChecker, 1, 10) node_ips = getNodeIPs(k8s_api) ips = pod_ips + node_ips print("ip's are: {}".format(ips)) pod_list = k8s_api.list_namespaced_pod("default", label_selector="app=busybox") pod = next(iter(pod_list.items), None) assert pod != None tutils.tcLog("Verify default connectivity") def pingChecker(): ping_cmd = ['ping', '-c', '3'] for ip in ips: v1 = client.CoreV1Api() cmd = list(ping_cmd) cmd.append(ip) resp = stream(v1.connect_get_namespaced_pod_exec, pod.metadata.name, 'default', command=cmd, stderr=True, stdin=False, stdout=True, tty=False) print("=>Resp is {}".format(resp)) if "3 packets received" not in resp: return "3 packets not received" return "" tutils.assertEventually(pingChecker, 1, 60) tutils.tcLog("Check pods access to API server") v1 = client.CoreV1Api() s = v1.read_namespaced_service("kubernetes", "default") svcIP = s.spec.cluster_ip logging.debug("=>K8s svc IP is {}".format(svcIP)) pod_list = k8s_api.list_namespaced_pod("default", label_selector="app=busybox") cmd2 = ['nc', '-zvnw', '1', svcIP, '443'] def k8sChecker(): for pod in pod_list.items: resp = stream(v1.connect_get_namespaced_pod_exec, pod.metadata.name, 'default', command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>Resp is {}".format(resp)) if "open" not in resp: return resp return "" tutils.assertEventually(k8sChecker, 1, 10) if delete_rc: tutils.tcLog("Delete pods") tutils.scaleRc("busybox", 0) k8s_api.delete_namespaced_replication_controller( "busybox", "default", client.V1DeleteOptions()) else: logging.debug("Skipping rc delete")
def test_np(object): tutils.tcLog("Verify GW flows are present") tutils.checkGwFlows("11.3.0.1") # create a service/deployment in default namespace svcIP = createNsSvc("default", "hostnames-svc") k8s_api = utils.create_from_yaml(k8s_client, "yamls/hostnames-dep.yaml") v1 = client.CoreV1Api() # check deployment is ready svc_hosts = [] def depChecker(): svc_hosts = getPodNames(v1, "default", "app=hostnames") if len(svc_hosts) >= 3: return "" return "Need 3 hosts, have {}".format(len(svc_hosts)) tutils.assertEventually(depChecker, 1, 180) # create two namespaces, with a client pod in each nsList = ["prod", "dev"] for ns in nsList: createNs(ns) logging.info("\nWaiting for namespaces to be available") sleep(3) for ns in nsList: createNsPod(ns, "client-pod") tutils.tcLog("Verify loadbalancing from dev and prod") # verify both clients can access the service, and the service load balances. for ns in nsList: cmd = ['curl', '--connect-timeout', '1', '-s', svcIP] backends = dict() for count in range(0, 12): resp = stream(v1.connect_get_namespaced_pod_exec, "client-pod", ns, command=cmd, stderr=True, stdin=False, stdout=True, tty=False) if resp == "": sleep(1) continue backends[resp] = True if len(backends) == 2: break lbWorkaround() #assert len(backends) == 2 logging.info("backends: {}".format(backends.keys())) tutils.tcLog("Verify there are no timeouts") nc_cmd = ['nc', '-zvnw', '1', svcIP, '80'] for ns in nsList: for count in range(0, 6): resp = stream(v1.connect_get_namespaced_pod_exec, "client-pod", ns, command=nc_cmd, stderr=True, stdin=False, stdout=True, tty=False) if 'open' not in resp: print("Error - {}, expected open".format(resp)) assert (False) lbWorkaround() # apply networkpolicy allowing access only to prod tutils.tcLog("Create k8s network policy") createNsNetPol("default", "hostnames-allow-prod") sleep(10) cmd = ['curl', '--connect-timeout', '1', svcIP] # wait for netpol to take effect def waiter(): count = 0 for ix in range(0, 10): resp1 = stream(v1.connect_get_namespaced_pod_exec, "client-pod", "dev", command=cmd, stderr=True, stdin=False, stdout=True, tty=False) if "timed out" in resp1: count += 1 else: sleep(1) if count == 10: return "" return "still accessible" tutils.tcLog("Verify dev cannot access the svc") tutils.assertEventually(waiter, 1, 120) tutils.tcLog("Verify prod can access the svc") # verify prod can access the svc cmd2 = ['nc', '-zvnw', '1', svcIP, '80'] def prodChecker(): for ix in range(0, 3): resp2 = stream(v1.connect_get_namespaced_pod_exec, "client-pod", "prod", command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("prod: {}".format(resp2)) if "open" not in resp2: return resp2 lbWorkaround() return "" clientIP = tutils.getPodIP("client-pod", "prod") ovs_pods = getPodNames(v1, "kube-system", "name=aci-containers-openvswitch") ovs_cmd = ['ovs-ofctl', 'dump-flows', 'br-access', '-OOpenFlow13'] def npInspector(): tutils.inspectLog( "Look for flows on br-access for {}".format(clientIP)) for ovs_pod in ovs_pods: ovs_resp = stream(v1.connect_get_namespaced_pod_exec, ovs_pod, "kube-system", command=ovs_cmd, stderr=True, stdin=False, stdout=True, tty=False) print("ovs_Pod: {}\n".format(ovs_pod)) for line in ovs_resp.splitlines(): if clientIP in line: print(line) tutils.assertEventually(prodChecker, 1, 5, npInspector) tutils.tcLog("Verify dev still can't access the svc") # and dev can't for ix in range(0, 3): resp3 = stream(v1.connect_get_namespaced_pod_exec, "client-pod", "dev", command=cmd2, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("dev: {}".format(resp3)) assert "timed out" in resp3 lbWorkaround() # delete everything nv1 = client.NetworkingV1Api() nv1.delete_namespaced_network_policy("hostnames-allow-prod", "default", client.V1DeleteOptions()) # verify access-br flows are deleted def flowChecker(): for ovs_pod in ovs_pods: ovs_resp = stream(v1.connect_get_namespaced_pod_exec, ovs_pod, "kube-system", command=ovs_cmd, stderr=True, stdin=False, stdout=True, tty=False) for line in ovs_resp.splitlines(): if clientIP in line: return line return "" tutils.tcLog("Verify network policy flows are deleted") tutils.assertEventually(flowChecker, 1, 120) tutils.scaleDep("default", "hostnames-dep", 0) for ns in nsList: v1.delete_namespace( ns, client.V1DeleteOptions()) # deletes the client-pod too v1.delete_namespaced_service("hostnames-svc", "default", client.V1DeleteOptions()) av1 = client.AppsV1Api() av1.delete_namespaced_deployment("hostnames-dep", "default", client.V1DeleteOptions()) def delChecker(): dList = av1.list_namespaced_deployment("default") for dep in dList.items: if dep.metadata.name == "hostnames-dep": return "hostnames-dep still present" sList = v1.list_namespaced_service("default") for svc in sList.items: if svc.metadata.name == "hostnames-svc": return "hostnames-svc still present" for ns in nsList: if tutils.namespaceExists(ns): return "{} exists".format(ns) return "" tutils.tcLog("Verify svc/dep is deleted") tutils.assertEventually(delChecker, 2, 40)
def getCSRs(object): tutils.tcLog("Get csr objects") qry = '/api/class/hcloudCsr.json' resp = object.apic.get(path=qry) resJson = json.loads(resp.content) return resJson["imdata"]
def test_cni_status(object): tutils.tcLog("Verify status of system pods") assertEventually(checkPodStatus, 1, 90) tutils.checkAgentLog()
def test_default(object): tutils.tcLog("Create 3 pods in default epg") k8s_api = utils.create_from_yaml(k8s_client, "yamls/busybox.yaml") # check rc is ready def rcChecker(): return checkRCStatus(k8s_api, 3) tutils.assertEventually(rcChecker, 1, 60) tutils.checkAgentLog() # check pods are ready def podChecker(): pod_list = k8s_api.list_namespaced_pod( "default", label_selector="app=busybox") if len(pod_list.items) != 3: return "Expected 3 pods, got {}".format(len(pod_list.items)) for pod in pod_list.items: if pod.status.phase != "Running": return "pod {} status is {}".format( pod.metadata.name, pod.status.phase) return "" tutils.assertEventually(podChecker, 1, 60) # verify connectivity ips = tutils.getPodIPs("default", "app=busybox") pod_list = k8s_api.list_namespaced_pod("default", label_selector="app=busybox") pod = next(iter(pod_list.items), None) assert pod != None tutils.tcLog("Verify default connectivity") def pingChecker(): ping_cmd = ['ping', '-c', '3'] for ip in ips: v1 = client.CoreV1Api() cmd = list(ping_cmd) cmd.append(ip) resp = stream(v1.connect_get_namespaced_pod_exec, pod.metadata.name, 'default', command=cmd, stderr=True, stdin=False, stdout=True, tty=False) logging.debug("=>Resp is {}".format(resp)) if "3 packets received" not in resp: return "3 packets not received" return "" tutils.assertEventually(pingChecker, 1, 60) tutils.tcLog("Delete pods") tutils.scaleRc("busybox", 0) k8s_api.delete_namespaced_replication_controller( "busybox", "default", client.V1DeleteOptions())
def test_np(object): tutils.tcLog("Verify GW flows are present") tutils.checkGwFlows("11.3.0.1") createNs("prod") # create a deployment in default namespace k8s_api = utils.create_from_yaml(k8s_client, "yamls/hostnames-dep.yaml") v1 = client.CoreV1Api() # check deployment is ready svc_hosts = [] def depChecker(): svc_hosts = getPodNames(v1, "default", "app=hostnames") if len(svc_hosts) >= 3: return "" return "Need 3 hosts, have {}".format(len(svc_hosts)) tutils.assertEventually(depChecker, 1, 180) createNsPod("default", "client-pod") tutils.tcLog("Verify access without netpol") podIPs = tutils.getPodIPs("default", "app=hostnames") assert len(podIPs) == 3 verifyAccess('default', 'open', podIPs) # apply networkpolicy allowing access only to prod tutils.tcLog("Create k8s network policy") createNsNetPol("default", "np/hostnames-deny-all") tutils.tcLog("Verify access denied") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Setup a netpol to allow access from prod") createNsNetPol("default", "hostnames-allow-prod") createNsPod("prod", "client-pod") tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.tcLog("Verify deny from default") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Delete deny-all") deleteNsNetPol('default', 'hostnames-deny-all') tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.tcLog("Verify deny from default") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Delete allow-prod") deleteNsNetPol('default', 'hostnames-allow-prod') tutils.tcLog("Verify allow from default") verifyAccess('default', 'open', podIPs) tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.tcLog("Allow port 9000 from prod") createNsNetPol("default", "np/hostnames-allow-prod9000") tutils.tcLog("Verify deny from default") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Verify deny 9376 from prod") verifyAccess('prod', 'timed out', podIPs) tutils.tcLog("Delete allow-prod9000") deleteNsNetPol('default', 'hostnames-allow-prod9000') tutils.tcLog("Allow port 9376 from prod") createNsNetPol("default", "np/hostnames-allow-prod9376") tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.tcLog("Verify deny from default") verifyAccess('default', 'timed out', podIPs) tutils.tcLog("Delete allow-prod9376") deleteNsNetPol('default', 'hostnames-allow-prod9376') tutils.tcLog("Verify allow from default") verifyAccess('default', 'open', podIPs) tutils.tcLog("Verify allow from prod") verifyAccess('prod', 'open', podIPs) tutils.scaleDep("default", "hostnames-dep", 0) v1.delete_namespace( 'prod', client.V1DeleteOptions()) # deletes the client-pod too av1 = client.AppsV1Api() av1.delete_namespaced_deployment("hostnames-dep", "default", client.V1DeleteOptions()) tutils.deletePod("default", "client-pod") def delChecker(): dList = av1.list_namespaced_deployment("default") for dep in dList.items: if dep.metadata.name == "hostnames-dep": return "hostnames-dep still present" if tutils.namespaceExists('prod'): return "prod exists" return "" tutils.tcLog("Verify cleanup") tutils.assertEventually(delChecker, 2, 40)
def test_kafka(object): # setup kafka services tutils.tcLog("Setup kafka services") for ky in kafkaYamls: try: utils.create_from_yaml(k8s_client, "yamls/"+ky) except ApiException as e: logging.debug("{} - ignored".format(e.reason)) k8s_api = client.CoreV1Api() # check kafka server is ready assertPodReady("default", "ut-kafka-0", 120) assertPodReady("default", "kafkakv", 120) sleep(5) # collect the current list of ep's from k8s tutils.tcLog("Collect ep's from k8s") initialEPList = readCniEPList() logging.info("EPList is {}".format(initialEPList)) tutils.tcLog("Verifying initial EPList with kafka") kafkaSyncChecker(initialEPList) tutils.tcLog("Adding a pod and checking it in kafka") utils.create_from_yaml(k8s_client, "yamls/alpine-pod.yaml") assertPodReady("default", "alpine-pod", 45) vtep = tutils.getPodNodeIP("alpine-pod", "default") tutils.tcLog("Check for podif") crdApi = client.CustomObjectsApi() group = "aci.aw" ns = "kube-system" podifName = "default.alpine-pod.{}".format(vtep) p = crdApi.get_namespaced_custom_object(group, "v1", ns, "podifs", podifName) logging.debug("podif: {}".format(p)) epStatus = p['status'] epID = "{}.{}.{}".format(epStatus['podns'], epStatus['podname'], epStatus['ifname']) toCheck = [] toCheck.append(epID) kafkaChecker(toCheck, "found") tutils.tcLog("Delete a pod and check removal from kafka") k8s_api.delete_namespaced_pod("alpine-pod", "default", client.V1DeleteOptions()) kafkaChecker(toCheck, "missing", 8) tutils.tcLog("Recheck ep sync between k8s and kafka") kafkaSyncChecker(initialEPList) tutils.tcLog("Add more endpoints") utils.create_from_yaml(k8s_client, "yamls/busybox.yaml") tutils.rcCheckScale("busybox", 3) tutils.tcLog("Get new eplist from k8s") EPList1 = readCniEPList() assert EPList1 != initialEPList tutils.tcLog("Recheck ep sync between k8s and kafka") kafkaSyncChecker(EPList1) tutils.tcLog("Bring controller down") scaleDep("kube-system", "aci-containers-controller", 0) sleep(10) tutils.tcLog("Change some endpoints") tutils.scaleRc("busybox", 0) tutils.checkPodsRemoved("app=busybox") EPList2 = readCniEPList() assert EPList2 != EPList1 tutils.scaleRc("busybox", 2) sleep(5) logging.debug("previous ep list:{}".format(EPList1)) EPList3 = readCniEPList() logging.debug("new ep list:{}".format(EPList3)) assert EPList3 != EPList1 tutils.tcLog("Bring controller up") scaleDep("kube-system", "aci-containers-controller", 1) sleep(10) tutils.tcLog("Verify controller is up") tutils.assertEventually(checkSystemStatus, 1, 60) tutils.tcLog("Check ep sync again") kafkaSyncChecker(EPList3, 6) tutils.tcLog("Clean up added endpoints") tutils.scaleRc("busybox", 0) tutils.checkPodsRemoved("app=busybox") k8s_api.delete_namespaced_replication_controller("busybox", "default", client.V1DeleteOptions())