Example #1
0
 def verifyULConnectivity(object, tepList):
     tutils.tcLog("Create pods in host ns")
     pods = tutils.createDs("ul-tester")
     for pod in pods:
         for tep in tepList:
             tutils.verifyPing(pod, "default", tep)
     tutils.tcLog("Delete pods in host ns")
     tutils.deleteDs("ul-tester")
Example #2
0
 def countCompHv(object):
     vmm_name = object.config["aci_config"]["system_id"]
     tutils.tcLog("Count compHv objects for {}".format(vmm_name))
     qry = '/api/mo/comp/prov-Kubernetes/ctrlr-[{}]-{}.json?query-target=children&target-subtree-class=compHv'.format(
         vmm_name, vmm_name)
     resp = object.apic.get(path=qry)
     resJson = json.loads(resp.content)
     return len(resJson["imdata"])
Example #3
0
 def verifyOLConnectivity(object, vmIP):
     tutils.tcLog("Create pods in epg-a")
     pods = tutils.createDs("ol-tester")
     object.verifyCapicEPs(pods, "epg-a")
     for pod in pods:
         tutils.verifyPing(pod, "default", vmIP)
     tutils.tcLog("Delete pods in epg-a")
     tutils.deleteDs("ol-tester")
Example #4
0
 def getCniTep(object):
     tutils.tcLog("Get CniTep objects")
     qry = '/api/class/hcl3CniTep.json'
     resp = object.apic.get(path=qry)
     resJson = json.loads(resp.content)
     tepList = []
     for mo in resJson['imdata']:
         addr = mo['hcl3CniTep']['attributes']['addr']
         tep = addr.split("/")[0]
         tepList.append(tep)
     return tepList
Example #5
0
 def verifyClusterInfo(object):
     vmm_name = object.config["aci_config"]["system_id"]
     tn_name = object.config["aci_config"]["tenant"]["name"]
     tutils.tcLog("Verify cluster info objects for {}/{}".format(
         tn_name, vmm_name))
     moClasses = ["compClusterInfo", "hcloudClusterInfo"]
     for moCl in moClasses:
         qry = '/api/class/{}.json?query-target-filter=and(eq({}.name,"{}"),eq({}.accountName,"{}"))'.format(
             moCl, moCl, moCl, vmm_name, tn_name)
         resp = object.apic.get(path=qry)
         resJson = json.loads(resp.content)
         assert len(resJson["imdata"]) == 1
Example #6
0
 def getVmIP(object, epg):
     tutils.tcLog("Get vm ip for epg {}".format(epg))
     qry = '/api/class/hcloudRtEpToSecurityGroup.json?query-target-filter=and(wcard(hcloudRtEpToSecurityGroup.dn,"{}"))'.format(
         epg)
     resp = object.apic.get(path=qry)
     epDns = object.getFromResp(resp, 'hcloudRtEpToSecurityGroup', 'tDn')
     assert len(epDns) > 0
     print("epDn: {}".format(epDns[0]))
     qry = '/api/mo/{}.json'.format(epDns[0])
     resp = object.apic.get(path=qry)
     epIPs = object.getFromResp(resp, 'hcloudEndPoint', 'primaryIpV4Addr')
     assert len(epIPs) > 0
     return epIPs[0]
Example #7
0
 def getAllEpIPs(object, epg):
     tutils.tcLog("Get all hcloudEP ip for epg {}".format(epg))
     ip_dict = {}
     qry = '/api/class/hcloudRtEpToSecurityGroup.json?query-target-filter=and(wcard(hcloudRtEpToSecurityGroup.dn,"{}]"))'.format(
         epg)
     resp = object.apic.get(path=qry)
     epDns = object.getFromResp(resp, 'hcloudRtEpToSecurityGroup', 'tDn')
     for epDn in epDns:
         qry = '/api/mo/{}.json'.format(epDn)
         resp = object.apic.get(path=qry)
         epIPs = object.getFromResp(resp, 'hcloudEndPoint',
                                    'primaryIpV4Addr')
         if len(epIPs) > 0:
             ep_ip = epIPs[0].split("/")[0]
             ip_dict[ep_ip] = True
     return ip_dict
Example #8
0
    def verifyCsr(object, csr, tunnelInfo, vrf_encap_id):
        dn = csr["hcloudCsr"]["attributes"]["dn"]
        qry = '/api/mo/{}.json?query-target=subtree&target-subtree-class=hcloudEndPointOper&query-target-filter=and(ne(hcloudEndPointOper.publicIpv4Addr,"0.0.0.0"))'.format(
            dn)
        resp = object.apic.get(path=qry)
        resJson = json.loads(resp.content)

        def getCsrIP():
            for epOper in resJson["imdata"]:
                if "hcloudEndPointOper" in epOper:
                    att = epOper["hcloudEndPointOper"]["attributes"]
                    operDn = att["dn"]
                    if "nwif-0" in operDn:
                        print("CSR IP: {}".format(att["publicIpv4Addr"]))
                        return att["publicIpv4Addr"]
            return None

        csrIP = getCsrIP()
        assert csrIP is not None
        print(csrIP)
        csr_c = paramiko.SSHClient()
        csr_c.load_system_host_keys()
        csr_c.set_missing_host_key_policy(paramiko.AutoAddPolicy())
        pw = object.config["aci_config"]["apic_login"]["password"]
        csr_c.connect(csrIP, username='******', password=pw)
        stdin, stdout, stderr = csr_c.exec_command(
            'show run | sec interface Tunn')
        # format output
        out_dict = {}
        key = ""
        for line in stdout:
            ll = line.strip('\n')
            #print(ll)
            if "interface Tunnel" in ll:
                key = ll.split()[1]
                out_dict[key] = {}
            else:
                if key:
                    out_dict[key][ll.strip()] = True
        csr_c.close()
        tutils.tcLog("Verify tunnels {} is in CSR".format(tunnelInfo))
        object.verifyTunnels(out_dict, tunnelInfo, vrf_encap_id)
        tutils.tcLog("Verify routes in CSR({})".format(csrIP))
        object.verifyCsrRoutes(csr_c, csrIP)
Example #9
0
    def test_concrete(object):
        object.setup()
        addr = object.config["aci_config"]["apic_hosts"][0]
        tutils.tcLog("Verify capic {} access".format(addr))
        assert object.apic is not None
        object.verifyClusterInfo()
        nodeCount = object.countCompHv()
        print("nodeCount: {}".format(nodeCount))
        csrList = object.getCSRs()
        csrCount = len(csrList)
        print("csrCount: {}".format(csrCount))
        if nodeCount == 0 or csrCount == 0:
            print("No further tests possible with these csr/compHv count")
            return
        tepList = object.getCniTep()
        assert len(tepList) > 0

        tutils.tcLog(
            "Verify connectivity from host ns to teps {}".format(tepList))
        object.verifyULConnectivity(tepList)

        tunnelInfo = object.getTunnelInfo()
        print(tunnelInfo)
        vrf_encap_id = tutils.read_vrf_encap_id()
        print("vrf-encap-id is {}".format(vrf_encap_id))
        for csr in csrList:
            object.verifyCsr(csr, tunnelInfo, vrf_encap_id)

        vmIP = object.getVmIP("test-vm-epg")
        print("vmIP: {}".format(vmIP))
        assert vmIP

        tutils.tcLog("Verify connectivity from epg-a to {}".format(vmIP))
        object.verifyOLConnectivity(vmIP)
Example #10
0
    def test_nslookup(object):
        v1 = client.CoreV1Api()
        # use alpine because busybox nslookup seems broken
        tutils.tcLog("Create a dns client pod")
        createPod("alpine-pod")

        s = v1.read_namespaced_service("kubernetes", "default")
        svcIP = s.spec.cluster_ip
        # verify ping fails across epgs
        print("\nVerify nslookup")
        tutils.tcLog("Verify dns lookup of kubernetes service")
        ns_lookup_cmd = ['nslookup', 'kubernetes']
        def respChecker():
            logging.debug("===svcIP is {}".format(svcIP))
            resp = stream(v1.connect_get_namespaced_pod_exec, "alpine-pod", 'default',
                      command=ns_lookup_cmd, stderr=True, stdin=False, stdout=True, tty=False)
            if svcIP in resp:
                logging.debug("=>command {}".format(ns_lookup_cmd))
                logging.debug("=>Resp is {}".format(resp))
                return ""
            else:
                logging.debug("NR=>Resp is {}".format(resp))
                #return ""
                #FIXME this is not reliable yet...
                return "svc not resolved {}".format(ns_lookup_cmd)
        
        # Basic inspection in case of a dns test failer
        def dnsInspector():
            tutils.inspectLog("Checking coredns status on failure")
            av1 = client.AppsV1Api()
            sr = av1.read_namespaced_deployment_status("coredns", "kube-system")
            tutils.inspectLog("CoreDNS available: {}, ready: {}".format(sr.status.available_replicas, sr.status.ready_replicas))

        tutils.assertEventually(respChecker, 1, 5, dnsInspector)

        tutils.tcLog("Create a service and check dns resolution")
        ns_lookup_cmd = ['nslookup', 'dns-test-svc']
        svcIP = createNsSvc("default", "dns-test-svc")
        logging.info("***svcIP is {}".format(svcIP))
        tutils.assertEventually(respChecker, 1, 60)

        v1.delete_namespaced_pod("alpine-pod", "default", client.V1DeleteOptions())
        v1.delete_namespaced_service("dns-test-svc", "default", client.V1DeleteOptions())
    def test_ovs_restart(object):
        tutils.tcLog("Getting list of ovs pods")
        node_list = v1.list_node()
        ns = tutils.getSysNs()
        pod_list = v1.list_namespaced_pod(ns, label_selector="name=aci-containers-openvswitch")
        assert len(node_list.items) > 0
        assert len(node_list.items) == len(pod_list.items)
        tutils.tcLog("Delete current ovs pods")
        for pod in pod_list.items:
            tutils.deletePod(ns, pod.metadata.name)

        tutils.tcLog("Check new ovs pods")
        def checker():
            new_pod_list = v1.list_namespaced_pod(ns, label_selector="name=aci-containers-openvswitch")
            if len(pod_list.items) == len(new_pod_list.items):
                return ""
            return "{} ne {}".format(len(new_pod_list.items), len(pod_list.items))
 
        tutils.assertEventually(checker, 2, 30)
Example #12
0
    def test_policy(object):
        v1 = client.CoreV1Api()
        createCRD("contracts", "tcp-6020")
        createCRD("contracts", "tcp-6021")
        sleep(1)
        createCRD("epgs", "epg-a")
        createCRD("epgs", "epg-b")
        # pod in epg-a
        createPod("pod-a")
        # pods in epg-b
        ip_6020 = createPod("pod-b6020")
        ip_6021 = createPod("pod-b6021")
        tutils.checkAgentLog()

        # verify ping fails across epgs
        tutils.tcLog("Verify ping failure across epgs")
        ping_cmd = ['ping', '-c', '6', '-t', '1', ip_6020]
        resp = stream(v1.connect_get_namespaced_pod_exec,
                      "pod-a",
                      'default',
                      command=ping_cmd,
                      stderr=True,
                      stdin=False,
                      stdout=True,
                      tty=False)
        logging.debug("=>Resp is {}".format(resp))
        assert "0 packets received" in resp

        tutils.tcLog("Verify tcp contract")
        # verify port 6020 access from pod-a to epg-b
        tutils.tcLog("port 6020 access from pod-a to epg-b")
        cmd1 = ['nc', '-zvnw', '1', ip_6020, '6020']
        resp = stream(v1.connect_get_namespaced_pod_exec,
                      "pod-a",
                      'default',
                      command=cmd1,
                      stderr=True,
                      stdin=False,
                      stdout=True,
                      tty=False)
        logging.debug("=>pod-a to epg-b[6020] Resp is {}".format(resp))
        assert "open" in resp

        sleep(5)
        # verify port 6021 is inaccessible from pod-a to epg-b
        tutils.tcLog("port 6021 deny from pod-a to epg-b")
        cmd2 = ['nc', '-zvnw', '1', ip_6021, '6021']
        resp = stream(v1.connect_get_namespaced_pod_exec,
                      "pod-a",
                      'default',
                      command=cmd2,
                      stderr=True,
                      stdin=False,
                      stdout=True,
                      tty=False)
        logging.debug("=>pod-a to epg-b[6021] Resp is {}".format(resp))
        assert "timed out" in resp

        sleep(5)
        # verify port 6021 is accessible within epg-b
        tutils.tcLog("port 6021 accessible within epg-b")
        resp = stream(v1.connect_get_namespaced_pod_exec,
                      "pod-b6020",
                      'default',
                      command=cmd2,
                      stderr=True,
                      stdin=False,
                      stdout=True,
                      tty=False)
        logging.debug("=>pod-b6020 to pod-b[6021] Resp is {}".format(resp))
        assert "open" in resp

        tutils.tcLog("Change allowed port to 6021")
        replaceCRD("epgs", "epg-a", "epg-a-upd")
        replaceCRD("epgs", "epg-b", "epg-b-upd")
        tutils.tcLog("Verify new contract is on agents")

        def contractChecker():
            resp = tutils.verifyAgentContracts(["GbpeL24Classifier/tcp-6021"],
                                               True)
            if resp == "":
                print("GbpeL24Classifier/tcp-6021 present")
            else:
                print(resp)

            return resp

        tutils.assertEventually(contractChecker, 1, 10)
        sleep(5)
        tutils.tcLog("port 6021 now allowed from pod-a to epg-b")
        cmd2 = ['nc', '-zvnw', '1', ip_6021, '6021']
        resp = stream(v1.connect_get_namespaced_pod_exec,
                      "pod-a",
                      'default',
                      command=cmd2,
                      stderr=True,
                      stdin=False,
                      stdout=True,
                      tty=False)
        logging.debug("=>pod-a to epg-b[6021] Resp is {}".format(resp))
        assert "open" in resp

        def contractRemChecker():
            resp = tutils.verifyAgentContracts([
                "GbpEpGroupToConsContractRSrc/288/tcp-6020",
                "GbpEpGroupToProvContractRSrc/288/tcp-6020"
            ], False)
            if resp == "":
                print(
                    "GbpEpGroupToConsContractRSrc/288/tcp-6020, GbpEpGroupToProvContractRSrc/288/tcp-6020 removed"
                )
            else:
                print(resp)

            return resp

        tutils.assertEventually(contractRemChecker, 1, 10)
        tutils.tcLog("port 6020 now denied from pod-a to epg-b")
        cmd1 = ['nc', '-zvnw', '1', ip_6020, '6020']
        resp = stream(v1.connect_get_namespaced_pod_exec,
                      "pod-a",
                      'default',
                      command=cmd1,
                      stderr=True,
                      stdin=False,
                      stdout=True,
                      tty=False)
        logging.debug("=>pod-a to epg-b[6020] Resp is {}".format(resp))
        assert "timed out" in resp

        toDelete = ["pod-a", "pod-b6020", "pod-b6021"]
        logging.info("Deleting {}\n".format(toDelete))
        for pod in toDelete:
            v1.delete_namespaced_pod(pod, "default", client.V1DeleteOptions())
        for pod in toDelete:
            tutils.checkPodDeleted(v1, "default", pod, 120)

        deleteCRD("contracts", "tcp-6020")
        deleteCRD("contracts", "tcp-6021")
        deleteCRD("epgs", "epg-a")
        deleteCRD("epgs", "epg-b")
        tutils.checkAgentLog()
Example #13
0
    def test_np(object):
        tutils.tcLog("Verify GW flows are present")
        tutils.checkGwFlows("11.3.0.1")
        tutils.tcLog("Create a namespace, with simple service")
        createNs("e2e")
        createNsPod("e2e", "srvr-80")
        createNsPod("e2e", "srvr-81")
        svcIP = createNsSvc("e2e", "simple-svc")
        
        tutils.tcLog("Create two client pods")
        client_pods = ["client-a", "client-b"]
        for pod in client_pods:
            createNsPod("e2e", pod)
        tutils.tcLog("Verify access without netpol")
        for pod in client_pods:
            verifyAccess("e2e", pod, svcIP, "80", "open")
            verifyAccess("e2e", pod, svcIP, "81", "open")

        tutils.tcLog("Delete clients")
        for pod in client_pods:
            tutils.deletePod("e2e", pod)

        tutils.tcLog("Apply network policy client-a->port80")
        createNsNetPol("e2e", "upd_policy1")
        tutils.tcLog("Create two client pods")
        for pod in client_pods:
            createNsPod("e2e", pod)
        tutils.tcLog("Verify client-a can access port 80")
        verifyAccess("e2e", "client-a", svcIP, "80", "open")
        tutils.tcLog("Verify failures for a/81, b/[80,81]")
        verifyAccess("e2e", "client-a", svcIP, "81", "timed out")
        verifyAccess("e2e", "client-b", svcIP, "80", "timed out")
        verifyAccess("e2e", "client-b", svcIP, "81", "timed out")
        tutils.tcLog("Delete clients")
        for pod in client_pods:
            tutils.deletePod("e2e", pod)

        tutils.tcLog("Update network policy client-b->port81")
        replaceNsNetPol("e2e", "upd_policy2")
        tutils.tcLog("Create two client pods")
        for pod in client_pods:
            createNsPod("e2e", pod)

        tutils.tcLog("Verify client-b can access port 81")
        verifyAccess("e2e", "client-b", svcIP, "81", "open")
        tutils.tcLog("Verify failures for b/81, a/[80,81]")
        verifyAccess("e2e", "client-b", svcIP, "80", "timed out")
        verifyAccess("e2e", "client-a", svcIP, "80", "timed out")
        verifyAccess("e2e", "client-a", svcIP, "81", "timed out")
        v1 = client.CoreV1Api()
        v1.delete_namespace("e2e", client.V1DeleteOptions())
        def nsDelChecker():
            if tutils.namespaceExists("e2e"):
                return "e2e exists"
            return ""

        tutils.tcLog("Verify ns is deleted")
        tutils.assertEventually(nsDelChecker, 2, 40)
    def test_default(object):
        tutils.tcLog("Verify GW flows are present")
        tutils.checkGwFlows("11.3.0.1")
        delete_rc = True
        k8s_api = client.CoreV1Api()
        if not checkRCStatus(k8s_api, 3):
            logging.debug("Using existing rc")
            delete_rc = False
        else:
            tutils.tcLog("Create 3 pods in default epg")
            k8s_api = utils.create_from_yaml(k8s_client, "yamls/busybox.yaml")

        # check rc is ready
        def rcChecker():
            return checkRCStatus(k8s_api, 3)

        tutils.assertEventually(rcChecker, 1, 60)
        tutils.checkAgentLog()

        # check pods are ready
        def podChecker():
            pod_list = k8s_api.list_namespaced_pod(
                "default", label_selector="app=busybox")
            if len(pod_list.items) < 3:
                return "Expected 3 pods, got {}".format(len(pod_list.items))

            for pod in pod_list.items:
                if pod.status.phase != "Running":
                    return "pod {} status is {}".format(
                        pod.metadata.name, pod.status.phase)

            return ""

        tutils.assertEventually(podChecker, 1, 60)

        # verify connectivity
        pod_ips = tutils.getPodIPs("default", "app=busybox")

        tutils.tcLog("Verify EPs are populated")

        def remEPChecker():
            res = tutils.verifyAgentEPs(pod_ips)
            if res == "":
                print("{} present".format(pod_ips))
            else:
                print(res)

            return res

        tutils.assertEventually(remEPChecker, 1, 10)

        node_ips = getNodeIPs(k8s_api)
        ips = pod_ips + node_ips
        print("ip's are: {}".format(ips))
        pod_list = k8s_api.list_namespaced_pod("default",
                                               label_selector="app=busybox")
        pod = next(iter(pod_list.items), None)
        assert pod != None

        tutils.tcLog("Verify default connectivity")

        def pingChecker():
            ping_cmd = ['ping', '-c', '3']
            for ip in ips:
                v1 = client.CoreV1Api()
                cmd = list(ping_cmd)
                cmd.append(ip)
                resp = stream(v1.connect_get_namespaced_pod_exec,
                              pod.metadata.name,
                              'default',
                              command=cmd,
                              stderr=True,
                              stdin=False,
                              stdout=True,
                              tty=False)
                print("=>Resp is {}".format(resp))
                if "3 packets received" not in resp:
                    return "3 packets not received"
            return ""

        tutils.assertEventually(pingChecker, 1, 60)
        tutils.tcLog("Check pods access to API server")

        v1 = client.CoreV1Api()
        s = v1.read_namespaced_service("kubernetes", "default")
        svcIP = s.spec.cluster_ip
        logging.debug("=>K8s svc IP is {}".format(svcIP))

        pod_list = k8s_api.list_namespaced_pod("default",
                                               label_selector="app=busybox")
        cmd2 = ['nc', '-zvnw', '1', svcIP, '443']

        def k8sChecker():
            for pod in pod_list.items:
                resp = stream(v1.connect_get_namespaced_pod_exec,
                              pod.metadata.name,
                              'default',
                              command=cmd2,
                              stderr=True,
                              stdin=False,
                              stdout=True,
                              tty=False)
                logging.debug("=>Resp is {}".format(resp))
                if "open" not in resp:
                    return resp

            return ""

        tutils.assertEventually(k8sChecker, 1, 10)

        if delete_rc:
            tutils.tcLog("Delete pods")
            tutils.scaleRc("busybox", 0)
            k8s_api.delete_namespaced_replication_controller(
                "busybox", "default", client.V1DeleteOptions())
        else:
            logging.debug("Skipping rc delete")
Example #15
0
    def test_np(object):
        tutils.tcLog("Verify GW flows are present")
        tutils.checkGwFlows("11.3.0.1")
        # create a service/deployment in default namespace
        svcIP = createNsSvc("default", "hostnames-svc")
        k8s_api = utils.create_from_yaml(k8s_client,
                                         "yamls/hostnames-dep.yaml")
        v1 = client.CoreV1Api()

        # check deployment is ready
        svc_hosts = []

        def depChecker():
            svc_hosts = getPodNames(v1, "default", "app=hostnames")
            if len(svc_hosts) >= 3:
                return ""
            return "Need 3 hosts, have {}".format(len(svc_hosts))

        tutils.assertEventually(depChecker, 1, 180)

        # create two namespaces, with a client pod in each
        nsList = ["prod", "dev"]
        for ns in nsList:
            createNs(ns)
        logging.info("\nWaiting for namespaces to be available")
        sleep(3)
        for ns in nsList:
            createNsPod(ns, "client-pod")

        tutils.tcLog("Verify loadbalancing from dev and prod")
        # verify both clients can access the service, and the service load balances.
        for ns in nsList:
            cmd = ['curl', '--connect-timeout', '1', '-s', svcIP]
            backends = dict()
            for count in range(0, 12):
                resp = stream(v1.connect_get_namespaced_pod_exec,
                              "client-pod",
                              ns,
                              command=cmd,
                              stderr=True,
                              stdin=False,
                              stdout=True,
                              tty=False)
                if resp == "":
                    sleep(1)
                    continue

                backends[resp] = True
                if len(backends) == 2:
                    break

                lbWorkaround()
            #assert len(backends) == 2
            logging.info("backends: {}".format(backends.keys()))

        tutils.tcLog("Verify there are no timeouts")
        nc_cmd = ['nc', '-zvnw', '1', svcIP, '80']
        for ns in nsList:
            for count in range(0, 6):
                resp = stream(v1.connect_get_namespaced_pod_exec,
                              "client-pod",
                              ns,
                              command=nc_cmd,
                              stderr=True,
                              stdin=False,
                              stdout=True,
                              tty=False)
                if 'open' not in resp:
                    print("Error - {}, expected open".format(resp))
                    assert (False)
                lbWorkaround()

        # apply networkpolicy allowing access only to prod
        tutils.tcLog("Create k8s network policy")
        createNsNetPol("default", "hostnames-allow-prod")
        sleep(10)

        cmd = ['curl', '--connect-timeout', '1', svcIP]

        # wait for netpol to take effect
        def waiter():
            count = 0
            for ix in range(0, 10):
                resp1 = stream(v1.connect_get_namespaced_pod_exec,
                               "client-pod",
                               "dev",
                               command=cmd,
                               stderr=True,
                               stdin=False,
                               stdout=True,
                               tty=False)
                if "timed out" in resp1:
                    count += 1
                else:
                    sleep(1)

                if count == 10:
                    return ""
            return "still accessible"

        tutils.tcLog("Verify dev cannot access the svc")
        tutils.assertEventually(waiter, 1, 120)

        tutils.tcLog("Verify prod can access the svc")
        # verify prod can access the svc
        cmd2 = ['nc', '-zvnw', '1', svcIP, '80']

        def prodChecker():
            for ix in range(0, 3):
                resp2 = stream(v1.connect_get_namespaced_pod_exec,
                               "client-pod",
                               "prod",
                               command=cmd2,
                               stderr=True,
                               stdin=False,
                               stdout=True,
                               tty=False)
                logging.debug("prod: {}".format(resp2))
                if "open" not in resp2:
                    return resp2
                lbWorkaround()

            return ""

        clientIP = tutils.getPodIP("client-pod", "prod")
        ovs_pods = getPodNames(v1, "kube-system",
                               "name=aci-containers-openvswitch")
        ovs_cmd = ['ovs-ofctl', 'dump-flows', 'br-access', '-OOpenFlow13']

        def npInspector():
            tutils.inspectLog(
                "Look for flows on br-access for {}".format(clientIP))
            for ovs_pod in ovs_pods:
                ovs_resp = stream(v1.connect_get_namespaced_pod_exec,
                                  ovs_pod,
                                  "kube-system",
                                  command=ovs_cmd,
                                  stderr=True,
                                  stdin=False,
                                  stdout=True,
                                  tty=False)
                print("ovs_Pod: {}\n".format(ovs_pod))
                for line in ovs_resp.splitlines():
                    if clientIP in line:
                        print(line)

        tutils.assertEventually(prodChecker, 1, 5, npInspector)
        tutils.tcLog("Verify dev still can't access the svc")
        # and dev can't
        for ix in range(0, 3):
            resp3 = stream(v1.connect_get_namespaced_pod_exec,
                           "client-pod",
                           "dev",
                           command=cmd2,
                           stderr=True,
                           stdin=False,
                           stdout=True,
                           tty=False)
            logging.debug("dev: {}".format(resp3))
            assert "timed out" in resp3
            lbWorkaround()

        # delete everything
        nv1 = client.NetworkingV1Api()
        nv1.delete_namespaced_network_policy("hostnames-allow-prod", "default",
                                             client.V1DeleteOptions())

        # verify access-br flows are deleted
        def flowChecker():
            for ovs_pod in ovs_pods:
                ovs_resp = stream(v1.connect_get_namespaced_pod_exec,
                                  ovs_pod,
                                  "kube-system",
                                  command=ovs_cmd,
                                  stderr=True,
                                  stdin=False,
                                  stdout=True,
                                  tty=False)
                for line in ovs_resp.splitlines():
                    if clientIP in line:
                        return line

            return ""

        tutils.tcLog("Verify network policy flows are deleted")
        tutils.assertEventually(flowChecker, 1, 120)

        tutils.scaleDep("default", "hostnames-dep", 0)
        for ns in nsList:
            v1.delete_namespace(
                ns, client.V1DeleteOptions())  # deletes the client-pod too

        v1.delete_namespaced_service("hostnames-svc", "default",
                                     client.V1DeleteOptions())
        av1 = client.AppsV1Api()
        av1.delete_namespaced_deployment("hostnames-dep", "default",
                                         client.V1DeleteOptions())

        def delChecker():
            dList = av1.list_namespaced_deployment("default")
            for dep in dList.items:
                if dep.metadata.name == "hostnames-dep":
                    return "hostnames-dep still present"
            sList = v1.list_namespaced_service("default")
            for svc in sList.items:
                if svc.metadata.name == "hostnames-svc":
                    return "hostnames-svc still present"
            for ns in nsList:
                if tutils.namespaceExists(ns):
                    return "{} exists".format(ns)
            return ""

        tutils.tcLog("Verify svc/dep is deleted")
        tutils.assertEventually(delChecker, 2, 40)
Example #16
0
 def getCSRs(object):
     tutils.tcLog("Get csr objects")
     qry = '/api/class/hcloudCsr.json'
     resp = object.apic.get(path=qry)
     resJson = json.loads(resp.content)
     return resJson["imdata"]
Example #17
0
 def test_cni_status(object):
     tutils.tcLog("Verify status of system pods")
     assertEventually(checkPodStatus, 1, 90)
     tutils.checkAgentLog()
Example #18
0
    def test_default(object):
        tutils.tcLog("Create 3 pods in default epg")
        k8s_api = utils.create_from_yaml(k8s_client, "yamls/busybox.yaml")

        # check rc is ready
        def rcChecker():
            return checkRCStatus(k8s_api, 3)

        tutils.assertEventually(rcChecker, 1, 60)
        tutils.checkAgentLog()

        # check pods are ready
        def podChecker():
            pod_list = k8s_api.list_namespaced_pod(
                "default", label_selector="app=busybox")
            if len(pod_list.items) != 3:
                return "Expected 3 pods, got {}".format(len(pod_list.items))

            for pod in pod_list.items:
                if pod.status.phase != "Running":
                    return "pod {} status is {}".format(
                        pod.metadata.name, pod.status.phase)

            return ""

        tutils.assertEventually(podChecker, 1, 60)

        # verify connectivity
        ips = tutils.getPodIPs("default", "app=busybox")
        pod_list = k8s_api.list_namespaced_pod("default",
                                               label_selector="app=busybox")
        pod = next(iter(pod_list.items), None)
        assert pod != None

        tutils.tcLog("Verify default connectivity")

        def pingChecker():
            ping_cmd = ['ping', '-c', '3']
            for ip in ips:
                v1 = client.CoreV1Api()
                cmd = list(ping_cmd)
                cmd.append(ip)
                resp = stream(v1.connect_get_namespaced_pod_exec,
                              pod.metadata.name,
                              'default',
                              command=cmd,
                              stderr=True,
                              stdin=False,
                              stdout=True,
                              tty=False)
                logging.debug("=>Resp is {}".format(resp))
                if "3 packets received" not in resp:
                    return "3 packets not received"
            return ""

        tutils.assertEventually(pingChecker, 1, 60)

        tutils.tcLog("Delete pods")
        tutils.scaleRc("busybox", 0)
        k8s_api.delete_namespaced_replication_controller(
            "busybox", "default", client.V1DeleteOptions())
    def test_np(object):
        tutils.tcLog("Verify GW flows are present")
        tutils.checkGwFlows("11.3.0.1")
        createNs("prod")
        # create a deployment in default namespace
        k8s_api = utils.create_from_yaml(k8s_client,
                                         "yamls/hostnames-dep.yaml")
        v1 = client.CoreV1Api()

        # check deployment is ready
        svc_hosts = []

        def depChecker():
            svc_hosts = getPodNames(v1, "default", "app=hostnames")
            if len(svc_hosts) >= 3:
                return ""
            return "Need 3 hosts, have {}".format(len(svc_hosts))

        tutils.assertEventually(depChecker, 1, 180)
        createNsPod("default", "client-pod")

        tutils.tcLog("Verify access without netpol")
        podIPs = tutils.getPodIPs("default", "app=hostnames")
        assert len(podIPs) == 3
        verifyAccess('default', 'open', podIPs)
        # apply networkpolicy allowing access only to prod
        tutils.tcLog("Create k8s network policy")
        createNsNetPol("default", "np/hostnames-deny-all")
        tutils.tcLog("Verify access denied")
        verifyAccess('default', 'timed out', podIPs)

        tutils.tcLog("Setup a netpol to allow access from prod")
        createNsNetPol("default", "hostnames-allow-prod")
        createNsPod("prod", "client-pod")
        tutils.tcLog("Verify allow from prod")
        verifyAccess('prod', 'open', podIPs)
        tutils.tcLog("Verify deny from default")
        verifyAccess('default', 'timed out', podIPs)

        tutils.tcLog("Delete deny-all")
        deleteNsNetPol('default', 'hostnames-deny-all')
        tutils.tcLog("Verify allow from prod")
        verifyAccess('prod', 'open', podIPs)
        tutils.tcLog("Verify deny from default")
        verifyAccess('default', 'timed out', podIPs)
        tutils.tcLog("Delete allow-prod")
        deleteNsNetPol('default', 'hostnames-allow-prod')
        tutils.tcLog("Verify allow from default")
        verifyAccess('default', 'open', podIPs)
        tutils.tcLog("Verify allow from prod")
        verifyAccess('prod', 'open', podIPs)

        tutils.tcLog("Allow port 9000 from prod")
        createNsNetPol("default", "np/hostnames-allow-prod9000")
        tutils.tcLog("Verify deny from default")
        verifyAccess('default', 'timed out', podIPs)
        tutils.tcLog("Verify deny 9376 from prod")
        verifyAccess('prod', 'timed out', podIPs)
        tutils.tcLog("Delete allow-prod9000")
        deleteNsNetPol('default', 'hostnames-allow-prod9000')

        tutils.tcLog("Allow port 9376 from prod")
        createNsNetPol("default", "np/hostnames-allow-prod9376")
        tutils.tcLog("Verify allow from prod")
        verifyAccess('prod', 'open', podIPs)
        tutils.tcLog("Verify deny from default")
        verifyAccess('default', 'timed out', podIPs)

        tutils.tcLog("Delete allow-prod9376")
        deleteNsNetPol('default', 'hostnames-allow-prod9376')
        tutils.tcLog("Verify allow from default")
        verifyAccess('default', 'open', podIPs)
        tutils.tcLog("Verify allow from prod")
        verifyAccess('prod', 'open', podIPs)

        tutils.scaleDep("default", "hostnames-dep", 0)
        v1.delete_namespace(
            'prod', client.V1DeleteOptions())  # deletes the client-pod too
        av1 = client.AppsV1Api()
        av1.delete_namespaced_deployment("hostnames-dep", "default",
                                         client.V1DeleteOptions())
        tutils.deletePod("default", "client-pod")

        def delChecker():
            dList = av1.list_namespaced_deployment("default")
            for dep in dList.items:
                if dep.metadata.name == "hostnames-dep":
                    return "hostnames-dep still present"
            if tutils.namespaceExists('prod'):
                return "prod exists"
            return ""

        tutils.tcLog("Verify cleanup")
        tutils.assertEventually(delChecker, 2, 40)
    def test_kafka(object):
        # setup kafka services
        tutils.tcLog("Setup kafka services")
        for ky in kafkaYamls:
            try:
                utils.create_from_yaml(k8s_client, "yamls/"+ky)
            except ApiException as e:
                logging.debug("{} - ignored".format(e.reason))

        k8s_api = client.CoreV1Api()
        # check kafka server is ready
        assertPodReady("default", "ut-kafka-0", 120)
        assertPodReady("default", "kafkakv", 120)
        sleep(5)

        # collect the current list of ep's from k8s
        tutils.tcLog("Collect ep's from k8s")
        initialEPList = readCniEPList()
        logging.info("EPList is {}".format(initialEPList))

        tutils.tcLog("Verifying initial EPList with kafka")
        kafkaSyncChecker(initialEPList)

        tutils.tcLog("Adding a pod and checking it in kafka")
        utils.create_from_yaml(k8s_client, "yamls/alpine-pod.yaml")
        assertPodReady("default", "alpine-pod", 45)
        vtep = tutils.getPodNodeIP("alpine-pod", "default")

        tutils.tcLog("Check for podif")
        crdApi = client.CustomObjectsApi()
        group = "aci.aw"
        ns = "kube-system"
        podifName = "default.alpine-pod.{}".format(vtep)
        p = crdApi.get_namespaced_custom_object(group, "v1", ns, "podifs", podifName)
        logging.debug("podif: {}".format(p))
        epStatus = p['status']
        epID = "{}.{}.{}".format(epStatus['podns'], epStatus['podname'], epStatus['ifname'])
        toCheck = []
        toCheck.append(epID)
        kafkaChecker(toCheck, "found")
        tutils.tcLog("Delete a pod and check removal from kafka")
        k8s_api.delete_namespaced_pod("alpine-pod", "default", client.V1DeleteOptions())
        kafkaChecker(toCheck, "missing", 8)

        tutils.tcLog("Recheck ep sync between k8s and kafka")
        kafkaSyncChecker(initialEPList)
        tutils.tcLog("Add more endpoints")
        utils.create_from_yaml(k8s_client, "yamls/busybox.yaml")
        tutils.rcCheckScale("busybox", 3)
        tutils.tcLog("Get new eplist from k8s")
        EPList1 = readCniEPList()
        assert EPList1 != initialEPList
        tutils.tcLog("Recheck ep sync between k8s and kafka")
        kafkaSyncChecker(EPList1)

        tutils.tcLog("Bring controller down")
        scaleDep("kube-system", "aci-containers-controller", 0)
        sleep(10)
        tutils.tcLog("Change some endpoints")
        tutils.scaleRc("busybox", 0)
        tutils.checkPodsRemoved("app=busybox")
        EPList2 = readCniEPList()
        assert EPList2 != EPList1
        tutils.scaleRc("busybox", 2)
        sleep(5)

        logging.debug("previous ep list:{}".format(EPList1))
        EPList3 = readCniEPList()
        logging.debug("new ep list:{}".format(EPList3))
        assert EPList3 != EPList1
        tutils.tcLog("Bring controller up")
        scaleDep("kube-system", "aci-containers-controller", 1)
        sleep(10)
        tutils.tcLog("Verify controller is up")
        tutils.assertEventually(checkSystemStatus, 1, 60)
        tutils.tcLog("Check ep sync again")
        kafkaSyncChecker(EPList3, 6)
        tutils.tcLog("Clean up added endpoints")
        tutils.scaleRc("busybox", 0)
        tutils.checkPodsRemoved("app=busybox")
        k8s_api.delete_namespaced_replication_controller("busybox", "default", client.V1DeleteOptions())