def setUp(self): super(_TestBGPAdvert, self).setUp() # Create bgp test namespace self.ns = "bgp-test" self.create_namespace(self.ns) self.nodes, self.ips, _ = node_info() self.external_node_ip = start_external_node_with_bgp( "kube-node-extra", bird_peer_config=self.get_bird_conf(), ) # Enable debug logging self.update_ds_env("calico-node", "kube-system", "BGP_LOGSEVERITYSCREEN", "debug") # Establish BGPPeer from cluster nodes to node-extra calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: node-extra.peer%s EOF """ % self.get_extra_peer_spec())
def setUpRR(self): super(TestBGPAdvert, self).setUp() # Create bgp test namespace self.ns = "bgp-test" self.create_namespace(self.ns) start_external_node_with_bgp("kube-node-extra", bird_conf_rr) # Enable debug logging self.update_ds_env("calico-node", "kube-system", "BGP_LOGSEVERITYSCREEN", "debug") # Establish BGPPeer from cluster nodes to node-extra using calicoctl # External peer has IP 10.192.0.5 calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: node-extra.peer spec: node: kube-node-2 peerIP: 10.192.0.5 asNumber: 64512 EOF """)
def setUp(self): super(TestBGPAdvert, self).setUp() # Create bgp test namespace self.ns = "bgp-test" self.create_namespace(self.ns) start_external_node_with_bgp("kube-node-extra", bird_conf) # set CALICO_ADVERTISE_CLUSTER_IPS=10.96.0.0/12 self.update_ds_env("calico-node", "kube-system", "CALICO_ADVERTISE_CLUSTER_IPS", "10.96.0.0/12") # Enable debug logging self.update_ds_env("calico-node", "kube-system", "BGP_LOGSEVERITYSCREEN", "debug") # Establish BGPPeer from cluster nodes to node-extra calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: name: node-extra.peer spec: peerIP: 10.192.0.5 asNumber: 64512 EOF """)
def test_many_services(self): """ Creates a lot of services quickly """ with DiagsCollector(): calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceClusterIPs: - cidr: 10.96.0.0/12 EOF """) # Assert that a route to the service IP range is present. retry_until_success( lambda: self.assertIn("10.96.0.0/12", self.get_routes())) # Create a local service and deployment. local_svc = "nginx-local" self.deploy("nginx:1.7.9", local_svc, self.ns, 80) self.wait_for_deployment(local_svc, self.ns) # Get clusterIPs. cluster_ips = [] cluster_ips.append(self.get_svc_cluster_ip(local_svc, self.ns)) # Create many more services which select this deployment. num_svc = 300 for i in range(num_svc): name = "nginx-svc-%s" % i self.create_service(name, local_svc, self.ns, 80) # Get all of their IPs. for i in range(num_svc): name = "nginx-svc-%s" % i cluster_ips.append(self.get_svc_cluster_ip(name, self.ns)) # Assert they are all advertised to the other node. This should happen # quickly enough that by the time we have queried all services from # the k8s API, they should be programmed on the remote node. routes = self.get_routes() for cip in cluster_ips: self.assertIn(cip, routes) # Scale to 0 replicas, assert all routes are removed. self.scale_deployment(local_svc, self.ns, 0) self.wait_for_deployment(local_svc, self.ns) def check_routes_gone(): routes = self.get_routes() for cip in cluster_ips: self.assertNotIn(cip, routes) retry_until_success(check_routes_gone, retries=10, wait_time=5)
def tearDown(self): super(_TestBGPAdvert, self).tearDown() self.delete_and_confirm(self.ns, "ns") try: # Delete the extra node. run("docker rm -f kube-node-extra") except subprocess.CalledProcessError: pass # Delete BGPPeers. calicoctl("delete bgppeer node-extra.peer", allow_fail=True) calicoctl("delete bgppeer peer-with-rr", allow_fail=True) # Restore node-to-node mesh. calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: {name: default} spec: nodeToNodeMeshEnabled: true asNumber: 64512 EOF """) # Remove node-2's route-reflector config. json_str = calicoctl("get node %s -o json" % self.nodes[2]) node_dict = json.loads(json_str) node_dict['metadata']['labels'].pop('i-am-a-route-reflector', '') node_dict['spec']['bgp'].pop('routeReflectorClusterID', '') calicoctl("""apply -f - << EOF %s EOF """ % json.dumps(node_dict))
def tearDown(self): # Delete deployment self.delete_and_confirm(self.ns_name, "ns") # Change pool to use IPIP default_pool = json.loads(calicoctl("get ippool default-ipv4-ippool -o json")) default_pool["spec"]["vxlanMode"] = "Never" default_pool["spec"]["ipipMode"] = "Always" calicoctl_apply_dict(default_pool) # restart calico-nodes kubectl("delete po -n kube-system -l k8s-app=calico-node") kubectl("wait --timeout=2m --for=condition=ready" + " pods -l k8s-app=calico-node -n kube-system")
def test_ipip_spoof(self): with DiagsCollector(): # Change pool to use IPIP if necessary default_pool = json.loads( calicoctl("get ippool default-ipv4-ippool -o json")) if default_pool["spec"]["vxlanMode"] != "Never" or default_pool[ "spec"]["ipipMode"] != "Always": default_pool["spec"]["vxlanMode"] = "Never" default_pool["spec"]["ipipMode"] = "Always" calicoctl_apply_dict(default_pool) # restart calico-nodes kubectl("delete po -n kube-system -l k8s-app=calico-node") kubectl("wait --timeout=2m --for=condition=ready" + " pods -l k8s-app=calico-node -n kube-system") # get busybox pod IP remote_pod_ip = retry_until_success( self.get_pod_ip, function_args=["access", self.ns_name]) print(remote_pod_ip) # clear conntrack table on all hosts self.clear_conntrack() # test connectivity works pod-pod retry_until_success(self.send_and_check, function_args=["ipip-normal", remote_pod_ip]) # clear conntrack table on all hosts self.clear_conntrack() def send_and_check_ipip_spoof(): self.send_spoofed_ipip_packet(self.ns_name, "scapy", "10.192.0.3", remote_pod_ip, "ipip-spoofed") kubectl( "exec -t -n %s access grep -- ipip-spoofed /root/snoop.txt" % self.ns_name) def assert_cannot_spoof_ipip(): failed = True try: send_and_check_ipip_spoof() except subprocess.CalledProcessError: failed = False if failed: print("ERROR - succeeded in sending spoofed IPIP packet") raise ConnectionError # test connectivity does NOT work when spoofing retry_until_success(assert_cannot_spoof_ipip)
def test_rr(self): # Create ExternalTrafficPolicy Local service with one endpoint on node-1 kubectl("""apply -f - << EOF apiVersion: apps/v1 kind: Deployment metadata: name: nginx-rr namespace: bgp-test labels: app: nginx spec: replicas: 1 selector: matchLabels: app: nginx run: nginx-rr template: metadata: labels: app: nginx run: nginx-rr spec: containers: - name: nginx-rr image: nginx:1.7.9 ports: - containerPort: 80 nodeSelector: beta.kubernetes.io/os: linux kubernetes.io/hostname: %s --- apiVersion: v1 kind: Service metadata: name: nginx-rr namespace: bgp-test labels: app: nginx run: nginx-rr spec: externalIPs: - 175.200.1.1 ports: - port: 80 targetPort: 80 selector: app: nginx run: nginx-rr type: NodePort externalTrafficPolicy: Local EOF """ % self.nodes[1]) calicoctl("get nodes -o yaml") calicoctl("get bgppeers -o yaml") calicoctl("get bgpconfigs -o yaml") # Update the node-2 to behave as a route-reflector json_str = calicoctl("get node %s -o json" % self.nodes[2]) node_dict = json.loads(json_str) node_dict['metadata']['labels']['i-am-a-route-reflector'] = 'true' node_dict['spec']['bgp']['routeReflectorClusterID'] = '224.0.0.1' calicoctl("""apply -f - << EOF %s EOF """ % json.dumps(node_dict)) # Disable node-to-node mesh, add cluster and external IP CIDRs to # advertise, and configure BGP peering between the cluster nodes and the # RR. (The BGP peering from the external node to the RR is included in # bird_conf_rr above.) calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: nodeToNodeMeshEnabled: false asNumber: 64512 serviceClusterIPs: - cidr: 10.96.0.0/12 serviceExternalIPs: - cidr: 175.200.0.0/16 EOF """) calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: {name: peer-with-rr} spec: peerIP: %s asNumber: 64512 EOF """ % self.ips[2]) svc_json = kubectl("get svc nginx-rr -n bgp-test -o json") svc_dict = json.loads(svc_json) cluster_ip = svc_dict['spec']['clusterIP'] external_ip = svc_dict['spec']['externalIPs'][0] retry_until_success( lambda: self.assertIn(cluster_ip, self.get_routes())) retry_until_success( lambda: self.assertIn(external_ip, self.get_routes()))
def test_external_ip_advertisement(self): """ Runs the tests for service external IP advertisement """ with DiagsCollector(): # Whitelist two IP ranges for the external IPs we'll test with calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceExternalIPs: - cidr: 175.200.0.0/16 - cidr: 200.255.0.0/24 EOF """) # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("nginx:1.7.9", local_svc, self.ns, 80) self.deploy("nginx:1.7.9", cluster_svc, self.ns, 80, traffic_policy="Cluster") self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get clusterIPs. local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert that clusterIPs are not advertised. retry_until_success( lambda: self.assertNotIn(local_svc_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(cluster_svc_ip, self.get_routes())) # Create a network policy that only accepts traffic from the external node. kubectl("""apply -f - << EOF apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-tcp-80-ex namespace: bgp-test spec: podSelector: {} policyTypes: - Ingress ingress: - from: - ipBlock: { cidr: %s/32 } ports: - protocol: TCP port: 80 EOF """ % self.external_node_ip) # Get host IPs for the nginx pods. local_svc_host_ip = self.get_svc_host_ip(local_svc, self.ns) cluster_svc_host_ip = self.get_svc_host_ip(cluster_svc, self.ns) # Select an IP from each external IP CIDR. local_svc_external_ip = "175.200.1.1" cluster_svc_external_ip = "200.255.255.1" # Add external IPs to the two services. self.add_svc_external_ips(local_svc, self.ns, [local_svc_external_ip]) self.add_svc_external_ips(cluster_svc, self.ns, [cluster_svc_external_ip]) # Verify that external IPs for local service is advertised but not the cluster service. local_svc_externalips_route = "%s via %s" % (local_svc_external_ip, local_svc_host_ip) cluster_svc_externalips_route = "%s via %s" % ( cluster_svc_external_ip, cluster_svc_host_ip) retry_until_success(lambda: self.assertIn( local_svc_externalips_route, self.get_routes())) retry_until_success(lambda: self.assertNotIn( cluster_svc_externalips_route, self.get_routes())) # Scale the local_svc to 4 replicas. self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) # Verify that we have ECMP routes for the external IP of the local service. retry_until_success(lambda: self.assert_ecmp_routes( local_svc_external_ip, [self.ips[1], self.ips[2], self.ips[3]]) ) # Delete both services, assert only cluster CIDR route is advertised. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that external IP is no longer an advertised route. retry_until_success(lambda: self.assertNotIn( local_svc_externalips_route, self.get_routes()))
def test_cluster_ip_advertisement(self): """ Runs the tests for service cluster IP advertisement - Create both a Local and a Cluster type NodePort service with a single replica. - assert only local and cluster CIDR routes are advertised. - assert /32 routes are used, source IP is preserved. - Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved. - Delete both services, assert only cluster CIDR route is advertised. """ with DiagsCollector(): calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceClusterIPs: - cidr: 10.96.0.0/12 EOF """) # Assert that a route to the service IP range is present. retry_until_success( lambda: self.assertIn("10.96.0.0/12", self.get_routes())) # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("nginx:1.7.9", local_svc, self.ns, 80) self.deploy("nginx:1.7.9", cluster_svc, self.ns, 80, traffic_policy="Cluster") self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get clusterIPs. local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert that both nginx service can be curled from the external node. retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Assert that local clusterIP is an advertised route and cluster clusterIP is not. retry_until_success( lambda: self.assertIn(local_svc_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(cluster_svc_ip, self.get_routes())) # Create a network policy that only accepts traffic from the external node. kubectl("""apply -f - << EOF apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-tcp-80-ex namespace: bgp-test spec: podSelector: {} policyTypes: - Ingress ingress: - from: - ipBlock: { cidr: %s/32 } ports: - protocol: TCP port: 80 EOF """ % self.external_node_ip) # Connectivity to nginx-local should always succeed. for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Connectivity to nginx-cluster will rarely succeed because it is load-balanced across all nodes. # When the traffic hits a node that doesn't host one of the service's pod, it will be re-routed # to another node and SNAT will cause the policy to drop the traffic. # Try to curl 10 times. try: for i in range(attempts): curl(cluster_svc_ip) self.fail( "external node should not be able to consistently access the cluster svc" ) except subprocess.CalledProcessError: pass # Scale the local_svc to 4 replicas self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) self.assert_ecmp_routes(local_svc_ip, [self.ips[1], self.ips[2], self.ips[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Delete both services. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that clusterIP is no longer an advertised route. retry_until_success( lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def test_loadbalancer_ip_advertisement(self): """ Runs the tests for service LoadBalancer IP advertisement """ with DiagsCollector(): # Whitelist IP ranges for the LB IPs we'll test with calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceLoadBalancerIPs: - cidr: 80.15.0.0/24 EOF """) # Create a dummy service first to occupy the first LB IP. This is # a hack to make sure the chosen IP we use in the tests below # isn't the same as the zero address in the range. self.create_service("dummy-service", "dummy-service", self.ns, 80, svc_type="LoadBalancer") # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("nginx:1.7.9", cluster_svc, self.ns, 80, traffic_policy="Cluster", svc_type="LoadBalancer") self.deploy("nginx:1.7.9", local_svc, self.ns, 80, svc_type="LoadBalancer") self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get the allocated LB IPs. local_lb_ip = self.get_svc_loadbalancer_ip(local_svc, self.ns) cluster_lb_ip = self.get_svc_loadbalancer_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Get host IPs for the nginx pods. local_svc_host_ip = self.get_svc_host_ip(local_svc, self.ns) cluster_svc_host_ip = self.get_svc_host_ip(cluster_svc, self.ns) # Verify that LB IP for local service is advertised but not the cluster service. local_svc_lb_route = "%s via %s" % (local_lb_ip, local_svc_host_ip) cluster_svc_lb_route = "%s via %s" % (cluster_lb_ip, cluster_svc_host_ip) retry_until_success( lambda: self.assertIn(local_svc_lb_route, self.get_routes())) retry_until_success(lambda: self.assertNotIn( cluster_svc_lb_route, self.get_routes())) # The full range should be advertised from each node. lb_cidr = "80.15.0.0/24" retry_until_success(lambda: self.assert_ecmp_routes( lb_cidr, [self.ips[0], self.ips[1], self.ips[2], self.ips[3]])) # Scale the local_svc to 4 replicas. self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) # Verify that we have ECMP routes for the LB IP of the local service from nodes running it. retry_until_success(lambda: self.assert_ecmp_routes( local_lb_ip, [self.ips[1], self.ips[2], self.ips[3]])) # Apply a modified BGP config that no longer enables advertisement # for LoadBalancer IPs. calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: {} EOF """) # Assert routes are withdrawn. retry_until_success( lambda: self.assertNotIn(local_lb_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(lb_cidr, self.get_routes())) # Apply a modified BGP config that has a mismatched CIDR specified. calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceLoadBalancerIPs: - cidr: 90.15.0.0/24 EOF """) # Assert routes are still withdrawn. retry_until_success( lambda: self.assertNotIn(local_lb_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(lb_cidr, self.get_routes())) # Reapply the correct configuration, we should see routes come back. calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceLoadBalancerIPs: - cidr: 80.15.0.0/24 EOF """) # Verify that we have ECMP routes for the LB IP of the local service from nodes running it. retry_until_success(lambda: self.assert_ecmp_routes( local_lb_ip, [self.ips[1], self.ips[2], self.ips[3]])) retry_until_success( lambda: self.assertIn(lb_cidr, self.get_routes())) retry_until_success(lambda: self.assertNotIn( cluster_svc_lb_route, self.get_routes())) # Services should be reachable from the external node. retry_until_success(curl, function_args=[local_lb_ip]) retry_until_success(curl, function_args=[cluster_lb_ip]) # Delete both services, assert only CIDR route is advertised. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that LB IP is no longer an advertised route. retry_until_success( lambda: self.assertNotIn(local_lb_ip, self.get_routes()))
def test_node_exclusion(self): """ Tests the node exclusion label. - Create services, assert advertised from all nodes. - Label one node so that it is excluded, assert that routes are withdrawn from that node. - Delete / recreate service, assert it is still advertised from the correct nodes. - Remove the exclusion label, assert that the node re-advertises the svc. """ with DiagsCollector(): calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceClusterIPs: - cidr: 10.96.0.0/12 serviceExternalIPs: - cidr: 175.200.0.0/16 EOF """) # Assert that a route to the service IP range is present. cluster_cidr = "10.96.0.0/12" retry_until_success( lambda: self.assertIn(cluster_cidr, self.get_routes())) # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("nginx:1.7.9", local_svc, self.ns, 80) self.deploy("nginx:1.7.9", cluster_svc, self.ns, 80, traffic_policy="Cluster") self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get clusterIPs. local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert that both nginx service can be curled from the external node. retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Assert that local clusterIP is an advertised route and cluster clusterIP is not. retry_until_success( lambda: self.assertIn(local_svc_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(cluster_svc_ip, self.get_routes())) # Connectivity should always succeed. for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Scale local service to 4 replicas self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert routes are correct and services are accessible. # Local service should only be advertised from nodes that can run pods. # The cluster CIDR should be advertised from all nodes. self.assert_ecmp_routes(local_svc_ip, [self.ips[1], self.ips[2], self.ips[3]]) self.assert_ecmp_routes( cluster_cidr, [self.ips[0], self.ips[1], self.ips[2], self.ips[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Label one node in order to exclude it from service advertisement. # After this, we should expect that all routes from that node are # withdrawn. kubectl( "label node %s node.kubernetes.io/exclude-from-external-load-balancers=true" % self.nodes[1]) # Assert routes are correct and services are accessible. # It should no longer have a route via self.nodes[1] self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]]) self.assert_ecmp_routes(cluster_cidr, [self.ips[0], self.ips[2], self.ips[3]]) # Should work the same for external IP cidr. external_ip_cidr = "175.200.0.0/16" self.assert_ecmp_routes(external_ip_cidr, [self.ips[0], self.ips[2], self.ips[3]]) # Should still be reachable through other nodes. for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Delete the local service, confirm that it is no longer advertised. self.delete_and_confirm(local_svc, "svc", self.ns) retry_until_success( lambda: self.assertNotIn(local_svc_ip, self.get_routes())) # Re-create the local service. Assert it is advertised from the correct nodes, # but not from the excluded node. self.create_service(local_svc, local_svc, self.ns, 80) self.wait_until_exists(local_svc, "svc", self.ns) local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Add an external IP to the local svc and assert it follows the same # advertisement rules. local_svc_external_ip = "175.200.1.1" self.add_svc_external_ips(local_svc, self.ns, [local_svc_external_ip]) self.assert_ecmp_routes(local_svc_external_ip, [self.ips[2], self.ips[3]]) # Enable the excluded node. Assert that the node starts # advertising service routes again. kubectl( "label node %s node.kubernetes.io/exclude-from-external-load-balancers=false --overwrite" % self.nodes[1]) self.assert_ecmp_routes(local_svc_ip, [self.ips[1], self.ips[2], self.ips[3]]) self.assert_ecmp_routes(local_svc_external_ip, [self.ips[1], self.ips[2], self.ips[3]]) self.assert_ecmp_routes( cluster_cidr, [self.ips[0], self.ips[1], self.ips[2], self.ips[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Delete both services. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that clusterIP is no longer an advertised route. retry_until_success( lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def test_cluster_ip_advertisement(self): """ Runs the tests for service cluster IPv6 advertisement - Create both a Local and a Cluster type NodePort service with a single replica. - assert only local and cluster CIDR routes are advertised. - assert /128 routes are used, source IP is preserved. - Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved. - Delete both services, assert only cluster CIDR route is advertised. """ with DiagsCollector(): calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceClusterIPs: - cidr: fd00:10:96::/112 EOF """) # Assert that a route to the service IP range is present. retry_until_success(lambda: self.assertIn("fd00:10:96::/112", self.get_routes())) # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", local_svc, self.ns, 80, ipv6=True) self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", cluster_svc, self.ns, 80, traffic_policy="Cluster", ipv6=True) self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get clusterIPs. local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert that both nginx service can be curled from the external node. retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Assert that local clusterIP is an advertised route and cluster clusterIP is not. retry_until_success(lambda: self.assertIn(local_svc_ip, self.get_routes())) retry_until_success(lambda: self.assertNotIn(cluster_svc_ip, self.get_routes())) # Create a network policy that only accepts traffic from the external node. kubectl("""apply -f - << EOF apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-tcp-80-ex namespace: bgp-test spec: podSelector: {} policyTypes: - Ingress ingress: - from: - ipBlock: { cidr: %s/128 } ports: - protocol: TCP port: 80 EOF """ % self.external_node_ip) # Connectivity to nginx-local should always succeed. for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # NOTE: Unlike in the IPv6 case (in test_bgp_advert.py) we cannot successfully test that # connectivity to nginx-cluster is load-balanced across all nodes (and hence, with the # above policy in place, will sometimes fail and sometimes succeed), because our current # observation is that Linux's IPv6 ECMP route choice does _not_ depend on source port, # even though it is documented as such when fib_multipath_hash_policy == 1. # Scale the local_svc to 4 replicas self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) self.assert_ecmp_routes(local_svc_ip, [self.ipv6s[1], self.ipv6s[2], self.ipv6s[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Delete both services. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that clusterIP is no longer an advertised route. retry_until_success(lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def test_rr(self): self.tearDown() self.setUpRR() # Create ExternalTrafficPolicy Local service with one endpoint on node-1 kubectl("""apply -f - << EOF apiVersion: apps/v1 kind: Deployment metadata: name: nginx-rr namespace: bgp-test labels: app: nginx spec: replicas: 1 selector: matchLabels: app: nginx run: nginx-rr template: metadata: labels: app: nginx run: nginx-rr spec: containers: - name: nginx-rr image: nginx:1.7.9 ports: - containerPort: 80 nodeSelector: beta.kubernetes.io/os: linux kubernetes.io/hostname: kube-node-1 --- apiVersion: v1 kind: Service metadata: name: nginx-rr namespace: bgp-test labels: app: nginx run: nginx-rr spec: ports: - port: 80 targetPort: 80 selector: app: nginx run: nginx-rr type: NodePort externalTrafficPolicy: Local EOF """) calicoctl("get nodes -o yaml") calicoctl("get bgppeers -o yaml") calicoctl("get bgpconfigs -o yaml") # Update the node-2 to behave as a route-reflector json_str = calicoctl("get node kube-node-2 -o json") node_dict = json.loads(json_str) node_dict['metadata']['labels']['i-am-a-route-reflector'] = 'true' node_dict['spec']['bgp']['routeReflectorClusterID'] = '224.0.0.1' calicoctl("""apply -f - << EOF %s EOF """ % json.dumps(node_dict)) # Disable node-to-node mesh and configure bgp peering # between node-1 and RR and also between external node and RR calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: {name: default} spec: nodeToNodeMeshEnabled: false asNumber: 64512 EOF """) calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPPeer metadata: {name: kube-node-1} spec: node: kube-node-1 peerIP: 10.192.0.4 asNumber: 64512 EOF """) svc_json = kubectl("get svc nginx-rr -n bgp-test -o json") svc_dict = json.loads(svc_json) svcRoute = svc_dict['spec']['clusterIP'] retry_until_success(lambda: self.assertIn(svcRoute, self.get_routes()))