def test_many_services(self): """ Creates a lot of services quickly """ with DiagsCollector(): calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceClusterIPs: - cidr: 10.96.0.0/12 EOF """) # Assert that a route to the service IP range is present. retry_until_success( lambda: self.assertIn("10.96.0.0/12", self.get_routes())) # Create a local service and deployment. local_svc = "nginx-local" self.deploy("nginx:1.7.9", local_svc, self.ns, 80) self.wait_for_deployment(local_svc, self.ns) # Get clusterIPs. cluster_ips = [] cluster_ips.append(self.get_svc_cluster_ip(local_svc, self.ns)) # Create many more services which select this deployment. num_svc = 300 for i in range(num_svc): name = "nginx-svc-%s" % i self.create_service(name, local_svc, self.ns, 80) # Get all of their IPs. for i in range(num_svc): name = "nginx-svc-%s" % i cluster_ips.append(self.get_svc_cluster_ip(name, self.ns)) # Assert they are all advertised to the other node. This should happen # quickly enough that by the time we have queried all services from # the k8s API, they should be programmed on the remote node. routes = self.get_routes() for cip in cluster_ips: self.assertIn(cip, routes) # Scale to 0 replicas, assert all routes are removed. self.scale_deployment(local_svc, self.ns, 0) self.wait_for_deployment(local_svc, self.ns) def check_routes_gone(): routes = self.get_routes() for cip in cluster_ips: self.assertNotIn(cip, routes) retry_until_success(check_routes_gone, retries=10, wait_time=5)
def test_ipip_spoof(self): with DiagsCollector(): # Change pool to use IPIP if necessary default_pool = json.loads( calicoctl("get ippool default-ipv4-ippool -o json")) if default_pool["spec"]["vxlanMode"] != "Never" or default_pool[ "spec"]["ipipMode"] != "Always": default_pool["spec"]["vxlanMode"] = "Never" default_pool["spec"]["ipipMode"] = "Always" calicoctl_apply_dict(default_pool) # restart calico-nodes kubectl("delete po -n kube-system -l k8s-app=calico-node") kubectl("wait --timeout=2m --for=condition=ready" + " pods -l k8s-app=calico-node -n kube-system") # get busybox pod IP remote_pod_ip = retry_until_success( self.get_pod_ip, function_args=["access", self.ns_name]) print(remote_pod_ip) # clear conntrack table on all hosts self.clear_conntrack() # test connectivity works pod-pod retry_until_success(self.send_and_check, function_args=["ipip-normal", remote_pod_ip]) # clear conntrack table on all hosts self.clear_conntrack() def send_and_check_ipip_spoof(): self.send_spoofed_ipip_packet(self.ns_name, "scapy", "10.192.0.3", remote_pod_ip, "ipip-spoofed") kubectl( "exec -t -n %s access grep -- ipip-spoofed /root/snoop.txt" % self.ns_name) def assert_cannot_spoof_ipip(): failed = True try: send_and_check_ipip_spoof() except subprocess.CalledProcessError: failed = False if failed: print("ERROR - succeeded in sending spoofed IPIP packet") raise ConnectionError # test connectivity does NOT work when spoofing retry_until_success(assert_cannot_spoof_ipip)
def _test_restart_route_churn(self, num_repeats, restart_func, expect_churn): with DiagsCollector(): # Get 2 worker node names, one to monitor routes and one # to have its calico-node restarted. The first name # returned is always the master, so skip that. nodes, ips, _ = node_info() self.assertGreater(len(nodes), 2) monitor_node = nodes[1] self.restart_node = nodes[2] self.restart_node_ip = ips[2] # Start running ip monitor on the monitor node, to monitor # IPv4 route changes. We use "fd00:10:244" to identify # and exclude IPv6 workload block routes like # fd00:10:244:0:1cc0:b1ac:ad47:e7c0/122. These definitely # _do_ flap when the host of that block restarts, but it # is not yet clear why this is; specifically it is not yet # known if it indicates anything wrong with calico/node's # GR setup. See # https://marc.info/?l=bird-users&m=158298182509702&w=2 # for the mailing list discussion so far. run("docker exec -d %s sh -c 'stdbuf -oL ip -ts monitor route | stdbuf -oL grep -v fd00:10:244 > rmon.txt'" % monitor_node) # Find the name of the calico-node pod on the restart node. self.get_restart_node_pod_name() # Restart the calico-node several times, on the other node. for i in range(num_repeats): # Restart it. _log.info("Iteration %d: restart pod %s", i, self.restart_pod_name) restart_func(self) # Kill the ip monitor process. run("docker exec %s pkill ip" % monitor_node) # Dump the monitor output. monitor_output = run("docker exec %s cat rmon.txt" % monitor_node) if expect_churn: # Assert that it is not empty. self.assertNotEqual(monitor_output, "") else: # Assert that it is empty. self.assertEqual(monitor_output, "")
def test_external_ip_advertisement(self): """ Runs the tests for service external IP advertisement """ with DiagsCollector(): # Whitelist two IP ranges for the external IPs we'll test with calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceExternalIPs: - cidr: 175.200.0.0/16 - cidr: 200.255.0.0/24 EOF """) # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("nginx:1.7.9", local_svc, self.ns, 80) self.deploy("nginx:1.7.9", cluster_svc, self.ns, 80, traffic_policy="Cluster") self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get clusterIPs. local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert that clusterIPs are not advertised. retry_until_success( lambda: self.assertNotIn(local_svc_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(cluster_svc_ip, self.get_routes())) # Create a network policy that only accepts traffic from the external node. kubectl("""apply -f - << EOF apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-tcp-80-ex namespace: bgp-test spec: podSelector: {} policyTypes: - Ingress ingress: - from: - ipBlock: { cidr: %s/32 } ports: - protocol: TCP port: 80 EOF """ % self.external_node_ip) # Get host IPs for the nginx pods. local_svc_host_ip = self.get_svc_host_ip(local_svc, self.ns) cluster_svc_host_ip = self.get_svc_host_ip(cluster_svc, self.ns) # Select an IP from each external IP CIDR. local_svc_external_ip = "175.200.1.1" cluster_svc_external_ip = "200.255.255.1" # Add external IPs to the two services. self.add_svc_external_ips(local_svc, self.ns, [local_svc_external_ip]) self.add_svc_external_ips(cluster_svc, self.ns, [cluster_svc_external_ip]) # Verify that external IPs for local service is advertised but not the cluster service. local_svc_externalips_route = "%s via %s" % (local_svc_external_ip, local_svc_host_ip) cluster_svc_externalips_route = "%s via %s" % ( cluster_svc_external_ip, cluster_svc_host_ip) retry_until_success(lambda: self.assertIn( local_svc_externalips_route, self.get_routes())) retry_until_success(lambda: self.assertNotIn( cluster_svc_externalips_route, self.get_routes())) # Scale the local_svc to 4 replicas. self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) # Verify that we have ECMP routes for the external IP of the local service. retry_until_success(lambda: self.assert_ecmp_routes( local_svc_external_ip, [self.ips[1], self.ips[2], self.ips[3]]) ) # Delete both services, assert only cluster CIDR route is advertised. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that external IP is no longer an advertised route. retry_until_success(lambda: self.assertNotIn( local_svc_externalips_route, self.get_routes()))
def test_cluster_ip_advertisement(self): """ Runs the tests for service cluster IP advertisement - Create both a Local and a Cluster type NodePort service with a single replica. - assert only local and cluster CIDR routes are advertised. - assert /32 routes are used, source IP is preserved. - Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved. - Delete both services, assert only cluster CIDR route is advertised. """ with DiagsCollector(): calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceClusterIPs: - cidr: 10.96.0.0/12 EOF """) # Assert that a route to the service IP range is present. retry_until_success( lambda: self.assertIn("10.96.0.0/12", self.get_routes())) # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("nginx:1.7.9", local_svc, self.ns, 80) self.deploy("nginx:1.7.9", cluster_svc, self.ns, 80, traffic_policy="Cluster") self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get clusterIPs. local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert that both nginx service can be curled from the external node. retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Assert that local clusterIP is an advertised route and cluster clusterIP is not. retry_until_success( lambda: self.assertIn(local_svc_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(cluster_svc_ip, self.get_routes())) # Create a network policy that only accepts traffic from the external node. kubectl("""apply -f - << EOF apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-tcp-80-ex namespace: bgp-test spec: podSelector: {} policyTypes: - Ingress ingress: - from: - ipBlock: { cidr: %s/32 } ports: - protocol: TCP port: 80 EOF """ % self.external_node_ip) # Connectivity to nginx-local should always succeed. for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Connectivity to nginx-cluster will rarely succeed because it is load-balanced across all nodes. # When the traffic hits a node that doesn't host one of the service's pod, it will be re-routed # to another node and SNAT will cause the policy to drop the traffic. # Try to curl 10 times. try: for i in range(attempts): curl(cluster_svc_ip) self.fail( "external node should not be able to consistently access the cluster svc" ) except subprocess.CalledProcessError: pass # Scale the local_svc to 4 replicas self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) self.assert_ecmp_routes(local_svc_ip, [self.ips[1], self.ips[2], self.ips[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Delete both services. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that clusterIP is no longer an advertised route. retry_until_success( lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def test_simple_policy(self): with DiagsCollector(): # Check we can talk to service. retry_until_success(self.can_connect, retries=10, wait_time=1, function_args=["access"]) _log.info("Client 'access' connected to open service") retry_until_success(self.can_connect, retries=10, wait_time=1, function_args=["no-access"]) _log.info("Client 'no-access' connected to open service") # Create default-deny policy policy = client.V1NetworkPolicy(metadata=client.V1ObjectMeta( name="default-deny", namespace="policy-demo"), spec={ "podSelector": { "matchLabels": {}, }, }) client.ExtensionsV1beta1Api().create_namespaced_network_policy( body=policy, namespace="policy-demo", ) _log.debug("Isolation policy created") # Check we cannot talk to service retry_until_success(self.cannot_connect, retries=10, wait_time=1, function_args=["access"]) _log.info("Client 'access' failed to connect to isolated service") retry_until_success(self.cannot_connect, retries=10, wait_time=1, function_args=["no-access"]) _log.info( "Client 'no-access' failed to connect to isolated service") # Create allow policy policy = client.V1NetworkPolicy(metadata=client.V1ObjectMeta( name="access-nginx", namespace="policy-demo"), spec={ 'ingress': [{ 'from': [{ 'podSelector': { 'matchLabels': { 'run': 'access' } } }] }], 'podSelector': { 'matchLabels': { 'app': 'nginx' } } }) client.ExtensionsV1beta1Api().create_namespaced_network_policy( body=policy, namespace="policy-demo", ) _log.debug("Allow policy created.") # Check we can talk to service as 'access' retry_until_success(self.can_connect, retries=10, wait_time=1, function_args=["access"]) _log.info("Client 'access' connected to protected service") # Check we cannot talk to service as 'no-access' retry_until_success(self.cannot_connect, retries=10, wait_time=1, function_args=["no-access"]) _log.info( "Client 'no-access' failed to connect to protected service")
def test_calico_monitoring_pods_running(self): with DiagsCollector(): self.check_pod_status('calico-monitoring')
def test_default_pods_running(self): with DiagsCollector(): self.check_pod_status('default')
def test_kubesystem_pods_running(self): with DiagsCollector(): self.check_pod_status('kube-system')
def test_loadbalancer_ip_advertisement(self): """ Runs the tests for service LoadBalancer IP advertisement """ with DiagsCollector(): # Whitelist IP ranges for the LB IPs we'll test with calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceLoadBalancerIPs: - cidr: 80.15.0.0/24 EOF """) # Create a dummy service first to occupy the first LB IP. This is # a hack to make sure the chosen IP we use in the tests below # isn't the same as the zero address in the range. self.create_service("dummy-service", "dummy-service", self.ns, 80, svc_type="LoadBalancer") # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("nginx:1.7.9", cluster_svc, self.ns, 80, traffic_policy="Cluster", svc_type="LoadBalancer") self.deploy("nginx:1.7.9", local_svc, self.ns, 80, svc_type="LoadBalancer") self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get the allocated LB IPs. local_lb_ip = self.get_svc_loadbalancer_ip(local_svc, self.ns) cluster_lb_ip = self.get_svc_loadbalancer_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Get host IPs for the nginx pods. local_svc_host_ip = self.get_svc_host_ip(local_svc, self.ns) cluster_svc_host_ip = self.get_svc_host_ip(cluster_svc, self.ns) # Verify that LB IP for local service is advertised but not the cluster service. local_svc_lb_route = "%s via %s" % (local_lb_ip, local_svc_host_ip) cluster_svc_lb_route = "%s via %s" % (cluster_lb_ip, cluster_svc_host_ip) retry_until_success( lambda: self.assertIn(local_svc_lb_route, self.get_routes())) retry_until_success(lambda: self.assertNotIn( cluster_svc_lb_route, self.get_routes())) # The full range should be advertised from each node. lb_cidr = "80.15.0.0/24" retry_until_success(lambda: self.assert_ecmp_routes( lb_cidr, [self.ips[0], self.ips[1], self.ips[2], self.ips[3]])) # Scale the local_svc to 4 replicas. self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) # Verify that we have ECMP routes for the LB IP of the local service from nodes running it. retry_until_success(lambda: self.assert_ecmp_routes( local_lb_ip, [self.ips[1], self.ips[2], self.ips[3]])) # Apply a modified BGP config that no longer enables advertisement # for LoadBalancer IPs. calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: {} EOF """) # Assert routes are withdrawn. retry_until_success( lambda: self.assertNotIn(local_lb_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(lb_cidr, self.get_routes())) # Apply a modified BGP config that has a mismatched CIDR specified. calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceLoadBalancerIPs: - cidr: 90.15.0.0/24 EOF """) # Assert routes are still withdrawn. retry_until_success( lambda: self.assertNotIn(local_lb_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(lb_cidr, self.get_routes())) # Reapply the correct configuration, we should see routes come back. calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceLoadBalancerIPs: - cidr: 80.15.0.0/24 EOF """) # Verify that we have ECMP routes for the LB IP of the local service from nodes running it. retry_until_success(lambda: self.assert_ecmp_routes( local_lb_ip, [self.ips[1], self.ips[2], self.ips[3]])) retry_until_success( lambda: self.assertIn(lb_cidr, self.get_routes())) retry_until_success(lambda: self.assertNotIn( cluster_svc_lb_route, self.get_routes())) # Services should be reachable from the external node. retry_until_success(curl, function_args=[local_lb_ip]) retry_until_success(curl, function_args=[cluster_lb_ip]) # Delete both services, assert only CIDR route is advertised. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that LB IP is no longer an advertised route. retry_until_success( lambda: self.assertNotIn(local_lb_ip, self.get_routes()))
def test_node_exclusion(self): """ Tests the node exclusion label. - Create services, assert advertised from all nodes. - Label one node so that it is excluded, assert that routes are withdrawn from that node. - Delete / recreate service, assert it is still advertised from the correct nodes. - Remove the exclusion label, assert that the node re-advertises the svc. """ with DiagsCollector(): calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceClusterIPs: - cidr: 10.96.0.0/12 serviceExternalIPs: - cidr: 175.200.0.0/16 EOF """) # Assert that a route to the service IP range is present. cluster_cidr = "10.96.0.0/12" retry_until_success( lambda: self.assertIn(cluster_cidr, self.get_routes())) # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("nginx:1.7.9", local_svc, self.ns, 80) self.deploy("nginx:1.7.9", cluster_svc, self.ns, 80, traffic_policy="Cluster") self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get clusterIPs. local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert that both nginx service can be curled from the external node. retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Assert that local clusterIP is an advertised route and cluster clusterIP is not. retry_until_success( lambda: self.assertIn(local_svc_ip, self.get_routes())) retry_until_success( lambda: self.assertNotIn(cluster_svc_ip, self.get_routes())) # Connectivity should always succeed. for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Scale local service to 4 replicas self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert routes are correct and services are accessible. # Local service should only be advertised from nodes that can run pods. # The cluster CIDR should be advertised from all nodes. self.assert_ecmp_routes(local_svc_ip, [self.ips[1], self.ips[2], self.ips[3]]) self.assert_ecmp_routes( cluster_cidr, [self.ips[0], self.ips[1], self.ips[2], self.ips[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Label one node in order to exclude it from service advertisement. # After this, we should expect that all routes from that node are # withdrawn. kubectl( "label node %s node.kubernetes.io/exclude-from-external-load-balancers=true" % self.nodes[1]) # Assert routes are correct and services are accessible. # It should no longer have a route via self.nodes[1] self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]]) self.assert_ecmp_routes(cluster_cidr, [self.ips[0], self.ips[2], self.ips[3]]) # Should work the same for external IP cidr. external_ip_cidr = "175.200.0.0/16" self.assert_ecmp_routes(external_ip_cidr, [self.ips[0], self.ips[2], self.ips[3]]) # Should still be reachable through other nodes. for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Delete the local service, confirm that it is no longer advertised. self.delete_and_confirm(local_svc, "svc", self.ns) retry_until_success( lambda: self.assertNotIn(local_svc_ip, self.get_routes())) # Re-create the local service. Assert it is advertised from the correct nodes, # but not from the excluded node. self.create_service(local_svc, local_svc, self.ns, 80) self.wait_until_exists(local_svc, "svc", self.ns) local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Add an external IP to the local svc and assert it follows the same # advertisement rules. local_svc_external_ip = "175.200.1.1" self.add_svc_external_ips(local_svc, self.ns, [local_svc_external_ip]) self.assert_ecmp_routes(local_svc_external_ip, [self.ips[2], self.ips[3]]) # Enable the excluded node. Assert that the node starts # advertising service routes again. kubectl( "label node %s node.kubernetes.io/exclude-from-external-load-balancers=false --overwrite" % self.nodes[1]) self.assert_ecmp_routes(local_svc_ip, [self.ips[1], self.ips[2], self.ips[3]]) self.assert_ecmp_routes(local_svc_external_ip, [self.ips[1], self.ips[2], self.ips[3]]) self.assert_ecmp_routes( cluster_cidr, [self.ips[0], self.ips[1], self.ips[2], self.ips[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Delete both services. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that clusterIP is no longer an advertised route. retry_until_success( lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def test_cluster_ip_advertisement(self): """ Runs the tests for service cluster IPv6 advertisement - Create both a Local and a Cluster type NodePort service with a single replica. - assert only local and cluster CIDR routes are advertised. - assert /128 routes are used, source IP is preserved. - Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved. - Delete both services, assert only cluster CIDR route is advertised. """ with DiagsCollector(): calicoctl("""apply -f - << EOF apiVersion: projectcalico.org/v3 kind: BGPConfiguration metadata: name: default spec: serviceClusterIPs: - cidr: fd00:10:96::/112 EOF """) # Assert that a route to the service IP range is present. retry_until_success(lambda: self.assertIn("fd00:10:96::/112", self.get_routes())) # Create both a Local and a Cluster type NodePort service with a single replica. local_svc = "nginx-local" cluster_svc = "nginx-cluster" self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", local_svc, self.ns, 80, ipv6=True) self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", cluster_svc, self.ns, 80, traffic_policy="Cluster", ipv6=True) self.wait_until_exists(local_svc, "svc", self.ns) self.wait_until_exists(cluster_svc, "svc", self.ns) # Get clusterIPs. local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns) cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns) # Wait for the deployments to roll out. self.wait_for_deployment(local_svc, self.ns) self.wait_for_deployment(cluster_svc, self.ns) # Assert that both nginx service can be curled from the external node. retry_until_success(curl, function_args=[local_svc_ip]) retry_until_success(curl, function_args=[cluster_svc_ip]) # Assert that local clusterIP is an advertised route and cluster clusterIP is not. retry_until_success(lambda: self.assertIn(local_svc_ip, self.get_routes())) retry_until_success(lambda: self.assertNotIn(cluster_svc_ip, self.get_routes())) # Create a network policy that only accepts traffic from the external node. kubectl("""apply -f - << EOF apiVersion: networking.k8s.io/v1 kind: NetworkPolicy metadata: name: allow-tcp-80-ex namespace: bgp-test spec: podSelector: {} policyTypes: - Ingress ingress: - from: - ipBlock: { cidr: %s/128 } ports: - protocol: TCP port: 80 EOF """ % self.external_node_ip) # Connectivity to nginx-local should always succeed. for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # NOTE: Unlike in the IPv6 case (in test_bgp_advert.py) we cannot successfully test that # connectivity to nginx-cluster is load-balanced across all nodes (and hence, with the # above policy in place, will sometimes fail and sometimes succeed), because our current # observation is that Linux's IPv6 ECMP route choice does _not_ depend on source port, # even though it is documented as such when fib_multipath_hash_policy == 1. # Scale the local_svc to 4 replicas self.scale_deployment(local_svc, self.ns, 4) self.wait_for_deployment(local_svc, self.ns) self.assert_ecmp_routes(local_svc_ip, [self.ipv6s[1], self.ipv6s[2], self.ipv6s[3]]) for i in range(attempts): retry_until_success(curl, function_args=[local_svc_ip]) # Delete both services. self.delete_and_confirm(local_svc, "svc", self.ns) self.delete_and_confirm(cluster_svc, "svc", self.ns) # Assert that clusterIP is no longer an advertised route. retry_until_success(lambda: self.assertNotIn(local_svc_ip, self.get_routes()))