def test_many_services(self):
"""
Creates a lot of services quickly
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: 10.96.0.0/12
EOF
""")
# Assert that a route to the service IP range is present.
retry_until_success(
lambda: self.assertIn("10.96.0.0/12", self.get_routes()))
# Create a local service and deployment.
local_svc = "nginx-local"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.wait_for_deployment(local_svc, self.ns)
# Get clusterIPs.
cluster_ips = []
cluster_ips.append(self.get_svc_cluster_ip(local_svc, self.ns))
# Create many more services which select this deployment.
num_svc = 300
for i in range(num_svc):
name = "nginx-svc-%s" % i
self.create_service(name, local_svc, self.ns, 80)
# Get all of their IPs.
for i in range(num_svc):
name = "nginx-svc-%s" % i
cluster_ips.append(self.get_svc_cluster_ip(name, self.ns))
# Assert they are all advertised to the other node. This should happen
# quickly enough that by the time we have queried all services from
# the k8s API, they should be programmed on the remote node.
routes = self.get_routes()
for cip in cluster_ips:
self.assertIn(cip, routes)
# Scale to 0 replicas, assert all routes are removed.
self.scale_deployment(local_svc, self.ns, 0)
self.wait_for_deployment(local_svc, self.ns)
def check_routes_gone():
routes = self.get_routes()
for cip in cluster_ips:
self.assertNotIn(cip, routes)
retry_until_success(check_routes_gone, retries=10, wait_time=5)
def test_ipip_spoof(self):
with DiagsCollector():
# Change pool to use IPIP if necessary
default_pool = json.loads(
calicoctl("get ippool default-ipv4-ippool -o json"))
if default_pool["spec"]["vxlanMode"] != "Never" or default_pool[
"spec"]["ipipMode"] != "Always":
default_pool["spec"]["vxlanMode"] = "Never"
default_pool["spec"]["ipipMode"] = "Always"
calicoctl_apply_dict(default_pool)
# restart calico-nodes
kubectl("delete po -n kube-system -l k8s-app=calico-node")
kubectl("wait --timeout=2m --for=condition=ready" +
" pods -l k8s-app=calico-node -n kube-system")
# get busybox pod IP
remote_pod_ip = retry_until_success(
self.get_pod_ip, function_args=["access", self.ns_name])
print(remote_pod_ip)
# clear conntrack table on all hosts
self.clear_conntrack()
# test connectivity works pod-pod
retry_until_success(self.send_and_check,
function_args=["ipip-normal", remote_pod_ip])
# clear conntrack table on all hosts
self.clear_conntrack()
def send_and_check_ipip_spoof():
self.send_spoofed_ipip_packet(self.ns_name, "scapy",
"10.192.0.3", remote_pod_ip,
"ipip-spoofed")
kubectl(
"exec -t -n %s access grep -- ipip-spoofed /root/snoop.txt"
% self.ns_name)
def assert_cannot_spoof_ipip():
failed = True
try:
send_and_check_ipip_spoof()
except subprocess.CalledProcessError:
failed = False
if failed:
print("ERROR - succeeded in sending spoofed IPIP packet")
raise ConnectionError
# test connectivity does NOT work when spoofing
retry_until_success(assert_cannot_spoof_ipip)
def _test_restart_route_churn(self, num_repeats, restart_func,
expect_churn):
with DiagsCollector():
# Get 2 worker node names, one to monitor routes and one
# to have its calico-node restarted. The first name
# returned is always the master, so skip that.
nodes, ips, _ = node_info()
self.assertGreater(len(nodes), 2)
monitor_node = nodes[1]
self.restart_node = nodes[2]
self.restart_node_ip = ips[2]
# Start running ip monitor on the monitor node, to monitor
# IPv4 route changes. We use "fd00:10:244" to identify
# and exclude IPv6 workload block routes like
# fd00:10:244:0:1cc0:b1ac:ad47:e7c0/122. These definitely
# _do_ flap when the host of that block restarts, but it
# is not yet clear why this is; specifically it is not yet
# known if it indicates anything wrong with calico/node's
# GR setup. See
# https://marc.info/?l=bird-users&m=158298182509702&w=2
# for the mailing list discussion so far.
run("docker exec -d %s sh -c 'stdbuf -oL ip -ts monitor route | stdbuf -oL grep -v fd00:10:244 > rmon.txt'"
% monitor_node)
# Find the name of the calico-node pod on the restart node.
self.get_restart_node_pod_name()
# Restart the calico-node several times, on the other node.
for i in range(num_repeats):
# Restart it.
_log.info("Iteration %d: restart pod %s", i,
self.restart_pod_name)
restart_func(self)
# Kill the ip monitor process.
run("docker exec %s pkill ip" % monitor_node)
# Dump the monitor output.
monitor_output = run("docker exec %s cat rmon.txt" % monitor_node)
if expect_churn:
# Assert that it is not empty.
self.assertNotEqual(monitor_output, "")
else:
# Assert that it is empty.
self.assertEqual(monitor_output, "")
def test_external_ip_advertisement(self):
"""
Runs the tests for service external IP advertisement
"""
with DiagsCollector():
# Whitelist two IP ranges for the external IPs we'll test with
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceExternalIPs:
- cidr: 175.200.0.0/16
- cidr: 200.255.0.0/24
EOF
""")
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that clusterIPs are not advertised.
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Create a network policy that only accepts traffic from the external node.
kubectl("""apply -f - << EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-tcp-80-ex
namespace: bgp-test
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- ipBlock: { cidr: %s/32 }
ports:
- protocol: TCP
port: 80
EOF
""" % self.external_node_ip)
# Get host IPs for the nginx pods.
local_svc_host_ip = self.get_svc_host_ip(local_svc, self.ns)
cluster_svc_host_ip = self.get_svc_host_ip(cluster_svc, self.ns)
# Select an IP from each external IP CIDR.
local_svc_external_ip = "175.200.1.1"
cluster_svc_external_ip = "200.255.255.1"
# Add external IPs to the two services.
self.add_svc_external_ips(local_svc, self.ns,
[local_svc_external_ip])
self.add_svc_external_ips(cluster_svc, self.ns,
[cluster_svc_external_ip])
# Verify that external IPs for local service is advertised but not the cluster service.
local_svc_externalips_route = "%s via %s" % (local_svc_external_ip,
local_svc_host_ip)
cluster_svc_externalips_route = "%s via %s" % (
cluster_svc_external_ip, cluster_svc_host_ip)
retry_until_success(lambda: self.assertIn(
local_svc_externalips_route, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(
cluster_svc_externalips_route, self.get_routes()))
# Scale the local_svc to 4 replicas.
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
# Verify that we have ECMP routes for the external IP of the local service.
retry_until_success(lambda: self.assert_ecmp_routes(
local_svc_external_ip, [self.ips[1], self.ips[2], self.ips[3]])
)
# Delete both services, assert only cluster CIDR route is advertised.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that external IP is no longer an advertised route.
retry_until_success(lambda: self.assertNotIn(
local_svc_externalips_route, self.get_routes()))
def test_cluster_ip_advertisement(self):
"""
Runs the tests for service cluster IP advertisement
- Create both a Local and a Cluster type NodePort service with a single replica.
- assert only local and cluster CIDR routes are advertised.
- assert /32 routes are used, source IP is preserved.
- Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved.
- Delete both services, assert only cluster CIDR route is advertised.
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: 10.96.0.0/12
EOF
""")
# Assert that a route to the service IP range is present.
retry_until_success(
lambda: self.assertIn("10.96.0.0/12", self.get_routes()))
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that both nginx service can be curled from the external node.
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Assert that local clusterIP is an advertised route and cluster clusterIP is not.
retry_until_success(
lambda: self.assertIn(local_svc_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Create a network policy that only accepts traffic from the external node.
kubectl("""apply -f - << EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-tcp-80-ex
namespace: bgp-test
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- ipBlock: { cidr: %s/32 }
ports:
- protocol: TCP
port: 80
EOF
""" % self.external_node_ip)
# Connectivity to nginx-local should always succeed.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Connectivity to nginx-cluster will rarely succeed because it is load-balanced across all nodes.
# When the traffic hits a node that doesn't host one of the service's pod, it will be re-routed
# to another node and SNAT will cause the policy to drop the traffic.
# Try to curl 10 times.
try:
for i in range(attempts):
curl(cluster_svc_ip)
self.fail(
"external node should not be able to consistently access the cluster svc"
)
except subprocess.CalledProcessError:
pass
# Scale the local_svc to 4 replicas
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
self.assert_ecmp_routes(local_svc_ip,
[self.ips[1], self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Delete both services.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that clusterIP is no longer an advertised route.
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def test_simple_policy(self):
with DiagsCollector():
# Check we can talk to service.
retry_until_success(self.can_connect,
retries=10,
wait_time=1,
function_args=["access"])
_log.info("Client 'access' connected to open service")
retry_until_success(self.can_connect,
retries=10,
wait_time=1,
function_args=["no-access"])
_log.info("Client 'no-access' connected to open service")
# Create default-deny policy
policy = client.V1NetworkPolicy(metadata=client.V1ObjectMeta(
name="default-deny", namespace="policy-demo"),
spec={
"podSelector": {
"matchLabels": {},
},
})
client.ExtensionsV1beta1Api().create_namespaced_network_policy(
body=policy,
namespace="policy-demo",
)
_log.debug("Isolation policy created")
# Check we cannot talk to service
retry_until_success(self.cannot_connect,
retries=10,
wait_time=1,
function_args=["access"])
_log.info("Client 'access' failed to connect to isolated service")
retry_until_success(self.cannot_connect,
retries=10,
wait_time=1,
function_args=["no-access"])
_log.info(
"Client 'no-access' failed to connect to isolated service")
# Create allow policy
policy = client.V1NetworkPolicy(metadata=client.V1ObjectMeta(
name="access-nginx", namespace="policy-demo"),
spec={
'ingress': [{
'from': [{
'podSelector': {
'matchLabels': {
'run': 'access'
}
}
}]
}],
'podSelector': {
'matchLabels': {
'app': 'nginx'
}
}
})
client.ExtensionsV1beta1Api().create_namespaced_network_policy(
body=policy,
namespace="policy-demo",
)
_log.debug("Allow policy created.")
# Check we can talk to service as 'access'
retry_until_success(self.can_connect,
retries=10,
wait_time=1,
function_args=["access"])
_log.info("Client 'access' connected to protected service")
# Check we cannot talk to service as 'no-access'
retry_until_success(self.cannot_connect,
retries=10,
wait_time=1,
function_args=["no-access"])
_log.info(
"Client 'no-access' failed to connect to protected service")
def test_calico_monitoring_pods_running(self):
with DiagsCollector():
self.check_pod_status('calico-monitoring')
def test_default_pods_running(self):
with DiagsCollector():
self.check_pod_status('default')
def test_kubesystem_pods_running(self):
with DiagsCollector():
self.check_pod_status('kube-system')
def test_loadbalancer_ip_advertisement(self):
"""
Runs the tests for service LoadBalancer IP advertisement
"""
with DiagsCollector():
# Whitelist IP ranges for the LB IPs we'll test with
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceLoadBalancerIPs:
- cidr: 80.15.0.0/24
EOF
""")
# Create a dummy service first to occupy the first LB IP. This is
# a hack to make sure the chosen IP we use in the tests below
# isn't the same as the zero address in the range.
self.create_service("dummy-service",
"dummy-service",
self.ns,
80,
svc_type="LoadBalancer")
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster",
svc_type="LoadBalancer")
self.deploy("nginx:1.7.9",
local_svc,
self.ns,
80,
svc_type="LoadBalancer")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get the allocated LB IPs.
local_lb_ip = self.get_svc_loadbalancer_ip(local_svc, self.ns)
cluster_lb_ip = self.get_svc_loadbalancer_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Get host IPs for the nginx pods.
local_svc_host_ip = self.get_svc_host_ip(local_svc, self.ns)
cluster_svc_host_ip = self.get_svc_host_ip(cluster_svc, self.ns)
# Verify that LB IP for local service is advertised but not the cluster service.
local_svc_lb_route = "%s via %s" % (local_lb_ip, local_svc_host_ip)
cluster_svc_lb_route = "%s via %s" % (cluster_lb_ip,
cluster_svc_host_ip)
retry_until_success(
lambda: self.assertIn(local_svc_lb_route, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(
cluster_svc_lb_route, self.get_routes()))
# The full range should be advertised from each node.
lb_cidr = "80.15.0.0/24"
retry_until_success(lambda: self.assert_ecmp_routes(
lb_cidr, [self.ips[0], self.ips[1], self.ips[2], self.ips[3]]))
# Scale the local_svc to 4 replicas.
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
# Verify that we have ECMP routes for the LB IP of the local service from nodes running it.
retry_until_success(lambda: self.assert_ecmp_routes(
local_lb_ip, [self.ips[1], self.ips[2], self.ips[3]]))
# Apply a modified BGP config that no longer enables advertisement
# for LoadBalancer IPs.
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec: {}
EOF
""")
# Assert routes are withdrawn.
retry_until_success(
lambda: self.assertNotIn(local_lb_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(lb_cidr, self.get_routes()))
# Apply a modified BGP config that has a mismatched CIDR specified.
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceLoadBalancerIPs:
- cidr: 90.15.0.0/24
EOF
""")
# Assert routes are still withdrawn.
retry_until_success(
lambda: self.assertNotIn(local_lb_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(lb_cidr, self.get_routes()))
# Reapply the correct configuration, we should see routes come back.
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceLoadBalancerIPs:
- cidr: 80.15.0.0/24
EOF
""")
# Verify that we have ECMP routes for the LB IP of the local service from nodes running it.
retry_until_success(lambda: self.assert_ecmp_routes(
local_lb_ip, [self.ips[1], self.ips[2], self.ips[3]]))
retry_until_success(
lambda: self.assertIn(lb_cidr, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(
cluster_svc_lb_route, self.get_routes()))
# Services should be reachable from the external node.
retry_until_success(curl, function_args=[local_lb_ip])
retry_until_success(curl, function_args=[cluster_lb_ip])
# Delete both services, assert only CIDR route is advertised.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that LB IP is no longer an advertised route.
retry_until_success(
lambda: self.assertNotIn(local_lb_ip, self.get_routes()))
def test_node_exclusion(self):
"""
Tests the node exclusion label.
- Create services, assert advertised from all nodes.
- Label one node so that it is excluded, assert that routes are withdrawn from that node.
- Delete / recreate service, assert it is still advertised from the correct nodes.
- Remove the exclusion label, assert that the node re-advertises the svc.
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: 10.96.0.0/12
serviceExternalIPs:
- cidr: 175.200.0.0/16
EOF
""")
# Assert that a route to the service IP range is present.
cluster_cidr = "10.96.0.0/12"
retry_until_success(
lambda: self.assertIn(cluster_cidr, self.get_routes()))
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that both nginx service can be curled from the external node.
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Assert that local clusterIP is an advertised route and cluster clusterIP is not.
retry_until_success(
lambda: self.assertIn(local_svc_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Connectivity should always succeed.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Scale local service to 4 replicas
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert routes are correct and services are accessible.
# Local service should only be advertised from nodes that can run pods.
# The cluster CIDR should be advertised from all nodes.
self.assert_ecmp_routes(local_svc_ip,
[self.ips[1], self.ips[2], self.ips[3]])
self.assert_ecmp_routes(
cluster_cidr,
[self.ips[0], self.ips[1], self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Label one node in order to exclude it from service advertisement.
# After this, we should expect that all routes from that node are
# withdrawn.
kubectl(
"label node %s node.kubernetes.io/exclude-from-external-load-balancers=true"
% self.nodes[1])
# Assert routes are correct and services are accessible.
# It should no longer have a route via self.nodes[1]
self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]])
self.assert_ecmp_routes(cluster_cidr,
[self.ips[0], self.ips[2], self.ips[3]])
# Should work the same for external IP cidr.
external_ip_cidr = "175.200.0.0/16"
self.assert_ecmp_routes(external_ip_cidr,
[self.ips[0], self.ips[2], self.ips[3]])
# Should still be reachable through other nodes.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Delete the local service, confirm that it is no longer advertised.
self.delete_and_confirm(local_svc, "svc", self.ns)
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
# Re-create the local service. Assert it is advertised from the correct nodes,
# but not from the excluded node.
self.create_service(local_svc, local_svc, self.ns, 80)
self.wait_until_exists(local_svc, "svc", self.ns)
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Add an external IP to the local svc and assert it follows the same
# advertisement rules.
local_svc_external_ip = "175.200.1.1"
self.add_svc_external_ips(local_svc, self.ns,
[local_svc_external_ip])
self.assert_ecmp_routes(local_svc_external_ip,
[self.ips[2], self.ips[3]])
# Enable the excluded node. Assert that the node starts
# advertising service routes again.
kubectl(
"label node %s node.kubernetes.io/exclude-from-external-load-balancers=false --overwrite"
% self.nodes[1])
self.assert_ecmp_routes(local_svc_ip,
[self.ips[1], self.ips[2], self.ips[3]])
self.assert_ecmp_routes(local_svc_external_ip,
[self.ips[1], self.ips[2], self.ips[3]])
self.assert_ecmp_routes(
cluster_cidr,
[self.ips[0], self.ips[1], self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Delete both services.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that clusterIP is no longer an advertised route.
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def test_cluster_ip_advertisement(self):
"""
Runs the tests for service cluster IPv6 advertisement
- Create both a Local and a Cluster type NodePort service with a single replica.
- assert only local and cluster CIDR routes are advertised.
- assert /128 routes are used, source IP is preserved.
- Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved.
- Delete both services, assert only cluster CIDR route is advertised.
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: fd00:10:96::/112
EOF
""")
# Assert that a route to the service IP range is present.
retry_until_success(lambda: self.assertIn("fd00:10:96::/112", self.get_routes()))
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", local_svc, self.ns, 80, ipv6=True)
self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", cluster_svc, self.ns, 80, traffic_policy="Cluster", ipv6=True)
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that both nginx service can be curled from the external node.
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Assert that local clusterIP is an advertised route and cluster clusterIP is not.
retry_until_success(lambda: self.assertIn(local_svc_ip, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Create a network policy that only accepts traffic from the external node.
kubectl("""apply -f - << EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-tcp-80-ex
namespace: bgp-test
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- ipBlock: { cidr: %s/128 }
ports:
- protocol: TCP
port: 80
EOF
""" % self.external_node_ip)
# Connectivity to nginx-local should always succeed.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# NOTE: Unlike in the IPv6 case (in test_bgp_advert.py) we cannot successfully test that
# connectivity to nginx-cluster is load-balanced across all nodes (and hence, with the
# above policy in place, will sometimes fail and sometimes succeed), because our current
# observation is that Linux's IPv6 ECMP route choice does _not_ depend on source port,
# even though it is documented as such when fib_multipath_hash_policy == 1.
# Scale the local_svc to 4 replicas
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
self.assert_ecmp_routes(local_svc_ip, [self.ipv6s[1], self.ipv6s[2], self.ipv6s[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Delete both services.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that clusterIP is no longer an advertised route.
retry_until_success(lambda: self.assertNotIn(local_svc_ip, self.get_routes()))