def test_dual_stack_status(self):
retry_until_success(lambda: read_status(self.status_name), retries=5, wait_time=1)
status = read_status(self.status_name)
# Should have correct agent status.
self.assert_is_subdict({'state': 'Ready', 'routerID': self.test_node_ip}, status['agent']['birdV4'])
self.assert_is_subdict({'state': 'Ready', 'routerID': self.test_node_ip}, status['agent']['birdV6'])
# Should have correct BGP status.
self.assert_is_subdict({'numberEstablishedV4': 3, 'numberEstablishedV6': 3,
'numberNotEstablishedV4': 0, 'numberNotEstablishedV6': 0}, status['bgp'])
# Should have correct peers and routes status.
for i in [0, 2, 3]:
peers = status['bgp']['peersV4']
self.assert_dictlist_has_subdict({'peerIP': self.ips[i], 'state': 'Established', 'type': 'NodeMesh'}, peers)
peers = status['bgp']['peersV6']
self.assert_dictlist_has_subdict({'peerIP': self.ip6s[i], 'state': 'Established', 'type': 'NodeMesh'}, peers)
# Should have correct route status.
routes = status['routes']['routesV4']
self.assert_dictlist_has_subdict({"gateway": self.ips[i], "interface": "eth0", 'type': 'FIB',
"learnedFrom": {
"sourceType": "NodeMesh"
}}, routes)
routes = status['routes']['routesV6']
self.assert_dictlist_has_subdict({"gateway": self.ip6s[i], "interface": "eth0", 'type': 'FIB',
"learnedFrom": {
"sourceType": "NodeMesh"
}}, routes)
def kill_bird(self):
run("docker exec %s pkill bird" % self.restart_node)
def check_bird_running():
run("docker exec %s pgrep bird" % self.restart_node)
retry_until_success(check_bird_running, retries=10, wait_time=1)
time.sleep(5)
def test_many_services(self):
"""
Creates a lot of services quickly
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: 10.96.0.0/12
EOF
""")
# Assert that a route to the service IP range is present.
retry_until_success(
lambda: self.assertIn("10.96.0.0/12", self.get_routes()))
# Create a local service and deployment.
local_svc = "nginx-local"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.wait_for_deployment(local_svc, self.ns)
# Get clusterIPs.
cluster_ips = []
cluster_ips.append(self.get_svc_cluster_ip(local_svc, self.ns))
# Create many more services which select this deployment.
num_svc = 300
for i in range(num_svc):
name = "nginx-svc-%s" % i
self.create_service(name, local_svc, self.ns, 80)
# Get all of their IPs.
for i in range(num_svc):
name = "nginx-svc-%s" % i
cluster_ips.append(self.get_svc_cluster_ip(name, self.ns))
# Assert they are all advertised to the other node. This should happen
# quickly enough that by the time we have queried all services from
# the k8s API, they should be programmed on the remote node.
routes = self.get_routes()
for cip in cluster_ips:
self.assertIn(cip, routes)
# Scale to 0 replicas, assert all routes are removed.
self.scale_deployment(local_svc, self.ns, 0)
self.wait_for_deployment(local_svc, self.ns)
def check_routes_gone():
routes = self.get_routes()
for cip in cluster_ips:
self.assertNotIn(cip, routes)
retry_until_success(check_routes_gone, retries=10, wait_time=5)
def delete_calico_node_pod(self):
run("kubectl delete po %s -n kube-system" % self.restart_pod_name)
# Wait until a replacement calico-node pod has been created.
retry_until_success(self.get_restart_node_pod_name, retries=10, wait_time=1)
# Wait until it is ready, before returning.
run("kubectl wait po %s -n kube-system --timeout=2m --for=condition=ready" %
self.restart_pod_name)
def test_ipip_spoof(self):
with DiagsCollector():
# Change pool to use IPIP if necessary
default_pool = json.loads(
calicoctl("get ippool default-ipv4-ippool -o json"))
if default_pool["spec"]["vxlanMode"] != "Never" or default_pool[
"spec"]["ipipMode"] != "Always":
default_pool["spec"]["vxlanMode"] = "Never"
default_pool["spec"]["ipipMode"] = "Always"
calicoctl_apply_dict(default_pool)
# restart calico-nodes
kubectl("delete po -n kube-system -l k8s-app=calico-node")
kubectl("wait --timeout=2m --for=condition=ready" +
" pods -l k8s-app=calico-node -n kube-system")
# get busybox pod IP
remote_pod_ip = retry_until_success(
self.get_pod_ip, function_args=["access", self.ns_name])
print(remote_pod_ip)
# clear conntrack table on all hosts
self.clear_conntrack()
# test connectivity works pod-pod
retry_until_success(self.send_and_check,
function_args=["ipip-normal", remote_pod_ip])
# clear conntrack table on all hosts
self.clear_conntrack()
def send_and_check_ipip_spoof():
self.send_spoofed_ipip_packet(self.ns_name, "scapy",
"10.192.0.3", remote_pod_ip,
"ipip-spoofed")
kubectl(
"exec -t -n %s access grep -- ipip-spoofed /root/snoop.txt"
% self.ns_name)
def assert_cannot_spoof_ipip():
failed = True
try:
send_and_check_ipip_spoof()
except subprocess.CalledProcessError:
failed = False
if failed:
print("ERROR - succeeded in sending spoofed IPIP packet")
raise ConnectionError
# test connectivity does NOT work when spoofing
retry_until_success(assert_cannot_spoof_ipip)
def test_rr(self):
# Create ExternalTrafficPolicy Local service with one endpoint on node-1
kubectl("""apply -f - << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-rr
namespace: bgp-test
labels:
app: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
run: nginx-rr
template:
metadata:
labels:
app: nginx
run: nginx-rr
spec:
containers:
- name: nginx-rr
image: nginx:1.7.9
ports:
- containerPort: 80
nodeSelector:
beta.kubernetes.io/os: linux
kubernetes.io/hostname: %s
---
apiVersion: v1
kind: Service
metadata:
name: nginx-rr
namespace: bgp-test
labels:
app: nginx
run: nginx-rr
spec:
externalIPs:
- 175.200.1.1
ports:
- port: 80
targetPort: 80
selector:
app: nginx
run: nginx-rr
type: NodePort
externalTrafficPolicy: Local
EOF
""" % self.nodes[1])
calicoctl("get nodes -o yaml")
calicoctl("get bgppeers -o yaml")
calicoctl("get bgpconfigs -o yaml")
# Update the node-2 to behave as a route-reflector
json_str = calicoctl("get node %s -o json" % self.nodes[2])
node_dict = json.loads(json_str)
node_dict['metadata']['labels']['i-am-a-route-reflector'] = 'true'
node_dict['spec']['bgp']['routeReflectorClusterID'] = '224.0.0.1'
calicoctl("""apply -f - << EOF
%s
EOF
""" % json.dumps(node_dict))
# Disable node-to-node mesh, add cluster and external IP CIDRs to
# advertise, and configure BGP peering between the cluster nodes and the
# RR. (The BGP peering from the external node to the RR is included in
# bird_conf_rr above.)
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
nodeToNodeMeshEnabled: false
asNumber: 64512
serviceClusterIPs:
- cidr: 10.96.0.0/12
serviceExternalIPs:
- cidr: 175.200.0.0/16
EOF
""")
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata: {name: peer-with-rr}
spec:
peerIP: %s
asNumber: 64512
EOF
""" % self.ips[2])
svc_json = kubectl("get svc nginx-rr -n bgp-test -o json")
svc_dict = json.loads(svc_json)
cluster_ip = svc_dict['spec']['clusterIP']
external_ip = svc_dict['spec']['externalIPs'][0]
retry_until_success(
lambda: self.assertIn(cluster_ip, self.get_routes()))
retry_until_success(
lambda: self.assertIn(external_ip, self.get_routes()))
def test_external_ip_advertisement(self):
"""
Runs the tests for service external IP advertisement
"""
with DiagsCollector():
# Whitelist two IP ranges for the external IPs we'll test with
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceExternalIPs:
- cidr: 175.200.0.0/16
- cidr: 200.255.0.0/24
EOF
""")
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that clusterIPs are not advertised.
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Create a network policy that only accepts traffic from the external node.
kubectl("""apply -f - << EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-tcp-80-ex
namespace: bgp-test
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- ipBlock: { cidr: %s/32 }
ports:
- protocol: TCP
port: 80
EOF
""" % self.external_node_ip)
# Get host IPs for the nginx pods.
local_svc_host_ip = self.get_svc_host_ip(local_svc, self.ns)
cluster_svc_host_ip = self.get_svc_host_ip(cluster_svc, self.ns)
# Select an IP from each external IP CIDR.
local_svc_external_ip = "175.200.1.1"
cluster_svc_external_ip = "200.255.255.1"
# Add external IPs to the two services.
self.add_svc_external_ips(local_svc, self.ns,
[local_svc_external_ip])
self.add_svc_external_ips(cluster_svc, self.ns,
[cluster_svc_external_ip])
# Verify that external IPs for local service is advertised but not the cluster service.
local_svc_externalips_route = "%s via %s" % (local_svc_external_ip,
local_svc_host_ip)
cluster_svc_externalips_route = "%s via %s" % (
cluster_svc_external_ip, cluster_svc_host_ip)
retry_until_success(lambda: self.assertIn(
local_svc_externalips_route, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(
cluster_svc_externalips_route, self.get_routes()))
# Scale the local_svc to 4 replicas.
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
# Verify that we have ECMP routes for the external IP of the local service.
retry_until_success(lambda: self.assert_ecmp_routes(
local_svc_external_ip, [self.ips[1], self.ips[2], self.ips[3]])
)
# Delete both services, assert only cluster CIDR route is advertised.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that external IP is no longer an advertised route.
retry_until_success(lambda: self.assertNotIn(
local_svc_externalips_route, self.get_routes()))
def test_cluster_ip_advertisement(self):
"""
Runs the tests for service cluster IP advertisement
- Create both a Local and a Cluster type NodePort service with a single replica.
- assert only local and cluster CIDR routes are advertised.
- assert /32 routes are used, source IP is preserved.
- Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved.
- Delete both services, assert only cluster CIDR route is advertised.
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: 10.96.0.0/12
EOF
""")
# Assert that a route to the service IP range is present.
retry_until_success(
lambda: self.assertIn("10.96.0.0/12", self.get_routes()))
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that both nginx service can be curled from the external node.
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Assert that local clusterIP is an advertised route and cluster clusterIP is not.
retry_until_success(
lambda: self.assertIn(local_svc_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Create a network policy that only accepts traffic from the external node.
kubectl("""apply -f - << EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-tcp-80-ex
namespace: bgp-test
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- ipBlock: { cidr: %s/32 }
ports:
- protocol: TCP
port: 80
EOF
""" % self.external_node_ip)
# Connectivity to nginx-local should always succeed.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Connectivity to nginx-cluster will rarely succeed because it is load-balanced across all nodes.
# When the traffic hits a node that doesn't host one of the service's pod, it will be re-routed
# to another node and SNAT will cause the policy to drop the traffic.
# Try to curl 10 times.
try:
for i in range(attempts):
curl(cluster_svc_ip)
self.fail(
"external node should not be able to consistently access the cluster svc"
)
except subprocess.CalledProcessError:
pass
# Scale the local_svc to 4 replicas
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
self.assert_ecmp_routes(local_svc_ip,
[self.ips[1], self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Delete both services.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that clusterIP is no longer an advertised route.
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def assert_ecmp_routes(self, dst, via):
matchStr = dst + " proto bird "
# sort ips and construct match string for ECMP routes.
for ip in sorted(via):
matchStr += "\n\tnexthop via %s dev eth0 weight 1 " % ip
retry_until_success(lambda: self.assertIn(matchStr, self.get_routes()))
def test_rr(self):
self.tearDown()
self.setUpRR()
# Create ExternalTrafficPolicy Local service with one endpoint on node-1
run("""kubectl apply -f - << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-rr
namespace: bgp-test
labels:
app: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
run: nginx-rr
template:
metadata:
labels:
app: nginx
run: nginx-rr
spec:
containers:
- name: nginx-rr
image: nginx:1.7.9
ports:
- containerPort: 80
nodeSelector:
beta.kubernetes.io/os: linux
kubernetes.io/hostname: kube-node-1
---
apiVersion: v1
kind: Service
metadata:
name: nginx-rr
namespace: bgp-test
labels:
app: nginx
run: nginx-rr
spec:
ports:
- port: 80
targetPort: 80
selector:
app: nginx
run: nginx-rr
type: NodePort
externalTrafficPolicy: Local
EOF
""")
run("kubectl exec -i -n kube-system calicoctl -- /calicoctl get nodes -o yaml"
)
run("kubectl exec -i -n kube-system calicoctl -- /calicoctl get bgppeers -o yaml"
)
run("kubectl exec -i -n kube-system calicoctl -- /calicoctl get bgpconfigs -o yaml"
)
# Update the node-2 to behave as a route-reflector
json_str = run(
"kubectl exec -i -n kube-system calicoctl -- /calicoctl get node kube-node-2 -o json"
)
node_dict = json.loads(json_str)
node_dict['metadata']['labels']['i-am-a-route-reflector'] = 'true'
node_dict['spec']['bgp']['routeReflectorClusterID'] = '224.0.0.1'
run("""kubectl exec -i -n kube-system calicoctl -- /calicoctl apply -f - << EOF
%s
EOF
""" % json.dumps(node_dict))
# Disable node-to-node mesh and configure bgp peering
# between node-1 and RR and also between external node and RR
run("""kubectl exec -i -n kube-system calicoctl -- /calicoctl apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata: {name: default}
spec:
nodeToNodeMeshEnabled: false
asNumber: 64512
EOF
""")
run("""kubectl exec -i -n kube-system calicoctl -- /calicoctl apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata: {name: kube-node-1}
spec:
node: kube-node-1
peerIP: 10.192.0.4
asNumber: 64512
EOF
""")
svc_json = run("kubectl get svc nginx-rr -n bgp-test -o json")
svc_dict = json.loads(svc_json)
svcRoute = svc_dict['spec']['clusterIP']
retry_until_success(lambda: self.assertIn(svcRoute, self.get_routes()))
def assert_ecmp_routes(self, dst, via=["10.192.0.3", "10.192.0.4"]):
matchStr = dst + " proto bird "
for ip in via:
matchStr += "\n\tnexthop via %s dev eth0 weight 1" % ip
retry_until_success(lambda: self.assertIn(matchStr, self.get_routes()))
def test_simple_policy(self):
with DiagsCollector():
# Check we can talk to service.
retry_until_success(self.can_connect,
retries=10,
wait_time=1,
function_args=["access"])
_log.info("Client 'access' connected to open service")
retry_until_success(self.can_connect,
retries=10,
wait_time=1,
function_args=["no-access"])
_log.info("Client 'no-access' connected to open service")
# Create default-deny policy
policy = client.V1NetworkPolicy(metadata=client.V1ObjectMeta(
name="default-deny", namespace="policy-demo"),
spec={
"podSelector": {
"matchLabels": {},
},
})
client.ExtensionsV1beta1Api().create_namespaced_network_policy(
body=policy,
namespace="policy-demo",
)
_log.debug("Isolation policy created")
# Check we cannot talk to service
retry_until_success(self.cannot_connect,
retries=10,
wait_time=1,
function_args=["access"])
_log.info("Client 'access' failed to connect to isolated service")
retry_until_success(self.cannot_connect,
retries=10,
wait_time=1,
function_args=["no-access"])
_log.info(
"Client 'no-access' failed to connect to isolated service")
# Create allow policy
policy = client.V1NetworkPolicy(metadata=client.V1ObjectMeta(
name="access-nginx", namespace="policy-demo"),
spec={
'ingress': [{
'from': [{
'podSelector': {
'matchLabels': {
'run': 'access'
}
}
}]
}],
'podSelector': {
'matchLabels': {
'app': 'nginx'
}
}
})
client.ExtensionsV1beta1Api().create_namespaced_network_policy(
body=policy,
namespace="policy-demo",
)
_log.debug("Allow policy created.")
# Check we can talk to service as 'access'
retry_until_success(self.can_connect,
retries=10,
wait_time=1,
function_args=["access"])
_log.info("Client 'access' connected to protected service")
# Check we cannot talk to service as 'no-access'
retry_until_success(self.cannot_connect,
retries=10,
wait_time=1,
function_args=["no-access"])
_log.info(
"Client 'no-access' failed to connect to protected service")
def test_loadbalancer_ip_advertisement(self):
"""
Runs the tests for service LoadBalancer IP advertisement
"""
with DiagsCollector():
# Whitelist IP ranges for the LB IPs we'll test with
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceLoadBalancerIPs:
- cidr: 80.15.0.0/24
EOF
""")
# Create a dummy service first to occupy the first LB IP. This is
# a hack to make sure the chosen IP we use in the tests below
# isn't the same as the zero address in the range.
self.create_service("dummy-service",
"dummy-service",
self.ns,
80,
svc_type="LoadBalancer")
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster",
svc_type="LoadBalancer")
self.deploy("nginx:1.7.9",
local_svc,
self.ns,
80,
svc_type="LoadBalancer")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get the allocated LB IPs.
local_lb_ip = self.get_svc_loadbalancer_ip(local_svc, self.ns)
cluster_lb_ip = self.get_svc_loadbalancer_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Get host IPs for the nginx pods.
local_svc_host_ip = self.get_svc_host_ip(local_svc, self.ns)
cluster_svc_host_ip = self.get_svc_host_ip(cluster_svc, self.ns)
# Verify that LB IP for local service is advertised but not the cluster service.
local_svc_lb_route = "%s via %s" % (local_lb_ip, local_svc_host_ip)
cluster_svc_lb_route = "%s via %s" % (cluster_lb_ip,
cluster_svc_host_ip)
retry_until_success(
lambda: self.assertIn(local_svc_lb_route, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(
cluster_svc_lb_route, self.get_routes()))
# The full range should be advertised from each node.
lb_cidr = "80.15.0.0/24"
retry_until_success(lambda: self.assert_ecmp_routes(
lb_cidr, [self.ips[0], self.ips[1], self.ips[2], self.ips[3]]))
# Scale the local_svc to 4 replicas.
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
# Verify that we have ECMP routes for the LB IP of the local service from nodes running it.
retry_until_success(lambda: self.assert_ecmp_routes(
local_lb_ip, [self.ips[1], self.ips[2], self.ips[3]]))
# Apply a modified BGP config that no longer enables advertisement
# for LoadBalancer IPs.
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec: {}
EOF
""")
# Assert routes are withdrawn.
retry_until_success(
lambda: self.assertNotIn(local_lb_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(lb_cidr, self.get_routes()))
# Apply a modified BGP config that has a mismatched CIDR specified.
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceLoadBalancerIPs:
- cidr: 90.15.0.0/24
EOF
""")
# Assert routes are still withdrawn.
retry_until_success(
lambda: self.assertNotIn(local_lb_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(lb_cidr, self.get_routes()))
# Reapply the correct configuration, we should see routes come back.
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceLoadBalancerIPs:
- cidr: 80.15.0.0/24
EOF
""")
# Verify that we have ECMP routes for the LB IP of the local service from nodes running it.
retry_until_success(lambda: self.assert_ecmp_routes(
local_lb_ip, [self.ips[1], self.ips[2], self.ips[3]]))
retry_until_success(
lambda: self.assertIn(lb_cidr, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(
cluster_svc_lb_route, self.get_routes()))
# Services should be reachable from the external node.
retry_until_success(curl, function_args=[local_lb_ip])
retry_until_success(curl, function_args=[cluster_lb_ip])
# Delete both services, assert only CIDR route is advertised.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that LB IP is no longer an advertised route.
retry_until_success(
lambda: self.assertNotIn(local_lb_ip, self.get_routes()))
def test_node_exclusion(self):
"""
Tests the node exclusion label.
- Create services, assert advertised from all nodes.
- Label one node so that it is excluded, assert that routes are withdrawn from that node.
- Delete / recreate service, assert it is still advertised from the correct nodes.
- Remove the exclusion label, assert that the node re-advertises the svc.
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: 10.96.0.0/12
serviceExternalIPs:
- cidr: 175.200.0.0/16
EOF
""")
# Assert that a route to the service IP range is present.
cluster_cidr = "10.96.0.0/12"
retry_until_success(
lambda: self.assertIn(cluster_cidr, self.get_routes()))
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that both nginx service can be curled from the external node.
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Assert that local clusterIP is an advertised route and cluster clusterIP is not.
retry_until_success(
lambda: self.assertIn(local_svc_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Connectivity should always succeed.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Scale local service to 4 replicas
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert routes are correct and services are accessible.
# Local service should only be advertised from nodes that can run pods.
# The cluster CIDR should be advertised from all nodes.
self.assert_ecmp_routes(local_svc_ip,
[self.ips[1], self.ips[2], self.ips[3]])
self.assert_ecmp_routes(
cluster_cidr,
[self.ips[0], self.ips[1], self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Label one node in order to exclude it from service advertisement.
# After this, we should expect that all routes from that node are
# withdrawn.
kubectl(
"label node %s node.kubernetes.io/exclude-from-external-load-balancers=true"
% self.nodes[1])
# Assert routes are correct and services are accessible.
# It should no longer have a route via self.nodes[1]
self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]])
self.assert_ecmp_routes(cluster_cidr,
[self.ips[0], self.ips[2], self.ips[3]])
# Should work the same for external IP cidr.
external_ip_cidr = "175.200.0.0/16"
self.assert_ecmp_routes(external_ip_cidr,
[self.ips[0], self.ips[2], self.ips[3]])
# Should still be reachable through other nodes.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Delete the local service, confirm that it is no longer advertised.
self.delete_and_confirm(local_svc, "svc", self.ns)
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
# Re-create the local service. Assert it is advertised from the correct nodes,
# but not from the excluded node.
self.create_service(local_svc, local_svc, self.ns, 80)
self.wait_until_exists(local_svc, "svc", self.ns)
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Add an external IP to the local svc and assert it follows the same
# advertisement rules.
local_svc_external_ip = "175.200.1.1"
self.add_svc_external_ips(local_svc, self.ns,
[local_svc_external_ip])
self.assert_ecmp_routes(local_svc_external_ip,
[self.ips[2], self.ips[3]])
# Enable the excluded node. Assert that the node starts
# advertising service routes again.
kubectl(
"label node %s node.kubernetes.io/exclude-from-external-load-balancers=false --overwrite"
% self.nodes[1])
self.assert_ecmp_routes(local_svc_ip,
[self.ips[1], self.ips[2], self.ips[3]])
self.assert_ecmp_routes(local_svc_external_ip,
[self.ips[1], self.ips[2], self.ips[3]])
self.assert_ecmp_routes(
cluster_cidr,
[self.ips[0], self.ips[1], self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Delete both services.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that clusterIP is no longer an advertised route.
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def test_cluster_ip_advertisement(self):
"""
Runs the tests for service cluster IPv6 advertisement
- Create both a Local and a Cluster type NodePort service with a single replica.
- assert only local and cluster CIDR routes are advertised.
- assert /128 routes are used, source IP is preserved.
- Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved.
- Delete both services, assert only cluster CIDR route is advertised.
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: fd00:10:96::/112
EOF
""")
# Assert that a route to the service IP range is present.
retry_until_success(lambda: self.assertIn("fd00:10:96::/112", self.get_routes()))
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", local_svc, self.ns, 80, ipv6=True)
self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", cluster_svc, self.ns, 80, traffic_policy="Cluster", ipv6=True)
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that both nginx service can be curled from the external node.
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Assert that local clusterIP is an advertised route and cluster clusterIP is not.
retry_until_success(lambda: self.assertIn(local_svc_ip, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Create a network policy that only accepts traffic from the external node.
kubectl("""apply -f - << EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-tcp-80-ex
namespace: bgp-test
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- ipBlock: { cidr: %s/128 }
ports:
- protocol: TCP
port: 80
EOF
""" % self.external_node_ip)
# Connectivity to nginx-local should always succeed.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# NOTE: Unlike in the IPv6 case (in test_bgp_advert.py) we cannot successfully test that
# connectivity to nginx-cluster is load-balanced across all nodes (and hence, with the
# above policy in place, will sometimes fail and sometimes succeed), because our current
# observation is that Linux's IPv6 ECMP route choice does _not_ depend on source port,
# even though it is documented as such when fib_multipath_hash_policy == 1.
# Scale the local_svc to 4 replicas
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
self.assert_ecmp_routes(local_svc_ip, [self.ipv6s[1], self.ipv6s[2], self.ipv6s[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Delete both services.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that clusterIP is no longer an advertised route.
retry_until_success(lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
