def setUp(self):
TestBase.setUp(self)
self.create_namespace("policy-demo")
self.deploy("nginx:1.7.9", "nginx", "policy-demo", 80)
# Create two client pods that live for the duration of the
# test. We will use 'kubectl exec' to try wgets from these at
# particular times.
#
# We do it this way - instead of one-shot pods that are
# created, try wget, and then exit - because it takes a
# relatively long time (7 seconds?) in this test setup for
# Calico routing and policy to be set up correctly for a newly
# created pod. In particular it's possible that connection
# from a just-created pod will fail because that pod's IP has
# not yet propagated to the IP set for the ingress policy on
# the server pod - which can confuse test code that is
# expecting connection failure for some other reason.
kubectl("run --generator=run-pod/v1 access -n policy-demo" +
" --image busybox --command /bin/sleep -- 3600")
kubectl("run --generator=run-pod/v1 no-access -n policy-demo" +
" --image busybox --command /bin/sleep -- 3600")
kubectl("wait --timeout=2m --for=condition=available" +
" deployment/nginx -n policy-demo")
kubectl("wait --timeout=2m --for=condition=ready" +
" pod/access -n policy-demo")
kubectl("wait --timeout=2m --for=condition=ready" +
" pod/no-access -n policy-demo")
def send_and_check_vxlan_spoof():
self.send_spoofed_vxlan_packet(self.ns_name, "scapy",
"10.192.0.3", remote_pod_ip,
"vxlan-spoofed")
kubectl(
"exec -t -n %s access grep -- vxlan-spoofed /root/snoop.txt"
% self.ns_name)
def send_and_check_ipip_spoof():
self.send_spoofed_ipip_packet(self.ns_name, "scapy",
"10.192.0.3", remote_pod_ip,
"ipip-spoofed")
kubectl(
"exec -t -n %s access -- grep ipip-spoofed /root/snoop.txt"
% self.ns_name)
def check_connected(name):
try:
kubectl("exec " + name + " -n policy-demo" +
" -- /bin/wget -O /dev/null -q --timeout=1 nginx")
except subprocess.CalledProcessError:
_log.exception("Failed to wget from nginx service")
return False
_log.debug("Contacted service")
return True
def clear_conntrack():
node_dict = json.loads(kubectl("get po "
"-n kube-system "
"-l k8s-app=calico-node "
"-o json"))
# Flush conntrack in every calico-node pod
for entry in node_dict["items"]:
node = entry["metadata"]["name"]
kubectl("exec -n kube-system %s conntrack -- -F" % node)
def tearDown(self):
# Delete deployment
self.delete_and_confirm(self.ns_name, "ns")
# Change pool to use IPIP
default_pool = json.loads(calicoctl("get ippool default-ipv4-ippool -o json"))
default_pool["spec"]["vxlanMode"] = "Never"
default_pool["spec"]["ipipMode"] = "Always"
calicoctl_apply_dict(default_pool)
# restart calico-nodes
kubectl("delete po -n kube-system -l k8s-app=calico-node")
kubectl("wait --timeout=2m --for=condition=ready" +
" pods -l k8s-app=calico-node -n kube-system")
def send_packet(ns_name, name, remote_pod_ip, message):
try:
kubectl("exec " + name + " -ti -n %s -- "
"scapy << EOF\n"
"send("
"IP(dst='%s')/"
"UDP(dport=5000, sport=5000)/"
"Raw(load='%s'))\n" % (ns_name, remote_pod_ip, message))
except subprocess.CalledProcessError:
_log.exception("Failed to send from scapy")
return False
_log.debug("scapy sent direct packet")
return True
def setUp(self):
TestBase.setUp(self)
self.ns_name = generate_unique_id(5, prefix="spoof")
self.create_namespace(self.ns_name)
# Create two client pods that live for the duration of the
# test. We will use 'kubectl exec' to try sending/receiving
# from these at particular times.
#
# We do it this way because it takes a
# relatively long time (7 seconds?) in this test setup for
# Calico routing and policy to be set up correctly for a newly
# created pod.
kubectl(
"run --generator=run-pod/v1 "
"access "
"-n %s "
"--image busybox "
"--overrides='{\"spec\": {\"nodeName\":\"kube-node-1\"}}' "
"--command /bin/sh -- -c \"nc -l -u -p 5000 &> /root/snoop.txt\"" %
self.ns_name)
kubectl("run --generator=run-pod/v1 "
"scapy "
"-n %s "
"--image ehlers/scapy "
"--overrides='{\"spec\": {\"nodeName\":\"kube-node-2\"}}' "
"--command /bin/sleep -- 3600" % self.ns_name)
kubectl("wait --timeout=2m --for=condition=ready" +
" pod/scapy -n %s" % self.ns_name)
kubectl("wait --timeout=2m --for=condition=ready" +
" pod/access -n %s" % self.ns_name)
def test_ipip_spoof(self):
with DiagsCollector():
# Change pool to use IPIP if necessary
default_pool = json.loads(
calicoctl("get ippool default-ipv4-ippool -o json"))
if default_pool["spec"]["vxlanMode"] != "Never" or default_pool[
"spec"]["ipipMode"] != "Always":
default_pool["spec"]["vxlanMode"] = "Never"
default_pool["spec"]["ipipMode"] = "Always"
calicoctl_apply_dict(default_pool)
# restart calico-nodes
kubectl("delete po -n kube-system -l k8s-app=calico-node")
kubectl("wait --timeout=2m --for=condition=ready" +
" pods -l k8s-app=calico-node -n kube-system")
# get busybox pod IP
remote_pod_ip = retry_until_success(
self.get_pod_ip, function_args=["access", self.ns_name])
print(remote_pod_ip)
# clear conntrack table on all hosts
self.clear_conntrack()
# test connectivity works pod-pod
retry_until_success(self.send_and_check,
function_args=["ipip-normal", remote_pod_ip])
# clear conntrack table on all hosts
self.clear_conntrack()
def send_and_check_ipip_spoof():
self.send_spoofed_ipip_packet(self.ns_name, "scapy",
"10.192.0.3", remote_pod_ip,
"ipip-spoofed")
kubectl(
"exec -t -n %s access grep -- ipip-spoofed /root/snoop.txt"
% self.ns_name)
def assert_cannot_spoof_ipip():
failed = True
try:
send_and_check_ipip_spoof()
except subprocess.CalledProcessError:
failed = False
if failed:
print("ERROR - succeeded in sending spoofed IPIP packet")
raise ConnectionError
# test connectivity does NOT work when spoofing
retry_until_success(assert_cannot_spoof_ipip)
def create_status(name, node, interval):
kubectl("""apply -f - <<EOF
apiVersion: projectcalico.org/v3
kind: CalicoNodeStatus
metadata:
name: %s
spec:
node: %s
classes:
- Agent
- BGP
- Routes
updatePeriodSeconds: %d
EOF
""" % (name, node, interval))
def get_svc_host_ipv6(self, svc, ns):
ipv4 = kubectl("get po -l app=%s -n %s -o json | jq -r .items[0].status.hostIP" %
(svc, ns)).strip()
for i in range(len(self.ipv4s)):
if ipv4 == self.ipv4s[i]:
return self.ipv6s[i]
assert False
def send_spoofed_vxlan_packet(ns_name, name, remote_node_ip, remote_pod_ip, message):
try:
kubectl("exec " + name + " -ti -n %s -- "
"scapy3 << EOF\n"
"send("
"IP(dst='%s')/"
"UDP(dport=4789)/"
"VXLAN(vni=4096)/"
"Ether()/"
"IP(dst='%s')/"
"UDP(dport=5000, sport=5000)/"
"Raw(load='%s'))\n" % (ns_name, remote_node_ip, remote_pod_ip, message))
except subprocess.CalledProcessError:
_log.exception("Failed to send spoofed VXLAN packet from scapy")
return False
_log.debug("scapy sent spoofed VXLAN packet")
return True
def get_svc_loadbalancer_ip(self, svc, ns):
for i in range(10):
lb_ip = kubectl(
"get svc %s -n %s -o json | jq -r .status.loadBalancer.ingress[0].ip"
% (svc, ns)).strip()
if lb_ip != "null":
return lb_ip
time.sleep(1)
raise Exception("No LoadBalancer IP found for service: %s/%s" %
(ns, svc))
def setUp(self):
super(_TestBGPAdvert, self).setUp()
# Create bgp test namespace
self.ns = "bgp-test"
self.create_namespace(self.ns)
self.nodes, self.ips, _ = node_info()
self.external_node_ip = start_external_node_with_bgp(
"kube-node-extra",
bird_peer_config=self.get_bird_conf(),
)
# Enable debug logging
self.update_ds_env("calico-node", "kube-system",
{"BGP_LOGSEVERITYSCREEN": "debug"})
# Establish BGPPeer from cluster nodes to node-extra
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata:
name: node-extra.peer%s
EOF
""" % self.get_extra_peer_spec())
kubectl("""apply -f - <<EOF
apiVersion: v1
kind: Secret
metadata:
name: bgp-secrets
namespace: kube-system
type: Opaque
stringData:
rr-password: very-secret
EOF
""")
def test_cluster_ip_advertisement(self):
"""
Runs the tests for service cluster IP advertisement
- Create both a Local and a Cluster type NodePort service with a single replica.
- assert only local and cluster CIDR routes are advertised.
- assert /32 routes are used, source IP is preserved.
- Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved.
- Delete both services, assert only cluster CIDR route is advertised.
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: 10.96.0.0/12
EOF
""")
# Assert that a route to the service IP range is present.
retry_until_success(
lambda: self.assertIn("10.96.0.0/12", self.get_routes()))
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that both nginx service can be curled from the external node.
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Assert that local clusterIP is an advertised route and cluster clusterIP is not.
retry_until_success(
lambda: self.assertIn(local_svc_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Create a network policy that only accepts traffic from the external node.
kubectl("""apply -f - << EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-tcp-80-ex
namespace: bgp-test
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- ipBlock: { cidr: %s/32 }
ports:
- protocol: TCP
port: 80
EOF
""" % self.external_node_ip)
# Connectivity to nginx-local should always succeed.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Connectivity to nginx-cluster will rarely succeed because it is load-balanced across all nodes.
# When the traffic hits a node that doesn't host one of the service's pod, it will be re-routed
# to another node and SNAT will cause the policy to drop the traffic.
# Try to curl 10 times.
try:
for i in range(attempts):
curl(cluster_svc_ip)
self.fail(
"external node should not be able to consistently access the cluster svc"
)
except subprocess.CalledProcessError:
pass
# Scale the local_svc to 4 replicas
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
self.assert_ecmp_routes(local_svc_ip,
[self.ips[1], self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Delete both services.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that clusterIP is no longer an advertised route.
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def add_svc_external_ips(self, svc, ns, ips):
ipsStr = ','.join('"{0}"'.format(ip) for ip in ips)
patchStr = "{\"spec\": {\"externalIPs\": [%s]}}" % (ipsStr)
return kubectl("patch svc %s -n %s --patch '%s'" %
(svc, ns, patchStr)).strip()
def get_svc_host_ip(self, svc, ns):
return kubectl(
"get po -l app=%s -n %s -o json | jq -r .items[0].status.hostIP" %
(svc, ns)).strip()
def get_svc_cluster_ip(self, svc, ns):
return kubectl("get svc %s -n %s -o json | jq -r .spec.clusterIP" %
(svc, ns)).strip()
def tearDown(self):
# Delete deployment
kubectl("delete --grace-period 0 pod access -n policy-demo")
kubectl("delete --grace-period 0 pod no-access -n policy-demo")
self.delete_and_confirm("policy-demo", "ns")
def test_rr(self):
self.tearDown()
self.setUpRR()
# Create ExternalTrafficPolicy Local service with one endpoint on node-1
kubectl("""apply -f - << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-rr
namespace: bgp-test
labels:
app: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
run: nginx-rr
template:
metadata:
labels:
app: nginx
run: nginx-rr
spec:
containers:
- name: nginx-rr
image: nginx:1.7.9
ports:
- containerPort: 80
nodeSelector:
beta.kubernetes.io/os: linux
kubernetes.io/hostname: kube-node-1
---
apiVersion: v1
kind: Service
metadata:
name: nginx-rr
namespace: bgp-test
labels:
app: nginx
run: nginx-rr
spec:
ports:
- port: 80
targetPort: 80
selector:
app: nginx
run: nginx-rr
type: NodePort
externalTrafficPolicy: Local
EOF
""")
calicoctl("get nodes -o yaml")
calicoctl("get bgppeers -o yaml")
calicoctl("get bgpconfigs -o yaml")
# Update the node-2 to behave as a route-reflector
json_str = calicoctl("get node kube-node-2 -o json")
node_dict = json.loads(json_str)
node_dict['metadata']['labels']['i-am-a-route-reflector'] = 'true'
node_dict['spec']['bgp']['routeReflectorClusterID'] = '224.0.0.1'
calicoctl("""apply -f - << EOF
%s
EOF
""" % json.dumps(node_dict))
# Disable node-to-node mesh and configure bgp peering
# between node-1 and RR and also between external node and RR
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata: {name: default}
spec:
nodeToNodeMeshEnabled: false
asNumber: 64512
EOF
""")
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata: {name: kube-node-1}
spec:
node: kube-node-1
peerIP: 10.192.0.4
asNumber: 64512
EOF
""")
svc_json = kubectl("get svc nginx-rr -n bgp-test -o json")
svc_dict = json.loads(svc_json)
svcRoute = svc_dict['spec']['clusterIP']
retry_until_success(lambda: self.assertIn(svcRoute, self.get_routes()))
def delete_status(name):
kubectl("delete caliconodestatuses.crd.projectcalico.org %s" % name)
def test_cluster_ip_advertisement(self):
"""
Runs the tests for service cluster IPv6 advertisement
- Create both a Local and a Cluster type NodePort service with a single replica.
- assert only local and cluster CIDR routes are advertised.
- assert /128 routes are used, source IP is preserved.
- Scale the Local NP service so it is running on multiple nodes, assert ECMP routing, source IP is preserved.
- Delete both services, assert only cluster CIDR route is advertised.
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: fd00:10:96::/112
EOF
""")
# Assert that a route to the service IP range is present.
retry_until_success(lambda: self.assertIn("fd00:10:96::/112", self.get_routes()))
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", local_svc, self.ns, 80, ipv6=True)
self.deploy("gcr.io/kubernetes-e2e-test-images/test-webserver:1.0", cluster_svc, self.ns, 80, traffic_policy="Cluster", ipv6=True)
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that both nginx service can be curled from the external node.
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Assert that local clusterIP is an advertised route and cluster clusterIP is not.
retry_until_success(lambda: self.assertIn(local_svc_ip, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Create a network policy that only accepts traffic from the external node.
kubectl("""apply -f - << EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-tcp-80-ex
namespace: bgp-test
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- ipBlock: { cidr: %s/128 }
ports:
- protocol: TCP
port: 80
EOF
""" % self.external_node_ip)
# Connectivity to nginx-local should always succeed.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# NOTE: Unlike in the IPv6 case (in test_bgp_advert.py) we cannot successfully test that
# connectivity to nginx-cluster is load-balanced across all nodes (and hence, with the
# above policy in place, will sometimes fail and sometimes succeed), because our current
# observation is that Linux's IPv6 ECMP route choice does _not_ depend on source port,
# even though it is documented as such when fib_multipath_hash_policy == 1.
# Scale the local_svc to 4 replicas
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
self.assert_ecmp_routes(local_svc_ip, [self.ipv6s[1], self.ipv6s[2], self.ipv6s[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Delete both services.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that clusterIP is no longer an advertised route.
retry_until_success(lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
def test_external_ip_advertisement(self):
"""
Runs the tests for service external IP advertisement
"""
with DiagsCollector():
# Whitelist two IP ranges for the external IPs we'll test with
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceExternalIPs:
- cidr: 175.200.0.0/16
- cidr: 200.255.0.0/24
EOF
""")
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that clusterIPs are not advertised.
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Create a network policy that only accepts traffic from the external node.
kubectl("""apply -f - << EOF
apiVersion: networking.k8s.io/v1
kind: NetworkPolicy
metadata:
name: allow-tcp-80-ex
namespace: bgp-test
spec:
podSelector: {}
policyTypes:
- Ingress
ingress:
- from:
- ipBlock: { cidr: %s/32 }
ports:
- protocol: TCP
port: 80
EOF
""" % self.external_node_ip)
# Get host IPs for the nginx pods.
local_svc_host_ip = self.get_svc_host_ip(local_svc, self.ns)
cluster_svc_host_ip = self.get_svc_host_ip(cluster_svc, self.ns)
# Select an IP from each external IP CIDR.
local_svc_external_ip = "175.200.1.1"
cluster_svc_external_ip = "200.255.255.1"
# Add external IPs to the two services.
self.add_svc_external_ips(local_svc, self.ns,
[local_svc_external_ip])
self.add_svc_external_ips(cluster_svc, self.ns,
[cluster_svc_external_ip])
# Verify that external IPs for local service is advertised but not the cluster service.
local_svc_externalips_route = "%s via %s" % (local_svc_external_ip,
local_svc_host_ip)
cluster_svc_externalips_route = "%s via %s" % (
cluster_svc_external_ip, cluster_svc_host_ip)
retry_until_success(lambda: self.assertIn(
local_svc_externalips_route, self.get_routes()))
retry_until_success(lambda: self.assertNotIn(
cluster_svc_externalips_route, self.get_routes()))
# Scale the local_svc to 4 replicas.
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
# Verify that we have ECMP routes for the external IP of the local service.
retry_until_success(lambda: self.assert_ecmp_routes(
local_svc_external_ip, [self.ips[1], self.ips[2], self.ips[3]])
)
# Delete both services, assert only cluster CIDR route is advertised.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that external IP is no longer an advertised route.
retry_until_success(lambda: self.assertNotIn(
local_svc_externalips_route, self.get_routes()))
def send_and_check(self, payload, remote_pod_ip):
self.send_packet(self.ns_name, "scapy", remote_pod_ip, payload)
kubectl("exec -t -n %s access grep -- %s /root/snoop.txt" %
(self.ns_name, payload))
def read_status(name):
status_json = kubectl("get caliconodestatus %s -o json" % name)
status_dict = json.loads(status_json)
return status_dict['status']
def get_pod_ip(podname, ns_name):
pod = json.loads(
kubectl("get po -n %s %s -o json" % (ns_name, podname)))
return pod["status"]["podIP"]
def test_rr(self):
# Create ExternalTrafficPolicy Local service with one endpoint on node-1
kubectl("""apply -f - << EOF
apiVersion: apps/v1
kind: Deployment
metadata:
name: nginx-rr
namespace: bgp-test
labels:
app: nginx
spec:
replicas: 1
selector:
matchLabels:
app: nginx
run: nginx-rr
template:
metadata:
labels:
app: nginx
run: nginx-rr
spec:
containers:
- name: nginx-rr
image: nginx:1.7.9
ports:
- containerPort: 80
nodeSelector:
beta.kubernetes.io/os: linux
kubernetes.io/hostname: %s
---
apiVersion: v1
kind: Service
metadata:
name: nginx-rr
namespace: bgp-test
labels:
app: nginx
run: nginx-rr
spec:
externalIPs:
- 175.200.1.1
ports:
- port: 80
targetPort: 80
selector:
app: nginx
run: nginx-rr
type: NodePort
externalTrafficPolicy: Local
EOF
""" % self.nodes[1])
calicoctl("get nodes -o yaml")
calicoctl("get bgppeers -o yaml")
calicoctl("get bgpconfigs -o yaml")
# Update the node-2 to behave as a route-reflector
json_str = calicoctl("get node %s -o json" % self.nodes[2])
node_dict = json.loads(json_str)
node_dict['metadata']['labels']['i-am-a-route-reflector'] = 'true'
node_dict['spec']['bgp']['routeReflectorClusterID'] = '224.0.0.1'
calicoctl("""apply -f - << EOF
%s
EOF
""" % json.dumps(node_dict))
# Disable node-to-node mesh, add cluster and external IP CIDRs to
# advertise, and configure BGP peering between the cluster nodes and the
# RR. (The BGP peering from the external node to the RR is included in
# bird_conf_rr above.)
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
nodeToNodeMeshEnabled: false
asNumber: 64512
serviceClusterIPs:
- cidr: 10.96.0.0/12
serviceExternalIPs:
- cidr: 175.200.0.0/16
EOF
""")
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPPeer
metadata: {name: peer-with-rr}
spec:
peerIP: %s
asNumber: 64512
EOF
""" % self.ips[2])
svc_json = kubectl("get svc nginx-rr -n bgp-test -o json")
svc_dict = json.loads(svc_json)
cluster_ip = svc_dict['spec']['clusterIP']
external_ip = svc_dict['spec']['externalIPs'][0]
retry_until_success(
lambda: self.assertIn(cluster_ip, self.get_routes()))
retry_until_success(
lambda: self.assertIn(external_ip, self.get_routes()))
def test_node_exclusion(self):
"""
Tests the node exclusion label.
- Create services, assert advertised from all nodes.
- Label one node so that it is excluded, assert that routes are withdrawn from that node.
- Delete / recreate service, assert it is still advertised from the correct nodes.
- Remove the exclusion label, assert that the node re-advertises the svc.
"""
with DiagsCollector():
calicoctl("""apply -f - << EOF
apiVersion: projectcalico.org/v3
kind: BGPConfiguration
metadata:
name: default
spec:
serviceClusterIPs:
- cidr: 10.96.0.0/12
serviceExternalIPs:
- cidr: 175.200.0.0/16
EOF
""")
# Assert that a route to the service IP range is present.
cluster_cidr = "10.96.0.0/12"
retry_until_success(
lambda: self.assertIn(cluster_cidr, self.get_routes()))
# Create both a Local and a Cluster type NodePort service with a single replica.
local_svc = "nginx-local"
cluster_svc = "nginx-cluster"
self.deploy("nginx:1.7.9", local_svc, self.ns, 80)
self.deploy("nginx:1.7.9",
cluster_svc,
self.ns,
80,
traffic_policy="Cluster")
self.wait_until_exists(local_svc, "svc", self.ns)
self.wait_until_exists(cluster_svc, "svc", self.ns)
# Get clusterIPs.
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
cluster_svc_ip = self.get_svc_cluster_ip(cluster_svc, self.ns)
# Wait for the deployments to roll out.
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert that both nginx service can be curled from the external node.
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Assert that local clusterIP is an advertised route and cluster clusterIP is not.
retry_until_success(
lambda: self.assertIn(local_svc_ip, self.get_routes()))
retry_until_success(
lambda: self.assertNotIn(cluster_svc_ip, self.get_routes()))
# Connectivity should always succeed.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Scale local service to 4 replicas
self.scale_deployment(local_svc, self.ns, 4)
self.wait_for_deployment(local_svc, self.ns)
self.wait_for_deployment(cluster_svc, self.ns)
# Assert routes are correct and services are accessible.
# Local service should only be advertised from nodes that can run pods.
# The cluster CIDR should be advertised from all nodes.
self.assert_ecmp_routes(local_svc_ip,
[self.ips[1], self.ips[2], self.ips[3]])
self.assert_ecmp_routes(
cluster_cidr,
[self.ips[0], self.ips[1], self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Label one node in order to exclude it from service advertisement.
# After this, we should expect that all routes from that node are
# withdrawn.
kubectl(
"label node %s node.kubernetes.io/exclude-from-external-load-balancers=true"
% self.nodes[1])
# Assert routes are correct and services are accessible.
# It should no longer have a route via self.nodes[1]
self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]])
self.assert_ecmp_routes(cluster_cidr,
[self.ips[0], self.ips[2], self.ips[3]])
# Should work the same for external IP cidr.
external_ip_cidr = "175.200.0.0/16"
self.assert_ecmp_routes(external_ip_cidr,
[self.ips[0], self.ips[2], self.ips[3]])
# Should still be reachable through other nodes.
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
retry_until_success(curl, function_args=[cluster_svc_ip])
# Delete the local service, confirm that it is no longer advertised.
self.delete_and_confirm(local_svc, "svc", self.ns)
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
# Re-create the local service. Assert it is advertised from the correct nodes,
# but not from the excluded node.
self.create_service(local_svc, local_svc, self.ns, 80)
self.wait_until_exists(local_svc, "svc", self.ns)
local_svc_ip = self.get_svc_cluster_ip(local_svc, self.ns)
self.assert_ecmp_routes(local_svc_ip, [self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Add an external IP to the local svc and assert it follows the same
# advertisement rules.
local_svc_external_ip = "175.200.1.1"
self.add_svc_external_ips(local_svc, self.ns,
[local_svc_external_ip])
self.assert_ecmp_routes(local_svc_external_ip,
[self.ips[2], self.ips[3]])
# Enable the excluded node. Assert that the node starts
# advertising service routes again.
kubectl(
"label node %s node.kubernetes.io/exclude-from-external-load-balancers=false --overwrite"
% self.nodes[1])
self.assert_ecmp_routes(local_svc_ip,
[self.ips[1], self.ips[2], self.ips[3]])
self.assert_ecmp_routes(local_svc_external_ip,
[self.ips[1], self.ips[2], self.ips[3]])
self.assert_ecmp_routes(
cluster_cidr,
[self.ips[0], self.ips[1], self.ips[2], self.ips[3]])
for i in range(attempts):
retry_until_success(curl, function_args=[local_svc_ip])
# Delete both services.
self.delete_and_confirm(local_svc, "svc", self.ns)
self.delete_and_confirm(cluster_svc, "svc", self.ns)
# Assert that clusterIP is no longer an advertised route.
retry_until_success(
lambda: self.assertNotIn(local_svc_ip, self.get_routes()))
