def setUp(self):
"""
Clean up host containers before every test.
"""
containers = docker.ps("-qa").split()
for container in containers:
DockerHost.delete_container(container)
def tearDown(self):
"""
Clean up host containers after every test.
"""
containers = docker.ps("-qa").split()
for container in containers:
DockerHost.delete_container(container)
def test_diags(self):
"""
Test that the diags command successfully uploads the diags file.
"""
host = DockerHost('host')
link = host.execute("/code/dist/calicoctl diags")
assert "https://transfer.sh/" in link
def test_diags(self):
"""
Test that the diags command successfully uploads the diags file.
"""
host = DockerHost('host')
results = host.calicoctl("diags")
self.assertIn(".tar.gz", results)
def __enter__(self):
"""
Set up the route reflector clusters when entering context.
:return: self.
"""
# Create the route reflector hosts, grouped by redundancy.
for ii in range(self.num_redundancy_groups):
cluster_id = str(IPAddress(0xFF000001 + ii))
redundancy_group = []
for jj in range(self.num_in_redundancy_group):
rr = DockerHost('RR.%d.%d' % (ii, jj), start_calico=False)
rr.add_resource({
'apiVersion': 'projectcalico.org/v3',
'kind': 'Node',
'metadata': {
'name': rr.get_hostname(),
'labels': {
'routeReflectorClusterID': cluster_id,
},
},
'spec': {
'bgp': {
'routeReflectorClusterID': cluster_id,
},
},
})
rr.start_calico_node()
# Store the redundancy group.
redundancy_group.append(rr)
self.redundancy_groups.append(redundancy_group)
# If there is more than one of them, configure full mesh
# peering between the route reflectors.
if self.num_redundancy_groups * self.num_in_redundancy_group > 1:
rr.add_resource({
'apiVersion': 'projectcalico.org/v3',
'kind': 'BGPPeer',
'metadata': {
'name': 'rr-mesh',
},
'spec': {
'nodeSelector': 'has(routeReflectorClusterID)',
'peerSelector': 'has(routeReflectorClusterID)',
},
})
return self
def test_add_container(self):
"""
Test adding container to calico networking after it exists.
"""
host = DockerHost('host')
node = host.create_workload("node")
# Use the `container add` command instead of passing a CALICO_IP on
# container creation. Note this no longer needs DOCKER_HOST specified.
host.calicoctl("container add %s 192.168.1.1" % node.name)
host.calicoctl("profile add TEST_GROUP")
host.calicoctl("profile TEST_GROUP member add %s" % node.name)
# Wait for felix to program down the route.
check_route = partial(host.execute, "ip route | grep '192\.168\.1\.1'")
retry_until_success(check_route, ex_class=ErrorReturnCode)
def __enter__(self):
"""
Set up the route reflector clusters when entering context.
:return: self.
"""
# Construct the common environment variables passed in when starting
# the route reflector.
etcd_auth = "-e ETCD_AUTHORITY=%s:2379" % get_ip()
# Create the route reflector hosts, grouped by redundancy.
for ii in range(self.num_redundancy_groups):
cluster_id = str(IPAddress(0xFF000001 + ii))
redundancy_group = []
for jj in range(self.num_in_redundancy_group):
rr = DockerHost('RR.%d.%d' % (ii, jj), start_calico=False)
ip = "-e IP=%s" % rr.ip
rr.execute(
"docker load --input /code/calico_containers/routereflector.tar"
)
rr.execute("docker run --privileged --net=host -d "
"--name rr %s %s "
"calico/routereflector" % (etcd_auth, ip))
# Invoke the suggested curl command to add the RR entry to
# etcd.
#
# See https://github.com/projectcalico/calico-bird/tree/feature-ipinip/build_routereflector
# for details.
rr.execute(
r'curl -L http://%s:2379/v2/keys/calico/bgp/v1/rr_v4/%s '
r'-XPUT -d value="{'
r'\"ip\":\"%s\",'
r'\"cluster_id\":\"%s\"'
r'}"' % (get_ip(), rr.ip, rr.ip, cluster_id))
# Store the redundancy group.
redundancy_group.append(rr)
self.redundancy_groups.append(redundancy_group)
return self
def __enter__(self):
"""
Set up the route reflector clusters when entering context.
:return: self.
"""
# Construct the common environment variables passed in when starting
# the route reflector.
etcd_auth = "-e ETCD_AUTHORITY=%s:2379" % get_ip()
# Create the route reflector hosts, grouped by redundancy.
for ii in range(self.num_redundancy_groups):
cluster_id = str(IPAddress(0xFF000001 + ii))
redundancy_group = []
for jj in range(self.num_in_redundancy_group):
rr = DockerHost('RR.%d.%d' % (ii, jj), start_calico=False)
ip = "-e IP=%s" % rr.ip
rr.execute("docker load --input /code/calico_containers/routereflector.tar")
rr.execute("docker run --privileged --net=host -d "
"--name rr %s %s "
"calico/routereflector" % (etcd_auth, ip))
# Invoke the suggested curl command to add the RR entry to
# etcd.
#
# See https://github.com/projectcalico/calico-bird/tree/feature-ipinip/build_routereflector
# for details.
rr.execute(r'curl -L http://%s:2379/v2/keys/calico/bgp/v1/rr_v4/%s '
r'-XPUT -d value="{'
r'\"ip\":\"%s\",'
r'\"cluster_id\":\"%s\"'
r'}"' % (get_ip(), rr.ip, rr.ip, cluster_id))
# Store the redundancy group.
redundancy_group.append(rr)
self.redundancy_groups.append(redundancy_group)
return self
def test_ipv6(self):
"""
Test mainline functionality with IPv6 addresses.
"""
host = DockerHost("host")
ip1, ip2 = "fd80:24e2:f998:72d6::1:1", "fd80:24e2:f998:72d6::1:2"
# We use this image here because busybox doesn't have ping6.
node1 = host.create_workload("node1", ip=ip1, image="phusion/baseimage:0.9.16")
node2 = host.create_workload("node2", ip=ip2, image="phusion/baseimage:0.9.16")
# Configure the nodes with the same profiles.
host.calicoctl("profile add TEST_GROUP")
host.calicoctl("profile TEST_GROUP member add %s" % node1)
host.calicoctl("profile TEST_GROUP member add %s" % node2)
node1.assert_can_ping(ip2, retries=3)
# Check connectivity.
self.assert_connectivity([node1, node2])
def test_no_powerstrip(self):
"""
Test mainline functionality without using powerstrip.
"""
host = DockerHost("host")
host.calicoctl("profile add TEST_GROUP")
# Remove the environment variable such that docker run does not utilize
# powerstrip.
node1 = host.create_workload("node1", use_powerstrip=False)
node2 = host.create_workload("node2", use_powerstrip=False)
# Attempt to configure the nodes with the same profiles. This will fail
# since we didn't use powerstrip to create the nodes.
with self.assertRaises(ErrorReturnCode):
host.calicoctl("profile TEST_GROUP member add %s" % node1)
with self.assertRaises(ErrorReturnCode):
host.calicoctl("profile TEST_GROUP member add %s" % node2)
# Add the nodes to Calico networking.
ip1, ip2 = "192.168.1.1", "192.168.1.2"
host.calicoctl("container add %s %s" % (node1, ip1))
host.calicoctl("container add %s %s" % (node2, ip2))
# Now add the profiles.
host.calicoctl("profile TEST_GROUP member add %s" % node1)
host.calicoctl("profile TEST_GROUP member add %s" % node2)
# Inspect the nodes (ensure this works without powerstrip)
host.execute("docker inspect %s" % node1)
host.execute("docker inspect %s" % node2)
# Check it works
node1.assert_can_ping(ip1, retries=3)
node1.assert_can_ping(ip2)
node2.assert_can_ping(ip1)
node2.assert_can_ping(ip2)
# Test the teardown commands
host.calicoctl("profile remove TEST_GROUP")
host.calicoctl("container remove %s" % node1)
host.calicoctl("container remove %s" % node2)
host.calicoctl("pool remove 192.168.0.0/16")
host.calicoctl("node stop")
def test_endpoint_commands_mainline(self):
"""
Run a mainline multi-host test using endpoint commands.
This test uses the "endpoint profile set" command to assign
endpoints to profiles according to the following topology:
Host1: [workload_A, workload_B, workload_C]
Host2: [workload_D, workload_E]
Creates a profile that connects A, C, & E
Creates an additional isolated profile for B.
Creates an additional isolated profile for D.
IP Connectivity is then tested to ensure that only workloads
in the same profile can ping one another
"""
host1 = DockerHost('host1')
host2 = DockerHost('host2')
ip_a = "192.168.1.1"
ip_b = "192.168.1.2"
ip_c = "192.168.1.3"
ip_d = "192.168.1.4"
ip_e = "192.168.1.5"
workload_a = host1.create_workload("workload_a", ip_a)
workload_b = host1.create_workload("workload_b", ip_b)
workload_c = host1.create_workload("workload_c", ip_c)
workload_d = host2.create_workload("workload_d", ip_d)
workload_e = host2.create_workload("workload_e", ip_e)
host1.calicoctl("profile add PROF_1_3_5")
host1.calicoctl("profile add PROF_2")
host1.calicoctl("profile add PROF_4")
workload_a_endpoint_id = host1.calicoctl("container workload_a endpoint-id show").strip()
workload_b_endpoint_id = host1.calicoctl("container workload_b endpoint-id show").strip()
workload_c_endpoint_id = host1.calicoctl("container workload_c endpoint-id show").strip()
workload_d_endpoint_id = host2.calicoctl("container workload_d endpoint-id show").strip()
workload_e_endpoint_id = host2.calicoctl("container workload_e endpoint-id show").strip()
host1.calicoctl("endpoint %s profile set PROF_1_3_5" % workload_a_endpoint_id)
host1.calicoctl("endpoint %s profile set PROF_2" % workload_b_endpoint_id)
host1.calicoctl("endpoint %s profile set PROF_1_3_5" % workload_c_endpoint_id)
host2.calicoctl("endpoint %s profile set PROF_4" % workload_d_endpoint_id)
host2.calicoctl("endpoint %s profile set PROF_1_3_5" % workload_e_endpoint_id)
self.assert_connectivity(pass_list=[workload_a, workload_c, workload_e],
fail_list=[workload_b, workload_d])
self.assert_connectivity(pass_list=[workload_b],
fail_list=[workload_a, workload_c, workload_d, workload_e])
self.assert_connectivity(pass_list=[workload_d],
fail_list=[workload_a, workload_b, workload_c, workload_e])
def test_arg_parsing(self):
"""
Test that calicoctl correctly accepts or rejects given argument.
"""
host = DockerHost('host', start_calico=False)
# Run various commands with invalid IPs.
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl('node --ip=127.a.0.1')
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl('node --ip=aa:bb::cc')
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl('node --ip=127.0.0.1 --ip6=127.0.0.1')
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl('node --ip=127.0.0.1 --ip6=aa:bb::zz')
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("bgppeer add 127.a.0.1 as 12345")
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("bgppeer add aa:bb::zz as 12345")
with self.assertRaises(ErrorReturnCode_255):
host.calicoctl("pool add 127.a.0.1")
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("pool add aa:bb::zz")
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("container node1 ip add 127.a.0.1")
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("container node1 ip add aa:bb::zz")
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("container add node1 127.a.0.1")
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("container add node1 aa:bb::zz")
# Add some pools and BGP peers and check the show commands
examples = [
["1.2.3.4", 4],
["aa:bb::ff", 6],
]
for [peer, version] in examples:
host.calicoctl("bgppeer add %s as 12345" % peer)
self.assertIn(peer, host.calicoctl("bgppeer show").stdout.rstrip())
self.assertIn(peer, host.calicoctl("bgppeer show --ipv%s" % version).stdout.rstrip())
self.assertNotIn(peer, host.calicoctl("bgppeer show --ipv%s" % self.ip_not(version)).stdout.rstrip())
host.calicoctl("bgppeer remove %s" % peer)
self.assertNotIn(peer, host.calicoctl("bgppeer show").stdout.rstrip())
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("bgppeer remove %s" % peer)
examples = [
["1.2.3.4", "1.2.3.4/32", 4],
["1.2.3.0/24", "1.2.3.0/24", 4],
["aa:bb::ff", "aa:bb::ff/128", 6],
]
for [ip, subnet, version] in examples:
host.calicoctl("pool add %s" % ip)
self.assertIn(subnet, host.calicoctl("pool show").stdout.rstrip())
self.assertIn(subnet, host.calicoctl("pool show --ipv%s" % version).stdout.rstrip())
self.assertNotIn(subnet, host.calicoctl("pool show --ipv%s" % self.ip_not(version)).stdout.rstrip())
host.calicoctl("pool remove %s" % subnet)
self.assertNotIn(subnet, host.calicoctl("pool show").stdout.rstrip())
# Check default AS command
self.assertEquals("64511",
host.calicoctl("default-node-as").stdout.strip())
host.calicoctl("default-node-as 12345")
self.assertEquals("12345",
host.calicoctl("default-node-as").stdout.strip())
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("default-node-as 99999999999999999999999")
with self.assertRaises(ErrorReturnCode_1):
host.calicoctl("default-node-as abcde")
# Check BGP mesh command
self.assertEquals("on",
host.calicoctl("bgp-node-mesh").stdout.strip())
host.calicoctl("bgp-node-mesh off")
self.assertEquals("off",
host.calicoctl("bgp-node-mesh").stdout.strip())
host.calicoctl("bgp-node-mesh on")
self.assertEquals("on",
host.calicoctl("bgp-node-mesh").stdout.strip())
# Spin up calicoctl specifying an AS number.
host2 = DockerHost('host2', as_num=64512)
# Add some peers
examples = [
["1.2.3.4", 4],
["aa:cc::ff", 6],
]
for [peer, version] in examples:
host2.calicoctl("node bgppeer add %s as 12345" % peer)
self.assertIn(peer, host2.calicoctl("node bgppeer show").stdout.rstrip())
self.assertIn(peer, host2.calicoctl("node bgppeer show --ipv%s" % version).stdout.rstrip())
self.assertNotIn(peer, host2.calicoctl("node bgppeer show --ipv%s" % self.ip_not(version)).stdout.rstrip())
host2.calicoctl("node bgppeer remove %s" % peer)
self.assertNotIn(peer, host2.calicoctl("node bgppeer show").stdout.rstrip())
with self.assertRaises(ErrorReturnCode_1):
host2.calicoctl("node bgppeer remove %s" % peer)
def run_mainline(self, ip1, ip2):
"""
Setup two endpoints on one host and check connectivity.
"""
host = DockerHost('host')
node1 = host.create_workload("node1", ip1)
node2 = host.create_workload("node2", ip2)
# Configure the nodes with the same profiles.
host.calicoctl("profile add TEST_GROUP")
host.calicoctl("profile TEST_GROUP member add %s" % node1)
host.calicoctl("profile TEST_GROUP member add %s" % node2)
# Check connectivity.
self.assert_connectivity([node1, node2])
# Test calicoctl teardown commands.
host.calicoctl("profile remove TEST_GROUP")
host.calicoctl("container remove %s" % node1)
host.calicoctl("container remove %s" % node2)
host.calicoctl("pool remove 192.168.0.0/16")
host.calicoctl("node stop")
def run_mainline(self, ip1, ip2):
"""
Setup two endpoints on one host and check connectivity.
"""
host = DockerHost('host')
host.start_etcd()
host_ip = docker.inspect("--format", "'{{ .NetworkSettings.IPAddress }}'", host.name).stdout.rstrip()
etcd_port = "ETCD_AUTHORITY=%s:2379" % host_ip
calicoctl = etcd_port + " /code/dist/calicoctl %s"
calico_port = "DOCKER_HOST=localhost:2377"
host.execute("docker run --rm -v `pwd`:/target jpetazzo/nsenter", _ok_code=[0, 1])
host.execute(calicoctl % "node --ip=127.0.0.1")
host.execute(calicoctl % "profile add TEST_GROUP")
# Wait for powerstrip to come up.
for i in range(5):
try:
host.listen("%s docker ps" % calico_port)
break
except ErrorReturnCode:
if i == 4:
raise AssertionError("Powerstrip failed to come up.")
else:
sleep(1)
host.listen("%s docker run -e CALICO_IP=%s -tid --name=node1 busybox" % (calico_port, ip1))
host.listen("%s docker run -e CALICO_IP=%s -tid --name=node2 busybox" % (calico_port, ip2))
# Perform a docker inspect to extract the configured IP addresses.
node1_ip = host.execute("%s docker inspect --format '{{ .NetworkSettings.IPAddress }}' node1" % calico_port).stdout.rstrip()
node2_ip = host.execute("%s docker inspect --format '{{ .NetworkSettings.IPAddress }}' node2" % calico_port).stdout.rstrip()
# Configure the nodes with the same profiles.
host.listen(calicoctl % "profile TEST_GROUP member add node1")
host.listen(calicoctl % "profile TEST_GROUP member add node2")
node1_pid = host.execute("docker inspect --format {{.State.Pid}} node1").stdout.rstrip()
node2_pid = host.execute("docker inspect --format {{.State.Pid}} node2").stdout.rstrip()
for i in range(10):
try:
host.listen("./nsenter -t %s ping %s -c 1 -W 1" % (node1_pid, node2_ip))
break
except ErrorReturnCode:
if i == 9:
raise AssertionError("Network failed to come up.")
else:
sleep(1)
# Check connectivity.
host.execute("./nsenter -t %s ping %s -c 1" % (node1_pid, node1_ip))
host.execute("./nsenter -t %s ping %s -c 1" % (node1_pid, node2_ip))
host.execute("./nsenter -t %s ping %s -c 1" % (node2_pid, node1_ip))
host.execute("./nsenter -t %s ping %s -c 1" % (node2_pid, node2_ip))
# Test calicoctl teardown commands.
host.execute(calicoctl % "profile remove TEST_GROUP")
host.execute(calicoctl % "container remove node1")
host.execute(calicoctl % "container remove node2")
host.execute(calicoctl % "pool remove 192.168.0.0/16")
host.execute(calicoctl % "node stop")
def __enter__(self):
"""
Set up the route reflector clusters when entering context.
:return: self.
"""
# Create the route reflector hosts, grouped by redundancy.
for ii in range(self.num_redundancy_groups):
cluster_id = str(IPAddress(0xFF000001 + ii))
redundancy_group = []
for jj in range(self.num_in_redundancy_group):
rr = DockerHost('RR.%d.%d' % (ii, jj), start_calico=False)
ip_env = "-e IP=%s" % rr.ip
rr.execute("docker load --input /code/routereflector.tar")
# Check which type of etcd is being run, then invoke the
# suggested curl command to add the RR entry to etcd.
#
# See https://github.com/projectcalico/calico-bird/tree/feature-ipinip/build_routereflector
# for details.
if os.getenv("ETCD_SCHEME", None) == "https":
# Etcd is running with SSL/TLS, pass the key values
rr.execute("docker run --privileged --net=host -d "
"--name rr %s "
"-e ETCD_ENDPOINTS=https://%s:2379 "
"-e ETCD_CA_CERT_FILE=%s "
"-e ETCD_CERT_FILE=%s "
"-e ETCD_KEY_FILE=%s "
"-v %s/certs:%s/certs "
"calico/routereflector" %
(ip_env, ETCD_HOSTNAME_SSL, ETCD_CA, ETCD_CERT,
ETCD_KEY, CHECKOUT_DIR,CHECKOUT_DIR))
rr.execute(r'curl --cacert %s --cert %s --key %s '
r'-L https://%s:2379/v2/keys/calico/bgp/v1/rr_v4/%s '
r'-XPUT -d value="{'
r'\"ip\":\"%s\",'
r'\"cluster_id\":\"%s\"'
r'}"' % (ETCD_CA, ETCD_CERT, ETCD_KEY,
ETCD_HOSTNAME_SSL, rr.ip, rr.ip,
cluster_id))
else:
rr.execute("docker run --privileged --net=host -d "
"--name rr %s "
"-e ETCD_ENDPOINTS=http://%s:2379 "
"calico/routereflector" % (ip_env, get_ip()))
rr.execute(r'curl -L http://%s:2379/v2/keys/calico/bgp/v1/rr_v4/%s '
r'-XPUT -d value="{'
r'\"ip\":\"%s\",'
r'\"cluster_id\":\"%s\"'
r'}"' % (get_ip(), rr.ip, rr.ip, cluster_id))
# Store the redundancy group.
redundancy_group.append(rr)
self.redundancy_groups.append(redundancy_group)
return self
def test_duplicate_ips(self):
"""
Start two workloads with the same IP on different hosts. Make sure they
can be reached from all places even after one of them is deleted.
"""
host1 = DockerHost('host1')
host2 = DockerHost('host2')
host3 = DockerHost('host3')
# Set up three workloads on three hosts
workload1 = host1.create_workload("workload1", "192.168.1.1")
workload2 = host2.create_workload("workload2", "192.168.1.2")
workload3 = host3.create_workload("workload3", "192.168.1.3")
# Set up the workloads with duplicate IPs
dup_ip = "192.168.1.4"
dup1 = host1.create_workload("dup1", dup_ip)
dup2 = host2.create_workload("dup2", dup_ip)
host1.calicoctl("profile add TEST_PROFILE")
# Add everyone to the same profile
workload1_epid = host1.calicoctl("container %s endpoint-id show" % workload1).strip()
host1.calicoctl("endpoint %s profile append TEST_PROFILE" % workload1_epid)
dup1_epid = host1.calicoctl("container %s endpoint-id show" % dup1).strip()
host1.calicoctl("endpoint %s profile append TEST_PROFILE" % dup1_epid)
workload2_epid = host2.calicoctl("container %s endpoint-id show" % workload2).strip()
host2.calicoctl("endpoint %s profile append TEST_PROFILE" % workload2_epid)
dup2_epid = host2.calicoctl("container %s endpoint-id show" % dup2).strip()
host2.calicoctl("endpoint %s profile append TEST_PROFILE" % dup2_epid)
workload3_dpid = host3.calicoctl("container %s endpoint-id show" % workload3).strip()
host3.calicoctl("endpoint %s profile append TEST_PROFILE" % workload3_dpid)
# Check for standard connectivity
workload1.assert_can_ping(dup_ip, retries=3)
workload2.assert_can_ping(dup_ip, retries=3)
workload3.assert_can_ping(dup_ip, retries=3)
# Delete one of the duplciates.
host2.execute("docker rm -f dup2")
# Check standard connectivity still works.
workload1.assert_can_ping(dup_ip, retries=3)
workload2.assert_can_ping(dup_ip, retries=3)
workload3.assert_can_ping(dup_ip, retries=3)
def test_multi_host(self):
"""
Run a mainline multi-host test. Almost identical in function to the vagrant coreOS demo.
"""
host1 = DockerHost('host1')
host2 = DockerHost('host2')
host1.start_etcd()
host1_ip = docker.inspect("--format", "'{{ .NetworkSettings.IPAddress }}'", host1.name).stdout.rstrip()
host2_ip = docker.inspect("--format", "'{{ .NetworkSettings.IPAddress }}'", host2.name).stdout.rstrip()
etcd_port = "ETCD_AUTHORITY=%s:2379" % host1_ip
calicoctl = etcd_port + " /code/dist/calicoctl %s"
host1.listen(calicoctl % "reset || true")
host1.listen(calicoctl % ("node --ip=%s" % host1_ip))
host2.listen(calicoctl % ("node --ip=%s" % host2_ip))
calico_port = "DOCKER_HOST=localhost:2377"
# Wait for the Calico nodes to be created.
sleep(3)
host1.listen("%s docker run -e CALICO_IP=192.168.1.1 --name workload-A -tid busybox" % calico_port)
host1.listen("%s docker run -e CALICO_IP=192.168.1.2 --name workload-B -tid busybox" % calico_port)
host1.listen("%s docker run -e CALICO_IP=192.168.1.3 --name workload-C -tid busybox" % calico_port)
host2.listen("%s docker run -e CALICO_IP=192.168.1.4 --name workload-D -tid busybox" % calico_port)
host2.listen("%s docker run -e CALICO_IP=192.168.1.5 --name workload-E -tid busybox" % calico_port)
host1.listen(calicoctl % "profile add PROF_A_C_E")
host1.listen(calicoctl % "profile add PROF_B")
host1.listen(calicoctl % "profile add PROF_D")
host1.listen(calicoctl % "profile PROF_A_C_E member add workload-A")
host1.listen(calicoctl % "profile PROF_B member add workload-B")
host1.listen(calicoctl % "profile PROF_A_C_E member add workload-C")
host2.listen(calicoctl % "profile PROF_D member add workload-D")
host2.listen(calicoctl % "profile PROF_A_C_E member add workload-E")
# Wait for the workload networking to converge.
sleep(1)
host1.execute("docker exec workload-A ping -c 4 192.168.1.3")
try:
host1.execute("docker exec workload-A ping -c 4 192.168.1.2")
raise
except ErrorReturnCode_1:
pass
try:
host1.execute("docker exec workload-A ping -c 4 192.168.1.4")
raise
except ErrorReturnCode_1:
pass
host1.execute("docker exec workload-A ping -c 4 192.168.1.5")
def run_multi_host(self, default_as=None, per_node_as=None):
"""
Run a mainline multi-host test.
Almost identical in function to the vagrant coreOS demo.
"""
host1 = DockerHost('host1', as_num=per_node_as)
host2 = DockerHost('host2', as_num=per_node_as)
if default_as:
host1.calicoctl("default-node-as 12345")
ip1 = "192.168.1.1"
ip2 = "192.168.1.2"
ip3 = "192.168.1.3"
ip4 = "192.168.1.4"
ip5 = "192.168.1.5"
workload1 = host1.create_workload("workload1", ip1)
workload2 = host1.create_workload("workload2", ip2)
workload3 = host1.create_workload("workload3", ip3)
workload4 = host2.create_workload("workload4", ip4)
workload5 = host2.create_workload("workload5", ip5)
host1.calicoctl("profile add PROF_1_3_5")
host1.calicoctl("profile add PROF_2")
host1.calicoctl("profile add PROF_4")
host1.calicoctl("profile PROF_1_3_5 member add %s" % workload1)
host1.calicoctl("profile PROF_2 member add %s" % workload2)
host1.calicoctl("profile PROF_1_3_5 member add %s" % workload3)
host2.calicoctl("profile PROF_4 member add %s" % workload4)
host2.calicoctl("profile PROF_1_3_5 member add %s" % workload5)
self.assert_connectivity(pass_list=[workload1, workload3, workload5],
fail_list=[workload2, workload4])
self.assert_connectivity(pass_list=[workload2],
fail_list=[workload1, workload3, workload4, workload5])
self.assert_connectivity(pass_list=[workload4],
fail_list=[workload1, workload2, workload3, workload5])
def run_ipv6_multi_host(self, default_as=None, per_node_as=None):
"""
Run a mainline multi-host test with IPv6.
Almost identical in function to the vagrant coreOS demo.
"""
host1 = DockerHost('host1', as_num=per_node_as)
host2 = DockerHost('host2', as_num=per_node_as)
if default_as:
host1.calicoctl("default-node-as 12345")
ip1 = "fd80:24e2:f998:72d6::1:1"
ip2 = "fd80:24e2:f998:72d6::1:2"
ip3 = "fd80:24e2:f998:72d6::1:3"
ip4 = "fd80:24e2:f998:72d6::1:4"
ip5 = "fd80:24e2:f998:72d6::1:5"
# We use this image here because busybox doesn't have ping6.
workload1 = host1.create_workload("workload1", ip1, image="phusion/baseimage:0.9.16")
workload2 = host1.create_workload("workload2", ip2, image="phusion/baseimage:0.9.16")
workload3 = host1.create_workload("workload3", ip3, image="phusion/baseimage:0.9.16")
workload4 = host2.create_workload("workload4", ip4, image="phusion/baseimage:0.9.16")
workload5 = host2.create_workload("workload5", ip5, image="phusion/baseimage:0.9.16")
host1.calicoctl("profile add PROF_1_3_5")
host1.calicoctl("profile add PROF_2")
host1.calicoctl("profile add PROF_4")
host1.calicoctl("profile PROF_1_3_5 member add %s" % workload1)
host1.calicoctl("profile PROF_2 member add %s" % workload2)
host1.calicoctl("profile PROF_1_3_5 member add %s" % workload3)
host2.calicoctl("profile PROF_4 member add %s" % workload4)
host2.calicoctl("profile PROF_1_3_5 member add %s" % workload5)
self.assert_connectivity(pass_list=[workload1, workload3, workload5],
fail_list=[workload2, workload4])
self.assert_connectivity(pass_list=[workload2],
fail_list=[workload1, workload3, workload4, workload5])
self.assert_connectivity(pass_list=[workload4],
fail_list=[workload1, workload2, workload3, workload5])
def test_add_ip(self):
"""
Test adding multiple IPs per workload.
"""
host = DockerHost('host')
ip11 = "192.168.1.1"
ip12 = "192.168.1.2"
ip21 = "192.168.2.1"
ip22 = "192.168.2.2"
ip31 = "192.168.3.1"
node1 = host.create_workload("node1", ip11)
node2 = host.create_workload("node2")
host.calicoctl("container add %s %s --interface=hello" % (node2, ip12))
host.calicoctl("profile add TEST_GROUP")
host.calicoctl("profile TEST_GROUP member add %s" % node1)
host.calicoctl("profile TEST_GROUP member add %s" % node2)
node1.assert_can_ping(ip12, retries=3)
# Add two more addresses to node1 and one more to node2
host.calicoctl("container node1 ip add %s" % ip21)
host.calicoctl("container node1 ip add %s" % ip31)
host.calicoctl("container %s ip add %s --interface=hello" % (node2, ip22))
node1.assert_can_ping(ip22)
node2.assert_can_ping(ip11)
node2.assert_can_ping(ip21)
node2.assert_can_ping(ip31)
# Now stop and restart node 1 and node 2.
host.execute("docker stop %s" % node1, use_powerstrip=True)
host.execute("docker stop %s" % node2, use_powerstrip=True)
host.execute("docker start %s" % node1, use_powerstrip=True)
host.execute("docker start %s" % node2, use_powerstrip=True)
# Test pings between the IPs.
node1.assert_can_ping(ip12, retries=3)
node1.assert_can_ping(ip22)
node2.assert_can_ping(ip11)
node2.assert_can_ping(ip21)
node2.assert_can_ping(ip31)
# Now remove and check pings to the removed addresses no longer work.
host.calicoctl("container %s ip remove %s" % (node1, ip21))
host.calicoctl("container %s ip remove %s --interface=hello" % (node2, ip22))
node1.assert_can_ping(ip12)
node2.assert_can_ping(ip11)
with self.assertRaises(ErrorReturnCode):
node1.assert_can_ping(ip22)
with self.assertRaises(ErrorReturnCode):
node2.assert_can_ping(ip21)
node2.assert_can_ping(ip31)
# Check that we can't remove addresses twice
with self.assertRaises(ErrorReturnCode):
host.calicoctl("container %s ip remove %s" % (node1, ip21))
def test_profile_commands(self):
"""
Test that the profile rule update command successfully updates.
"""
host = DockerHost('host', start_calico=False)
host.calicoctl("profile add TEST_PROFILE")
json = ('{ "id": "TEST_PROFILE", '
'"inbound_rules": [ { "action": "allow", "src_tag": "TEST_PROFILE" }, '
'{ "action": "deny" } ], '
'"outbound_rules": [ { "action": "deny" } ] }')
calicoctl = "/code/dist/calicoctl %s"
host.execute("echo '%s' | " % json + calicoctl % "profile TEST_PROFILE rule update")
self.assertIn('1 deny', host.calicoctl("profile TEST_PROFILE rule show").stdout.rstrip())
json_piece = '"outbound_rules": [\n {\n "action": "deny"'
self.assertIn(json_piece, host.calicoctl("profile TEST_PROFILE rule json").stdout.rstrip())
# Test that adding and removing a tag works.
self.assertNotIn("TEST_TAG", self.show_tag(host))
host.calicoctl("profile TEST_PROFILE tag add TEST_TAG")
self.assertIn("TEST_TAG", self.show_tag(host))
host.calicoctl("profile TEST_PROFILE tag remove TEST_TAG")
self.assertNotIn("TEST_TAG", self.show_tag(host))
def test_endpoint_commands(self):
"""
Run a mainline multi-host test using endpoint commands
Performs more complicated endpoint profile assignments to test
the append, set, and remove commands in situations where the commands
specify multiple profiles at once.
"""
host1 = DockerHost('host1')
host2 = DockerHost('host2')
ip_main = "192.168.1.1"
ip_a = "192.168.1.2"
ip_b = "192.168.1.3"
ip_c = "192.168.1.4"
workload_main = host1.create_workload("workload_main", ip_main)
host2.create_workload("workload_a", ip_a)
host2.create_workload("workload_b", ip_b)
host2.create_workload("workload_c", ip_c)
workload_main_endpoint_id = host1.calicoctl("container workload_main endpoint-id show").strip()
workload_a_endpoint_id = host2.calicoctl("container workload_a endpoint-id show").strip()
workload_b_endpoint_id = host2.calicoctl("container workload_b endpoint-id show").strip()
workload_c_endpoint_id = host2.calicoctl("container workload_c endpoint-id show").strip()
host1.calicoctl("profile add PROF_A")
host1.calicoctl("profile add PROF_B")
host1.calicoctl("profile add PROF_C")
host2.calicoctl("endpoint %s profile set PROF_A" % workload_a_endpoint_id)
host2.calicoctl("endpoint %s profile set PROF_B" % workload_b_endpoint_id)
host2.calicoctl("endpoint %s profile set PROF_C" % workload_c_endpoint_id)
# Test set single profile
host1.calicoctl("endpoint %s profile set PROF_A" % workload_main_endpoint_id)
workload_main.assert_can_ping(ip_a, retries=4)
workload_main.assert_cant_ping(ip_b)
workload_main.assert_cant_ping(ip_c)
# Test set multiple profiles (note: PROF_A should now be removed)
host1.calicoctl("endpoint %s profile set PROF_B PROF_C" % workload_main_endpoint_id)
workload_main.assert_cant_ping(ip_a, retries=4)
workload_main.assert_can_ping(ip_b)
workload_main.assert_can_ping(ip_c)
# Test set profile to None
host1.calicoctl("endpoint %s profile set" % workload_main_endpoint_id)
workload_main.assert_cant_ping(ip_a, retries=4)
workload_main.assert_cant_ping(ip_b)
workload_main.assert_cant_ping(ip_c)
# Append a single profile
host1.calicoctl("endpoint %s profile append PROF_A" % workload_main_endpoint_id)
workload_main.assert_can_ping(ip_a, retries=4)
workload_main.assert_cant_ping(ip_b)
workload_main.assert_cant_ping(ip_c)
# Append two profiles at once
host1.calicoctl("endpoint %s profile append PROF_B PROF_C" % workload_main_endpoint_id)
workload_main.assert_can_ping(ip_a, retries=4)
workload_main.assert_can_ping(ip_b)
workload_main.assert_can_ping(ip_c)
# Remove a single profile
host1.calicoctl("endpoint %s profile remove PROF_C" % workload_main_endpoint_id)
workload_main.assert_can_ping(ip_a, retries=4)
workload_main.assert_can_ping(ip_b)
workload_main.assert_cant_ping(ip_c)
# Remove two profiles at once
host1.calicoctl("endpoint %s profile remove PROF_A PROF_B" % workload_main_endpoint_id)
workload_main.assert_cant_ping(ip_a, retries=4)
workload_main.assert_cant_ping(ip_b)
workload_main.assert_cant_ping(ip_c)
def __enter__(self):
"""
Set up the route reflector clusters when entering context.
:return: self.
"""
# Create the route reflector hosts, grouped by redundancy.
for ii in range(self.num_redundancy_groups):
cluster_id = str(IPAddress(0xFF000001 + ii))
redundancy_group = []
for jj in range(self.num_in_redundancy_group):
rr = DockerHost('RR.%d.%d' % (ii, jj), start_calico=False)
ip_env = "-e IP=%s" % rr.ip
rr.execute("docker load --input /code/routereflector.tar")
# Check which type of etcd is being run, then invoke the
# suggested curl command to add the RR entry to etcd.
#
# See https://github.com/projectcalico/calico-bird/tree/feature-ipinip/build_routereflector
# for details.
rr_ver = os.getenv("RR_VER", "latest")
if os.getenv("ETCD_SCHEME", None) == "https":
# Etcd is running with SSL/TLS, pass the key values
rr.execute("docker run --privileged --net=host -d "
"--name rr %s "
"-e ETCD_ENDPOINTS=https://%s:2379 "
"-e ETCD_CA_CERT_FILE=%s "
"-e ETCD_CERT_FILE=%s "
"-e ETCD_KEY_FILE=%s "
"-v %s/certs:%s/certs "
"calico/routereflector:%s" %
(ip_env, ETCD_HOSTNAME_SSL, ETCD_CA, ETCD_CERT,
ETCD_KEY, CHECKOUT_DIR, CHECKOUT_DIR, rr_ver))
rr.execute(
r'curl --cacert %s --cert %s --key %s '
r'-L https://%s:2379/v2/keys/calico/bgp/v1/rr_v4/%s '
r'-XPUT -d value="{'
r'\"ip\":\"%s\",'
r'\"cluster_id\":\"%s\"'
r'}"' % (ETCD_CA, ETCD_CERT, ETCD_KEY,
ETCD_HOSTNAME_SSL, rr.ip, rr.ip, cluster_id))
else:
rr.execute("docker run --privileged --net=host -d "
"--name rr %s "
"-e ETCD_ENDPOINTS=http://%s:2379 "
"calico/routereflector:%s" %
(ip_env, get_ip(), rr_ver))
rr.execute(
r'curl -L http://%s:2379/v2/keys/calico/bgp/v1/rr_v4/%s '
r'-XPUT -d value="{'
r'\"ip\":\"%s\",'
r'\"cluster_id\":\"%s\"'
r'}"' % (get_ip(), rr.ip, rr.ip, cluster_id))
# Store the redundancy group.
redundancy_group.append(rr)
self.redundancy_groups.append(redundancy_group)
return self
def test_unix_socket(self):
host = DockerHost('host')
calicoctl = "sudo /code/dist/calicoctl %s"
host._listen(calicoctl % "restart-docker-without-alternative-unix-socket")
# host.start_etcd(restart="always")
host._listen("docker run --restart=always -d --net=host --name etcd quay.io/coreos/etcd:v2.0.10")
# Run without the unix socket. Check that docker can be accessed though both
# the unix socket and the powerstrip TCP port.
host._listen(calicoctl % "node --ip=127.0.0.1")
host._listen("docker ps")
self.assert_powerstrip_up(host)
# Run with the unix socket. Check that docker can be access through both
# unix sockets.
# TODO: Currently hangs here.
host._listen(calicoctl % "restart-docker-with-alternative-unix-socket")
# etcd is running under docker, so wait for it to come up.
sleep(5)
host._listen(calicoctl % "node --ip=127.0.0.1")
host._listen("docker ps")
# Switch back to without the unix socket and check that everything still works.
host._listen(calicoctl % "restart-docker-without-alternative-unix-socket")
# etcd is running under docker, so wait for it to come up.
sleep(5)
host._listen(calicoctl % "node --ip=127.0.0.1")
host._listen("docker ps")
self.assert_powerstrip_up(host)
