def __init__(self, name):
self.name = name
self.workloads = set()
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
log_and_run("docker rm -f %s || true" % self.name)
log_and_run("docker run --privileged -tid"
"-v `pwd`/docker:/usr/local/bin/docker "
"-v %s:/code --name %s "
"calico/dind:libnetwork --cluster-store=etcd://%s:2379" %
(os.getcwd(), self.name, utils.get_ip()))
self.ip = log_and_run("docker inspect --format "
"'{{ .NetworkSettings.IPAddress }}' %s" % self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps, ex_class=CalledProcessError,
retries=10)
self.execute("gunzip -c /code/calico-node.tgz | docker load")
self.execute("gunzip -c /code/busybox.tgz | docker load")
self.execute("gunzip -c /code/calico-node-libnetwork.tgz | docker load")
self.start_calico_node()
def __init__(self, name):
self.name = name
self.workloads = set()
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
log_and_run("docker rm -f %s || true" % self.name)
log_and_run("docker run --privileged -tid"
"-v `pwd`/docker:/usr/local/bin/docker "
"-v %s:/code --name %s "
"calico/dind:libnetwork --cluster-store=etcd://%s:2379" %
(os.getcwd(), self.name, utils.get_ip()))
self.ip = log_and_run("docker inspect --format "
"'{{ .NetworkSettings.IPAddress }}' %s" %
self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps, ex_class=CalledProcessError, retries=10)
self.execute("gunzip -c /code/calico-node.tgz | docker load")
self.execute("gunzip -c /code/busybox.tgz | docker load")
self.execute(
"gunzip -c /code/calico-node-libnetwork.tgz | docker load")
self.start_calico_node()
def tearDownClass(cls):
# Tidy up
for host in cls.hosts:
host.remove_workloads()
for host in cls.hosts:
host.cleanup()
del host
log_and_run("docker rm -f cali-st-ext-nginx || true")
def set_ext_container_default_route(cls, container_name):
pid = log_and_run("docker inspect -f '{{.State.Pid}}' %s" %
container_name)
_log.info("pid is %s", pid)
log_and_run("mkdir -p /var/run/netns; "
"ln -s /proc/%s/ns/net /var/run/netns/%s; "
"ip netns exec %s ip route del default; "
"ip netns exec %s ip route add default via %s" %
(pid, pid, pid, pid, cls.gateway_ext_ip))
def set_ext_container_default_route(cls, container_name):
pid = log_and_run("docker inspect -f '{{.State.Pid}}' %s" %
container_name)
_log.info("pid is %s", pid)
log_and_run("mkdir -p /var/run/netns; "
"ln -s /proc/%s/ns/net /var/run/netns/%s; "
"ip netns exec %s ip route del default; "
"ip netns exec %s ip route add default via %s" %
(pid, pid, pid, pid, cls.gateway_ext_ip))
def tearDownClass(cls):
# Tidy up
for host in cls.hosts:
host.remove_workloads()
for host in cls.hosts:
host.cleanup()
del host
log_and_run("docker rm -f cali-st-ext-nginx || true")
def tearDownClass(cls):
# Tidy up
for host in cls.hosts:
host.remove_workloads()
for host in cls.hosts:
host.cleanup()
del host
cls.calinet.delete()
log_and_run("docker rm -f cali-st-ext-nginx || true")
clear_on_failures()
def cleanup(self):
"""
Clean up this host, including removing any containers is created. This
is necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
:return:
"""
# For Docker-in-Docker, we need to remove all containers and
# all images...
self.remove_containers()
self.remove_images()
# ...and the outer container.
log_and_run("docker rm -f %s || true" % self.name)
self._cleaned = True
def cleanup(self):
"""
Clean up this host, including removing any containers is created. This
is necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
:return:
"""
# For Docker-in-Docker, we need to remove all containers and
# all images...
self.remove_containers()
self.remove_images()
# ...and the outer container.
log_and_run("docker rm -f %s || true" % self.name)
self._cleaned = True
def get_hostname(self):
"""
Get the hostname from Docker
The hostname is a randomly generated string.
Note, this function only works with a host with dind enabled.
Raises an exception if dind is not enabled.
:param host: DockerHost object
:return: hostname of DockerHost
"""
command = "docker inspect --format {{.Config.Hostname}} %s" % self.name
return log_and_run(command)
def get_hostname(self):
"""
Get the hostname from Docker
The hostname is a randomly generated string.
Note, this function only works with a host with dind enabled.
Raises an exception if dind is not enabled.
:param host: DockerHost object
:return: hostname of DockerHost
"""
command = "docker inspect --format {{.Config.Hostname}} %s" % self.name
return log_and_run(command)
def execute(self, command):
"""
Pass a command into a host container.
Raises a CommandExecError() if the command returns a non-zero
return code.
:param command: The command to execute.
:return: The output from the command with leading and trailing
whitespace removed.
"""
command = self.escape_shell_single_quotes(command)
command = "docker exec -it %s sh -c '%s'" % (self.name, command)
return log_and_run(command)
def execute(self, command):
"""
Pass a command into a host container.
Raises a CommandExecError() if the command returns a non-zero
return code.
:param command: The command to execute.
:return: The output from the command with leading and trailing
whitespace removed.
"""
command = self.escape_shell_single_quotes(command)
command = "docker exec -it %s sh -c '%s'" % (self.name,
command)
return log_and_run(command)
def get_container_ip(cls, container_name):
ip = log_and_run(
"docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' %s" %
container_name)
return ip.strip()
def setUpClass(cls):
# Wipe etcd once before any test in this class runs.
_log.debug("Wiping etcd")
wipe_etcd(HOST_IPV4)
# We set up an additional docker network to act as the external
# network. The Gateway container is connected to both networks.
# and we configure it as a NAT gateway.
#
# "cali-st-ext" host
# container
# |
# "cali-st-ext" docker
# bridge
# |
# Gateway Host
# container container
# \ /
# default docker
# bridge
# We are testing two host endpoints including
# gw_int connecting gateway with host through internal network.
# gw_ext connecting gateway with external server.
#
# We are testing five access patterns.
# Host to external server through gateway.
# Host -> gw_int(untracked ingress, preDNAT) -> gw_int(forward ingress) ->
# gw_ext(forward egress) -> gw_ext(untracked egress) -> external server.
#
# Host to workload running on gateway.
# Host -> gw_int(untracked ingress, preDNAT) -> gw_int(forward ingress) ->
# workload (workload ingress)
#
# Host to process running on gateway.
# Host -> gw_int(untracked ingress, preDNAT) -> gw_int(normal ingress)
#
# Process running on gateway to external server.
# Process -> gw_ext(normal egress) -> gw_ext(untracked egress)
#
# Workload running on gateway to external server.
# Workload (workload egress) -> gw_ext(forward egress) -> gw_ext(untracked egress)
# First, create the hosts and the gateway.
cls.hosts = []
cls.gateway = DockerHost(
"cali-st-gw",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False)
cls.gateway_hostname = cls.gateway.execute("hostname")
cls.host = DockerHost(
"cali-st-host",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False)
cls.host_hostname = cls.host.execute("hostname")
cls.hosts.append(cls.gateway)
cls.hosts.append(cls.host)
# Delete the nginx container if it still exists. We need to do this
# before we try to remove the network.
log_and_run("docker rm -f cali-st-ext-nginx || true")
# Create the external network.
log_and_run("docker network rm cali-st-ext || true")
# Use 172.19.0.0 to avoid clash with normal docker subnet and
# docker-in-docker subnet
log_and_run(
"docker network create --driver bridge --subnet 172.19.0.0/16 cali-st-ext"
)
# And an nginx server on the external network only.
log_and_run("docker run"
" --network=cali-st-ext"
" -d"
" --name=cali-st-ext-nginx"
" nginx")
for host in cls.hosts:
host.start_calico_node()
# Run local httpd server on gateway.
cls.gateway.execute(
"echo '<HTML> Local process </HTML>' > $HOME/index.html && httpd -p 80 -h $HOME"
)
# Get the internal IP of the gateway. We do this before we add the second
# network since it means we don't have to figure out which IP is which.
int_ip = str(cls.gateway.ip)
cls.gateway_int_ip = int_ip
_log.info("Gateway internal IP: %s", cls.gateway_int_ip)
# Add the gateway to the external network.
log_and_run("docker network connect cali-st-ext cali-st-gw")
# Get the external IP of the gateway.
ext_ip = log_and_run(
"docker inspect --format "
"'{{with index .NetworkSettings.Networks"
" \"cali-st-ext\"}}{{.IPAddress}}{{end}}' cali-st-gw")
cls.gateway_ext_ip = ext_ip
_log.info("Gateway external IP: %s", cls.gateway_ext_ip)
# Get the IP of the external server.
ext_ip = cls.get_container_ip("cali-st-ext-nginx")
cls.ext_server_ip = ext_ip
_log.info("External server IP: %s", cls.ext_server_ip)
# Configure the internal host to use the gateway for the external IP.
cls.host.execute("ip route add %s via %s" %
(cls.ext_server_ip, cls.gateway_int_ip))
# Configure the gateway to forward and NAT.
cls.gateway.execute("sysctl -w net.ipv4.ip_forward=1")
cls.gateway.execute(
"iptables -t nat -A POSTROUTING --destination %s -j MASQUERADE" %
cls.ext_server_ip)
cls.calinet = cls.gateway.create_network("calinet")
cls.gateway_workload = cls.gateway.create_workload(
"gw-wl",
image="workload",
network=cls.calinet,
labels=["org.projectcalico.label.wep=gateway"])
cls.host_workload = cls.host.create_workload(
"host-wl",
image="workload",
network=cls.calinet,
labels=["org.projectcalico.label.wep=host"])
clear_on_failures()
add_on_failure(cls.host.log_extra_diags)
add_on_failure(cls.gateway.log_extra_diags)
def setUpClass(cls):
# Wipe etcd once before any test in this class runs.
wipe_etcd()
# We set up an additional docker network to act as the external
# network. The Gateway container is connected to both networks.
# and we configure it as a NAT gateway.
#
# "cali-st-ext" host
# container
# |
# "cali-st-ext" docker
# bridge
# |
# Gateway Host
# container container
# \ /
# default docker
# bridge
# First, create the hosts and the gateway.
cls.hosts = []
cls.gateway = DockerHost(
"cali-st-gw",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False)
cls.gateway_hostname = cls.gateway.execute("hostname")
cls.host = DockerHost(
"cali-st-host",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False)
cls.host_hostname = cls.host.execute("hostname")
cls.hosts.append(cls.gateway)
cls.hosts.append(cls.host)
# Delete the nginx container if it still exists. We need to do this
# before we try to remove the network.
log_and_run("docker rm -f cali-st-ext-nginx || true")
# Create the external network.
log_and_run("docker network rm cali-st-ext || true")
# Use 172.19.0.0 to avoid clash with normal docker subnet and
# docker-in-docker subnet
log_and_run(
"docker network create --driver bridge --subnet 172.19.0.0/16 cali-st-ext"
)
# And an nginx server on the external network only.
log_and_run("docker run"
" --network=cali-st-ext"
" -d"
" --name=cali-st-ext-nginx"
" nginx")
for host in cls.hosts:
host.start_calico_node()
# Get the internal IP of the gateway. We do this before we add the second
# network since it means we don't have to figure out which IP is which.
int_ip = str(cls.gateway.ip)
cls.gateway_int_ip = int_ip
_log.info("Gateway internal IP: %s", cls.gateway_int_ip)
# Add the gateway to the external network.
log_and_run("docker network connect cali-st-ext cali-st-gw")
cls.gateway.execute("ip addr")
# Get the IP of the external server.
ext_ip = cls.get_container_ip("cali-st-ext-nginx")
cls.ext_server_ip = ext_ip
_log.info("External workload IP: %s", cls.ext_server_ip)
# Configure the internal host to use the gateway for the external IP.
cls.host.execute("ip route add %s via %s" %
(cls.ext_server_ip, cls.gateway_int_ip))
# Configure the gateway to forward and NAT.
cls.gateway.execute("sysctl -w net.ipv4.ip_forward=1")
cls.gateway.execute(
"iptables -t nat -A POSTROUTING --destination %s -j MASQUERADE" %
cls.ext_server_ip)
def setUpClass(cls):
# Wipe etcd once before any test in this class runs.
wipe_etcd()
# We set up an additional docker network to act as the external
# network. The Gateway container is connected to both networks.
# and we configure it as a NAT gateway.
#
# "cali-st-ext" host
# container
# |
# "cali-st-ext" docker
# bridge
# |
# Gateway Host
# container container
# \ /
# default docker
# bridge
# We are testing two host endpoints including
# gw_int connecting gateway with host through internal network.
# gw_ext connecting gateway with external server.
#
# We are testing five access patterns.
# Host to external server through gateway.
# Host -> gw_int(untracked ingress, preDNAT) -> gw_int(forward ingress) ->
# gw_ext(forward egress) -> gw_ext(untracked egress) -> external server.
#
# Host to workload running on gateway.
# Host -> gw_int(untracked ingress, preDNAT) -> gw_int(forward ingress) ->
# workload (workload ingress)
#
# Host to process running on gateway.
# Host -> gw_int(untracked ingress, preDNAT) -> gw_int(normal ingress)
#
# Process running on gateway to external server.
# Process -> gw_ext(normal egress) -> gw_ext(untracked egress)
#
# Workload running on gateway to external server.
# Workload (workload egress) -> gw_ext(forward egress) -> gw_ext(untracked egress)
# First, create the hosts and the gateway.
cls.hosts = []
cls.gateway = DockerHost("cali-st-gw",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False)
cls.gateway_hostname = cls.gateway.execute("hostname")
cls.host = DockerHost("cali-st-host",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False)
cls.host_hostname = cls.host.execute("hostname")
cls.hosts.append(cls.gateway)
cls.hosts.append(cls.host)
# Delete the nginx container if it still exists. We need to do this
# before we try to remove the network.
log_and_run("docker rm -f cali-st-ext-nginx || true")
# Create the external network.
log_and_run("docker network rm cali-st-ext || true")
# Use 172.19.0.0 to avoid clash with normal docker subnet and
# docker-in-docker subnet
log_and_run("docker network create --driver bridge --subnet 172.19.0.0/16 cali-st-ext")
# And an nginx server on the external network only.
log_and_run("docker run"
" --network=cali-st-ext"
" -d"
" --name=cali-st-ext-nginx"
" nginx")
for host in cls.hosts:
host.start_calico_node()
# Run local httpd server on gateway.
cls.gateway.execute(
"echo '<HTML> Local process </HTML>' > $HOME/index.html && httpd -p 80 -h $HOME")
# Get the internal IP of the gateway. We do this before we add the second
# network since it means we don't have to figure out which IP is which.
int_ip = str(cls.gateway.ip)
cls.gateway_int_ip = int_ip
_log.info("Gateway internal IP: %s", cls.gateway_int_ip)
# Add the gateway to the external network.
log_and_run("docker network connect cali-st-ext cali-st-gw")
# Get the external IP of the gateway.
ext_ip = log_and_run("docker inspect --format "
"'{{with index .NetworkSettings.Networks"
" \"cali-st-ext\"}}{{.IPAddress}}{{end}}' cali-st-gw")
cls.gateway_ext_ip = ext_ip
_log.info("Gateway external IP: %s", cls.gateway_ext_ip)
# Get the IP of the external server.
ext_ip = cls.get_container_ip("cali-st-ext-nginx")
cls.ext_server_ip = ext_ip
_log.info("External workload IP: %s", cls.ext_server_ip)
# Configure the internal host to use the gateway for the external IP.
cls.host.execute("ip route add %s via %s" %
(cls.ext_server_ip, cls.gateway_int_ip))
# Configure the gateway to forward and NAT.
cls.gateway.execute("sysctl -w net.ipv4.ip_forward=1")
cls.gateway.execute("iptables -t nat -A POSTROUTING --destination %s -j MASQUERADE" %
cls.ext_server_ip)
cls.calinet = cls.gateway.create_network("calinet")
cls.gateway_workload = cls.gateway.create_workload(
"gw-wl",
image="workload",
network=cls.calinet,
labels=["org.projectcalico.label.wep=gateway"])
cls.host_workload = cls.host.create_workload(
"host-wl",
image="workload",
network=cls.calinet,
labels=["org.projectcalico.label.wep=host"])
def get_container_ip(cls, container_name):
ip = log_and_run(
"docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' %s" %
container_name)
return ip.strip()
def setUpClass(cls):
# Wipe etcd once before any test in this class runs.
wipe_etcd()
# We set up an additional docker network to act as the external
# network. The Gateway container is connected to both networks.
# and we configure it as a NAT gateway.
#
# "cali-st-ext" host
# container
# |
# "cali-st-ext" docker
# bridge
# |
# Gateway Host
# container container
# \ /
# default docker
# bridge
# First, create the hosts and the gateway.
cls.hosts = []
cls.gateway = DockerHost("cali-st-gw",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False)
cls.gateway_hostname = cls.gateway.execute("hostname")
cls.host = DockerHost("cali-st-host",
additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS,
post_docker_commands=POST_DOCKER_COMMANDS,
start_calico=False)
cls.host_hostname = cls.host.execute("hostname")
cls.hosts.append(cls.gateway)
cls.hosts.append(cls.host)
# Delete the nginx container if it still exists. We need to do this
# before we try to remove the network.
log_and_run("docker rm -f cali-st-ext-nginx || true")
# Create the external network.
log_and_run("docker network rm cali-st-ext || true")
# Use 172.19.0.0 to avoid clash with normal docker subnet and
# docker-in-docker subnet
log_and_run("docker network create --driver bridge --subnet 172.19.0.0/16 cali-st-ext")
# And an nginx server on the external network only.
log_and_run("docker run"
" --network=cali-st-ext"
" -d"
" --name=cali-st-ext-nginx"
" nginx")
for host in cls.hosts:
host.start_calico_node()
# Get the internal IP of the gateway. We do this before we add the second
# network since it means we don't have to figure out which IP is which.
int_ip = str(cls.gateway.ip)
cls.gateway_int_ip = int_ip
_log.info("Gateway internal IP: %s", cls.gateway_int_ip)
# Add the gateway to the external network.
log_and_run("docker network connect cali-st-ext cali-st-gw")
cls.gateway.execute("ip addr")
# Get the IP of the external server.
ext_ip = cls.get_container_ip("cali-st-ext-nginx")
cls.ext_server_ip = ext_ip
_log.info("External workload IP: %s", cls.ext_server_ip)
# Configure the internal host to use the gateway for the external IP.
cls.host.execute("ip route add %s via %s" %
(cls.ext_server_ip, cls.gateway_int_ip))
# Configure the gateway to forward and NAT.
cls.gateway.execute("sysctl -w net.ipv4.ip_forward=1")
cls.gateway.execute("iptables -t nat -A POSTROUTING --destination %s -j MASQUERADE" %
cls.ext_server_ip)
