def __init__(self, name): self.name = name self.workloads = set() # This variable is used to assert on destruction that this object was # cleaned up. If not used as a context manager, users of this object self._cleaned = False log_and_run("docker rm -f %s || true" % self.name) log_and_run("docker run --privileged -tid" "-v `pwd`/docker:/usr/local/bin/docker " "-v %s:/code --name %s " "calico/dind:libnetwork --cluster-store=etcd://%s:2379" % (os.getcwd(), self.name, utils.get_ip())) self.ip = log_and_run("docker inspect --format " "'{{ .NetworkSettings.IPAddress }}' %s" % self.name) # Make sure docker is up docker_ps = partial(self.execute, "docker ps") retry_until_success(docker_ps, ex_class=CalledProcessError, retries=10) self.execute("gunzip -c /code/calico-node.tgz | docker load") self.execute("gunzip -c /code/busybox.tgz | docker load") self.execute("gunzip -c /code/calico-node-libnetwork.tgz | docker load") self.start_calico_node()
def __init__(self, name): self.name = name self.workloads = set() # This variable is used to assert on destruction that this object was # cleaned up. If not used as a context manager, users of this object self._cleaned = False log_and_run("docker rm -f %s || true" % self.name) log_and_run("docker run --privileged -tid" "-v `pwd`/docker:/usr/local/bin/docker " "-v %s:/code --name %s " "calico/dind:libnetwork --cluster-store=etcd://%s:2379" % (os.getcwd(), self.name, utils.get_ip())) self.ip = log_and_run("docker inspect --format " "'{{ .NetworkSettings.IPAddress }}' %s" % self.name) # Make sure docker is up docker_ps = partial(self.execute, "docker ps") retry_until_success(docker_ps, ex_class=CalledProcessError, retries=10) self.execute("gunzip -c /code/calico-node.tgz | docker load") self.execute("gunzip -c /code/busybox.tgz | docker load") self.execute( "gunzip -c /code/calico-node-libnetwork.tgz | docker load") self.start_calico_node()
def tearDownClass(cls): # Tidy up for host in cls.hosts: host.remove_workloads() for host in cls.hosts: host.cleanup() del host log_and_run("docker rm -f cali-st-ext-nginx || true")
def set_ext_container_default_route(cls, container_name): pid = log_and_run("docker inspect -f '{{.State.Pid}}' %s" % container_name) _log.info("pid is %s", pid) log_and_run("mkdir -p /var/run/netns; " "ln -s /proc/%s/ns/net /var/run/netns/%s; " "ip netns exec %s ip route del default; " "ip netns exec %s ip route add default via %s" % (pid, pid, pid, pid, cls.gateway_ext_ip))
def set_ext_container_default_route(cls, container_name): pid = log_and_run("docker inspect -f '{{.State.Pid}}' %s" % container_name) _log.info("pid is %s", pid) log_and_run("mkdir -p /var/run/netns; " "ln -s /proc/%s/ns/net /var/run/netns/%s; " "ip netns exec %s ip route del default; " "ip netns exec %s ip route add default via %s" % (pid, pid, pid, pid, cls.gateway_ext_ip))
def tearDownClass(cls): # Tidy up for host in cls.hosts: host.remove_workloads() for host in cls.hosts: host.cleanup() del host log_and_run("docker rm -f cali-st-ext-nginx || true")
def tearDownClass(cls): # Tidy up for host in cls.hosts: host.remove_workloads() for host in cls.hosts: host.cleanup() del host cls.calinet.delete() log_and_run("docker rm -f cali-st-ext-nginx || true") clear_on_failures()
def cleanup(self): """ Clean up this host, including removing any containers is created. This is necessary especially for Docker-in-Docker so we don't leave dangling volumes. :return: """ # For Docker-in-Docker, we need to remove all containers and # all images... self.remove_containers() self.remove_images() # ...and the outer container. log_and_run("docker rm -f %s || true" % self.name) self._cleaned = True
def cleanup(self): """ Clean up this host, including removing any containers is created. This is necessary especially for Docker-in-Docker so we don't leave dangling volumes. :return: """ # For Docker-in-Docker, we need to remove all containers and # all images... self.remove_containers() self.remove_images() # ...and the outer container. log_and_run("docker rm -f %s || true" % self.name) self._cleaned = True
def get_hostname(self): """ Get the hostname from Docker The hostname is a randomly generated string. Note, this function only works with a host with dind enabled. Raises an exception if dind is not enabled. :param host: DockerHost object :return: hostname of DockerHost """ command = "docker inspect --format {{.Config.Hostname}} %s" % self.name return log_and_run(command)
def get_hostname(self): """ Get the hostname from Docker The hostname is a randomly generated string. Note, this function only works with a host with dind enabled. Raises an exception if dind is not enabled. :param host: DockerHost object :return: hostname of DockerHost """ command = "docker inspect --format {{.Config.Hostname}} %s" % self.name return log_and_run(command)
def execute(self, command): """ Pass a command into a host container. Raises a CommandExecError() if the command returns a non-zero return code. :param command: The command to execute. :return: The output from the command with leading and trailing whitespace removed. """ command = self.escape_shell_single_quotes(command) command = "docker exec -it %s sh -c '%s'" % (self.name, command) return log_and_run(command)
def execute(self, command): """ Pass a command into a host container. Raises a CommandExecError() if the command returns a non-zero return code. :param command: The command to execute. :return: The output from the command with leading and trailing whitespace removed. """ command = self.escape_shell_single_quotes(command) command = "docker exec -it %s sh -c '%s'" % (self.name, command) return log_and_run(command)
def get_container_ip(cls, container_name): ip = log_and_run( "docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' %s" % container_name) return ip.strip()
def setUpClass(cls): # Wipe etcd once before any test in this class runs. _log.debug("Wiping etcd") wipe_etcd(HOST_IPV4) # We set up an additional docker network to act as the external # network. The Gateway container is connected to both networks. # and we configure it as a NAT gateway. # # "cali-st-ext" host # container # | # "cali-st-ext" docker # bridge # | # Gateway Host # container container # \ / # default docker # bridge # We are testing two host endpoints including # gw_int connecting gateway with host through internal network. # gw_ext connecting gateway with external server. # # We are testing five access patterns. # Host to external server through gateway. # Host -> gw_int(untracked ingress, preDNAT) -> gw_int(forward ingress) -> # gw_ext(forward egress) -> gw_ext(untracked egress) -> external server. # # Host to workload running on gateway. # Host -> gw_int(untracked ingress, preDNAT) -> gw_int(forward ingress) -> # workload (workload ingress) # # Host to process running on gateway. # Host -> gw_int(untracked ingress, preDNAT) -> gw_int(normal ingress) # # Process running on gateway to external server. # Process -> gw_ext(normal egress) -> gw_ext(untracked egress) # # Workload running on gateway to external server. # Workload (workload egress) -> gw_ext(forward egress) -> gw_ext(untracked egress) # First, create the hosts and the gateway. cls.hosts = [] cls.gateway = DockerHost( "cali-st-gw", additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) cls.gateway_hostname = cls.gateway.execute("hostname") cls.host = DockerHost( "cali-st-host", additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) cls.host_hostname = cls.host.execute("hostname") cls.hosts.append(cls.gateway) cls.hosts.append(cls.host) # Delete the nginx container if it still exists. We need to do this # before we try to remove the network. log_and_run("docker rm -f cali-st-ext-nginx || true") # Create the external network. log_and_run("docker network rm cali-st-ext || true") # Use 172.19.0.0 to avoid clash with normal docker subnet and # docker-in-docker subnet log_and_run( "docker network create --driver bridge --subnet 172.19.0.0/16 cali-st-ext" ) # And an nginx server on the external network only. log_and_run("docker run" " --network=cali-st-ext" " -d" " --name=cali-st-ext-nginx" " nginx") for host in cls.hosts: host.start_calico_node() # Run local httpd server on gateway. cls.gateway.execute( "echo '<HTML> Local process </HTML>' > $HOME/index.html && httpd -p 80 -h $HOME" ) # Get the internal IP of the gateway. We do this before we add the second # network since it means we don't have to figure out which IP is which. int_ip = str(cls.gateway.ip) cls.gateway_int_ip = int_ip _log.info("Gateway internal IP: %s", cls.gateway_int_ip) # Add the gateway to the external network. log_and_run("docker network connect cali-st-ext cali-st-gw") # Get the external IP of the gateway. ext_ip = log_and_run( "docker inspect --format " "'{{with index .NetworkSettings.Networks" " \"cali-st-ext\"}}{{.IPAddress}}{{end}}' cali-st-gw") cls.gateway_ext_ip = ext_ip _log.info("Gateway external IP: %s", cls.gateway_ext_ip) # Get the IP of the external server. ext_ip = cls.get_container_ip("cali-st-ext-nginx") cls.ext_server_ip = ext_ip _log.info("External server IP: %s", cls.ext_server_ip) # Configure the internal host to use the gateway for the external IP. cls.host.execute("ip route add %s via %s" % (cls.ext_server_ip, cls.gateway_int_ip)) # Configure the gateway to forward and NAT. cls.gateway.execute("sysctl -w net.ipv4.ip_forward=1") cls.gateway.execute( "iptables -t nat -A POSTROUTING --destination %s -j MASQUERADE" % cls.ext_server_ip) cls.calinet = cls.gateway.create_network("calinet") cls.gateway_workload = cls.gateway.create_workload( "gw-wl", image="workload", network=cls.calinet, labels=["org.projectcalico.label.wep=gateway"]) cls.host_workload = cls.host.create_workload( "host-wl", image="workload", network=cls.calinet, labels=["org.projectcalico.label.wep=host"]) clear_on_failures() add_on_failure(cls.host.log_extra_diags) add_on_failure(cls.gateway.log_extra_diags)
def setUpClass(cls): # Wipe etcd once before any test in this class runs. wipe_etcd() # We set up an additional docker network to act as the external # network. The Gateway container is connected to both networks. # and we configure it as a NAT gateway. # # "cali-st-ext" host # container # | # "cali-st-ext" docker # bridge # | # Gateway Host # container container # \ / # default docker # bridge # First, create the hosts and the gateway. cls.hosts = [] cls.gateway = DockerHost( "cali-st-gw", additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) cls.gateway_hostname = cls.gateway.execute("hostname") cls.host = DockerHost( "cali-st-host", additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) cls.host_hostname = cls.host.execute("hostname") cls.hosts.append(cls.gateway) cls.hosts.append(cls.host) # Delete the nginx container if it still exists. We need to do this # before we try to remove the network. log_and_run("docker rm -f cali-st-ext-nginx || true") # Create the external network. log_and_run("docker network rm cali-st-ext || true") # Use 172.19.0.0 to avoid clash with normal docker subnet and # docker-in-docker subnet log_and_run( "docker network create --driver bridge --subnet 172.19.0.0/16 cali-st-ext" ) # And an nginx server on the external network only. log_and_run("docker run" " --network=cali-st-ext" " -d" " --name=cali-st-ext-nginx" " nginx") for host in cls.hosts: host.start_calico_node() # Get the internal IP of the gateway. We do this before we add the second # network since it means we don't have to figure out which IP is which. int_ip = str(cls.gateway.ip) cls.gateway_int_ip = int_ip _log.info("Gateway internal IP: %s", cls.gateway_int_ip) # Add the gateway to the external network. log_and_run("docker network connect cali-st-ext cali-st-gw") cls.gateway.execute("ip addr") # Get the IP of the external server. ext_ip = cls.get_container_ip("cali-st-ext-nginx") cls.ext_server_ip = ext_ip _log.info("External workload IP: %s", cls.ext_server_ip) # Configure the internal host to use the gateway for the external IP. cls.host.execute("ip route add %s via %s" % (cls.ext_server_ip, cls.gateway_int_ip)) # Configure the gateway to forward and NAT. cls.gateway.execute("sysctl -w net.ipv4.ip_forward=1") cls.gateway.execute( "iptables -t nat -A POSTROUTING --destination %s -j MASQUERADE" % cls.ext_server_ip)
def setUpClass(cls): # Wipe etcd once before any test in this class runs. wipe_etcd() # We set up an additional docker network to act as the external # network. The Gateway container is connected to both networks. # and we configure it as a NAT gateway. # # "cali-st-ext" host # container # | # "cali-st-ext" docker # bridge # | # Gateway Host # container container # \ / # default docker # bridge # We are testing two host endpoints including # gw_int connecting gateway with host through internal network. # gw_ext connecting gateway with external server. # # We are testing five access patterns. # Host to external server through gateway. # Host -> gw_int(untracked ingress, preDNAT) -> gw_int(forward ingress) -> # gw_ext(forward egress) -> gw_ext(untracked egress) -> external server. # # Host to workload running on gateway. # Host -> gw_int(untracked ingress, preDNAT) -> gw_int(forward ingress) -> # workload (workload ingress) # # Host to process running on gateway. # Host -> gw_int(untracked ingress, preDNAT) -> gw_int(normal ingress) # # Process running on gateway to external server. # Process -> gw_ext(normal egress) -> gw_ext(untracked egress) # # Workload running on gateway to external server. # Workload (workload egress) -> gw_ext(forward egress) -> gw_ext(untracked egress) # First, create the hosts and the gateway. cls.hosts = [] cls.gateway = DockerHost("cali-st-gw", additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) cls.gateway_hostname = cls.gateway.execute("hostname") cls.host = DockerHost("cali-st-host", additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) cls.host_hostname = cls.host.execute("hostname") cls.hosts.append(cls.gateway) cls.hosts.append(cls.host) # Delete the nginx container if it still exists. We need to do this # before we try to remove the network. log_and_run("docker rm -f cali-st-ext-nginx || true") # Create the external network. log_and_run("docker network rm cali-st-ext || true") # Use 172.19.0.0 to avoid clash with normal docker subnet and # docker-in-docker subnet log_and_run("docker network create --driver bridge --subnet 172.19.0.0/16 cali-st-ext") # And an nginx server on the external network only. log_and_run("docker run" " --network=cali-st-ext" " -d" " --name=cali-st-ext-nginx" " nginx") for host in cls.hosts: host.start_calico_node() # Run local httpd server on gateway. cls.gateway.execute( "echo '<HTML> Local process </HTML>' > $HOME/index.html && httpd -p 80 -h $HOME") # Get the internal IP of the gateway. We do this before we add the second # network since it means we don't have to figure out which IP is which. int_ip = str(cls.gateway.ip) cls.gateway_int_ip = int_ip _log.info("Gateway internal IP: %s", cls.gateway_int_ip) # Add the gateway to the external network. log_and_run("docker network connect cali-st-ext cali-st-gw") # Get the external IP of the gateway. ext_ip = log_and_run("docker inspect --format " "'{{with index .NetworkSettings.Networks" " \"cali-st-ext\"}}{{.IPAddress}}{{end}}' cali-st-gw") cls.gateway_ext_ip = ext_ip _log.info("Gateway external IP: %s", cls.gateway_ext_ip) # Get the IP of the external server. ext_ip = cls.get_container_ip("cali-st-ext-nginx") cls.ext_server_ip = ext_ip _log.info("External workload IP: %s", cls.ext_server_ip) # Configure the internal host to use the gateway for the external IP. cls.host.execute("ip route add %s via %s" % (cls.ext_server_ip, cls.gateway_int_ip)) # Configure the gateway to forward and NAT. cls.gateway.execute("sysctl -w net.ipv4.ip_forward=1") cls.gateway.execute("iptables -t nat -A POSTROUTING --destination %s -j MASQUERADE" % cls.ext_server_ip) cls.calinet = cls.gateway.create_network("calinet") cls.gateway_workload = cls.gateway.create_workload( "gw-wl", image="workload", network=cls.calinet, labels=["org.projectcalico.label.wep=gateway"]) cls.host_workload = cls.host.create_workload( "host-wl", image="workload", network=cls.calinet, labels=["org.projectcalico.label.wep=host"])
def get_container_ip(cls, container_name): ip = log_and_run( "docker inspect -f '{{range .NetworkSettings.Networks}}{{.IPAddress}}{{end}}' %s" % container_name) return ip.strip()
def setUpClass(cls): # Wipe etcd once before any test in this class runs. wipe_etcd() # We set up an additional docker network to act as the external # network. The Gateway container is connected to both networks. # and we configure it as a NAT gateway. # # "cali-st-ext" host # container # | # "cali-st-ext" docker # bridge # | # Gateway Host # container container # \ / # default docker # bridge # First, create the hosts and the gateway. cls.hosts = [] cls.gateway = DockerHost("cali-st-gw", additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) cls.gateway_hostname = cls.gateway.execute("hostname") cls.host = DockerHost("cali-st-host", additional_docker_options=CLUSTER_STORE_DOCKER_OPTIONS, post_docker_commands=POST_DOCKER_COMMANDS, start_calico=False) cls.host_hostname = cls.host.execute("hostname") cls.hosts.append(cls.gateway) cls.hosts.append(cls.host) # Delete the nginx container if it still exists. We need to do this # before we try to remove the network. log_and_run("docker rm -f cali-st-ext-nginx || true") # Create the external network. log_and_run("docker network rm cali-st-ext || true") # Use 172.19.0.0 to avoid clash with normal docker subnet and # docker-in-docker subnet log_and_run("docker network create --driver bridge --subnet 172.19.0.0/16 cali-st-ext") # And an nginx server on the external network only. log_and_run("docker run" " --network=cali-st-ext" " -d" " --name=cali-st-ext-nginx" " nginx") for host in cls.hosts: host.start_calico_node() # Get the internal IP of the gateway. We do this before we add the second # network since it means we don't have to figure out which IP is which. int_ip = str(cls.gateway.ip) cls.gateway_int_ip = int_ip _log.info("Gateway internal IP: %s", cls.gateway_int_ip) # Add the gateway to the external network. log_and_run("docker network connect cali-st-ext cali-st-gw") cls.gateway.execute("ip addr") # Get the IP of the external server. ext_ip = cls.get_container_ip("cali-st-ext-nginx") cls.ext_server_ip = ext_ip _log.info("External workload IP: %s", cls.ext_server_ip) # Configure the internal host to use the gateway for the external IP. cls.host.execute("ip route add %s via %s" % (cls.ext_server_ip, cls.gateway_int_ip)) # Configure the gateway to forward and NAT. cls.gateway.execute("sysctl -w net.ipv4.ip_forward=1") cls.gateway.execute("iptables -t nat -A POSTROUTING --destination %s -j MASQUERADE" % cls.ext_server_ip)