def log_extra_diags(self):
# Run a set of commands to trace ip routes, iptables and ipsets.
self.execute("ip route", raise_exception_on_failure=False)
self.execute("iptables-save -c", raise_exception_on_failure=False)
self.execute("ip6tables-save -c", raise_exception_on_failure=False)
self.execute("ipset save", raise_exception_on_failure=False)
self.execute("ps waux", raise_exception_on_failure=False)
self.execute("docker logs calico-node",
raise_exception_on_failure=False)
self.execute("docker exec calico-node ls -l /var/log/calico/felix",
raise_exception_on_failure=False)
self.execute("docker exec calico-node cat /var/log/calico/felix/*",
raise_exception_on_failure=False)
self.execute("docker exec calico-node ls -l /var/log/calico/confd",
raise_exception_on_failure=False)
self.execute("docker exec calico-node cat /var/log/calico/confd/*",
raise_exception_on_failure=False)
self.execute("docker exec calico-node ls -l /var/log/calico/bird",
raise_exception_on_failure=False)
self.execute("docker exec calico-node cat /var/log/calico/bird/*",
raise_exception_on_failure=False)
self.execute(
"docker exec calico-node cat /etc/calico/confd/config/bird.cfg",
raise_exception_on_failure=False)
self.execute("docker ps -a", raise_exception_on_failure=False)
for wl in self.workloads:
wl.host.execute("docker logs %s" % wl.name,
raise_exception_on_failure=False)
log_and_run("docker logs %s" % self.name,
raise_exception_on_failure=False)
def cleanup(self):
"""
Clean up this host, including removing any containers created. This is
necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
:return:
"""
# Check for logs before tearing down
if self.log_analyzer is not None:
self.log_analyzer.check_logs_for_exceptions()
logger.info("# Cleaning up host %s", self.name)
if self.dind:
# For Docker-in-Docker, we need to remove all containers and
# all images.
# Start by just removing the workloads and then attempt cleanup of
# networks...
self.remove_workloads()
self.cleanup_networks()
# ...delete any remaining containers and the images...
self.remove_containers()
self.remove_images()
# ...and the outer container for DinD.
log_and_run("docker rm -f %s || true" % self.name)
else:
# For non Docker-in-Docker, we can only remove the containers we
# created - so remove the workloads, attempt cleanup of networks
# and delete the calico node.
self.remove_workloads()
self.cleanup_networks()
log_and_run("docker rm -f calico-node || true")
self._cleaned = True
def __init__(self, name, start_calico=True, dind=True,
additional_docker_options="",
post_docker_commands=["docker load -i /code/calico-node.tar",
"docker load -i /code/busybox.tar"]):
self.name = name
self.dind = dind
self.workloads = set()
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
if dind:
log_and_run("docker rm -f %s || true" % self.name)
log_and_run("docker run --privileged -tid "
"-v %s/docker:/usr/local/bin/docker "
"-v %s:/code --name %s "
"calico/dind:latest docker daemon --storage-driver=aufs %s" %
(CHECKOUT_DIR, CHECKOUT_DIR, self.name, additional_docker_options))
self.ip = log_and_run("docker inspect --format "
"'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" % self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps, ex_class=CalledProcessError,
retries=10)
for command in post_docker_commands:
self.execute(command)
else:
self.ip = get_ip(v6=False)
self.ip6 = get_ip(v6=True)
if start_calico:
self.start_calico_node()
def cleanup(self):
"""
Clean up this host, including removing any containers created. This is
necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
:return:
"""
# Check for logs before tearing down
if self.log_analyzer is not None:
self.log_analyzer.check_logs_for_exceptions()
logger.info("# Cleaning up host %s", self.name)
if self.dind:
# For Docker-in-Docker, we need to remove all containers and
# all images.
# Start by just removing the workloads and then attempt cleanup of
# networks...
self.remove_workloads()
self.cleanup_networks()
# ...delete any remaining containers and the images...
self.remove_containers()
self.remove_images()
# ...and the outer container for DinD.
log_and_run("docker rm -f %s || true" % self.name)
else:
# For non Docker-in-Docker, we can only remove the containers we
# created - so remove the workloads, attempt cleanup of networks
# and delete the calico node.
self.remove_workloads()
self.cleanup_networks()
log_and_run("docker rm -f calico-node || true")
self._cleaned = True
def __init__(self, name, start_calico=True, dind=True,
additional_docker_options="",
post_docker_commands=["docker load -i /code/calico-node.tar",
"docker load -i /code/busybox.tar"],
calico_node_autodetect_ip=False):
self.name = name
self.dind = dind
self.workloads = set()
self.ip = None
"""
An IP address value to pass to calicoctl as `--ip`. If left as None, no value will be passed,
forcing calicoctl to do auto-detection.
"""
self.ip6 = None
"""
An IPv6 address value to pass to calicoctl as `--ipv6`. If left as None, no value will be passed.
"""
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
docker_args = "--privileged -tid " \
"-v /lib/modules:/lib/modules " \
"-v %s/certs:%s/certs -v %s:/code --name %s" % \
(CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR,
self.name)
if ETCD_SCHEME == "https":
docker_args += " --add-host %s:%s" % (ETCD_HOSTNAME_SSL, get_ip())
if dind:
log_and_run("docker rm -f %s || true" % self.name)
# Pass the certs directory as a volume since the etcd SSL/TLS
# environment variables use the full path on the host.
# Set iptables=false to prevent iptables error when using dind libnetwork
log_and_run("docker run %s "
"calico/dind:latest "
" --storage-driver=aufs "
"--iptables=false "
"%s" %
(docker_args, additional_docker_options))
self.ip = log_and_run("docker inspect --format "
"'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" % self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps, ex_class=CalledProcessError,
retries=10)
for command in post_docker_commands:
self.execute(command)
elif not calico_node_autodetect_ip:
# Find the IP so it can be specified as `--ip` when launching node later.
self.ip = get_ip(v6=False)
self.ip6 = get_ip(v6=True)
if start_calico:
self.start_calico_node()
def __init__(self,
name,
start_calico=True,
dind=True,
additional_docker_options="",
post_docker_commands=[
"docker load -i /code/calico-node.tar",
"docker load -i /code/busybox.tar"
]):
self.name = name
self.dind = dind
self.workloads = set()
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
docker_args = "--privileged -tid -v %s/docker:/usr/local/bin/docker " \
"-v %s/certs:%s/certs -v %s:/code --name %s" % \
(CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR,
self.name)
if ETCD_SCHEME == "https":
docker_args += " --add-host %s:%s" % (ETCD_HOSTNAME_SSL, get_ip())
if dind:
log_and_run("docker rm -f %s || true" % self.name)
# Pass the certs directory as a volume since the etcd SSL/TLS
# environment variables use the full path on the host.
log_and_run("docker run %s "
"calico/dind:latest "
"docker daemon --storage-driver=aufs %s" %
(docker_args, additional_docker_options))
self.ip = log_and_run(
"docker inspect --format "
"'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" %
self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps,
ex_class=CalledProcessError,
retries=10)
for command in post_docker_commands:
self.execute(command)
else:
self.ip = get_ip(v6=False)
self.ip6 = get_ip(v6=True)
if start_calico:
self.start_calico_node()
def execute(self,
command,
raise_exception_on_failure=True,
daemon_mode=False):
"""
Pass a command into a host container.
Raises a CommandExecError() if the command returns a non-zero
return code if raise_exception_on_failure=True.
:param command: The command to execute.
:param raise_exception_on_failure: Raises an exception if the command exits with
non-zero return code.
:param daemon_mode: The command will be executed as a daemon process. Useful
to start a background service.
:return: The output from the command with leading and trailing
whitespace removed.
"""
if self.dind:
option = "-d" if daemon_mode else "-it"
command = self.escape_shell_single_quotes(command)
command = "docker exec %s %s sh -c '%s'" % (option, self.name,
command)
return log_and_run(
command, raise_exception_on_failure=raise_exception_on_failure)
def writefile(self, filename, data):
"""
Writes a file on a host (e.g. a yaml file for loading into calicoctl).
:param filename: string, the filename to create
:param data: string, the data to put inthe file
:return: Return code of execute operation.
"""
if self.dind:
with tempfile.NamedTemporaryFile() as tmp:
tmp.write(data)
tmp.flush()
log_and_run("docker cp %s %s:%s" % (tmp.name, self.name, filename))
else:
with open(filename, 'w') as f:
f.write(data)
self.execute("cat %s" % filename)
def __init__(
self,
name,
start_calico=True,
dind=True,
additional_docker_options="",
post_docker_commands=[
"docker load --input /code/calico_containers/calico-node.tar",
"docker load --input /code/calico_containers/busybox.tar"
]):
self.name = name
self.dind = dind
self.workloads = set()
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
if dind:
log_and_run("docker rm -f %s || true" % self.name)
log_and_run(
"docker run --privileged -tid "
"-v %s/docker:/usr/local/bin/docker "
"-v %s:/code --name %s "
"calico/dind:latest docker daemon --storage-driver=aufs %s" %
(os.getcwd(), os.getcwd(), self.name,
additional_docker_options))
self.ip = log_and_run(
"docker inspect --format "
"'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" %
self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps,
ex_class=CalledProcessError,
retries=10)
for command in post_docker_commands:
self.execute(command)
else:
self.ip = get_ip()
if start_calico:
self.start_calico_node()
def __init__(self, name, start_calico=True, dind=True,
additional_docker_options="",
post_docker_commands=["docker load -i /code/calico-node.tar",
"docker load -i /code/busybox.tar"]):
self.name = name
self.dind = dind
self.workloads = set()
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
docker_args = "--privileged -tid -v %s/docker:/usr/local/bin/docker " \
"-v %s/certs:%s/certs -v %s:/code --name %s" % \
(CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR,
self.name)
if ETCD_SCHEME == "https":
docker_args += " --add-host %s:%s" % (ETCD_HOSTNAME_SSL, get_ip())
if dind:
log_and_run("docker rm -f %s || true" % self.name)
# Pass the certs directory as a volume since the etcd SSL/TLS
# environment variables use the full path on the host.
log_and_run("docker run %s "
"calico/dind:latest "
"docker daemon --storage-driver=aufs %s" %
(docker_args, additional_docker_options))
self.ip = log_and_run("docker inspect --format "
"'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" % self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps, ex_class=CalledProcessError,
retries=10)
for command in post_docker_commands:
self.execute(command)
else:
self.ip = get_ip(v6=False)
self.ip6 = get_ip(v6=True)
if start_calico:
self.start_calico_node()
def execute_readline(self, command, filename):
"""
Execute a command and return a list of the lines in the file. If
running dind, the file to read will be copied to the host before
reading since docker exec can be unreliable when piping large
amounts of data.
Raises an exception if the return code is non-zero. Stderr is ignored.
Use this rather than execute if the command outputs a large amount of
data that cannot be handled as a single string.
:return: List of individual lines.
"""
logger.debug("Running command on %s", self.name)
logger.debug(" - Command: %s", command % filename)
if self.dind:
# Copy file to host before reading.
logger.debug("Running dind, copy file to host first")
# Make a tmp directory where we can place the file.
log_and_run("mkdir -p /tmp/%s" % self.name,
raise_exception_on_failure=True)
# Generate the filename to use on the host.
# /var/log/calico/felix/current becomes /tmp/<host>/var-log-calico-felix-current.
hostfile = "/tmp/%s/%s" % (self.name, filename.strip("/").replace(
"/", "-"))
# Copy to the host.
c = "docker cp %s:%s %s" % (self.name, filename, hostfile)
log_and_run(c, raise_exception_on_failure=True)
logger.debug("Copied file from container to %s" % hostfile)
command = command % hostfile
else:
# Otherwise just run the provided command.
command = command % filename
logger.debug("Final command: %s", command.split(" "))
out = check_output(command.split(" "))
return out.split("\n")
def get_hostname(self):
"""
Get the hostname from Docker
The hostname is a randomly generated string.
Note, this function only works with a host with dind enabled.
Raises an exception if dind is not enabled.
:param host: DockerHost object
:return: hostname of DockerHost
"""
command = "docker inspect --format {{.Config.Hostname}} %s" % self.name
return log_and_run(command)
def get_hostname(self):
"""
Get the hostname from Docker
The hostname is a randomly generated string.
Note, this function only works with a host with dind enabled.
Raises an exception if dind is not enabled.
:param host: DockerHost object
:return: hostname of DockerHost
"""
command = "docker inspect --format {{.Config.Hostname}} %s" % self.name
return log_and_run(command)
def cleanup(self):
"""
Clean up this host, including removing any containers created. This is
necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
:return:
"""
logger.info("# Cleaning up host %s", self.name)
if self.dind:
# For Docker-in-Docker, we need to remove all containers and
# all images...
self.remove_containers()
self.remove_images()
# ...and the outer container for DinD.
log_and_run("docker rm -f %s || true" % self.name)
else:
# For non Docker-in-Docker, we can only remove the containers we
# created - so remove the workloads and the calico node.
self.remove_workloads()
log_and_run("docker rm -f calico-node || true")
self._cleaned = True
def cleanup(self):
"""
Clean up this host, including removing any containers created. This is
necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
:return:
"""
logger.info("# Cleaning up host %s", self.name)
if self.dind:
# For Docker-in-Docker, we need to remove all containers and
# all images...
self.remove_containers()
self.remove_images()
# ...and the outer container for DinD.
log_and_run("docker rm -f %s || true" % self.name)
else:
# For non Docker-in-Docker, we can only remove the containers we
# created - so remove the workloads and the calico node.
self.remove_workloads()
log_and_run("docker rm -f calico-node || true")
self._cleaned = True
def get_hostname(self):
"""
Get the hostname from Docker
The hostname is a randomly generated string.
Note, this function only works with a host with dind enabled.
Raises an exception if dind is not enabled.
:return: hostname of DockerHost
"""
# If overriding the hostname, return that one.
if self.override_hostname:
return self.override_hostname
command = "docker inspect --format {{.Config.Hostname}} %s" % self.name
return log_and_run(command)
def get_hostname(self):
"""
Get the hostname from Docker
The hostname is a randomly generated string.
Note, this function only works with a host with dind enabled.
Raises an exception if dind is not enabled.
:return: hostname of DockerHost
"""
# If overriding the hostname, return that one.
if self.override_hostname:
return self.override_hostname
command = "docker inspect --format {{.Config.Hostname}} %s" % self.name
return log_and_run(command)
def execute(self, command):
"""
Pass a command into a host container.
Raises a CommandExecError() if the command returns a non-zero
return code.
:param command: The command to execute.
:return: The output from the command with leading and trailing
whitespace removed.
"""
if self.dind:
command = self.escape_shell_single_quotes(command)
command = "docker exec -it %s sh -c '%s'" % (self.name, command)
return log_and_run(command)
def execute(self, command):
"""
Pass a command into a host container.
Raises a CommandExecError() if the command returns a non-zero
return code.
:param command: The command to execute.
:return: The output from the command with leading and trailing
whitespace removed.
"""
if self.dind:
command = self.escape_shell_single_quotes(command)
command = "docker exec -it %s sh -c '%s'" % (self.name, command)
return log_and_run(command)
def __init__(self,
name,
start_calico=True,
dind=True,
additional_docker_options="",
post_docker_commands=[
"docker load -i /code/calico-node.tar",
"docker load -i /code/busybox.tar"
],
calico_node_autodetect_ip=False,
simulate_gce_routing=False,
override_hostname=False,
networking=None):
self.name = name
self.dind = dind
self.workloads = set()
self.ip = None
self.log_analyzer = None
"""
An IP address value to pass to calicoctl as `--ip`. If left as None,
no value will be passed, forcing calicoctl to do auto-detection.
"""
self.ip6 = None
"""
An IPv6 address value to pass to calicoctl as `--ipv6`. If left as
None, no value will be passed.
"""
self.override_hostname = None if not override_hostname else \
uuid.uuid1().hex[:16]
"""
Create an arbitrary hostname if we want to override.
"""
if networking is None:
self.networking = global_setting()
else:
self.networking = networking
assert self.networking in [NETWORKING_CNI, NETWORKING_LIBNETWORK]
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
# must invoke cleanup.
self._cleaned = False
docker_args = "--privileged -tid " \
"-v /lib/modules:/lib/modules " \
"-v %s/certs:%s/certs -v %s:/code --name %s" % \
(CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR,
self.name)
if ETCD_SCHEME == "https":
docker_args += " --add-host %s:%s" % (ETCD_HOSTNAME_SSL, get_ip())
if dind:
log_and_run("docker rm -f %s || true" % self.name)
# Pass the certs directory as a volume since the etcd SSL/TLS
# environment variables use the full path on the host.
# Set iptables=false to prevent iptables error when using dind
# libnetwork
log_and_run("docker run %s "
"calico/dind:latest "
"--iptables=false "
"%s" % (docker_args, additional_docker_options))
self.ip = log_and_run(
"docker inspect --format "
"'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" %
self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps,
ex_class=CalledProcessError,
retries=10)
if simulate_gce_routing:
# Simulate addressing and routing setup as on a GCE instance:
# the instance has a /32 address (which means that it appears
# not to be directly connected to anything) and a default route
# that does not have the 'onlink' flag to override that.
#
# First check that we can ping the Docker bridge, and trace out
# initial state.
self.execute("ping -c 1 -W 2 172.17.0.1")
self.execute("ip a")
self.execute("ip r")
# Change the normal /16 IP address to /32.
self.execute("ip a del %s/16 dev eth0" % self.ip)
self.execute("ip a add %s/32 dev eth0" % self.ip)
# Add a default route via the Docker bridge.
self.execute("ip r a 172.17.0.1 dev eth0")
self.execute("ip r a default via 172.17.0.1 dev eth0")
# Trace out final state, and check that we can still ping the
# Docker bridge.
self.execute("ip a")
self.execute("ip r")
self.execute("ping -c 1 -W 2 172.17.0.1")
for command in post_docker_commands:
self.execute(command)
elif not calico_node_autodetect_ip:
# Find the IP so it can be specified as `--ip` when launching
# node later.
self.ip = get_ip(v6=False)
self.ip6 = get_ip(v6=True)
if start_calico:
self.start_calico_node()
class DockerHost(object):
"""
A host container which will hold workload containers to be networked by
Calico.
:param calico_node_autodetect_ip: When set to True, the test framework
will not perform IP detection, and will run `calicoctl node` without
explicitly passing in a value for --ip. This means calico-node will be
forced to do its IP detection.
:param override_hostname: When set to True, the test framework will
choose an alternate hostname for the host which it will pass to all
calicoctl components as the HOSTNAME environment variable. If set
to False, the HOSTNAME environment is not explicitly set.
:param networking: What plugin to use to set up the networking for
workloads on this host. Possible values are None (the default), meaning to
use the global setting for the test run; "cni", meaning to use the Calico
CNI plugin; and "libnetwork", meaning to use the Docker libnetwork plugin.
The global setting for the test run is taken from the environment variable
ST_NETWORKING, and is "cni" if that variable is not set.
"""
# A static list of Docker networks that are created by the tests. This
# list covers all Docker hosts.
docker_networks = []
def __init__(self,
name,
start_calico=True,
dind=True,
additional_docker_options="",
post_docker_commands=[
"docker load -i /code/calico-node.tar",
"docker load -i /code/busybox.tar"
],
calico_node_autodetect_ip=False,
simulate_gce_routing=False,
override_hostname=False,
networking=None):
self.name = name
self.dind = dind
self.workloads = set()
self.ip = None
self.log_analyzer = None
"""
An IP address value to pass to calicoctl as `--ip`. If left as None,
no value will be passed, forcing calicoctl to do auto-detection.
"""
self.ip6 = None
"""
An IPv6 address value to pass to calicoctl as `--ipv6`. If left as
None, no value will be passed.
"""
self.override_hostname = None if not override_hostname else \
uuid.uuid1().hex[:16]
"""
Create an arbitrary hostname if we want to override.
"""
if networking is None:
self.networking = global_setting()
else:
self.networking = networking
assert self.networking in [NETWORKING_CNI, NETWORKING_LIBNETWORK]
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
# must invoke cleanup.
self._cleaned = False
docker_args = "--privileged -tid " \
"-v /lib/modules:/lib/modules " \
"-v %s/certs:%s/certs -v %s:/code --name %s" % \
(CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR,
self.name)
if ETCD_SCHEME == "https":
docker_args += " --add-host %s:%s" % (ETCD_HOSTNAME_SSL, get_ip())
if dind:
log_and_run("docker rm -f %s || true" % self.name)
# Pass the certs directory as a volume since the etcd SSL/TLS
# environment variables use the full path on the host.
# Set iptables=false to prevent iptables error when using dind
# libnetwork
log_and_run("docker run %s "
"calico/dind:latest "
"--iptables=false "
"%s" % (docker_args, additional_docker_options))
self.ip = log_and_run(
"docker inspect --format "
"'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" %
self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps,
ex_class=CalledProcessError,
retries=10)
if simulate_gce_routing:
# Simulate addressing and routing setup as on a GCE instance:
# the instance has a /32 address (which means that it appears
# not to be directly connected to anything) and a default route
# that does not have the 'onlink' flag to override that.
#
# First check that we can ping the Docker bridge, and trace out
# initial state.
self.execute("ping -c 1 -W 2 172.17.0.1")
self.execute("ip a")
self.execute("ip r")
# Change the normal /16 IP address to /32.
self.execute("ip a del %s/16 dev eth0" % self.ip)
self.execute("ip a add %s/32 dev eth0" % self.ip)
# Add a default route via the Docker bridge.
self.execute("ip r a 172.17.0.1 dev eth0")
self.execute("ip r a default via 172.17.0.1 dev eth0")
# Trace out final state, and check that we can still ping the
# Docker bridge.
self.execute("ip a")
self.execute("ip r")
self.execute("ping -c 1 -W 2 172.17.0.1")
for command in post_docker_commands:
self.execute(command)
elif not calico_node_autodetect_ip:
# Find the IP so it can be specified as `--ip` when launching
# node later.
self.ip = get_ip(v6=False)
self.ip6 = get_ip(v6=True)
if start_calico:
self.start_calico_node()
def execute(self,
command,
raise_exception_on_failure=True,
daemon_mode=False):
"""
Pass a command into a host container.
Raises a CommandExecError() if the command returns a non-zero
return code if raise_exception_on_failure=True.
:param command: The command to execute.
:param raise_exception_on_failure: Raises an exception if the command exits with
non-zero return code.
:param daemon_mode: The command will be executed as a daemon process. Useful
to start a background service.
:return: The output from the command with leading and trailing
whitespace removed.
"""
if self.dind:
option = "-d" if daemon_mode else "-it"
command = self.escape_shell_single_quotes(command)
command = "docker exec %s %s sh -c '%s'" % (option, self.name,
command)
return log_and_run(
command, raise_exception_on_failure=raise_exception_on_failure)
def execute_readline(self, command):
"""
Execute a command and return individual lines as a generator.
Raises an exception if the return code is non-zero. Stderr is ignored.
Use this rather than execute if the command outputs a large amount of
data that cannot be handled as a single string.
:return: Generator of individual lines.
"""
logger.debug("Running command on %s", self.name)
logger.debug(" - Command: %s", command)
if self.dind:
command = self.escape_shell_single_quotes(command)
command = "docker exec -it %s sh -c '%s'" % (self.name, command)
logger.debug("Final command: %s", command)
proc = Popen(command, stdout=PIPE, shell=True)
try:
# Read and return one line at a time until no more data is
# returned.
for line in proc.stdout:
yield line
finally:
status = proc.wait()
logger.debug("- return: %s", status)
if status:
raise Exception("Command %s returned non-zero exit code %s" %
(command, status))
def calicoctl(self,
command,
version=None,
raise_exception_on_failure=True):
"""
Convenience function for abstracting away calling the calicoctl
command.
Raises a CommandExecError() if the command returns a non-zero
return code.
:param command: The calicoctl command line parms as a single string.
:param version: The calicoctl version to use (this is appended to the
executable name. It is assumed the Makefile will ensure
the required versions are downloaded.
:return: The output from the command with leading and trailing
whitespace removed.
"""
if not version:
calicoctl = os.environ.get("CALICOCTL", "/code/dist/calicoctl")
else:
calicoctl = "/code/dist/calicoctl-" + version
if ETCD_SCHEME == "https":
etcd_auth = "%s:2379" % ETCD_HOSTNAME_SSL
else:
etcd_auth = "%s:2379" % get_ip()
# Export the environment, in case the command has multiple parts, e.g.
# use of | or ;
#
# Pass in all etcd params, the values will be empty if not set anyway
calicoctl = "export ETCD_ENDPOINTS=%s://%s; " \
"export ETCD_CA_CERT_FILE=%s; " \
"export ETCD_CERT_FILE=%s; " \
"export ETCD_KEY_FILE=%s; %s" % \
(ETCD_SCHEME, etcd_auth, ETCD_CA, ETCD_CERT, ETCD_KEY,
calicoctl)
# If the hostname is being overriden, then export the HOSTNAME
# environment.
if self.override_hostname:
calicoctl = "export HOSTNAME=%s; %s" % (self.override_hostname,
calicoctl)
return self.execute(
calicoctl + " " + command,
raise_exception_on_failure=raise_exception_on_failure)
def start_calico_node(self,
options="",
with_ipv4pool_cidr_env_var=True,
env_options=""):
"""
Start calico in a container inside a host by calling through to the
calicoctl node command.
:param env_options: Docker environment options.
:param options: calico node options.
:param with_ipv4pool_cidr_env_var: dryrun options.
"""
args = ['node', 'run']
if with_ipv4pool_cidr_env_var:
args.append('--dryrun')
if "--node-image" not in options:
args.append('--node-image=%s' % NODE_CONTAINER_NAME)
# Add the IP addresses if required and we aren't explicitly specifying
# them in the options. The --ip and --ip6 options can be specified
# using "=" or space-separated parms.
if self.ip and "--ip=" not in options and "--ip " not in options:
args.append('--ip=%s' % self.ip)
if self.ip6 and "--ip6=" not in options and "--ip6 " not in options:
args.append('--ip6=%s' % self.ip6)
args.append(options)
cmd = ' '.join(args)
if with_ipv4pool_cidr_env_var:
# Run the dryrun command, then modify and execute the command that
# that tells us.
assert "--dryrun" in cmd
output = self.calicoctl(cmd)
# Look for the line in the output that includes "docker run",
# "--net=host" and "--name=calico-node".
for line in output.split('\n'):
if re.match(r'docker run .*--net=host .*--name=calico-node',
line):
# This is the line we want to modify.
break
else:
raise AssertionError("No node run line in %s" % output)
# Break the line at the first occurrence of " -e ".
prefix, _, suffix = line.rstrip().partition(" -e ")
felix_logsetting = ""
if FELIX_LOGLEVEL != "":
felix_logsetting = " -e FELIX_LOGSEVERITYSCREEN=" + FELIX_LOGLEVEL
# Construct the calicoctl command that we want, including the
# CALICO_IPV4POOL_CIDR setting.
modified_cmd = (
prefix +
(" -e CALICO_IPV4POOL_CIDR=%s " % DEFAULT_IPV4_POOL_CIDR) +
felix_logsetting + env_options + " -e " + suffix)
# Now run that.
self.execute(modified_cmd)
else:
# Run the non-dryrun calicoctl node run command.
self.calicoctl(cmd)
self.attach_log_analyzer()
def set_ipip_enabled(self, enabled):
pools_output = self.calicoctl("get ippool -o yaml")
pools_dict = yaml.safe_load(pools_output)
for pool in pools_dict['items']:
print "Pool is %s" % pool
if ':' not in pool['spec']['cidr']:
pool['spec']['ipipMode'] = 'Always' if enabled else 'Never'
if 'creationTimestamp' in pool['metadata']:
del pool['metadata']['creationTimestamp']
self.writefile("ippools.yaml", yaml.dump(pools_dict))
self.calicoctl("apply -f ippools.yaml")
def attach_log_analyzer(self):
self.log_analyzer = LogAnalyzer(self, "/var/log/calico/felix/current",
FELIX_LOG_FORMAT, TIMESTAMP_FORMAT)
def start_calico_node_with_docker(self):
"""
Start calico in a container inside a host by calling docker directly.
"""
if ETCD_SCHEME == "https":
etcd_auth = "%s:2379" % ETCD_HOSTNAME_SSL
ssl_args = "-e ETCD_CA_CERT_FILE=%s " \
"-e ETCD_CERT_FILE=%s " \
"-e ETCD_KEY_FILE=%s " \
"-v %s/certs:%s/certs " \
% (ETCD_CA, ETCD_CERT, ETCD_KEY,
CHECKOUT_DIR, CHECKOUT_DIR)
else:
etcd_auth = "%s:2379" % get_ip()
ssl_args = ""
# If the hostname has been overridden on this host, then pass it in
# as an environment variable.
if self.override_hostname:
hostname_args = "-e HOSTNAME=%s" % self.override_hostname
else:
hostname_args = ""
self.execute("docker run -d --net=host --privileged "
"--name=calico-node "
"%s "
"-e IP=%s "
"-e ETCD_ENDPOINTS=%s://%s %s "
"-v /var/log/calico:/var/log/calico "
"-v /var/run/calico:/var/run/calico "
"%s" % (hostname_args, self.ip, ETCD_SCHEME, etcd_auth,
ssl_args, NODE_CONTAINER_NAME))
def remove_workloads(self):
"""
Remove all containers running on this host.
Useful for test shut down to ensure the host is cleaned up.
:return: None
"""
for workload in self.workloads:
try:
if self.networking == NETWORKING_CNI:
workload.run_cni("DEL")
self.execute("docker rm -f %s" % workload.name)
except CalledProcessError:
# Make best effort attempt to clean containers. Don't fail the
# test if a container can't be removed.
pass
def remove_images(self):
"""
Remove all images running on this host.
Useful for test shut down to ensure the host is cleaned up.
:return: None
"""
cmd = "docker rmi $(docker images -qa)"
try:
self.execute(cmd)
except CalledProcessError:
# Best effort only.
pass
def remove_containers(self):
"""
Remove all containers running on this host.
Useful for test shut down to ensure the host is cleaned up.
:return: None
"""
cmd = "docker rm -f $(docker ps -qa)"
try:
self.execute(cmd)
except CalledProcessError:
# Best effort only.
pass
def __enter__(self):
return self
def __exit__(self, exc_type, exc_val, exc_tb):
"""
Exit the context of this host.
:return: None
"""
self.cleanup(log_extra_diags=bool(exc_type))
def cleanup(self, log_extra_diags=False, err_words=None, ignore_list=[]):
"""
Clean up this host, including removing any containers created. This is
necessary especially for Docker-in-Docker so we don't leave dangling
volumes.
Also, perform log analysis to check for any errors, raising an exception
if any were found.
If log_extra_is set to True we will log some extra diagnostics (this is
set to True if the DockerHost context manager exits with an exception).
Extra logs will also be output if the log analyzer detects any errors.
"""
# Check for logs before tearing down, log extra diags if we spot an error.
log_exception = None
try:
if self.log_analyzer is not None:
self.log_analyzer.check_logs_for_exceptions(
err_words, ignore_list)
except Exception, e:
log_exception = e
log_extra_diags = True
# Log extra diags if we need to.
if log_extra_diags:
self.log_extra_diags()
logger.info("# Cleaning up host %s", self.name)
if self.dind:
# For Docker-in-Docker, we need to remove all containers and
# all images.
# Start by just removing the workloads and then attempt cleanup of
# networks...
self.remove_workloads()
self.cleanup_networks()
# ...delete any remaining containers and the images...
self.remove_containers()
self.remove_images()
# ...and the outer container for DinD.
log_and_run("docker rm -f %s || true" % self.name)
else:
# For non Docker-in-Docker, we can only remove the containers we
# created - so remove the workloads, attempt cleanup of networks
# and delete the calico node.
self.remove_workloads()
self.cleanup_networks()
log_and_run("docker rm -f calico-node || true")
self._cleaned = True
# Now that tidy-up is complete, re-raise any exceptions found in the logs.
if log_exception:
raise log_exception
def __init__(self, name, start_calico=True, dind=True,
additional_docker_options="",
post_docker_commands=["docker load -i /code/calico-node.tar",
"docker load -i /code/busybox.tar"],
calico_node_autodetect_ip=False,
simulate_gce_routing=False,
override_hostname=False):
self.name = name
self.dind = dind
self.workloads = set()
self.ip = None
self.log_analyzer = None
"""
An IP address value to pass to calicoctl as `--ip`. If left as None,
no value will be passed, forcing calicoctl to do auto-detection.
"""
self.ip6 = None
"""
An IPv6 address value to pass to calicoctl as `--ipv6`. If left as
None, no value will be passed.
"""
self.override_hostname = None if not override_hostname else \
uuid.uuid1().hex[:16]
"""
Create an arbitrary hostname if we want to override.
"""
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
# must invoke cleanup.
self._cleaned = False
docker_args = "--privileged -tid " \
"-v /lib/modules:/lib/modules " \
"-v %s/certs:%s/certs -v %s:/code --name %s" % \
(CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR,
self.name)
if ETCD_SCHEME == "https":
docker_args += " --add-host %s:%s" % (ETCD_HOSTNAME_SSL, get_ip())
if dind:
log_and_run("docker rm -f %s || true" % self.name)
# Pass the certs directory as a volume since the etcd SSL/TLS
# environment variables use the full path on the host.
# Set iptables=false to prevent iptables error when using dind
# libnetwork
log_and_run("docker run %s "
"calico/dind:latest "
"--iptables=false "
"%s" %
(docker_args, additional_docker_options))
self.ip = log_and_run(
"docker inspect --format "
"'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" %
self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps, ex_class=CalledProcessError,
retries=10)
if simulate_gce_routing:
# Simulate addressing and routing setup as on a GCE instance:
# the instance has a /32 address (which means that it appears
# not to be directly connected to anything) and a default route
# that does not have the 'onlink' flag to override that.
#
# First check that we can ping the Docker bridge, and trace out
# initial state.
self.execute("ping -c 1 -W 2 172.17.0.1")
self.execute("ip a")
self.execute("ip r")
# Change the normal /16 IP address to /32.
self.execute("ip a del %s/16 dev eth0" % self.ip)
self.execute("ip a add %s/32 dev eth0" % self.ip)
# Add a default route via the Docker bridge.
self.execute("ip r a 172.17.0.1 dev eth0")
self.execute("ip r a default via 172.17.0.1 dev eth0")
# Trace out final state, and check that we can still ping the
# Docker bridge.
self.execute("ip a")
self.execute("ip r")
self.execute("ping -c 1 -W 2 172.17.0.1")
for command in post_docker_commands:
self.execute(command)
elif not calico_node_autodetect_ip:
# Find the IP so it can be specified as `--ip` when launching
# node later.
self.ip = get_ip(v6=False)
self.ip6 = get_ip(v6=True)
if start_calico:
self.start_calico_node()
def __init__(self,
name,
start_calico=True,
dind=True,
additional_docker_options="",
post_docker_commands=[
"docker load -i /code/calico-node.tar",
"docker load -i /code/busybox.tar"
],
calico_node_autodetect_ip=False,
override_hostname=False):
self.name = name
self.dind = dind
self.workloads = set()
self.ip = None
"""
An IP address value to pass to calicoctl as `--ip`. If left as None,
no value will be passed, forcing calicoctl to do auto-detection.
"""
self.ip6 = None
"""
An IPv6 address value to pass to calicoctl as `--ipv6`. If left as
None, no value will be passed.
"""
self.override_hostname = None if not override_hostname else \
uuid.uuid1().hex[:16]
"""
Create an arbitrary hostname if we want to override.
"""
# This variable is used to assert on destruction that this object was
# cleaned up. If not used as a context manager, users of this object
self._cleaned = False
docker_args = "--privileged -tid " \
"-v /lib/modules:/lib/modules " \
"-v %s/certs:%s/certs -v %s:/code --name %s" % \
(CHECKOUT_DIR, CHECKOUT_DIR, CHECKOUT_DIR,
self.name)
if ETCD_SCHEME == "https":
docker_args += " --add-host %s:%s" % (ETCD_HOSTNAME_SSL, get_ip())
if dind:
log_and_run("docker rm -f %s || true" % self.name)
# Pass the certs directory as a volume since the etcd SSL/TLS
# environment variables use the full path on the host.
# Set iptables=false to prevent iptables error when using dind
# libnetwork
log_and_run("docker run %s "
"calico/dind:latest "
"--iptables=false "
"%s" % (docker_args, additional_docker_options))
self.ip = log_and_run(
"docker inspect --format "
"'{{.NetworkSettings.Networks.bridge.IPAddress}}' %s" %
self.name)
# Make sure docker is up
docker_ps = partial(self.execute, "docker ps")
retry_until_success(docker_ps,
ex_class=CalledProcessError,
retries=10)
for command in post_docker_commands:
self.execute(command)
elif not calico_node_autodetect_ip:
# Find the IP so it can be specified as `--ip` when launching
# node later.
self.ip = get_ip(v6=False)
self.ip6 = get_ip(v6=True)
if start_calico:
self.start_calico_node()
