def get_networks(process_id=None):
docker_client = docker.from_env()
docker_api_client = APIClient(kwargs_from_env())
networks = docker_client.networks.list(
filters={'label': 'atomix-test=true'})
nets = set()
for network in networks:
if process_id is None or process_id == docker_api_client.inspect_network(
network.name)['Labels']['atomix-process']:
network_name = docker_api_client.inspect_network(
network.name)['Labels']['atomix-cluster']
nets.add(network_name)
return [Network(name) for name in nets]
class Network(object):
"""Atomix test network."""
def __init__(self, name, process_id=None):
self.name = name
self.process_id = process_id
self._docker_client = docker.from_env()
self._docker_api_client = APIClient(kwargs_from_env())
self._hosts = None
@property
def docker_network(self):
try:
return self._docker_client.networks.get(self.name)
except docker.errors.NotFound:
raise UnknownNetworkError(self.name)
@property
def subnet(self):
return str(
self._docker_api_client.inspect_network(
self.name)['IPAM']['Config'][0]['Subnet'])
@property
def gateway(self):
return str(
self._docker_api_client.inspect_network(
self.name)['IPAM']['Config'][0]['Gateway'])
@property
def hosts(self):
if self._hosts is None:
self._hosts = self._create_hosts_iterator()
return self._hosts
def _create_hosts_iterator(self):
"""Creates a host iterator from available hosts by inspecting existing containers attached to the network."""
ips = set([self.gateway] + [
str(IPv4Interface(container['IPv4Address']).ip)
for container in self._docker_api_client.inspect_network(self.name)
['Containers'].values()
])
for host in IPv4Network(unicode(self.subnet)).hosts():
host = str(host)
if host not in ips:
yield host
def setup(self, supernet='172.18.0.0/16', subnet=None, gateway=None):
"""Sets up the network."""
def find_subnet():
docker_subnets = []
for network in self._docker_client.networks.list():
network_info = self._docker_api_client.inspect_network(
network.name)
if len(network_info['IPAM']['Config']) > 0:
docker_subnets.append(
str(network_info['IPAM']['Config'][0]['Subnet']))
for subnet in IPv4Network(
unicode(supernet)).subnets(new_prefix=24):
if str(subnet) not in docker_subnets:
return str(subnet)
raise UnknownNetworkError(
"Cannot find available subnet from supernet {}".format(
supernet))
if subnet is None:
subnet = find_subnet()
self._hosts = iter(
[str(host) for host in IPv4Network(unicode(subnet)).hosts()])
if gateway is None:
gateway = str(next(self._hosts))
ipam_pool = docker.types.IPAMPool(subnet=subnet, gateway=gateway)
ipam_config = docker.types.IPAMConfig(pool_configs=[ipam_pool])
logger.info("Creating network")
labels = {
'atomix-test': 'true',
'atomix-process': self.process_id or '',
'atomix-cluster': self.name
}
self._docker_client.networks.create(self.name,
driver='bridge',
ipam=ipam_config,
labels=labels)
def teardown(self):
"""Tears down the network."""
logger.info("Removing network")
try:
self.docker_network.remove()
except:
pass
def partition(self, local, remote=None):
"""Partitions the given local from the given remote using a bi-directional partition."""
local, remote = self._get_node(local), self._get_node(remote)
return self.bipartition(local, remote)
def unipartition(self, local, remote=None):
"""Partitions the given local from the given remote."""
local, remote = self._get_node(local), self._get_node(remote)
if remote is not None:
return self._partition(local, remote)
else:
disruptions = []
for name, ip in self._interfaces():
if name != local:
disruptions.append(self._partition(local, name))
return with_context(*disruptions)
def bipartition(self, node1, node2=None):
"""Creates a bi-directional partition between the two nodes."""
node1, node2 = self._get_node(node1), self._get_node(node2)
if node2 is not None:
return with_context(self._partition(node1, node2),
self._partition(node2, node1))
else:
disruptions = []
for name, ip in self._interfaces():
if name != node1:
disruptions.append(self.bipartition(node1, name))
return with_context(*disruptions)
def heal(self, local=None, remote=None):
"""Heals partitions."""
local, remote = self._get_node(local), self._get_node(remote)
if local is not None and remote is not None:
self._heal(local, remote)
self._heal(remote, local)
elif local is not None:
for name, ip in self._interfaces():
if name != local:
self._heal(local, name)
self._heal(name, local)
else:
for name1, ip1 in self._interfaces():
for name2, ip2 in self._interfaces():
if name1 != name2:
self._heal(name1, name2)
self._heal(name2, name1)
def partition_halves(self):
"""Partitions the network into two halves."""
disruptions = []
ips = self._interfaces()
for i in range(len(ips)):
if i % 2 == 0:
for j in range(len(ips)):
if i != j and j % 2 == 1:
disruptions.append(
self.bipartition(ips[i][0], ips[j][0]))
return with_context(*disruptions)
def partition_random(self):
"""Partitions a random node."""
node1 = self._random_interface()[0]
node2 = self._random_interface()[0]
while node1 == node2:
node2 = self._random_interface()
return with_context(self.bipartition(node1, node2))
def partition_bridge(self, node=None):
"""Partitions a node as a bridge to two sides of a cluster."""
if node is None:
node = self._random_interface()[0]
else:
node = self._get_node(node)
interfaces = self._interfaces()
disruptions = []
for i in range(len(interfaces)):
if i % 2 == 0 and interfaces[i][0] != node:
for j in range(len(interfaces)):
if i != j and j % 2 == 1 and interfaces[j][0] != node:
disruptions.append(
self.bipartition(interfaces[i][0],
interfaces[j][0]))
return with_context(*disruptions)
def partition_isolate(self, node=None):
"""Isolates the given node from all its peers."""
if node is None:
node = self._random_interface()[0]
else:
node = self._get_node(node)
disruptions = []
for name, ip in self._interfaces():
if name != node:
disruptions.append(self.bipartition(node, name))
return with_context(*disruptions)
def delay(self,
node=None,
latency=50,
jitter=10,
correlation=.75,
distribution='normal'):
"""Delays packets to the given node."""
if node is None:
return with_context(*[
self._delay(name, latency, jitter, correlation, distribution)
for name, ip in self._interfaces()
])
return self._delay(self._get_node(node), latency, jitter, correlation,
distribution)
def drop(self, node=None, probability=.02, correlation=.25):
"""Drops packets to the given node."""
if node is None:
return with_context(*[
self._drop(name, probability, correlation)
for name, ip in self._interfaces()
])
return self._drop(self._get_node(node), probability, correlation)
def reorder(self, node=None, probability=.02, correlation=.5):
"""Reorders packets to the given node."""
if node is None:
return with_context(*[
self._reorder(name, probability, correlation)
for name, ip in self._interfaces()
])
return self._reorder(self._get_node(node), probability, correlation)
def duplicate(self, node=None, probability=.005, correlation=.05):
"""Duplicates packets to the given node."""
if node is None:
return with_context(*[
self._duplicate(name, probability, correlation)
for name, ip in self._interfaces()
])
return self._duplicate(self._get_node(node), probability, correlation)
def corrupt(self, node=None, probability=.02):
"""Duplicates packets to the given node."""
if node is None:
return with_context(*[
self._corrupt(name, probability)
for name, ip in self._interfaces()
])
return self._corrupt(self._get_node(node), probability)
def restore(self, node=None):
"""Restores packets to the given node to normal order."""
if node is None:
for name, ip in self._interfaces():
self._restore(name)
else:
self._restore(self._get_node(node))
def _partition(self, local, remote):
"""Partitions the given local from the given remote."""
logger.info("Cutting off link %s->%s", local, remote)
self._run_in_container(local, 'iptables', '-A', 'INPUT', '-s',
self._get_ip(remote), '-j', 'DROP', '-w')
return with_context(lambda: self.heal(local, remote))
def _heal(self, local, remote):
"""Heals a partition from the given local to the given remote."""
logger.info("Restoring link %s->%s", local, remote)
self._run_in_container(local, 'iptables', '-D', 'INPUT', '-s',
self._get_ip(remote), '-j', 'DROP', '-w')
def _delay(self,
node,
latency=50,
jitter=10,
correlation=.75,
distribution='normal'):
"""Delays packets to the given node."""
latency, jitter, correlation = self._millize(latency), self._millize(
jitter), self._percentize(correlation)
logger.info(
"Delaying packets to %s (latency=%s, jitter=%s, correlation=%s, distribution=%s)",
node, latency, jitter, correlation, distribution)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'delay', latency, jitter,
correlation, 'distribution', distribution)
return with_context(lambda: self.restore(node))
def _drop(self, node, probability=.02, correlation=.25):
"""Drops packets to the given node."""
probability, correlation = self._percentize(
probability), self._percentize(correlation)
logger.info("Dropping packets to %s (probability=%s, correlation=%s)",
node, probability, correlation)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'loss', probability,
correlation)
return with_context(lambda: self.restore(node))
def _reorder(self, node, probability=.02, correlation=.5):
"""Reorders packets to the given node."""
probability, correlation = self._percentize(
probability), self._percentize(correlation)
logger.info(
"Reordering packets to %s (probability=%s, correlation=%s)", node,
probability, correlation)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'reorder', probability,
correlation)
return with_context(lambda: self.restore(node))
def _duplicate(self, node, probability=.005, correlation=.05):
"""Duplicates packets to the given node."""
probability, correlation = self._percentize(
probability), self._percentize(correlation)
logger.info(
"Duplicating packets to %s (probability=%s, correlation=%s)", node,
probability, correlation)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'duplicate', probability,
correlation)
return with_context(lambda: self.restore(node))
def _corrupt(self, node, probability=.02):
"""Duplicates packets to the given node."""
probability = self._percentize(probability)
logger.info("Corrupting packets to %s (probability=%s)", node,
probability)
self._run_in_container(node, 'tc', 'qdisc', 'add', 'dev', 'eth0',
'root', 'netem', 'corrupt', probability)
return with_context(lambda: self.restore(node))
def _restore(self, node):
"""Restores packets to the given node to normal order."""
logger.info("Restoring packets to %s", node)
self._run_in_container(node, 'tc', 'qdisc', 'del', 'dev', 'eth0',
'root')
def _run_in_container(self, node, *command):
command = ' '.join([shlex_quote(str(arg)) for arg in command])
self._get_container(node).exec_run(command)
def _percentize(self, d, digits=2):
return '{}%'.format(round(d * 100, digits))
def _millize(self, ms):
return '{}ms'.format(ms)
def _interfaces(self):
"""Lists the IPs used in the network."""
containers = self._docker_api_client.inspect_network(
self.name)['Containers']
return sorted([(container['Name'],
str(IPv4Interface(container['IPv4Address']).ip))
for container in containers.values()],
key=lambda x: x[0])
def _random_interface(self):
"""Returns a random interface name, ip pair."""
return random.choice(self._interfaces())
def _get_container(self, name):
return self._docker_client.containers.get(name)
def _get_ip(self, name):
return dict(self._interfaces())[name]
def _get_node(self, name):
if name is None:
return None
try:
return '{}-{}'.format(self.name, int(name))
except ValueError:
return name