def get_routing_table(ip_version, namespace=None):
"""Return a list of dictionaries, each representing a route.
:param ip_version: IP version of routes to return, for example 4
:param namespace: The name of the namespace from which to get the routes
:return: a list of dictionaries, each representing a route.
The dictionary format is: {'destination': cidr,
'nexthop': ip,
'device': device_name,
'scope': scope}
"""
family = _IP_VERSION_FAMILY_MAP[ip_version]
try:
netns = pyroute2.NetNS(namespace, flags=0) if namespace else None
except OSError as e:
if e.errno == errno.ENOENT:
raise NetworkNamespaceNotFound(netns_name=namespace)
raise
with pyroute2.IPDB(nl=netns) as ipdb:
ipdb_routes = ipdb.routes
ipdb_interfaces = ipdb.interfaces
routes = [{'destination': route['dst'],
'nexthop': route.get('gateway'),
'device': ipdb_interfaces[route['oif']]['ifname'],
'scope': _get_scope_name(route['scope'])}
for route in ipdb_routes if route['family'] == family]
return routes
def get_frontend_ip(self):
with pyroute2.NetNS(constants.AMPHORA_NAMESPACE) as ns:
addr = ns.get_addr(label=constants.NETNS_PRIMARY_INTERFACE)[0]
for item in addr['attrs']:
if 'IFA_ADDRESS' in item:
return item[1]
raise exceptions.Conflict(description="Didn't get frontent ip.")
def enact(self, endpoint_map, ip):
print('REMOVE_TUNNEL', self.service, self.endpoint)
# Open network namespace inside the endpoint if we have it.
# If the pod is not local, we do not have it - but another tunnel
# router will.
netns = None
try:
netns = (pyroute2.NetNS(self.endpoint.networkNs)
if self.endpoint.networkNs else None)
except FileNotFoundError:
# If the namespace has gone away the interface is also gone
pass
for iface in endpoint_map[self.service][self.endpoint]:
if iface.internal and netns:
print('REMOVE_POD_IFACE', self.service, self.endpoint, iface)
netns.link('delete', index=iface.ifx)
else:
print('REMOVE_HOST_IFACE', self.service, self.endpoint, iface)
ip.link('delete', index=iface.ifx)
if netns:
netns.close()
del endpoint_map[self.service][self.endpoint]
def netns_with_veth():
"""High level API for netns based debugging"""
assert os.getuid() == 0
# Configure namespace and interfaces
ip = pyroute2.IPDB()
ip.create(kind="veth", ifname="pycoz0", peer="pycoz1")
ip.commit()
ns = pyroute2.IPDB(nl=pyroute2.NetNS("pycoz"))
with ip.interfaces.pycoz0 as veth:
veth.net_ns_fd = "pycoz"
with ns.interfaces.pycoz0 as veth:
veth.add_ip("192.168.0.1/24")
veth.up()
with ip.interfaces.pycoz1 as veth:
veth.add_ip("192.168.0.2/24")
veth.up()
ip.release()
ns.release()
# Switch namespace
pyroute2.netns.setns("pycoz")
def get_ipdb(netns=None):
if netns:
netns = utils.convert_netns(netns)
ipdb = pyroute2.IPDB(nl=pyroute2.NetNS(netns))
else:
ipdb = pyroute2.IPDB()
return ipdb
def get_interface_name(ip_address, net_ns=None):
"""Gets the interface name from an IP address.
:param ip_address: The IP address to lookup.
:param net_ns: The network namespace to find the interface in.
:returns: The interface name.
:raises exceptions.InvalidIPAddress: Invalid IP address provided.
:raises octavia.common.exceptions.NotFound: No interface was found.
"""
# We need to normalize the address as IPv6 has multiple representations
# fe80:0000:0000:0000:f816:3eff:fef2:2058 == fe80::f816:3eff:fef2:2058
try:
normalized_addr = ipaddress.ip_address(ip_address).compressed
except ValueError as e:
raise exceptions.InvalidIPAddress(ip_addr=ip_address) from e
if net_ns:
with pyroute2.NetNS(net_ns) as rtnl_api:
interface = _find_interface(ip_address, rtnl_api, normalized_addr)
else:
with pyroute2.IPRoute() as rtnl_api:
interface = _find_interface(ip_address, rtnl_api, normalized_addr)
if interface is not None:
return interface
raise exceptions.NotFound(resource='IP address', id=ip_address)
def __init__(self):
self.host_ns = pyroute2.NetNS('host')
self.ipdb = pyroute2.IPDB(nl=self.host_ns)
self.dockerc = docker.from_env()
self.challenges = {}
self.challenges_lock = threading.RLock()
def create_virtual_servers_from_list(
to_create: List[Tuple[
ContextLocal, ContextCommunication,
str, ContextLocal, ContextCommunication
]], client_netns_name: Optional[str]
) -> Tuple[List[subprocess.Popen], List[pyroute2.NetNS]]:
"""
Create virtual servers from contexts specified in a list.
Returns a list containing processes of created servers
and a list containing their network namespaces.
"""
created_servers = []
netns = {}
netns[None] = pyroute2.IPRoute()
for (cl, cc, sn, sl, sc) in to_create:
if sn not in netns:
netns[sn] = pyroute2.NetNS(sn)
clean_traffic("default", netns[sn])
new_server = create_virtual_server(netns[client_netns_name], cl, cc,
netns[sn], sl, sc)
created_servers.append(new_server)
netns.pop(None)
return (created_servers, list(netns.values()))
def plug_network(self, mac_address, fixed_ips, mtu=None):
# Check if the interface is already in the network namespace
# Do not attempt to re-plug the network if it is already in the
# network namespace
if self._netns_interface_exists(mac_address):
return webob.Response(
json=dict(message="Interface already exists"), status=409)
# This is the interface as it was initially plugged into the
# default network namespace, this will likely always be eth1
try:
self._check_ip_addresses(fixed_ips=fixed_ips)
except socket.error:
return webob.Response(json=dict(message="Invalid network port"),
status=400)
default_netns_interface = self._interface_by_mac(mac_address)
# We need to determine the interface name when inside the namespace
# to avoid name conflicts
with pyroute2.NetNS(consts.AMPHORA_NAMESPACE,
flags=os.O_CREAT) as netns:
# 1 means just loopback, but we should already have a VIP. This
# works for the add/delete/add case as we don't delete interfaces
# Note, eth0 is skipped because that is the VIP interface
netns_interface = 'eth{0}'.format(len(netns.get_links()))
LOG.info('Plugged interface %s will become %s in the namespace %s',
default_netns_interface, netns_interface,
consts.AMPHORA_NAMESPACE)
interface_file_path = self._osutils.get_network_interface_file(
netns_interface)
self._osutils.write_port_interface_file(
netns_interface=netns_interface,
fixed_ips=fixed_ips,
mtu=mtu,
interface_file_path=interface_file_path)
# Update the list of interfaces to add to the namespace
self._update_plugged_interfaces_file(netns_interface, mac_address)
with pyroute2.IPRoute() as ipr:
# Move the interfaces into the namespace
idx = ipr.link_lookup(ifname=default_netns_interface)[0]
ipr.link('set',
index=idx,
net_ns_fd=consts.AMPHORA_NAMESPACE,
IFLA_IFNAME=netns_interface)
self._osutils._bring_if_down(netns_interface)
self._osutils._bring_if_up(netns_interface, 'network')
return webob.Response(json=dict(
message="OK",
details="Plugged on interface {interface}".format(
interface=netns_interface)),
status=202)
def _netns_interface_exists(self, mac_address):
with pyroute2.NetNS(consts.AMPHORA_NAMESPACE,
flags=os.O_CREAT) as netns:
for link in netns.get_links():
for attr in link['attrs']:
if attr[0] == 'IFLA_ADDRESS' and attr[1] == mac_address:
return True
return False
def _get_iproute(namespace):
# From iproute.py:
# `IPRoute` -- RTNL API to the current network namespace
# `NetNS` -- RTNL API to another network namespace
if namespace:
# do not try and create the namespace
return pyroute2.NetNS(namespace, flags=0)
else:
return pyroute2.IPRoute()
def get_ipdb(netns=None):
try:
return _IPDB[netns]
except KeyError:
if netns:
ipdb = pyroute2.IPDB(nl=pyroute2.NetNS(netns))
else:
ipdb = pyroute2.IPDB()
_IPDB[netns] = ipdb
return ipdb
def enact(self, endpoint_map, ip):
print('NEW_TUNNEL', self.service, self.endpoint)
ifs = []
# Open network namespace inside the endpoint if we have it.
# If the pod is not local, we do not have it - but another tunnel
# router will.
netns = (pyroute2.NetNS(self.endpoint.networkNs)
if self.endpoint.networkNs else None)
if MODE == 'gre':
ifname = TUNNEL_PREFIX + str(
binascii.hexlify(socket.inet_aton(self.endpoint.ip)), 'utf-8')
try:
ip.link('add',
ifname=ifname,
kind='gre',
gre_remote=self.endpoint.ip)
except pyroute2.netlink.exceptions.NetlinkError as e:
if e.code != errno.EEXIST:
raise
ifx = ip.link_lookup(ifname=ifname)[0]
ifs.append(Interface(ifx, internal=False))
ip.link('set', state='up', index=ifx)
if netns:
print('NEW_POD_TUNNEL', self.service, self.endpoint)
ifname = self.service.name
try:
netns.link('add',
ifname=ifname,
kind='gre',
gre_local=self.endpoint.ip)
except pyroute2.netlink.exceptions.NetlinkError as e:
if e.code != errno.EEXIST:
raise
ifx = netns.link_lookup(ifname=ifname)[0]
ifs.append(Interface(ifx, internal=True))
try:
netns.addr('add',
address=self.service.tunnel_ip,
prefixlen=32,
index=ifx)
except pyroute2.netlink.exceptions.NetlinkError as e:
if e.code != errno.EEXIST:
raise
netns.link('set', state='up', index=ifx)
if netns:
netns.close()
endpoint_map[self.service][self.endpoint] = ifs
def _get_networks():
networks = dict()
with pyroute2.NetNS(consts.AMPHORA_NAMESPACE) as netns:
for interface in netns.get_links():
interface_name = None
for item in interface['attrs']:
if item[0] == 'IFLA_IFNAME' and not item[1].startswith('eth'):
break
elif item[0] == 'IFLA_IFNAME':
interface_name = item[1]
if item[0] == 'IFLA_STATS64':
networks[interface_name] = dict(
network_tx=item[1]['tx_bytes'],
network_rx=item[1]['rx_bytes'])
return networks
def plug_network(mac_address):
# This is the interface as it was initially plugged into the
# default network namespace, this will likely always be eth1
default_netns_interface = _interface_by_mac(mac_address)
# We need to determine the interface name when inside the namespace
# to avoid name conflicts
with pyroute2.NetNS(consts.AMPHORA_NAMESPACE, flags=os.O_CREAT) as netns:
# 1 means just loopback, but we should already have a VIP
# This works for the add/delete/add case as we don't delete interfaces
# Note, eth0 is skipped because that is the VIP interface
netns_interface = 'eth{0}'.format(len(netns.get_links()))
LOG.info(_LI('Plugged interface {0} will become {1} in the '
'namespace {2}').format(default_netns_interface,
netns_interface,
consts.AMPHORA_NAMESPACE))
interface_file_path = util.get_network_interface_file(netns_interface)
# write interface file
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
# mode 00644
mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
with os.fdopen(os.open(interface_file_path, flags, mode),
'w') as text_file:
text = template_port.render(interface=netns_interface)
text_file.write(text)
# Update the list of interfaces to add to the namespace
_update_plugged_interfaces_file(netns_interface, mac_address)
with pyroute2.IPRoute() as ipr:
# Move the interfaces into the namespace
idx = ipr.link_lookup(ifname=default_netns_interface)[0]
ipr.link('set', index=idx,
net_ns_fd=consts.AMPHORA_NAMESPACE,
IFLA_IFNAME=netns_interface)
_bring_if_down(netns_interface)
_bring_if_up(netns_interface, 'network')
return flask.make_response(flask.jsonify(dict(
message="OK",
details="Plugged on interface {interface}".format(
interface=netns_interface))), 202)
def endpoint_container_iface(n, e):
for cid, info in client.networks.get(n).attrs['Containers'].items():
if info['EndpointID'] == e:
container = client.containers.get(cid)
netns = f'/proc/{container.attrs["State"]["Pid"]}/ns/net'
with pyroute2.NetNS(netns) as rtnl:
for link in rtnl.get_links():
attrs = dict(link['attrs'])
if attrs['IFLA_ADDRESS'] == info['MacAddress']:
return {
'netns': netns,
'ifname': attrs['IFLA_IFNAME'],
'address': attrs['IFLA_ADDRESS']
}
break
return None
def get_routing_table(ip_version, namespace=None):
"""Return a list of dictionaries, each representing a route.
:param ip_version: IP version of routes to return, for example 4
:param namespace: The name of the namespace from which to get the routes
:return: a list of dictionaries, each representing a route.
The dictionary format is: {'destination': cidr,
'nexthop': ip,
'device': device_name,
'scope': scope}
"""
family = _IP_VERSION_FAMILY_MAP[ip_version]
try:
netns = pyroute2.NetNS(namespace, flags=0) if namespace else None
except OSError as e:
if e.errno == errno.ENOENT:
raise NetworkNamespaceNotFound(netns_name=namespace)
raise
routes = []
with pyroute2.IPDB(nl=netns) as ipdb:
ipdb_routes = ipdb.routes
ipdb_interfaces = ipdb.interfaces
for route in ipdb_routes:
if route['family'] != family:
continue
dst = route['dst']
nexthop = route.get('gateway')
oif = route.get('oif')
scope = _get_scope_name(route['scope'])
# If there is not a valid outgoing interface id, check if
# this is a multipath route (i.e. same destination with
# multiple outgoing interfaces)
if oif:
device = ipdb_interfaces[oif]['ifname']
rt = _make_route_dict(dst, nexthop, device, scope)
routes.append(rt)
elif route.get('multipath'):
for mpr in route['multipath']:
oif = mpr['oif']
device = ipdb_interfaces[oif]['ifname']
rt = _make_route_dict(dst, nexthop, device, scope)
routes.append(rt)
return routes
def get_veth_name_for_container(container_info):
"""Returns the name of the veth interface associated with the container
:param container_info: the container info dictionary returned by Docker API
:returns: the veth name as string
"""
logger.info(container_info)
if not os.path.exists(NETNS_PREFIX):
os.mkdir(NETNS_PREFIX)
pid = container_info['State']['Pid']
proc_dir = PROC_TEMPLATE.format(pid)
netns_symlink_path = NETNS_PREFIX + str(pid)
veth_name = ''
try:
if not os.path.exists(netns_symlink_path):
os.symlink(proc_dir, netns_symlink_path)
logger.debug('Created a symlink {0}'.format(netns_symlink_path))
container_netns = pyroute2.IPDB(nl=pyroute2.NetNS(str(pid)))
main_netns = pyroute2.IPDB()
try:
logger.debug(container_netns.interfaces)
# logger.debug(main_netns.interfaces)
with container_netns.by_name['eth0'] as eth0:
eth0_index = eth0['index']
veth_index = eth0_index + 1
with main_netns.by_index[veth_index] as veth:
veth_name = veth['ifname']
finally:
container_netns.release()
main_netns.release()
finally:
if os.path.exists(netns_symlink_path):
os.remove(netns_symlink_path)
logger.debug('Deleted the symlink {0}'.format(netns_symlink_path))
return veth_name
def configure(self):
name = self.context.make_network_namespace_name()
nsip = pyroute2.IPDB(nl=pyroute2.NetNS(name))
self.context.shared_registry.add(type(self), nsip)
def get_interface(ip_addr):
try:
if six.PY2:
ip_version = ipaddress.ip_address(unicode(ip_addr)).version
else:
ip_version = ipaddress.ip_address(ip_addr).version
except Exception:
return flask.make_response(
flask.jsonify(dict(message="Invalid IP address")), 400)
if ip_version == 4:
address_format = netifaces.AF_INET
elif ip_version == 6:
address_format = netifaces.AF_INET6
else:
return flask.make_response(
flask.jsonify(dict(message="Bad IP address version")), 400)
# We need to normalize the address as IPv6 has multiple representations
# fe80:0000:0000:0000:f816:3eff:fef2:2058 == fe80::f816:3eff:fef2:2058
normalized_addr = socket.inet_ntop(
address_format, socket.inet_pton(address_format, ip_addr))
with pyroute2.NetNS(consts.AMPHORA_NAMESPACE) as netns:
for addr in netns.get_addr():
# Save the interface index as IPv6 records don't list a
# textual interface
interface_idx = addr['index']
# Save the address family (IPv4/IPv6) for use normalizing
# the IP address for comparison
interface_af = addr['family']
# Search through the attributes of each address record
for attr in addr['attrs']:
# Look for the attribute name/value pair for the address
if attr[0] == 'IFA_ADDRESS':
# Compare the normalized address with the address we
# we are looking for. Since we have matched the name
# above, attr[1] is the address value
if normalized_addr == socket.inet_ntop(
interface_af,
socket.inet_pton(interface_af, attr[1])):
# Lookup the matching interface name by
# getting the interface with the index we found
# in the above address search
lookup_int = netns.get_links(interface_idx)
# Search through the attributes of the matching
# interface record
for int_attr in lookup_int[0]['attrs']:
# Look for the attribute name/value pair
# that includes the interface name
if int_attr[0] == 'IFLA_IFNAME':
# Return the response with the matching
# interface name that is in int_attr[1]
# for the matching interface attribute
# name
return flask.make_response(
flask.jsonify(
dict(message='OK',
interface=int_attr[1])), 200)
return flask.make_response(
flask.jsonify(
dict(message="Error interface not found "
"for IP address")), 404)
def get_ipdb(netns=None):
if netns:
ipdb = pyroute2.IPDB(nl=pyroute2.NetNS(netns))
else:
ipdb = pyroute2.IPDB()
return ipdb
def plug_vip(self, vip, subnet_cidr, gateway,
mac_address, mtu=None, vrrp_ip=None, host_routes=None):
# Validate vip and subnet_cidr, calculate broadcast address and netmask
try:
render_host_routes = []
ip = ipaddress.ip_address(
vip if isinstance(vip, six.text_type) else six.u(vip))
network = ipaddress.ip_network(
subnet_cidr if isinstance(subnet_cidr, six.text_type)
else six.u(subnet_cidr))
vip = ip.exploded
broadcast = network.broadcast_address.exploded
netmask = (network.prefixlen if ip.version == 6
else network.netmask.exploded)
vrrp_version = None
if vrrp_ip:
vrrp_ip_obj = ipaddress.ip_address(
vrrp_ip if isinstance(vrrp_ip, six.text_type)
else six.u(vrrp_ip)
)
vrrp_version = vrrp_ip_obj.version
if host_routes:
for hr in host_routes:
network = ipaddress.ip_network(
hr['destination'] if isinstance(
hr['destination'], six.text_type) else
six.u(hr['destination']))
render_host_routes.append({'network': network,
'gw': hr['nexthop']})
except ValueError:
return webob.Response(json=dict(message="Invalid VIP"),
status=400)
# Check if the interface is already in the network namespace
# Do not attempt to re-plug the VIP if it is already in the
# network namespace
if self._netns_interface_exists(mac_address):
return webob.Response(
json=dict(message="Interface already exists"), status=409)
# Check that the interface has been fully plugged
self._interface_by_mac(mac_address)
# Always put the VIP interface as eth1
primary_interface = consts.NETNS_PRIMARY_INTERFACE
secondary_interface = "{interface}:0".format(
interface=primary_interface)
interface_file_path = self._osutils.get_network_interface_file(
primary_interface)
self._osutils.create_netns_dir()
self._osutils.write_interfaces_file()
self._osutils.write_vip_interface_file(
interface_file_path=interface_file_path,
primary_interface=primary_interface,
vip=vip,
ip=ip,
broadcast=broadcast,
netmask=netmask,
gateway=gateway,
mtu=mtu,
vrrp_ip=vrrp_ip,
vrrp_version=vrrp_version,
render_host_routes=render_host_routes)
# Update the list of interfaces to add to the namespace
# This is used in the amphora reboot case to re-establish the namespace
self._update_plugged_interfaces_file(primary_interface, mac_address)
# Create the namespace
netns = pyroute2.NetNS(consts.AMPHORA_NAMESPACE, flags=os.O_CREAT)
netns.close()
# Load sysctl in new namespace
sysctl = pyroute2.NSPopen(consts.AMPHORA_NAMESPACE,
[consts.SYSCTL_CMD, '--system'],
stdout=subprocess.PIPE)
sysctl.communicate()
sysctl.wait()
sysctl.release()
cmd_list = [['modprobe', 'ip_vs'],
[consts.SYSCTL_CMD, '-w', 'net.ipv4.vs.conntrack=1']]
if ip.version == 4:
# For lvs function, enable ip_vs kernel module, enable ip_forward
# conntrack in amphora network namespace.
cmd_list.append([consts.SYSCTL_CMD, '-w', 'net.ipv4.ip_forward=1'])
elif ip.version == 6:
cmd_list.append([consts.SYSCTL_CMD, '-w',
'net.ipv6.conf.all.forwarding=1'])
for cmd in cmd_list:
ns_exec = pyroute2.NSPopen(consts.AMPHORA_NAMESPACE, cmd,
stdout=subprocess.PIPE)
ns_exec.wait()
ns_exec.release()
with pyroute2.IPRoute() as ipr:
# Move the interfaces into the namespace
idx = ipr.link_lookup(address=mac_address)[0]
ipr.link('set', index=idx, net_ns_fd=consts.AMPHORA_NAMESPACE,
IFLA_IFNAME=primary_interface)
# In an ha amphora, keepalived should bring the VIP interface up
if (CONF.controller_worker.loadbalancer_topology ==
consts.TOPOLOGY_ACTIVE_STANDBY):
secondary_interface = None
# bring interfaces up
self._osutils.bring_interfaces_up(
ip, primary_interface, secondary_interface)
return webob.Response(json=dict(
message="OK",
details="VIP {vip} plugged on interface {interface}".format(
vip=vip, interface=primary_interface)), status=202)
def start(self):
for name, net in self.conf['networks'].items():
net = ipaddress.ip_network(net)
for existing in self.networks.values():
if net.overlaps(existing['net']):
raise Exception(
'network {} overlaps with existing network {}'.format(
net, existing['net']))
print('creating network {} ({})'.format(name, net))
self.networks[name] = {
'net':
net,
'used_ips':
set(),
'bridge':
self.main_db.create(kind='bridge',
ifname='{}-sw'.format(name)).up().commit(),
}
for name, node in self.nodes.items():
if name in pyroute2.netns.listnetns():
raise Exception(
'[{}] network namespace already exists'.format(name))
print('creating node "{}"'.format(name))
node['ns'] = pyroute2.NetNS(name)
node['db'] = pyroute2.IPDB(nl=node['ns'])
with node['db'].interfaces['lo'] as lo:
lo.up()
for net_name, net in node['networks'].items():
dest = self.networks[net_name]
if 'ip' in net:
net['ip'] = ipaddress.IPv4Interface(
(net['ip'], dest['net'].prefixlen))
if net['ip'].ip not in dest['net']:
raise Exception(
'[{}] ip address {} is not in network {} ({})'.
format(name, net['ip'].ip, net_name, dest['net']))
if net['ip'].ip in dest['used_ips']:
raise Exception(
'[{}] ip address {} is already in use in network {}'
.format(name, net['ip'].ip, net_name))
dest['used_ips'].add(net['ip'].ip)
else:
net['ip'] = self.next_ip(net_name)
print('[{}] adding to network {} (ip: {})'.format(
name, net_name, net['ip']))
host_connector = (self.main_db.create(
ifname='{}-{}'.format(name, net_name),
kind='veth',
peer=net_name).up().commit())
(dest['bridge'].add_port(host_connector).commit())
with self.main_db.interfaces[net_name] as nc:
nc.net_ns_fd = name
time.sleep(0.2)
with node['db'].interfaces[net_name] as nc:
nc.up()
nc.add_ip(str(net['ip']))
if 'routes' in node:
for route, via in node['routes'].items():
print('[{}] adding route {} via {}'.format(
name, route, via))
(node['db'].routes.add(dst=route, gateway=via).commit())
def open_namespace(namespace):
"""Open namespace to test if the namespace is ready to be manipulated"""
with pyroute2.NetNS(namespace, flags=0):
pass
def open(self):
self.netns = pyroute2.NetNS(self.namespace_name)
self.__setup_veth_pair()
self.__setup_ip_namespace_route()
def Run(image_id, command, **kwargs):
try:
images = Images()
if image_id not in images:
raise Exception("No image with id {0}".format(image_id))
while True:
uuid = ''.join(random.choices('0123456789', k=6))
containers = Ps()
if uuid not in containers:
break
uuid_path = os.path.join('/var/myOwnDocker/ps', uuid)
if not os.path.exists(uuid_path):
os.makedirs(uuid_path)
print("run :: Creating container with id {0}...".format(uuid))
ip = str(random.randint(100, 255))
mac = str(int(uuid[-2:]))
with pyroute2.IPDB() as ipdb:
veth0_name = 'veth0_{0}'.format(uuid)
veth1_name = 'veth1_{0}'.format(uuid)
netns_name = 'netns_{0}'.format(uuid)
bridge_interface_name = 'bridge0'
with ipdb.create(kind='veth', ifname=veth0_name,
peer=veth1_name) as interface:
interface.up()
if bridge_interface_name not in ipdb.interfaces.keys():
ipdb.create(kind='bridge',
ifname=bridge_interface_name).commit()
interface.set_target('master', bridge_interface_name)
pyroute2.netns.create(netns_name)
with ipdb.interfaces[veth1_name] as veth1:
veth1.net_ns_fd = netns_name
ns = pyroute2.IPDB(nl=pyroute2.NetNS(netns_name))
with ns.interfaces.lo as lo:
lo.up()
with ns.interfaces[veth1_name] as veth1:
veth1.address = "02:42:ac:11:00:{0}".format(mac)
veth1.add_ip('10.0.0.{0}/24'.format(ip))
veth1.up()
ns.routes.add({'dst': 'default', 'gateway': '10.0.0.1'}).commit()
print("run :: Creating snapshot...")
subprocess.run([
'btrfs', 'subvolume', 'snapshot',
'/var/myOwnDocker/images/{0}'.format(image_id),
'/var/myOwnDocker/ps/{0}'.format(uuid)
],
check=True)
with open(
'/var/myOwnDocker/ps/{0}/{1}/etc/resolv.conf'.format(
uuid, image_id), 'w') as f:
f.write('nameserver 8.8.8.8\n')
with open(
'/var/myOwnDocker/ps/{0}/{1}/{2}.cmd'.format(
uuid, image_id, uuid), 'w') as f:
f.write(command + '\n')
try:
print("run :: Creating Cgroup...")
user = os.getlogin()
cgroups.user.create_user_cgroups(user)
cg = cgroups.Cgroup(uuid)
cg.set_cpu_limit(50)
cg.set_memory_limit(512)
new_root_path = '/var/myOwnDocker/ps/{0}/{1}'.format(
uuid, image_id)
def put_in_cgroup():
try:
print(
"run :: put_in_cgroup :: Putting process in Cgroup..."
)
pid = os.getpid()
print("run :: put_in_cgroup :: pid={}".format(pid))
cg = cgroups.Cgroup(uuid)
cg.add(pid)
print("run :: put_in_cgroup :: Added pid in Cgroup.")
pyroute2.netns.setns(netns_name)
print("run :: put_in_cgroup :: Isolated NetNS.")
os.chdir(new_root_path)
os.chroot(new_root_path)
print(
"run :: put_in_cgroup :: Successfully put process in Cgroup."
)
except Exception as e:
print(
"run :: put_in_cgroup :: Failed to put process in Cgroup."
)
traceback.print_exc()
print(
"run :: Running container with id {0} and command {1}...".
format(uuid, command))
process = subprocess.run([
'unshare -fmuip --mount-proc && /bin/mount -t proc proc /proc && sleep 2 && '
+ command
],
preexec_fn=put_in_cgroup,
universal_newlines=True,
check=True,
executable='/bin/bash')
out, err = process.stdout, process.stderr
if out is None:
out = ""
if err is None:
err = ""
with open(
os.path.join('/var/myOwnDocker/ps/{0}'.format(uuid),
'out.log'), 'a') as f:
f.write(out + '\n')
with open(
os.path.join('/var/myOwnDocker/ps/{0}'.format(uuid),
'err.log'), 'a') as f:
f.write(err + '\n')
print(
"run :: Container with id {0} and command {1} ran successfully."
.format(uuid, command))
except:
print(
"run :: Failed to run container with id {0} and command {1}."
.format(uuid, command))
traceback.print_exc()
finally:
pyroute2.NetNS(netns_name).close()
pyroute2.netns.remove(netns_name)
ipdb.interfaces[veth0_name].remove()
except Exception as e:
print("run :: Failed to create container.")
traceback.print_exc()
def plug_vip(self,
vip,
subnet_cidr,
gateway,
mac_address,
mtu=None,
vrrp_ip=None,
host_routes=None):
# Validate vip and subnet_cidr, calculate broadcast address and netmask
try:
render_host_routes = []
ip = ipaddress.ip_address(
vip if isinstance(vip, six.text_type) else six.u(vip))
network = ipaddress.ip_network(subnet_cidr if isinstance(
subnet_cidr, six.text_type) else six.u(subnet_cidr))
vip = ip.exploded
broadcast = network.broadcast_address.exploded
netmask = (network.prefixlen
if ip.version is 6 else network.netmask.exploded)
vrrp_version = None
if vrrp_ip:
vrrp_ip_obj = ipaddress.ip_address(vrrp_ip if isinstance(
vrrp_ip, six.text_type) else six.u(vrrp_ip))
vrrp_version = vrrp_ip_obj.version
if host_routes:
for hr in host_routes:
network = ipaddress.ip_network(
hr['destination'] if isinstance(
hr['destination'], six.text_type
) else six.u(hr['destination']))
render_host_routes.append({
'network': network,
'gw': hr['nexthop']
})
except ValueError:
return webob.Response(json=dict(message="Invalid VIP"), status=400)
# Check if the interface is already in the network namespace
# Do not attempt to re-plug the VIP if it is already in the
# network namespace
if self._netns_interface_exists(mac_address):
return webob.Response(
json=dict(message="Interface already exists"), status=409)
# This is the interface prior to moving into the netns
default_netns_interface = self._interface_by_mac(mac_address)
# Always put the VIP interface as eth1
primary_interface = consts.NETNS_PRIMARY_INTERFACE
secondary_interface = "{interface}:0".format(
interface=primary_interface)
interface_file_path = self._osutils.get_network_interface_file(
primary_interface)
self._osutils.create_netns_dir()
self._osutils.write_interfaces_file()
self._osutils.write_vip_interface_file(
interface_file_path=interface_file_path,
primary_interface=primary_interface,
vip=vip,
ip=ip,
broadcast=broadcast,
netmask=netmask,
gateway=gateway,
mtu=mtu,
vrrp_ip=vrrp_ip,
vrrp_version=vrrp_version,
render_host_routes=render_host_routes)
# Update the list of interfaces to add to the namespace
# This is used in the amphora reboot case to re-establish the namespace
self._update_plugged_interfaces_file(primary_interface, mac_address)
# Create the namespace
netns = pyroute2.NetNS(consts.AMPHORA_NAMESPACE, flags=os.O_CREAT)
netns.close()
# Load sysctl in new namespace
sysctl = pyroute2.NSPopen(consts.AMPHORA_NAMESPACE,
[consts.SYSCTL_CMD, '--system'],
stdout=subprocess.PIPE)
sysctl.communicate()
sysctl.wait()
sysctl.release()
with pyroute2.IPRoute() as ipr:
# Move the interfaces into the namespace
idx = ipr.link_lookup(ifname=default_netns_interface)[0]
ipr.link('set',
index=idx,
net_ns_fd=consts.AMPHORA_NAMESPACE,
IFLA_IFNAME=primary_interface)
# bring interfaces up
self._osutils.bring_interfaces_up(ip, primary_interface,
secondary_interface)
return webob.Response(json=dict(
message="OK",
details="VIP {vip} plugged on interface {interface}".format(
vip=vip, interface=primary_interface)),
status=202)
def plug_vip(vip, subnet_cidr, gateway, mac_address):
# validate vip
try:
socket.inet_aton(vip)
except socket.error:
return flask.make_response(flask.jsonify(dict(
message="Invalid VIP")), 400)
interface = _interface_by_mac(mac_address)
primary_interface = "{interface}".format(interface=interface)
secondary_interface = "{interface}:0".format(interface=interface)
# assume for now only a fixed subnet size
sections = vip.split('.')[:3]
sections.append('255')
broadcast = '.'.join(sections)
# We need to setup the netns network directory so that the ifup
# commands used here and in the startup scripts "sees" the right
# interfaces and scripts.
interface_file_path = util.get_network_interface_file(interface)
os.makedirs('/etc/netns/' + consts.AMPHORA_NAMESPACE)
shutil.copytree('/etc/network',
'/etc/netns/{}/network'.format(consts.AMPHORA_NAMESPACE),
symlinks=True,
ignore=shutil.ignore_patterns('eth0*', 'openssh*'))
name = '/etc/netns/{}/network/interfaces'.format(consts.AMPHORA_NAMESPACE)
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
# mode 00644
mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
with os.fdopen(os.open(name, flags, mode), 'w') as file:
file.write('auto lo\n')
file.write('iface lo inet loopback\n')
file.write('source /etc/netns/{}/network/interfaces.d/*.cfg\n'.format(
consts.AMPHORA_NAMESPACE))
# write interface file
mode = stat.S_IRUSR | stat.S_IWUSR | stat.S_IRGRP | stat.S_IROTH
flags = os.O_WRONLY | os.O_CREAT | os.O_TRUNC
with os.fdopen(os.open(interface_file_path, flags, mode),
'w') as text_file:
text = template_vip.render(
interface=interface,
vip=vip,
broadcast=broadcast,
# assume for now only a fixed subnet size
netmask='255.255.255.0')
text_file.write(text)
# Update the list of interfaces to add to the namespace
# This is used in the amphora reboot case to re-establish the namespace
_update_plugged_interfaces_file(interface, mac_address)
# Create the namespace
netns = pyroute2.NetNS(consts.AMPHORA_NAMESPACE, flags=os.O_CREAT)
netns.close()
with pyroute2.IPRoute() as ipr:
# Move the interfaces into the namespace
idx = ipr.link_lookup(ifname=primary_interface)[0]
ipr.link('set', index=idx, net_ns_fd=consts.AMPHORA_NAMESPACE)
# bring interfaces up
_bring_if_down(primary_interface)
_bring_if_down(secondary_interface)
_bring_if_up(primary_interface, 'VIP')
_bring_if_up(secondary_interface, 'VIP')
return flask.make_response(flask.jsonify(dict(
message="OK",
details="VIP {vip} plugged on interface {interface}".format(
vip=vip, interface=interface))), 202)