def close(self):
"""Close the SSH connection."""
if self.netns_file_path:
with Namespace(self.netns_file_path, 'net'):
self.ssh_client.close()
else:
self.ssh_client.close()
def create_veth(self):
'''
Create veth pair
The ifname inside container can be same for multiple containers.
Hence, the veth pair cannot be created inside the host namespace.
Create veth pair inside container and move one-end to host namespace
'''
ip = IPRoute()
iface = ip.link_lookup(ifname=self.host_ifname)
if len(iface) != 0:
return
# Switch to container namespace
with Namespace(self.container_pid, 'net'):
ip_ns = IPRoute()
try:
# Create veth pairs
ip_ns.link_create(ifname=self.host_ifname,
peer=self.container_ifname,
kind='veth')
except NetlinkError as e:
if e.code != errno.EEXIST:
raise CniError(
CNI_ERROR_ADD_VETH,
'Error creating veth device ' + self.host_ifname +
' code ' + str(e.code) + ' message ' + e.message)
# Move one end of veth pair to host network namespace
idx = ip_ns.link_lookup(ifname=self.host_ifname)[0]
ip_ns.link('set', index=idx, net_ns_pid=self.main_pid)
return
def _exec(self, cmd):
"""Private function that handles the ssh client invocation."""
def _exec_raw(_cmd):
# pylint: disable=subprocess-run-check
cp = utils.run_cmd([
"ssh",
"-q",
"-o", "ConnectTimeout=1",
"-o", "StrictHostKeyChecking=no",
"-o", "UserKnownHostsFile=/dev/null",
"-i", self.ssh_config["ssh_key_path"],
"{}@{}".format(
self.ssh_config["username"],
self.ssh_config["hostname"]
),
_cmd],
ignore_return_code=True)
_res = (
cp.returncode,
cp.stdout,
cp.stderr
)
return _res
# TODO: If a microvm runs in a particular network namespace, we have to
# temporarily switch to that namespace when doing something that routes
# packets over the network, otherwise the destination will not be
# reachable. Use a better setup/solution at some point!
if self.netns_file_path:
with Namespace(self.netns_file_path, 'net'):
res = _exec_raw(cmd)
else:
res = _exec_raw(cmd)
return res
def setup_host_interfaces(self):
"""Setup the interfaces (bridge, tap, VETH pair) on the host and connect
them to the container."""
for name, interface in self.interfaces.items():
logger.debug('Setting up interface %s on %s.', name, self.name)
interface.setup_bridge()
interface.connect_tap_to_bridge()
self.container.devices.update({
name: {
'name': name,
'type': 'nic',
'nictype': 'bridged',
'parent': interface.bridge_name,
'hwaddr': interface.mac_address,
'host_name': interface.veth_name,
}
})
self.container.save(wait=True)
container_state = pylxd.Client().api.containers[
self.name].state.get().json()
pid = container_state['metadata']['pid']
# Get container's namespace and setup the interface in the container
with Namespace(pid, 'net'):
interface.setup_veth_container_end(name)
def stop_server(self, port, protocol):
try:
server_process = self._server_registry.get_server(
self._ns, port, protocol)
self.log.info(
"Stop %s server on port %s in namespace %s" %
(protocol, port, self._ns))
if server_process and server_process.is_running():
with Namespace(self._ns_full_path, 'net'):
server_process.stop()
self._server_registry.remove_server(
self._ns, port, protocol)
elif server_process and not server_process.is_running():
self.log.warning("%s server is not running on %s" %
(protocol, port))
self._server_registry.remove_server(
self.ROOT_NAMESPACE_NAME, port, protocol)
else:
self.log.warning("%s server is not running on %s" %
(protocol, port))
except Exception as e:
self.log.exception(
"Stopping %s server on port %s failed in namespace %s" %
(protocol, port, self._ns))
raise e
def do_test():
fd, filename = tempfile.mkstemp()
os.close(fd)
os.remove(filename)
with Namespace(filename, 'net'):
pass
def test_namespaces_as_root(self):
"""Test entering all namespaces the pid has as root"""
for name in filter(lambda x: x != 'user', NAMESPACE_NAMES):
if os.path.exists(
os.path.join('/proc', str(self._child.pid), 'ns', name)):
with Namespace(self._child.pid, name):
pass
class QuaggaTelnetDaemon(object):
TELNET_PASSWORD = "******"
TELNET_PORT = 2605
def __init__(self, ctn):
self.ns = Namespace(ctn.get_pid(), 'net')
def __enter__(self):
self.ns.__enter__()
self.tn = telnetlib.Telnet('127.0.0.1', self.TELNET_PORT)
self.tn.read_until("Password: "******"\n")
self.tn.write("enable\n")
self.tn.read_until('bgpd#')
return self.tn
def __exit__(self, type, value, traceback):
self.tn.close()
self.ns.__exit__(type, value, traceback)
class QuaggaTelnetDaemon(object):
TELNET_PASSWORD = "******"
TELNET_PORT = 2605
def __init__(self, ctn):
self.ns = Namespace(ctn.get_pid(), 'net')
def __enter__(self):
self.ns.__enter__()
self.tn = telnetlib.Telnet('127.0.0.1', self.TELNET_PORT)
self.tn.read_until("Password: "******"\n")
self.tn.write("enable\n")
self.tn.read_until('bgpd#')
return self.tn
def __exit__(self, type, value, traceback):
self.tn.close()
self.ns.__exit__(type, value, traceback)
def get_fdb_endpoints(self):
with Namespace(self.vxlan['filepath'], 'net'):
o = subprocess.check_output(['bridge', 'fdb', 'show', 'br0'])
fdb_ep = filter(lambda x: 'dst' in x and self.vxlan['device'] in x,
o.split('\n'))
for ep in fdb_ep:
(mac, locator) = (lambda x: [x[0], x[4]])(ep.split(' '))
self.fdb_endpoints.append(tuple([mac, locator]))
def run_tests_for_target(network_ns, ports, target):
"""
Enters a desired network namespace and attempts to reach a target on a list of ports.
"""
# resolve host directly here
# https://stackoverflow.com/questions/2805231/how-can-i-do-dns-lookups-in-python-including-referring-to-etc-hosts
LOGGER.info("Target: %s", target)
port_on_nums = {port.replace("-", ""): port for port in ports}
port_string = ",".join(port_on_nums.keys())
# ToDo do we really need this -> we know the service already
# DNS could be blocked
# resolve target ip
# Only IPv4 currently
# ToDo try catch -- > socket.gaierror: [Errno -2] Name or service not known
svc_dns_entry = get_domain_name_for(target)
LOGGER.info(svc_dns_entry)
svc_ip = socket.gethostbyname(svc_dns_entry)
LOGGER.info("Service IP: %s for Service: %s", svc_ip, target)
with tempfile.NamedTemporaryFile() as result_file:
LOGGER.debug("Results will be stored to %s", result_file)
# remove the need for nmap!
# e.g. https://gist.github.com/betrcode/0248f0fda894013382d7
# nmap that target TODO: handle None ip
# Replace bare nmap call with a better integrated solution like: https://pypi.org/project/python-nmap/ ?
nmap_cmd = [
"nmap", "-oX", result_file.name, "-Pn", "-p", port_string, svc_ip
]
LOGGER.info("running nmap with cmd %s", nmap_cmd)
prc = None
with Namespace(network_ns, "net"):
prc = subprocess.run(nmap_cmd,
stdout=subprocess.PIPE,
stderr=subprocess.PIPE)
if prc is None or prc.returncode:
LOGGER.error("Executing nmap in foreign net ns failed! output:")
LOGGER.error(prc.stderr)
LOGGER.debug(prc)
return {
port_string: {
"success":
False,
"error":
"Couldn't nmap host %s with hostname %s" %
(target, svc_dns_entry),
}
}
LOGGER.info("finished running nmap")
LOGGER.debug("Error log: %s", prc.stderr)
# when not using shell the output from nmap contains \\n instead of newline characters
LOGGER.info("Stdout: %s", str(prc.stdout).split("\\n"))
return extract_results_from_nmap_xml_file(result_file, port_on_nums,
target)
def scp_get_file(self, remote_path, local_path):
"""Copy files from the VM using scp."""
cmd = ('scp -o StrictHostKeyChecking=no'
' -o UserKnownHostsFile=/dev/null'
' -i {} {}@{}:{} {}').format(self.ssh_config['ssh_key_path'],
self.ssh_config['username'],
self.ssh_config['hostname'],
remote_path, local_path)
if self.netns_file_path:
with Namespace(self.netns_file_path, 'net'):
utils.run_cmd(cmd)
else:
utils.run_cmd(cmd)
def scp_file(self, local_path, remote_path):
"""Copy a files to the VM using scp."""
cmd = ('scp -o StrictHostKeyChecking=no'
' -o UserKnownHostsFile=/dev/null'
' -i {} {} {}@{}:{}').format(self.ssh_config['ssh_key_path'],
local_path,
self.ssh_config['username'],
self.ssh_config['hostname'],
remote_path)
if self.netns_file_path:
with Namespace(self.netns_file_path, 'net'):
run(cmd, shell=True, check=True)
else:
run(cmd, shell=True, check=True)
def sig_ns_validator(name, value):
sig_ns = os.environ.get('signaling_namespace')
# If we have a configured signaling DNS server
# use it in preference for both DNS Python requests
# and none DNS Python requests
sig_dns = os.environ.get('signaling_dns_server')
if sig_ns:
with Namespace('/var/run/netns/' + sig_ns, 'net'):
return run_validator_with_dns(validator, name, value, sig_dns)
else:
return run_validator_with_dns(validator, name, value, sig_dns)
def setup_host_interfaces(self):
"""Setup the interfaces (bridge, tap, VETH pair) on the host and connect
them to the container."""
for name, interface in self.interfaces.items():
interface.setup_bridge()
interface.connect_tap_to_bridge()
interface.setup_veth_pair({
'ifname': name,
"net_ns_fd": f"/proc/{self.container_pid}/ns/net"
})
# Get container's namespace and setup the interface in the container
with Namespace(self.container_pid, 'net'):
interface.setup_veth_container_end(name)
def get_proc_net_dev_by_task_id(task_id):
'''Get /proc/net/dev contents from inside a container via its net ns'''
if not task_id:
return None
try:
with Namespace(task_id, 'net'):
# To read network metric in namespace,
# "open" method don't work with namespaces
network_data = subprocess.check_output(
['/bin/cat', '/proc/net/dev'])
return network_data.split('\n')
except:
collectd.debug("cannot get /proc/net/dev " +
"from netns for pid {0}".format(task_id))
return None
def start_client(self, src, clients):
client = self._client_registry.get_client(self._ns)
if client and client.is_running():
self.log.warning("Client is already running on %s" % src)
return
try:
process = WorkerProcess(
TrafficClient, (src, clients), {})
with Namespace(self._ns_full_path, 'net'):
process.start()
self._client_registry.add_client(self._ns, process)
except Exception as e:
self.log.exception(
"Starting client process on interface %s failed" % src)
raise e
def scp_get_file(self, remote_path, local_path):
"""Copy files from the VM using scp."""
cmd = ("scp -o StrictHostKeyChecking=no"
" -o UserKnownHostsFile=/dev/null"
" -i {} {}@{}:{} {}").format(
self.ssh_config["ssh_key_path"],
self.ssh_config["username"],
self.ssh_config["hostname"],
remote_path,
local_path,
)
if self.netns_file_path:
with Namespace(self.netns_file_path, "net"):
utils.run_cmd(cmd)
else:
utils.run_cmd(cmd)
def resolve_domain_address_in_namespace(addr, is_signaling): # pragma: no cover
dns_answers = []
if is_signaling:
sig_ns = os.environ.get('signaling_namespace')
sig_dns = os.environ.get('signaling_dns_server')
if sig_ns:
with Namespace('/var/run/netns/' + sig_ns, 'net'):
dns_answers = resolve_domain_address(addr, sig_dns)
else:
dns_answers = resolve_domain_address(addr, sig_dns)
else:
dns_answers = resolve_domain_address(addr)
ip_addrs = [ip.address for ip in dns_answers]
return ip_addrs
def stop_client(self):
try:
client = self._client_registry.get_client(self._ns)
if client and client.is_running():
with Namespace(self._ns_full_path, 'net'):
client.stop()
self._client_registry.remove_client(self._ns)
elif client and not client.is_running():
self.log.warning("Client is not running in namespace %s" %
self._ns)
self._client_registry.remove_client(self._ns)
else:
self.log.warning("Client is not running in namespace %s" %
self._ns)
except Exception:
self.log.exception("Stopping client failed in namespace %s" %
self._ns)
def run_tests_for_target(network_ns, ports, target):
"""
Enters a desired network namespace and attempts to reach a target on a list of ports.
"""
# resolve host directly here
# https://stackoverflow.com/questions/2805231/how-can-i-do-dns-lookups-in-python-including-referring-to-etc-hosts
LOGGER.info("Target: %s", target)
port_on_nums = {port.replace("-", ""): port for port in ports}
port_string = ",".join(port_on_nums.keys())
ipv6_arg = ""
if ipaddress.ip_address(target).version == 6:
ipv6_arg = "-6"
nm_scanner = nmap.PortScanner()
with Namespace(network_ns, "net"):
nm_scanner.scan(target,
arguments=f"-n -Pn -p {port_string} {ipv6_arg}")
LOGGER.info("Ran nmap with cmd %s", nm_scanner.command_line())
return extract_results_from_nmap(nm_scanner, port_on_nums, target)
def addif(self, ctn, ifname='', mac=''):
with docker_netns(ctn.name) as pid:
host_ifname = '{0}_{1}'.format(self.name, ctn.name)
guest_ifname = random_str(5)
ip = IPRoute()
ip.link('add', ifname=host_ifname, kind='veth', peer=guest_ifname)
host = ip.link_lookup(ifname=host_ifname)[0]
ip.link('set', index=host, master=self.br)
ip.link('set', index=host, state='up')
self.ctns.append(ctn)
guest = ip.link_lookup(ifname=guest_ifname)[0]
ip.link('set', index=guest, net_ns_fd=pid)
with Namespace(pid, 'net'):
ip = IPRoute()
if ifname == '':
links = [x.get_attr('IFLA_IFNAME') for x in ip.get_links()]
n = [
int(l[len('eth'):]) for l in links
if l.startswith('eth')
]
idx = 0
if len(n) > 0:
idx = max(n) + 1
ifname = 'eth{0}'.format(idx)
ip.link('set', index=guest, ifname=ifname)
ip.link('set', index=guest, state='up')
if mac != '':
ip.link('set', index=guest, address=mac)
if self.with_ip:
address, mask = self.next_ip_address().split('/')
ip.addr('add',
index=guest,
address=address,
mask=int(mask))
ctn.ip_addrs.append((ifname, address, self.name))
return ifname
def configure_veth(self, mac, ip4, plen, gw):
'''
Configure the interface inside container with,
- Mac-address
- IP Address
- Default gateway
- Link-up
'''
ip = IPRoute()
idx = ip.link_lookup(ifname=self.host_ifname)[0]
ip.link('set', index=idx, state='up')
with Namespace(self.container_pid, 'net'):
ip_ns = IPRoute()
idx_ns = ip_ns.link_lookup(ifname=self.container_ifname)[0]
try:
ip_ns.link('set', index=idx_ns, state='up')
except NetlinkError as e:
self.veth_error(e, self.container_ifname,
'Error setting link state for veth device')
try:
ip_ns.link('set', index=idx_ns, address=mac)
except NetlinkError as e:
self.veth_error(e, self.container_ifname,
'Error setting mac-address for veth device')
try:
ip_ns.addr('add', index=idx_ns, address=ip4, prefixlen=plen)
except NetlinkError as e:
if e.code != errno.EEXIST:
self.veth_error(
e, self.container_ifname,
'Error setting ip-address for veth device')
try:
ip_ns.route('add', dst='0.0.0.0/0', gateway=gw)
except NetlinkError as e:
if e.code != errno.EEXIST:
self.veth_error(e, self.container_ifname,
'Error adding default route in container')
return
def start_server(self, protocol, port, src="0.0.0.0"):
server_process = self._server_registry.get_server(
self._ns, port, protocol)
if server_process and server_process.is_running():
self.log.warning("%s server on port %s is already running" %
(protocol, port))
return
self.log.info(
"Starting %s server on port %s on interface %s in namespace %s" %
(protocol, port, src, self._ns))
try:
server_cls, args, kwargs = create_server_class(protocol, port, src)
process = WorkerProcess(server_cls, args, kwargs)
with Namespace(self._ns_full_path, 'net'):
process.start()
self._server_registry.add_server(
self._ns, port, protocol, process)
except Exception as e:
self.log.exception(
"Starting %s server on port %s on interface %s failed" %
(protocol, port, src))
raise e
def __init__(self, ssh_config):
"""Instantiate a SSH client and connect to a microVM."""
self.netns_file_path = ssh_config['netns_file_path']
self.ssh_client = SSHClient() # pylint: disable=no-value-for-parameter
self.ssh_client.set_missing_host_key_policy(AutoAddPolicy())
assert os.path.exists(ssh_config['ssh_key_path'])
# Retry to connect to the host as long as the delay between calls
# is less than 30 seconds. Sleep 1, 2, 4, 8, ... seconds between
# attempts. These parameters might need additional tweaking when we
# add tests with 1000 Firecracker microvm.
# TODO: If a microvm runs in a particular network namespace, we have to
# temporarily switch to that namespace when doing something that routes
# packets over the network, otherwise the destination will not be
# reachable. Use a better setup/solution at some point!
if self.netns_file_path:
with Namespace(self.netns_file_path, 'net'):
self.initial_connect(ssh_config)
else:
self.initial_connect(ssh_config)
def test_namespace_good_path(self):
"""Test entering an arbirtrary namespace"""
try:
# get the path to it's network namespace
ns_path = os.path.join('/proc', str(self._child.pid), 'ns', 'net')
# bind mount it to a temp location
fd, filename = tempfile.mkstemp()
os.close(fd)
assert self._libc.mount(ns_path.encode('ascii'),
filename.encode('ascii'), 0, 4096, 0) == 0
# enter the bind mount
with Namespace(filename, 'net'):
pass
finally:
# ensure we clean up the bind
self._libc.umount(filename.encode('ascii'))
os.remove(filename)
def run(self):
if self.name in get_containers():
self.stop()
binds = [
'{0}:{1}'.format(os.path.abspath(sv[0]), sv[1])
for sv in self.shared_volumes
]
config = dckr.create_host_config(binds=binds, privileged=True)
ctn = dckr.create_container(
image=self.image,
detach=True,
name=self.name,
stdin_open=True,
volumes=[sv[1] for sv in self.shared_volumes],
host_config=config,
network_disabled=True)
dckr.start(container=self.name)
self.id = ctn['Id']
self.is_running = True
with docker_netns(self.name) as pid:
with Namespace(pid, 'net'):
ip = IPRoute()
lo = ip.link_lookup(ifname='lo')[0]
ip.link('set', index=lo, state='up')
def connect_ctn_to_br(ctn, brname):
with docker_netns(ctn) as pid:
ip = IPRoute()
br = ip.link_lookup(ifname=brname)
if len(br) == 0:
ip.link_create(ifname=brname, kind='bridge')
br = ip.link_lookup(ifname=brname)
br = br[0]
ip.link('set', index=br, state='up')
ifs = ip.link_lookup(ifname=ctn)
if len(ifs) > 0:
ip.link_remove(ifs[0])
ip.link_create(ifname=ctn, kind='veth', peer=pid)
host = ip.link_lookup(ifname=ctn)[0]
ip.link('set', index=host, master=br)
ip.link('set', index=host, state='up')
guest = ip.link_lookup(ifname=pid)[0]
ip.link('set', index=guest, net_ns_fd=pid)
with Namespace(pid, 'net'):
ip = IPRoute()
ip.link('set', index=guest, ifname='eth1')
ip.link('set', index=guest, state='up')
def __init__(self, ctn):
self.ns = Namespace(ctn.get_pid(), 'net')
def execute_command(self, cmd_string):
"""Execute the command passed as a string in the ssh context."""
if self.netns_file_path:
with Namespace(self.netns_file_path, 'net'):
return self.ssh_client.exec_command(cmd_string)
return self.ssh_client.exec_command(cmd_string)
def find_messy_entries(self):
if len(self.consul_endpoints) == 0:
self.get_consul_endpoints()
if len(self.fdb_endpoints) == 0:
self.get_fdb_endpoints()
# check different entries in consul_endpoints and fdb_endpoints and
# replace them
replace_list = [
tuple([c_tup[0], f_tup[1], c_tup[1]])
for c_tup in self.consul_endpoints for f_tup in self.fdb_endpoints
if f_tup[0] == c_tup[0] and f_tup[1] != c_tup[1]
and f_tup[1] != '127.0.0.1'
]
if len(replace_list) > 0:
print '--- To replace ---'
for r in replace_list:
print '{0:} from {1:} to {2:}'.format(r[0], r[1], r[2])
else:
print 'Nothing to replace'
# check missing entries in consul_endpoints and remove them
# from fdb_endpoints
delete_list = [
tuple([tup[0]]) for tup in self.fdb_endpoints
if tup[0] not in [x[0] for x in self.consul_endpoints]
]
if len(delete_list) > 0:
print '\n--- To remove ---'
for d in delete_list:
print d[0]
else:
print 'Nothing to delete'
if self.config['dry_run']:
print 'This is a dry run, no modifications will be made to ' \
+ 'your system'
elif len(replace_list) or len(delete_list):
with Namespace(self.vxlan['filepath'], 'net'):
for tup in replace_list:
o = subprocess.check_output([
'bridge',
'fdb',
'replace',
tup[0],
'dev',
self.vxlan['device'],
'dst',
tup[2],
]).split('\n')
if o[0] != '':
print o
for tup in delete_list:
o = subprocess.check_output([
'bridge',
'fdb',
'delete',
tup[0],
'dev',
self.vxlan['device'],
]).split('\n')
if o[0] != '':
print o
def __init__(self, ctn):
self.ns = Namespace(ctn.get_pid(), 'net')
