def cleanup(self):
"""
Clean up left-over ipsets that existed at start-of-day.
"""
_log.info("Cleaning up left-over ipsets.")
all_ipsets = list_ipset_names()
# only clean up our own rubbish.
pfx = IPSET_PREFIX[self.ip_type]
tmppfx = IPSET_TMP_PREFIX[self.ip_type]
felix_ipsets = set([n for n in all_ipsets if (n.startswith(pfx) or
n.startswith(tmppfx))])
whitelist = set()
live_ipsets = self.objects_by_id.itervalues()
# stopping_objects_by_id is a dict of sets of TagIpset objects,
# chain them together.
stopping_ipsets = chain.from_iterable(
self.stopping_objects_by_id.itervalues())
for ipset in chain(live_ipsets, stopping_ipsets):
# Ask the ipset for all the names it may use and whitelist.
whitelist.update(ipset.owned_ipset_names())
_log.debug("Whitelisted ipsets: %s", whitelist)
ipsets_to_delete = felix_ipsets - whitelist
_log.debug("Deleting ipsets: %s", ipsets_to_delete)
# Delete the ipsets before we return. We can't queue these up since
# that could conflict if someone increffed one of the ones we're about
# to delete.
for ipset_name in ipsets_to_delete:
try:
futils.check_call(["ipset", "destroy", ipset_name])
except FailedSystemCall:
_log.exception("Failed to clean up dead ipset %s, will "
"retry on next cleanup.", ipset_name)
def _execute_iptables(self, input_lines, fail_log_level=logging.ERROR):
"""
Runs ip(6)tables-restore with the given input. Retries iff
the COMMIT fails.
:raises FailedSystemCall: if the command fails on a non-commit
line or if it repeatedly fails and retries are exhausted.
"""
backoff = 0.01
num_tries = 0
success = False
while not success:
input_str = "\n".join(input_lines) + "\n"
_log.debug("%s input:\n%s", self._restore_cmd, input_str)
# Run iptables-restore in noflush mode so that it doesn't
# blow away all the tables we're not touching.
cmd = [self._restore_cmd, "--noflush", "--verbose"]
try:
futils.check_call(cmd, input_str=input_str)
except FailedSystemCall as e:
# Parse the output to determine if error is retryable.
retryable, detail = _parse_ipt_restore_error(
input_lines, e.stderr)
num_tries += 1
if retryable:
if num_tries < MAX_IPT_RETRIES:
_log.info(
"%s failed with retryable error. Retry in "
"%.2fs", self._iptables_cmd, backoff)
self._stats.increment("iptables commit failure "
"(retryable)")
gevent.sleep(backoff)
if backoff > MAX_IPT_BACKOFF:
backoff = MAX_IPT_BACKOFF
backoff *= (1.5 + random.random())
continue
else:
_log.log(
fail_log_level,
"Failed to run %s. Out of retries: %s.\n"
"Output:\n%s\n"
"Error:\n%s\n"
"Input was:\n%s", self._restore_cmd, detail,
e.stdout, e.stderr, input_str)
self._stats.increment("iptables commit failure "
"(out of retries)")
else:
_log.log(
fail_log_level,
"%s failed with non-retryable error: %s.\n"
"Output:\n%s\n"
"Error:\n%s\n"
"Input was:\n%s", self._restore_cmd, detail, e.stdout,
e.stderr, input_str)
self._stats.increment("iptables non-retryable failure")
raise
else:
self._stats.increment("iptables success")
success = True
def set_routes(ip_type, ips, interface, mac=None, reset_arp=False):
"""
Set the routes on the interface to be the specified set.
:param ip_type: Type of IP (IPV4 or IPV6)
:param set ips: IPs to set up (any not in the set are removed)
:param str interface: Interface name
:param str mac|NoneType: MAC address. May not be none unless ips is empty.
:param bool reset_arp: Reset arp. Only valid if IPv4.
"""
if mac is None and ips:
raise ValueError("mac must be supplied if ips is not empty")
if reset_arp and ip_type != futils.IPV4:
raise ValueError("reset_arp may only be supplied for IPv4")
current_ips = list_interface_ips(ip_type, interface)
removed_ips = (current_ips - ips)
for ip in removed_ips:
del_route(ip_type, ip, interface)
remove_conntrack_flows(removed_ips, 4 if ip_type == futils.IPV4 else 6)
for ip in (ips - current_ips):
add_route(ip_type, ip, interface, mac)
if reset_arp:
for ip in (ips & current_ips):
futils.check_call(['arp', '-s', ip, mac, '-i', interface])
def list_interface_route_ips(ip_type, interface):
"""
List IP addresses for which there are routes to a given interface.
:param str ip_type: IP type, either futils.IPV4 or futils.IPV6
:param str interface: Interface name
:returns: a set of all addresses for which there is a route to the device.
"""
ips = set()
if ip_type == futils.IPV4:
data = futils.check_call(["ip", "route", "list", "dev",
interface]).stdout
else:
data = futils.check_call(
["ip", "-6", "route", "list", "dev", interface]).stdout
lines = data.split("\n")
_log.debug("Existing routes to %s : %s", interface, lines)
for line in lines:
# Example of the lines we care about is (having specified the
# device above): "10.11.2.66 proto static scope link"
words = line.split()
if len(words) > 1:
ip = words[0]
if common.validate_ip_addr(ip, futils.IP_TYPE_TO_VERSION[ip_type]):
# Looks like an IP address. Note that we here are ignoring
# routes to networks configured when the interface is created.
ips.add(words[0])
_log.debug("Found existing IP addresses : %s", ips)
return ips
def list_interface_route_ips(ip_type, interface):
"""
List IP addresses for which there are routes to a given interface.
:param str ip_type: IP type, either futils.IPV4 or futils.IPV6
:param str interface: Interface name
:returns: a set of all addresses for which there is a route to the device.
"""
ips = set()
if ip_type == futils.IPV4:
data = futils.check_call(
["ip", "route", "list", "dev", interface]).stdout
else:
data = futils.check_call(
["ip", "-6", "route", "list", "dev", interface]).stdout
lines = data.split("\n")
_log.debug("Existing routes to %s : %s", interface, lines)
for line in lines:
# Example of the lines we care about is (having specified the
# device above): "10.11.2.66 proto static scope link"
words = line.split()
if len(words) > 1:
ip = words[0]
if common.validate_ip_addr(ip, futils.IP_TYPE_TO_VERSION[ip_type]):
# Looks like an IP address. Note that we here are ignoring
# routes to networks configured when the interface is created.
ips.add(words[0])
_log.debug("Found existing IP addresses : %s", ips)
return ips
def remove_conntrack_flows(ip_addresses, ip_version):
"""
Removes any conntrack entries that use any of the given IP
addresses in their source/destination.
"""
assert ip_version in (4, 6)
for ip in ip_addresses:
_log.debug("Removing conntrack rules for %s", ip)
for direction in [
"--orig-src", "--orig-dst", "--reply-src", "--reply-dst"
]:
try:
futils.check_call([
"conntrack", "--family",
"ipv%s" % ip_version, "--delete", direction, ip
])
except FailedSystemCall as e:
if e.retcode == 1 and "0 flow entries" in e.stderr:
# Expected if there are no flows.
_log.debug("No conntrack entries found for %s/%s.", ip,
direction)
else:
# Suppress the exception, conntrack entries will timeout
# and it's hard to think of an example where killing and
# restarting felix would help.
_log.exception(
"Failed to remove conntrack flows for %s. "
"Ignoring.", ip)
def set_routes(ip_type, ips, interface, mac=None, reset_arp=False):
"""
Set the routes on the interface to be the specified set.
:param ip_type: Type of IP (IPV4 or IPV6)
:param set ips: IPs to set up (any not in the set are removed)
:param str interface: Interface name
:param str mac|NoneType: MAC address. May not be none unless ips is empty.
:param bool reset_arp: Reset arp. Only valid if IPv4.
"""
if mac is None and ips:
raise ValueError("mac must be supplied if ips is not empty")
if reset_arp and ip_type != futils.IPV4:
raise ValueError("reset_arp may only be supplied for IPv4")
current_ips = list_interface_route_ips(ip_type, interface)
removed_ips = (current_ips - ips)
for ip in removed_ips:
del_route(ip_type, ip, interface)
for ip in (ips - current_ips):
add_route(ip_type, ip, interface, mac)
if reset_arp:
for ip in (ips & current_ips):
futils.check_call(['arp', '-s', ip, mac, '-i', interface])
def list_ips_by_iface(ip_type):
"""
List the local IPs assigned to all interfaces.
:param str ip_type: IP type, either futils.IPV4 or futils.IPV6
:returns: a set of all addresses directly assigned to the device.
"""
assert ip_type in (futils.IPV4,
futils.IPV6), ("Expected an IP type, got %s" % ip_type)
if ip_type == futils.IPV4:
data = futils.check_call(["ip", "-4", "addr", "list"]).stdout
regex = r'^ inet ([0-9.]+)'
else:
data = futils.check_call(["ip", "-6", "addr", "list"]).stdout
regex = r'^ inet6 ([0-9a-fA-F:.]+)'
ips_by_iface = defaultdict(set)
iface_name = None
for line in data.splitlines():
m = re.match(r"^\d+: ([^:]+):", line)
if m:
iface_name = m.group(1)
else:
assert iface_name
m = re.match(regex, line)
if m:
ip = IPAddress(m.group(1))
ips_by_iface[iface_name].add(ip)
return ips_by_iface
def cleanup(self):
"""
Clean up left-over ipsets that existed at start-of-day.
"""
_log.info("Cleaning up left-over ipsets.")
all_ipsets = list_ipset_names()
# only clean up our own rubbish.
pfx = IPSET_PREFIX[self.ip_type]
tmppfx = IPSET_TMP_PREFIX[self.ip_type]
felix_ipsets = set([n for n in all_ipsets if (n.startswith(pfx) or
n.startswith(tmppfx))])
whitelist = set()
live_ipsets = self.objects_by_id.itervalues()
# stopping_objects_by_id is a dict of sets of RefCountedIpsetActor
# objects, chain them together.
stopping_ipsets = chain.from_iterable(
self.stopping_objects_by_id.itervalues())
for ipset in chain(live_ipsets, stopping_ipsets):
# Ask the ipset for all the names it may use and whitelist.
whitelist.update(ipset.owned_ipset_names())
_log.debug("Whitelisted ipsets: %s", whitelist)
ipsets_to_delete = felix_ipsets - whitelist
_log.debug("Deleting ipsets: %s", ipsets_to_delete)
# Delete the ipsets before we return. We can't queue these up since
# that could conflict if someone increffed one of the ones we're about
# to delete.
for ipset_name in ipsets_to_delete:
try:
futils.check_call(["ipset", "destroy", ipset_name])
except FailedSystemCall:
_log.exception("Failed to clean up dead ipset %s, will "
"retry on next cleanup.", ipset_name)
def list_ips_by_iface(ip_type):
"""
List the local IPs assigned to all interfaces.
:param str ip_type: IP type, either futils.IPV4 or futils.IPV6
:returns: a set of all addresses directly assigned to the device.
"""
assert ip_type in (futils.IPV4, futils.IPV6), (
"Expected an IP type, got %s" % ip_type
)
if ip_type == futils.IPV4:
data = futils.check_call(["ip", "-4", "addr", "list"]).stdout
regex = r'^ inet ([0-9.]+)'
else:
data = futils.check_call(["ip", "-6", "addr", "list"]).stdout
regex = r'^ inet6 ([0-9a-fA-F:.]+)'
ips_by_iface = defaultdict(set)
iface_name = None
for line in data.splitlines():
m = re.match(r"^\d+: ([^:]+):", line)
if m:
iface_name = m.group(1)
else:
assert iface_name
m = re.match(regex, line)
if m:
ip = IPAddress(m.group(1))
ips_by_iface[iface_name].add(ip)
return ips_by_iface
def _sync_to_ipset(self):
_log.info("Rewriting %s ipset %s for tag %s with %d members.",
self.ip_type, self.name, self._id, len(self.members))
_log.debug("Setting ipset %s to %s", self.name, self.members)
# We use ipset restore, which processes a batch of ipset updates.
# The only operation that we're sure is atomic is swapping two ipsets
# so we build up the complete set of members in a temporary ipset,
# swap it into place and then delete the old ipset.
create_cmd = "create %s hash:ip family %s --exist"
input_lines = [
# Ensure both the main set and the temporary set exist.
create_cmd % (self.name, self.family),
create_cmd % (self.tmpname, self.family),
# Flush the temporary set. This is a no-op unless we had a
# left-over temporary set before.
"flush %s" % self.tmpname,
]
# Add all the members to the temporary set,
input_lines += ["add %s %s" % (self.tmpname, m) for m in self.members]
# Then, atomically swap the temporary set into place.
input_lines.append("swap %s %s" % (self.name, self.tmpname))
# Finally, delete the temporary set (which was the old active set).
input_lines.append("destroy %s" % self.tmpname)
# COMMIT tells ipset restore to actually execute the changes.
input_lines.append("COMMIT")
input_str = "\n".join(input_lines) + "\n"
futils.check_call(["ipset", "restore"], input_str=input_str)
# We have got the set into the correct state.
self.programmed_members = self.members.copy()
def remove_conntrack_flows(ip_addresses, ip_version):
"""
Removes any conntrack entries that use any of the given IP
addresses in their source/destination.
"""
assert ip_version in (4, 6)
for ip in ip_addresses:
_log.debug("Removing conntrack rules for %s", ip)
for direction in ["--orig-src", "--orig-dst",
"--reply-src", "--reply-dst"]:
try:
futils.check_call(["conntrack", "--family",
"ipv%s" % ip_version, "--delete",
direction, ip])
except FailedSystemCall as e:
if e.retcode == 1 and "0 flow entries" in e.stderr:
# Expected if there are no flows.
_log.debug("No conntrack entries found for %s/%s.",
ip, direction)
else:
# Suppress the exception, conntrack entries will timeout
# and it's hard to think of an example where killing and
# restarting felix would help.
_log.exception("Failed to remove conntrack flows for %s. "
"Ignoring.", ip)
def _exec_and_commit(self, input_lines):
"""
Executes the the given lines of "ipset restore" input and
follows them with a COMMIT call.
"""
input_lines.append("COMMIT")
input_str = "\n".join(input_lines) + "\n"
futils.check_call(["ipset", "restore"], input_str=input_str)
def on_unreferenced(self):
try:
if self.set_exists:
futils.check_call(["ipset", "destroy", self.name])
if self.tmpset_exists:
futils.check_call(["ipset", "destroy", self.tmpname])
finally:
self._notify_cleanup_complete()
def _exec_and_commit(self, input_lines):
"""
Executes the the given lines of "ipset restore" input and
follows them with a COMMIT call.
"""
input_lines.append("COMMIT")
input_str = "\n".join(input_lines) + "\n"
futils.check_call(["ipset", "restore"], input_str=input_str)
def on_unreferenced(self):
try:
if self.set_exists:
futils.check_call(["ipset", "destroy", self.name])
if self.tmpset_exists:
futils.check_call(["ipset", "destroy", self.tmpname])
finally:
self._notify_cleanup_complete()
def _execute_iptables(self, input_lines, fail_log_level=logging.ERROR):
"""
Runs ip(6)tables-restore with the given input. Retries iff
the COMMIT fails.
:raises FailedSystemCall: if the command fails on a non-commit
line or if it repeatedly fails and retries are exhausted.
"""
backoff = 0.01
num_tries = 0
success = False
while not success:
input_str = "\n".join(input_lines) + "\n"
_log.debug("%s input:\n%s", self._restore_cmd, input_str)
# Run iptables-restore in noflush mode so that it doesn't
# blow away all the tables we're not touching.
cmd = [self._restore_cmd, "--noflush", "--verbose"]
try:
futils.check_call(cmd, input_str=input_str)
except FailedSystemCall as e:
# Parse the output to determine if error is retryable.
retryable, detail = _parse_ipt_restore_error(input_lines,
e.stderr)
num_tries += 1
if retryable:
if num_tries < MAX_IPT_RETRIES:
_log.info("%s failed with retryable error. Retry in "
"%.2fs", self._iptables_cmd, backoff)
gevent.sleep(backoff)
if backoff > MAX_IPT_BACKOFF:
backoff = MAX_IPT_BACKOFF
backoff *= (1.5 + random.random())
continue
else:
_log.log(
fail_log_level,
"Failed to run %s. Out of retries: %s.\n"
"Output:\n%s\n"
"Error:\n%s\n"
"Input was:\n%s",
self._restore_cmd, detail, e.stdout, e.stderr,
input_str)
else:
_log.log(
fail_log_level,
"%s failed with non-retryable error: %s.\n"
"Output:\n%s\n"
"Error:\n%s\n"
"Input was:\n%s",
self._restore_cmd, detail, e.stdout, e.stderr,
input_str)
raise
else:
success = True
def exists(self):
try:
futils.check_call(["ipset", "list", self.set_name])
except FailedSystemCall as e:
if e.retcode == 1 and "does not exist" in e.stderr:
return False
else:
_log.exception("Failed to check if ipset exists")
raise
else:
return True
def exists(self):
try:
futils.check_call(["ipset", "list", self.set_name])
except FailedSystemCall as e:
if e.retcode == 1 and "does not exist" in e.stderr:
return False
else:
_log.exception("Failed to check if ipset exists")
raise
else:
return True
def test_bad_check_call(self):
# Test an invalid command - must parse but not return anything.
args = ["ls", "wibble_wobble"]
try:
futils.check_call(args)
self.assertTrue(False)
except futils.FailedSystemCall as e:
self.assertNotEqual(e.retcode, 0)
self.assertEqual(list(e.args), args)
self.assertNotEqual(e.stdout, None)
self.assertNotEqual(e.stderr, None)
self.assertTrue("wibble_wobble" in str(e))
def test_bad_check_call(self):
# Test an invalid command - must parse but not return anything.
try:
args = ["ls", "wibble_wobble"]
futils.check_call(args)
self.assertTrue(False)
except futils.FailedSystemCall as e:
self.assertNotEqual(e.retcode, 0)
self.assertEqual(list(e.args), args)
self.assertNotEqual(e.stdout, None)
self.assertNotEqual(e.stderr, None)
self.assertTrue("wibble_wobble" in str(e))
def create(name, typename, family):
"""
Create an ipset. If it already exists, do nothing.
*name* is the name of the ipset.
*typename* must be a valid type, such as "hash:net" or "hash:net,port"
*family* must be *inet* or *inet6*
"""
if futils.call_silent(["ipset", "list", name]) != 0:
# ipset list failed - either does not exist, or an error. Either way,
# try creation, throwing an error if it does not work.
futils.check_call(
["ipset", "create", name, typename, "family", family])
def _sync_to_ipset(self):
_log.debug("Setting ipset %s to %s", self.name, self.members)
fd, filename = tempfile.mkstemp(text=True)
f = os.fdopen(fd, "w")
if not self.set_exists:
# ipset does not exist, so just create it and put the data in it.
set_name = self.name
create = True
swap = False
elif not self.tmpset_exists:
# Set exists, but tmpset does not
set_name = self.tmpname
create = True
swap = True
else:
# Both set and tmpset exist
set_name = self.tmpname
create = False
swap = True
if create:
f.write("create %s hash:ip family %s\n" % (set_name, self.family))
else:
f.write("flush %s\n" % (set_name))
for member in self.members:
f.write("add %s %s\n" % (set_name, member))
if swap:
f.write("swap %s %s\n" % (self.name, self.tmpname))
f.write("destroy %s\n" % (self.tmpname))
f.close()
# Load that data.
futils.check_call(["ipset", "restore", "-file", filename])
# By the time we get here, the set exists, and the tmpset does not if
# we just destroyed it after a swap (it might still exist if it did and
# the main set did not when we started, unlikely though that seems!).
self.set_exists = True
if swap:
self.tmpset_exists = False
# Tidy up the tmp file.
os.remove(filename)
# We have got the set into the correct state.
self.programmed_members = self.members.copy()
def _sync_to_ipset(self):
_log.debug("Setting ipset %s to %s", self.name, self.members)
fd, filename = tempfile.mkstemp(text=True)
f = os.fdopen(fd, "w")
if not self.set_exists:
# ipset does not exist, so just create it and put the data in it.
set_name = self.name
create = True
swap = False
elif not self.tmpset_exists:
# Set exists, but tmpset does not
set_name = self.tmpname
create = True
swap = True
else:
# Both set and tmpset exist
set_name = self.tmpname
create = False
swap = True
if create:
f.write("create %s hash:ip family %s\n" % (set_name, self.family))
else:
f.write("flush %s\n" % (set_name))
for member in self.members:
f.write("add %s %s\n" % (set_name, member))
if swap:
f.write("swap %s %s\n" % (self.name, self.tmpname))
f.write("destroy %s\n" % (self.tmpname))
f.close()
# Load that data.
futils.check_call(["ipset", "restore", "-file", filename])
# By the time we get here, the set exists, and the tmpset does not if
# we just destroyed it after a swap (it might still exist if it did and
# the main set did not when we started, unlikely though that seems!).
self.set_exists = True
if swap:
self.tmpset_exists = False
# Tidy up the tmp file.
os.remove(filename)
# We have got the set into the correct state.
self.programmed_members = self.members.copy()
def load_nf_conntrack():
"""
Try to force the nf_conntrack_netlink kernel module to be loaded.
"""
_log.info("Running conntrack command to force load of "
"nf_conntrack_netlink module.")
try:
# Running the stats command is enough to make conntrack load the
# kernel module if needed. We don't use modprobe here because it's
# not smart enough to tell if the module is compiled in.
futils.check_call(["conntrack", "-S"])
except FailedSystemCall:
_log.exception("Failed to execute conntrack command to force load of "
"nf_conntrack_netlink module. conntrack commands may "
"fail later.")
def load_nf_conntrack():
"""
Try to force the nf_conntrack_netlink kernel module to be loaded.
"""
_log.info("Running conntrack command to force load of "
"nf_conntrack_netlink module.")
try:
# Running the stats command is enough to make conntrack load the
# kernel module if needed. We don't use modprobe here because it's
# not smart enough to tell if the module is compiled in.
futils.check_call(["conntrack", "-S"])
except FailedSystemCall:
_log.exception("Failed to execute conntrack command to force load of "
"nf_conntrack_netlink module. conntrack commands may "
"fail later.")
def test_good_check_call(self):
# Test a command. Result must include "calico" given where it is run from.
args = ["ls"]
result = futils.check_call(args)
self.assertNotEqual(result.stdout, None)
self.assertNotEqual(result.stderr, None)
self.assertTrue("calico" in result.stdout)
self.assertEqual(result.stderr, "")
def test_good_check_call(self):
# Test a command. Result must include "calico" given where it is run from.
args = ["ls"]
result = futils.check_call(args)
self.assertNotEqual(result.stdout, None)
self.assertNotEqual(result.stderr, None)
self.assertTrue("calico" in result.stdout)
self.assertEqual(result.stderr, "")
def configure_interface_ipv6(if_name, proxy_target):
"""
Configure an interface to support IPv6 traffic from an endpoint.
- Enable proxy NDP on the interface.
- Program the given proxy target (gateway the endpoint will use).
:param if_name: The name of the interface to configure.
:param proxy_target: IPv6 address which is proxied on this interface for
NDP.
:returns: None
:raises: FailedSystemCall
"""
_write_proc_sys("/proc/sys/net/ipv6/conf/%s/proxy_ndp" % if_name, 1)
# Allows None if no IPv6 proxy target is required.
if proxy_target:
futils.check_call(["ip", "-6", "neigh", "add", "proxy", str(proxy_target), "dev", if_name])
def load_nf_conntrack():
"""
Try to force the nf_conntrack_netlink kernel module to be loaded.
"""
_log.info("Running conntrack command to force load of "
"nf_conntrack_netlink module.")
try:
# Run a conntrack command to trigger it to load the kernel module if
# it's not already compiled in. We list rules with a randomly-chosen
# link local address. That makes it very unlikely that we generate
# any wasteful output. We used to use "-S" (show stats) here but it
# seems to be bugged on some platforms, generating an error.
futils.check_call(["conntrack", "-L", "-s", "169.254.45.169"])
except FailedSystemCall:
_log.exception("Failed to execute conntrack command to force load of "
"nf_conntrack_netlink module. conntrack commands may "
"fail later.")
def list_interface_ips(ip_type, interface):
"""
List the local IPs assigned to an interface.
:param str ip_type: IP type, either futils.IPV4 or futils.IPV6
:param str interface: Interface name
:returns: a set of all addresses directly assigned to the device.
"""
assert ip_type in (futils.IPV4, futils.IPV6), "Expected an IP type, got %s" % ip_type
if ip_type == futils.IPV4:
data = futils.check_call(["ip", "addr", "list", "dev", interface]).stdout
regex = r"^ inet ([0-9.]+)"
else:
data = futils.check_call(["ip", "-6", "addr", "list", "dev", interface]).stdout
regex = r"^ inet6 ([0-9a-fA-F:.]+)"
# Search the output for lines beginning " inet(6)".
ips = re.findall(regex, data, re.MULTILINE)
_log.debug("Interface %s has %s IPs %s", interface, ip_type, ips)
return set(IPAddress(ip) for ip in ips)
def load_nf_conntrack():
"""
Try to force the nf_conntrack_netlink kernel module to be loaded.
"""
_log.info("Running conntrack command to force load of " "nf_conntrack_netlink module.")
try:
# Run a conntrack command to trigger it to load the kernel module if
# it's not already compiled in. We list rules with a randomly-chosen
# link local address. That makes it very unlikely that we generate
# any wasteful output. We used to use "-S" (show stats) here but it
# seems to be bugged on some platforms, generating an error.
futils.check_call(["conntrack", "-L", "-s", "169.254.45.169"])
except FailedSystemCall:
_log.exception(
"Failed to execute conntrack command to force load of "
"nf_conntrack_netlink module. conntrack commands may "
"fail later."
)
def configure_interface_ipv6(if_name, proxy_targets):
"""
Configure an interface to support IPv6 traffic from an endpoint.
- Enable proxy NDP on the interface.
- Program the given proxy targets (gateway(s) the endpoint will use).
:param if_name: The name of the interface to configure.
:param proxy_targets: IPv6 addresses which are proxied on this interface
for NDP.
:return: None
"""
with open("/proc/sys/net/ipv6/conf/%s/proxy_ndp" % if_name, 'wb') as f:
f.write('1')
for target in proxy_targets:
futils.check_call(
["ip", "-6", "neigh", "add", "proxy",
str(target), "dev", if_name])
def configure_interface_ipv6(if_name, proxy_target):
"""
Configure an interface to support IPv6 traffic from an endpoint.
- Enable proxy NDP on the interface.
- Program the given proxy target (gateway the endpoint will use).
:param if_name: The name of the interface to configure.
:param proxy_target: IPv6 address which is proxied on this interface for
NDP.
:returns: None
:raises: FailedSystemCall
"""
_write_proc_sys("/proc/sys/net/ipv6/conf/%s/proxy_ndp" % if_name, 1)
# Allows None if no IPv6 proxy target is required.
if proxy_target:
futils.check_call(["ip", "-6", "neigh", "add",
"proxy", str(proxy_target), "dev", if_name])
def read_table(self, type, name):
"""
Runs the iptables command to load a table and returns it. Separated out
purely for convenience of testing (since we can trivially mock it out).
"""
data = futils.check_call(
[IPTABLES_CMD[type], "--wait", "--list-rules", "--table",
name]).stdout
return data
def cleanup(self):
"""
Clean up left-over ipsets that existed at start-of-day.
"""
_log.info("Cleaning up left-over ipsets.")
all_ipsets = list_ipset_names()
# Filter deletion candidates to only ipsets that we could have created.
felix_ipsets = set()
for ipset in all_ipsets:
print "ipset: %s" % ipset
for prefix in ALL_FELIX_PREFIXES[self.ip_type]:
print "prefix: %s" % prefix
if ipset.startswith(prefix):
print "matched"
felix_ipsets.add(ipset)
whitelist = set()
live_ipsets = self.objects_by_id.itervalues()
# stopping_objects_by_id is a dict of sets of RefCountedIpsetActor
# objects, chain them together.
stopping_ipsets = chain.from_iterable(
self.stopping_objects_by_id.itervalues())
for ipset in chain(live_ipsets, stopping_ipsets):
# Ask the ipset for all the names it may use and whitelist.
whitelist.update(ipset.owned_ipset_names())
_log.debug("Whitelisted ipsets: %s", whitelist)
print "Whitelisted ipsets: %s" % whitelist
ipsets_to_delete = felix_ipsets - whitelist
_log.debug("Deleting ipsets: %s", ipsets_to_delete)
# Delete the ipsets before we return. We can't queue these up since
# that could conflict if someone increffed one of the ones we're about
# to delete.
for ipset_name in ipsets_to_delete:
try:
futils.check_call(["ipset", "destroy", ipset_name])
except FailedSystemCall:
_log.exception("Failed to clean up dead ipset %s, will "
"retry on next cleanup.", ipset_name)
_log.info("All ipsets: %s", all_ipsets)
_log.info("Whitelist: %s", whitelist)
_log.info("ipsets to delete: %s", ipsets_to_delete)
def interface_exists(interface):
"""
Checks if an interface exists.
:param str interface: Interface name
:returns: True if interface device exists
:raises: FailedSystemCall if ip link list fails.
We could check under /sys/class/net here, but there's a window where
/sys/class/net/<iface> might still exist for a link that is in the process
of being deleted, so we use ip link list instead.
"""
try:
futils.check_call(["ip", "link", "list", interface])
return True
except futils.FailedSystemCall as fsc:
if fsc.stderr.count("does not exist") != 0:
return False
else:
# An error other than does not exist; just pass on up
raise
def interface_exists(interface):
"""
Checks if an interface exists.
:param str interface: Interface name
:returns: True if interface device exists
:raises: FailedSystemCall if ip link list fails.
We could check under /sys/class/net here, but there's a window where
/sys/class/net/<iface> might still exist for a link that is in the process
of being deleted, so we use ip link list instead.
"""
try:
futils.check_call(["ip", "link", "list", interface])
return True
except futils.FailedSystemCall as fsc:
if fsc.stderr.count("does not exist") != 0:
return False
else:
# An error other than does not exist; just pass on up
raise
def test_good_check_call(self):
with mock.patch("calico.felix.futils._call_semaphore", wraps=futils._call_semaphore) as m_sem:
# Test a command. Result must include "calico" given where it is
# run from.
args = ["ls"]
result = futils.check_call(args)
self.assertNotEqual(result.stdout, None)
self.assertNotEqual(result.stderr, None)
self.assertTrue("calico" in result.stdout)
self.assertEqual(result.stderr, "")
self.assertTrue(m_sem.__enter__.called)
def remove_conntrack_flows(ip_addresses, ip_version):
"""
Removes any conntrack entries that use any of the given IP
addresses in their source/destination.
"""
assert ip_version in (4, 6)
for ip in ip_addresses:
_log.debug("Removing conntrack rules for %s", ip)
for direction in [
"--orig-src", "--orig-dst", "--reply-src", "--reply-dst"
]:
remaining_attempts = 3
while remaining_attempts > 0:
remaining_attempts -= 1
try:
futils.check_call([
"conntrack", "--family",
"ipv%s" % ip_version, "--delete", direction, ip
])
except FailedSystemCall as e:
if e.retcode == 1 and "0 flow entries" in e.stderr:
# Success: there are no flows.
_log.debug("No conntrack entries found for %s/%s.", ip,
direction)
break
if remaining_attempts == 0:
# Log the failure but the cause is likely a conntrack
# or Felix bug so killing the process is unlikely to
# help hence we suppress the exception and let the
# conntrack flows time out.
_log.exception(
"Failed to remove conntrack flows for "
"%s/%s after multiple attempts.", ip, direction)
else:
_log.warning(
"Failed to remove conntrack flows for "
"%s/%s; will retry: %s", ip, direction, e)
else:
_log.debug("Removed conntrack flows for %s/%s.", ip,
direction)
break
def test_good_check_call(self):
with mock.patch("calico.felix.futils._call_semaphore",
wraps=futils._call_semaphore) as m_sem:
# Test a command. Result must include "calico" given where it is
# run from.
args = ["ls"]
result = futils.check_call(args)
self.assertNotEqual(result.stdout, None)
self.assertNotEqual(result.stderr, None)
self.assertTrue("calico" in result.stdout)
self.assertEqual(result.stderr, "")
self.assertTrue(m_sem.__enter__.called)
def set_interface_ips(ip_type, interface, ips):
"""
Set the IPs directly assigned to an interface. Idempotent: does not
flap addresses if they're already in place.
:param str ip_type: IP type, either futils.IPV4 or futils.IPV6
:param str interface: Interface name
:param set[IPAddress] ips: The IPs to set or an empty set to remove all
IPs.
"""
assert ip_type in (futils.IPV4, futils.IPV6), "Expected an IP type, got %s" % ip_type
old_ips = list_interface_ips(ip_type, interface)
ips_to_add = ips - old_ips
ips_to_remove = old_ips - ips
ip_cmd = ["ip", "-6"] if ip_type == futils.IPV6 else ["ip"]
for ip in ips_to_remove:
_log.info("Removing IP %s from interface %s", ip, interface)
futils.check_call(ip_cmd + ["addr", "del", str(ip), "dev", interface])
for ip in ips_to_add:
_log.info("Adding IP %s to interface %s", ip, interface)
futils.check_call(ip_cmd + ["addr", "add", str(ip), "dev", interface])
def list_interface_ips(ip_type, interface):
"""
List the local IPs assigned to an interface.
:param str ip_type: IP type, either futils.IPV4 or futils.IPV6
:param str interface: Interface name
:returns: a set of all addresses directly assigned to the device.
"""
assert ip_type in (futils.IPV4,
futils.IPV6), ("Expected an IP type, got %s" % ip_type)
if ip_type == futils.IPV4:
data = futils.check_call(["ip", "addr", "list", "dev",
interface]).stdout
regex = r'^ inet ([0-9.]+)'
else:
data = futils.check_call(
["ip", "-6", "addr", "list", "dev", interface]).stdout
regex = r'^ inet6 ([0-9a-fA-F:.]+)'
# Search the output for lines beginning " inet(6)".
ips = re.findall(regex, data, re.MULTILINE)
_log.debug("Interface %s has %s IPs %s", interface, ip_type, ips)
return set(IPAddress(ip) for ip in ips)
def _configure_ipip_device(config):
"""Creates and enables the IPIP tunnel device.
:raises FailedSystemCall on failure.
"""
if not devices.interface_exists(IP_IN_IP_DEV_NAME):
# Make sure the IP-in-IP device exists; since we use the global
# device, this command actually creates it as a side-effect of
# initialising the kernel module rather than explicitly creating
# it.
_log.info("Tunnel device didn't exist; creating.")
futils.check_call(["ip", "tunnel", "add", IP_IN_IP_DEV_NAME,
"mode", "ipip"])
futils.check_call(["ip", "link", "set", IP_IN_IP_DEV_NAME, "mtu",
str(config.IP_IN_IP_MTU)])
if not devices.interface_up(IP_IN_IP_DEV_NAME):
_log.info("Tunnel device wasn't up; enabling.")
futils.check_call(["ip", "link", "set", IP_IN_IP_DEV_NAME, "up"])
# Allow an IP address to be added to the tunnel. This is useful to
# allow the host to have an IP on a private IPIP network so that it can
# originate traffic and have it routed correctly.
_log.info("Setting IPIP device IP to %s", config.IP_IN_IP_ADDR)
tunnel_addrs = [config.IP_IN_IP_ADDR] if config.IP_IN_IP_ADDR else []
devices.set_interface_ips(futils.IPV4, IP_IN_IP_DEV_NAME,
set(tunnel_addrs))
_log.info("Configured IPIP device.")
def _configure_ipip_device(config):
"""Creates and enables the IPIP tunnel device.
:raises FailedSystemCall on failure.
"""
if not devices.interface_exists(IP_IN_IP_DEV_NAME):
# Make sure the IP-in-IP device exists; since we use the global
# device, this command actually creates it as a side-effect of
# initialising the kernel module rather than explicitly creating
# it.
_log.info("Tunnel device didn't exist; creating.")
futils.check_call(
["ip", "tunnel", "add", IP_IN_IP_DEV_NAME, "mode", "ipip"])
futils.check_call([
"ip", "link", "set", IP_IN_IP_DEV_NAME, "mtu",
str(config.IP_IN_IP_MTU)
])
if not devices.interface_up(IP_IN_IP_DEV_NAME):
_log.info("Tunnel device wasn't up; enabling.")
futils.check_call(["ip", "link", "set", IP_IN_IP_DEV_NAME, "up"])
# Allow an IP address to be added to the tunnel. This is useful to
# allow the host to have an IP on a private IPIP network so that it can
# originate traffic and have it routed correctly.
_log.info("Setting IPIP device IP to %s", config.IP_IN_IP_ADDR)
tunnel_addrs = [config.IP_IN_IP_ADDR] if config.IP_IN_IP_ADDR else []
devices.set_interface_ips(futils.IPV4, IP_IN_IP_DEV_NAME,
set(tunnel_addrs))
_log.info("Configured IPIP device.")
def set_interface_ips(ip_type, interface, ips):
"""
Set the IPs directly assigned to an interface. Idempotent: does not
flap addresses if they're already in place.
:param str ip_type: IP type, either futils.IPV4 or futils.IPV6
:param str interface: Interface name
:param set[IPAddress] ips: The IPs to set or an empty set to remove all
IPs.
"""
assert ip_type in (futils.IPV4,
futils.IPV6), ("Expected an IP type, got %s" % ip_type)
old_ips = list_interface_ips(ip_type, interface)
ips_to_add = ips - old_ips
ips_to_remove = old_ips - ips
ip_cmd = ["ip", "-6"] if ip_type == futils.IPV6 else ["ip"]
for ip in ips_to_remove:
_log.info("Removing IP %s from interface %s", ip, interface)
futils.check_call(ip_cmd + ["addr", "del", str(ip), "dev", interface])
for ip in ips_to_add:
_log.info("Adding IP %s to interface %s", ip, interface)
futils.check_call(ip_cmd + ["addr", "add", str(ip), "dev", interface])
def list_tap_ips(type, tap):
"""
List IP addresses for which there are routes to a given tap interface.
Returns a set with all addresses for which there is a route to the device.
"""
ips = set()
if type == futils.IPV4:
data = futils.check_call(["ip", "route", "list", "dev", tap]).stdout
else:
data = futils.check_call(["ip", "-6", "route", "list", "dev",
tap]).stdout
lines = data.split("\n")
log.debug("Existing routes to %s : %s" % (tap, ",".join(lines)))
for line in lines:
#*********************************************************************#
#* Example of the lines we care about is (having specified the *#
#* device above) : *#
#* 10.11.2.66 proto static scope link *#
#*********************************************************************#
words = line.split()
if len(words) > 1:
ip = words[0]
if common.validate_ipv4_addr(ip) or common.validate_ipv6_addr(ip):
# Looks like an IP address to me
ips.add(words[0])
else:
# Not an IP address; seems odd.
log.warning("No IP address found in line %s for %s" %
(line, tap))
log.debug("Found existing IP addresses : %s" % ips)
return ips
def list_interface_ips(type, interface):
"""
List IP addresses for which there are routes to a given interface.
Returns a set with all addresses for which there is a route to the device.
"""
ips = set()
if type == futils.IPV4:
data = futils.check_call(["ip", "route", "list", "dev",
interface]).stdout
else:
data = futils.check_call(
["ip", "-6", "route", "list", "dev", interface]).stdout
lines = data.split("\n")
log.debug("Existing routes to %s : %s" % (interface, ",".join(lines)))
for line in lines:
#*********************************************************************#
#* Example of the lines we care about is (having specified the *#
#* device above) : *#
#* 10.11.2.66 proto static scope link *#
#*********************************************************************#
words = line.split()
if len(words) > 1:
ip = words[0]
if common.validate_ip_addr(ip, None):
# Looks like an IP address. Note that we here are ignoring
# routes to networks configured when the interface is created.
ips.add(words[0])
log.debug("Found existing IP addresses : %s", ips)
return ips
def add_route(type, ip, tap, mac):
"""
Add a route to a given tap interface (including arp config).
Errors lead to exceptions that are not handled here.
"""
if type == futils.IPV4:
futils.check_call(['arp', '-s', ip, mac, '-i', tap])
futils.check_call(["ip", "route", "add", ip, "dev", tap])
else:
futils.check_call(["ip", "-6", "route", "add", ip, "dev", tap])
def del_route(type, ip, tap):
"""
Delete a route to a given tap interface (including arp config).
Errors lead to exceptions that are not handled here.
"""
if type == futils.IPV4:
futils.check_call(['arp', '-d', ip, '-i', tap])
futils.check_call(["ip", "route", "del", ip, "dev", tap])
else:
futils.check_call(["ip", "-6", "route", "del", ip, "dev", tap])
def list_ipset_names():
"""
List all names of ipsets. Note that this is *not* the same as the ipset
list command which lists contents too (hence the name change).
"""
data = futils.check_call(["ipset", "list"]).stdout
lines = data.split("\n")
names = []
for line in lines:
words = line.split()
if len(words) > 1 and words[0] == "Name:":
names.append(words[1])
return names
def list_ipset_names():
"""
List all names of ipsets. Note that this is *not* the same as the ipset
list command which lists contents too (hence the name change).
"""
data = futils.check_call(["ipset", "list"]).stdout
lines = data.split("\n")
names = []
for line in lines:
words = line.split()
if len(words) > 1 and words[0] == "Name:":
names.append(words[1])
return names
def del_route(ip_type, ip, interface):
"""
Delete a route to a given interface (including arp config).
:param ip_type: Type of IP (IPV4 or IPV6)
:param str ip: IP address
:param str interface: Interface name
:raises FailedSystemCall
"""
if ip_type == futils.IPV4:
futils.check_call(['arp', '-d', ip, '-i', interface])
futils.check_call(["ip", "route", "del", ip, "dev", interface])
else:
futils.check_call(["ip", "-6", "route", "del", ip, "dev", interface])
def _configure_ipip_device(config):
"""Creates and enables the IPIP tunnel device.
:raises FailedSystemCall on failure.
"""
if not devices.interface_exists(IP_IN_IP_DEV_NAME):
# Make sure the IP-in-IP device exists; since we use the global
# device, this command actually creates it as a side-effect of
# initialising the kernel module rather than explicitly creating
# it.
_log.info("Tunnel device didn't exist; creating.")
futils.check_call(["ip", "tunnel", "add", IP_IN_IP_DEV_NAME,
"mode", "ipip"])
futils.check_call(["ip", "link", "set", IP_IN_IP_DEV_NAME, "mtu",
str(config.IP_IN_IP_MTU)])
if not devices.interface_up(IP_IN_IP_DEV_NAME):
_log.info("Tunnel device wasn't up; enabling.")
futils.check_call(["ip", "link", "set", IP_IN_IP_DEV_NAME, "up"])
_log.info("Configured IPIP device.")
def add_route(ip_type, ip, interface, mac):
"""
Add a route to a given interface (including arp config).
Errors lead to exceptions that are not handled here.
Note that we use "ip route replace", since that overrides any imported
routes to the same IP, which might exist in the middle of a migration.
:param ip_type: Type of IP (IPV4 or IPV6)
:param str ip: IP address
:param str interface: Interface name
:param str mac: MAC address or None to skip programming the ARP cache.
:raises FailedSystemCall
"""
if ip_type == futils.IPV4:
if mac:
futils.check_call(['arp', '-s', ip, mac, '-i', interface])
futils.check_call(["ip", "route", "replace", ip, "dev", interface])
else:
futils.check_call(["ip", "-6", "route", "replace", ip, "dev",
interface])
def add_route(ip_type, ip, interface, mac):
"""
Add a route to a given interface (including arp config).
Errors lead to exceptions that are not handled here.
Note that we use "ip route replace", since that overrides any imported
routes to the same IP, which might exist in the middle of a migration.
:param ip_type: Type of IP (IPV4 or IPV6)
:param str ip: IP address
:param str interface: Interface name
:param str mac: MAC address. May not be None unless ip is None.
:raises FailedSystemCall
"""
if mac is None and ip:
raise ValueError("mac must be supplied if ip is provided")
if ip_type == futils.IPV4:
futils.check_call(['arp', '-s', ip, mac, '-i', interface])
futils.check_call(["ip", "route", "replace", ip, "dev", interface])
else:
futils.check_call(["ip", "-6", "route", "replace", ip, "dev", interface])
def install_global_rules(config, v4_filter_updater, v6_filter_updater,
v4_nat_updater):
"""
Set up global iptables rules. These are rules that do not change with
endpoint, and are expected never to change (such as the rules that send all
traffic through the top level Felix chains).
This method therefore :
- ensures that all the required global tables are present;
- applies any changes required.
"""
# The interface matching string; for example, if interfaces start "tap"
# then this string is "tap+".
iface_match = config.IFACE_PREFIX + "+"
# If enabled, create the IP-in-IP device
if config.IP_IN_IP_ENABLED:
_log.info("IP-in-IP enabled, ensuring device exists.")
if not devices.interface_exists(IP_IN_IP_DEV_NAME):
# Make sure the IP-in-IP device exists; since we use the global
# device, this command actually creates it as a side-effect of
# initialising the kernel module rather than explicitly creating
# it.
_log.info("Tunnel device didn't exist; creating.")
futils.check_call(["ip", "tunnel", "add", IP_IN_IP_DEV_NAME,
"mode", "ipip"])
if not devices.interface_up(IP_IN_IP_DEV_NAME):
_log.info("Tunnel device wasn't up; enabling.")
futils.check_call(["ip", "link", "set", IP_IN_IP_DEV_NAME, "up"])
# The IPV4 nat table first. This must have a felix-PREROUTING chain.
nat_pr = []
if config.METADATA_IP is not None:
# Need to expose the metadata server on a link-local.
# DNAT tcp -- any any anywhere 169.254.169.254
# tcp dpt:http to:127.0.0.1:9697
nat_pr.append("--append " + CHAIN_PREROUTING + " "
"--protocol tcp "
"--dport 80 "
"--destination 169.254.169.254/32 "
"--jump DNAT --to-destination %s:%s" %
(config.METADATA_IP, config.METADATA_PORT))
v4_nat_updater.rewrite_chains({CHAIN_PREROUTING: nat_pr}, {}, async=False)
v4_nat_updater.ensure_rule_inserted(
"PREROUTING --jump %s" % CHAIN_PREROUTING, async=False)
# Now the filter table. This needs to have calico-filter-FORWARD and
# calico-filter-INPUT chains, which we must create before adding any
# rules that send to them.
for iptables_updater, hosts_set in [(v4_filter_updater, HOSTS_IPSET_V4),
# FIXME support IP-in-IP for IPv6.
(v6_filter_updater, None)]:
if hosts_set and config.IP_IN_IP_ENABLED:
hosts_set_name = hosts_set.set_name
hosts_set.ensure_exists()
else:
hosts_set_name = None
if iptables_updater is v4_filter_updater:
input_chain, input_deps = _build_input_chain(
iface_match=iface_match,
metadata_addr=config.METADATA_IP,
metadata_port=config.METADATA_PORT,
dhcp_src_port=68,
dhcp_dst_port=67,
ipv6=False,
default_action=config.DEFAULT_INPUT_CHAIN_ACTION,
hosts_set_name=hosts_set_name,
)
else:
input_chain, input_deps = _build_input_chain(
iface_match=iface_match,
metadata_addr=None,
metadata_port=None,
dhcp_src_port=546,
dhcp_dst_port=547,
ipv6=True,
default_action=config.DEFAULT_INPUT_CHAIN_ACTION,
hosts_set_name=hosts_set_name,
)
forward_chain, forward_deps = _build_forward_chain(iface_match)
iptables_updater.rewrite_chains(
{
CHAIN_FORWARD: forward_chain,
CHAIN_INPUT: input_chain
},
{
CHAIN_FORWARD: forward_deps,
CHAIN_INPUT: input_deps,
},
async=False)
iptables_updater.ensure_rule_inserted(
"INPUT --jump %s" % CHAIN_INPUT,
async=False)
iptables_updater.ensure_rule_inserted(
"FORWARD --jump %s" % CHAIN_FORWARD,
async=False)