class LSDB(object):
BASE_NET = ip_network(CFG.get(DEFAULTSECT, 'base_net'))
def __init__(self):
self.BASE_NET = ip_network(CFG.get(DEFAULTSECT, 'base_net'))
self.private_address_network = ip_network(CFG.get(DEFAULTSECT,
'private_net'))
try:
with open(CFG.get(DEFAULTSECT, 'private_ips'), 'r') as f:
self.private_address_binding = json.load(f)
self.router_private_address = {}
for subnets in self.private_address_binding.itervalues():
for rid, ip in subnets.iteritems():
try:
iplist = self.router_private_address[rid]
except KeyError:
iplist = self.router_private_address[rid] = []
# Enable single private address as string
if isinstance(ip, str):
ip = [ip]
iplist.extend(ip)
except Exception as e:
log.warning('Incorrect private IP addresses binding file')
log.warning(str(e))
self.private_address_binding = {}
self.router_private_address = {}
self.last_line = ''
self.leader_watchdog = None
self.transaction = None
self.graph = IGPGraph()
self.routers = {} # router-id : lsa
self.networks = {} # DR IP : lsa
self.ext_networks = {} # (router-id, dest) : lsa
self.controllers = defaultdict(list) # controller nr : ip_list
self.listener = {}
self.keep_running = True
self.queue = Queue()
self.processing_thread = Thread(target=self.process_lsa,
name="lsa_processing_thread")
self.processing_thread.setDaemon(True)
self.processing_thread.start()
def set_leader_watchdog(self, wd):
self.leader_watchdog = wd
def get_leader(self):
return min(self.controllers.iterkeys()) if self.controllers else None
def stop(self):
for l in self.listener.values():
l.session.stop()
self.keep_running = False
self.queue.put('')
def lsdb(self, lsa):
if lsa.TYPE == RouterLSA.TYPE:
return self.routers
elif lsa.TYPE == NetworkLSA.TYPE:
return self.networks
elif lsa.TYPE == ASExtLSA.TYPE:
return self.ext_networks
def register_change_listener(self, listener):
try:
del self.listener[listener]
log.info('Shapeshifter disconnected.')
except KeyError:
log.info('Shapeshifter connected.')
l = ProxyCloner(ShapeshifterProxy, listener)
self.listener[listener] = l
l.bootstrap_graph(graph=[(u, v, d.get('metric', -1))
for u, v, d in self.graph.edges(data=True)
],
node_properties={n: data for n, data in
self.graph.nodes_iter(data=True)
})
@staticmethod
def extract_lsa_properties(lsa_part):
d = {}
for prop in lsa_part.split(SEP_INTER_FIELD):
if not prop:
continue
key, val = prop.split(SEP_INTRA_FIELD)
d[key] = val
return d
def commit_change(self, line):
# Check that this is not a duplicate of a previous update ...
if self.last_line == line:
return
self.queue.put(line)
def forwarding_address_of(self, src, dst):
"""
Return the forwarding address for a src, dst pair. If src is specified, return
the private 'link-local' address of the src-dst link, otherwise return a 'public'
IP belonging to dst
:param src: the source node of the link towards the FA, possibly null
:param dst: the node owning the forwarding address
:return: forwarding address (str) or None if no compatible address was found
"""
try:
return self.graph[src][dst]['dst_address'] if src \
else self.graph[dst][self.graph.neighbors(dst)[0]]['src_address']
except KeyError:
log.debug('%s-%s not found in graph', src, dst)
return None
def remove_lsa(self, lsa):
lsdb = self.lsdb(lsa)
try:
del lsdb[lsa.key()]
except KeyError:
pass
def add_lsa(self, lsa):
lsdb = self.lsdb(lsa)
lsdb[lsa.key()] = lsa
def process_lsa(self):
while self.keep_running:
commit = False
try:
line = self.queue.get(timeout=5)
if not line:
self.queue.task_done()
continue
# Start parsing the LSA log
action, lsa_info = line.split(SEP_ACTION)
if action == BEGIN:
self.transaction = Transaction()
elif action == COMMIT:
if self.transaction:
self.transaction.commit(self)
self.transaction = None
commit = True
else:
lsa_parts = [self.extract_lsa_properties(part)
for part in lsa_info.split(SEP_GROUP) if part]
lsa = LSA.parse(LSAHeader.parse(lsa_parts.pop(0)),
lsa_parts)
log.debug('Parsed %s: %s', action, lsa)
if action == REM:
if not self.transaction:
self.remove_lsa(lsa)
else:
self.transaction.remove_lsa(lsa)
elif action == ADD:
if not self.transaction:
self.add_lsa(lsa)
else:
self.transaction.add_lsa(lsa)
if lsa.push_update_on_remove() or not action == REM:
commit = True
self.queue.task_done()
except Empty:
if self.transaction:
log.debug('Splitting transaction due to timeout')
self.transaction.commit(self)
self.transaction = Transaction()
commit = True
if commit:
# Update graph accordingly
new_graph = self.build_graph()
# Compute graph difference and update it
self.update_graph(new_graph)
def __str__(self):
strs = [str(lsa) for lsa in chain(self.routers.values(),
self.networks.values(),
self.ext_networks.values())]
strs.insert(0, '* LSDB Content [%d]:' % len(strs))
return '\n'.join(strs)
def build_graph(self):
self.controllers.clear()
new_graph = IGPGraph()
# Rebuild the graph from the LSDB
for lsa in chain(self.routers.values(),
self.networks.values(),
self.ext_networks.values()):
lsa.apply(new_graph, self)
# Contract all IPs to their respective router-id
for lsa in self.routers.values():
lsa.contract_graph(new_graph, self.router_private_address.get(
lsa.routerid, []))
# Figure out the controllers layout
controller_prefix = CFG.getint(DEFAULTSECT, 'controller_prefixlen')
# Group by controller and log them
for ip in new_graph.nodes_iter():
try:
addr = ip_address(ip)
except ValueError:
continue # Have a prefix
if addr in self.BASE_NET:
"""1. Compute address diff to remove base_net
2. Right shift to remove host bits
3. Mask with controller mask
"""
id = (((int(addr) - int(self.BASE_NET.network_address)) >>
self.BASE_NET.max_prefixlen - controller_prefix) &
((1 << controller_prefix) - 1))
self.controllers[id].append(ip)
# Contract them on the graph
for id, ips in self.controllers.iteritems():
cname = 'C_%s' % id
new_graph.add_controller(cname)
new_graph.contract(cname, ips)
# Remove generated self loops
new_graph.remove_edges_from(new_graph.selfloop_edges())
self.apply_secondary_addresses(new_graph)
return new_graph
def update_graph(self, new_graph):
self.leader_watchdog.check_leader(self.get_leader())
added_edges = new_graph.difference(self.graph)
removed_edges = self.graph.difference(new_graph)
node_prop_diff = {n: data
for n, data in new_graph.nodes_iter(data=True)
if n not in self.graph or
(data.viewitems() - self.graph.node[n].viewitems())}
# Propagate differences
if added_edges or removed_edges or node_prop_diff:
log.debug('Pushing changes')
for u, v in added_edges:
self.for_all_listeners('add_edge',
u, v, metric=new_graph.metric(u, v))
for u, v in removed_edges:
self.for_all_listeners('remove_edge', u, v)
if node_prop_diff:
self.for_all_listeners('update_node_properties',
**node_prop_diff)
if CFG.getboolean(DEFAULTSECT, 'draw_graph'):
new_graph.draw(CFG.get(DEFAULTSECT, 'graph_loc'))
self.graph = new_graph
log.info('LSA update yielded +%d -%d edges changes, '
'%d node property changes', len(added_edges),
len(removed_edges), len(node_prop_diff))
self.for_all_listeners('commit')
def for_all_listeners(self, funcname, *args, **kwargs):
"""Apply funcname to all listeners"""
for i in self.listener.itervalues():
getattr(i, funcname)(*args, **kwargs)
def apply_secondary_addresses(self, graph):
for subnet in self.private_address_binding.itervalues():
for dst, ip in subnet.iteritems():
for src in subnet.iterkeys():
if src == dst:
continue
try:
graph[src][dst]['dst_address'] = ip
except KeyError:
pass
class LSDB(object):
def __init__(self):
self.BASE_NET = ip_network(CFG.get(DEFAULTSECT, 'base_net'))
self.private_addresses = PrivateAddressStore(CFG.get(DEFAULTSECT,
'private_ips'))
self.last_line = ''
self.leader_watchdog = None
self.transaction = False
self.uncommitted_changes = 0
self.graph = IGPGraph()
self._lsdb = {NetworkLSA.TYPE: {},
RouterLSA.TYPE: {},
ASExtLSA.TYPE: {}}
self.controllers = defaultdict(list) # controller nr : ip_list
self.listener = {}
self.keep_running = True
self.queue = Queue()
self.processing_thread = start_daemon_thread(
target=self.process_lsa, name='lsa processing thread')
@property
def routers(self):
return self._lsdb[RouterLSA.TYPE]
@property
def networks(self):
return self._lsdb[NetworkLSA.TYPE]
@property
def ext_networks(self):
return self._lsdb[ASExtLSA.TYPE]
def set_leader_watchdog(self, wd):
self.leader_watchdog = wd
def get_leader(self):
return min(self.controllers.iterkeys()) if self.controllers else None
def stop(self):
for l in self.listener.values():
l.session.stop()
self.keep_running = False
self.queue.put('')
def lsdb(self, lsa):
return self._lsdb.get(lsa.TYPE, None)
def register_change_listener(self, listener):
try:
del self.listener[listener]
log.info('Shapeshifter disconnected.')
except KeyError:
log.info('Shapeshifter connected.')
l = ProxyCloner(ShapeshifterProxy, listener)
self.listener[listener] = l
l.bootstrap_graph(graph=[(u, v, d)
for u, v, d in self.graph.export_edges()
],
node_properties={n: data for n, data in
self.graph.nodes_iter(data=True)
})
def commit_change(self, line):
# Check that this is not a duplicate of a previous update ...
if self.last_line == line or not line:
return
self.queue.put(line)
def forwarding_address_of(self, src, dst):
"""
Return the forwarding address for a src, dst pair.
If src is specified, return the private 'link-local' address of
the src-dst link, otherwise return a 'public' IP belonging to dst
:param src: the source node of the link towards the FA, possibly null
:param dst: the node owning the forwarding address
:return: forwarding address (str)
or None if no compatible address was found
"""
# If we have a src address, we want the set of private IPs
# Otherwise we want any IP of dst
if src:
try:
return self.graph[src][dst]['dst_address']
except KeyError as e:
log.error("Couldn't resolve local forwarding of %s-%s, missing"
" key %s", src, dst, e)
else:
try:
data = filter(lambda v: v is not None,
(self.graph[dst][succ].get('src_address', None)
for succ in self.graph.successors_iter(dst)))
if data:
return min(data)
log.error("Cannot use %s as nexthop as it has no physical "
"link to other routers!", dst)
except KeyError:
log.error("Couldn't find nexthop %s when resolving global "
"forwarding address", dst)
return None
def remove_lsa(self, lsa):
lsdb = self.lsdb(lsa)
try:
del lsdb[lsa.key()]
except (KeyError, TypeError): # LSA not found, lsdb is None
pass
def add_lsa(self, lsa):
lsdb = self.lsdb(lsa)
try:
lsdb[lsa.key()] = lsa
except TypeError: # LSDB is None
pass
def get_current_seq_number(self, lsa):
try:
return self.lsdb(lsa)[lsa.key()].seqnum
except (KeyError, TypeError): # LSA not found, LSDB is None
return None
def is_old_seqnum(self, lsa):
"""Return whether the lsa is older than the copy in the lsdb is any"""
c_seqnum = self.get_current_seq_number(lsa)
new_seqnum = lsa.seqnum
return (c_seqnum and # LSA already present in the LSDB
not is_newer_seqnum(new_seqnum, c_seqnum) and
# We allow duplicate as they are used to flush LSAs
new_seqnum != c_seqnum)
def handle_lsa_line(self, line):
"""We received a line describing an lsa, handle it"""
action, lsa_info = line.split(SEP_ACTION)
if action == BEGIN:
self.start_transaction()
elif action == COMMIT:
self.reset_transaction()
else: # ADD/REM LSA messages
lsa = parse_lsa(lsa_info)
log.debug('Parsed %s: %s [%d]', action, lsa, lsa.seqnum)
# Sanity checks
if self.is_old_seqnum(lsa):
log.debug("OLD seqnum for LSA, ignoring it...")
action = None
# Perform the update if it is still applicable
if action == REM:
self.remove_lsa(lsa)
self.uncommitted_changes += 1
elif action == ADD:
self.add_lsa(lsa)
self.uncommitted_changes += 1
def process_lsa(self):
"""Parse new LSAs, and update the graph if needed"""
while self.keep_running:
try:
line = self.queue.get(timeout=5)
self.queue.task_done()
if not line:
continue
self.handle_lsa_line(line)
except Empty:
self.reset_transaction()
if (self.queue.empty() and # Try to empty the queue before update
not self.transaction and self.uncommitted_changes):
self.commit()
def reset_transaction(self):
"""Reset the transaction"""
self.transaction = False
def start_transaction(self):
"""Record a new LSA transaction"""
self.transaction = True
def commit(self):
"""Updates have been made on the LSDB, update the graph"""
# Update graph accordingly
new_graph = self.build_graph()
# Compute graph difference and update it
self.update_graph(new_graph)
self.uncommitted_changes = 0
def __str__(self):
strs = [str(lsa) for lsa in chain(self.routers.values(),
self.networks.values(),
self.ext_networks.values())]
strs.insert(0, '* LSDB Content [%d]:' % len(strs))
return '\n'.join(strs)
def build_graph(self):
self.controllers.clear()
new_graph = IGPGraph()
# Rebuild the graph from the LSDB
for lsa in chain(self.routers.itervalues(),
self.networks.itervalues(),
self.ext_networks.itervalues()):
if is_expired_lsa(lsa):
log.debug("LSA %s is too old (%d) ignoring it!",
lsa, lsa.age)
else:
lsa.apply(new_graph, self)
# Contract all IPs to their respective router-id
for rlsa in self.routers.itervalues():
rlsa.contract_graph(new_graph,
self.private_addresses
.addresses_of(rlsa.routerid))
# Figure out the controllers layout
controller_prefix = CFG.getint(DEFAULTSECT, 'controller_prefixlen')
# Group by controller and log them
for ip in new_graph.nodes_iter():
try:
addr = ip_address(ip)
except ValueError:
continue # Have a prefix
if addr in self.BASE_NET:
"""1. Compute address diff to remove base_net
2. Right shift to remove host bits
3. Mask with controller mask"""
cid = (((int(addr) - int(self.BASE_NET.network_address)) >>
self.BASE_NET.max_prefixlen - controller_prefix) &
((1 << controller_prefix) - 1))
self.controllers[cid].append(ip)
# Contract them on the graph
for id, ips in self.controllers.iteritems():
cname = 'C_%s' % id
new_graph.add_controller(cname)
new_graph.contract(cname, ips)
# Remove generated self loops
new_graph.remove_edges_from(new_graph.selfloop_edges())
self.apply_secondary_addresses(new_graph)
return new_graph
def update_graph(self, new_graph):
self.leader_watchdog.check_leader(self.get_leader())
added_edges = new_graph.difference(self.graph)
removed_edges = self.graph.difference(new_graph)
node_prop_diff = {n: data
for n, data in new_graph.nodes_iter(data=True)
if n not in self.graph or
(data.viewitems() - self.graph.node[n].viewitems())}
# Propagate differences
if added_edges or removed_edges or node_prop_diff:
log.debug('Pushing changes')
for u, v in added_edges:
self.for_all_listeners('add_edge', u, v,
new_graph.export_edge_data(u, v))
for u, v in removed_edges:
self.for_all_listeners('remove_edge', u, v)
if node_prop_diff:
self.for_all_listeners('update_node_properties',
**node_prop_diff)
if CFG.getboolean(DEFAULTSECT, 'draw_graph'):
new_graph.draw(CFG.get(DEFAULTSECT, 'graph_loc'))
self.graph = new_graph
log.info('LSA update yielded +%d -%d edges changes, '
'%d node property changes', len(added_edges),
len(removed_edges), len(node_prop_diff))
self.for_all_listeners('commit')
def for_all_listeners(self, funcname, *args, **kwargs):
"""Apply funcname to all listeners"""
f = methodcaller(funcname, *args, **kwargs)
map(f, self.listener.itervalues())
def apply_secondary_addresses(self, graph):
for src, dst in graph.router_links:
try:
graph[src][dst]['dst_address'] = self.private_addresses\
.addresses_of(dst, src)
except KeyError:
log.debug('%(src)-%(dst)s does not yet exists on the graph'
', ignoring private addresses.', locals())
pass
class LSDB(object):
def __init__(self):
self.BASE_NET = ip_network(CFG.get(DEFAULTSECT, 'base_net'))
self.private_addresses = PrivateAddressStore(
CFG.get(DEFAULTSECT, 'private_ips'))
self.last_line = ''
self.leader_watchdog = None
self.transaction = None
self.graph = IGPGraph()
self.routers = {} # router-id : lsa
self.networks = {} # DR IP : lsa
self.ext_networks = {} # (router-id, dest) : lsa
self.controllers = defaultdict(list) # controller nr : ip_list
self.listener = {}
self.keep_running = True
self.queue = Queue()
self.processing_thread = Thread(target=self.process_lsa,
name="lsa_processing_thread")
self.processing_thread.setDaemon(True)
self.processing_thread.start()
def set_leader_watchdog(self, wd):
self.leader_watchdog = wd
def get_leader(self):
return min(self.controllers.iterkeys()) if self.controllers else None
def stop(self):
for l in self.listener.values():
l.session.stop()
self.keep_running = False
self.queue.put('')
def lsdb(self, lsa):
if lsa.TYPE == RouterLSA.TYPE:
return self.routers
elif lsa.TYPE == NetworkLSA.TYPE:
return self.networks
elif lsa.TYPE == ASExtLSA.TYPE:
return self.ext_networks
def register_change_listener(self, listener):
try:
del self.listener[listener]
log.info('Shapeshifter disconnected.')
except KeyError:
log.info('Shapeshifter connected.')
l = ProxyCloner(ShapeshifterProxy, listener)
self.listener[listener] = l
l.bootstrap_graph(
graph=[(u, v, d) for u, v, d in self.graph.export_edges()],
node_properties={
n: data
for n, data in self.graph.nodes_iter(data=True)
})
@staticmethod
def extract_lsa_properties(lsa_part):
d = {}
for prop in lsa_part.split(SEP_INTER_FIELD):
if not prop:
continue
key, val = prop.split(SEP_INTRA_FIELD)
d[key] = val
return d
def commit_change(self, line):
# Check that this is not a duplicate of a previous update ...
if self.last_line == line:
return
self.queue.put(line)
def forwarding_address_of(self, src, dst):
"""
Return the forwarding address for a src, dst pair.
If src is specified, return the private 'link-local' address of
the src-dst link, otherwise return a 'public' IP belonging to dst
:param src: the source node of the link towards the FA, possibly null
:param dst: the node owning the forwarding address
:return: forwarding address (str)
or None if no compatible address was found
"""
# If we have a src address, we want the set of private IPs
# Otherwise we want any IP of dst
u, v, key = ((src, dst, 'dst_address') if src else
(dst, self.graph.neighbors(dst)[0], 'src_address'))
try:
edge = self.graph[u][v]
except KeyError:
log.error(
'%s-%s not found in graph when resolving '
'forwarding address of (%s,%s)', u, v, src, dst)
return None
try:
return edge[key]
except KeyError:
log.error(
'%s not found in the properties of edge %s-%s '
'when resolving forwarding address of (%s, %s)\n%s', key, u, v,
src, dst, edge)
return None
def remove_lsa(self, lsa):
lsdb = self.lsdb(lsa)
try:
del lsdb[lsa.key()]
except KeyError:
pass
except TypeError:
pass # LSDB is None
def add_lsa(self, lsa):
lsdb = self.lsdb(lsa)
try:
lsdb[lsa.key()] = lsa
except TypeError:
pass # LSDB is None
def process_lsa(self):
while self.keep_running:
commit = False
try:
line = self.queue.get(timeout=5)
if not line:
self.queue.task_done()
continue
# Start parsing the LSA log
action, lsa_info = line.split(SEP_ACTION)
if action == BEGIN:
self.transaction = Transaction()
elif action == COMMIT:
if self.transaction:
self.transaction.commit(self)
self.transaction = None
commit = True
else:
lsa_parts = [
self.extract_lsa_properties(part)
for part in lsa_info.split(SEP_GROUP) if part
]
lsa = LSA.parse(LSAHeader.parse(lsa_parts.pop(0)),
lsa_parts)
log.debug('Parsed %s: %s', action, lsa)
provider = self.transaction if self.transaction else self
if action == REM:
provider.remove_lsa(lsa)
elif action == ADD:
provider.add_lsa(lsa)
if lsa.push_update_on_remove() or not action == REM:
commit = True
self.queue.task_done()
except Empty:
if self.transaction:
log.debug('Splitting transaction due to timeout')
self.transaction.commit(self)
self.transaction = Transaction()
commit = True
if commit:
# Update graph accordingly
new_graph = self.build_graph()
# Compute graph difference and update it
self.update_graph(new_graph)
def __str__(self):
strs = [
str(lsa)
for lsa in chain(self.routers.values(), self.networks.values(),
self.ext_networks.values())
]
strs.insert(0, '* LSDB Content [%d]:' % len(strs))
return '\n'.join(strs)
def build_graph(self):
self.controllers.clear()
new_graph = IGPGraph()
# Rebuild the graph from the LSDB
for lsa in chain(self.routers.itervalues(), self.networks.itervalues(),
self.ext_networks.itervalues()):
lsa.apply(new_graph, self)
# Contract all IPs to their respective router-id
for rlsa in self.routers.itervalues():
rlsa.contract_graph(
new_graph, self.private_addresses.addresses_of(rlsa.routerid))
# Figure out the controllers layout
controller_prefix = CFG.getint(DEFAULTSECT, 'controller_prefixlen')
# Group by controller and log them
for ip in new_graph.nodes_iter():
try:
addr = ip_address(ip)
except ValueError:
continue # Have a prefix
if addr in self.BASE_NET:
"""1. Compute address diff to remove base_net
2. Right shift to remove host bits
3. Mask with controller mask"""
cid = (((int(addr) - int(self.BASE_NET.network_address)) >>
self.BASE_NET.max_prefixlen - controller_prefix) &
((1 << controller_prefix) - 1))
self.controllers[cid].append(ip)
# Contract them on the graph
for id, ips in self.controllers.iteritems():
cname = 'C_%s' % id
new_graph.add_controller(cname)
new_graph.contract(cname, ips)
# Remove generated self loops
new_graph.remove_edges_from(new_graph.selfloop_edges())
self.apply_secondary_addresses(new_graph)
return new_graph
def update_graph(self, new_graph):
self.leader_watchdog.check_leader(self.get_leader())
added_edges = new_graph.difference(self.graph)
removed_edges = self.graph.difference(new_graph)
node_prop_diff = {
n: data
for n, data in new_graph.nodes_iter(data=True)
if n not in self.graph or (data.viewitems() -
self.graph.node[n].viewitems())
}
# Propagate differences
if added_edges or removed_edges or node_prop_diff:
log.debug('Pushing changes')
for u, v in added_edges:
self.for_all_listeners('add_edge', u, v,
new_graph.export_edge_data(u, v))
for u, v in removed_edges:
self.for_all_listeners('remove_edge', u, v)
if node_prop_diff:
self.for_all_listeners('update_node_properties',
**node_prop_diff)
if CFG.getboolean(DEFAULTSECT, 'draw_graph'):
new_graph.draw(CFG.get(DEFAULTSECT, 'graph_loc'))
self.graph = new_graph
log.info(
'LSA update yielded +%d -%d edges changes, '
'%d node property changes', len(added_edges),
len(removed_edges), len(node_prop_diff))
self.for_all_listeners('commit')
def for_all_listeners(self, funcname, *args, **kwargs):
"""Apply funcname to all listeners"""
for i in self.listener.itervalues():
getattr(i, funcname)(*args, **kwargs)
def apply_secondary_addresses(self, graph):
for src, dst in graph.router_links:
try:
graph[src][dst]['dst_address'] = self.private_addresses\
.addresses_of(dst, src)
except KeyError:
log.debug(
'%(src)-%(dst)s does not yet exists on the graph'
', ignoring private addresses.', locals())
pass
