def test_no_subscribers(self):
"""Tests case where we have no subscribers left"""
imp_graph = RideD.get_importance_graph(self.tree, [], self.root)
for u, v, imp in imp_graph.edges(data=RideD.IMPORTANCE_ATTRIBUTE_NAME):
self.assertEqual(imp_graph[u][v][RideD.IMPORTANCE_ATTRIBUTE_NAME],
0)
self.assertEqual(self.tree.number_of_edges(),
imp_graph.number_of_edges())
def on_start(self):
"""
Build and configure the RideD middleware
"""
# TODO: probably run this in the background?
if self.rided is not None:
if not BUILD_RIDED_IN_INIT:
assert isinstance(self.rided, dict)
self.rided = RideD(**self.rided)
assert isinstance(self.rided, RideD)
# Rather than periodically update the topology, which in our experiments would result in perfectly routing
# around all the failures due to 0-latency control plane, we just update it once for now...
self.timed_call(self.maintenance_interval, self.__class__.__maintain_topology, repeat=False)
# self.timed_call(self.maintenance_interval, self.__class__.__maintain_topology, repeat=True)
super(RideDEventSink, self).on_start()
def test_basic(self):
"""Basic test case where all leaves are subscribers"""
imp_graph = RideD.get_importance_graph(self.tree, self.subscribers,
self.root)
self.assertEqual(imp_graph[0][1][RideD.IMPORTANCE_ATTRIBUTE_NAME], 3)
self.assertEqual(imp_graph[1][3][RideD.IMPORTANCE_ATTRIBUTE_NAME], 2)
for u, v in ((0, 7), (1, 2), (3, 4), (3, 5), (5, 6), (7, 8), (8, 9)):
self.assertEqual(imp_graph[u][v][RideD.IMPORTANCE_ATTRIBUTE_NAME],
1)
def test_subscriber_subset(self):
"""Tests case where not all leaf nodes are subscribers"""
imp_graph = RideD.get_importance_graph(self.tree, [2, 4, 6], self.root)
self.assertEqual(imp_graph[0][1][RideD.IMPORTANCE_ATTRIBUTE_NAME], 3)
self.assertEqual(imp_graph[1][3][RideD.IMPORTANCE_ATTRIBUTE_NAME], 2)
for u, v in ((1, 2), (3, 4), (3, 5), (5, 6)):
self.assertEqual(imp_graph[u][v][RideD.IMPORTANCE_ATTRIBUTE_NAME],
1)
for u, v in ((0, 7), (7, 8), (8, 9)):
self.assertEqual(imp_graph[u][v][RideD.IMPORTANCE_ATTRIBUTE_NAME],
0)
def test_internal_subscribers(self):
"""Tests case where some subscribers are non-leaf nodes"""
subs = [2, 4, 6, 3]
imp_graph = RideD.get_importance_graph(self.tree, subs, self.root)
# Only these two links should have increased 'importance'
self.assertEqual(imp_graph[0][1][RideD.IMPORTANCE_ATTRIBUTE_NAME], 4)
self.assertEqual(imp_graph[1][3][RideD.IMPORTANCE_ATTRIBUTE_NAME], 3)
for u, v in ((1, 2), (3, 4), (3, 5), (5, 6)):
self.assertEqual(imp_graph[u][v][RideD.IMPORTANCE_ATTRIBUTE_NAME],
1)
for u, v in ((0, 7), (7, 8), (8, 9)):
self.assertEqual(imp_graph[u][v][RideD.IMPORTANCE_ATTRIBUTE_NAME],
0)
def setUp(self):
# Our test topology is a basic campus network topology (constructed with the campus_topo_gen.py script) with:
# 4 core, 2 buildings per core, 2 hosts/building, and 2 inter-building links;
# TODO: see example diagram to visualize the relevant parts
topo_file = os.path.join(os.path.split(__file__)[0], 'test_topo.json')
self.topology = NetworkxSdnTopology(topo_file)
self.root = self.topology.get_servers()[0]
# self.topology.draw()
# set up some manual MDMTs by just building networkx Graphs using collections of links
self.ntrees = 4
self.mdmts = [ # tree1
nx.Graph(
((self.root, 'c0'), ('c0', 'c1'), ('c1', 'b0'),
('b0', 'h0-b0'), ('c0', 'c2'), ('c2', 'b1'), ('b1', 'h0-b1'),
('b1', 'h1-b1'), ('c2', 'b3'), ('b3', 'h0-b3'))),
# tree2
nx.Graph(
((self.root, 'c3'), ('c3', 'c2'), ('c2', 'b1'), ('b1',
'h0-b1'),
('b1', 'h1-b1'), ('b1', 'b0'), ('b0', 'h0-b0'), ('b0', 'c1'),
('c1', 'b5'), ('b5', 'b3'), ('b3', 'h0-b3'))),
# tree3
nx.Graph(((self.root, 'c0'), ('c0', 'c1'), ('c1', 'b0'),
('b0', 'h0-b0'), ('b0', 'b1'), ('b1', 'h0-b1'),
('b1', 'h1-b1'), (self.root, 'c3'), ('c3', 'c2'),
('c2', 'b3'), ('b3', 'h0-b3'))),
# tree4
nx.Graph(((self.root, 'c0'), ('c0', 'c1'), ('c1', 'b0'),
('b0', 'h0-b0'), ('c2', 'b1'), ('b1', 'h0-b1'),
('b1', 'h1-b1'), (self.root, 'c3'), ('c3', 'c2'),
('c2', 'b3'), ('b3', 'h0-b3')))
]
# self.topology.draw_multicast_trees(self.mdmts[2:3])
mdmt_addresses = ['tree%d' % (d + 1) for d in range(self.ntrees)]
self.rided = RideD(
topology_mgr=self.topology,
ntrees=self.ntrees,
dpid=self.root,
addresses=mdmt_addresses,
tree_choosing_heuristic=RideD.MAX_LINK_IMPORTANCE,
# This test callback notifies us of subscribers reached and ensures the right MDMT was selected
alert_sending_callback=self.__send_alert_test_callback)
# XXX: manually set the MDMTs to avoid calling RideD.update(), which will try to run SDN operations in addition
# to creating the MDMTs using the construction algorithms
self.rided.mdmts[ALERT_TOPIC] = self.mdmts
for mdmt, addr in zip(self.mdmts, mdmt_addresses):
self.rided.set_address_for_mdmt(mdmt, addr)
# set up manual publisher routes
self.publishers = ['h1-b5', 'h1-b1']
self.publisher_routes = [['h1-b5', 'b5', 'c1', 'c0', self.root],
['h1-b1', 'b1', 'c2', 'c3', self.root]]
for pub_route in self.publisher_routes:
self.rided.set_publisher_route(pub_route[0], pub_route)
for pub in self.publishers:
self.rided.notify_publication(pub)
# register the subscribers
self.subscribers = ['h0-b0', 'h0-b1', 'h1-b1', 'h0-b3']
for sub in self.subscribers:
self.rided.add_subscriber(sub, ALERT_TOPIC)
# We expect the MDMTs to be selected (via 'importance' policy) in this order for the following tests...
self.expected_mdmts = [('tree4', ), ('tree2', ), ('tree3', ),
('tree1', ), ('tree2', ),
('tree1', 'tree3', 'tree4')]
# ... based on these subscribers being reached during each attempt.
self.subs_reached_at_attempt = [
('h0-b1', 'h1-b1'), #0
tuple(),
tuple(),
tuple(), # 1-3 no responses...
('h0-b3', ), #4
('h0-b0', ) #5 ; all done!
]
# NOTES about the test cases:
# NOTE: we only do these tests for 'importance' since the others will have a tie between tree3/4
# we should choose tree2 second due to update about subs reached...
# because AlertContext tracks trees tried we should use tree3 third
# furthermore, we should lastly try tree1 even though it had lowest importance!
# then, we should try tree2 as the highest current importance after a notification since we've tried all of them
# finally, since we have a tie among all the others
self.attempt_num = 0
self.alert = self.rided._make_new_alert(ALERT_MSG, ALERT_TOPIC)
class TestMdmtSelection(unittest.TestCase):
"""Tests the RideD algorithms but NOT the SDN mechanisms"""
def setUp(self):
# Our test topology is a basic campus network topology (constructed with the campus_topo_gen.py script) with:
# 4 core, 2 buildings per core, 2 hosts/building, and 2 inter-building links;
# TODO: see example diagram to visualize the relevant parts
topo_file = os.path.join(os.path.split(__file__)[0], 'test_topo.json')
self.topology = NetworkxSdnTopology(topo_file)
self.root = self.topology.get_servers()[0]
# self.topology.draw()
# set up some manual MDMTs by just building networkx Graphs using collections of links
self.ntrees = 4
self.mdmts = [ # tree1
nx.Graph(
((self.root, 'c0'), ('c0', 'c1'), ('c1', 'b0'),
('b0', 'h0-b0'), ('c0', 'c2'), ('c2', 'b1'), ('b1', 'h0-b1'),
('b1', 'h1-b1'), ('c2', 'b3'), ('b3', 'h0-b3'))),
# tree2
nx.Graph(
((self.root, 'c3'), ('c3', 'c2'), ('c2', 'b1'), ('b1',
'h0-b1'),
('b1', 'h1-b1'), ('b1', 'b0'), ('b0', 'h0-b0'), ('b0', 'c1'),
('c1', 'b5'), ('b5', 'b3'), ('b3', 'h0-b3'))),
# tree3
nx.Graph(((self.root, 'c0'), ('c0', 'c1'), ('c1', 'b0'),
('b0', 'h0-b0'), ('b0', 'b1'), ('b1', 'h0-b1'),
('b1', 'h1-b1'), (self.root, 'c3'), ('c3', 'c2'),
('c2', 'b3'), ('b3', 'h0-b3'))),
# tree4
nx.Graph(((self.root, 'c0'), ('c0', 'c1'), ('c1', 'b0'),
('b0', 'h0-b0'), ('c2', 'b1'), ('b1', 'h0-b1'),
('b1', 'h1-b1'), (self.root, 'c3'), ('c3', 'c2'),
('c2', 'b3'), ('b3', 'h0-b3')))
]
# self.topology.draw_multicast_trees(self.mdmts[2:3])
mdmt_addresses = ['tree%d' % (d + 1) for d in range(self.ntrees)]
self.rided = RideD(
topology_mgr=self.topology,
ntrees=self.ntrees,
dpid=self.root,
addresses=mdmt_addresses,
tree_choosing_heuristic=RideD.MAX_LINK_IMPORTANCE,
# This test callback notifies us of subscribers reached and ensures the right MDMT was selected
alert_sending_callback=self.__send_alert_test_callback)
# XXX: manually set the MDMTs to avoid calling RideD.update(), which will try to run SDN operations in addition
# to creating the MDMTs using the construction algorithms
self.rided.mdmts[ALERT_TOPIC] = self.mdmts
for mdmt, addr in zip(self.mdmts, mdmt_addresses):
self.rided.set_address_for_mdmt(mdmt, addr)
# set up manual publisher routes
self.publishers = ['h1-b5', 'h1-b1']
self.publisher_routes = [['h1-b5', 'b5', 'c1', 'c0', self.root],
['h1-b1', 'b1', 'c2', 'c3', self.root]]
for pub_route in self.publisher_routes:
self.rided.set_publisher_route(pub_route[0], pub_route)
for pub in self.publishers:
self.rided.notify_publication(pub)
# register the subscribers
self.subscribers = ['h0-b0', 'h0-b1', 'h1-b1', 'h0-b3']
for sub in self.subscribers:
self.rided.add_subscriber(sub, ALERT_TOPIC)
# We expect the MDMTs to be selected (via 'importance' policy) in this order for the following tests...
self.expected_mdmts = [('tree4', ), ('tree2', ), ('tree3', ),
('tree1', ), ('tree2', ),
('tree1', 'tree3', 'tree4')]
# ... based on these subscribers being reached during each attempt.
self.subs_reached_at_attempt = [
('h0-b1', 'h1-b1'), #0
tuple(),
tuple(),
tuple(), # 1-3 no responses...
('h0-b3', ), #4
('h0-b0', ) #5 ; all done!
]
# NOTES about the test cases:
# NOTE: we only do these tests for 'importance' since the others will have a tie between tree3/4
# we should choose tree2 second due to update about subs reached...
# because AlertContext tracks trees tried we should use tree3 third
# furthermore, we should lastly try tree1 even though it had lowest importance!
# then, we should try tree2 as the highest current importance after a notification since we've tried all of them
# finally, since we have a tie among all the others
self.attempt_num = 0
self.alert = self.rided._make_new_alert(ALERT_MSG, ALERT_TOPIC)
def test_basic_mdmt_selection(self):
"""Tests MDMT-selection (without alerting context) for the default policy by manually assigning
MDMTs, publisher routes, notifying RideD about a few publications and verifying that the selected MDMT is
the one expected given this information."""
mdmt = self.rided.get_best_mdmt(
self.alert, heuristic=self.rided.MAX_LINK_IMPORTANCE)
self.assertIn(self.rided.get_address_for_mdmt(mdmt),
self.expected_mdmts[0])
mdmt = self.rided.get_best_mdmt(
self.alert, heuristic=self.rided.MAX_OVERLAPPING_LINKS)
self.assertEqual(self.rided.get_address_for_mdmt(mdmt), 'tree4')
mdmt = self.rided.get_best_mdmt(self.alert,
heuristic=self.rided.MIN_MISSING_LINKS)
self.assertEqual(self.rided.get_address_for_mdmt(mdmt), 'tree4')
# TODO: ENHANCE: additional test...
# Now, if we reset the STT and change the publisher routes we should get different MDMTs
# self.rided.stt_mgr.reset()
# self.rided.set_publisher_route('h1-b5', ['h1-b5', 'b5', 'c1', 'c0', self.root])
#
# for pub in self.publishers:
# self.rided.notify_publication(pub)
#
# mdmt = self.rided.get_best_mdmt(ALERT_TOPIC, heuristic=self.rided.MAX_OVERLAPPING_LINKS)
# self.assertEqual(self.rided.get_address_for_mdmt(mdmt), 'tree4')
#
# mdmt = self.rided.get_best_mdmt(ALERT_TOPIC, heuristic=self.rided.MAX_LINK_IMPORTANCE)
# self.assertEqual(self.rided.get_address_for_mdmt(mdmt), 'tree4')
#
# mdmt = self.rided.get_best_mdmt(ALERT_TOPIC, heuristic=self.rided.MIN_MISSING_LINKS)
# self.assertEqual(self.rided.get_address_for_mdmt(mdmt), 'tree4')
def test_mdmt_selection_with_context(self):
"""Tests MDMT-selection WITH alerting context in a similar manner to the basic tests. Here we use an
AlertContext object to change the MDMT choice based on claiming that some subscribers have already been alerted."""
# NOTE: we're actually using the _do_send_alert method instead of manually recording and doing notifications.
# The callback used to actually 'send the alert packet' (no network operations) will handle notifying subs.
for attempt_num, subs_reached in enumerate(
self.subs_reached_at_attempt):
mdmt = self.rided._do_send_alert(self.alert)
self.assertIn(self.rided.get_address_for_mdmt(mdmt),
self.expected_mdmts[attempt_num])
#### TEST ACTUAL send_alert(...) API ######
def test_send_alert(self):
"""
Tests the main send_alert API that exercises everything previously tested along with the retransmit
capability. This uses a custom testing callback instead of opening a socket and test servers to receive alerts.
"""
expected_num_attempts = len(self.subs_reached_at_attempt)
# Send the alert and ensure it took the right # retries
alert = self.rided.send_alert(ALERT_MSG,
ALERT_TOPIC,
timeout=TIMEOUT,
max_retries=expected_num_attempts + 1)
sleep((expected_num_attempts + 1) * TIMEOUT)
self.assertFalse(alert.active)
self.assertEqual(self.attempt_num, expected_num_attempts)
self.assertEqual(len(alert.subscribers_reached),
len(self.subscribers)) # not all subs reached????
def test_cancel_alert(self):
"""Ensure that cancelling alerts works properly by cancelling it before it finishes and verify that some
subscribers remain unreached."""
expected_num_attempts = len(self.subs_reached_at_attempt)
alert = self.rided.send_alert(ALERT_MSG,
ALERT_TOPIC,
timeout=TIMEOUT,
max_retries=expected_num_attempts + 1)
# instead of waiting for it to finish, cancel the alert right before the last one gets sent
sleep((expected_num_attempts - 1.5) * TIMEOUT)
self.rided.cancel_alert(alert)
sleep(TIMEOUT)
# Now we should note that the last alert message wasn't sent!
self.assertFalse(alert.active)
self.assertEqual(self.attempt_num, expected_num_attempts - 1)
self.assertEqual(len(alert.subscribers_reached),
len(self.subscribers) - 1)
def test_send_alert_unsuccessfully(self):
expected_num_attempts = len(self.subs_reached_at_attempt)
# since we set max_retries to be less than the number required this alert should stop early despite not reaching all subs
alert = self.rided.send_alert(ALERT_MSG,
ALERT_TOPIC,
timeout=TIMEOUT,
max_retries=expected_num_attempts - 2)
sleep((expected_num_attempts + 1) * TIMEOUT)
self.assertFalse(alert.active)
self.assertEqual(self.attempt_num, expected_num_attempts - 1)
self.assertEqual(len(alert.subscribers_reached),
len(self.subscribers) - 1) # not all subs reached????
def __send_alert_test_callback(self, alert, mdmt):
"""
Custom callback to handle verifying that the expected MDMT was used in between each attempt and
notifies RideD of which subscribers were reached.
:param alert:
:type alert: RideD.AlertContext
:param mdmt:
:return:
"""
self.assertTrue(
alert.active,
"__send_alert_test_callback should not fire if alert isn't active!"
)
expected_mdmt = self.expected_mdmts[self.attempt_num]
self.assertIn(
self.rided.get_address_for_mdmt(mdmt), expected_mdmt,
"incorrect MDMT selected for attempt %d: expected one of %s but got %s"
% (self.attempt_num, expected_mdmt, mdmt))
for s in self.subs_reached_at_attempt[self.attempt_num]:
# XXX: because this callback is fired while the alert's thread_lock is acquired, we have to do this
# from inside another thread so that it will run after this callback returns. Otherwise, deadlock!
# self.rided.notify_alert_response(s, alert, mdmt)
Thread(target=self.rided.notify_alert_response,
args=(s, alert, mdmt)).start()
self.attempt_num += 1
# ENHANCE: test_send_alert_multi_threaded????
# ENHANCE: test_send_alert_network_socket
## helper functions
def _build_mdmts(self, subscribers=None):
mdmts = self.rided.build_mdmts(subscribers=subscribers)
self.rided.mdmts = mdmts
return mdmts
def run_experiment(self):
"""Check what percentage of subscribers are still reachable
from the server after the failure model has been applied
by removing the failed_nodes and failed_links from each tree
as well as a copy of the overall topology (for the purpose of
establishing an upper bound oracle heuristic).
We also explore the use of an intelligent multicast tree-choosing
heuristic that picks the tree with the most overlap with the paths
each publisher's sensor data packet arrived on.
:rtype dict:
"""
# IDEA: we can determine the reachability by the following:
# for each topology, remove the failed nodes and links,
# determine all reachable nodes in topology,
# consider only those that are subscribers,
# record % reachable by any/all topologies.
# We also have different methods of choosing which tree to use
# and two non-multicast comparison heuristics.
# NOTE: the reachability from the whole topology (oracle) gives us an
# upper bound on how well the edge server could possibly do,
# even without using multicast.
subscribers = set(self.subscribers)
result = dict()
heuristic = self.get_mcast_heuristic_name()
# we'll record reachability for various choices of trees
result[heuristic] = dict()
failed_topology = self.get_failed_topology(self.topo.topo,
self.failed_nodes,
self.failed_links)
# start up and configure RideD middleware for building/choosing trees
# We need to specify dummy addresses that won't actually be used for anything.
addresses = ["10.0.0.%d" % d for d in range(self.ntrees)]
rided = RideD(
self.topo,
self.server,
addresses,
self.ntrees,
construction_algorithm=self.tree_construction_algorithm[0],
const_args=self.tree_construction_algorithm[1:])
# HACK: since we never made an actual API for the controller, we just do this manually...
for s in subscribers:
rided.add_subscriber(s, PUBLICATION_TOPIC)
# Build up the Successfully Traversed Topology (STT) from each publisher
# by determining which path the packet would take in the functioning
# topology and add its edges to the STT only if that path is
# functioning in the failed topology.
# BIG OH: O(T) + O(S), where S = |STT|
# XXX: because the RideC implementation requires an actual SDN controller adapter, we just repeat the logic
# for computing 'redirection' routes (publisher-->edge after cloud failure) here...
if self.reroute_policy == 'shortest':
pub_routes = {
pub: self.topo.get_path(pub,
self.server,
weight=DISTANCE_METRIC)
for pub in self.publishers
}
else:
if self.reroute_policy != 'disjoint':
log.error(
"unknown reroute_policy '%s'; defaulting to 'disjoint'...")
pub_routes = {
p[0]: p
for p in self.topo.get_multi_source_disjoint_paths(
self.publishers, self.server, weight=DISTANCE_METRIC)
}
assert list(sorted(pub_routes.keys())) == list(
sorted(self.publishers)
), "not all hosts accounted for in disjoint paths: %s" % pub_routes.values(
)
# Determine which publishers successfully reached the edge to build the STT in Ride-D and report pub_rate
pub_rate = 0
for pub in self.publishers:
path = pub_routes[pub]
rided.set_publisher_route(pub, path)
if random.random() >= self.error_rate and nx.is_simple_path(
failed_topology, path):
rided.notify_publication(pub)
pub_rate += 1
pub_rate /= float(len(self.publishers))
result['pub_rate'] = pub_rate
# build and get multicast trees
trees = rided.build_mdmts()[PUBLICATION_TOPIC]
# XXX: rather than use the install_mdmts API, which would try to install flow rules, we just set them directly
rided.mdmts[PUBLICATION_TOPIC] = trees
# record which heuristic we used
for tree in trees:
tree.graph['heuristic'] = self.get_mcast_heuristic_name()
# sanity check that the returned trees reach all destinations
assert all(
nx.has_path(tree, self.server, sub) for sub in subscribers)
# ORACLE
# First, use a copy of whole topology as the 'oracle' heuristic,
# which sees what subscribers are even reachable by ANY path.
reach = self.get_oracle_reachability(subscribers, self.server,
failed_topology)
result['oracle'] = reach
# UNICAST
# Second, get the reachability for the 'unicast' heuristic,
# which sees what subscribers are reachable on the failed topology
# via the path they'd normally be reached on the original topology
paths = [
nx.shortest_path(self.topo.topo,
self.server,
s,
weight=DISTANCE_METRIC) for s in subscribers
]
# record the cost of the paths whether they would succeed or not
unicast_cost = sum(self.topo.topo[u][v].get(COST_METRIC, 1) for p in paths\
for u, v in zip(p, p[1:]))
# now filter only paths that are still functioning and record the reachability
paths = [p for p in paths if nx.is_simple_path(failed_topology, p)]
result['unicast'] = len(paths) / float(len(subscribers))
# TODO: disjoint unicast paths comparison!
# ALL TREES' REACHABILITIES: all, min, max, mean, stdev
# Next, check all the redundant multicast trees together to get their respective (and aggregate) reachabilities
topos_to_check = [
self.get_failed_topology(t, self.failed_nodes, self.failed_links)
for t in trees
]
reaches = self.get_reachability(self.server, subscribers,
topos_to_check)
heuristic = trees[0].graph[
'heuristic'] # we assume all trees from same heuristic
result[heuristic]['all'] = reaches[-1]
reaches = reaches[:-1]
result[heuristic]['max'] = max(reaches)
result[heuristic]['min'] = min(reaches)
result[heuristic]['mean'] = np.mean(reaches)
result[heuristic]['stdev'] = np.std(reaches)
# CHOSEN
# Finally, check the tree chosen by the edge server heuristic(s)
# for having the best estimated chance of data delivery
choices = dict()
for method in RideD.MDMT_SELECTION_POLICIES:
alert_ctx = rided._make_new_alert("dummy msg", PUBLICATION_TOPIC)
choices[method] = rided.get_best_mdmt(alert_ctx, method)
for choice_method, best_tree in choices.items():
best_tree_idx = trees.index(best_tree)
reach = reaches[best_tree_idx]
result[heuristic]['%s-chosen' % choice_method] = reach
### RECORDING METRICS ###
# Record the distance to the subscribers in terms of # hops
# TODO: make this latency instead?
nhops = []
for t in trees:
for s in subscribers:
nhops.append(len(nx.shortest_path(t, s, self.server)) - 1)
result['nhops'] = dict(mean=np.mean(nhops),
stdev=np.std(nhops),
min=min(nhops),
max=max(nhops))
# Record the pair-wise overlap between the trees
tree_edges = [set(t.edges()) for t in trees]
overlap = [
len(t1.intersection(t2)) for t1 in tree_edges for t2 in tree_edges
]
result['overlap'] = sum(overlap)
# TODO: try to get this working on topos > 20?
# the ILP will need some work if we're going to get even the relaxed version running on large topologies
# overlap_lower_bound = ilp_redundant_multicast(self.topo.topo, server, subscribers, len(trees), get_lower_bound=True)
# result['overlap_lower_bound'] = overlap_lower_bound
# Record the average size of the trees
costs = [
sum(e[2].get(COST_METRIC, 1) for e in t.edges(data=True))
for t in trees
]
result['cost'] = dict(mean=np.mean(costs),
stdev=np.std(costs),
min=min(costs),
max=max(costs),
unicast=unicast_cost)
return result
def __init__(self, broker,
# RideD parameters
# TODO: sublcass RideD in order to avoid code repetition here for extracting parameters?
dpid, addresses=None, topology_mgr='onos', ntrees=2,
tree_choosing_heuristic='importance', tree_construction_algorithm=('red-blue',),
max_retries=None,
# XXX: rather than running a separate service that would intercept incoming publications matching the
# specified flow for use in the STT, we simply wait for seismic picks and use them as if they're
# incoming packets. This ignores other potential packets from those hosts, but this will have to do
# for now since running such a separated service would require more systems programming than this...
subscriptions=(SEISMIC_PICK_TOPIC,
# RideD gathers the publisher routes from RideC via events when they change
PUBLISHER_ROUTE_TOPIC,
),
maintenance_interval=10,
multicast=True, dst_port=DEFAULT_COAP_PORT, topics_to_sink=(SEISMIC_ALERT_TOPIC,), **kwargs):
"""
See also the parameters for RideD constructor!
:param ntrees: # MDMTs to build (passed to RideD constructor); note that setting this to 0 disables multicast!
:param broker:
:param addresses: iterable of network addresses (i.e. tuples of (str[ipv4_src_addr], udp_src_port))
that can be used to register multicast trees and send alert packets through them
****NOTE: we use udp_src_port rather than the expected dst_port because this allows the clients to
respond to this port# and have the response routed via the proper MDMT
:param dst_port: port number to send events to (NOTE: we expect all subscribers to listen on the same port OR
for you to configure the flow rules to convert this port to the expected one before delivery to subscriber)
:param topics_to_sink: a SensedEvent whose topic matches one in this list will be
resiliently multicast delivered; others will be ignored
:param maintenance_interval: seconds between running topology updates and reconstructing MDMTs if necessary,
accounting for topology changes or new/removed subscribers
:param multicast: if True (default unless ntrees==0), build RideD for using multicast; otherwise, subscribers are alerting one
at a time (async) using unicast
:param kwargs:
"""
super(RideDEventSink, self).__init__(broker, topics_to_sink=topics_to_sink, subscriptions=subscriptions, **kwargs)
# Catalogue active subscribers' host addresses (indexed by topic with value being a set of subscribers)
self.subscribers = dict()
self.dst_port = dst_port
self.maintenance_interval = maintenance_interval
# If we need to do anything with the server right away or expect some logic to be called
# that will not directly check whether the server is running currently, we should wait
# for a CoapServerRunning event before accessing the actual server.
# NOTE: make sure to do this here, not on_start, as we currently only send the ready notification once!
ev = CoapServer.CoapServerRunning(None)
self.subscribe(ev, callback=self.__class__.__on_coap_ready)
# Store parameters for RideD resilient multicast middleware; we'll actually build it later since it takes a while...
self.use_multicast = multicast if ntrees else False
if self.use_multicast and addresses is None:
raise NotImplementedError("you must specify the multicast 'addresses' parameter if multicast is enabled!")
# If we aren't doing multicast, we can create a single CoapClient without a specified src_port/address and this
# will be filled in for us...
# COAPTHON-SPECIFIC: unclear that we'd be able to do this in all future versions...
if not self.use_multicast:
self.rided = None
srv_ip = '10.0.0.1'
self._coap_clients = {'unicast': CoapClient(server_hostname=srv_ip, server_port=self.dst_port,
confirmable_messages=not self.use_multicast)}
# Configure RideD and necessary CoapClient instances...
# Use a single client for EACH MDMT to connect with each server. We do this so that we can specify the source
# port and have the 'server' (remote subscribers) respond to this port# and therefore route responses along the
# same path. Hence, we need to ensure addresses contains some useable addresses or we'll get null exceptions!
else:
# cmd-line specified addresses might convert them to a list of lists, so make them tuples for hashing!
addresses = [tuple(address) for address in addresses]
# This callback is essentially the CoAP implementation for RideD: it uses CoAPthon to send a request to the
# given address through a 'helper client' and register a callback for receiving subscriber responses and
# notifying RideD of them.
# NOTE: a different CoAP message is created for each alert re-try since they're sent as non-CONfirmable!
do_send_cb = self.__sendto
# We may opt to build RideD in on_start() instead depending on what resources we want available first...
self.rided = dict(topology_mgr=topology_mgr, dpid=dpid, addresses=addresses, ntrees=ntrees,
tree_choosing_heuristic=tree_choosing_heuristic, tree_construction_algorithm=tree_construction_algorithm,
alert_sending_callback=do_send_cb, max_retries=max_retries)
if BUILD_RIDED_IN_INIT:
self.rided = RideD(**self.rided)
# NOTE: we store CoapClient instances in a dict so that we can index them by MDMT address for easily
# accessing the proper instance for the chosen MDMT
self._coap_clients = dict()
for address in addresses:
dst_ip, src_port = address
self._coap_clients[address] = CoapClient(server_hostname=dst_ip, server_port=self.dst_port,
src_port=src_port,
confirmable_messages=not self.use_multicast)
# Need to track outstanding alerts as we can only have a single one for each topic at a time
# since they're updates: index them by topic --> AlertContext
self._outstanding_alerts = dict()
# Use thread locks to prevent simultaneous write access to data structures due to e.g.
# handling multiple simultaneous subscription registrations.
self.__subscriber_lock = Lock()
class RideDEventSink(ThreadedEventSink):
"""
An EventSink that delivers events using the RIDE-D middleware for resilient IP multicast-based publishing.
"""
def __init__(self, broker,
# RideD parameters
# TODO: sublcass RideD in order to avoid code repetition here for extracting parameters?
dpid, addresses=None, topology_mgr='onos', ntrees=2,
tree_choosing_heuristic='importance', tree_construction_algorithm=('red-blue',),
max_retries=None,
# XXX: rather than running a separate service that would intercept incoming publications matching the
# specified flow for use in the STT, we simply wait for seismic picks and use them as if they're
# incoming packets. This ignores other potential packets from those hosts, but this will have to do
# for now since running such a separated service would require more systems programming than this...
subscriptions=(SEISMIC_PICK_TOPIC,
# RideD gathers the publisher routes from RideC via events when they change
PUBLISHER_ROUTE_TOPIC,
),
maintenance_interval=10,
multicast=True, dst_port=DEFAULT_COAP_PORT, topics_to_sink=(SEISMIC_ALERT_TOPIC,), **kwargs):
"""
See also the parameters for RideD constructor!
:param ntrees: # MDMTs to build (passed to RideD constructor); note that setting this to 0 disables multicast!
:param broker:
:param addresses: iterable of network addresses (i.e. tuples of (str[ipv4_src_addr], udp_src_port))
that can be used to register multicast trees and send alert packets through them
****NOTE: we use udp_src_port rather than the expected dst_port because this allows the clients to
respond to this port# and have the response routed via the proper MDMT
:param dst_port: port number to send events to (NOTE: we expect all subscribers to listen on the same port OR
for you to configure the flow rules to convert this port to the expected one before delivery to subscriber)
:param topics_to_sink: a SensedEvent whose topic matches one in this list will be
resiliently multicast delivered; others will be ignored
:param maintenance_interval: seconds between running topology updates and reconstructing MDMTs if necessary,
accounting for topology changes or new/removed subscribers
:param multicast: if True (default unless ntrees==0), build RideD for using multicast; otherwise, subscribers are alerting one
at a time (async) using unicast
:param kwargs:
"""
super(RideDEventSink, self).__init__(broker, topics_to_sink=topics_to_sink, subscriptions=subscriptions, **kwargs)
# Catalogue active subscribers' host addresses (indexed by topic with value being a set of subscribers)
self.subscribers = dict()
self.dst_port = dst_port
self.maintenance_interval = maintenance_interval
# If we need to do anything with the server right away or expect some logic to be called
# that will not directly check whether the server is running currently, we should wait
# for a CoapServerRunning event before accessing the actual server.
# NOTE: make sure to do this here, not on_start, as we currently only send the ready notification once!
ev = CoapServer.CoapServerRunning(None)
self.subscribe(ev, callback=self.__class__.__on_coap_ready)
# Store parameters for RideD resilient multicast middleware; we'll actually build it later since it takes a while...
self.use_multicast = multicast if ntrees else False
if self.use_multicast and addresses is None:
raise NotImplementedError("you must specify the multicast 'addresses' parameter if multicast is enabled!")
# If we aren't doing multicast, we can create a single CoapClient without a specified src_port/address and this
# will be filled in for us...
# COAPTHON-SPECIFIC: unclear that we'd be able to do this in all future versions...
if not self.use_multicast:
self.rided = None
srv_ip = '10.0.0.1'
self._coap_clients = {'unicast': CoapClient(server_hostname=srv_ip, server_port=self.dst_port,
confirmable_messages=not self.use_multicast)}
# Configure RideD and necessary CoapClient instances...
# Use a single client for EACH MDMT to connect with each server. We do this so that we can specify the source
# port and have the 'server' (remote subscribers) respond to this port# and therefore route responses along the
# same path. Hence, we need to ensure addresses contains some useable addresses or we'll get null exceptions!
else:
# cmd-line specified addresses might convert them to a list of lists, so make them tuples for hashing!
addresses = [tuple(address) for address in addresses]
# This callback is essentially the CoAP implementation for RideD: it uses CoAPthon to send a request to the
# given address through a 'helper client' and register a callback for receiving subscriber responses and
# notifying RideD of them.
# NOTE: a different CoAP message is created for each alert re-try since they're sent as non-CONfirmable!
do_send_cb = self.__sendto
# We may opt to build RideD in on_start() instead depending on what resources we want available first...
self.rided = dict(topology_mgr=topology_mgr, dpid=dpid, addresses=addresses, ntrees=ntrees,
tree_choosing_heuristic=tree_choosing_heuristic, tree_construction_algorithm=tree_construction_algorithm,
alert_sending_callback=do_send_cb, max_retries=max_retries)
if BUILD_RIDED_IN_INIT:
self.rided = RideD(**self.rided)
# NOTE: we store CoapClient instances in a dict so that we can index them by MDMT address for easily
# accessing the proper instance for the chosen MDMT
self._coap_clients = dict()
for address in addresses:
dst_ip, src_port = address
self._coap_clients[address] = CoapClient(server_hostname=dst_ip, server_port=self.dst_port,
src_port=src_port,
confirmable_messages=not self.use_multicast)
# Need to track outstanding alerts as we can only have a single one for each topic at a time
# since they're updates: index them by topic --> AlertContext
self._outstanding_alerts = dict()
# Use thread locks to prevent simultaneous write access to data structures due to e.g.
# handling multiple simultaneous subscription registrations.
self.__subscriber_lock = Lock()
@property
def coap_clients(self):
# this obscures the fact that we store the clients in a dict
return self._coap_clients.values()
def __maintain_topology(self):
"""Runs periodically to check for topology updates, reconstruct the MDMTs if necessary, and update flow
rules to account for these topology changes or newly-joined/leaving subscribers."""
# ENHANCE: only update the necessary changes: old subscribers are easy to trim, new ones could be added directly,
# and topologies could be compared for differences (though that's probably about the same work as just refreshing the whole thing)
# TODO: probably need to lock rided during this so we don't e.g. send_event to an MDMT that's currently being reconfigured.... maybe that's okay though?
self.rided.update()
def on_start(self):
"""
Build and configure the RideD middleware
"""
# TODO: probably run this in the background?
if self.rided is not None:
if not BUILD_RIDED_IN_INIT:
assert isinstance(self.rided, dict)
self.rided = RideD(**self.rided)
assert isinstance(self.rided, RideD)
# Rather than periodically update the topology, which in our experiments would result in perfectly routing
# around all the failures due to 0-latency control plane, we just update it once for now...
self.timed_call(self.maintenance_interval, self.__class__.__maintain_topology, repeat=False)
# self.timed_call(self.maintenance_interval, self.__class__.__maintain_topology, repeat=True)
super(RideDEventSink, self).on_start()
def __sendto(self, alert_ctx, mdmt):
"""
Sends msg to the specified address using CoAP. topic is used to define the path of the CoAP
resource we PUT the msg in.
:param alert_ctx:
:type alert_ctx: RideD.AlertContext
:param mdmt: the MDMT to use for sending this alert; must extract address from it!
:return:
"""
address = self.rided.get_address_for_mdmt(mdmt)
topic = alert_ctx.topic
msg = alert_ctx.msg
# The response callback needs to know which MDMT was used so that it can notify RideD about it.
def __mdmt_response_callback(response):
self.__put_event_callback(response, alert_context=alert_ctx, mdmt_used=mdmt)
coap_client = self._coap_clients[address]
return self.__send_alert_from_client(msg, topic, coap_client, __mdmt_response_callback)
def __send_alert_from_client(self, msg, topic, coap_client, response_callback=None):
"""
Actually sends the message via the specified CoapClient.
:param msg:
:param topic:
:param coap_client: the CoapClient instance to use, which will already have its src/dst_port/addr set
:param response_callback: called when the destination responds (e.g. with OK); default is self.__put_event_callback
:return:
"""
# TODO: don't hardcode this...
path = "/events/%s" % topic
if response_callback is None:
response_callback = self.__put_event_callback
# Use async mode to send this message as otherwise sending a bunch of them can lead to a back log...
coap_client.put(path=path, payload=msg, callback=response_callback)
log.debug("RIDE-D message sent: topic=%s ; address=%s ; payload_length=%d" % (topic, coap_client.server, len(msg)))
def __put_event_callback(self, response, alert_context=None, mdmt_used=None):
"""
This callback handles the CoAP response for a PUT message. In addition to logging the success or failure it
notifies RideD of the response's route (using the provided mdmt_used parameter) if configured for
reliable multicast delivery.
:param response:
:type response: coapthon.messages.response.Response
:param alert_context: the state of this alert
:type alert_context: RideD.AlertContext
:param mdmt_used: if specified, the request was sent via reliable multicast and this parameter represents the
multicast tree used
:type mdmt_used: nx.Graph
:return:
"""
# TODO: record results to output later?
responder_addr = response.source
responder_ip_addr = responder_addr[0]
# XXX: when client closes the last response is a NoneType
if response is None:
return
elif coap_response_success(response):
log.debug("successfully sent alert to " + str(response.source))
if alert_context and mdmt_used: # multicast alert!
# notify RideD about this successful response
responder = self.rided.topology_manager.get_host_by_ip(responder_ip_addr)
self.rided.notify_alert_response(responder, alert_context, mdmt_used)
elif response.code == CoapCodes.NOT_FOUND.number:
log.warning("remote %s rejected PUT request for uncreated object: did you forget to add that resource?" % str(responder_addr))
else:
log.error("failed to send aggregated events due to Coap error: %s" % coap_code_to_name(response.code))
def send_event(self, event):
"""
When charged with sending an event, we will send it to each subscriber. If configured for using multicast,
we first choose the best MDMT for resilient multicast delivery."""
topic = event.topic
encoded_event = self.encode_event(event)
log.debug("Sending event via RIDE-D with topic %s" % topic)
# Send the event as we're configured to
try:
# Determine the best MDMT, get the destination associated with it, and send the event.
if self.use_multicast:
# if we ever encounter this, replace it with some real error handling...
assert self.rided is not None, "woops! Ride-D should be set up but it isn't..."
try:
# XXX: we can only have a single outstanding alert at a time for a given topic so
# we need to cancel the last one if it exists.
if topic in self._outstanding_alerts:
self.rided.cancel_alert(self._outstanding_alerts.pop(topic))
self._outstanding_alerts[topic] = self.rided.send_alert(encoded_event, topic)
except KeyError:
log.error("currently-unhandled error likely caused by trying to MDMT-multicast"
" an alert to an unregistered topic with no MDMTs!")
return False
# Configured as unicast, so send a message to each subscriber individually
else:
# For unicast case, we only needed to create one client!
coap_client = self.coap_clients[0]
for dst_ip_address in self.subscribers.get(topic, []):
# But, we do need to set the destination address for that client...
coap_client.server = (dst_ip_address, self.dst_port)
self.__send_alert_from_client(encoded_event, topic=topic, coap_client=coap_client)
return True
except IOError as e:
log.error("failed to send event via CoAP PUT due to error: %s" % e)
return False
def on_event(self, event, topic):
"""
We receive sensor-publisher route updates via events from RideC.
HACK: any seismic picks we receive are treated as incoming publications for the purposes of updating the
STT. This clearly does not belong in a finalized version of the RideD middleware, which would instead
intercept actual packets matching a particular flow and use them to update the STT.
:param event:
:type event: scale_client.core.sensed_event.SensedEvent
:param topic:
:return:
"""
if topic == SEISMIC_PICK_TOPIC:
if self.rided and not event.is_local:
# Find the publishing host's IP address and use that to notify RideD
publisher = event.source
# ENHANCE: accept full address (e.g. ipv4_add, port) as publisher IDs just like RideC!
publisher = get_hostname_from_path(publisher)
assert publisher is not None, "error processing publication with no source hostname: %s" % event.source
# TODO: may need to wrap this with mutex
self.rided.notify_publication(publisher, id_type='ip')
elif topic == PUBLISHER_ROUTE_TOPIC:
if self.rided:
for host, route in event.data.items():
log.debug("setting publisher route from event: host(%s) --> %s" % (host, route))
host = self.rided.topology_manager.get_host_by_ip(host)
self.rided.set_publisher_route(host, route)
else:
assert False, "received non-seismic event we didn't subscribe to! topic=%s" % topic
def process_subscription(self, topic, host):
"""
Handles a subscription request by adding the host to the current subscribers.
Note that we don't collect a port number or protocol type as we currently assume it will be
CoAP and its well-known port number.
:param topic:
:param host: IP address or hostname of subscribing host (likely taken from CoAP request)
:return:
"""
log.debug("processing RIDE-D subscription for topic '%s' by host '%s'" % (topic, host))
with self.__subscriber_lock:
self.subscribers.setdefault(topic, set()).add(host)
if self.rided:
# WARNING: supposedly we should only register subscribers that are reachable in our topology view or
# we'll cause errors later... we should try to handle those errors instead!
try:
# ENHANCE: handle port numbers? all ports will be same for our scenario and OF could convert them anyway so no hurry...
host = self.rided.topology_manager.get_host_by_ip(host)
# If we can't find a path, how did we even get this subscription? Path failed after it was sent?
self.rided.topology_manager.get_path(host, self.rided.dpid)
with self.__subscriber_lock:
self.rided.add_subscriber(host, topic_id=SEISMIC_ALERT_TOPIC)
except BaseException as e:
log.warning("Route between subscriber %s and server %s not found: skipping...\nError: %s" % (host, self.rided.dpid, e))
return False
return True
def __on_coap_ready(self, server):
"""
Register a CoAP API endpoint for subscribers to register their subscriptions through.
:param CoapServer server:
:return:
"""
if self.use_multicast:
# TODO: if we ever encounter this, we should delay registering the subscriptions API until after ride-d is setup
# maybe we could just defer the arriving subscription by not sending a response?
assert self.rided is not None, "woops coap is set up but ride-d isn't!!"
# ENHANCE: could save server name to make sure we've got the right one her?
# if self._server_name is None or self._server_name == server.name:
self._server = server
def __process_coap_subscription(coap_request, coap_resource):
"""
Extract the relevant subscription information from the CoAP request object and pass it along to self.process_subscription()
:param coap_request:
:type coap_request: coapthon.messages.request.Request
:param coap_resource:
:return:
"""
host, port = coap_request.source
payload = coap_request.payload
# ENHANCE: check the content-type?
topic = payload
# TODO: remove this hack later
assert topic == SEISMIC_ALERT_TOPIC, "unrecognized subscription topic %s" % topic
if self.process_subscription(topic, host):
return coap_resource
else:
return False
# ENHANCE: how to handle an unsubscribe?
path = SUBSCRIPTION_API_PATH
server.register_api(path, name="%s subscription registration" % SEISMIC_ALERT_TOPIC,
post_callback=__process_coap_subscription, allow_children=True)
def check_available(self, event):
"""We only deliver events whose topic matches those that have been registered
with RIDE-D and currently have subscribers."""
return super(RideDEventSink, self).check_available(event) and event.topic in self.subscribers
def on_stop(self):
"""Close any open network connections e.g. CoapClient"""
for client in self.coap_clients:
client.close()
super(RideDEventSink, self).on_stop()
# TODO: log error when no subscribers ever connected?
def encode_event(self, event):
"""Encodes the given event with several fields stripped out and only the most recent event IDs in order to save
space in the single CoAP packet it will be sunk in."""
return compress_alert_one_coap_packet(event)