def sidr_reject_by_peer(SDXes, sdxi):
"""
Reject policy by checking peers
"""
policy = SDXes[sdxi]['policy']
reject_cnt = 0
tp_reject_cnt = 0
peer_policy = [SDXes[p]['policy'] for p in SDXes[sdxi]['peers']]
merge_peer_policy = PyTricia()
for p_policy in peer_policy:
for p in p_policy.keys():
if p not in merge_peer_policy.keys():
merge_peer_policy[p] = {}
merge_peer_policy[p].update(p_policy[p])
rejected_prefixes = set(policy.keys()).intersection(
merge_peer_policy.keys())
fp_rejected_policy = PyTricia()
for prefix in rejected_prefixes:
deflect_port = {p for p in policy[prefix].keys()}
peer_deflect_port = {p for p in merge_peer_policy[prefix].keys()}
reject_cnt += len(deflect_port)
tp_reject_cnt += len(deflect_port.intersection(peer_deflect_port))
fp_rejected_policy[prefix] = {
p: 1
for p in deflect_port if p not in peer_deflect_port
}
return reject_cnt, tp_reject_cnt, fp_rejected_policy
def initiate_bubblecast(G):
"""
Initiate bubblecast table for each network in G.
"""
for n in G.nodes():
G.node[n]['qcast'] = PyTricia()
G.node[n]['qcast-in'] = PyTricia()
G.node[n]['dcast'] = PyTricia()
G.node[n]['dcast-in'] = PyTricia()
def initiate_ribs(G, overwrite=True):
"""
Initiate ribs for each network in G.
"""
for n in G.nodes():
G.node[n]['rib'] = PyTricia()
for prefix in G.node[n]['ip-prefixes']:
update_initial_rib(G.node[n]['rib'], prefix, overwrite)
# out_ribs = G.node[n]['adj-ribs-out']
# for d in out_ribs:
# update_initial_rib(out_ribs[d], prefix, n)
if overwrite:
G.node[n]['adj-ribs-in'] = {n: PyTricia() for n in G.neighbors(n)}
G.node[n]['adj-ribs-out'] = {n: PyTricia() for n in G.neighbors(n)}
def sidr_deflection_sim(G,
flows,
SDXes,
service_types=DEFAULT_SERVICE_TYPES,
network_ratio=0.5,
prefix_ratio=0.2,
**kwargs):
"""
Simulate false-positive cases of SIDR
SDXes: [sdx]
sdx: (sdxi, dests, peers)
"""
for sdxi in SDXes.keys():
policy = PyTricia()
for dest in SDXes[sdxi]['dests']:
node_prefixes = G.node[dest]['ip-prefixes']
sidr_random_deflect(node_prefixes, policy, service_types,
network_ratio, prefix_ratio)
SDXes[sdxi]['policy'] = policy
total_reject_cnt = 0
total_tp_reject_cnt = 0
total_fp_volume = 0
for sdxi in SDXes.keys():
reject_cnt, tp_reject_cnt, fp_rejected_policy = sidr_reject_by_peer(
SDXes, sdxi)
fp_volume = fp_affected_volume(G, fp_rejected_policy,
SDXes[sdxi]['srcs'], flows)
print(sdxi, reject_cnt, tp_reject_cnt, fp_volume)
total_reject_cnt += reject_cnt
total_tp_reject_cnt += tp_reject_cnt
total_fp_volume += fp_volume
print(total_reject_cnt, total_reject_cnt - total_tp_reject_cnt,
total_fp_volume)
def __init__(self, name, measurement_config, **kwargs):
Node.__init__(self, name, measurement_config, **kwargs)
self.dpid = abs(hash(name))
self.dstmac_cache = {}
self.pox_switch = PoxBridgeSoftwareSwitch(self.dpid, name=name,
ports=0, miss_send_len=2**16, max_buffers=2**8, features=None)
self.pox_switch.set_connection(self)
self.pox_switch.set_output_packet_callback(self. send_packet)
self.controller_name = kwargs.get('controller', 'controller')
self.autoack = bool(eval(kwargs.get('autoack', 'False')))
self.controller_links = {}
self.interface_to_port_map = {}
self.trace = bool(eval(kwargs.get('trace', 'False')))
self.ipdests = PyTricia()
for prefix in kwargs.get('ipdests','').split():
self.ipdests[prefix] = True
# explicitly add a localhost link/interface
ipa,ipb = [ ip for ip in next(FsConfigurator.link_subnetter).iterhosts() ]
remotemac = default_ip_to_macaddr(ipb)
self.add_link(NullLink, ipa, ipb, 'remote', remotemac=remotemac)
self.trafgen_ip = str(ipa)
self.trafgen_mac = remotemac
self.dstmac_cache[self.name] = remotemac
def __init__(self, address_family, fib, log, log_id):
assert fib.address_family == address_family
self.address_family = address_family
self.destinations = PyTricia()
self.fib = fib
self._log = log
self._log_id = log_id
def __init__(self, debug=False):
super(NetworkTopology, self).__init__()
# name
self.name = 'unnamed'
# initialize logging
self.debug = debug
self.logger = get_logger('NetworkTopology', 'DEBUG' if debug else 'INFO')
# dataplane engine that computes forwarding and dominator graphs
self.dp_engine = None
# dict of router name to router
self.routers = dict()
# mapping of router and interface to the next hop router
self.next_hops = defaultdict(dict)
# dict of all subnets in network to router and interface name directly on that subnet
self.subnets = defaultdict(PyTricia)
# dict of dst subnet to edges which are blocked by simple ACLs
self.simple_acls = PyTricia()
self.links = list()
def __init__(self):
super(ForwardingTable, self).__init__()
# initialize logging
self.logger = get_logger('FECFinder', 'INFO')
# init Trie
self.fib = PyTricia()
def __configure_routing(self):
for n in self.graph:
self.routing[n] = single_source_dijkstra_path(self.graph, n)
self.ipdestlpm = PyTricia()
for n, d in self.graph.nodes_iter(data=True):
dlist = d.get('ipdests', '').split()
for destipstr in dlist:
ipnet = ipaddr.IPNetwork(destipstr)
xnode = {}
self.ipdestlpm[str(ipnet)] = xnode
if 'dests' in xnode:
xnode['dests'].append(n)
else:
xnode['net'] = ipnet
xnode['dests'] = [n]
# install static forwarding table entries to each node
# FIXME: there's a problematic bit of code here that triggers
# pytricia-related (iterator) core dump
for nodename, nodeobj in self.nodes.iteritems():
if isinstance(nodeobj, Router):
for prefix in self.ipdestlpm.keys():
lpmnode = self.ipdestlpm.get(prefix)
if nodename not in lpmnode['dests']:
routes = self.routing[nodename]
for d in lpmnode['dests']:
try:
path = routes[d]
except KeyError:
self.logger.warn(
"No route from {} to {}".format(
nodename, d))
continue
nexthop = path[1]
nodeobj.addForwardingEntry(prefix, nexthop)
self.owdhash = {}
for a in self.graph:
for b in self.graph:
key = a + ':' + b
rlist = [a]
while rlist[-1] != b:
nh = self.nexthop(rlist[-1], b)
if not nh:
self.logger.debug(
'No route from %s to %s (in owd; ignoring)' %
(a, b))
return None
rlist.append(nh)
owd = 0.0
for i in xrange(len(rlist) - 1):
owd += self.delay(rlist[i], rlist[i + 1])
self.owdhash[key] = owd
def __init__(self, name, measurement_config, **kwargs):
Node.__init__(self, name, measurement_config, **kwargs)
self.autoack=bool(eval(str(kwargs.get('autoack','False'))))
self.forwarding_table = PyTricia(32)
self.default_link = None
from fslib.configurator import FsConfigurator
ipa,ipb = [ ip for ip in next(FsConfigurator.link_subnetter).iterhosts() ]
self.add_link(NullLink, ipa, ipb, 'remote')
self.trafgen_ip = str(ipa)
def sidr_peer_deflection_sim(G,
dests,
service_types=DEFAULT_SERVICE_TYPES,
network_ratio=0.5,
prefix_ratio=0.2,
**kwargs):
"""
Simulate false-positive cases of SIDR between peer SDXes
"""
peer1 = PyTricia()
peer2 = PyTricia()
for dest in dests:
node_prefixes = G.node[dest]['ip-prefixes']
sidr_random_deflect(node_prefixes, peer1, service_types, network_ratio,
prefix_ratio)
sidr_random_deflect(node_prefixes, peer2, service_types, network_ratio,
prefix_ratio)
total_reject_cnt, tp_reject_cnt = sidr_reject(peer1, peer2)
print(total_reject_cnt, tp_reject_cnt)
def __init__(self, fib):
self._fib = fib
# 1. The real production code for the RIB (but not the FIB) needs Destination objects and
# Route objects to support multiple routes to the same destination (prefix): one per owner
# (e.g. south-SPF and north-SPF). This prototype only supports a single route per
# destination (prefix) to keep things simple and avoid distracting attention from the
# negative disaggregation algorithm.
# 2. The PyTricia code takes prefixes as strings. Not sure if this is the best choice
# for production code. I suspect it is better to accept prefixes as Prefix objects that
# have an internal binary representation.
self._routes = PyTricia()
def read_topo(filepath):
# type: (str) -> networkx.Graph
topo = yaml.load(open(filepath))
nodes = topo["nodes"]
links = topo["links"]
G = networkx.DiGraph()
G.ip_prefixes = PyTricia()
for node_name in nodes:
node = nodes[node_name]
node_id = node['id']
G.add_node(node_id)
node_obj = G.nodes[node_id]
node_obj['id'] = node_id
node_obj['type'] = node['type']
node_obj['asn'] = node.get('asn', None)
node_obj['ip-prefixes'] = node.get('ip-prefixes', set()) or set()
node_obj['ip'] = node.get('ip', None)
node_obj['providers'] = set(node.get('providers', []))
node_obj['customers'] = set(node.get('customers', []))
node_obj['peers'] = set(node.get('peers', []))
node_obj['name'] = node_name
if 'topology-name' in node:
node_obj['topology-name'] = node['topology-name']
for prefix in node_obj['ip-prefixes']:
G.ip_prefixes[prefix] = node_id
for link in links:
G.add_edge(*link)
G.add_edge(*link[::-1])
for node_id in G.nodes():
node_obj = G.node[node_id]
node_obj['adj-ribs-in'] = {n: PyTricia() for n in G.neighbors(node_id)}
node_obj['rib'] = PyTricia()
node_obj['adj-ribs-out'] = {
n: PyTricia()
for n in G.neighbors(node_id)
}
node_obj['local_policy'] = PyTricia()
return G
def fp_affected_volume(G, fp_rejected_policy, srcs, flows):
"""
"""
src_trie = PyTricia()
volume = 0
for src in srcs:
node_prefixes = G.node[src]['ip-prefixes']
for prefix in node_prefixes:
src_trie[prefix] = 1
for flow in flows:
if src_trie.get(flow['src_ip'], 0) and \
fp_rejected_policy.get(flow['dst_ip'], {}).get(flow['dst_port'], 0):
volume += flow['volume']
return volume
def transform(g: Graph, c_as):
g.ip_prefixes = PyTricia()
for n in g.nodes():
cust = set()
prov = set()
peer = set()
for n1, n2 in g.edges(n):
e = g.edges[n1, n2]
if e['rel'] == 'pc':
cust.add(n2)
elif e['rel'] == 'cp':
prov.add(n2)
elif e['rel'] == 'pp':
peer.add(n2)
# g.node[n]['asn'] = n
g.node[n]['customers'] = cust
g.node[n]['providers'] = prov
g.node[n]['peers'] = peer
g.node[n]['ip-prefixes'] = set()
# g.node[n]['ip'] = None
g.node[c_as]['ip-prefixes'].add("1.1.1.1/24")
g.ip_prefixes["1.1.1.1/24"] = c_as
for node_id in g.nodes():
node_obj = g.node[node_id]
node_obj['adj-ribs-in'] = {n: PyTricia() for n in g.neighbors(node_id)}
node_obj['rib'] = PyTricia()
node_obj['adj-ribs-out'] = {
n: PyTricia()
for n in g.neighbors(node_id)
}
node_obj['local_policy'] = PyTricia()
for n in g.nodes():
g.node[n]['rib'] = PyTricia()
for prefix in g.node[n]['ip-prefixes']:
update_initial_rib(g.node[n]['rib'], prefix, True)
# out_ribs = G.node[n]['adj-ribs-out']
# for d in out_ribs:
# update_initial_rib(out_ribs[d], prefix, n)
g.node[n]['adj-ribs-in'] = {n: PyTricia() for n in g.neighbors(n)}
g.node[n]['adj-ribs-out'] = {n: PyTricia() for n in g.neighbors(n)}
def generate_funny_block_policy(G,
service_types=DEFAULT_SERVICE_TYPES,
network_ratio=0.5,
prefix_ratio=0.2,
policy_type='both',
**kwargs):
magic_ratio = 3 # Magic!
funny_ports = [
p for p in range(1, 65536) if p not in DEFAULT_SERVICE_TYPES.keys()
]
# funny_ports = [21, 80, 22, 23, 24, 25, 32, 44, 98]
edge_networks = [
n for n in G.nodes() if G.node[n].get('type', '') == 'edge'
]
transit_networks = [
n for n in G.nodes()
if G.node[n].get('type', '') == 'transit' and n != 1
]
for d in random.sample(
transit_networks,
math.ceil(
len(transit_networks) * float(network_ratio) / magic_ratio)):
# print(d)
if 'local_policy' not in G.node[d]:
G.node[d]['local_policy'] = PyTricia()
local_policy = G.node[d]['local_policy']
for dest in random.sample(
[n for n in edge_networks if n != d],
math.ceil(len(edge_networks) * float(network_ratio))):
node_prefixes = G.node[dest]['ip-prefixes']
for prefix in random.sample(
node_prefixes,
math.ceil(len(node_prefixes) * float(prefix_ratio))):
local_policy[prefix] = {
port: None
for port in random.sample(funny_ports, random.randint(
1, 4))
}
return G
def generate_fully_random_policy(G,
service_types=DEFAULT_SERVICE_TYPES,
network_ratio=0.5,
prefix_ratio=0.2,
policy_place='transit',
policy_type='both',
**kwargs):
edge_networks = [
n for n in G.nodes() if G.node[n].get('type', '') == 'edge'
]
transit_networks = [
n for n in G.nodes() if G.node[n].get('type', '') == 'transit'
]
if policy_place == 'transit':
policy_networks = transit_networks
elif policy_place == 'edge':
policy_networks = edge_networks
else:
policy_networks = G.nodes()
for d in policy_networks:
if 'local_policy' not in G.node[d]:
G.node[d]['local_policy'] = PyTricia()
local_policy = G.node[d]['local_policy']
for dest in random.sample(
[n for n in edge_networks if n != d],
math.ceil(len(edge_networks) * float(network_ratio))):
node_prefixes = G.node[dest]['ip-prefixes']
for prefix in random.sample(
node_prefixes,
math.ceil(len(node_prefixes) * float(prefix_ratio))):
local_policy[prefix] = {
port: gen_single_policy(G, d, dest, policy_type)
for port in random.sample(service_types.keys(),
random.randint(1, 4))
}
return G
def main() -> None:
args = parser.parse_args()
# Set source IP addresses
addrs: Sequence[AnyIPAddress] = args.bind or list(_select_addrs())
ipv4_src = next((a for a in addrs if isinstance(a, IPv4Address)), None)
ipv6_src = next((a for a in addrs if isinstance(a, IPv6Address)), None)
del addrs
# Create blacklist patricia trees from multiple sources
ipv4_bl: Optional["PyTricia[bool]"] = None if ipv4_src is None else PyTricia(32, socket.AF_INET)
ipv6_bl: Optional["PyTricia[bool]"] = None if ipv6_src is None else PyTricia(128, socket.AF_INET6)
for net in itertools.chain.from_iterable(args.blacklist):
if isinstance(net, IPv4Network) and ipv4_bl is not None and net not in ipv4_bl:
ipv4_bl[net] = True
elif ipv6_bl is not None and isinstance(net, IPv6Network) and net not in ipv6_bl:
ipv6_bl[net] = True
duplicate_filter: Set[ScanHost] = set()
output_mod = get_output_module(args.output)
# Aggregate targets from multiple sources and filter them by IP version and blacklist
# Discarded and filtered targets are output immediately to allow their memory to be freed
def target_filter(tgt: ScanHost) -> bool:
if isinstance(tgt.ip, IPv4Address):
if ipv4_bl is None or any(tgt.ip in net for net in _IPV4_DISC_NETS) or tgt.ip.is_multicast:
output_mod.discarded_target(tgt)
return False
elif tgt.ip in ipv4_bl or tgt in duplicate_filter:
output_mod.filtered_target(tgt)
return False
elif isinstance(tgt.ip, IPv6Address):
if ipv6_bl is None or any(tgt.ip in net for net in _IPV6_DISC_NETS) or tgt.ip.is_multicast:
output_mod.discarded_target(tgt)
return False
elif tgt.ip in ipv6_bl or tgt in duplicate_filter:
output_mod.filtered_target(tgt)
return False
else:
raise ValueError("tgt is neither IPv4 nor IPv6")
duplicate_filter.add(tgt)
return True
# Filtering happens lazily within this generator (whenever a new chunk is requested)
tgt_gen: Iterable[ScanHost] = filter(target_filter, itertools.chain(
args.target, itertools.chain.from_iterable(args.nmap),
itertools.chain.from_iterable(args.zmap), itertools.chain.from_iterable(args.json)
))
# Setup sockets/multiplexers
# Both multiplexers share a token bucket with a precision of 1/8th of a second
# and which allows bursts of up to half a second's worth of packets
limiter = TokenBucket(args.rate // 8 or 1, 0.125, args.rate // 2 or 1)
loop = asyncio.SelectorEventLoop()
ipv4_plex = None if ipv4_src is None else IPv4TestMultiplexer(ipv4_src, limiter, loop=loop)
ipv6_plex = None if ipv6_src is None else IPv6TestMultiplexer(ipv6_src, limiter, loop=loop)
try:
pairs: Iterable[Tuple[Type[BaseTest], Type[BaseTest]]] = zip(itertools.repeat(LivenessTest),
parse_test_list(args.tests))
active_tests = [test for pair in pairs for test in pair]
active_tests.append(LivenessTest) # Additional final liveness test
del pairs
except ValueError as e:
parser.error(str(e))
return # not strictly necessary, but helps type checkers
# Run tests sequentially for chunks of targets such that they don't
# run into the packet rate limit. Since MAX_PACKET_RATE is generally
# an overestimation, we do not include a safety buffer here.
chunksize = round(args.rate / overall_packet_rate(active_tests))
chunk_gen = (list(itertools.islice(tgt_gen, chunksize)) for _ in itertools.repeat(None))
first = True
for chunk in chunk_gen:
output_mod.flush() # chunk creation may have triggered new DESC/FLTR outputs
if not chunk:
if first:
parser.error("at least one valid target is required")
break # chunk is empty -> tgt_gen is empty -> all targets have been processed
first = False
tgt_futs: Dict[ScanHost, List["asyncio.Future[TestResult]"]] = {tgt: [] for tgt in chunk}
for test in active_tests:
all_futs: List["asyncio.Future[TestResult]"] = []
random.shuffle(chunk)
src_port = next(_PORT_SEQ)
ipv4_host = None if ipv4_src is None else ScanHost(ipv4_src, src_port)
ipv6_host = None if ipv6_src is None else ScanHost(ipv6_src, src_port)
# Passing the test as a default parameter to the lambda ensures
# that the variable is not overwritten by further iterations of the loop
for tgt in chunk:
if isinstance(tgt.ip, IPv4Address):
t = test(ipv4_host, tgt, loop=loop) # type: ignore
ipv4_plex.register_test(t) # type: ignore
fut = loop.create_task(t.run())
fut.add_done_callback(lambda f, t=t: ipv4_plex.unregister_test(t)) # type: ignore
else:
t = test(ipv6_host, tgt, loop=loop) # type: ignore
ipv6_plex.register_test(t) # type: ignore
fut = loop.create_task(t.run())
fut.add_done_callback(lambda f, t=t: ipv6_plex.unregister_test(t)) # type: ignore
tgt_futs[tgt].append(fut)
all_futs.append(fut)
print("Running", test.__name__)
loop.run_until_complete(asyncio.wait(all_futs, loop=loop))
if test.FAIL_EARLY:
# Skip further tests for targets with a failure in the current test
chunk = [tgt for tgt, fut in zip(chunk, all_futs) if result_not_failed(fut)]
time.sleep(3)
for tgt, results in tgt_futs.items():
output_mod(tgt, active_tests, results)
output_mod.flush()
from pytricia import PyTricia
# FIXME: They are global variable and need to be removed
ip_prefixes = PyTricia()
global_policy = dict() # type: dict[str, PyTricia]
def update_initial_rib(rib, prefix, overwrite=True):
"""
Update the initial the rib for a prefix
"""
if overwrite or prefix not in rib:
rib[prefix] = {0: []}
else:
rib[prefix].update({0: []})
def initiate_ribs(G, overwrite=True):
"""
Initiate ribs for each network in G.
"""
for n in G.nodes():
G.node[n]['rib'] = PyTricia()
for prefix in G.node[n]['ip-prefixes']:
update_initial_rib(G.node[n]['rib'], prefix, overwrite)
# out_ribs = G.node[n]['adj-ribs-out']
# for d in out_ribs:
# update_initial_rib(out_ribs[d], prefix, n)
if overwrite:
G.node[n]['adj-ribs-in'] = {n: PyTricia() for n in G.neighbors(n)}
G.node[n]['adj-ribs-out'] = {n: PyTricia() for n in G.neighbors(n)}
def __init__(self):
self._routes = PyTricia()
def get_routes(ifdict):
def get_value(name, info, headers):
if name == 'dst':
dstip = info[headers.index('Destination')]
if dstip == 'default':
dstip = '0.0.0.0/0'
if 'Genmask' in headers:
dstip = "{}/{}".format(dstip, info[headers.index('Genmask')])
try:
rv = IPv4Network(normalize_ipv4(dstip), strict=False)
return rv
except:
return None
elif name == 'gw':
return info[headers.index('Gateway')]
elif name == 'iface':
if 'Iface' in headers:
return info[headers.index('Iface')]
elif 'Netif' in headers:
return info[headers.index('Netif')]
def is_ipaddr(s):
return re.match(r'\d+(\.\d+){2,3}', s)
def normalize_ipv4(s):
if '/' in s:
prefix, plen = s.split('/')
while prefix.count('.') < 3:
prefix += ".0"
s = '/'.join((prefix, plen))
return s
def is_ethaddr(s):
return re.match(r'[0-9a-fA-F]{1,2}(:[0-9a-fA-F]{1,2}){5}', s)
routes = PyTricia(32)
gwips = defaultdict(list)
cmd = "netstat -r -n"
if sys.platform == 'darwin':
cmd += ' -f inet'
s = check_output(cmd, shell=True, universal_newlines=True)
headers = []
for line in s.split('\n'):
dst = gwip = iface = None
if line.startswith('Destination'):
headers = line.split()
elif len(line) > 0 and \
(line[0].isdigit() or line.startswith('default')):
info = line.split()
dst = get_value('dst', info, headers)
if dst is None:
continue
gwip = get_value('gw', info, headers)
iface = get_value('iface', info, headers)
# skip localhost and multicast
if dst.is_multicast or dst.is_loopback:
continue
if iface not in ifdict:
continue
if is_ipaddr(gwip):
gwip = IPv4Address(gwip)
elif is_ethaddr(gwip):
continue
else:
gwip = IPv4Address('0.0.0.0')
# print(dst,gwip,iface)
if routes.has_key(dst):
# print("Already have nh for {}: {}".format(dst, routes[dst]))
ii = routes[dst]
else:
nh = NextHop(dst, iface, gwip)
routes[dst] = nh
return routes
