def test_any_node():
"""Any Node."""
r = AnyNode()
a = AnyNode()
b = AnyNode(foo=4)
eq_(r.parent, None)
eq_(a.parent, None)
eq_(b.parent, None)
a.parent = r
b.parent = r
eq_(r.children, (a, b))
eq_(repr(r), "AnyNode()")
eq_(repr(a), "AnyNode()")
eq_(repr(b), "AnyNode(foo=4)")
def get_node_tree():
""" reads nodes from topology and builds tree """
topology = read()
tree_nodes = []
for link in topology['links']:
parent_ip = link['source']
child_ip = link['target']
link_capacity = link['properties']['capacity'][0]
parent = next((x for x in tree_nodes if x.id == parent_ip), None)
if not parent:
parent_node = next((x for x in topology['nodes'] if x['id'] == parent_ip), None)
parent = AnyNode(id=parent_node['id'], label=parent_node['label'],
properties=parent_node['properties'], capacity=float("inf"))
tree_nodes.append(parent)
child = next((x for x in tree_nodes if x.id == child_ip), None)
if not child:
child_node = next((x for x in topology['nodes'] if x['id'] == child_ip), None)
child = AnyNode(id=child_node['id'], parent=parent, label=child_node['label'],
properties=child_node['properties'], capacity=min(parent.capacity, link_capacity))
tree_nodes.append(child)
child.parent = parent
root = next((x for x in tree_nodes if x.parent is None), None)
# with open(config.TOPOLOGY_PATH + '.txt', 'w') as topology_file:
# topology_file.write(u' '.join(RenderTree(root).by_attr('id')).encode('utf-8'))
return root
def NormalState(self, node: AnyNode):
"""
This function sets the state of a node in the GraphTree.
:param node:AnyNode: a node
"""
try:
if (node.is_leaf) and not (node.is_root):
node.state = 'destination'
node.name = 'destination_' + str(node.id)
elif (node.is_root):
node.state = 'root'
node.name = 'root_' + str(node.id)
node.parent = None
else:
node.state = 'subnode'
node.name = 'subnode_' + str(node.id)
except Exception as ex:
template = "An exception of type {0} occurred in [TParser.NormalState]. Arguments:\n{1!r}"
message = template.format(type(ex).__name__, ex.args)
print(message)
def question4(filename,
new_names,
prefix_length_max,
prefix_length_min,
prefix_length_step,
rows,
use_saved_model=False):
"""Aggregate the subnetworks responsible the top 10, the top 1 and the top 0.1 percent of traffic by volume.
This is done using an N-ary tree using the anytree library.
Writes the subnets to a file and creates a graphical representation of the resulting tree.
Parameters:
filename (String): The name of the file containing the data
new_names (String): The new names of the columns of the data
prefix_length_max (int): Max prefix length to consider
prefix_length_min (int): Min prefix length to consider
prefix_length_step (int): Steps from one prefix length to another, positive
rows (int): Number of rows in the dataframe to consider
use_saved_model (boolean): Whether we wish to use the grouped dataframe saved to disk or not
Return:
/
"""
# Use saved pickle of grouped dataframe if wished so
if (not use_saved_model):
df = pd.read_csv(filename,
header=0,
delimiter=',',
names=new_names,
usecols=['src_addr', 'in_bytes'],
nrows=rows)
total_traffic = df['in_bytes'].sum()
np.save('total_traf.npy', total_traffic)
countSubnets(df, "77.102.101.0/16", 24)
# Count the IP addresses and sum their traffic
df = df.groupby('src_addr', sort=False).agg({
'src_addr': 'count',
'in_bytes': 'sum'
})
df = df.rename_axis(None).reset_index()
df.columns = ['src_addr', 'src_addr_frequency', 'sum_in_bytes']
df.to_pickle("df_groupby_q4.pkl")
else:
df = pd.read_pickle("df_groupby_q4.pkl")
total_traffic = np.load('total_traf.npy')
# Node pointing at upper_level subnets
curr_root = AnyNode(ip="root", frequency=0, traffic=0)
# Create a tree to aggregate subnets
for cnt, prefix_length in enumerate(
range(prefix_length_max, prefix_length_min, -prefix_length_step)):
if (cnt == 0):
# Process every IP address
for _, row in df.iterrows():
subnet_of_ip = extractPrefix(row['src_addr'], prefix_length,
True)
# Find in the tree if the subnet exists alerady
existing_subnet = find(
node=curr_root,
filter_=lambda node: node.ip == subnet_of_ip,
maxlevel=2)
# If not, create a new one
subnet = None
if (existing_subnet is None):
subnet = AnyNode(parent=curr_root,
ip=subnet_of_ip,
frequency=0,
traffic=0)
else:
subnet = existing_subnet
# Add the child (the ip address) and update the parent (subnet)
AnyNode(parent=subnet,
ip=row['src_addr'],
frequency=row['src_addr_frequency'],
traffic=row['sum_in_bytes'])
subnet.traffic += row['sum_in_bytes']
subnet.frequency += row['src_addr_frequency']
else:
new_root = AnyNode(
ip="new_root", frequency=0,
traffic=0) # Node pointing at upper_level subnets
# The current root contains the subnets
for subnet in curr_root.children:
subnet_ip = str(subnet.ip).split('/')[0]
subnet_of_subnet = extractPrefix(subnet_ip, prefix_length,
True)
# Find in the tree if the subnet exists alerady
existing_subnet = find(
node=new_root,
filter_=lambda node: node.ip == subnet_of_subnet,
maxlevel=2)
# If not, create a new one and add the current subnet as his child
if (existing_subnet is None):
AnyNode(parent=new_root,
children=[subnet],
ip=subnet_of_subnet,
frequency=subnet.frequency,
traffic=subnet.traffic)
else:
subnet.parent = existing_subnet
existing_subnet.traffic += subnet.traffic
existing_subnet.frequency += subnet.frequency
# Update the current root
curr_root = new_root
curr_root.ip = "root"
# Update root node with it's childs attributes
curr_root.traffic = sum(x.traffic for x in curr_root.children)
curr_root.frequency = sum(x.frequency for x in curr_root.children)
# # Save the tree as a graph
# createTree(curr_root)
# Count the number of prefix above a certain traffic threshold
max_level = int(
(prefix_length_max - prefix_length_min) / prefix_length_step) + 1
top10_prefix_nodes = findall(
node=curr_root,
filter_=lambda node: node.traffic / total_traffic >= 0.10,
maxlevel=max_level)
top1_prefix_nodes = findall(
node=curr_root,
filter_=lambda node: node.traffic / total_traffic >= 0.01,
maxlevel=max_level)
top01_prefix_nodes = findall(
node=curr_root,
filter_=lambda node: node.traffic / total_traffic >= 0.001,
maxlevel=max_level)
top10_prefix = [x.ip for x in top10_prefix_nodes]
top1_prefix = [x.ip for x in top1_prefix_nodes]
top01_prefix = [x.ip for x in top01_prefix_nodes]
with open('top10_prefix.txt', 'w') as f:
for item in top10_prefix:
f.write("{}\n".format(item))
with open('top1_prefix.txt', 'w') as f:
for item in top1_prefix:
f.write("{}\n".format(item))
with open('top01_prefix.txt', 'w') as f:
for item in top01_prefix:
f.write("{}\n".format(item))
def BuildTree(std: dict, stl: dict):
idTree = None
idNodes = dict()
idDirMap = dict()
for dir_id in sorted(std):
dir = std[dir_id]
n = AnyNode(name=dir_id)
if dir["parent"] is None:
idTree = n
idNodes[dir_id] = n
idDirMap[dir_id] = dir
for dir_id in sorted(std):
dir = std[dir_id]
n = idNodes[dir_id]
if dir["parent"] is None:
continue
n.parent = idNodes[dir["parent"]]
for dir_id in sorted(stl):
dir = stl[dir_id]
n = AnyNode(name=dir_id)
if dir["parent"] is None:
idTree = n
idNodes[dir_id] = n
idDirMap[dir_id] = dir
for dir_id in sorted(stl):
dir = stl[dir_id]
n = idNodes[dir_id]
if dir["parent"] is None:
continue
n.parent = idNodes[dir["parent"]]
# print(RenderTree(idTree))
nameNode = None
namedNodeDict = dict()
node: Node
for node in PreOrderIter(idTree):
dir = idDirMap[node.name]
# Generic attributes
name = dir["name"]
deleted = dir["deleted"]
updated = dir["updated"]
t = "Directory" if "markAsAddon" in dir else "File"
if t == "Directory":
# Folder only attributes
markAsAddon = dir["markAsAddon"]
hidden = dir["hidden"]
n = AnyNode(name=name,
type=t,
deleted=deleted,
updated=updated,
markAsAddon=markAsAddon,
hidden=hidden)
else:
# File only attributes
size = dir["size"]
compressedSize = dir["compressedSize"]
compressed = dir["compressed"]
sha1 = dir["sha1"]
n = AnyNode(name=name,
type=t,
deleted=deleted,
updated=updated,
size=size,
compressedSize=compressedSize,
compressed=compressed,
sha1=sha1)
namedNodeDict[node.name] = n
if node.parent is None:
nameNode = n
continue
parent = node.parent
n.parent = namedNodeDict[parent.name]
return nameNode
