def _build_clusters_no_overlapping(self, graph):
print(nx.dijkstra_path(graph, '0', '7', 'weight'))
print(nx.dijkstra_path_length(graph, '0', '7', 'weight'))
paths_for_diameter = nx.shortest_path_length(graph, weight='weight')
for path in nx.shortest_path_length(graph, weight='weight'):
print(path)
ecc = nx.eccentricity(graph, sp=dict(paths_for_diameter))
# ecc = nx.eccentricity(graph, sp=shortest_paths_for_diameter)
diameter = nx.diameter(graph, e=ecc)
# for path in paths_for_diameter:
# print(path)
print('The graph diameter is ', diameter)
height_of_cluster = math.ceil(math.log(diameter, 2)) + 1
print('height_of_the hierarchy is ', height_of_cluster)
# lowest level clusters
print('lowest level clusters')
for n in graph.nodes():
paths = nx.single_source_dijkstra_path_length(graph, n, 0, weight='weight')
print(paths)
cluster_graph = graph.subgraph([n])
cluster = Cluster('c' + str(n) + '_l' + '0', cluster_graph, 0)
self._network.add_cluster_level(0)
self._network.add_cluster(0, cluster)
self._network.draw_cluster(cluster.cluster_id)
for i in range(int(height_of_cluster)):
self._network.add_cluster_level(i+1)
print('AT LEVEL ------- ', i + 1)
distance = pow(2, i + 1)
print('THE DISTANCE LIMIT IS ', distance)
clustered_peers = []
for n in range(self._peer_count):
# for n in graph.nodes():
print('clustering peer ', n)
if n in clustered_peers:
print('peer ', n, ' already clustered')
continue
paths_found = nx.single_source_dijkstra_path_length(graph, str(n), distance, weight='weight')
peers_to_cluster = []
print('paths found in the level ', paths_found)
for peer in paths_found:
if int(peer) in clustered_peers:
print('peer already clustered in another cluster')
else:
clustered_peers.append(int(peer))
peers_to_cluster.append(int(peer))
cluster_graph = graph.subgraph([str(i) for i in peers_to_cluster])
# cluster = Cluster('c' + str(n) + '_l' + str(i + 1), peers_to_cluster)
cluster = Cluster('c' + str(n) + '_l' + str(i + 1), cluster_graph, i + 1)
self._network.add_cluster(i + 1, cluster)
def __init__(self, operationCount, peerCount, graphInput):
print("Inside Simulation init")
self._op_count = int(operationCount)
self._peer_count = int(peerCount)
plt.axis('off')
graph = nx.read_weighted_edgelist(graphInput)
print(graph)
# some properties
print("node degree clustering")
for v in nx.nodes(graph):
print('%s %d %f' % (v, nx.degree(graph, v), nx.clustering(graph, v)))
# print the adjacency list to terminal
try:
nx.write_adjlist(graph, sys.stdout)
except TypeError:
nx.write_adjlist(graph, sys.stdout.buffer)
# node_pos = nx.spring_layout(graph)
#
# edge_weight = nx.get_edge_attributes(graph, 'weight')
# # Draw the nodes
# nx.draw_networkx(graph, node_pos, node_color='grey', node_size=100)
# # Draw the edges
# nx.draw_networkx_edges(graph, node_pos, edge_color='black')
# # Draw the edge labels
# nx.draw_networkx_edge_labels(graph, node_pos, edge_color='red', edge_labels=edge_weight)
# plt.show()
paths_for_diameter = nx.shortest_path_length(graph, weight='weight')
ecc = nx.eccentricity(graph, sp=dict(paths_for_diameter))
self._diameter = nx.diameter(graph, e=ecc)
print('The graph diameter is ', self._diameter)
self._height_of_cluster = math.ceil(math.log(self._diameter, 2)) + 1
if not os.path.exists(graphInput+'_network'):
print("CREATING NEW NETWORK")
self._network = Network(graph)
self._setup_peers(graph)
# self._build_clusters_exactly_logn_membership(graph)
self._build_clusters_at_least_one_and_at_most_logn(graph)
# self._build_clusters_no_overlapping(graph)
self._build_tree(graph)
self.save_network(graphInput + '_network')
exit(0)
else:
print("LOADING NETWORK FROM INPUT FILE")
# load network
self.load_network(graphInput + '_network')
def graph_properties(G, n, p):
# graph properties
A = nx.adjacency_matrix(G).toarray()
L = nx.laplacian_matrix(G).toarray()
betwn_ctrlity = nx.betweenness_centrality(G)
names = ['id', 'data']
formats = ['i', 'f8']
dtype = dict(names=names, formats=formats)
betwn_ctrlity_array = np.array(list(betwn_ctrlity.items()), dtype=dtype)
c = nx.average_clustering(G)
d = nx.diameter(G)
np.savetxt('Q8_Assets/Adjacency_N_' + str(n) + '_P_' + str(p) + '.csv',
A,
delimiter=",")
np.savetxt('Q8_Assets/Laplacian_N_' + str(n) + '_P_' + str(p) + '.csv',
L,
delimiter=",")
np.savetxt('Q8_Assets/Betweeness_N_' + str(n) + '_P_' + str(p) + '.csv',
betwn_ctrlity_array,
delimiter=",")
print('Clustering = ', c, '\n')
print('Diameter = ', d, '\n')
# write to a file
# nx.write_gml(G, "test_gml")
# json_graph.node_link_data(G)
# nx.write_edgelist(G,"test.edgeList")
# print the adjacency list to terminal
print("THE ADJACENCY LIST IS ")
# try:
# nx.write_adjlist(G, sys.stdout)
# except TypeError:
# nx.write_adjlist(G, sys.stdout.buffer)
for a in [(n, nbrdict) for n, nbrdict in G.adjacency()]:
print(a)
for i in range(0, n):
print("NODE ", i)
temp = []
for a in [n for n in G.neighbors(i)]:
temp.append(a)
print(temp)
assert len(temp) > 0
# diameter
paths_for_diameter = nx.shortest_path_length(G, weight='weight')
ecc = nx.eccentricity(G, sp=dict(paths_for_diameter))
diameter = nx.diameter(G, e=ecc)
print("OK diameter without weight?", nx.diameter(G))
print('The graph diameter with weight is ', diameter)
# nx.write_weighted_edgelist(G, str(n)+'_'+str(p)+'diamter'+str(diameter)+'_newtest.weighted.edgelist')
for p in spl:
pathlengths.append(spl[p])
print('')
print("average shortest path length %s" %
(sum(pathlengths) / len(pathlengths)))
# histogram of path lengths
dist = {}
for p in pathlengths:
if p in dist:
dist[p] += 1
else:
dist[p] = 1
print('')
print("length #paths")
verts = dist.keys()
for d in sorted(verts):
print('%s %d' % (d, dist[d]))
print("radius: %d" % nx.radius(G))
print("diameter: %d" % nx.diameter(G))
print("eccentricity: %s" % nx.eccentricity(G))
print("center: %s" % nx.center(G))
print("periphery: %s" % nx.periphery(G))
print("density: %s" % nx.density(G))
nx.draw(G, with_labels=True)
plt.show()
print('{} {} '.format(v, spl))
for p in spl:
pathlengths.append(spl[p])
print('')
print("average shortest path length %s" % (sum(pathlengths) / len(pathlengths)))
# histogram of path lengths
dist = {}
for p in pathlengths:
if p in dist:
dist[p] += 1
else:
dist[p] = 1
print('')
print("length #paths")
verts = dist.keys()
for d in sorted(verts):
print('%s %d' % (d, dist[d]))
print("radius: %d" % nx.radius(G))
print("diameter: %d" % nx.diameter(G))
print("eccentricity: %s" % nx.eccentricity(G))
print("center: %s" % nx.center(G))
print("periphery: %s" % nx.periphery(G))
print("density: %s" % nx.density(G))
nx.draw(G, with_labels=True)
plt.show()
def _build_clusters_at_least_one_and_at_most_logn(self, graph):
paths_for_diameter = nx.shortest_path_length(graph, weight='weight')
for path in nx.shortest_path_length(graph, weight='weight'):
print(path)
ecc = nx.eccentricity(graph, sp=dict(paths_for_diameter))
diameter = nx.diameter(graph, e=ecc)
print('The graph diameter is ', diameter)
height_of_cluster = math.ceil(math.log(diameter, 2)) + 1
self._network.set_height_of_clusters(height_of_cluster)
print('height_of_the hierarchy is ', height_of_cluster)
# lowest level clusters
print('lowest level clusters')
self._network.add_cluster_level(0)
# level 0 clusters
for n in graph.nodes():
paths = nx.single_source_dijkstra_path_length(graph, n, 0, weight='weight')
print(paths)
cluster_graph = graph.subgraph([n])
cluster = Cluster('c' + str(n) + '_l' + '0', cluster_graph, 0, str(n))
self._network.add_cluster(0, cluster)
# self._network.draw_cluster(cluster.cluster_id)
# form upper level clusters
for i in range(int(height_of_cluster)):
self._network.add_cluster_level(i + 1)
clustered_peers_list = []
# for naming the cluster properly
cluster_ids_list = []
print ('AT LEVEL ----- ', i + 1)
distance = pow(2, i + 1)
print('THE DISTANCE LIMIT IS ', distance)
clusterize = {}
n = 0
# iterate over the peers once
while n < self._peer_count:
print('clustering peer ', n)
paths_found = nx.single_source_dijkstra_path_length(graph, str(n), distance, weight='weight')
peers_to_cluster = []
print('paths found in the level ', paths_found)
tmp_peers_list_to_cluster = []
for peer in paths_found:
peers_to_cluster.append(peer)
# clustered_peers_list.append(peer)
tmp_peers_list_to_cluster.append(peer)
c_id = 'c' + str(n) + '_l' + str(i + 1)
# for naming the clusters properly
cluster_ids_list.append(c_id)
cluster_ids_count = Counter(cluster_ids_list)
c_id = c_id + "_" + str(cluster_ids_count[c_id])
temp_clustered_peers_list = copy.deepcopy(clustered_peers_list)
temp_clustered_peers_list.extend(tmp_peers_list_to_cluster)
duplicate = False
if self.at_most_logn(Counter(temp_clustered_peers_list)) < 0:
# check if duplicate peers or not
for inner_key, inner_value in clusterize.items():
if Counter(inner_value) == Counter(tmp_peers_list_to_cluster):
print ("FOUND DUPLICATE CLUSTER")
duplicate = True
break
print("DOESN'T VIOLATE AT MOST LOG N")
if not duplicate:
clustered_peers_list.extend(tmp_peers_list_to_cluster)
clusterize[c_id] = peers_to_cluster
n += 1
print("CHECKING MEMBERSHIP")
print(self.at_least_one(Counter(clustered_peers_list)) < 0)
print(self.at_most_logn(Counter(clustered_peers_list)) < 0)
assert (self.at_most_logn(Counter(clustered_peers_list)) < 0)
# if not built yet build a cluster and remove peers who appear more than logn times
missing_cluster_id = self.at_least_one(Counter(clustered_peers_list))
while missing_cluster_id > -1:
print("PEER ", missing_cluster_id, " IS MISSING, BUILDING A CLUSTER AROUND IT.")
paths_found = nx.single_source_dijkstra_path_length(graph, str(missing_cluster_id), distance, weight='weight')
peers_to_cluster = []
print('paths found in the level ', paths_found)
tmp_peers_list_to_cluster = []
for peer in paths_found:
peers_to_cluster.append(peer)
# clustered_peers_list.append(peer)
tmp_peers_list_to_cluster.append(peer)
# todo creating single node cluster
tmp_peers_list_to_cluster = [str(missing_cluster_id)]
c_id = 'c' + str(missing_cluster_id) + '_l' + str(i + 1)
# for naming the clusters properly
cluster_ids_list.append(c_id)
cluster_ids_count = Counter(cluster_ids_list)
c_id = c_id + "_" + str(cluster_ids_count[c_id])
temp_clustered_peers_list = copy.deepcopy(clustered_peers_list)
temp_clustered_peers_list.extend(tmp_peers_list_to_cluster)
duplicate = False
if self.at_most_logn(Counter(temp_clustered_peers_list)) < 0:
# check if duplicate peers or not
for inner_key, inner_value in clusterize.items():
if Counter(inner_value) == Counter(tmp_peers_list_to_cluster):
print("FOUND DUPLICATE CLUSTER")
duplicate = True
break
print("DOESN'T VIOLATE AT MOST LOG N")
if not duplicate:
print("WHY HERE")
clustered_peers_list.extend(tmp_peers_list_to_cluster)
clusterize[c_id] = tmp_peers_list_to_cluster
else:
# find the peer that appears more than logn
excess_cluster_id = self.at_most_logn(Counter(temp_clustered_peers_list))
while excess_cluster_id != -1:
print(tmp_peers_list_to_cluster)
print(excess_cluster_id)
tmp_peers_list_to_cluster.remove(str(excess_cluster_id))
tmp_clustered_peers_list = copy.deepcopy(clustered_peers_list)
tmp_clustered_peers_list.extend(tmp_peers_list_to_cluster)
# removing
print(excess_cluster_id, " APPEARS IN MORE THAN LOGN CLUSTER, REMOVING IT")
excess_cluster_id = self.at_most_logn(Counter(tmp_clustered_peers_list))
print("PREPARING THE CLUSTER FROM MODIFIED PEERS LIST ", tmp_peers_list_to_cluster)
clustered_peers_list.extend(tmp_peers_list_to_cluster)
clusterize[c_id] = tmp_peers_list_to_cluster
assert (self.at_most_logn(Counter(clustered_peers_list)) < 0)
missing_cluster_id = self.at_least_one(Counter(clustered_peers_list))
print("ASSERTING")
assert (self.at_least_one(Counter(clustered_peers_list)) < 0)
assert (self.at_most_logn(Counter(clustered_peers_list)) < 0)
print(clusterize)
print(Counter(clustered_peers_list))
print("FINALLY ADDING CLUSTERS")
for key in clusterize:
print (len(clusterize[key]))
print(nx.number_of_nodes(graph.subgraph([str(i) for i in clusterize[key]])))
assert(nx.number_of_nodes(graph.subgraph([str(i) for i in clusterize[key]])) == len(clusterize[key]))
cluster_graph = graph.subgraph([str(i) for i in clusterize[key]])
cluster = Cluster(key, cluster_graph, i + 1)
if i + 1 == height_of_cluster:
cluster.root = True
self._network._root_cluster = cluster
self._network.add_cluster(i + 1, cluster)
# self._network.draw_cluster(cluster.cluster_id)
print("CLUSTERIZE ", clusterize)
return
def _build_clusters_exactly_logn_membership(self, graph):
paths_for_diameter = nx.shortest_path_length(graph, weight='weight')
for path in nx.shortest_path_length(graph, weight='weight'):
print(path)
ecc = nx.eccentricity(graph, sp=dict(paths_for_diameter))
# ecc = nx.eccentricity(graph, sp=shortest_paths_for_diameter)
diameter = nx.diameter(graph, e=ecc)
# for path in paths_for_diameter:
# print(path)
print('The graph diameter is ', diameter)
height_of_cluster = math.ceil(math.log(diameter, 2)) + 1
print('height_of_the hierarchy is ', height_of_cluster)
# lowest level clusters
print('lowest level clusters')
self._network.add_cluster_level(0)
for n in graph.nodes():
paths = nx.single_source_dijkstra_path_length(graph, n, 0, weight='weight')
print(paths)
cluster_graph = graph.subgraph([n])
cluster = Cluster('c' + str(n) + '_l' + '0', cluster_graph, 0)
self._network.add_cluster(0, cluster)
# self._network.draw_cluster(cluster.cluster_id)
for i in range(int(height_of_cluster)):
self._network.add_cluster_level(i + 1)
clustered_peers_list = []
cluster_ids_list = []
print('AT LEVEL ------- ', i + 1)
distance = pow(2, i + 1)
print('THE DISTANCE LIMIT IS ', distance)
clustered_peers = []
# for naming the clusters properly
n = 0
while n < self._peer_count:
incomplete = False
# for n in graph.nodes():
print('clustering peer ', n)
paths_found = nx.single_source_dijkstra_path_length(graph, str(n), distance, weight='weight')
peers_to_cluster = []
print('paths found in the level ', paths_found)
for peer in paths_found:
clustered_peers.append(int(peer))
peers_to_cluster.append(int(peer))
clustered_peers_list.append(peer)
cluster_graph = graph.subgraph([str(i) for i in peers_to_cluster])
c_id = 'c' + str(n) + '_l' + str(i + 1)
cluster_ids_list.append(c_id)
cluster_ids_count = Counter(cluster_ids_list)
c_id = c_id + "_" + str(cluster_ids_count[c_id])
cluster = Cluster(c_id, cluster_graph, i + 1)
self._network.add_cluster(i + 1, cluster)
# self._network.draw_cluster(cluster.cluster_id)
clustered_peers_count = Counter(clustered_peers_list)
# print(self._network._clusters)
for j in range(self._peer_count):
if clustered_peers_count[str(j)] < math.log(self._peer_count, 2):
incomplete = True
if not incomplete:
# make peers fall in exactly log(n) clusters
for peer in range(self._peer_count):
# count of peers in all clusters
peer_count = clustered_peers_count[str(peer)]
while peer_count > math.log(self._peer_count, 2):
for cluster in reversed(self._network.clusters_by_level(i + 1)):
if clustered_peers_count[str(peer)] > math.log(self._peer_count, 2) and cluster.graph.has_node(
str(peer)):
cluster.graph.remove_node(str(peer))
peer_count = peer_count - 1
if str(peer) in clustered_peers_list:
clustered_peers_list.remove(str(peer))
clustered_peers_count = Counter(clustered_peers_list)
break
break
n += 1
if n == self._peer_count:
n = 0
# delete empty clusters
for i in range(int(height_of_cluster)):
for cluster in self._network.clusters_by_level(i):
if nx.number_of_nodes(cluster.graph) == 0:
self._network.remove_cluster_by_id(cluster.cluster_id, cluster.level)
for i in range(int(height_of_cluster)):
for cluster in self._network.clusters_by_level(i):
assert (nx.number_of_nodes(cluster.graph) is not 0)
# make sure that there are log(n) peers in every level
for i in range(int(height_of_cluster) - 1):
print("Verifying in level ", i + 1)
clustered_peer_list = []
for cluster in self._network.clusters_by_level(i + 1):
for node in cluster.graph.nodes:
clustered_peer_list.append(str(node))
clustered_peer_count = Counter(clustered_peer_list)
for cl in clustered_peer_count:
assert (clustered_peer_count[cl] == math.log(self._peer_count, 2))
