def create_topology_and_pathlist(a,g,arrangement):
G = create_topology(a,g,arrangement)
all_min_paths_DF = all_pair_all_min_paths_DF(G, edge_weight = "w", a = a, g = g)
all_djkstra_paths = all_pair_all_shortest_paths_djkstra(G, edge_weight = "w")
return G, all_min_paths_DF, all_djkstra_paths
def compare_paths_by_djkstra_and_floyd_warshall():
'''
As the name suggests. Just a test function.
'''
G = intro_code_to_test_something.create_dragonfly(a=8, g = 17, arrangement = "absolute_improved")
intro_code_to_test_something.draw_graph(G)
paths_fw = all_pair_all_shortest_paths_floyd_warshall(G, verbose = False)
paths_dj = djkstra.all_pair_all_shortest_paths_djkstra(G, print_paths = False)
#paths_dj[5][5].append((1,2))
path_validness = djkstra.check_path_validness(paths_fw, paths_dj)
print("path validity: ", path_validness)
#TODO:
# issue: FW seems to be an order of magnitude slow that djkstra. Not sure why.
# for the time being, djkstra seems to be the way forward.
pass
def intro_code():
N = 6
graph = [[] for x in range(N)]
edges = [(0, 1, 1), (0, 2, 1), (1, 2, 1), (3, 4, 1), (3, 5, 1), (4, 5, 1),
(2, 3, 3), (1, 5, 3)] #(src, dest, weight)
for (src, dst, weight) in edges:
graph[src].append((dst, weight))
graph[dst].append((src, weight))
print("the graph:")
for idx in range(len(graph)):
print(idx, graph[idx])
G = nx.Graph()
for (src, dst, weight) in edges:
G.add_edge(src, dst, w=weight)
print("the graph from networkx:")
print(G.edges(data=True))
draw_graph(G)
# check the paths in the graph
#paths = djkstra(0, N, graph)
#check djkstra
print("\ncalling all_pair_djkstra: ")
start_time = time.time()
paths = all_pair_all_shortest_paths_djkstra(G, edge_weight="w")
end_time = time.time()
print("djkstra returned.")
print("execution time: ", end_time - start_time)
#now check floyd-warshall
start_time = time.time()
print("\ncalling floyd-warshall")
paths2 = ut.floyd_warshall_all_pairs_sp(G, weight='w')
end_time = time.time()
print("FW returned.")
print("execution time: ", end_time - start_time)
# print("\nfloyd warshall paths:")
# for src in range(N):
# print("\nsrc:",src)
# for dst in range(N):
# print(src,dst, l2[src][dst])
#
pass
def create_MIN_paths_list(G,
traffic,
all_pair_min_paths=None,
*,
verbose=False):
'''
A function that creates a MIN path list for only the s-d pairs present in the traffic pattern.
This is needed to make MIN-path method compatible to the corresponding VLB-path one.
Previously we were using all_pair_all_shortest_paths_djkstra() for the passing the paths,
but that creates a 3D list. On the other hand, VLB-path functions creates a dict with (s,d)
pair tuple as the key. This causes access incoherence in the functions which intend to
work with the path list.
Returns: a dict with keys (s,d) and value: list of path tuples.
'''
if all_pair_min_paths == None:
all_pair_min_paths = all_pair_all_shortest_paths_djkstra(
G, edge_weight="w", print_paths=False)
all_MINpaths = {}
for src, dst in traffic:
minpaths = [tuple(path) for path in all_pair_min_paths[src][dst]]
all_MINpaths[(src, dst)] = minpaths
if verbose:
print("\nMin paths for each pair:")
for sdpair, paths in all_MINpaths.items():
print(sdpair, " -> ", paths)
return all_MINpaths
pass
print("Hello world!")
a = 4
g = 5
p = a//2
arrangement = "absolute_improved"
#G = create_topology(a,g,arrangement)
G, local_links, global_links = topologies.build_drgonfly(a=a, p=a // 2, h=a // 2, gl_arrangmnt=arrangement, g=g, seed = 0)
graph_adj_list = generate_graph_adjacency_list(G)
group_pair_vs_global_links = get_list_of_links_between_each_group_pair(G, edge_weight = "w", a = a, g = g )
paths_dj = all_pair_all_shortest_paths_djkstra(G, print_paths = False)
group_pair_vs_nodes = generate_list_of_nodes_connected_to_group_pairs(graph_adj_list, a)
two_hop_neighbor_list = generate_list_of_2hop_neighbors(G, group_pair_vs_global_links, edge_weight = "w", a = a, g = g)
from traffic import create_random_permutation_pattern
SDpairs = create_random_permutation_pattern(a = a, g = g, seed = 10, verbose = False)
print("\nTraffic pattern:")
sd_pair_vs_3hop_inodes = inodes_for_3hop_paths_for_all_SD_pairs(SDpairs, G, graph_adj_list, two_hop_neighbor_list, edge_weight = "w", a = a, g = g, p = p)
for pair in SDpairs:
src = pair[0]
dst = pair[1]
def test_jains(a, g, pattern = None, routing = "MIN", file_pointer = None, *, draw = False):
'''
So far, routing can be only MIN or VLB, or Combined
'''
print("\n\ninside test_jains ...")
start_time = time.time()
#create a Dragonfly topology
#a = 8
#g = 29
h = a//2
p = a//2
arrangement = "absolute_improved"
print("a, g, arrangement : ", a, g, arrangement)
G = create_topology(a,g,arrangement)
if draw == True:
draw_graph_circular(G, a = a)
if pattern == "randperm":
seeds = [0, 2]
#TODO: we need to parameterize seeds later
elif pattern == "shift":
seeds = [0] # seed is not needed for shift
else:
print("Error! Unsupported traffic pattern!")
sys.exit(1)
for seed in seeds:
#create a traffic pattern
if pattern == "randperm":
traffic = create_random_permutation_pattern(a = a, g = g, seed = seed, verbose = False)
elif pattern == "shift":
traffic = create_shift_traffic_pattern(a = a, g = g, verbose = False)
tot_sd_pairs = len(traffic)
traffic_and_freqs = check_pattern_for_duplicates(traffic, verbose = False)
tot_unique_pairs = len(traffic_and_freqs)
#create path list
#test with both min and vlb, one by one
all_djkstra_paths = all_pair_all_shortest_paths_djkstra(G, edge_weight = "w")
modes = [0, 1, 3] #mode 1 and 2 are practically equivalent, so skipping 2
for mode in modes:
if routing == "MIN":
paths = create_MIN_paths_list(G, traffic, all_pair_min_paths = all_djkstra_paths, verbose = False)
elif routing == "VLB":
paths = create_VLB_paths_list(G, a, traffic, all_pair_min_paths = all_djkstra_paths, verbose = False)
elif routing == "Combined":
paths = merge_MIN_and_VLB_paths(G, a, traffic, all_pair_min_paths = all_djkstra_paths, mode = mode, verbose = False)
#call JM
print("\ncalling JM for routing {}, seed {} and mode {} ".format(routing, seed, mode))
#total_bw, res = jains_model(G = G, traffic = traffic, path_list = paths, verbose = False)
total_bw, result_bw = jains_model_with_unique_pairs(G = G, traffic_and_freqs = traffic_and_freqs, path_list = paths, verbose = False)
print("out of JM ...")
print("\ntotal_bw: " , total_bw)
print("result: " , result_bw)
#do file writing here.
print("{},{},{},{},{},{},{},{},{},".format(a,g,h,p,arrangement,pattern,seed,tot_sd_pairs,tot_unique_pairs))
if file_pointer:
file_pointer.write("{},{},{},{},{},{},{},{},{},".format(a,g,h,p,arrangement,pattern,seed,tot_sd_pairs,tot_unique_pairs))
print("{},{},{:.4f},{:.4f}".format(routing, mode, total_bw, result_bw))
if file_pointer:
file_pointer.write("{},{},{:.4f},{:.4f}".format(routing, mode, total_bw, result_bw))
file_pointer.write("\n")
#win
end_time = time.time()
print("total time spent:", end_time - start_time)
pass
def driverForGlobalLinkUsageCount():
#create a topology
a = 8
gs = [9]
traffics = ["shift"]
#traffic = "rperm"
traffic_params = {"shift": "shift_by"}
traffic_param_values = {"shift": 1}
for g in gs:
#g = 33
p = 1 #router_based traffic
h = a // 2
arrangement = "absolute_improved"
#not changing it anytime soon
G = create_topology(a, g, arrangement)
#draw_graph_circular(G, a = a)
#make the adjacency list
graph_adj_list = generate_graph_adjacency_list(G)
#make twoHopNeighborList
group_pair_vs_global_links = get_list_of_links_between_each_group_pair(
G, edge_weight="w", a=a, g=g)
twoHopNeighbors = generate_list_of_2hop_neighbors(
G, group_pair_vs_global_links, edge_weight="w", a=a, g=g)
#create path list
all_djkstra_paths = all_pair_all_shortest_paths_djkstra(
G, edge_weight="w")
#get a traffic
#traffic = "shift"
for traffic in traffics:
if traffic == "rperm":
SDpairs = create_random_permutation_pattern(
a=a,
g=g,
seed=traffic_param_values[traffic],
verbose=False)
elif traffic == "shift":
SDpairs = create_shift_traffic_pattern(
a=a,
g=g,
shift_by=traffic_param_values[traffic],
router_level=True,
verbose=True)
else:
SDpairs = None
#select a mode. Options: "src_only" or "src_and_dst"
modes = [
"3hop_paths_only", "4hop_paths_only", "5hop_src_only",
"5hop_src_and_dst"
]
#link_usage_data = {}
for mode in modes:
print("traffic:", traffic)
print("mode:", mode)
#link_usage_data[mode] = calculateLinkUsage(graph_adj_list, SDpairs, mode, all_djkstra_paths, twoHopNeighbors, a = a, g = g, p = p)
countGlobalLinkUsage(G,
SDpairs,
mode,
all_djkstra_paths,
a=a,
g=g,
p=p,
h=h)
#all_usage_data[(a,g,traffic,traffic_params[traffic], traffic_param_values[traffic])] = link_usage_data
pass
def driverForLinkUsageCalculation():
#create a topology
a = 8
gs = [9, 17, 25, 33]
traffics = ["rperm", "shift"]
#traffic = "rperm"
traffic_params = {"shift": "shift_by", "rperm": "seed"}
traffic_param_values = {"shift": 1, "rperm": 45}
all_usage_data = {}
for g in gs:
#g = 33
p = a // 2
h = a // 2
arrangement = "absolute_improved"
#not changing it anytime soon
G = create_topology(a, g, arrangement)
#draw_graph_circular(G, a = a)
#make the adjacency list
graph_adj_list = generate_graph_adjacency_list(G)
#make twoHopNeighborList
group_pair_vs_global_links = get_list_of_links_between_each_group_pair(
G, edge_weight="w", a=a, g=g)
twoHopNeighbors = generate_list_of_2hop_neighbors(
G, group_pair_vs_global_links, edge_weight="w", a=a, g=g)
#create path list
all_djkstra_paths = all_pair_all_shortest_paths_djkstra(
G, edge_weight="w")
#get a traffic
#traffic = "shift"
for traffic in traffics:
if traffic == "rperm":
SDpairs = create_random_permutation_pattern(
a=a,
g=g,
seed=traffic_param_values[traffic],
verbose=False)
elif traffic == "shift":
SDpairs = create_shift_traffic_pattern(
a=a,
g=g,
shift_by=traffic_param_values[traffic],
router_level=False,
verbose=False)
else:
SDpairs = None
#select a mode. Options: "src_only" or "src_and_dst"
modes = ["src_only", "src_and_dst"]
link_usage_data = {}
for mode in modes:
print("traffic:", traffic)
print("mode:", mode)
link_usage_data[mode] = calculateLinkUsage(graph_adj_list,
SDpairs,
mode,
all_djkstra_paths,
twoHopNeighbors,
a=a,
g=g,
p=p)
all_usage_data[(a, g, traffic, traffic_params[traffic],
traffic_param_values[traffic])] = link_usage_data
#title = "a : {}, g : {}, traffic : {}, {} : {}".format(a,g,traffic, traffic_params[traffic], traffic_param_values[traffic])
#process_link_usage_data(link_usage_data, a=a, h=h, g=g, p=p, title = "test")
process_all_link_usage_data(all_usage_data)
def reachability_test(Graph, a, h, g):
'''
We want to check, for each node, how many nodes can be reached in 1, 2 or 3 steps.
Get the stat for each node individually. Then compress them to get a summary result.
'''
'''
Logic:
#call djkstra, get the table
#for each pair of nodes, check the min path len. update counts for src and dst nodes accordingly.
#for each node, we need to save three counts: no of nodes reachable by hops 1,2,3
#when done, convert each (count_1, count_2, count_3) triplet to tuples and get their count. That is the summary we are looking for.
'''
N = len(Graph.nodes)
counts = [[0, 0, 0] for x in range(N)]
all_djkstra_paths = all_pair_all_shortest_paths_djkstra(Graph)
for src in range(N):
#print(src, end=" : ")
for dst in range(src + 1, N):
#print(dst, end= " ")
#don't want to consider same-group node pairs
if different_group_nodes((src, dst), a):
paths = all_djkstra_paths[src][dst]
length = len(paths[0]) - 1 # hop count = node count - 1
counts[src][length - 1] += 1
counts[dst][length - 1] += 1
#test print
# for ii in range(N):
# print("{} : {} {} {} ".format(ii, counts[ii][0], counts[ii][1], counts[ii][2]))
#
#get frequency
frequency_of_reachability_count = {}
for ii in range(N):
tup = tuple(counts[ii])
if tup in frequency_of_reachability_count:
frequency_of_reachability_count[tup] += 1
else:
frequency_of_reachability_count[tup] = 1
values_and_freq = []
for tup, count in frequency_of_reachability_count.items():
values_and_freq.append([a, h, g, tup[0], tup[1], tup[2], count])
records = pd.DataFrame(values_and_freq)
records.columns = [
"a", "h", "g", "dist_1", "dist_2", "dist_3", "frequency"
]
records["dist_1_pcnt"] = records["dist_1"] / (
records["dist_1"] + records["dist_2"] + records["dist_3"]) * 100
records["dist_2_pcnt"] = records["dist_2"] / (
records["dist_1"] + records["dist_2"] + records["dist_3"]) * 100
records["dist_3_pcnt"] = records["dist_3"] / (
records["dist_1"] + records["dist_2"] + records["dist_3"]) * 100
#print(records)
#test print
# print("frequency:")
# for row in values_and_freq:
# print(row)
#
return records
pass
def run_test_for_a_config(a, h, g, arrg, *, seed=0):
print(
"\nStarting test for: a {}, h {}, g {}, arrangment {}, seed {}".format(
a, h, g, arrg, seed))
config = (a, h, g, arrg)
#folder_name = "Path_Stats"
folder_name = "TestFolder_supposedly_unimportant_data"
os.makedirs(folder_name, exist_ok=True)
# file_name1 = folder_name + "/" + "a_{}_h{}_g{}_global_paths.csv".format(a,h,g)
# file_name2 = folder_name + "/" + "a_{}_h{}_g{}_global_paths_detailed.csv".format(a,h,g)
# file_name3 = folder_name + "/" + "a_{}_h{}_g{}_all_paths_overall.csv".format(a,h,g)
file_name1 = folder_name + "/" + arrg + "_global_paths.csv"
file_name2 = folder_name + "/" + arrg + "_global_paths_detailed.csv"
file_name3 = folder_name + "/" + arrg + "_all_paths_overall.csv"
if arrg == "practice":
arrg = "absolute_improved"
start_time = time.time()
G, local_links, global_links = tp.build_drgonfly(a=a,
p=a // 2,
h=a // 2,
gl_arrangmnt=arrg,
g=g,
seed=seed)
end_time = time.time()
#print("topology creation time:", end_time - start_time)
if ut.validate_DF(G, global_links, local_links, a, g):
#print("Valid topology")
validity = "Valid"
else:
#print("Invalid topology")
validity = "Invalid"
#print_graph(G)
#draw_graph_circular(G, a = a)
N = len(G)
#print("\ncalling all_pair_djkstra: ")
start_time = time.time()
paths_djk = all_pair_all_shortest_paths_djkstra(G, edge_weight="w")
end_time = time.time()
#print("djkstra returned.")
#print("execution time: ", end_time - start_time)
get_path_len_stat(paths_djk,
G,
config,
seed=seed,
validity=validity,
file_name1=file_name1,
file_name2=file_name2,
file_name3=file_name3)
# print("\ncalling all_pair_networkx: ")
# start_time = time.time()
# paths_nx = all_pair_all_shortest_paths_networkx(G, edge_weight = "w")
# end_time = time.time()
# print("netowkrx returned.")
# print("execution time: ", end_time - start_time)
# get_path_len_stat(paths_nx, G, config, seed = seed, validity = validity, file_name1 = file_name1, file_name2 = file_name2, file_name3 = file_name3)
pass
def create_VLB_paths_list(G,
a,
trafficPattern,
all_pair_min_paths=None,
*,
mode=0,
verbose=True):
'''
G: networkx Graph object
a = DF parameter a, reuqired to find the group id of a node
Traffic pattern: A list of (S,D) tuple.
S and D are routers, *NOT* PEs
mode: How the VLB paths will be selected.
This decides the max path len in first and second phase of VLB routes.
In Dragonfly, max path len for min routing can be 3.
So each phase can have 3 hops, or less.
Mode 0: <= 3 + <= 3 => unrestricted VLB routing, default case
Mode 1: < 3 + <= 3 => First phase restricted
Mode 2: <= 3 + < 3 => Second phase restricted
Mode 3: < 3 + < 3 => Both phase restricted
returns: A dict of (s,d) pair keys and pathlist in values.
pathlist is list of tuples. Each tuple contains the node ids in the path.
'''
'''
#for each S-D pair, we need all the VLB paths.
#for an SD, each other node can be a potential I node.
#So for each I node, we need all S-I-D paths.
#If there are multiple paths in the S-I and I-D portion, then it is even more complex.
#In that case, if there are m1 paths in the S-I part and m2 paths in the I-D part,
#then there are a total of m1*m2 number of total paths for this S-I-D quad only.
#how big is the total number of paths?
#In a DF, total number of PE is a*g*p. Say, N.
#Then in a randperm, there are N number of SD pairs.
#Each of the a*g-2 routers can act as an I.
#So total SID combo: N*(a*g-2)
#For single paths only, we are talking about N*(a*g-2) paths.
#For multipaths, it will be bigger.
#For a = 8, h = 4, p =4. If g = 33, then N = 33 * 8 * 4 = 264 * 4 = 1056
#So total SID combo: 1056 * (8*33-2) = 278 784 = 0.28 M
#if g = 15, N = 15 * 8 * 4 = 120 * 4 = 480
# so total SID combo: 480 * (8*15-2) = 57600 = 0.057 M
#For a = 16, h = 8, p = 8. If g = 129, then N = 129 * 16 * 8 = 2064 * 8 = 16512
#so total SID combo: 16512 * (16*129-2) = 34 080 768 = 34M
#if g = 40, then N = 40 * 16 * 8 = 640 * 8 = 5120
#so total SID combo: 5120 * (16*40) = 3 276 800 = 3M
'''
#ok, complexity aside. here are the steps.
#First, call djkstra to get the shortest paths among all the pairs.
#Then get the traffic pattern.
#For each SD router combo, get every other router as an intermediate router I.
#Get all the paths for (S,I) and (I,D) from the djkstra table.
#combine the paths. So essentially m1 * m2 number of paths for each SID combo.
#So for each SD, get a dict of (S,D) vs list of all paths from all (SID) triplet. Basically, all VLB paths.
#pass this (S,D) vs [All VLB paths] to the Jain's method.
if all_pair_min_paths == None:
all_djkstra_paths = all_pair_all_shortest_paths_djkstra(
G, edge_weight="w")
else:
all_djkstra_paths = all_pair_min_paths
N = len(G)
all_VLBpaths = {}
if mode == 0:
max_SI_pathlen = 3
max_ID_pathlen = 3
elif mode == 1:
max_SI_pathlen = 2
max_ID_pathlen = 3
elif mode == 2:
max_SI_pathlen = 3
max_ID_pathlen = 2
elif mode == 3:
max_SI_pathlen = 2
max_ID_pathlen = 2
else:
print("invalid mode: ", mode)
sys.exit(1)
#for each SD pair
for (src, dst) in trafficPattern:
if src == dst:
continue
SDpair_all_VLBpaths = set()
srcGroup = src // a
dstGroup = dst // a
#print("groups:", srcGroup, dstGroup)
#if src and dst are in the same group
if srcGroup == dstGroup:
#select imdt from only within that group
imdt_lst = [
x for x in range(srcGroup * a, (srcGroup + 1) * a)
if x != src and x != dst
]
else:
#we want to skip the routers that are in the same group as the src and the dst
imdt_lst = [
x for x in range(N)
if x // a != srcGroup and x // a != dstGroup
]
#print("imdt_lst:", imdt_lst)
#for each possible imdt router
for imdt in imdt_lst:
# if src == imdt or dst == imdt:
# continue
#not needed anymore as omitting srcGroup and dstGroup is taking care of this case as well.
#construct all VLB path
SIpaths = [
path for path in all_djkstra_paths[src][imdt]
if len(path) <= (max_SI_pathlen + 1)
]
IDpaths = [
path for path in all_djkstra_paths[imdt][dst]
if len(path) <= (max_ID_pathlen + 1)
]
#paths contain src and dst nodes, so hop number is one less than the number of nodes in the path
#print(src, imdt, " : ", SIpaths)
#print(imdt, dst, " : ", IDpaths)
#VLBpaths = [ tuple( list(path1) + list(path2)[1:]) for path1 in SIpaths for path2 in IDpaths ]
#print(VLBpaths)
for path1 in SIpaths:
for path2 in IDpaths:
SDpair_all_VLBpaths.add(
tuple(list(path1) + list(path2)[1:]))
# print("\n\n")
# print(src,dst,"all vlb paths:")
# print(SDpair_all_VLBpaths)
#
all_VLBpaths[(src, dst)] = list(SDpair_all_VLBpaths)
if verbose:
print("\nVLB paths for each pair:")
for sdpair, paths in all_VLBpaths.items():
print(sdpair, " -> ", paths)
return all_VLBpaths
pass
def merge_MIN_and_VLB_paths(G,
a,
trafficPattern,
all_pair_min_paths=None,
*,
mode=0,
verbose=True):
'''
As the name suggests, this function takes a topology and traffic pattern, then
generates all the min and VLB paths, and merges them together.
Can be used to apporximate something similar to UGAL routing.
As usual:
G: networkx Graph object
a = DF parameter a, reuqired to find the group id of a node
Traffic pattern: A list of (S,D) tuple.
S and D are routers, *NOT* PEs
all_pair_min_paths: the caller function may call djkstra itself and send the
paths as a parameter which will be used to find min and
vlb paths. If None, then the djkstra-paths will be generated
inside the min and vlb functions.
If sent, it's a 3D list of format [src][dst][lst of all path tuples]
mode: How the VLB paths will be selected.
This decides the max path len in frist and second phase of VLB routes.
In Dragonfly, max path len for min routing can be 3.
So each phase can have 3 hops, or less.
Mode 0: <= 3 + <= 3 => unrestricted VLB routing, default case
Mode 1: < 3 + <= 3 => First phase restricted
Mode 2: <= 3 + < 3 => Second phase restricted
Mode 3: < 3 + < 3 => Both phase restricted
returns: A dict of (s,d) pair keys and pathlist in values.
pathlist is list of tuples. Each tuple contains the node ids in the path.
'''
#will save time for one run
if all_pair_min_paths == None:
all_djkstra_paths = all_pair_all_shortest_paths_djkstra(
G, edge_weight="w")
else:
all_djkstra_paths = all_pair_min_paths
all_min_paths = create_MIN_paths_list(G,
trafficPattern,
all_pair_min_paths=all_djkstra_paths,
verbose=False)
all_vlb_paths = create_VLB_paths_list(G,
a,
trafficPattern,
all_pair_min_paths=all_djkstra_paths,
mode=mode,
verbose=False)
if verbose:
print("\n generated the paths.")
for (src, dst) in trafficPattern:
print("({}, {})".format(src, dst))
print("min: ", all_min_paths[(src, dst)])
print("vlb: ", all_vlb_paths[(src, dst)])
#paths returned, now merge
all_combined_paths = {}
for (src, dst) in trafficPattern:
all_combined_paths[(src, dst)] = [
*all_min_paths[(src, dst)], *all_vlb_paths[(src, dst)]
]
#shallow copy. But in this case it's fine.
if verbose:
print("\n merged paths.")
for (src, dst) in trafficPattern:
print("({}, {})".format(src, dst))
print(all_combined_paths[(src, dst)])
return all_combined_paths
pass