def _print_score(self, graph=None):
if graph == None:
graph = self.G
x = utils.score(graph, self.assignments, self.num_partitions)
edges_cut, steps = utils.base_metrics(graph, self.assignments)
mod = utils.modularity_wavg(graph, self.assignments,
self.num_partitions)
loneliness = utils.loneliness_score_wavg(graph,
self.loneliness_score_param,
self.assignments,
self.num_partitions)
max_perm = utils.run_max_perm(graph)
if self.verbose > 1:
print("{0:.5f}\t\t{1:.10f}\t{2}\t\t{3}\t\t\t{4}\t{5}\t{6}".format(
x[0], x[1], edges_cut, steps, mod, loneliness, max_perm))
return [x[0], x[1], edges_cut, steps, mod, loneliness, max_perm]
def generate_prediction_model(self, graph, num_iterations, num_partitions,
assignments, fixed):
self.runAssignments = {}
# STEP 0: sort the graph nodes
sortedNodes = sorted(graph.nodes())
# STEP 1: create a mapping of nodes for relabeling
nodeMapping = {}
for newID, nodeID in enumerate(sortedNodes):
# old label as key, new label as value
nodeMapping[nodeID] = newID
# Create a new graph with the new mapping
G = nx.relabel_nodes(graph, nodeMapping, copy=True)
# Copy over the node and edge weightings: double check this
for node in sortedNodes:
newNode = nodeMapping[node]
try:
G.node[newNode]['weight'] = graph.node[node]['weight']
for edge in graph.neighbors(node):
newEdge = nodeMapping[edge]
try:
G.edge[newNode][newEdge]['weight'] = graph.edge[node][
edge]['weight']
except Exception as err:
pass
except Exception as err:
pass
# Determine assignments
patoh_assignments = np.full(G.number_of_nodes(), -1)
for nodeID, assignment in enumerate(assignments):
if nodeID in nodeMapping:
# this nodeID is part of the mapping
newNodeID = nodeMapping[nodeID]
if fixed[nodeID] == 1:
patoh_assignments[newNodeID] = assignment
#print('assignments', assignments)
#print('fixed', fixed)
#print('mapping', nodeMapping)
#print('patoh_assignments', patoh_assignments)
iterations = {}
for i in range(0, self.partitioningIterations):
_assignments = self._runPartitioning(G, num_partitions,
patoh_assignments,
nodeMapping, assignments)
self.runAssignments[i] = _assignments
edges_cut, steps, cut_edges = gputils.base_metrics(
graph, _assignments)
#change to TCV
if steps not in list(iterations.keys()):
iterations[steps] = _assignments
#if edges_cut not in list(iterations.keys()):
# iterations[edges_cut] = _assignments
# return the maximum TCV
minTCV = 1000000000
for key in list(iterations.keys()):
if key < minTCV:
minTCV = key
assignments = iterations[minTCV]
# return the minimum edges cut
'''
minEdgesCut = graph.number_of_edges()
for key in list(iterations.keys()):
if key < minEdgesCut:
minEdgesCut = key
assignments = iterations[minEdgesCut]
'''
self._printIterationStats(iterations)
del iterations
return assignments
def _generate_prediction_model(self, graph, num_iterations, num_partitions,
assignments, fixed):
# STEP 0: sort the graph nodes
gSortedNodes = sorted(graph.nodes())
# create a mapping between the graph node ids and those used by SCOTCH
# ensures nodes are numbered from 0...n-1
node_indeces = self._createGraphIndeces(gSortedNodes, len(assignments))
#print('node_indeces', node_indeces)
# generate a new graph that only has the new nodes
G = nx.Graph()
for node in gSortedNodes:
# set the new node index used by scotch
G.add_node(node_indeces[node])
try:
G.node[
node_indeces[node]]['weight'] = graph.node[node]['weight']
#print("G.node[node_indeces[node]]['weight']", G.node[node_indeces[node]]['weight'])
except Exception as err:
pass
# add the edges for each node using the new ids
for node in gSortedNodes:
newNodeID = node_indeces[node]
for edge in graph.neighbors(node):
newEdgeID = node_indeces[edge]
G.add_edge(newNodeID, newEdgeID)
try:
weight = graph[node][edge]['weight']
G[newNodeID][newEdgeID]['weight'] = weight
except Exception as err:
pass
# determine the assignment partitions
patoh_assignments = []
for nodeID, assignment in enumerate(assignments):
if node_indeces[nodeID] >= 0:
# this nodeID is part of this graph and needs to be partitioned
# add node's fixed partition, if present
patoh_assignments.append(assignment)
iterations = {}
for i in range(0, self.partitioningIterations):
# perform an iteration of partitioning
_assignments = self._runPartitioning(G, num_partitions,
patoh_assignments,
node_indeces, assignments)
# compute the edges cut
edges_cut, steps = gputils.base_metrics(graph, _assignments)
#change to TCV
if steps not in list(iterations.keys()):
iterations[steps] = _assignments
#if edges_cut not in list(iterations.keys()):
# iterations[edges_cut] = _assignments
# return the maximum TCV
maxTCV = 0.0
for key in list(iterations.keys()):
if key > maxTCV:
maxTCV = key
print('max_tcv', maxTCV)
assignments = iterations[maxTCV]
# return the minimum edges cut
'''
minEdgesCut = graph.number_of_edges()
for key in list(iterations.keys()):
if key < minEdgesCut:
minEdgesCut = key
assignments = iterations[minEdgesCut]
'''
self._printIterationStats(iterations)
del iterations
return assignments
def get_graph_metrics(self):
self.metrics_timestamp = datetime.datetime.now().strftime('%H%M%S')
f, _ = os.path.splitext(os.path.basename(self.DATA_FILENAME))
self.metrics_filename = f + "-" + self.metrics_timestamp
graph_metrics = {
"file": self.metrics_timestamp,
"num_partitions": self.num_partitions,
"num_iterations": self.num_iterations,
"prediction_model_cut_off": self.prediction_model_cut_off,
"restream_batches": self.restream_batches,
"use_virtual_nodes": self.use_virtual_nodes,
"virtual_edge_weight": self.virtual_edge_weight,
}
graph_fieldnames = [
"file",
"num_partitions",
"num_iterations",
"prediction_model_cut_off",
"restream_batches",
"use_virtual_nodes",
"virtual_edge_weight",
"edges_cut",
"waste",
"cut_ratio",
"total_communication_volume",
"network_permanence",
"Q",
"NQ",
"Qds",
"intraEdges",
"interEdges",
"intraDensity",
"modularity degree",
"conductance",
"expansion",
"contraction",
"fitness",
"QovL",
]
if self.verbose > 0:
print("Complete graph with {} nodes".format(
self.G.number_of_nodes()))
file_oslom = utils.write_graph_files(self.OUTPUT_DIRECTORY,
"{}-all".format(
self.metrics_filename),
self.G,
quiet=True)
# original scoring algorithm
scoring = utils.score(self.G, self.assignments, self.num_partitions)
graph_metrics.update({
"waste": scoring[0],
"cut_ratio": scoring[1],
})
# edges cut and communication volume
edges_cut, steps = utils.base_metrics(self.G, self.assignments)
graph_metrics.update({
"edges_cut": edges_cut,
"total_communication_volume": steps,
})
# MaxPerm
max_perm = utils.run_max_perm(self.G)
graph_metrics.update({"network_permanence": max_perm})
# Community Quality metrics
community_metrics = utils.run_community_metrics(
self.OUTPUT_DIRECTORY, "{}-all".format(self.metrics_filename),
file_oslom)
graph_metrics.update(community_metrics)
if self.verbose > 0:
print("\nConfig")
print("-------\n")
for f in graph_fieldnames[:8]:
print("{}: {}".format(f, graph_metrics[f]))
print("\nMetrics")
print("-------\n")
for f in graph_fieldnames[8:]:
print("{}: {}".format(f, graph_metrics[f]))
# write metrics to CSV
csv_file = os.path.join(self.OUTPUT_DIRECTORY, "metrics.csv")
utils.write_metrics_csv(csv_file, graph_fieldnames, graph_metrics)
def _print_score(self, graph=None):
if self.compute_metrics_enabled == False:
return [0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0]
if graph == None:
graph = self.G
# waste, cut_ratio
x = utils.score(graph, self.assignments, self.num_partitions)
edges_cut, steps, cut_edges = utils.base_metrics(
graph, self.assignments)
#q_qds_conductance = utils.louvainModularityComQuality(graph, self.assignments, self.num_partitions)
# non-overlapping metrics
q_qds_conductance = utils.infomapModularityComQuality(
graph, self.assignments, self.num_partitions)
loneliness = utils.loneliness_score_wavg(graph,
self.loneliness_score_param,
self.assignments,
self.num_partitions)
#print('loneliness', loneliness)
#max_perm = utils.run_max_perm(graph)
max_perm = utils.wavg_max_perm(graph, self.assignments,
self.num_partitions)
rbse_list = utils.ratherBeSomewhereElseList(graph, self.assignments,
self.num_partitions)
rbse = utils.ratherBeSomewhereElseMetric(rbse_list)
#nmi_score = nmi_metrics.nmi(np.array([self.assignments_prediction_model, self.assignments]))
nmi_assignments = self.assignments.tolist()
pred_assignments = self.assignments_prediction_model.tolist()
pred_nmi_assignments = []
actual_nmi_assignments = []
if self.use_virtual_nodes:
#print(len(nmi_assignments), self.initial_number_of_nodes)
if (len(nmi_assignments) > self.initial_number_of_nodes):
nmi_assignments = nmi_assignments[0:self.
initial_number_of_nodes]
for i, partition in enumerate(nmi_assignments):
if partition >= 0:
pred_nmi_assignments.append(pred_assignments[i])
actual_nmi_assignments.append(partition)
#print('computing nmi', len(pred_nmi_assignments), len(actual_nmi_assignments))
nmi_score = normalized_mutual_info_score(pred_nmi_assignments,
actual_nmi_assignments)
#nmi_score = 0.0
# compute the sum of edge weights for all the cut edges for a total score
#total_cut_weight = 0
#for cutEdge in cut_edges:
# total_cut_weight += self.originalG.edge[cutEdge[0]][cutEdge[1]]['weight']
# compute fscores
#print('computing fscore...')
fscore, fscore_relabelled = utils.fscores2(
self.assignments_prediction_model, self.assignments,
self.num_partitions)
#fscore = 0.0
#fscore_relabelled = 0.0
#print('fscores:', fscore, fscore_relabelled)
if self.verbose > 1:
print(
"{0:.5f}\t\t{1:.10f}\t{2}\t\t{3}\t\t\t{4:.5f}\t{5:.5f}\t{6}\t{7:.5f}\t{8:.10f}\t{9}\t{10}\t{11:.5f}"
.format(x[0], x[1], edges_cut, steps, q_qds_conductance[1],
q_qds_conductance[2], max_perm, rbse, nmi_score,
fscore, abs(fscore - fscore_relabelled), loneliness))
#print("{0:.5f}\t\t{1:.10f}\t{2}\t\t{3}\t\t\t{4:.5f}\t{5:.5f}\t{6:.5f}\t{7}\t{8}\t{9:.10f}\t{10}\t{11}\t{12}".format(x[0], x[1], edges_cut, steps, q_qds_conductance[0], q_qds_conductance[1], q_qds_conductance[2], loneliness, max_perm, nmi_score, total_cut_weight, fscore, abs(fscore-fscore_relabelled)))
#print("{0:.5f}\t\t{1:.10f}\t{2}\t\t{3}\t\t\t{4}\t{5}\t{6}\t{7:.10f}\t{8}\t{9}\t{10}".format(x[0], x[1], edges_cut, steps, mod, loneliness, max_perm, nmi_score, total_cut_weight, fscore, abs(fscore-fscore_relabelled)))
return [
x[0], x[1], edges_cut, steps, q_qds_conductance[1],
q_qds_conductance[2], max_perm, rbse, nmi_score, fscore,
abs(fscore - fscore_relabelled), loneliness
]
