def test_FGW(args):
"""
Fused Gromov-Wasserstein distance
"""
args.m = 8
args.n = 4
if args.fix_seed:
torch.manual_seed(0)
g = nx.stochastic_block_model([4, 4], [[0.9, 0.1], [0.1, 0.9]], seed=8576)
#components = nx.connected_components(g)
g.remove_nodes_from(list(nx.isolates(g)))
args.m = len(g)
g2 = nx.stochastic_block_model([4, 4], [[0.9, 0.1], [0.1, 0.9]])
#components = nx.connected_components(g)
g2.remove_nodes_from(list(nx.isolates(g2)))
args.n = len(g2)
gwdist = Fused_Gromov_Wasserstein_distance(alpha=0.8,
features_metric='sqeuclidean')
graph.graph_dist(args)
'''
g = gwGraph.Graph(g)
g2 = gwGraph.Graph(g2)
dist = gwdist.graph_d(g,g2)
print('GW dist ', dist)
'''
pdb.set_trace()
def synthetic_data_sbm(N=1000):
"""sbm"""
graphs = []
graph_labels = []
import random
for _ in range(int(N / 2)):
G = nx.stochastic_block_model(
[
random.randint(10, 30),
random.randint(10, 30),
random.randint(10, 30)
],
[[0.6, 0.1, 0.1], [0.1, 0.6, 0.1], [0.1, 0.1, 0.6]],
)
graphs.append(G)
graph_labels.append(1)
for _ in range(int(N / 2)):
G = nx.stochastic_block_model(
[random.randint(20, 40),
random.randint(20, 40)], [[0.6, 0.1], [0.1, 0.6]])
graphs.append(G)
graph_labels.append(2)
return graphs, np.asarray(graph_labels)
def test_FGW(args):
"""
Fused Gromov-Wasserstein distance
"""
import lib.graph as gwGraph
from lib.ot_distances import Fused_Gromov_Wasserstein_distance
args.m = 8
args.n = 4
if args.fix_seed:
torch.manual_seed(0)
#args.Lx = torch.randn(args.m*(args.m-1)//2) #torch.FloatTensor([[1, -1], [-1, 2]])
#args.Lx = realize_upper(args.Lx, args.m)
#pdb.set_trace()
g = nx.stochastic_block_model([4,4],[[0.9,0.1],[0.1,0.9]], seed = 8576)
#components = nx.connected_components(g)
g.remove_nodes_from(list(nx.isolates(g)))
args.m = len(g)
Lx = nx.laplacian_matrix(g, range(args.m)).todense()
args.Lx = torch.from_numpy(Lx).to(dtype=torch.float32) #+ torch.ones(args.m, args.m)/args.m
args.n_epochs = 150
'''
g2 = nx.stochastic_block_model([4,4],[[0.9,0.1],[0.1,0.9]])
g2.remove_nodes_from(list(nx.isolates(g2)))
args.n = len(g2)
'''
loss, P, L = graph.graph_dist(args, plot=False)
if isinstance(L, torch.Tensor):
L = L.numpy()
np.fill_diagonal(L, 0)
A = -L
g2 = nx.from_numpy_array(A)
gwdist = Fused_Gromov_Wasserstein_distance(alpha=0.8,features_metric='sqeuclidean')
g = gwGraph.Graph(g)
g2 = gwGraph.Graph(g2)
dist = gwdist.graph_d(g,g2)
print('GW dist ', dist)
###
g3 = nx.stochastic_block_model([4,4],[[0.9,0.1],[0.1,0.9]],seed=452)
g3.remove_nodes_from(list(nx.isolates(g3)))
args.m = len(g3)
Lx = nx.laplacian_matrix(g3, range(args.m)).todense()
args.Lx = torch.from_numpy(Lx).to(dtype=torch.float32) #+ torch.ones(args.m, args.m)/args.m
loss2, P2, L2 = graph.graph_dist(args, plot=False)
L=L2
if isinstance(L, torch.Tensor):
L = L.numpy()
np.fill_diagonal(L, 0)
A = -L
g4 = nx.from_numpy_array(A)
#gwdist = Fused_Gromov_Wasserstein_distance(alpha=0.8,features_metric='sqeuclidean')
g3 = gwGraph.Graph(g3)
g4 = gwGraph.Graph(g4)
dist = gwdist.graph_d(g3,g4)
print('GW dist ', dist)
pdb.set_trace()
def set_graph(self, graphtype, graphparams):
if graphtype == 'hexagonal':
self.G = nx.triangular_lattice_graph(**graphparams)
elif graphtype == 'watts strogatz':
self.G = nx.watts_strogatz_graph(**graphparams)
elif graphtype == 'newman watts strogatz':
self.G = nx.newman_watts_strogatz_graph(**graphparams)
elif graphtype == 'square':
self.G = nx.grid_2d_graph(**graphparams)
elif graphtype == 'random blocks':
self.G = nx.stochastic_block_model(**graphparams)
elif graphtype == 'powerlaw cluster':
self.G = nx.powerlaw_cluster_graph(**graphparams)
elif graphtype == 'scale-free small world':
self.G = clustered_scalefree(**graphparams)
elif graphtype == 'barabasi-albert':
self.G = nx.barabasi_albert_graph(**graphparams)
else:
print('Using complete graph')
self.G = nx.complete_graph(**graphparams)
def generate_fakeDB():
"""
Generate fake dataset for testing
output: graphs - list of graph signal
labels - list of multi-class label
"""
#Gen SBM
sizes = [39255, 39255, 39256]
probs = [[0.25, 0.05, 0.02],
[0.05, 0.35, 0.07],
[0.02, 0.07, 0.40]]
graph = nx.stochastic_block_model(sizes, probs, seed=0)
W = nx.adjacency_matrix(graph)
G = pygsp.graphs.Graph(W)
W = W.astype(float)
d = W.sum(axis=0)
d += np.spacing(np.array(0, W.dtype))
d = 1.0 / np.sqrt(d)
D = sparse.diags(d.A.squeeze(), 0)
I = sparse.identity(d.size, dtype=W.dtype)
L_norm = I - D*W*D
L = L_norm.tocoo()
graph_signals = np.random.random([117766, 1921])
labels = (np.random.random([117766, 23]) > 0.5 )*1
return graph_signals, labels, L
def test_graph_generator():
n = 400
m = 5
seed = 0
# G = nx.barabasi_albert_graph(n, m, seed)
G = nx.random_partition_graph([100, 100, 200], .25, .01)
sizes = [10, 90, 300]
probs = [[0.25, 0.05, 0.02], [0.05, 0.35, 0.07], [0.02, 0.07, 0.40]]
G = nx.stochastic_block_model(sizes, probs, seed=0)
G = nx.newman_watts_strogatz_graph(400, 5, 0.5)
A = nx.to_numpy_array(G)
print(A)
plt.pcolormesh(A)
plt.show()
s = sorted(G.degree, key=lambda x: x[1], reverse=True)
# newmap = {s[i][0]:i for i in range(len(s))}
# H= nx.relabel_nodes(G,newmap)
# newmap = generate_node_mapping(G, type='community')
# H = networkX_reorder_nodes(G, newmap)
H = networkx_reorder_nodes(G, 'community')
# B = nx.to_numpy_array(H)
# # plt.pcolormesh(B)
# plt.imshow(B)
# plt.show()
visualize_graph_matrix(H)
def generate_Connectivity_CP(inter_prob, intra_prob, increment, alpha=0.49):
cps = [15, 30, 60, 75, 90, 105, 135]
fname = "CPConnect_" + str(inter_prob) + "_" + str(intra_prob) + "_" + str(
increment) + "_" + str(alpha) + ".txt"
sizes_2 = [125, 125, 125, 125]
probs_2 = construct_SBM_block(sizes_2, inter_prob, intra_prob)
sizes = sizes_2
probs = probs_2
maxt = 150
G_0 = nx.stochastic_block_model(sizes, probs)
G_0 = nx.Graph(G_0)
G_t = G_0
G_times = []
G_times.append(G_t)
for t in range(maxt):
if (t in cps):
for i in range(len(probs)):
for j in range(len(probs[0])):
if (probs[i][j] < intra_prob):
probs[i][j] = probs[i][j] + increment
G_t = SBM_snapshot(G_t, alpha, sizes, probs)
G_times.append(G_t)
else:
G_t = SBM_snapshot(G_t, alpha, sizes, probs)
G_times.append(G_t)
print("generating " + str(t), end="\r")
#write the entire history of snapshots
to_edgelist(G_times, fname)
def create_test_graphs(n,
block_sizes=[],
block_prob=[],
graph_type='inv_cov',
seed_nb=123):
if graph_type == 'inv_cov':
l1 = sklearn.datasets.make_spd_matrix(n)
elif graph_type == 'cov':
l1 = sklearn.datasets.make_spd_matrix(n)
l1 = lg.inv(l1)
else:
if graph_type == 'geo':
g1 = nx.random_geometric_graph(n, 0.55)
if graph_type == 'er':
g1 = nx.erdos_renyi_graph(n, 0.45)
if graph_type == 'sbm':
g1 = nx.stochastic_block_model(block_sizes,
block_prob,
seed=seed_nb)
g1.remove_nodes_from(list(nx.isolates(g1)))
n = len(g1)
l1 = nx.laplacian_matrix(g1, range(n))
l1 = np.array(l1.todense())
# Permutation and second graph
l2, P_true = create_permutation(n, l1, seed_nb)
x = np.double(l1)
y = l2
return [x, y, P_true]
def create_sbm():
"""simple test"""
# set a simple SBM model
sizes = [15, 35, 25]
probs = [[0.7, 0.08, 0.10], [0.08, 0.8, 0.02], [0.10, 0.02, 0.80]]
graph = nx.stochastic_block_model(sizes, probs, seed=0)
# need to set the weights to 1
for i, j in graph.edges():
graph[i][j]["weight"] = 1
# ground truth
community_labels = [graph.nodes[i]["block"] for i in graph]
# spring layout
pos = nx.spring_layout(graph, weight=None, scale=1)
for u in graph:
graph.nodes[u]["pos"] = pos[u]
# draw the graph with ground truth
plt.figure(figsize=(5, 4))
nx.draw(graph, pos=pos, node_color=community_labels)
plt.title("Ground truth communities")
plt.savefig("ground_truth.png", bbox_inches="tight")
with open("sbm_graph.pkl", "wb") as pickle_file:
pickle.dump(nx.adjacency_matrix(graph, weight="weight"), pickle_file)
nx.write_gpickle(graph, "sbm_graph.gpickle")
def process(self):
# Read data into huge `Data` list.
data_list = []
for i in range(self.num_samples):
y = np.random.randint(len(self.targets))
probs = [
[self.in_probs[0], self.targets[y]],
[self.targets[y], self.in_probs[1]],
]
block_sizes = self.sizes[np.random.randint(len(self.sizes))]
G = nx.stochastic_block_model(block_sizes, probs, seed=i + 1)
x = torch.zeros((sum(block_sizes), self.n_features))
for i, partition in enumerate(G.graph["partition"]):
partition = np.array(list(partition))
feat = np.random.choice(self.n_features,
len(partition),
p=self.feat_probs[i])
x[partition, feat] = 1
data = from_networkx(G)
data["x"] = x.float()
data["y"] = torch.tensor([y])
data_list.append(data)
if self.pre_filter is not None:
data_list = [data for data in data_list if self.pre_filter(data)]
if self.pre_transform is not None:
data_list = [self.pre_transform(data) for data in data_list]
data, slices = self.collate(data_list)
torch.save((data, slices), self.processed_paths[0])
def test_stochastic_block_model():
sizes = [75, 75, 300]
probs = [[0.25, 0.05, 0.02],
[0.05, 0.35, 0.07],
[0.02, 0.07, 0.40]]
G = nx.stochastic_block_model(sizes, probs, seed=0)
C = G.graph['partition']
assert_equal(len(C), 3)
assert_equal(len(G), 450)
assert_equal(G.size(), 22160)
GG = nx.stochastic_block_model(sizes, probs, range(450), seed=0)
assert_equal(G.nodes, GG.nodes)
# Test Exceptions
sbm = nx.stochastic_block_model
badnodelist = list(range(400)) # not enough nodes to match sizes
badprobs1 = [[0.25, 0.05, 1.02],
[0.05, 0.35, 0.07],
[0.02, 0.07, 0.40]]
badprobs2 = [[0.25, 0.05, 0.02],
[0.05, -0.35, 0.07],
[0.02, 0.07, 0.40]]
probs_rect1 = [[0.25, 0.05, 0.02],
[0.05, -0.35, 0.07]]
probs_rect2 = [[0.25, 0.05],
[0.05, -0.35],
[0.02, 0.07]]
asymprobs = [[0.25, 0.05, 0.01],
[0.05, -0.35, 0.07],
[0.02, 0.07, 0.40]]
assert_raises(nx.NetworkXException, sbm, sizes, badprobs1)
assert_raises(nx.NetworkXException, sbm, sizes, badprobs2)
assert_raises(nx.NetworkXException, sbm, sizes, probs_rect1, directed=True)
assert_raises(nx.NetworkXException, sbm, sizes, probs_rect2, directed=True)
assert_raises(nx.NetworkXException, sbm, sizes, asymprobs, directed=False)
assert_raises(nx.NetworkXException, sbm, sizes, probs, badnodelist)
nodelist = [0] + list(range(449)) # repeated node name in nodelist
assert_raises(nx.NetworkXException, sbm, sizes, probs, nodelist)
# Extra keyword arguments test
GG = nx.stochastic_block_model(sizes, probs, seed=0, selfloops=True)
assert_equal(G.nodes, GG.nodes)
GG = nx.stochastic_block_model(sizes, probs, selfloops=True, directed=True)
assert_equal(G.nodes, GG.nodes)
GG = nx.stochastic_block_model(sizes, probs, seed=0, sparse=False)
assert_equal(G.nodes, GG.nodes)
def test_stochastic_block_model():
sizes = [75, 75, 300]
probs = [[0.25, 0.05, 0.02],
[0.05, 0.35, 0.07],
[0.02, 0.07, 0.40]]
G = nx.stochastic_block_model(sizes, probs, seed=0)
C = G.graph['partition']
assert len(C) == 3
assert len(G) == 450
assert G.size() == 22160
GG = nx.stochastic_block_model(sizes, probs, range(450), seed=0)
assert G.nodes == GG.nodes
# Test Exceptions
sbm = nx.stochastic_block_model
badnodelist = list(range(400)) # not enough nodes to match sizes
badprobs1 = [[0.25, 0.05, 1.02],
[0.05, 0.35, 0.07],
[0.02, 0.07, 0.40]]
badprobs2 = [[0.25, 0.05, 0.02],
[0.05, -0.35, 0.07],
[0.02, 0.07, 0.40]]
probs_rect1 = [[0.25, 0.05, 0.02],
[0.05, -0.35, 0.07]]
probs_rect2 = [[0.25, 0.05],
[0.05, -0.35],
[0.02, 0.07]]
asymprobs = [[0.25, 0.05, 0.01],
[0.05, -0.35, 0.07],
[0.02, 0.07, 0.40]]
pytest.raises(nx.NetworkXException, sbm, sizes, badprobs1)
pytest.raises(nx.NetworkXException, sbm, sizes, badprobs2)
pytest.raises(nx.NetworkXException, sbm, sizes, probs_rect1, directed=True)
pytest.raises(nx.NetworkXException, sbm, sizes, probs_rect2, directed=True)
pytest.raises(nx.NetworkXException, sbm, sizes, asymprobs, directed=False)
pytest.raises(nx.NetworkXException, sbm, sizes, probs, badnodelist)
nodelist = [0] + list(range(449)) # repeated node name in nodelist
pytest.raises(nx.NetworkXException, sbm, sizes, probs, nodelist)
# Extra keyword arguments test
GG = nx.stochastic_block_model(sizes, probs, seed=0, selfloops=True)
assert G.nodes == GG.nodes
GG = nx.stochastic_block_model(sizes, probs, selfloops=True, directed=True)
assert G.nodes == GG.nodes
GG = nx.stochastic_block_model(sizes, probs, seed=0, sparse=False)
assert G.nodes == GG.nodes
def generate_random_traffic_network(
n_clients: int = 200,
n_servers={
"SMB": 1,
"HTTP": 1,
"RDP": 1,
},
seed: Optional[int] = 0,
tolerance: np.float32 = np.float32(1e-3),
alpha=np.array([(0.1, 0.3), (0.18, 0.09)], dtype=float),
beta=np.array([(100, 10), (10, 100)], dtype=float),
) -> nx.DiGraph:
"""
Randomly generate a directed multi-edge network graph representing
fictitious SMB, HTTP, and RDP traffic.
Arguments:
n_clients: number of workstation nodes that can initiate sessions with server nodes
n_servers: dictionary indicatin the numbers of each nodes listening to each protocol
seed: seed for the psuedo-random number generator
tolerance: absolute tolerance for bounding the edge probabilities in [tolerance, 1-tolerance]
alpha: beta distribution parameters alpha such that E(edge prob) = alpha / beta
beta: beta distribution parameters beta such that E(edge prob) = alpha / beta
Returns:
(nx.classes.multidigraph.MultiDiGraph): the randomly generated network from the hierarchical block model
"""
edges_labels = defaultdict(set) # set backed multidict
for protocol in list(n_servers.keys()):
sizes = [n_clients, n_servers[protocol]]
# sample edge probabilities from a beta distribution
np.random.seed(seed)
probs: np.ndarray = np.random.beta(a=alpha, b=beta, size=(2, 2))
# scale by edge type
if protocol == "SMB":
probs = 3 * probs
if protocol == "RDP":
probs = 4 * probs
# don't allow probs too close to zero or one
probs = np.clip(probs,
a_min=tolerance,
a_max=np.float32(1.0 - tolerance))
# sample edges using block models given edge probabilities
di_graph_for_protocol = nx.stochastic_block_model(sizes=sizes,
p=probs,
directed=True,
seed=seed)
for edge in di_graph_for_protocol.edges:
edges_labels[edge].add(protocol)
digraph = nx.DiGraph()
for (u, v), port in list(edges_labels.items()):
digraph.add_edge(u, v, protocol=port)
return digraph
def SBM( self, sizes, probs ): self.G = nx.stochastic_block_model( sizes, probs ) self.Adjacency = nx.to_scipy_sparse_matrix(self.G) self.Labels = [] [self.Labels.extend([i for _ in range(sizes[i])]) for i in range(len(sizes))] self.Labels = np.array( self.Labels ) self.labeled = True return
def generate_ChangePoint(inter_prob, intra_prob, alpha):
cps = [15, 30, 60, 75, 90, 105, 135]
fname = "ChangePoint_" + str(inter_prob) + "_" + str(
intra_prob) + "_" + str(alpha) + ".txt"
cps_sizes = []
cps_probs = []
#let there be 500 nodes
sizes_1 = [250, 250] #500 nodes total at all times
probs_1 = construct_SBM_block(sizes_1, inter_prob, intra_prob)
sizes_2 = [125, 125, 125, 125]
probs_2 = construct_SBM_block(sizes_2, inter_prob, intra_prob)
sizes_3 = [50] * 10
probs_3 = construct_SBM_block(sizes_3, inter_prob, intra_prob)
list_sizes = []
list_sizes.append(sizes_1)
list_sizes.append(sizes_2)
list_sizes.append(sizes_3)
list_probs = []
list_probs.append(probs_1)
list_probs.append(probs_2)
list_probs.append(probs_3)
list_idx = 1
sizes = sizes_2
probs = probs_2
maxt = 150
G_0 = nx.stochastic_block_model(sizes, probs)
G_0 = nx.Graph(G_0)
G_t = G_0
G_times = []
G_times.append(G_t)
for t in range(maxt):
if (t in cps):
if ((list_idx + 1) > len(list_sizes) - 1):
list_idx = 0
else:
list_idx = list_idx + 1
sizes = list_sizes[list_idx]
probs = list_probs[list_idx]
G_t = SBM_snapshot(G_t, alpha, sizes, probs)
G_times.append(G_t)
print("generating " + str(t), end="\r")
else:
G_t = SBM_snapshot(G_t, alpha, sizes, probs)
G_times.append(G_t)
print("generating " + str(t), end="\r")
#write the entire history of snapshots
to_edgelist(G_times, fname)
def sbms3(n=100, n1=100, n2=50, n3=50, p=0.5, q=0.1):
# generate n sbm graphs
gs = []
sizes = [n1, n2, n3]
probs = [[p, q, q], [q, p, q], [q, q, p]]
for i in range(n):
g = stochastic_block_model(sizes, probs, seed=i)
gs.append(g)
return gs
def generate_stochastic_block_model(sizes, probs):
"""
:param sizes: sizes of each blocks
:param probs: list of list of floats. Element(i, j) gives the density of edges
going from nodes of group i to nodes of group j.
:return: Stochastic block model NetworkX Graph
"""
return nx.stochastic_block_model(sizes, probs, seed=0)
def sbms(n=100, n1=100, n2=50, p=0.5, q=0.1):
# generate n sbm graphs of 2 blocks whose size are n1 and n2
gs = []
sizes = [n1, n2]
probs = [[p, q], [q, p]]
for i in range(n):
g = stochastic_block_model(sizes, probs, seed=i)
gs.append(g)
print(f'generating {n} sbm graphs of {n1} with p {p} and {n2} with p {q}')
return gs
def generate_initial_matrix(graph_size):
"""
Generates the an n x n adj graph with a given level of community separation
:param graph_size: the size of the graph.
:return: Adj matrix with 2 seperate communities.
"""
N = 20 # Time steps
n = graph_size # Nodes
p = .4
c = 2
adj = np.zeros([n, n, 2 * N + 2])
# Create random graph and store to the last lateral slice of adj
G = nx.fast_gnp_random_graph(n, p, seed=4652, directed=False)
G_adj = nx.to_numpy_matrix(G)
adj[0:n, 0:n,
0] = G_adj # Stoer the first adjacency matrix in the first frontal slice of adj
# Create another random graph and store to the last lateral slice of adj
# G = nx.fast_gnp_random_graph(n,p,seed = 4652, directed = False)
# G_adj = nx.to_numpy_matrix(G)
adj[0:n, 0:n, (2 * N + 2) -
1] = G_adj # Store the last adjacency matrix in the last frontal slice of adj
# Generate graph community over 20 graphs.
for i in range(1, N + 1):
q = p - p * i / (N + 1)
P = np.array([[p, q], [q, p]])
Gsbm = nx.to_numpy_matrix(
nx.stochastic_block_model([int(n / c), int(n / c)], P))
adj[0:n, 0:n, i] = Gsbm
# print(i)
eps = 0.0
for j in range(1, N + 2):
q = p * (j - 1) / (
N + 2) + eps # remove the eps for complete dissconnectivity #
P = np.array([[p, q], [q, p]])
Gsbm = nx.to_numpy_matrix(
nx.stochastic_block_model([int(n / c), int(n / c)], P))
adj[0:n, 0:n, i + j] = Gsbm
# print(i+j)
return adj[:, :, 20]
def synthetic(n=20, m=20):
# n communities of size m
sizes = [m] * n
# mixing matrix
probs = np.ones((n, n)) * 0.05
for i in range(n):
probs[i][i] = 0.8
#probs[:20][:20] += 0.1
G = nx.stochastic_block_model(sizes, probs, seed=0)
G.graph['name'] = 'synthetic'
print(nx.info(G))
############## what if the graph is large ##################
# the experiment shows that Wilk's theorem is true?!
# in that way, n->+inf, the distribution is indeed chi-squared
gnc = [list(range(i * m, i * m + m)) for i in range(n)]
import random
X = list(range(n * m))
random.shuffle(X)
tmp = {i: x for i, x in enumerate(X)}
indices = [list(range(i * m, i * m + m)) for i in range(n)]
# a completly random partition
rand_comms = [list(map(tmp.get, row)) for row in indices]
print("\n".join(map(str, rand_comms)))
LR_test = _2ll(G, rand_comms)
print("random communities", LR_test)
LR_test = _2ll(G, gnc)
print("ground truth", LR_test)
pvalue(G, gnc, LR_test, L=3000, plothist=True)
return
############# end of this experiment #######################
# community detection
comms = list(multiscale_community_detection(G, gamma=0.3))
map_comm = {v: i for i, c in enumerate(comms) for v in c}
# check NMI
a = [map_comm[k] for k in G.nodes()]
b = [k // m for k in G.nodes()]
print("NMI=", metrics.adjusted_mutual_info_score(a, b))
print("#Comm=", len(comms))
print(comms)
comms = greedy_modularity_communities(G)
print("#Comm=", len(comms))
print(comms)
def generate_SBM(self,Graphs_num=300,nodes_per_graph=60,block_size=10,fraction=0.3,mult_factor=1.2,avg_deg=10,test_size=0.2):
blocks_num=int(nodes_per_graph/block_size)
sizes=[block_size]*blocks_num
G,y=[],[]
for i in range (Graphs_num):
p_in=fraction if i <Graphs_num/2 else fraction*mult_factor
p_out=(avg_deg-(block_size-1)*p_in)/(nodes_per_graph-block_size)
p=p_out*np.ones([blocks_num]*2)+(p_in-p_out)*np.eye(blocks_num)
#print(p_in,p_out)
G.append(nx.stochastic_block_model(sizes, p))
y.append(-1 if i<Graphs_num/2 else 1)
G_train, G_test, y_train, y_test = train_test_split(G, y, test_size=test_size)
return (G_train,y_train),(G_test,y_test)
def generate_network(n, on_the_run_tweaks):
P_norm_dat = pd.read_csv('P_norm.csv', index_col=0)
P_norm = np.array(P_norm_dat)
#print(P_norm)
# n = 1000
Asi_population = int(0.08 * n)
Lat_population = int(0.29 * n)
Bla_population = int(0.30 * n)
Whi_population = int(0.33 * n)
#sizes = list([Asi_population, Lat_population, Bla_population, Whi_population])
sizes = list(
[Whi_population, Bla_population, Asi_population, Lat_population])
# probs = [[AA , AL, AB, AW ], # Asian
# [AL, LL , LB, LW ], # Latino
# [AB, LB, BB, BW ],# Black
# [AW, LW, BW, WW]] # White
#
probs = P_norm
if on_the_run_tweaks:
call = 58
cb = 1.08
ca = 0.83
cl = 1.04
probs /= call
probs[1, :] /= cb
probs[:, 1] /= cb
probs[2, :] /= ca
probs[:, 2] /= ca
probs[3, :] /= cl
probs[:, 3] /= cl
print(probs)
g = stochastic_block_model(sizes, probs, sparse=True)
# neigh = np.array(list(map(lambda i: len(list(g.neighbors(i))), range(len(g)))))
# isolated_neighbors = []
# if len(np.argwhere(neigh == 0)) != 0: isolated_neighbors = np.argwhere(neigh == 0)[:, 0]
#
# for i in isolated_neighbors:
# g.add_edge(i, np.random.randint(0, n))
# g.add_edge(i, np.random.randint(0, n))
return g
def generate_network(n, on_the_run_tweaks):
P_norm_dat = pd.read_csv('P_norm.csv', index_col = 0)
P_norm = np.array(P_norm_dat)
global sizes
#sizes = list([Asi_population, Lat_population, Bla_population, Whi_population])
#sizes = list([Whi_population, Bla_population, Asi_population, Lat_population])
sizes = np.array( pd.read_csv('Population_fraction.csv') )[0]
sizes = sizes / sizes.sum() * n
sizes = sizes.astype('int')
probs = P_norm
if on_the_run_tweaks:
call = 57.5
cb = 1.08
ca = 0.83
cl = 1.04
probs /= call
probs[1,:] /= cb
probs[:,1] /= cb
probs[2,:] /= ca
probs[:,2] /= ca
probs[3,:] /= cl
probs[:,3] /= cl
#print(P_norm_dat)
#plt.imshow(P_norm_dat)
#plt.colorbar()
#plt.show()
P_norm_dat[:] = probs
#print(P_norm_dat)
#plt.imshow(P_norm_dat)
#plt.colorbar()
P_norm_dat.to_csv('P_norm_adj.csv')
print(probs)
g = stochastic_block_model(sizes, probs, sparse=True)
# neigh = np.array(list(map(lambda i: len(list(g.neighbors(i))), range(len(g)))))
# isolated_neighbors = []
# if len(np.argwhere(neigh == 0)) != 0: isolated_neighbors = np.argwhere(neigh == 0)[:, 0]
#
# for i in isolated_neighbors:
# g.add_edge(i, np.random.randint(0, n))
# g.add_edge(i, np.random.randint(0, n))
return g
def run(self, num_itera=30):
if num_itera == 1:
self.G_ = nx.stochastic_block_model(self.sizes, self.probs)
self.A = nx.to_numpy_array(self.G_)
else:
# Run SBM
Anull = [
nx.to_numpy_array(
nx.stochastic_block_model(self.sizes, self.probs))
for i in range(num_itera)
]
Anull_mean = np.mean(Anull, axis=0)
self.A = Anull_mean
self.G_ = nx.stochastic_block_model(self.sizes, self.probs)
# Get partition/communities - loop through every node and find
# ... its RSN community based on the index bounds for each RSN
bounds = np.array(self.__rsn_index_change(self.labels)[1:])
bounds[-1] += 1
self.partition = {
node: np.where(node < bounds)[0][0]
for node in list(self.G_.nodes())
}
def generate_sbm(sizes, probs, maxweight=1):
"""Generate a Stochastic Block Model graph.
Assign random values drawn from U({1, ..., maxw}) to the edges.
sizes : list of sizes (int) of the blocks
probs : matrix of probabilities (in [0, 1]) of edge creation
between nodes depending on the blocks they belong to
maxweight : maximum value of the weights to randomly assign
(default 1, resulting in weights all equal to 1)
"""
graph = nx.stochastic_block_model(sizes, probs)
weights = 1 + np.random.choice(maxweight, len(graph.edges))
weights = dict(zip(graph.edges, weights))
nx.set_edge_attributes(graph, weights, 'weight')
return graph
def desymmetric_stochastic(sizes=[100, 100, 100],
probs=[[0.5, 0.45, 0.45], [0.45, 0.5, 0.45],
[0.45, 0.45, 0.5]],
seed=0,
off_diag_prob=0.9,
norm=False):
from sklearn.model_selection import train_test_split
g = nx.stochastic_block_model(sizes, probs, seed=seed)
original_A = nx.adjacency_matrix(g).todense()
A = original_A.copy()
# for blocks represent adj within clusters
accum_size = 0
for s in sizes:
x, y = np.where(
np.triu(original_A[accum_size:s + accum_size,
accum_size:s + accum_size]))
x1, x2, y1, y2 = train_test_split(x, y, test_size=0.5)
A[x1 + accum_size, y1 + accum_size] = 0
A[y2 + accum_size, x2 + accum_size] = 0
accum_size += s
# for blocks represent adj out of clusters (cluster2cluster edges)
accum_x, accum_y = 0, 0
n_cluster = len(sizes)
for i in range(n_cluster):
accum_y = accum_x + sizes[i]
for j in range(i + 1, n_cluster):
x, y = np.where(original_A[accum_x:sizes[i] + accum_x,
accum_y:sizes[j] + accum_y])
x1, x2, y1, y2 = train_test_split(x, y, test_size=off_diag_prob)
A[x1 + accum_x, y1 + accum_y] = 0
A[y2 + accum_y, x2 + accum_x] = 0
accum_y += sizes[j]
accum_x += sizes[i]
# label assignment based on parameter sizes
label = []
for i, s in enumerate(sizes):
label.extend([i] * s)
label = np.array(label)
return np.array(original_A), np.array(A), label
def create_sb_graph(self):
"""
Generating a stochastic block graph.
Example parameter dictionary: { "method": "sb", "n_blocks": 10, "p": 0.1, "k": 0.01 }
self.p : float
Probability of the edge between nodes belonging to different components
self.k : float
Probability of edges within a block/community
self.n_blocks : int
Number of blocks/communities that are to be created within the network
"""
n_blocks = self.params['n_blocks']
probabilities = np.full((n_blocks, n_blocks), self.params['p'])
np.fill_diagonal(probabilities, self.params['k'])
block_sizes = [int(self.n / n_blocks) for _ in range(n_blocks)]
self.G = nx.stochastic_block_model(block_sizes, probabilities)
def block_stochastic(parameters):
"""Construct the block stochastic graphs.
Parameters
----------
parameters : tuple
List of parameters:
n_range: range of graph sizes,
prob_type: type of edge probabilities to be set
rep_graph: number of graphs to be generated per n.
seed: random seed to be used
Returns
-------
list
list of networkx.Digraphs
"""
n_range, prob_type, rep_graph, seed = parameters
p_range = [0.03 * i for i in range(1, 10)]
graphs = []
for n in n_range:
for i in range(rep_graph):
sizes = [
ceil(4 * n / 12),
ceil(3 * n / 12),
ceil(2 * n / 12),
ceil(1 * n / 12),
ceil(1 * n / 12),
ceil(1 * n / 12)
]
for p in p_range:
probs = (.3 - p) * (np.ones(6) - np.eye(6)) + p * (np.eye(6))
U = nx.stochastic_block_model(sizes, probs, seed=seed)
G = random_direct(U)
G.graph['graphname'] = '_'.join(
['block_stochastic', 'n',
str(n), 'p',
str(p),
str(i)])
set_probabilities(G, prob_type=prob_type)
set_unit_node_weights(G)
graphs.append(G)
return graphs
def run_community_graph(args):
#test if Lx and Ly are the same, then dist should be small!
#laplacian make integral at end. Inverse is often quite small for images! ?? leading to tiny evals. even neg
#args.Lx = torch.eye(args.m)*torch.abs(torch.randn((args.m, args.m)))*2 #utils.symmetrize(torch.randn((args.m, args.m)))
args.m = 12
args.n = 4
if args.fix_seed:
torch.manual_seed(0)
g = nx.stochastic_block_model([6, 6], [[0.9, 0.1], [0.1, 0.9]], seed=8576)
#components = nx.connected_components(g)
g.remove_nodes_from(list(nx.isolates(g)))
args.m = len(g)
Lx = nx.laplacian_matrix(g, range(args.m)).todense()
args.Lx = torch.from_numpy(Lx).to(
dtype=torch.float32) #+ torch.ones(args.m, args.m)/args.m
args.n_epochs = 370 #100
graph.graph_dist(args)
def run_community_graph(args):
args.m = 12
args.n = 4
if args.fix_seed:
torch.manual_seed(0)
#args.Lx = torch.randn(args.m*(args.m-1)//2) #torch.FloatTensor([[1, -1], [-1, 2]])
#args.Lx = realize_upper(args.Lx, args.m)
#pdb.set_trace()
g = nx.stochastic_block_model([6,6],[[0.9,0.1],[0.1,0.9]], seed = 8576)
#components = nx.connected_components(g)
g.remove_nodes_from(list(nx.isolates(g)))
args.m = len(g)
Lx = nx.laplacian_matrix(g, range(args.m)).todense()
args.Lx = torch.from_numpy(Lx).to(dtype=torch.float32) #+ torch.ones(args.m, args.m)/args.m
args.n_epochs = 370 #100
graph.graph_dist(args)
def SBM_ER(N_group, p, weight, seed):
"""TODO: Docstring for SBM_ER.
:N_group: TODO
:p: TODO
:seed: TODO
:returns: TODO
"""
G = nx.stochastic_block_model(N_group, p, seed=seed)
A = nx.to_numpy_array(G)
A = A * weight
A_index = np.where(A > 0)
A_interaction = A[A_index]
index_i = A_index[0]
index_j = A_index[1]
degree = np.sum(A > 0, 1)
cum_index = np.hstack((0, np.cumsum(degree)))
return A, A_interaction, index_i, index_j, cum_index