def _gen_mat_data(n: int = 20,
m: int = 20,
p: int = 0.50,
mat_type: str = 'sb',
binary: bool = False,
asfile: bool = True,
n_graphs: int = 1):
if binary is True:
wt = 1
else:
wt = np.random.uniform
mat_list = []
mat_file_list = []
for nm in range(n_graphs):
if mat_type == 'er':
mat = largest_connected_component(
symmetrize(
remove_loops(
er_nm(n,
m,
wt=np.random.uniform,
wtargs=dict(low=0, high=1)))))
elif mat_type == 'sb':
if p is None:
raise ValueError(
f"for mat_type {mat_type}, p cannot be None")
mat = largest_connected_component(
symmetrize(
remove_loops(
sbm(np.array([n]),
np.array([[p]]),
wt=wt,
wtargs=dict(low=0, high=1)))))
else:
raise ValueError(f"mat_type {mat_type} not recognized!")
mat_list.append(mat)
if asfile is True:
mat_path_tmp = tempfile.NamedTemporaryFile(mode='w+',
suffix='.npy',
delete=False)
mat_path = str(mat_path_tmp.name)
np.save(mat_path, mat)
mat_file_list.append(mat_path)
mat_path_tmp.close()
return {'mat_list': mat_list, 'mat_file_list': mat_file_list}
def prune_graph(self):
import graspologic.utils as gu
from pynets.statistics.individual.algorithms import defragment, \
prune_small_components, most_important
hardcoded_params = utils.load_runconfig()
if int(self.prune) not in range(0, 4):
raise ValueError(f"Pruning option {self.prune} invalid!")
if self.prune != 0:
# Remove isolates
G_tmp = self.G.copy()
self.G = defragment(G_tmp)[0]
del G_tmp
if int(self.prune) == 1:
try:
self.G = prune_small_components(
self.G, min_nodes=hardcoded_params["min_nodes"][0])
except BaseException:
print(
UserWarning(f"Warning: pruning {self.est_path} "
f"failed..."))
elif int(self.prune) == 2:
try:
hub_detection_method = \
hardcoded_params["hub_detection_method"][0]
print(f"Filtering for hubs on the basis of "
f"{hub_detection_method}...\n")
self.G = most_important(self.G, method=hub_detection_method)[0]
except FileNotFoundError as e:
import sys
print(e, "Failed to parse advanced.yaml")
elif int(self.prune) == 3:
print("Pruning all but the largest connected "
"component subgraph...")
self.G = gu.largest_connected_component(self.G)
else:
print("No graph defragmentation applied...")
self.G = nx.from_numpy_array(self.in_mat)
if nx.is_empty(self.G) is True or \
(np.abs(self.in_mat) < 0.0000001).all() or \
self.G.number_of_edges() == 0:
print(
UserWarning(f"Warning: {self.est_path} "
f"empty after pruning!"))
return self.in_mat, None
# Saved pruned
if (self.prune != 0) and (self.prune is not None):
final_mat_path = f"{self.est_path.split('.npy')[0]}{'_pruned'}"
utils.save_mat(self.in_mat, final_mat_path, self.out_fmt)
print(f"{'Source File: '}{final_mat_path}")
return self.in_mat, final_mat_path
def _gen_mat_data(n: int=20, m: int=20, p: int=0.50,
mat_type: str='sb', binary: bool=False,
asfile: bool=True, n_graphs: int=1,
lcc: bool=False, modality: str='func'):
if binary is True:
wt = 1
else:
wt = np.random.uniform
mat_list = []
mat_file_list = []
if n_graphs > 0:
for nm in range(n_graphs):
if mat_type == 'er':
mat = symmetrize(
remove_loops(er_nm(n, m, wt=np.random.uniform,
wtargs=dict(low=0, high=1))))
elif mat_type == 'sb':
if p is None:
raise ValueError(
f"for mat_type {mat_type}, p cannot be None")
mat = symmetrize(
remove_loops(sbm(np.array([n]), np.array([[p]]),
wt=wt, wtargs=dict(low=0,
high=1))))
else:
raise ValueError(f"mat_type {mat_type} not recognized!")
if lcc is True:
mat = largest_connected_component(mat)
mat_list.append(autofix(mat))
if asfile is True:
path_tmp = tempfile.NamedTemporaryFile(mode='w+',
suffix='.npy',
delete=False)
mat_path_tmp = str(path_tmp.name)
out_folder = f"{str(Path.home())}/test_mats"
os.makedirs(out_folder, exist_ok=True)
if modality == 'func':
mat_path = f"{out_folder}/graph_sub-999_modality-func_" \
f"model-corr_template-" \
f"MNI152_2mm_" \
f"parc_tol-6fwhm_hpass-" \
f"0Hz_" \
f"signal-mean_thrtype-prop_thr-" \
f"{round(random.uniform(0, 1),2)}.npy"
elif modality == 'dwi':
mat_path = f"{out_folder}/graph_sub-999_modality-func_" \
f"model-csa_template-" \
f"MNI152_2mm_tracktype-local_" \
f"traversal-det_minlength-30_" \
f"tol-5_thrtype-prop_thr-" \
f"{round(random.uniform(0, 1),2)}.npy"
shutil.copyfile(mat_path_tmp, mat_path)
np.save(mat_path, mat)
mat_file_list.append(mat_path)
path_tmp.close()
return {'mat_list': mat_list, 'mat_file_list': mat_file_list}
def omnibus_embedding_pairwise(
graphs: List[Union[nx.Graph, nx.OrderedGraph, nx.DiGraph,
nx.OrderedDiGraph]],
dimensions: int = 100,
elbow_cut: Optional[int] = None,
svd_solver_algorithm: SvdAlgorithmType = "randomized",
svd_solver_iterations: int = 5,
svd_seed: Optional[int] = None,
weight_attribute: str = "weight",
use_laplacian: bool = False,
) -> List[Tuple[Embeddings, Embeddings]]:
"""
Generates a pairwise omnibus embedding for each pair of graphs in a list of graphs using the adjacency matrix.
If given graphs A, B, and C, the embeddings will be computed for A, B and B, C.
If the node labels differ between each pair of graphs, then those nodes will only be found in the resulting embedding
if they exist in the largest connected component of the union of all edges across all graphs in the time series.
Graphs will always have their diagonal augmented. In other words, a self-loop
will be created for each node with a weight corresponding to the weighted degree.
Lastly, all weights will be rescaled based on their relative rank in the graph,
which is beneficial in minimizing anomalous results if some edge weights are
extremely atypical of the rest of the graph.
Parameters
----------
graphs : List[Union[nx.Graph, nx.OrderedGraph, nx.DiGraph, nx.OrderedDiGraph]]
A list of undirected or directed graphs. The graphs **must**:
- be fully numerically weighted (every edge must have a real, numeric weight
or else it will be treated as an unweighted graph)
- be a basic graph (meaning it should not be a multigraph; if you have a
multigraph you must first decide how you want to handle the weights of the
edges between two nodes, whether summed, averaged, last-wins,
maximum-weight-only, etc)
dimensions : int (default=100)
Dimensions to use for the svd solver.
For undirected graphs, if ``elbow_cut==None``, you will receive an embedding
that has ``nodes`` rows and ``dimensions`` columns.
For directed graphs, if ``elbow_cut==None``, you will receive an embedding that
has ``nodes`` rows and ``2*dimensions`` columns.
If ``elbow_cut`` is specified to be not ``None``, we will cut the embedding at
``elbow_cut`` elbow, but the provided ``dimensions`` will be used in the
creation of the SVD.
elbow_cut : Optional[int] (default=None)
Using a process described by Zhu & Ghodsi in their paper "Automatic
dimensionality selection from the scree plot via the use of profile likelihood",
truncate the dimensionality of the return on the ``elbow_cut``-th elbow.
By default this value is ``None`` but can be used to reduce the dimensionality
of the returned tensors.
svd_solver_algorithm : str (default="randomized")
allowed values: {'randomized', 'full', 'truncated'}
SVD solver to use:
- 'randomized'
Computes randomized svd using
:func:`sklearn.utils.extmath.randomized_svd`
- 'full'
Computes full svd using :func:`scipy.linalg.svd`
Does not support ``graph`` input of type scipy.sparse.csr_matrix
- 'truncated'
Computes truncated svd using :func:`scipy.sparse.linalg.svds`
svd_solver_iterations : int (default=5)
Number of iterations for randomized SVD solver. Not used by 'full' or
'truncated'. The default is larger than the default in randomized_svd
to handle sparse matrices that may have large slowly decaying spectrum.
svd_seed : Optional[int] (default=None)
Used to seed the PRNG used in the ``randomized`` svd solver algorithm.
weight_attribute : str (default="weight")
The edge dictionary key that contains the weight of the edge.
use_laplacian : bool (default=False)
Determine whether to use the Laplacian matrix of each graph in order to
calculate the omnibus embedding using the Laplacian spectral embedding
technique.
Returns
-------
List[Tuple[Embeddings, Embeddings]]
Raises
------
beartype.roar.BeartypeCallHintPepParamException if parameters do not match type hints
ValueError if values are not within appropriate ranges or allowed values
See Also
--------
graspologic.pipeline.embed.Embeddings
graspologic.embed.OmnibusEmbed
graspologic.embed.AdjacencySpectralEmbed
graspologic.embed.select_svd
References
----------
.. [1] Levin, K., Athreya, A., Tang, M., Lyzinski, V., & Priebe, C. E. (2017,
November). A central limit theorem for an omnibus embedding of multiple random
dot product graphs. In Data Mining Workshops (ICDMW), 2017 IEEE International
Conference on (pp. 964-967). IEEE.
.. [2] Sussman, D.L., Tang, M., Fishkind, D.E., Priebe, C.E. "A
Consistent Adjacency Spectral Embedding for Stochastic Blockmodel Graphs,"
Journal of the American Statistical Association, Vol. 107(499), 2012
.. [3] Levin, K., Roosta-Khorasani, F., Mahoney, M. W., & Priebe, C. E. (2018).
Out-of-sample extension of graph adjacency spectral embedding. PMLR: Proceedings
of Machine Learning Research, 80, 2975-2984.
.. [4] Zhu, M. and Ghodsi, A. (2006). Automatic dimensionality selection from the
scree plot via the use of profile likelihood. Computational Statistics & Data
Analysis, 51(2), pp.918-930.
"""
check_argument(len(graphs) > 1, "more than one graph is required")
check_argument(dimensions >= 1, "dimensions must be positive")
check_argument(elbow_cut is None or elbow_cut >= 1,
"elbow_cut must be positive")
check_argument(
svd_solver_algorithm in __SVD_SOLVER_TYPES,
f"svd_solver_algorithm must be one of the values in {','.join(__SVD_SOLVER_TYPES)}",
)
check_argument(svd_solver_iterations >= 1,
"svd_solver_iterations must be positive")
check_argument(
svd_seed is None or 0 <= svd_seed <= 2**32 - 1,
"svd_seed must be a nonnegative, 32-bit integer",
)
used_weight_attribute = _graphs_precondition_checks(
graphs, weight_attribute)
perform_augment_diagonal = not use_laplacian
graph_embeddings = []
# create a graph that contains all nodes and edges across the entire corpus
union_graph = graphs[0].copy()
for graph in graphs[1:]:
union_graph.add_edges_from(graph.edges())
union_graph_lcc: Union[nx.Graph, nx.Digraph, nx.OrderedGraph,
nx.OrderedDiGraph] = largest_connected_component(
union_graph)
union_graph_lcc_nodes: Set[Any] = set(list(union_graph_lcc.nodes()))
union_node_ids = np.array(list(union_graph_lcc_nodes))
previous_graph = graphs[0].copy()
for graph in graphs[1:]:
current_graph = graph.copy()
# assure both graphs contain the exact same node set
# by removing nodes or adding isolates as needed
_sync_nodes(previous_graph, union_graph_lcc_nodes)
_sync_nodes(current_graph, union_graph_lcc_nodes)
# remove self loops, run pass to ranks and diagonal augmentation
previous_graph_augmented = _augment_graph(
previous_graph,
union_graph_lcc_nodes,
used_weight_attribute,
perform_augment_diagonal=perform_augment_diagonal,
)
current_graph_augmented = _augment_graph(
current_graph,
union_graph_lcc_nodes,
used_weight_attribute,
perform_augment_diagonal=perform_augment_diagonal,
)
model = OmnibusEmbed(
n_components=dimensions,
n_elbows=None, # we will do elbow cuts
algorithm=svd_solver_algorithm,
n_iter=svd_solver_iterations,
check_lcc=False,
diag_aug=False,
concat=False,
svd_seed=svd_seed,
lse=use_laplacian,
)
previous_embedding, current_embedding = model.fit_transform(
graphs=[previous_graph_augmented, current_graph_augmented])
previous_embedding_cut = _elbow_cut_if_needed(elbow_cut,
graph.is_directed(),
model.singular_values_,
previous_embedding)
current_embedding_cut = _elbow_cut_if_needed(elbow_cut,
graph.is_directed(),
model.singular_values_,
current_embedding)
graph_embeddings.append((
Embeddings(union_node_ids, previous_embedding_cut),
Embeddings(union_node_ids, current_embedding_cut),
))
return graph_embeddings
def to_largest_connected_component(adj, meta=None):
adj, lcc_inds = largest_connected_component(adj, return_inds=True)
if meta is not None:
return adj, meta.iloc[lcc_inds]
else:
return adj
"DCER": DCEREstimator(directed=True, loops=False, degree_directed=False),
}
rerun_test = False
if rerun_test:
currtime = time.time()
n_null_samples = 100
statistics = []
for edge_type in edge_types:
print(f"Edge type = {edge_type}")
edge_type_adj = mg.to_edge_type_graph(edge_type).adj
edge_type_adj = binarize(edge_type_adj)
largest_connected_component(edge_type_adj)
tstat = p_upper_tstat(edge_type_adj)
observed = pd.DataFrame(index=[0])
observed["estimated_p_upper"] = tstat
observed["edge_type"] = edge_type
observed["null_model"] = "Observed"
statistics.append(observed)
# estimate null distribution via bootstrap sampling
for null_name, NullEstimator in null_estimators.items():
ne = NullEstimator.fit(edge_type_adj)
def sampler():
return np.squeeze(ne.sample())