def yield_multiplex_network_in_layersets(imgdata,layerset_size,timewindow=100,overlap=0,nanlogfile=None):
# Yields small networks which contain layerset_size layers each,
# starting from the beginning of imgdata
# This is for running subgraph enumeration without having to load entire network at once (only create one relevant subnet at a time)
assert(layerset_size>0 and isinstance(layerset_size,int))
k = get_number_of_layers(imgdata.shape,timewindow,overlap)
start_times,end_times = get_start_and_end_times(k,timewindow,overlap)
# get all consecutive subsequences of length layerset_size in the range of all layers:
layersets = zip(*(range(k)[ii:] for ii in range(layerset_size)))
for layerset in layersets:
M = pn.MultiplexNetwork(couplings='ordinal',fullyInterconnected=True)
for tw_no in layerset:
A,voxellist = corrs_and_mask_calculations.make_adjacency_matrix(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],exclude_masked=True)
for ii in range(A.shape[0]):
node1 = str(voxellist[ii])
for jj in range(ii+1,A.shape[1]):
node2 = str(voxellist[jj])
if not np.isnan(A[ii,jj]):
M[node1,tw_no][node2,tw_no] = A[ii,jj]
else:
if nanlogfile != None:
with open(nanlogfile,'a+') as f:
f.write('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
else:
print('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
yield M
del(M)
def make_multiplex(imgdata,timewindow=100,overlap=0,nanlogfile=None):
# ignore remainder of timepoints
# maybe use nan-inclusive adjacency matrix to weed out masked voxels?
#xdim = imgdata.shape[0]
#ydim = imgdata.shape[1]
#zdim = imgdata.shape[2]
k = get_number_of_layers(imgdata.shape,timewindow,overlap)
start_times,end_times = get_start_and_end_times(k,timewindow,overlap)
M = pn.MultiplexNetwork(couplings='ordinal',fullyInterconnected=True)
for tw_no in range(k):
#start = tw_no*(timewindow-overlap)
#end = timewindow + tw_no*(timewindow-overlap)
A,voxellist = corrs_and_mask_calculations.make_adjacency_matrix(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],exclude_masked=True)
for ii in range(A.shape[0]):
# magic tricks to get true index from linearized form (disclaimer: not actual magic)
# node1 = str((ii//(ydim*zdim),(ii%(ydim*zdim))//zdim,(ii%(ydim*zdim))%zdim))
node1 = str(voxellist[ii])
for jj in range(ii+1,A.shape[1]):
#node2 = str((jj//(ydim*zdim),(jj%(ydim*zdim))//zdim,(jj%(ydim*zdim))%zdim))
node2 = str(voxellist[jj])
# only get unmasked voxels
if not np.isnan(A[ii,jj]):
M[node1,tw_no][node2,tw_no] = A[ii,jj]
else:
if nanlogfile != None:
with open(nanlogfile,'a+') as f:
f.write('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
else:
print('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
return M
def read_weighted_network(filename):
# There's no EOF in Python so we need to use clumsy flags (must iterate using 'for line in f'...)
# The closest thing to EOF is the empty string '', but if there are empty lines in the file there's no way to know if it's the end or
# a stripped newline
# Edges should be the last entry in the file
# Assuming layer names are integers (ordinally coupled multiplex)
nodeflag = 0
layerflag = 0
edgeflag = 0
with open(filename,'r') as f:
# Read first line which should contain the type of the network (multilayer/multiplex)
nettype = f.readline()
if nettype[0] != '#':
raise Exception('The first line should start with # and contain network type')
nettype = nettype.strip(' #\n').lower()
if nettype == 'multiplex':
M = pn.MultiplexNetwork(couplings='ordinal',fullyInterconnected=True)
else:
# If network is not multiplex, we use the general multilayer class
M = pn.MultilayerNetwork(aspects=1,fullyInterconnected=False)
if nettype == 'multiplex':
for line in f:
line = line.strip()
if line[0] == '#':
pass
else:
if nodeflag == 1:
for node in line.split(';'):
M.add_node(node)
nodeflag = 0
if layerflag == 1:
for layer in line.split(';'):
M.add_layer(int(layer),1)
layerflag = 0
if edgeflag == 1:
edge = line.split(';')
if edge[2] == edge[3]:
M[edge[0],int(edge[2])][edge[1],int(edge[3])] = float(edge[4])
elif edge[0] == edge[1]:
pass
else:
raise Exception('Illegal inter-layer edges')
if line == 'nodes:':
nodeflag = 1
if line == 'layers:':
layerflag = 1
if line == 'edges:':
edgeflag = 1
else:
# If the network is a general multilayer one, we just need to set all the edges
for line in f:
line = line.strip()
if edgeflag == 1:
edge = line.split(';')
M[edge[0],int(edge[2])][edge[1],int(edge[3])] = float(edge[4])
if line == 'edges:':
edgeflag = 1
return M
def threshold(net, threshold, method=">=", ignoreCouplingEdges=False):
def accept_edge(weight, threshold, rule):
if rule == ">=":
return weight >= threshold
elif rule == "<=":
return weight <= threshold
elif rule == ">":
return weight > threshold
elif rule == "<":
return weight < threshold
else:
raise Exception("Invalid method for thresholding: " + str(rule))
mplex = (type(net) == netmodule.MultiplexNetwork)
if mplex:
for coupling in net.couplings:
if coupling[0] != "none":
mplex = False
if mplex:
newNet = netmodule.MultiplexNetwork(
couplings=net.couplings,
directed=net.directed,
noEdge=net.noEdge,
fullyInterconnected=net.fullyInterconnected)
else:
newNet = netmodule.MultilayerNetwork(
aspects=net.aspects,
noEdge=net.noEdge,
directed=net.directed,
fullyInterconnected=net.fullyInterconnected)
#copy nodes,layers,node-layers
for node in net:
newNet.add_node(node)
for aspect in range(net.aspects):
for layer in net.slices[aspect + 1]:
newNet.add_layer(layer, aspect=aspect + 1)
if not net.fullyInterconnected:
for nodelayer in net.iter_node_layers():
layer = lnodelayer[1:]
if net.aspects == 1:
layer = layer[0]
newNet.add_node(nodelayer[0], layer=layer)
if mplex:
for layer in net.iter_layers():
for edge in net.A[layer].edges:
if accept_edge(edge[-1], threshold, rule=method):
newNet.A[layer][edge[0]][edge[1]] = edge[-1]
else:
for edge in net.edges:
if accept_edge(edge[-1], threshold, rule=method):
newNet[edge[:-1]] = edge[-1]
return newNet
def test_write_weighted_network(self):
M_multiplex = pn.MultiplexNetwork(couplings='ordinal',
fullyInterconnected=True)
M_multiplex['(1, 2, 3)', 0]['(2, 3, 4)', 0] = 0.5
M_multiplex['(2, 3, 4)', 1]['(3, 4, 5)', 1] = 0.999
M_multiplex['(3, 4, 5)', 2]['(1, 2, 3)', 2] = 1
possible_nodelines = set([
'(1, 2, 3);(2, 3, 4);(3, 4, 5)\n',
'(1, 2, 3);(3, 4, 5);(2, 3, 4)\n',
'(2, 3, 4);(1, 2, 3);(3, 4, 5)\n',
'(2, 3, 4);(3, 4, 5);(1, 2, 3)\n',
'(3, 4, 5);(1, 2, 3);(2, 3, 4)\n',
'(3, 4, 5);(2, 3, 4);(1, 2, 3)\n'
])
possible_layerlines = set(
['0;1;2\n', '0;2;1\n', '1;0;2\n', '1;2;0\n', '2;0;1\n', '2;1;0\n'])
edgeset = set([
'(1, 2, 3);(2, 3, 4);0;0;0.5\n', '(1, 2, 3);(1, 2, 3);0;1;1.0\n',
'(1, 2, 3);(1, 2, 3);1;2;1.0\n', '(1, 2, 3);(3, 4, 5);2;2;1\n',
'(2, 3, 4);(2, 3, 4);0;1;1.0\n', '(2, 3, 4);(3, 4, 5);1;1;0.999\n',
'(2, 3, 4);(2, 3, 4);1;2;1.0\n', '(3, 4, 5);(3, 4, 5);0;1;1.0\n',
'(3, 4, 5);(3, 4, 5);1;2;1.0\n'
])
network_io.write_weighted_network(
M_multiplex, 'test_for_network_writing_WILL_BE_REMOVED.txt',
'Created by test_write_weighted_network')
try:
with open('test_for_network_writing_WILL_BE_REMOVED.txt',
'r') as f:
self.assertEqual(f.readline(), '# Multiplex\n')
self.assertEqual(f.readline(),
'# Created by test_write_weighted_network\n')
self.assertEqual(f.readline(), 'nodes:\n')
self.assertTrue(f.readline() in possible_nodelines)
self.assertEqual(f.readline(), 'layers:\n')
self.assertTrue(f.readline() in possible_layerlines)
self.assertEqual(f.readline(), 'edges:\n')
for line in f:
self.assertTrue(line in edgeset)
edgeset.remove(line)
self.assertEqual(len(edgeset), 0)
self.assertEqual(f.readline(), '')
finally:
os.remove('test_for_network_writing_WILL_BE_REMOVED.txt')
M_multilayer = pn.MultilayerNetwork(aspects=1,
fullyInterconnected=False)
M_multilayer['[(1, 2, 3),(2, 3, 4)]', 0]['[(3, 4, 5)]', 0] = 0.123
M_multilayer['[(1, 2, 3)]', 1]['[(2, 3, 4),(3, 4, 5)]', 1] = 0.456
M_multilayer['[(1, 2, 3),(2, 3, 4)]', 0]['[(1, 2, 3)]', 1] = 0.5
M_multilayer['[(1, 2, 3),(2, 3, 4)]', 0]['[(2, 3, 4),(3, 4, 5)]',
1] = 0.333
M_multilayer['[(3, 4, 5)]', 0]['[(1, 2, 3)]', 1] = 0
M_multilayer['[(3, 4, 5)]', 0]['[(2, 3, 4),(3, 4, 5)]', 1] = 0.5
possible_nodelines = set([
x[0] + ';' + x[1] + ';' + x[2] + ';' + x[3] + '\n'
for x in itertools.permutations([
'[(1, 2, 3)]', '[(1, 2, 3),(2, 3, 4)]',
'[(2, 3, 4),(3, 4, 5)]', '[(3, 4, 5)]'
])
])
possible_layerlines = set(['0;1\n', '1;0\n'])
edgeset = set([
'[(1, 2, 3),(2, 3, 4)];[(2, 3, 4),(3, 4, 5)];0;1;0.333\n',
'[(1, 2, 3),(2, 3, 4)];[(3, 4, 5)];0;0;0.123\n',
'[(1, 2, 3),(2, 3, 4)];[(1, 2, 3)];0;1;0.5\n',
'[(3, 4, 5)];[(2, 3, 4),(3, 4, 5)];0;1;0.5\n',
'[(1, 2, 3)];[(2, 3, 4),(3, 4, 5)];1;1;0.456\n'
])
network_io.write_weighted_network(
M_multilayer, 'test_for_network_writing_WILL_BE_REMOVED.txt',
'Created by test_write_weighted_network')
try:
with open('test_for_network_writing_WILL_BE_REMOVED.txt',
'r') as f:
self.assertEqual(f.readline(), '# Multilayer\n')
self.assertEqual(f.readline(),
'# Created by test_write_weighted_network\n')
self.assertEqual(f.readline(), 'nodes:\n')
self.assertTrue(f.readline() in possible_nodelines)
self.assertEqual(f.readline(), 'layers:\n')
self.assertTrue(f.readline() in possible_layerlines)
self.assertEqual(f.readline(), 'edges:\n')
for line in f:
self.assertTrue(line in edgeset)
edgeset.remove(line)
self.assertEqual(len(edgeset), 0)
self.assertEqual(f.readline(), '')
finally:
os.remove('test_for_network_writing_WILL_BE_REMOVED.txt')
def make_clustered_multilayer(imgdata,timewindow=100,overlap=0,n_clusters=100,method='sklearn',template=None,nanlogfile=None):
'''
Possible methods:
'sklearn' : hierarchical clustering from sklearn, different for each time window
'template' : preconstructed clustering, same for each time window (requires parameter template : 3d ndarray with the same shape
as imgdata.shape[0:3] where 0 denotes masked voxel and other values denote cluster identity
Using this, n_clusters is ignored
'''
k = get_number_of_layers(imgdata.shape,timewindow,overlap)
start_times,end_times = get_start_and_end_times(k,timewindow,overlap)
if method == 'sklearn':
M = pn.MultilayerNetwork(aspects=1,fullyInterconnected=False)
previous_voxels_in_clusters = dict()
for tw_no in range(k):
# Create new object and make voxels_in_clusters refer to it (doesn't change previous_voxels_in_clusters)
voxels_in_clusters = dict()
#start = tw_no*(timewindow-overlap)
#end = timewindow + tw_no*(timewindow-overlap)
model,voxellist = clustering.cluster_timewindow_scikit(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],n_clusters=n_clusters)
for ii,label in enumerate(model.labels_):
voxels_in_clusters.setdefault(label,[]).append(voxellist[ii])
R = calculate_cluster_correlation_matrix(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],voxels_in_clusters)
for ii in range(R.shape[0]):
node1 = str(voxels_in_clusters[ii])
for jj in range(ii+1,R.shape[1]):
node2 = str(voxels_in_clusters[jj])
if not np.isnan(R[ii,jj]):
M[node1,tw_no][node2,tw_no] = R[ii,jj]
else:
if nanlogfile != None:
with open(nanlogfile,'a+') as f:
f.write('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
else:
print('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
for cluster_number in voxels_in_clusters:
for previous_cluster_number in previous_voxels_in_clusters:
cluster_overlap = get_overlap(set(voxels_in_clusters[cluster_number]),set(previous_voxels_in_clusters[previous_cluster_number]))
M[str(previous_voxels_in_clusters[previous_cluster_number]),tw_no-1][str(voxels_in_clusters[cluster_number]),tw_no] = cluster_overlap
previous_voxels_in_clusters = voxels_in_clusters # reference to the same object
elif method == 'template':
M = pn.MultiplexNetwork(couplings='ordinal',fullyInterconnected=True)
voxels_in_clusters = get_voxels_in_clusters(template)
for tw_no in range(k):
R = calculate_cluster_correlation_matrix(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],voxels_in_clusters)
for ii in range(R.shape[0]):
node1 = str(voxels_in_clusters[ii])
for jj in range(ii+1,R.shape[1]):
node2 = str(voxels_in_clusters[jj])
if not np.isnan(R[ii,jj]):
M[node1,tw_no][node2,tw_no] = R[ii,jj]
else:
if nanlogfile != None:
with open(nanlogfile,'a+') as f:
f.write('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
else:
print('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
else:
raise NotImplementedError('This clustering not implemented')
return M
def yield_clustered_multilayer_network_in_layersets(imgdata,layerset_size,timewindow=100,overlap=0,n_clusters=100,
method='sklearn',template=None,nanlogfile=None,
event_time_stamps=None,ROI_centroids=[],ROI_names=[],
consistency_threshold=-1,consistency_target_function='spatialConsistency',
f_transform_consistency=False,calculate_consistency_while_clustering=False,
n_consistency_CPUs=5,consistency_save_path='spatial-consistency.pkl',
n_consistency_iters=100,consistency_percentage_ROIs_for_thresholding=0,
n_ReHo_neighbors=6,percentage_min_centroid_distance=0,ReHo_measure='ReHo'):
"""
Consistency-related inputs:
---------------------------
ROI_centroids: n_ROIs x 3 np.array, coordinates (in voxels) of the ROI centroids, around which ROIs are grown in some clustering
methods (default=[]). Set to 'random' to use random seeds and to 'ReHo' to use seeds selected by maximal Regional
Homogeneity.
ROI_names: list of strs, names of ROIs (default=[])
consistency_threshold: float or string. The lowest centroid-voxel correlation that leads to adding a voxel. All
thresholding approaches may lead to parcellation where some voxels don't belong to any ROI.
Options:
- -1: no thresholding (the default option)
- float: no voxel is added to a ROI if the value of consistency target function < threshold
- 'data-driven': no voxel is added to a ROI if after adding the voxel the mean correlation of the
voxels of this ROI is lower than the mean correlation between the voxels of this ROI and the voxels
of any other ROI. However, the same voxel can be considered again later.
- 'strict data-driven': similar as above but when a voxel has been found to be sub-threshold
for a ROI, it is entirely removed from the base of possible voxels for this ROI
- 'voxel-wise': no voxel is added to a ROI if its average correlation to the voxels of this
ROI is lower than its average correlation at least consistency_percentage_ROIs_for_thresholding * n_ROIs
other ROIs
- 'voxel-neighbor': no voxel is added to a ROI if its average correlation to the voxels of this
ROI is lower than the average correlation of a voxel to its closest (6-voxel) neighborhood. This
threshold value is calculated as an average across all voxels before starting to build the ROIs.
consistency_target_function: str, homogeneity measure that will be optimized. Options:
'spatialConsistency': the mean Pearson correlation coefficient of the voxels already in the ROI and
the candidate voxel (the default option)
'correlationWithCentroid': the Pearson correlation between the ROI centroid time series and
voxel time series
f_transform_consistency: boolean; if True, Fischer z transform is applied to the correlations before averaging (default: False)
calculate_consistency_while_clustering: boolean; if True, the consistency of each cluster is calculated and saved in a pickle
file during the clustering. Note that setting this to True doesn't change clustering
method. So, this can be applied together with 'template' or 'sklearn' methods. (default: False)
n_consistency_CPUs: int, number of CPUs used for parallel consistency calculations if calculate_consistency_while_clustering == True.
(default: 5)
consistency_save_path: str, a path to which the consistency pickle will be saved (default: 'spatial_consistency.pkl')
n_consistency_iters: int, number of random seed sets to generate if ROI_centroids == 'random' (default = 100)
consistency_percentage_ROIs_for_thresholding: float (from 0 to 1), used in thresholding (see above) (default = 0 that is interprested
as 1/n_ROIs)
percentage_min_centroid_distance: float (from 0 to 1), the minimal distance between ReHo-based seeds is set as
percentageMinCentroidDistance times maximal dimension of imgdata (default = 0).
n_ReHo_neighbors: int, number or neighbors used for calculating ReHo if ReHo-based seeds are to be used; options: 6 (faces),
18 (faces + edges), 26 (faces + edges + corners) (default = 6)
ReHoMeasure: str, the measure of the neighbourhood similarity used to pick the ReHo-based seeds, options 'ReHo', 'spatialConsistency' (default = 'ReHo')
"""
# If event_time_stamps is specified, then they are used to compute start_times, end_times and k (and timewindow and overlap are ignored).
# Otherwise, timewindow and overlap are used ot compute start_times, end_times and k.
if event_time_stamps == None:
k = get_number_of_layers(imgdata.shape,timewindow,overlap)
start_times,end_times = get_start_and_end_times(k,timewindow,overlap)
else:
assert isinstance(event_time_stamps,list)
k = len(event_time_stamps) + 1
start_times = [0] + event_time_stamps
end_times = event_time_stamps + [imgdata.shape[3]]
layersets = zip(*(range(k)[ii:] for ii in range(layerset_size)))
if method == 'sklearn':
voxels_in_clusters_by_timewindow = dict()
for layerset in layersets:
M = pn.MultilayerNetwork(aspects=1,fullyInterconnected=False)
previous_voxels_in_clusters = dict()
for tw_no in layerset:
if not tw_no in voxels_in_clusters_by_timewindow:
voxels_in_clusters = dict()
model,voxellist = clustering.cluster_timewindow_scikit(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],n_clusters=n_clusters)
for ii,label in enumerate(model.labels_):
voxels_in_clusters.setdefault(label,[]).append(voxellist[ii])
voxels_in_clusters_by_timewindow[tw_no] = voxels_in_clusters
else:
voxels_in_clusters = voxels_in_clusters_by_timewindow[tw_no]
R = calculate_cluster_correlation_matrix(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],voxels_in_clusters)
for ii in range(R.shape[0]):
node1 = str(voxels_in_clusters[ii])
for jj in range(ii+1,R.shape[1]):
node2 = str(voxels_in_clusters[jj])
if not np.isnan(R[ii,jj]):
M[node1,tw_no][node2,tw_no] = R[ii,jj]
else:
if nanlogfile != None:
with open(nanlogfile,'a+') as f:
f.write('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
else:
print('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
for cluster_number in voxels_in_clusters:
for previous_cluster_number in previous_voxels_in_clusters:
cluster_overlap = get_overlap(set(voxels_in_clusters[cluster_number]),set(previous_voxels_in_clusters[previous_cluster_number]))
M[str(previous_voxels_in_clusters[previous_cluster_number]),tw_no-1][str(voxels_in_clusters[cluster_number]),tw_no] = cluster_overlap
previous_voxels_in_clusters = voxels_in_clusters # reference to the same object
if calculate_consistency_while_clustering: # calculating spatial consistency of formed clusters
windowdata = imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]]
if '.' in consistency_save_path:
name,extension = consistency_save_path.split('.')
consistency_save_path_final = name + '_' + str(tw_no) + '.' + extension
else:
consistency_save_path_final = name + '_' + str(tw_no) + '.pkl'
calculate_spatial_consistency(windowdata,voxels_in_clusters,f_transform_consistency,n_consistency_CPUs,consistency_save_path_final)
del(voxels_in_clusters_by_timewindow[min(voxels_in_clusters_by_timewindow)])
yield M
del(M)
elif method == 'template':
voxels_in_clusters = get_voxels_in_clusters(template)
for layerset in layersets:
M = pn.MultiplexNetwork(couplings='ordinal',fullyInterconnected=True)
for tw_no in layerset:
R = calculate_cluster_correlation_matrix(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],voxels_in_clusters)
for ii in range(R.shape[0]):
node1 = str(voxels_in_clusters[ii])
for jj in range(ii+1,R.shape[1]):
node2 = str(voxels_in_clusters[jj])
if not np.isnan(R[ii,jj]):
M[node1,tw_no][node2,tw_no] = R[ii,jj]
else:
if nanlogfile != None:
with open(nanlogfile,'a+') as f:
f.write('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
else:
print('NaN correlation at nodes '+node1+', '+node2+' at timewindow '+str(tw_no)+'\n')
if calculate_consistency_while_clustering: #calculating spatial consistency of formed clusters
windowdata = imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]]
if '.' in consistency_save_path:
name,extension = consistency_save_path.split('.')
consistency_save_path_final = name + '_' + str(tw_no) + '.' + extension
else:
consistency_save_path_final = name + '_' + str(tw_no) + '.pkl'
calculate_spatial_consistency(windowdata,voxels_in_clusters,f_transform_consistency,n_consistency_CPUs,consistency_save_path_final)
yield M
del(M)
elif method == 'consistency_optimized':
voxels_in_clusters_by_timewindow = dict()
for layerset in layersets:
M = pn.MultilayerNetwork(aspects=1,fullyInterconnected=False)
previous_voxels_in_clusters = dict()
for tw_no in layerset:
cfg = {'ROICentroids':ROI_centroids,'names':ROI_names,'imgdata':imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],
'threshold':consistency_threshold,'targetFunction':consistency_target_function,
'fTransform':f_transform_consistency,'nROIs':n_clusters,'template':template,
'percentageROIsForThresholding':consistency_percentage_ROIs_for_thresholding,'nCPUs':n_consistency_CPUs,
'nReHoNeighbors':n_ReHo_neighbors,'percentageMinCentroidDistance':percentage_min_centroid_distance,
'ReHoMeasure':ReHo_measure}
if not tw_no in voxels_in_clusters_by_timewindow:
voxels_in_clusters = dict()
if ROI_centroids == 'random':
voxel_labels, voxel_coordinates = cbc.growOptimizedROIsInParallel(cfg,n_consistency_iters,n_consistency_CPUs)
else:
voxel_labels, voxel_coordinates, _ = cbc.growOptimizedROIs(cfg)
for ii, label in enumerate(voxel_labels):
voxels_in_clusters.setdefault(label,[]).append(voxel_coordinates[ii]) # voxels_in_clusters will contain label:[voxels with label] pairs; here, coordinates of each voxel are added to the correct list
if -1 in voxels_in_clusters:
del voxels_in_clusters[-1] # Voxels that are not located in any ROI (because of thresholding) have label -1. These should not be considered further in the pipeline.
voxels_in_clusters_by_timewindow[tw_no] = voxels_in_clusters
else:
voxels_in_clusters = voxels_in_clusters_by_timewindow[tw_no]
R = calculate_cluster_correlation_matrix(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],voxels_in_clusters)
for ii in range(R.shape[0]):
node1 = str(voxels_in_clusters[ii])
for jj in range(ii+1,R.shape[1]):
node2 = str(voxels_in_clusters[jj])
if not np.isnan(R[ii,jj]):
M[node1,node2,tw_no] = R[ii,jj]
else:
if nanlogfile != None:
with open(nanlogfile,'a+') as f:
f.write('NaN correlation at nodes ' + node1 + ',' + node2 + 'at timewindow ' + str(tw_no) + '/n')
else:
print('NaN correlation at nodes ' + node1 + ',' + node2 + 'at timewindow ' + str(tw_no) + '/n')
for cluster_number in voxels_in_clusters:
for previous_cluster_number in previous_voxels_in_clusters:
# If previous_voxels_in_clusters is empty, this loop isn't performed at all
cluster_overlap = get_overlap(set(voxels_in_clusters[cluster_number]),set(previous_voxels_in_clusters[previous_cluster_number]))
M[str(previous_voxels_in_clusters[previous_cluster_number]),tw_no-1][str(voxels_in_clusters[cluster_number]),tw_no] = cluster_overlap
previous_voxels_in_clusters = voxels_in_clusters
if calculate_consistency_while_clustering:
windowdata = imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]]
if '.' in consistency_save_path:
name,extension = consistency_save_path.split('.')
consistency_save_path_final = name + '_' + str(tw_no) + '.' + extension
else:
consistency_save_path_final = name + '_' + str(tw_no) + '.pkl'
calculate_spatial_consistency(windowdata,voxels_in_clusters,f_transform_consistency,n_consistency_CPUs,consistency_save_path_final)
del(voxels_in_clusters_by_timewindow[min(voxels_in_clusters_by_timewindow)])
yield M
del(M)
elif method=='random_balls':
voxels_in_clusters_by_timewindow = dict()
for layerset in layersets:
M = pn.MultilayerNetwork(aspects=1,fullyInterconnected=False)
previous_voxels_in_clusters = dict()
for tw_no in layerset:
if not tw_no in voxels_in_clusters_by_timewindow:
voxels_in_clusters = dict()
voxel_labels, voxel_coordinates, _ = cbc.growSphericalROIs('random',imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],nROIs=n_clusters,template=template)
for ii, label in enumerate(voxel_labels):
voxels_in_clusters.setdefault(label,[]).append(voxel_coordinates[ii]) # voxels_in_clusters will contain label:[voxels with label] pairs; here, coordinates of each voxel are added to the correct list
if -1 in voxels_in_clusters:
del voxels_in_clusters[-1] # Voxels that are not located in any ROI (because of thresholding) have label -1. These should not be considered further in the pipeline.
voxels_in_clusters_by_timewindow[tw_no] = voxels_in_clusters
else:
voxels_in_clusters = voxels_in_clusters_by_timewindow[tw_no]
R = calculate_cluster_correlation_matrix(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],voxels_in_clusters)
for ii in range(R.shape[0]):
node1 = str(voxels_in_clusters[ii])
for jj in range(ii+1,R.shape[1]):
node2 = str(voxels_in_clusters[jj])
if not np.isnan(R[ii,jj]):
M[node1,node2,tw_no] = R[ii,jj]
else:
if nanlogfile != None:
with open(nanlogfile,'a+') as f:
f.write('NaN correlation at nodes ' + node1 + ',' + node2 + 'at timewindow ' + str(tw_no) + '/n')
else:
print('NaN correlation at nodes ' + node1 + ',' + node2 + 'at timewindow ' + str(tw_no) + '/n')
for cluster_number in voxels_in_clusters:
for previous_cluster_number in previous_voxels_in_clusters:
# If previous_voxels_in_clusters is empty, this loop isn't performed at all
cluster_overlap = get_overlap(set(voxels_in_clusters[cluster_number]),set(previous_voxels_in_clusters[previous_cluster_number]))
M[str(previous_voxels_in_clusters[previous_cluster_number]),tw_no-1][str(voxels_in_clusters[cluster_number]),tw_no] = cluster_overlap
previous_voxels_in_clusters = voxels_in_clusters
if calculate_consistency_while_clustering:
windowdata = imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]]
if '.' in consistency_save_path:
name,extension = consistency_save_path.split('.')
consistency_save_path_final = name + '_' + str(tw_no) + '.' + extension
else:
consistency_save_path_final = name + '_' + str(tw_no) + '.pkl'
calculate_spatial_consistency(windowdata,voxels_in_clusters,f_transform_consistency,n_consistency_CPUs,consistency_save_path_final)
del(voxels_in_clusters_by_timewindow[min(voxels_in_clusters_by_timewindow)])
elif method=='craddock':
voxels_in_clusters_by_timewindow = dict()
for layerset in layersets:
M = pn.MultilayerNetwork(aspects=1,fullyInterconnected=False)
previous_voxels_in_clusters = dict()
for tw_no in layerset:
cfg = {'imgdata':imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],'threshold':consistency_threshold,'nROIs':n_clusters}
if not tw_no in voxels_in_clusters_by_timewindow:
voxels_in_clusters = dict()
voxel_labels,voxel_coordinates = cbc.spectralNCutClustering(cfg)
if min(np.abs(voxel_labels))>0: # if cluster indexing starts from a non-zero value, let's fix it to start from zero
voxel_labels[np.where(voxel_labels)>0] -= np.min(voxel_labels[np.where(voxel_labels)>0])
for ii, label in enumerate(voxel_labels):
voxels_in_clusters.setdefault(label,[]).append(voxel_coordinates[ii]) # voxels_in_clusters will contain label:[voxels with label] pairs; here, coordinates of each voxel are added to the correct list
if -1 in voxels_in_clusters:
del voxels_in_clusters[-1] # Voxels that are not located in any ROI (because of thresholding) have label -1. These should not be considered further in the pipeline.
voxels_in_clusters_by_timewindow[tw_no] = voxels_in_clusters
else:
voxels_in_clusters = voxels_in_clusters_by_timewindow[tw_no]
R = calculate_cluster_correlation_matrix(imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]],voxels_in_clusters)
for ii in range(R.shape[0]):
node1 = str(voxels_in_clusters[ii])
for jj in range(ii+1,R.shape[1]):
node2 = str(voxels_in_clusters[jj])
if not np.isnan(R[ii,jj]):
M[node1,node2,tw_no] = R[ii,jj]
else:
if nanlogfile != None:
with open(nanlogfile,'a+') as f:
f.write('NaN correlation at nodes ' + node1 + ',' + node2 + 'at timewindow ' + str(tw_no) + '/n')
else:
print('NaN correlation at nodes ' + node1 + ',' + node2 + 'at timewindow ' + str(tw_no) + '/n')
for cluster_number in voxels_in_clusters:
for previous_cluster_number in previous_voxels_in_clusters:
# If previous_voxels_in_clusters is empty, this loop isn't performed at all
cluster_overlap = get_overlap(set(voxels_in_clusters[cluster_number]),set(previous_voxels_in_clusters[previous_cluster_number]))
M[str(previous_voxels_in_clusters[previous_cluster_number]),tw_no-1][str(voxels_in_clusters[cluster_number]),tw_no] = cluster_overlap
previous_voxels_in_clusters = voxels_in_clusters
if calculate_consistency_while_clustering:
windowdata = imgdata[:,:,:,start_times[tw_no]:end_times[tw_no]]
if '.' in consistency_save_path:
name,extension = consistency_save_path.split('.')
consistency_save_path_final = name + '_' + str(tw_no) + '.' + extension
else:
consistency_save_path_final = name + '_' + str(tw_no) + '.pkl'
calculate_spatial_consistency(windowdata,voxels_in_clusters,f_transform_consistency,n_consistency_CPUs,consistency_save_path_final)
yield M
del(M)
else:
raise NotImplementedError('Not implemented')
def relabel(net,nodeNames=None,layerNames=None):
"""Returns a copy of the network with nodes and layers relabeled.
Parameters
----------
net : MultilayerNetwork, or MultiplexNetwork
The original network.
nodeNames : None, or dict
The map from node names to node indices.
layerNames : None, dict, or sequence of dicts
The map(s) from (elementary) layer names to (elementary) layer indices.
Note that you can add empty dicts for aspects you do not want to relabel.
Return
------
newnet : type(net)
The normalized network.
"""
def dget(d,e):
if e in d:
return d[e]
else:
return e
def layer_to_indexlayer(layer,layerNames):
return tuple([dget(layerNames[i],elayer) for i,elayer in enumerate(layer)])
if nodeNames==None:
nodeNames={}
if layerNames==None:
layerNames=[]
if net.aspects==1:
if isinstance(layerNames,dict):
layerNames=[layerNames]
for aspect in range(net.aspects):
if len(layerNames)<aspect+1:
layerNames.append({})
if type(net)==netmodule.MultilayerNetwork:
newNet=netmodule.MultilayerNetwork(aspects=net.aspects,
noEdge=net.noEdge,
directed=net.directed,
fullyInterconnected=net.fullyInterconnected)
elif type(net)==netmodule.MultiplexNetwork:
newNet=netmodule.MultiplexNetwork(couplings=net.couplings,
directed=net.directed,
noEdge=net.noEdge,
fullyInterconnected=net.fullyInterconnected)
else:
raise Exception("Invalid type of net",type(net))
for node in net:
newNet.add_node(dget(nodeNames,node))
for aspect in range(net.aspects):
for layer in net.slices[aspect+1]:
newNet.add_layer(dget(layerNames[aspect],layer),aspect=aspect+1)
if not net.fullyInterconnected:
for nodelayer in net.iter_node_layers():
layer=layer_to_indexlayer(nodelayer[1:],layerNames)
if net.aspects==1:
layer=layer[0]
newNet.add_node(dget(nodeNames,nodelayer[0]),layer=layer)
if type(net)==netmodule.MultilayerNetwork:
for edge in net.edges:
newedge=[dget(nodeNames,edge[0]),dget(nodeNames,edge[1])]
for aspect in range(net.aspects):
newedge.append(dget(layerNames[aspect],edge[2+aspect*2]))
newedge.append(dget(layerNames[aspect],edge[2+aspect*2+1]))
newNet[tuple(newedge)]=edge[-1]
elif type(net)==netmodule.MultiplexNetwork:
for layer in net.iter_layers():
if net.aspects==1:
layertuple=(layer,)
else:
layertuple=layer
for node in net.A[layer]:
for neigh in net.A[layer][node]:
newNet[(dget(nodeNames,node),dget(nodeNames,neigh))+layer_to_indexlayer(layertuple,layerNames)]=net[(node,neigh)+layertuple]
return newNet
def subnet(net,nodes,*layers,**kwargs):
"""Returns an induced subgraph with given set of nodes and layers.
Parameters
----------
net : MultilayerNetwork, MultiplexNetwork
The original network.
nodes : sequence
The nodes that span the induces subgraph.
*layers : *sequence
(Elementary) layers included in the subgraph. One parameter for each aspect.
newNet : None, MultilayerNetwork, MultiplexNetwork
An empty new network or None. If None, the new network is created as a
an empty copy of the net. The edges and nodes are copied to this network.
nolinks : bool
If set True, this function does not copy any links. That is, the returned
network is _not_ an induced subnetwork but an empty network.
Return
------
subnet : type(net), or type(newNet)
The induced subgraph that contains only nodes given in
`nodes` and the edges between those nodes that are
present in `net`. Node properties etc are left untouched.
"""
if "newNet" in kwargs:
newNet=kwargs["newNet"]
else:
newNet=None
if "nolinks" in kwargs:
nolinks=kwargs["nolinks"]
else:
nolinks=False
assert len(layers)==net.aspects, "Please give layers for each aspect."
nodelayers=[]
for a,elayers in enumerate(itertools.chain([nodes],layers)):
if elayers==None:
nodelayers.append(set(net.get_layers(a)))
else:
nodelayers.append(set(elayers))
if newNet==None:
if isinstance(net,netmodule.MultiplexNetwork):
newNet=netmodule.MultiplexNetwork(couplings=net.couplings,
directed=net.directed,
noEdge=net.noEdge,
fullyInterconnected=net.fullyInterconnected)
elif isinstance(net,netmodule.MultilayerNetwork):
newNet=netmodule.MultilayerNetwork(aspects=net.aspects,
noEdge=net.noEdge,
directed=net.directed,
fullyInterconnected=net.fullyInterconnected)
else:
raise Exception("Invalid net type: "+str(type(net)))
addedElementaryLayers=[]
for a,elayers in enumerate(nodelayers):#enumerate(itertools.chain((nodes,),layers)):
if net.fullyInterconnected or a!=0:
addedElementaryLayers.append(0)
oldElementaryLayers=net.get_layers(a)
for elayer in elayers:
if elayer in oldElementaryLayers:
newNet.add_layer(elayer,a)
addedElementaryLayers[-1]+=1
if not net.fullyInterconnected:
totalNodeLayers=0
for nl in net.iter_node_layers():
if reduce(lambda x,y:x and y, (e in nodelayers[a] for a,e in enumerate(nl))):
if net.aspects==1:
newNet.add_node(nl[0],layer=nl[1])
else:
newNet.add_node(nl[0],layer=nl[1:])
totalNodeLayers+=1
else:
totalNodeLayers=reduce(lambda x,y:x*y,addedElementaryLayers)
#copy the links
if not nolinks:
if isinstance(net,netmodule.MultiplexNetwork):
#Go through all the combinations of new layers
for layer in itertools.product(*nodelayers[1:]):
layer=layer[0] if net.aspects==1 else layer
subnet(net.A[layer],nodelayers[0],newNet=newNet.A[layer],nolinks=nolinks)
elif isinstance(net,netmodule.MultilayerNetwork):
for nl1 in itertools.product(*nodelayers):
nl1 = nl1[0] if net.aspects==0 else nl1
if net[nl1].deg()>=totalNodeLayers:
for nl2 in itertools.product(*nodelayers):
nl2 = nl2[0] if net.aspects==0 else nl2
if net[nl1][nl2]!=net.noEdge:
newNet[nl1][nl2]=net[nl1][nl2]
else:
if net.aspects==0:
for nl2 in net[nl1]:
if nl2 in nodelayers[0]:
newNet[nl1][nl2]=net[nl1][nl2]
else:
for nl2 in net[nl1]:
if reduce(lambda x,y:x and y, (e in nodelayers[a] for a,e in enumerate(nl2))):
newNet[nl1][nl2]=net[nl1][nl2]
else:
raise Exception("Invalid net type: "+str(type(net)))
return newNet
def graphlets(n, layers, n_l=None, couplings=None, allowed_aspects="all"):
'''
Generate graphlets up to n nodes
Parameters
----------
n : int
maximum number of nodes
layers : list of layers
n_l : int
Number of layers in the generated graphlets, can be smaller than or equal
to the number of elements in layers
couplings : list, str, tuple, None, MultilayerNetwork
Parameter determining how the layers are coupled, i.e. what
inter-layer edges are present.
allowed_aspects : list, string
the aspects that can be permutated when computing isomorphisms
Returns
-------
nets : dict (key: n_nodes, value: list of MultiplexNetwork objects)
graphlets
invariants : dict (key: str(complete invariant), value: tuple(index in 'nets': n_nodes, index in the list of multiplex networks))
complete invariants of the graphlets, the values can be used to match the graphlets in 'nets'
'''
if n_l == None:
n_l = len(layers)
nets = {}
invariants = {}
nets2 = []
for net_layers in itertools.combinations(layers, n_l):
layer_combs = layer_combinations(net_layers)
for layer_comb in layer_combs:
net = pymnet.MultiplexNetwork(couplings=couplings,
fullyInterconnected=True)
for layer in layer_comb:
net[0, 1, layer] = 1
for layer in net_layers:
net.add_layer(layer)
ci = pymnet.get_complete_invariant(net,
allowed_aspects=allowed_aspects)
ci_s = str(ci)
if not ci_s in invariants:
invariants[ci_s] = (2, len(nets2))
nets2.append(net)
nets[2] = nets2
for i in range(2, n):
nets_i = nets[i]
nets_i_1 = []
for net in nets_i:
net_nodes = net.slices[0]
net_layers = list(net.slices[1])
net_layers.sort()
layer_combs = layer_combinations(net_layers)
for n_n in range(1, i + 1):
for node_comb in itertools.combinations(range(i), n_n):
node_layers = [layer_combs] * n_n
for node_layer_comb in itertools.product(*node_layers):
new_net = pymnet.subnet(net, net_nodes, net_layers)
for node_i in range(n_n):
node = node_comb[node_i]
for layer in node_layer_comb[node_i]:
new_net[node, i, layer] = 1
for layer in net_layers:
new_net.add_layer(layer)
# check if isomorphic with a previous graph & add only if not isomorphic
ci = pymnet.get_complete_invariant(
new_net, allowed_aspects=allowed_aspects)
ci_s = str(ci)
if not ci_s in invariants:
invariants[ci_s] = (i + 1, len(nets_i_1))
nets_i_1.append(new_net)
nets[i + 1] = nets_i_1
return nets, invariants
class test_network_io(unittest.TestCase):
timeseries0 = np.array(range(-10, 11), dtype=np.float64)
timeseries1 = np.array([
-0.46362334, -0.56982772, 0.06455791, -0.44209878, -0.31068497,
0.05360425, -0.41299186, -0.29082169, -0.07190158, -0.12474256,
-0.24997589, 0.01267206, 0.03601663, 0.29330202, -0.12646342,
0.13130587, 0.57496159, 0.77851974, 0.12816724, 0.63563011, 0.35058168
],
dtype=np.float64)
timeseries2 = np.array(range(100, 121), dtype=np.float64)
timeseries3 = np.zeros((21, ))
timeseries4 = np.array(
[1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 10, 9, 8, 7, 6, 5, 4, 3, 2, 1],
dtype=np.float64)
timeseries5 = np.array(range(0, -21, -1))
timeseries6 = np.copy(timeseries0)
timeseries7 = np.copy(timeseries1)
timeseries8 = np.copy(timeseries2)
timeseries9 = np.copy(timeseries3)
timeseries10 = np.copy(timeseries4)
timeseries11 = np.copy(timeseries5)
imgdata = np.block([[[[timeseries0], [timeseries1]],
[[timeseries2], [timeseries3]],
[[timeseries4], [timeseries5]]],
[[[timeseries6], [timeseries7]],
[[timeseries8], [timeseries9]],
[[timeseries10], [timeseries11]]]])
pickle_test_mplex = pn.MultiplexNetwork(couplings='ordinal',
fullyInterconnected=True)
pickle_test_mplex['(1, 2, 3)', 0]['(2, 3, 4)', 0] = 0.5
pickle_test_mplex['(2, 3, 4)', 1]['(3, 4, 5)', 1] = 0.999
pickle_test_mplex['(1, 2, 3)', 1]['(2, 3, 4)', 1] = 0.001
pickle_test_mplex['(1, 2, 3)', 1]['(3, 4, 5)', 1] = 0.234
pickle_test_mplex['(3, 4, 5)', 2]['(1, 2, 3)', 2] = 1
pickle_test_mlayer = pn.MultilayerNetwork(aspects=1,
fullyInterconnected=False)
pickle_test_mlayer['[(1, 2, 3),(2, 3, 4)]', 9]['[(3, 4, 5)]', 9] = 0.123
pickle_test_mlayer['[(1, 2, 3)]', 10]['[(2, 3, 4),(3, 4, 5)]', 10] = 0.456
pickle_test_mlayer['[(1, 2, 3),(2, 3, 4)]', 9]['[(1, 2, 3)]', 10] = 0.5
pickle_test_mlayer['[(1, 2, 3),(2, 3, 4)]', 9]['[(2, 3, 4),(3, 4, 5)]',
10] = 0.333
pickle_test_mlayer['[(3, 4, 5)]', 9]['[(1, 2, 3)]', 10] = 0
pickle_test_mlayer['[(3, 4, 5)]', 9]['[(2, 3, 4),(3, 4, 5)]', 10] = 0.5
pickle_test_mlayer['[(4,5,6)]', 10]['[(2, 3, 4),(3, 4, 5)]', 10] = 0.01
pickle_test_mlayer['[(2, 3, 4)]', 11]['[(2, 3, 4),(3, 4, 5)]', 10] = 0.999
def round_edge_weights(self, M):
# rounds edge weights to 10 decimals
if isinstance(M, pn.net.MultiplexNetwork):
for edge in M.edges:
if edge[2] == edge[3]:
M[edge[0], edge[2]][edge[1], edge[3]] = round(edge[4], 10)
return M
else:
for edge in M.edges:
M[edge[0], edge[2]][edge[1], edge[3]] = round(edge[4], 10)
return M
def test_write_weighted_network(self):
M_multiplex = pn.MultiplexNetwork(couplings='ordinal',
fullyInterconnected=True)
M_multiplex['(1, 2, 3)', 0]['(2, 3, 4)', 0] = 0.5
M_multiplex['(2, 3, 4)', 1]['(3, 4, 5)', 1] = 0.999
M_multiplex['(3, 4, 5)', 2]['(1, 2, 3)', 2] = 1
possible_nodelines = set([
'(1, 2, 3);(2, 3, 4);(3, 4, 5)\n',
'(1, 2, 3);(3, 4, 5);(2, 3, 4)\n',
'(2, 3, 4);(1, 2, 3);(3, 4, 5)\n',
'(2, 3, 4);(3, 4, 5);(1, 2, 3)\n',
'(3, 4, 5);(1, 2, 3);(2, 3, 4)\n',
'(3, 4, 5);(2, 3, 4);(1, 2, 3)\n'
])
possible_layerlines = set(
['0;1;2\n', '0;2;1\n', '1;0;2\n', '1;2;0\n', '2;0;1\n', '2;1;0\n'])
edgeset = set([
'(1, 2, 3);(2, 3, 4);0;0;0.5\n', '(1, 2, 3);(1, 2, 3);0;1;1.0\n',
'(1, 2, 3);(1, 2, 3);1;2;1.0\n', '(1, 2, 3);(3, 4, 5);2;2;1\n',
'(2, 3, 4);(2, 3, 4);0;1;1.0\n', '(2, 3, 4);(3, 4, 5);1;1;0.999\n',
'(2, 3, 4);(2, 3, 4);1;2;1.0\n', '(3, 4, 5);(3, 4, 5);0;1;1.0\n',
'(3, 4, 5);(3, 4, 5);1;2;1.0\n'
])
network_io.write_weighted_network(
M_multiplex, 'test_for_network_writing_WILL_BE_REMOVED.txt',
'Created by test_write_weighted_network')
try:
with open('test_for_network_writing_WILL_BE_REMOVED.txt',
'r') as f:
self.assertEqual(f.readline(), '# Multiplex\n')
self.assertEqual(f.readline(),
'# Created by test_write_weighted_network\n')
self.assertEqual(f.readline(), 'nodes:\n')
self.assertTrue(f.readline() in possible_nodelines)
self.assertEqual(f.readline(), 'layers:\n')
self.assertTrue(f.readline() in possible_layerlines)
self.assertEqual(f.readline(), 'edges:\n')
for line in f:
self.assertTrue(line in edgeset)
edgeset.remove(line)
self.assertEqual(len(edgeset), 0)
self.assertEqual(f.readline(), '')
finally:
os.remove('test_for_network_writing_WILL_BE_REMOVED.txt')
M_multilayer = pn.MultilayerNetwork(aspects=1,
fullyInterconnected=False)
M_multilayer['[(1, 2, 3),(2, 3, 4)]', 0]['[(3, 4, 5)]', 0] = 0.123
M_multilayer['[(1, 2, 3)]', 1]['[(2, 3, 4),(3, 4, 5)]', 1] = 0.456
M_multilayer['[(1, 2, 3),(2, 3, 4)]', 0]['[(1, 2, 3)]', 1] = 0.5
M_multilayer['[(1, 2, 3),(2, 3, 4)]', 0]['[(2, 3, 4),(3, 4, 5)]',
1] = 0.333
M_multilayer['[(3, 4, 5)]', 0]['[(1, 2, 3)]', 1] = 0
M_multilayer['[(3, 4, 5)]', 0]['[(2, 3, 4),(3, 4, 5)]', 1] = 0.5
possible_nodelines = set([
x[0] + ';' + x[1] + ';' + x[2] + ';' + x[3] + '\n'
for x in itertools.permutations([
'[(1, 2, 3)]', '[(1, 2, 3),(2, 3, 4)]',
'[(2, 3, 4),(3, 4, 5)]', '[(3, 4, 5)]'
])
])
possible_layerlines = set(['0;1\n', '1;0\n'])
edgeset = set([
'[(1, 2, 3),(2, 3, 4)];[(2, 3, 4),(3, 4, 5)];0;1;0.333\n',
'[(1, 2, 3),(2, 3, 4)];[(3, 4, 5)];0;0;0.123\n',
'[(1, 2, 3),(2, 3, 4)];[(1, 2, 3)];0;1;0.5\n',
'[(3, 4, 5)];[(2, 3, 4),(3, 4, 5)];0;1;0.5\n',
'[(1, 2, 3)];[(2, 3, 4),(3, 4, 5)];1;1;0.456\n'
])
network_io.write_weighted_network(
M_multilayer, 'test_for_network_writing_WILL_BE_REMOVED.txt',
'Created by test_write_weighted_network')
try:
with open('test_for_network_writing_WILL_BE_REMOVED.txt',
'r') as f:
self.assertEqual(f.readline(), '# Multilayer\n')
self.assertEqual(f.readline(),
'# Created by test_write_weighted_network\n')
self.assertEqual(f.readline(), 'nodes:\n')
self.assertTrue(f.readline() in possible_nodelines)
self.assertEqual(f.readline(), 'layers:\n')
self.assertTrue(f.readline() in possible_layerlines)
self.assertEqual(f.readline(), 'edges:\n')
for line in f:
self.assertTrue(line in edgeset)
edgeset.remove(line)
self.assertEqual(len(edgeset), 0)
self.assertEqual(f.readline(), '')
finally:
os.remove('test_for_network_writing_WILL_BE_REMOVED.txt')
def test_read_weighted_network(self):
M_multiplex = network_construction.make_multiplex(self.imgdata,
timewindow=7,
overlap=2)
network_io.write_weighted_network(
M_multiplex, 'test_for_network_reading_WILL_BE_REMOVED.txt',
'Created by test_read_weighted_network')
M_multiplex_read = network_io.read_weighted_network(
'test_for_network_reading_WILL_BE_REMOVED.txt')
try:
self.assertEqual(self.round_edge_weights(M_multiplex),
self.round_edge_weights(M_multiplex_read))
finally:
os.remove('test_for_network_reading_WILL_BE_REMOVED.txt')
M_multilayer = network_construction.make_clustered_multilayer(
self.imgdata, timewindow=7, overlap=2, n_clusters=3)
network_io.write_weighted_network(
M_multilayer, 'test_for_network_reading_WILL_BE_REMOVED.txt',
'Created by test_read_weighted_network')
M_multilayer_read = network_io.read_weighted_network(
'test_for_network_reading_WILL_BE_REMOVED.txt')
try:
self.assertEqual(self.round_edge_weights(M_multilayer),
self.round_edge_weights(M_multilayer_read))
finally:
os.remove('test_for_network_reading_WILL_BE_REMOVED.txt')
def test_pickle_file_io_for_networks(self):
try:
network_io.write_pickle_file(
self.pickle_test_mplex,
'test_for_pickle_io_mplex_network_WILL_BE_REMOVED.pkl')
network_io.write_pickle_file(
self.pickle_test_mlayer,
'test_for_pickle_io_mlayer_network_WILL_BE_REMOVED.pkl')
pickle_test_mplex_read = network_io.read_pickle_file(
'test_for_pickle_io_mplex_network_WILL_BE_REMOVED.pkl')
pickle_test_mlayer_read = network_io.read_pickle_file(
'test_for_pickle_io_mlayer_network_WILL_BE_REMOVED.pkl')
self.assertEqual(self.pickle_test_mplex, pickle_test_mplex_read)
self.assertEqual(self.pickle_test_mlayer, pickle_test_mlayer_read)
finally:
if os.path.exists(
'test_for_pickle_io_mplex_network_WILL_BE_REMOVED.pkl'):
os.remove(
'test_for_pickle_io_mplex_network_WILL_BE_REMOVED.pkl')
if os.path.exists(
'test_for_pickle_io_mlayer_network_WILL_BE_REMOVED.pkl'):
os.remove(
'test_for_pickle_io_mlayer_network_WILL_BE_REMOVED.pkl')
def test_write_layersetwise_network(self):
try:
os.mkdir('dir_for_test_write_layersetwise_network_WILL_BE_REMOVED')
network_io.write_layersetwise_network(
self.pickle_test_mplex,
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED')
network_io.write_layersetwise_network(
self.pickle_test_mlayer,
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED')
self.assertEqual(
sorted(
os.listdir(
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED'
)), ['0_1_2', '9_10_11'])
mplex = network_io.read_pickle_file(
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED/0_1_2'
)
mlayer = network_io.read_pickle_file(
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED/9_10_11'
)
self.assertEqual(self.pickle_test_mplex, mplex)
self.assertEqual(self.pickle_test_mlayer, mlayer)
finally:
if os.path.exists(
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED/0_1_2'
):
os.remove(
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED/0_1_2'
)
if os.path.exists(
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED/9_10_11'
):
os.remove(
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED/9_10_11'
)
if os.path.exists(
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED'):
os.rmdir(
'dir_for_test_write_layersetwise_network_WILL_BE_REMOVED')
def get_fruchterman_reingold_multilayer_layout(net,
nodeDist="auto",
boxSize=1.0,
alignedNodes=True,
nodelayerCoords=None,
nodeCoords=None,
fixedNodes=None,
fixedNodeLayers=None,
iterations=100):
"""A multilayer version of the Fructherman-Reingold algorithm for network layouts.
This is a mulitlayer variation of the standard FR algorithm, where the layout is
produced by simulating springs between linked nodes and repulsive forces between all
nodes. The main difference to the normal version of the algorithm is that the nodes
which are on different layers do not repulse each other.
Parameters
----------
net : MultilayerNetwork
The network for which the coordinates are calculated
nodeDist : float, string
The desired distance between pairs of nodes. If "auto", then inverse of the
square root of the number of nodes is used.
boxSize : float, tuple of floats
The size of the bounding box for the coordinates. If float is given then a square
box is used. Otherwise, provide a tuple with two floats.
alignedNodes : bool
Determines if the nodes-layer tuples with the same node should have the same
coordinates.
nodelayerCoords : dict, None
Initial coordinates for node-layer tuples. If None, random coordinates are used.
If alignedNodes is set to True these coordinates are not used.
nodeCoords : dict, None
Initial coordinates for nodes. If None, random coordinates are used. If a coordinate
for node-layer tuple is defined then that is used instead of the node coordinate.
fixedNodes : set, None
The set of nodes that are not moved from the initial coordinates. If None, then
all nodes are allowed to move. You can also use list or similar data structures, but
set is recommended for speed when the number of elements is large.
fixedNodeLayers : set, None
The set of nodes-layers that are not moved from the initial coordinates. If None, then
all node-layers are allowed to move. You can also use list or similar data structures, but
set is recommended for speed when the number of elements is large.
iterations : int
The number of times the nodes/node-layer tuples are moved.
"""
#Parsing parameters and sanity check for them
#net
assert isinstance(net, pymnet.MultilayerNetwork), "Invalid network type"
assert net.aspects >= 1, "No monoplex networks"
#If the network is fully interconnected, we just create network with one layer
if net.fullyInterconnected:
assert nodelayerCoords == None, "Only node coordinates for fully interconnected networks"
magg = pymnet.MultiplexNetwork(fullyInterconnected=False)
magg.add_layer("all")
magg.A["all"] = pymnet.transforms.aggregate(net, 1)
net = magg
#nodeDist
if nodeDist == "auto":
nodeDist = 1. / math.sqrt(len(net.slices[0]))
else:
nodeDist = float(nodeDist)
assert nodeDist > 0
#boxSize
if isinstance(boxSize, tuple) or isinstance(boxSize, list):
assert len(boxSize) == 2
else:
boxSize = float(boxSize)
boxSize = (boxSize, boxSize)
#nodeCoords
if nodeCoords == None:
nodeCoords = {}
#nodelayerCoords
if nodelayerCoords == None:
nodelayerCoords = {}
if alignedNodes: #use node coordinates
nc = {}
for node in net:
if node in nodeCoords:
nc[node] = nodeCoords[node]
else:
nc[node] = (boxSize[0] * random.random(),
boxSize[1] * random.random())
else: #use node-layer tuple coordinates
nlc = {}
for nl in net.iter_node_layers():
if nl in nodelayerCoords:
nlc[nl] = nodelayerCoords[nl]
elif nl[0] in nodeCoords:
nlc[nl] = nodeCoords[nl[0]]
else:
nlc[nl] = (boxSize[0] * random.random(),
boxSize[1] * random.random())
if fixedNodes == None:
fixedNodes = set()
if fixedNodeLayers == None:
fixedNodeLayers = set()
#Parsing complete
#Some internal parameters
temperature = 0.1 * max(boxSize)
delta_temperature = temperature / float(iterations)
min_dist = 0.01
for iteration in range(iterations):
if alignedNodes: #we have coordinates for nodes
delta_nc = dict(((k, (0., 0.)) for k in nc))
#Spring forces
for edge in net.edges:
node1 = edge[0]
node2 = edge[1]
if node1 != node2:
diff = (nc[node1][0] - nc[node2][0],
nc[node1][1] - nc[node2][1])
dist = math.sqrt(diff[0]**2 + diff[1]**2)
c = dist / float(nodeDist)
delta_nc[node1] = (delta_nc[node1][0] - c * diff[0],
delta_nc[node1][1] - c * diff[1])
delta_nc[node2] = (delta_nc[node2][0] + c * diff[0],
delta_nc[node2][1] + c * diff[1])
#Repulsive forces
for node1 in net:
for node2 in net:
if node1 != node2:
layer_overlap = len(
net._nodeToLayers[node1].intersection(
net._nodeToLayers[node2]))
diff = (nc[node1][0] - nc[node2][0],
nc[node1][1] - nc[node2][1])
dist = math.sqrt(diff[0]**2 + diff[1]**2)
dist = max(dist, min_dist)
c = layer_overlap * nodeDist**2 / float(dist**2)
delta_nc[node1] = (delta_nc[node1][0] + c * diff[0],
delta_nc[node1][1] + c * diff[1])
delta_nc[node2] = (delta_nc[node2][0] - c * diff[0],
delta_nc[node2][1] - c * diff[1])
#Normalize coordinate, and apply them
#for node,(x,y) in delta_nc.iteritems():
for node in delta_nc:
x, y = delta_nc[node]
if node not in fixedNodes:
delta_len = math.sqrt(x**2 + y**2)
nc[node] = (nc[node][0] + temperature * delta_len * x,
nc[node][1] + temperature * delta_len * y)
normalize_coordinates(nc, boxSize, inplace=True)
else: #we have coordinates for node-layer tuples
#There is currently a lot of code dublication here when compared to the
#case where nodes are aligned. Some of this should be removed, and some
#of it will probably disappear once the code is optimized a bit.
delta_nlc = dict(((k, (0., 0.)) for k in nlc))
#Spring forces
for edge in net.edges:
nl1, nl2 = net._link_to_nodes(edge[:-1])
diff = (nlc[nl1][0] - nlc[nl2][0], nlc[nl1][1] - nlc[nl2][1])
dist = math.sqrt(diff[0]**2 + diff[1]**2)
dist = max(dist, min_dist)
c = dist / float(nodeDist)
delta_nlc[nl1] = (delta_nlc[nl1][0] - c * diff[0],
delta_nlc[nl1][1] - c * diff[1])
delta_nlc[nl2] = (delta_nlc[nl2][0] + c * diff[0],
delta_nlc[nl2][1] + c * diff[1])
#Repulsive forces
for nl1 in net.iter_node_layers():
layer = nl1[1:][0] if net.aspects == 1 else nl1[1:]
for node2 in net.iter_nodes(layer=layer):
nl2 = (node2, ) + nl1[1:]
if nl1 != nl2:
diff = (nlc[nl1][0] - nlc[nl2][0],
nlc[nl1][1] - nlc[nl2][1])
dist = math.sqrt(diff[0]**2 + diff[1]**2)
dist = max(dist, min_dist)
c = nodeDist**2 / float(dist**2)
delta_nlc[nl1] = (delta_nlc[nl1][0] + c * diff[0],
delta_nlc[nl1][1] + c * diff[1])
delta_nlc[nl2] = (delta_nlc[nl2][0] - c * diff[0],
delta_nlc[nl2][1] - c * diff[1])
#Normalize coordinate, and apply them
#for nl,(x,y) in delta_nlc.iteritems():
for nl in delta_nlc:
x, y = delta_nlc[nl]
if nl not in fixedNodeLayers:
delta_len = math.sqrt(x**2 + y**2)
nlc[nl] = (nlc[nl][0] + temperature * delta_len * x,
nlc[nl][1] + temperature * delta_len * y)
normalize_coordinates(nlc, boxSize, inplace=True)
temperature -= delta_temperature
if alignedNodes:
return nc
else:
return nlc
def yield_clustered_multilayer_network_in_layersets(imgdata,
layerset_size,
timewindow=100,
overlap=0,
n_clusters=100,
method='sklearn',
template=None,
nanlogfile=None,
event_time_stamps=None):
# If event_time_stamps is specified, then they are used to compute start_times, end_times and k (and timewindow and overlap are ignored).
# Otherwise, timewindow and overlap are used ot compute start_times, end_times and k.
if event_time_stamps == None:
k = get_number_of_layers(imgdata.shape, timewindow, overlap)
start_times, end_times = get_start_and_end_times(
k, timewindow, overlap)
else:
assert isinstance(event_time_stamps, list)
k = len(event_time_stamps) + 1
start_times = [0] + event_time_stamps
end_times = event_time_stamps + [imgdata.shape[3]]
layersets = zip(*(range(k)[ii:] for ii in range(layerset_size)))
if method == 'sklearn':
voxels_in_clusters_by_timewindow = dict()
for layerset in layersets:
M = pn.MultilayerNetwork(aspects=1, fullyInterconnected=False)
previous_voxels_in_clusters = dict()
for tw_no in layerset:
if not tw_no in voxels_in_clusters_by_timewindow:
voxels_in_clusters = dict()
model, voxellist = clustering.cluster_timewindow_scikit(
imgdata[:, :, :, start_times[tw_no]:end_times[tw_no]],
n_clusters=n_clusters)
for ii, label in enumerate(model.labels_):
voxels_in_clusters.setdefault(label,
[]).append(voxellist[ii])
voxels_in_clusters_by_timewindow[
tw_no] = voxels_in_clusters
else:
voxels_in_clusters = voxels_in_clusters_by_timewindow[
tw_no]
R = calculate_cluster_correlation_matrix(
imgdata[:, :, :, start_times[tw_no]:end_times[tw_no]],
voxels_in_clusters)
for ii in range(R.shape[0]):
node1 = str(voxels_in_clusters[ii])
for jj in range(ii + 1, R.shape[1]):
node2 = str(voxels_in_clusters[jj])
if not np.isnan(R[ii, jj]):
M[node1, tw_no][node2, tw_no] = R[ii, jj]
else:
if nanlogfile != None:
with open(nanlogfile, 'a+') as f:
f.write('NaN correlation at nodes ' +
node1 + ', ' + node2 +
' at timewindow ' + str(tw_no) +
'\n')
else:
print('NaN correlation at nodes ' + node1 +
', ' + node2 + ' at timewindow ' +
str(tw_no) + '\n')
for cluster_number in voxels_in_clusters:
for previous_cluster_number in previous_voxels_in_clusters:
cluster_overlap = get_overlap(
set(voxels_in_clusters[cluster_number]),
set(previous_voxels_in_clusters[
previous_cluster_number]))
M[str(previous_voxels_in_clusters[
previous_cluster_number]),
tw_no - 1][str(voxels_in_clusters[cluster_number]),
tw_no] = cluster_overlap
previous_voxels_in_clusters = voxels_in_clusters # reference to the same object
del (voxels_in_clusters_by_timewindow[min(
voxels_in_clusters_by_timewindow)])
yield M
del (M)
elif method == 'template':
voxels_in_clusters = get_voxels_in_clusters(template)
for layerset in layersets:
M = pn.MultiplexNetwork(couplings='ordinal',
fullyInterconnected=True)
for tw_no in layerset:
R = calculate_cluster_correlation_matrix(
imgdata[:, :, :, start_times[tw_no]:end_times[tw_no]],
voxels_in_clusters)
for ii in range(R.shape[0]):
node1 = str(voxels_in_clusters[ii])
for jj in range(ii + 1, R.shape[1]):
node2 = str(voxels_in_clusters[jj])
if not np.isnan(R[ii, jj]):
M[node1, tw_no][node2, tw_no] = R[ii, jj]
else:
if nanlogfile != None:
with open(nanlogfile, 'a+') as f:
f.write('NaN correlation at nodes ' +
node1 + ', ' + node2 +
' at timewindow ' + str(tw_no) +
'\n')
else:
print('NaN correlation at nodes ' + node1 +
', ' + node2 + ' at timewindow ' +
str(tw_no) + '\n')
yield M
del (M)
else:
raise NotImplementedError('Not implemented')
def load_mplex(filenames, netnames, threshold=0):
net = pymnet.MultiplexNetwork(couplings='none')
for fn, nn in zip(filenames, netnames):
net.add_layer(nn)
load_layer(net.A[nn], fn, threshold=threshold)
return net