def save(self, filename=''):
r'''
Parameters
----------
filename : string
The filename to contain the saved object data in Numpy zip format (npz)
Examples
--------
>>> pn = OpenPNM.Network.Cubic(shape=[3,3,3])
>>> pn.save('test_pn')
>>> gn = OpenPNM.Network.GenericNetwork()
>>> gn.load('test_pn')
>>> # Remove newly created file
>>> import os
>>> os.remove('test_pn.npz')
'''
if filename == '':
filename = self.name
obj_dict = {}
obj_dict['data'] = self.copy()
obj_dict['info'] = {}
obj_dict['info']['name'] = self.name
obj_dict['info']['module'] = self.__module__
sp.savez_compressed(filename, **obj_dict)
def read_fmri(self, data_dir, reduce_dim=None, int_subid=1, roiwise=1):
""" Read fMRI data from disk
If reduce_dim = None, no dimesionality reduction is performed
data is Time x Vertices x sub"""
self.data_dir_fmri = data_dir
self.dirlst = glob.glob(self.data_dir_fmri + '/*.mat')
self.fmri_data = list()
self.subids = list()
print('\n=======\n Reading fMRI data\n=======\n')
if reduce_dim is not None:
pca = PCA(n_components=reduce_dim)
for subfile in self.dirlst:
subid = subfile.replace('_rest_bold.32k.GOrd.mat', '')
subid = subid.replace(self.data_dir_fmri + '/', '')
outfile = self.data_dir_fmri + '/processed/' + subid + 'pca_reduced.npz'
if int_subid:
subid = int(subid)
if os.path.isfile(outfile):
a = sp.load(outfile)
fmri_data = a['fmri_data']
print('Fast Read sub id = ' + str(subid))
else:
if os.path.isfile(subfile):
print('Reading sub id = ' + str(subid))
fmri_data = loadmat(subfile)['dtseries']
# Preprocess fMRI, replace Nan by avg of cortical activity at
# that time point and standardize this should be interesting
imp = Imputer(missing_values='NaN',
strategy='mean',
axis=0)
fmri_data = imp.fit_transform(fmri_data)
fmri_data = StandardScaler().fit_transform(fmri_data)
if reduce_dim != None:
fmri_data = pca.fit_transform(fmri_data)
sp.savez_compressed(outfile, fmri_data=fmri_data)
if roiwise > 0:
fmri_data = roiwise_timeseries(fmri_data,
self.gord_label)
self.subids.append(subid)
self.fmri_data.append(fmri_data)
for ind in range(nSub):
sub_data1[:, :, ind], _ = rot_sub_data(ref=sub_data1[:, :, 0],
sub=sub_data1[:, :, ind])
sub_data2[:, :, ind], _ = rot_sub_data(ref=sub_data1[:, :, 0],
sub=sub_data2[:, :, ind])
cat_data[sub_data1.shape[0] * ind:sub_data1.shape[0] *
(ind + 1), :] = sub_data1[:, :, ind]
ind1 = nSub + ind
cat_data[sub_data1.shape[0] * ind1:sub_data1.shape[0] *
(ind1 + 1), :] = sub_data2[:, :, ind]
print ind, sub_data1.shape, cat_data.shape
#sp.savez_compressed('data_bothsessions', cat_data=cat_data, lst=lst, nClusters=nClusters)
######
a = sp.load('data_bothsessions.npz')
cat_data = a['cat_data']
lst = a['lst']
nClusters = a['nClusters']
SC = KMeans(n_clusters=nClusters, random_state=5324, n_jobs=-1)
labs_all = SC.fit_predict(cat_data)
lab_sub = labs_all.reshape((sub_data1.shape[0], 2 * nSub), order='F')
sp.savez_compressed('labs_all_data_bothsessions_17_clusters',
lab_sub=lab_sub,
lst=lst,
nClusters=nClusters)
## calculate the interacting GF from local Dyson equation
GFint_F = GFlambda(En_A - SE_F)
DOSF = -sp.imag(GFint_F[int(N / 2)]) / sp.pi
Norm = IntDOS(GFint_F, En_A)
[HWHM, DOSmax, wmax] = CalculateHWHM(GFint_F, En_A)
if chat: print('# Int A(w)dw (int) = {0: .5f}, DOS[0] = {1: .5f}, HWHM = {2: .6f}'\
.format(float(Norm),float(DOSF),float(HWHM)))
## write intermediate step to file
if WriteFiles:
filename = 'gf_iter' + str(NLoop) + '.dat'
WriteFileX([En_A, GFbath_F, SE_F, GFint_F], WriteMax, WriteStep, 'gf',
filename, chat)
## save int. GF and SE in case we want to continue iterations and remove an old one (npz files are large)
sp.savez_compressed('dmft_' + str(NLoop),
GFint_F=GFint_F,
SE_F=SE_F,
Lambda=Lambda)
rm_filename = 'dmft_' + str(NLoop - 2) + '.npz'
if rm_filename in listdir('.'):
print('# Removing npz file from iteration {0: 3d} to save disk space.'.
format(NLoop - 2))
remove(rm_filename)
## write data about current iteration fo file
iter_file.write('{0: 3d}\t{1: .6f}\t{2: .6f}\t{3: .6f}\t{4: .6f}\t{5: .6f}\t{6: .6f}\t{7: .6f}\n'\
.format(NLoop,Lambda,float(a),float(Norm),float(DOSF),float(HWHM),float(Uc),float(DLambda)))
iter_file.close()
## write the final GF to file
filename = 'gf_U' + str(U) + '_dmft.dat'
WriteFileX([En_A, GFbath_F, SE_F, GFint_F], WriteMax, WriteStep, 'gf',
filename, chat)
m = np.mean(temp, 1)
temp = temp - m[:, None]
s = np.std(temp, 1) + 1e-16
temp = temp / s[:, None]
d1 = temp
if count1 == 0:
sub_data1 = sp.zeros((d1.shape[0], d1.shape[1], len(lst)))
sub_data1[:, :, count1] = d1
count1 += 1
print count1,
nSub = sub_data1.shape[2]
cat_data = sp.reshape(sub_data1,
(sub_data1.shape[0], sub_data1.shape[1] * nSub), 'F')
print cat_data.shape
del sub_data1, d1, temp, data1
SC = KMeans(n_clusters=nClusters, random_state=5324, verbose=1)
labs_cat = SC.fit_predict(cat_data)
sp.savez_compressed('labs_concat_data_100_clusters',
labs_cat=labs_cat,
lst=lst,
nClusters=nClusters)
sub_data = sp.zeros((d.shape[0],d.shape[1],len(lst)))
sub_data[:,:,count1] = d
count1+=1
print count1,
nSub=sub_data.shape[2]
cat_data1=sp.zeros((nSub*sub_data.shape[0]/2,sub_data.shape[1]))
cat_data2=sp.zeros((nSub*sub_data.shape[0]/2,sub_data.shape[1]))
for ind in range(nSub):
sub_data[:,:,ind] = rot_sub_data(ref=sub_data[:,:,0],sub=sub_data[:,:,ind])
if ind < nSub/2:
cat_data1[sub_data.shape[0]*ind:sub_data.shape[0]*(ind+1),:] = sub_data[:,:,ind]
else:
ind2=ind-nSub/2
cat_data2[sub_data.shape[0]*ind2:sub_data.shape[0]*(ind2+1),:] = sub_data[:,:,ind-1]
SC = KMeans(n_clusters=nClusters,random_state=5324)
labs_all1 = SC.fit_predict(cat_data1)
labs_all2 = SC.fit_predict(cat_data2)
lab_sub1=labs_all1.reshape((sub_data.shape[0],nSub/2),order='F')
lab_sub2=labs_all2.reshape((sub_data.shape[0],nSub/2),order='F')
sp.savez_compressed('labs_all_split2_data_1common_10clusters', lst=lst, lab_sub1=lab_sub1, lab_sub2=lab_sub2, cat_data1=cat_data1, cat_data2=cat_data2)
def save(network, filename=''):
r'''
Save the current simulation in it's entirity.
Parameters
----------
net : OpenPNM Network Object
The network object of the simulation to be saved. This will
automatically save all Geometry, Phases and Physics objects
associated with the Network, but will not save any Algorithms.
filename : string, optional
The file name to yse for saving. Defaults to the Network's name.
Notes
-----
This stores the simulation in a nested dictionary with the data dict
of the object stored under ['data'][object.name], the object linking
under ['tree'][object.name] and the information to reproduce the models
under ['mods'][object.name]. The ``load`` method knows how to unpack
this dictionary.
Examples
--------
>>> # Saving
>>> pn = OpenPNM.Network.Cubic(shape=[3,3,3])
>>> geo = OpenPNM.Geometry.Stick_and_Ball(network=pn,pores=pn.pores(),throats=pn.throats(),name='geo_1')
>>> air = OpenPNM.Phases.Air(network=pn)
>>> phys = OpenPNM.Physics.Standard(network=pn,phase=air,pores=pn.pores(),throats=pn.throats())
>>> import OpenPNM.Utilities.IO as io
>>> io.PNM.save(pn,'test_pn')
>>> # Loading
>>> import OpenPNM.Utilities.IO as io
>>> #pn = io.PNM.load('test_pn')
>>> # Delete the new file
>>> import os
>>> os.remove('test_pn.pnm')
See Also
--------
IO.PNM.load
'''
if filename != '':
filename = filename
else:
filename = network.name
#Setup nested dictionary to store simulation
sim = {}
sim['data'] = {}
sim['tree'] = {}
sim['mods'] = {}
#Collect all objects into a single list
all_objs = [network]
all_objs.extend(network._geometries)
all_objs.extend(network._phases)
all_objs.extend(network._physics)
#Enter each object's data, object tree and models into dictionary
for obj in all_objs:
module = obj.__module__.split('.')[1]
sim['data'][module + '.' + obj.name] = obj.copy()
sim['tree'][module + '.' + obj.name] = {
'SelfType': obj.__class__.__name__,
'Geometries': obj.geometries(),
'Phases': obj.phases(),
'Physics': obj.physics()
}
sim['mods'][module + '.' + obj.name] = {}
for prop in list(obj._models.keys()):
sim['mods'][module + '.' + obj.name][prop] = PNM._save_model(
obj, prop)
#Save nested dictionary as a Numpy zip file
_sp.savez_compressed(filename, **sim)
#Rename the zip extension to pnm for kicks
_os.rename(filename + '.npz', filename + '.pnm')
SEold_F = sp.copy(SE_F)
## calculate the interacting GF from local Dyson equation
GFint_F = GFlambda(En_A-SE_F)
DOSF = -sp.imag(GFint_F[int(N/2)])/sp.pi
Norm = IntDOS(GFint_F,En_A)
[HWHM,DOSmax,wmax] = CalculateHWHM(GFint_F,En_A)
if chat: print('# Int A(w)dw (int) = {0: .5f}, DOS[0] = {1: .5f}, HWHM = {2: .6f}'\
.format(float(Norm),float(DOSF),float(HWHM)))
## write intermediate step to file
if WriteFiles:
filename = 'gf_iter'+str(NLoop)+'.dat'
WriteFileX([En_A,GFbath_F,SE_F,GFint_F],WriteMax,WriteStep,'gf',filename,chat)
## save int. GF and SE in case we want to continue iterations and remove an old one (npz files are large)
sp.savez_compressed('dmft_'+str(NLoop),GFint_F = GFint_F, SE_F = SE_F, Lambda = Lambda)
rm_filename = 'dmft_'+str(NLoop-2)+'.npz'
if rm_filename in listdir('.'):
print('# Removing npz file from iteration {0: 3d} to save disk space.'.format(NLoop-2))
remove(rm_filename)
## write data about current iteration fo file
iter_file.write('{0: 3d}\t{1: .6f}\t{2: .6f}\t{3: .6f}\t{4: .6f}\t{5: .6f}\t{6: .6f}\t{7: .6f}\n'\
.format(NLoop,Lambda,float(a),float(Norm),float(DOSF),float(HWHM),float(Uc),float(DLambda)))
iter_file.close()
## write the final GF to file
filename = 'gf_U'+str(U)+'_dmft.dat'
WriteFileX([En_A,GFbath_F,SE_F,GFint_F],WriteMax,WriteStep,'gf',filename,chat)
print('{0: .3f}\t{1: .3f}\t{2: .3f}\t{3: .3f}\t{4: .6f}\t{5: .6f}\t{6: .6f}\t{7: .6f}\t{8: .6f}\t{9: .6f}\n'\
.format(U,Delta,ef,T,Lambda,float(a),float(Norm),float(DOSF),float(HWHM),float(DLambda)))
def spladder():
### get command line options
options = parse_options(sys.argv)
### parse parameters from options object
CFG = settings.parse_args(options)
### add dependencies provided in config section
#if 'paths' in CFG:
# for i in CFG['paths']:
# eval('import %s'% CFG['paths'][i])
### load confidence level settings
if not CFG['no_reset_conf']:
CFG = settings.set_confidence_level(CFG)
### do not compute components of merged set, if result file already exists
fn_out_merge = get_filename('fn_out_merge', CFG)
fn_out_merge_val = get_filename('fn_out_merge_val', CFG)
if not 'spladder_infile' in CFG and not os.path.exists(fn_out_merge):
### iterate over files, if merge strategy is single
if CFG['merge_strategy'] in ['single', 'merge_graphs']:
idxs = range(len(CFG['samples']))
else:
idxs = [0]
### set parallelization
if CFG['rproc']:
jobinfo = []
### create out-directory
if not os.path.exists(CFG['out_dirname']):
os.makedirs(CFG['out_dirname'])
### create spladder sub-directory
if not os.path.exists(os.path.join(CFG['out_dirname'], 'spladder')):
os.makedirs(os.path.join(CFG['out_dirname'], 'spladder'))
### pre-process annotation, if necessary
if CFG['anno_fname'].split('.')[-1] != 'pickle':
if not os.path.exists(CFG['anno_fname'] + '.pickle'):
if CFG['anno_fname'].split('.')[-1] in ['gff', 'gff3']:
(genes, CFG) = init.init_genes_gff3(CFG['anno_fname'], CFG, CFG['anno_fname'] + '.pickle')
elif CFG['anno_fname'].split('.')[-1] in ['gtf']:
(genes, CFG) = init.init_genes_gtf(CFG['anno_fname'], CFG, CFG['anno_fname'] + '.pickle')
else:
print >> sys.stderr, 'ERROR: Unknown annotation format. File needs to end in gtf or gff/gff3\nCurrent file: %s' % CFG['anno_fname']
sys.exit(1)
CFG['anno_fname'] += '.pickle'
### add anotation contigs into lookup table
if not 'genes' in CFG:
genes = cPickle.load(open(CFG['anno_fname'], 'r'))
else:
genes = CFG['genes']
CFG = init.append_chrms(sp.unique(sp.array([x.chr for x in genes], dtype='str')), CFG)
del genes
for idx in idxs:
CFG_ = dict()
if CFG['merge_strategy'] != 'merge_bams':
CFG_['bam_fnames'] = CFG['bam_fnames']
CFG_['samples'] = CFG['samples']
CFG['bam_fnames'] = CFG['bam_fnames'][idx]
CFG['samples'] = CFG['samples'][idx]
CFG['out_fname'] = '%s/spladder/genes_graph_conf%i.%s.pickle' % (CFG['out_dirname'], CFG['confidence_level'], CFG['samples'])
else:
CFG['out_fname'] = '%s/spladder/genes_graph_conf%i.%s.pickle' % (CFG['out_dirname'], CFG['confidence_level'], CFG['merge_strategy'])
### assemble out filename to check if we are already done
fn_out = CFG['out_fname']
if CFG['do_prune']:
fn_out = re.sub('.pickle$', '_pruned.pickle', fn_out)
if CFG['do_gen_isoforms']:
fn_out = re.sub('.pickle$', '_with_isoforms.pickle', fn_out)
if os.path.exists(fn_out):
print >> sys.stdout, '%s - All result files already exist.' % fn_out
else:
if CFG['rproc']:
jobinfo.append(rp.rproc('spladder_core', CFG, 15000, CFG['options_rproc'], 60*60))
else:
spladder_core(CFG)
for key in CFG_:
try:
CFG[key] = CFG_[key].copy()
except AttributeError:
CFG[key] = CFG_[key]
### collect results after parallelization
if CFG['rproc']:
rp.rproc_wait(jobinfo, 30, 1.0, -1)
### merge parts if necessary
if CFG['merge_strategy'] == 'merge_graphs':
run_merge(CFG)
if not 'spladder_infile' in CFG and CFG['validate_splicegraphs'] and not os.path.exists(fn_out_merge_val):
(genes, inserted) = cPickle.load(open(fn_out_merge, 'r'))
genes = filter_by_edgecount(genes, CFG)
cPickle.dump((genes, inserted), open(fn_out_merge_val, 'w'), -1)
del genes
### get count output file
fn_in_count = get_filename('fn_count_in', CFG)
fn_out_count = get_filename('fn_count_out', CFG)
### convert input BAMs to sparse arrays
if CFG['bam_to_sparse']:
for bfn in CFG['bam_fnames']:
if bfn.endswith('bam') and not os.path.exists(re.sub(r'.bam$', '', bfn) + '.npz'):
cnts = dict()
if not 'chrm_lookup' in CFG:
IN = pysam.Samfile(bfn, 'rb')
CFG = append_chrms([x['SN'] for x in parse_header(IN.text)['SQ']], CFG)
IN.close()
if CFG['parallel'] > 1:
import multiprocessing as mp
pool = mp.Pool(processes=CFG['parallel'])
result = [pool.apply_async(summarize_chr, args=(bfn, str(chrm), CFG,)) for chrm in sorted(CFG['chrm_lookup'])]
while result:
tmp = result.pop(0).get()
cnts[tmp[0] + '_reads_row'] = tmp[1].row.astype('uint8')
cnts[tmp[0] + '_reads_col'] = tmp[1].col
cnts[tmp[0] + '_reads_dat'] = tmp[1].data
cnts[tmp[0] + '_reads_shp'] = tmp[1].shape
cnts[tmp[0] + '_introns_m'] = tmp[2]
cnts[tmp[0] + '_introns_p'] = tmp[3]
else:
for chrm in CFG['chrm_lookup']:
tmp = summarize_chr(bfn, str(chrm), CFG)
cnts[chrm + '_reads_row'] = tmp[1].row.astype('uint8')
cnts[chrm + '_reads_col'] = tmp[1].col
cnts[chrm + '_reads_dat'] = tmp[1].data
cnts[chrm + '_reads_shp'] = tmp[1].shape
cnts[chrm + '_introns_m'] = tmp[2]
cnts[chrm + '_introns_p'] = tmp[3]
sp.savez_compressed(re.sub(r'.bam$', '', bfn), **cnts)
elif CFG['verbose']:
print >> sys.stdout, 'Sparse BAM representation for %s already exists.' % bfn
### count segment graph
if CFG['run_as_analysis'] or CFG['count_segment_graph']:
if not os.path.exists(fn_out_count):
count_graph_coverage_wrapper(fn_in_count, fn_out_count, CFG)
### count intron coverage phenotype
if CFG['count_intron_cov']:
fn_out_intron_count = fn_out_count.replace('mat', 'introns.pickle')
count_intron_coverage_wrapper(fn_in_count, fn_out_intron_count, CFG)
### handle alternative splicing part
if CFG['run_as_analysis']:
collect_events(CFG)
for idx in range(len(CFG['event_types'])):
analyze_events(CFG, CFG['event_types'][idx])
nSub = sub_data1.shape[2]
npts = sp.sum(msk_small_region)
cat_data = sp.zeros((2 * nSub * npts, sub_data1.shape[1]))
for ind in range(nSub):
d1 = rot_sub_data(ref=sub_data1[:, :, 0], sub=sub_data1[:, :, ind])
# sub_data1[:,:,ind] =d1[msk_small_region,:]
d2 = rot_sub_data(ref=sub_data1[:, :, 0], sub=sub_data2[:, :, ind])
# sub_data2[:,:,ind] =d[msk_small_region,:]
cat_data[npts * ind:npts * (ind + 1), :] = d1[msk_small_region, :]
ind1 = nSub + ind
cat_data[npts * ind1:npts * (ind1 + 1), :] = d2[msk_small_region, :]
print ind, sub_data1.shape, cat_data.shape
sp.savez_compressed('data_bothsessions_precuneus',
cat_data=cat_data,
msk_small_region=msk_small_region)
SC = KMeans(n_clusters=nClusters, random_state=5324)
labs_all = SC.fit_predict(cat_data)
lab_sub = labs_all.reshape((npts, 2 * nSub), order='F')
sp.savez_compressed('labs_all_data_bothsessions_8_clusters_precuneus',
msk_small_region=msk_small_region,
lab_sub=lab_sub,
cat_data=cat_data,
lst=lst,
nClusters=nClusters)
m = np.mean(temp, 1)
temp = temp - m[:, None]
s = np.std(temp, 1) + 1e-16
temp = temp / s[:, None]
d = temp
if count1 == 0:
sub_data = sp.zeros((d.shape[0], d.shape[1], len(lst)))
sub_data[:, :, count1] = d
count1 += 1
print count1,
nSub = sub_data.shape[2]
cat_data = sp.zeros((nSub * sub_data.shape[0], sub_data.shape[1]))
for ind in range(nSub):
sub_data[:, :, ind] = rot_sub_data(ref=sub_data[:, :, 0],
sub=sub_data[:, :, ind])
cat_data[sub_data.shape[0] * ind:sub_data.shape[0] *
(ind + 1), :] = sub_data[:, :, ind]
print ind, sub_data.shape, cat_data.shape
SC = KMeans(n_clusters=nClusters, random_state=5324)
labs_all = SC.fit_predict(cat_data)
lab_sub = labs_all.reshape((sub_data.shape[0], nSub), order='F')
sp.savez_compressed('labs_all_data2', lab_sub=lab_sub, cat_data=cat_data)
if GFtype in ['cubic','square']:
fin = GFtype+'_gf_'+str(W)+'_'+str(NE)+'_'+str(dE)+'.npz'
try:
GFzero_A = sp.load(fin)['GFzero_A']
i2 = sp.load(fin)['izero']
if i2 == izero:
if chat: print('# - reading non-interacting GF from file '+fin)
else:
print('- i0 value differs from the saved data')
raise FileNotFoundError()
except FileNotFoundError:
if chat: print('# - file '+fin+' not found, or i0 differs, generating new file')
t = time()
GFzero_A = GFlambda(En_A)
GFzero_A = GFzero_A/IntDOS(GFzero_A)
sp.savez_compressed(fin, En_A = En_A, GFzero_A = GFzero_A, izero = izero)
if chat: print('# - file generated in '+str(int(time()-t))+' seconds.')
else:
GFzero_A = GFlambda(En_A)
GFzero_A = GFzero_A/IntDOS(GFzero_A)
#WriteFileX([GFzero_A,GFzero_A,GFzero_A],WriteMax,WriteStep,'gf','GFzero.dat')
## calculating the charge-symmetric case (n=0.5) to get Lambda0
if chat: print('#\n# calculating the charge-symmetric solution:')
if chat: print('# norm[G0]: {0: .6f}, n[G0]: {1: .6f}'\
.format(float(IntDOS(GFzero_A)),float(Filling(GFzero_A))))
if chat: print('# calculating the charge-symmetric two-particle bubble...')
Bubble_A = TwoParticleBubble(GFzero_A,GFzero_A,'eh') # Bubble[0] is negative
BubZero = Bubble_A[int(N/2)]
Uc = -1.0/sp.real(BubZero)
labs1 = parcellate_motor(sub, nClusters+1, 0, 1, 0)
if os.path.isfile(os.path.join(p_dir, sub, sub +
'.rfMRI_REST2_RL.reduce3.ftdata.NLM_11N\
_hvar_25.mat')):
labs2 = parcellate_motor(sub, nClusters+1, 0, 2, 0)
count1 += 1
if count1 == 1:
labs_all_1 = sp.array(labs1)
labs_all_2 = sp.array(labs2)
else:
labs_all_1 = sp.vstack([labs_all_1, labs1])
labs_all_2 = sp.vstack([labs_all_2, labs2])
# sp.savez_compressed('clustering_results_sessions1' + str(nClusters),
# labs_all_1=labs_all_1, labs_all_2=labs_all_2)
R = sp.zeros(count1)
for a in range(count1):
R[a] = adjusted_rand_score(labs_all_1[a, :], labs_all_2[a, :])
R_all.append(R)
print('Clusters=', nClusters)
sp.savez_compressed('clustering_results_sessions_heirarchical', R_all=R_all,
labs_all_1=labs_all_1, labs_all_2=labs_all_2)
#%%
fig = plt.figure()
plt.boxplot(R_all)
fig.savefig('across_subjects_adj_rand_sessions_heirarchical.pdf')
# d_ref = d
#
drot, _ = rot_sub_data(ref=d2, sub=d1)
dist[iWinL, nb] = sp.linalg.norm((drot - d2) / sp.sqrt(WinL))
corr_diff[iWinL, nb] = sp.median(sp.sum(drot * d2, axis=1) / d2.shape[1])
corr_mtx = sp.sum(drot * d2, axis=1) / d2.shape[1]
corr_mtx_diff[iWinL, nb] = sp.linalg.norm(corr_mtx - full_corr)
print(nb, WinL, dist[iWinL, nb], corr_diff[iWinL, nb], corr_mtx_diff[iWinL,
nb])
sp.savez_compressed('Corr_dist_nsamples_200tNLM.npz',
corr_diff=corr_diff,
corr_mtx_diff=corr_mtx_diff,
dist=dist)
#%%
import seaborn as sns
import matplotlib.pyplot as plt
import pandas as pd
win_lengths = sp.arange(5, 1200, 20)
data = sp.load('Corr_dist_nsamples_200tNLM.npz')
corr_mtx_diff = data['corr_mtx_diff']
corr_diff = data['corr_diff']
dist = data['dist']
df = []
p_dir, sub, sub + '.rfMRI_REST1_RL.reduce3.ftdata.NLM_11N\
_hvar_25.mat')):
labs1 = parcellate_motor(sub, nClusters + 1, 1, 1, 2)
if os.path.isfile(
os.path.join(
p_dir, sub, sub + '.rfMRI_REST2_RL.reduce3.ftdata.NLM_11N\
_hvar_25.mat')):
labs2 = parcellate_motor(sub, nClusters + 1, 1, 2, 2)
count1 += 1
if count1 == 1:
labs_all_1 = sp.array(labs1)
labs_all_2 = sp.array(labs2)
else:
labs_all_1 = sp.vstack([labs_all_1, labs1])
labs_all_2 = sp.vstack([labs_all_2, labs2])
R = sp.zeros(count1)
for a in range(count1):
R = adjusted_rand_score(labs_all_1, labs_all_2)
R_all.append(R)
print('Clusters=', nClusters)
sp.savez_compressed('clustering_results_sessions_GMM', R_all=R_all)
#%%
fig = plt.figure()
plt.plot(R_all)
fig.savefig('across_subjects_adj_rand_sessions_GMM.pdf')
#%%
s = np.std(temp, 1) + 1e-116
temp = temp / s[:, None]
d1 = temp # [cc_msk, :]
win_lengths = sp.arange(5, d1.shape[1], 20)
nboot = 200
nbootiter = sp.arange(nboot)
cfull = sp.dot(d1, d1.T) / d1.shape[1]
cnt = 0
sz = sp.arange(2, d1.shape[1], 20)
err = sp.zeros((len(win_lengths), nboot))
for nb, iWinL in itertools.product(nbootiter, sp.arange(len(win_lengths))):
WinL = win_lengths[iWinL]
startpt = randint(0, data.shape[1])
t = sp.arange(startpt, startpt + WinL)
t = sp.mod(t, data.shape[1])
temp = data[LR_flag, :]
temp = temp[:, t]
m = np.mean(temp, 1)
temp = temp - m[:, None]
s = sp.std(temp, axis=1) + 1e-116
temp = temp / s[:, None]
d1 = temp
c = sp.dot(d1, d1.T) / WinL
err[iWinL, nb] = sp.linalg.norm(cfull - c)
print WinL, nb, err[iWinL, nb]
sp.savez_compressed('corr_samples.npz', err=err, win_lengths=win_lengths)
print count1,
nSub = sub_data.shape[2]
cat_data = sp.zeros((nSub * sub_data.shape[0], sub_data.shape[1]))
for ind in range(nSub):
sub_data[:, :, ind] = rot_sub_data(ref=sub_data[:, :, 0],
sub=sub_data[:, :, ind])
# cat_data[sub_data.shape[0]*ind:sub_data.shape[0]*(ind+1),:] = sub_data[:,:,ind]
print ind, sub_data.shape, cat_data.shape
SC = DBSCAN(
eps=.1
) #,min_samples=50,leaf_size=50)# min_cluster_size=5) #KMeans(n_clusters=nClusters,random_state=5324)
lab_sub = sp.zeros((sub_data.shape[0], nSub))
for ind in [0]: # range(nSub):
dat1 = (0.000001 + sub_data[:, :, ind]) / sp.sum(
0.000001 + sp.sqrt(sub_data[:, :, ind] * sub_data[:, :, ind]),
axis=1)[:, None]
met = sp.absolute(sp.arccos(sp.corrcoef(dat1)))
lab_sub[:, ind] = SC.fit_predict(dat1)
print ind, sp.amax(lab_sub)
#labs_all = SC.fit_predict(cat_data)
#lab_sub=labs_all.reshape((sub_data.shape[0],nSub),order='F')
sp.savez_compressed('labs_all_data1_rot_individual_nclusters_30_HDBSCAN_sub0',
lab_sub=lab_sub,
cat_data=cat_data,
lst=lst)