def compute_rab(self):
'''
Uses the current set of ICA realizations (pytabled) to compute K*(K-1)/2 cross-correlation matrices;
they are indexed via tuples. R(a,b) is much smaller than the ICA realizations (all R(a,b) matrices
are generally smaller than ONE realization), so R(a,b) is also retained in memory. Recomputation of
the R(a,b) matrices is forced.
'''
if not os.path.exists(self.rabDirectory):
try:
os.mkdir(self.rabDirectory)
except OSError:
pass
icaFiles = sorted(os.listdir(self.icaDirectory))
if len(icaFiles) == 0:
raise RAICARICAException
for fi in icaFiles:
fiPtr = tb.open_file(os.path.join(self.icaDirectory,fi),'r')
si = fiPtr.get_node('/decomps/sources').read()
fiPtr.close()
i = np.int(deconstruct_file_name(fi)[1])
print 'Working on R(%d,b)'%i
for fj in icaFiles:
j = np.int(deconstruct_file_name(fj)[1])
if j > i:
# sources assumed to have unit std. dev. but nonzero mean - will behave badly if not!
fjPtr = tb.open_file(os.path.join(self.icaDirectory,fj),'r')
sj = fjPtr.get_node('/decomps/sources').read()
fjPtr.close()
self.RabDict[(i,j)] = np.abs(corrmatrix(si,sj))
# pickle the result
rabPtr = open(os.path.join(self.rabDirectory,'rabmatrix.db'),'wb')
cPickle.dump(self.RabDict,rabPtr,protocol=-1)
rabPtr.close()
def compute_rab(self):
'''
Uses the current set of ICA realizations (pytabled) to compute K*(K-1)/2 cross-correlation matrices;
they are indexed via tuples. R(a,b) is much smaller than the ICA realizations (all R(a,b) matrices
are generally smaller than ONE realization), so R(a,b) is also retained in memory. Recomputation of
the R(a,b) matrices is forced.
'''
if not os.path.exists(self.rabDirectory):
try:
os.mkdir(self.rabDirectory)
except OSError:
pass
icaFiles = sorted(os.listdir(self.icaDirectory))
if len(icaFiles) == 0:
raise RAICARICAException
for fi in icaFiles:
fiPtr = tb.openFile(os.path.join(self.icaDirectory,fi),'r')
si = fiPtr.getNode('/decomps/sources').read()
fiPtr.close()
i = np.int(deconstruct_file_name(fi)[1])
print 'Working on R(%d,b)'%i
for fj in icaFiles:
j = np.int(deconstruct_file_name(fj)[1])
if j > i:
# sources assumed to have unit std. dev. but nonzero mean - will behave badly if not!
fjPtr = tb.openFile(os.path.join(self.icaDirectory,fj),'r')
sj = fjPtr.getNode('/decomps/sources').read()
fjPtr.close()
self.RabDict[(i,j)] = np.abs(corrmatrix(si,sj))
# pickle the result
rabPtr = open(os.path.join(self.rabDirectory,'rabmatrix.db'),'wb')
cPickle.dump(self.RabDict,rabPtr,protocol=-1)
rabPtr.close()
def match_sources_nondegen(self, similarity, transfer):
'''
Forces nondegenerate (one tICA -> one sICA) matching.
'''
matchDict = dict()
tICAFiles = sorted(os.listdir(self.temporalDirectory))
if len(tICAFiles) == 0:
raise RAICARICAException
for tfi in tICAFiles:
if self.reportLevel > 1:
print 'Matching sources from %s' % tfi
i = np.int(deconstruct_file_name(tfi)[1])
matchDict[i] = dict()
tfiPtr = tb.openFile(os.path.join(self.temporalDirectory, tfi),
'r')
# get the corresponding sICA file
try:
sfiPtr = tb.openFile(
os.path.join(self.spatialDirectory,
construct_file_name('sICARun', i, 'h5')), 'r')
except:
raise RAICARICAException
# similarities computed here
Sij = self.simFuncs[similarity](tfiPtr, sfiPtr, transfer)
# we have the similiarity matrices. search for successive maxima
for k in xrange(self.nSignals):
bigS = Sij.max()
row, col = np.unravel_index(Sij.argmax(), Sij.shape)
matchDict[i][row] = (col, bigS)
# zero out the row/col where we matched the pair
Sij[row, :] = 0.0
Sij[:, col] = 0.0
tfiPtr.close()
sfiPtr.close()
return matchDict
def match_sources_degen(self, similarity, transfer):
'''
Allows degenerate (many tICA -> one sICA) matching.
'''
matchDict = dict()
tICAFiles = sorted(os.listdir(self.temporalDirectory))
if len(tICAFiles) == 0:
raise RAICARICAException
for tfi in tICAFiles:
if self.reportLevel > 1:
print 'Matching sources from %s' % tfi
i = np.int(deconstruct_file_name(tfi)[1])
matchDict[i] = dict()
tfiPtr = tb.openFile(os.path.join(self.temporalDirectory, tfi),
'r')
# get the corresponding sICA file
try:
sfiPtr = tb.openFile(
os.path.join(self.spatialDirectory,
construct_file_name('sICARun', i, 'h5')), 'r')
except:
raise RAICARICAException
# similarities computed here
Sij = self.simFuncs[similarity](tfiPtr, sfiPtr, transfer)
# just find all the row maxima, even if they occur in the same columns
matchInd = Sij.argmax(axis=1)
for k in xrange(self.nSignals):
matchDict[i][k] = (matchInd[k], Sij[k, matchInd[k]])
tfiPtr.close()
sfiPtr.close()
return matchDict
def compute_raicar_distribution(self):
'''
Calcluates the RAICAR distribution (histogram of rz-rz absolute cross correlation coefficients).
This can be used to set a significance bound on the BICAR component reproducibility. If this
distribution already exists, the pickled value is read and used to perform the calculations.
'''
raicarDist = list()
if not os.path.exists(self.bkgDirectory):
try:
os.mkdir(self.bkgDirectory)
except OSError:
pass
icaFiles = sorted(os.listdir(self.spatialDirectory))
if len(icaFiles) == 0:
raise BICARICAException
for fi in icaFiles:
fiPtr = tb.openFile(os.path.join(self.spatialDirectory, fi), 'r')
si = fiPtr.getNode('/decomps/sources').read()
fiPtr.close()
i = np.int(deconstruct_file_name(fi)[1])
for fj in icaFiles:
j = np.int(deconstruct_file_name(fj)[1])
if j > i:
# sources assumed to have unit std. dev. but nonzero mean - will behave badly if not!
fjPtr = tb.openFile(
os.path.join(self.spatialDirectory, fj), 'r')
sj = fjPtr.getNode('/decomps/sources').read()
fjPtr.close()
# break up a complex line
siMean = np.reshape(si.mean(axis=1), (si.shape[0], 1))
sjMean = np.reshape(sj.mean(axis=1), (sj.shape[0], 1))
flatArray = np.abs((1.0 / si.shape[1]) * np.dot(si, sj.T) -
np.dot(siMean, sjMean.T)).flatten()
raicarDist.append(flatArray)
raicarDist = np.hstack(raicarDist)
# pickle the result
rPtr = open(os.path.join(self.bkgDirectory, 'raicardist.db'), 'wb')
cPickle.dump(raicarDist, rPtr, protocol=-1)
rPtr.close()
def align_component(self, k):
'''
Uses the alignment dictionary to assemble a single aligned component, which will be subsequently
averaged to make a raicar component.
'''
gc.collect()
if not os.path.exists(self.alnDirectory):
try:
os.mkdir(self.alnDirectory)
except OSError:
pass
if len(self.alignDict) == 0:
print 'No alignment information currently in storage; trying version on disk.'
if not os.path.exists(
os.path.join(self.alnDirectory, 'alignments.db')):
raise RAICARAlignmentException
else:
alnPtr = open(os.path.join(self.alnDirectory, 'alignments.db'),
'rb')
self.alignDict = cPickle.load(alnPtr)
alnPtr.close()
if not self.alignDict.has_key(k):
print 'Error. Requested component %d does not exist.' % k
return
icaFiles = sorted(os.listdir(self.icaDirectory))
if len(icaFiles) == 0:
raise RAICARICAException
print 'Aligning component %d' % k
sourcesToAlign = []
mixColsToAlign = []
for fi in icaFiles:
print 'Working on file %s' % fi
i = np.int(deconstruct_file_name(fi)[1])
h5Ptr = tb.open_file(os.path.join(self.icaDirectory, fi), 'r')
sourcesToAlign.append(h5Ptr.root.decomps.sources[
self.alignDict[k][i], :]) # source to fetch
mixColsToAlign.append(
h5Ptr.root.decomps.mixing[:, self.
alignDict[k][i]]) # mixing element
h5Ptr.close()
# source is aligned, form the aligned source and mixing matrix
alignedSources = np.vstack(sourcesToAlign)
alignedMixing = np.vstack(mixColsToAlign).T
fileName = os.path.join(self.alnDirectory,
construct_file_name('alnRun', k, 'h5'))
h5Ptr = tb.open_file(fileName, mode="w", title='Aligned Component')
aligned = h5Ptr.create_group(h5Ptr.root, 'aligned',
'Aligned Component')
h5Ptr.create_array(aligned, 'sources', alignedSources, "S")
h5Ptr.create_array(aligned, 'mixing', alignedMixing, "A")
h5Ptr.close()
def align_component(self,k):
'''
Uses the alignment dictionary to assemble a single aligned component, which will be subsequently
averaged to make a raicar component.
'''
gc.collect()
if not os.path.exists(self.alnDirectory):
try:
os.mkdir(self.alnDirectory)
except OSError:
pass
if len(self.alignDict) == 0:
print 'No alignment information currently in storage; trying version on disk.'
if not os.path.exists(os.path.join(self.alnDirectory,'alignments.db')):
raise RAICARAlignmentException
else:
alnPtr = open(os.path.join(self.alnDirectory,'alignments.db'),'rb')
self.alignDict = cPickle.load(alnPtr)
alnPtr.close()
if not self.alignDict.has_key(k):
print 'Error. Requested component %d does not exist.' % k
return
icaFiles = sorted(os.listdir(self.icaDirectory))
if len(icaFiles) == 0:
raise RAICARICAException
print 'Aligning component %d' % k
sourcesToAlign = []
mixColsToAlign = []
for fi in icaFiles:
print 'Working on file %s' % fi
i = np.int(deconstruct_file_name(fi)[1])
h5Ptr = tb.open_file(os.path.join(self.icaDirectory,fi),'r')
sourcesToAlign.append(h5Ptr.root.decomps.sources[self.alignDict[k][i],:]) # source to fetch
mixColsToAlign.append(h5Ptr.root.decomps.mixing[:,self.alignDict[k][i]]) # mixing element
h5Ptr.close()
# source is aligned, form the aligned source and mixing matrix
alignedSources = np.vstack(sourcesToAlign)
alignedMixing = np.vstack(mixColsToAlign).T
fileName = os.path.join(self.alnDirectory,construct_file_name('alnRun',k,'h5'))
h5Ptr = tb.open_file(fileName,mode="w",title='Aligned Component')
aligned = h5Ptr.create_group(h5Ptr.root,'aligned','Aligned Component')
h5Ptr.create_array(aligned,'sources',alignedSources,"S")
h5Ptr.create_array(aligned,'mixing',alignedMixing,"A")
h5Ptr.close()
def align_component(self, k):
'''
Uses the calculated alignment dictionaries (spatial and temporal) to assemble pairs of a single
aligned component, which will be subsequently averaged to make a bicar component.
'''
if len(self.alignDict) == 0:
if self.reportLevel > 0:
print 'No temporal alignment information currently in storage; trying version on disk.'
try:
alnPtr = open(os.path.join(self.alnDirectory, 'alignments.db'),
'rb')
self.alignDict = cPickle.load(alnPtr)
alnPtr.close()
except:
raise RAICARAlignmentException
if len(self.spatialAlignDict) == 0:
if self.reportLevel > 0:
print 'No spatial alignment information currently in storage; trying version on disk.'
try:
alnPtr = open(
os.path.join(self.alnDirectory, 'spatial_alignments.db'),
'rb')
self.spatialAlignDict = cPickle.load(alnPtr)
alnPtr.close()
except:
raise RAICARAlignmentException
if not self.alignDict.has_key(k):
if self.reportLevel > 0:
print 'Error. Requested component %d does not exist.' % k
return
# temporal alignment
tICAFiles = sorted(os.listdir(self.temporalDirectory))
sICAFiles = sorted(os.listdir(self.spatialDirectory))
if len(tICAFiles) == 0 or len(sICAFiles) == 0:
raise RAICARICAException
if self.reportLevel > 0:
print 'Aligning temporal component %d' % k
sourcesToAlign = []
mixColsToAlign = []
for fi in tICAFiles:
if self.reportLevel > 1:
print 'Working on file %s' % fi
i = np.int(deconstruct_file_name(fi)[1])
h5Ptr = tb.open_file(os.path.join(self.temporalDirectory, fi), 'r')
sourcesToAlign.append(h5Ptr.root.decomps.sources[
self.alignDict[k][i], :]) # source to fetch
mixColsToAlign.append(
h5Ptr.root.decomps.mixing[:, self.
alignDict[k][i]]) # mixing element
h5Ptr.close()
# temporal source is aligned, form the aligned source and mixing matrix
alignedSources = np.vstack(sourcesToAlign)
alignedMixing = np.vstack(mixColsToAlign).T
fileName = os.path.join(self.alnDirectory,
construct_file_name('alnRun_t', k, 'h5'))
h5Ptr = tb.open_file(fileName, mode="w", title='Aligned Component')
aligned = h5Ptr.create_group(h5Ptr.root, 'aligned',
'Aligned Component')
h5Ptr.create_array(aligned, 'sources', alignedSources, "S")
h5Ptr.create_array(aligned, 'mixing', alignedMixing, "A")
h5Ptr.close()
# repeat for the spatial source
if self.reportLevel > 0:
print 'Aligning spatial component %d' % k
sourcesToAlign = []
mixColsToAlign = []
for fi in sICAFiles:
if self.reportLevel > 1:
print 'Working on file %s' % fi
i = np.int(deconstruct_file_name(fi)[1])
h5Ptr = tb.open_file(os.path.join(self.spatialDirectory, fi), 'r')
sourcesToAlign.append(h5Ptr.root.decomps.sources[
self.spatialAlignDict[k][i], :]) # source to fetch
mixColsToAlign.append(
h5Ptr.root.decomps.mixing[:, self.spatialAlignDict[k][i]]
) # mixing element
h5Ptr.close()
# spatial source is aligned, form the aligned source and mixing matrix
alignedSources = np.vstack(sourcesToAlign)
alignedMixing = np.vstack(mixColsToAlign).T
fileName = os.path.join(self.alnDirectory,
construct_file_name('alnRun_s', k, 'h5'))
h5Ptr = tb.open_file(fileName, mode="w", title='Aligned Component')
aligned = h5Ptr.create_group(h5Ptr.root, 'aligned',
'Aligned Component')
h5Ptr.create_array(aligned, 'sources', alignedSources, "S")
h5Ptr.create_array(aligned, 'mixing', alignedMixing, "A")
h5Ptr.close()
def compute_rab(self):
'''
Uses the current set of ICA realizations (pytabled) to compute K*(K-1)/2 cross-correlation matrices;
they are indexed via tuples. R(a,b) is much smaller than the ICA realizations (all R(a,b) matrices
are generally smaller than ONE realization), so R(a,b) is also retained in memory. Recomputation of
the R(a,b) matrices is forced. R(a,b) matrices from paired temporal/spatial sources are combined
using:
R(a,b) = 0.5*R_t(a,b) + 0.5*R_s(a,b)
This assumes the number of samples (timepoints in the tICA sources and locations in the sICA sources)
in the two datasets are comparable; otherwise a weighted sum should be used.
'''
if not os.path.exists(self.rabDirectory):
try:
os.mkdir(self.rabDirectory)
except OSError:
pass
if len(self.matchDict) == 0:
try:
matPtr = open(os.path.join(self.matDirectory, 'matching.db'),
'rb')
self.matchDict = cPickle.load(matPtr)
matPtr.close()
except:
raise BICARMatchingException
if self.nSignals is None:
# need the number of signals for matrix sizing, available from the matching dictionary
self.nSignals = len(zip(*self.matchDict[0])[0])
# temporal files to loop over
tICAFiles = sorted(os.listdir(self.temporalDirectory))
if len(tICAFiles) == 0:
raise RAICARICAException
for tif in tICAFiles:
i = np.int(deconstruct_file_name(tif)[1])
tiPtr = tb.openFile(os.path.join(self.temporalDirectory, tif), 'r')
if self.reportLevel > 1:
print 'Working on R(%d,b)' % i
try:
siPtr = tb.openFile(
os.path.join(self.spatialDirectory,
construct_file_name('sICARun', i, 'h5')), 'r')
except:
raise RAICARICAException
# used to link temporal and spatial sources
for tjf in tICAFiles:
j = np.int(deconstruct_file_name(tjf)[1])
if j > i:
try:
sjPtr = tb.openFile(
os.path.join(
self.spatialDirectory,
construct_file_name('sICARun', j, 'h5')), 'r')
except:
raise RAICARICAException
self.RabDict[(i, j)] = np.zeros(
(self.nSignals, self.nSignals))
# all sources assumed to have unit std. dev. but nonzero mean - will behave badly otherwise!
tjPtr = tb.openFile(
os.path.join(self.temporalDirectory, tjf), 'r')
# double loop over signals
for l in range(0, self.nSignals):
for m in range(0, self.nSignals):
# temporal cross correlation
tsi = tiPtr.root.decomps.sources[l, :]
tsj = tjPtr.root.decomps.sources[m, :]
self.RabDict[(
i, j)][l, m] += 0.5 * (np.abs(
(1.0 / len(tsi)) * np.dot(tsi, tsj)) -
tsi.mean() * tsj.mean())
# corresponding spatial cross correlation
lmatch = self.matchDict[i][l][0]
mmatch = self.matchDict[j][m][0]
ssi = siPtr.root.decomps.sources[lmatch, :]
ssj = sjPtr.root.decomps.sources[mmatch, :]
self.RabDict[(
i, j)][l, m] += 0.5 * (np.abs(
(1.0 / len(ssi)) * np.dot(ssi, ssj)) -
ssi.mean() * ssj.mean())
tjPtr.close()
sjPtr.close()
siPtr.close()
tiPtr.close()
# pickle the result
rabPtr = open(os.path.join(self.rabDirectory, 'rabmatrix.db'), 'wb')
cPickle.dump(self.RabDict, rabPtr, protocol=-1)
rabPtr.close()