def apply(self, loressignal, extraoffset):
"""
Parameters
----------
loressignal: 1D array
The aliased waveform to match
extraoffset: float
Additional offset to apply to hiressignal (e.g. for slice offset)
Returns
-------
corrfunc: 1D array
The correlation function evaluated at timepoints of timerange
"""
loresaxis = np.arange(0.0, len(loressignal)) * (
1.0 / self.lores_Fs) - self.loresstarttime
targetsignal = tide_math.corrnormalize(loressignal)
corrfunc = self.timerange * 0.0
for i in range(len(self.timerange)):
theoffset = self.timerange[i] + extraoffset
offsetkey = "{:.3f}".format(theoffset)
try:
aliasedhiressignal = self.aliasedsignals[offsetkey]
#print(offsetkey, ' - cache hit')
except KeyError:
#print(offsetkey, ' - cache miss')
self.aliasedsignals[offsetkey] = tide_math.corrnormalize(
self.tcgenerator.yfromx(loresaxis + theoffset))
aliasedhiressignal = self.aliasedsignals[offsetkey]
corrfunc[i] = np.dot(aliasedhiressignal, targetsignal)
return corrfunc
def preptc(self, thetc, isreftc=False):
# prepare timecourse by filtering, normalizing, detrending, and applying a window function
if isreftc or (not self.negativegradient):
thenormtc = tide_math.corrnormalize(
self.ncprefilter.apply(self.Fs, thetc),
detrendorder=self.detrendorder,
windowfunc=self.windowfunc,
)
else:
thenormtc = tide_math.corrnormalize(
-np.gradient(self.ncprefilter.apply(self.Fs, thetc)),
detrendorder=self.detrendorder,
windowfunc=self.windowfunc,
)
return thenormtc
def onecorrelation(thetc,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
ncprefilter,
referencetc,
usewindowfunc=True,
detrendorder=1,
windowfunc='hamming',
corrweighting='none'):
thetc_classfilter = ncprefilter.apply(oversampfreq, thetc)
thetc = thetc_classfilter
# prepare timecourse by normalizing, detrending, and applying a window function
preppedtc = tide_math.corrnormalize(thetc,
prewindow=usewindowfunc,
detrendorder=detrendorder,
windowfunc=windowfunc)
# now actually do the correlation
thexcorr = tide_corr.fastcorrelate(preppedtc, referencetc, usefft=True, weighting=corrweighting)
# find the global maximum value
theglobalmax = np.argmax(thexcorr)
return thexcorr[corrorigin - lagmininpts:corrorigin + lagmaxinpts], theglobalmax
def preptc(self, thetc):
# prepare timecourse by filtering, normalizing, detrending, and applying a window function
return tide_math.corrnormalize(
self.ncprefilter.apply(self.Fs, thetc),
detrendorder=self.detrendorder,
windowfunc="None",
)
def onecorrelation(thetc,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
ncprefilter,
referencetc,
usewindowfunc=True,
detrendorder=1,
windowfunc='hamming',
corrweighting='none'):
thetc_classfilter = ncprefilter.apply(oversampfreq, thetc)
thetc = thetc_classfilter
# prepare timecourse by normalizing, detrending, and applying a window function
preppedtc = tide_math.corrnormalize(thetc,
prewindow=usewindowfunc,
detrendorder=detrendorder,
windowfunc=windowfunc)
# now actually do the correlation
thexcorr = tide_corr.fastcorrelate(preppedtc, referencetc, usefft=True, weighting=corrweighting)
# find the global maximum value
theglobalmax = np.argmax(thexcorr)
return thexcorr[corrorigin - lagmininpts:corrorigin + lagmaxinpts], theglobalmax
def readTimeData(self, thename):
if self.isbids:
dummy, dummy, columns, indata, dummy = tide_io.readbidstsv(
self.filename)
try:
self.timedata = indata[columns.index(thename), :]
except ValueError:
print("no column named", thename, "in", columns)
self.timedata = None
return
else:
self.timedata = tide_io.readvec(self.filename)
self.length = len(self.timedata)
self.timeaxis = (
np.linspace(0.0, self.length, num=self.length, endpoint=False) /
self.samplerate) - self.starttime
self.specaxis, self.specdata = tide_filt.spectrum(
tide_math.corrnormalize(self.timedata), self.samplerate)
self.kurtosis, self.kurtosis_z, self.kurtosis_p = tide_stats.kurtosisstats(
self.timedata)
if self.verbose:
print("Timecourse data range:", np.min(self.timedata),
np.max(self.timedata))
print("sample rate:", self.samplerate)
print("Timecourse length:", self.length)
print("timeaxis length:", len(self.timeaxis))
print("kurtosis:", self.kurtosis)
print("kurtosis_z:", self.kurtosis_z)
print("kurtosis_p:", self.kurtosis_p)
print()
def shorttermcorr_1D(data1,
data2,
sampletime,
windowtime,
samplestep=1,
prewindow=False,
detrendorder=0,
windowfunc='hamming'):
"""
Parameters
----------
data1
data2
sampletime
windowtime
samplestep
prewindow
detrendorder
windowfunc
Returns
-------
"""
windowsize = int(windowtime // sampletime)
halfwindow = int((windowsize + 1) // 2)
times = []
corrpertime = []
ppertime = []
for i in range(halfwindow, np.shape(data1)[0] - halfwindow, samplestep):
dataseg1 = tide_math.corrnormalize(data1[i - halfwindow:i +
halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
dataseg2 = tide_math.corrnormalize(data2[i - halfwindow:i +
halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
thepcorr = sp.stats.stats.pearsonr(dataseg1, dataseg2)
times.append(i * sampletime)
corrpertime.append(thepcorr[0])
ppertime.append(thepcorr[1])
return np.asarray(times, dtype='float64'), np.asarray(
corrpertime, dtype='float64'), np.asarray(ppertime, dtype='float64')
def aliasedcorrelate(
hiressignal,
hires_Fs,
lowressignal,
lowres_Fs,
timerange,
hiresstarttime=0.0,
lowresstarttime=0.0,
padtime=30.0,
):
"""Perform an aliased correlation.
This function is deprecated, and is retained here as a reference against
which to test AliasedCorrelator.
Parameters
----------
hiressignal: 1D array
The unaliased waveform to match
hires_Fs: float
The sample rate of the unaliased waveform
lowressignal: 1D array
The aliased waveform to match
lowres_Fs: float
The sample rate of the aliased waveform
timerange: 1D array
The delays for which to calculate the correlation function
Returns
-------
corrfunc: 1D array
The correlation function evaluated at timepoints of timerange
"""
highresaxis = np.arange(
0.0, len(hiressignal)) * (1.0 / hires_Fs) - hiresstarttime
lowresaxis = np.arange(
0.0, len(lowressignal)) * (1.0 / lowres_Fs) - lowresstarttime
tcgenerator = tide_resample.FastResampler(highresaxis,
hiressignal,
padtime=padtime)
targetsignal = tide_math.corrnormalize(lowressignal)
corrfunc = timerange * 0.0
for i in range(len(timerange)):
aliasedhiressignal = tide_math.corrnormalize(
tcgenerator.yfromx(lowresaxis + timerange[i]))
corrfunc[i] = np.dot(aliasedhiressignal, targetsignal)
return corrfunc
def shorttermcorr_1D(
data1, data2, sampletime, windowtime, samplestep=1, detrendorder=0, windowfunc="hamming",
):
"""Calculate short-term sliding-window correlation between two 1D arrays.
Parameters
----------
data1
data2
sampletime
windowtime
samplestep
detrendorder
windowfunc
Returns
-------
times
corrpertime
ppertime
"""
windowsize = int(windowtime // sampletime)
halfwindow = int((windowsize + 1) // 2)
times = []
corrpertime = []
ppertime = []
for i in range(halfwindow, np.shape(data1)[0] - halfwindow, samplestep):
dataseg1 = tide_math.corrnormalize(
data1[i - halfwindow : i + halfwindow],
detrendorder=detrendorder,
windowfunc=windowfunc,
)
dataseg2 = tide_math.corrnormalize(
data2[i - halfwindow : i + halfwindow],
detrendorder=detrendorder,
windowfunc=windowfunc,
)
thepcorr = sp.stats.stats.pearsonr(dataseg1, dataseg2)
times.append(i * sampletime)
corrpertime.append(thepcorr[0])
ppertime.append(thepcorr[1])
return (
np.asarray(times, dtype="float64"),
np.asarray(corrpertime, dtype="float64"),
np.asarray(ppertime, dtype="float64"),
)
def _get_null_distribution(indata, xcorr_x, thefilter, prewindow, detrendorder,
searchstart, searchend, Fs, dofftcorr,
windowfunc='hamming', corrweighting='none',
numreps=1000):
"""
Get an empirical null distribution from the data.
"""
print('estimating significance distribution using {0} '
'repetitions'.format(numreps))
corrlist = zeros(numreps, dtype='float')
corrlist_pear = zeros(numreps, dtype='float')
xcorr_x_trim = xcorr_x[searchstart:searchend + 1]
filteredindata = tide_math.corrnormalize(thefilter.apply(Fs, indata),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
for i in range(numreps):
# make a shuffled copy of the regressors
shuffleddata = permutation(indata)
# filter it
filteredshuffleddata = np.nan_to_num(
tide_math.corrnormalize(thefilter.apply(Fs, shuffleddata),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc))
# crosscorrelate with original
theshuffledxcorr = tide_corr.fastcorrelate(filteredindata,
filteredshuffleddata,
usefft=dofftcorr,
weighting=corrweighting)
# find and tabulate correlation coefficient at optimal lag
theshuffledxcorr_trim = theshuffledxcorr[searchstart:searchend + 1]
maxdelay = xcorr_x_trim[argmax(theshuffledxcorr_trim)]
corrlist[i] = theshuffledxcorr_trim[argmax(theshuffledxcorr_trim)]
# find and tabulate correlation coefficient at 0 lag
corrlist_pear[i] = pearsonr(filteredindata, filteredshuffleddata)[0]
# return the distribution data
return corrlist, corrlist_pear
def aliasedcorrelate(hiressignal,
hires_Fs,
lowressignal,
lowres_Fs,
timerange,
hiresstarttime=0.0,
lowresstarttime=0.0,
padvalue=30.0):
"""
Parameters
----------
hiressignal: 1D array
The unaliased waveform to match
hires_Fs: float
The sample rate of the unaliased waveform
lowressignal: 1D array
The aliased waveform to match
lowres_Fs: float
The sample rate of the aliased waveform
timerange: 1D array
The delays for which to calculate the correlation function
Returns
-------
corrfunc: 1D array
The correlation function evaluated at timepoints of timerange
"""
highresaxis = np.arange(
0.0, len(hiressignal)) * (1.0 / hires_Fs) - hiresstarttime
lowresaxis = np.arange(
0.0, len(lowressignal)) * (1.0 / lowres_Fs) - lowresstarttime
tcgenerator = tide_resample.fastresampler(highresaxis,
hiressignal,
padvalue=padvalue)
targetsignal = tide_math.corrnormalize(lowressignal)
corrfunc = timerange * 0.0
for i in range(len(timerange)):
aliasedhiressignal = tide_math.corrnormalize(
tcgenerator.yfromx(lowresaxis + timerange[i]))
corrfunc[i] = np.dot(aliasedhiressignal, targetsignal)
return corrfunc
def quickcorr(data1, data2, windowfunc='hamming'):
"""
Parameters
----------
data1
data2
windowfunc
Returns
-------
"""
thepcorr = sp.stats.stats.pearsonr(tide_math.corrnormalize(data1,
prewindow=True,
detrendorder=1,
windowfunc=windowfunc),
tide_math.corrnormalize(data2,
prewindow=True,
detrendorder=1,
windowfunc=windowfunc))
return thepcorr
def arbcorr(
input1,
Fs1,
input2,
Fs2,
start1=0.0,
start2=0.0,
windowfunc="hamming",
method="univariate",
debug=False,
):
"""Calculate something."""
if Fs1 > Fs2:
corrFs = Fs1
matchedinput1 = input1
matchedinput2 = tide_resample.upsample(input2, Fs2, corrFs, method=method, debug=debug)
elif Fs2 > Fs1:
corrFs = Fs2
matchedinput1 = tide_resample.upsample(input1, Fs1, corrFs, method=method, debug=debug)
matchedinput2 = input2
else:
corrFs = Fs1
matchedinput1 = input1
matchedinput2 = input2
norm1 = tide_math.corrnormalize(matchedinput1, detrendorder=1, windowfunc=windowfunc)
norm2 = tide_math.corrnormalize(matchedinput2, detrendorder=1, windowfunc=windowfunc)
thexcorr_y = signal.fftconvolve(norm1, norm2[::-1], mode="full")
thexcorr_x = (
np.linspace(0.0, len(thexcorr_y) / corrFs, num=len(thexcorr_y), endpoint=False)
- (len(norm1) // 2 + len(norm2) // 2) / corrFs
+ start1
- start2
)
zeroloc = int(np.argmin(np.fabs(thexcorr_x)))
LGR.debug(f"len(norm1) = {len(norm1)}")
LGR.debug(f"len(norm2) = {len(norm2)}")
LGR.debug(f"len(thexcorr_y) = {len(thexcorr_y)}")
LGR.debug(f"zeroloc = {zeroloc}")
return thexcorr_x, thexcorr_y, corrFs, zeroloc
def shorttermcorr_1D(data1, data2, sampletime, windowtime, samplestep=1, prewindow=False, detrendorder=0,
windowfunc='hamming'):
"""
Parameters
----------
data1
data2
sampletime
windowtime
samplestep
prewindow
detrendorder
windowfunc
Returns
-------
"""
windowsize = int(windowtime // sampletime)
halfwindow = int((windowsize + 1) // 2)
times = []
corrpertime = []
ppertime = []
for i in range(halfwindow, np.shape(data1)[0] - halfwindow, samplestep):
dataseg1 = tide_math.corrnormalize(data1[i - halfwindow:i + halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
dataseg2 = tide_math.corrnormalize(data2[i - halfwindow:i + halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
thepcorr = sp.stats.stats.pearsonr(dataseg1, dataseg2)
times.append(i * sampletime)
corrpertime.append(thepcorr[0])
ppertime.append(thepcorr[1])
return np.asarray(times, dtype='float64'), np.asarray(corrpertime, dtype='float64'), np.asarray(ppertime,
dtype='float64')
def quickcorr(data1, data2, windowfunc='hamming'):
"""
Parameters
----------
data1
data2
windowfunc
Returns
-------
"""
thepcorr = sp.stats.stats.pearsonr(
tide_math.corrnormalize(data1,
prewindow=True,
detrendorder=1,
windowfunc=windowfunc),
tide_math.corrnormalize(data2,
prewindow=True,
detrendorder=1,
windowfunc=windowfunc))
return thepcorr
def shorttermcorr_2D(data1,
data2,
sampletime,
windowtime,
samplestep=1,
laglimit=None,
weighting='none',
prewindow=False,
windowfunc='hamming',
detrendorder=0,
display=False):
"""
Parameters
----------
data1
data2
sampletime
windowtime
samplestep
laglimit
weighting
prewindow
windowfunc
detrendorder
display
Returns
-------
"""
windowsize = int(windowtime // sampletime)
halfwindow = int((windowsize + 1) // 2)
if laglimit is None:
laglimit = windowtime / 2.0
dataseg1 = tide_math.corrnormalize(data1[0:2 * halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
dataseg2 = tide_math.corrnormalize(data2[0:2 * halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
thexcorr = fastcorrelate(dataseg1, dataseg2, weighting=weighting)
xcorrlen = np.shape(thexcorr)[0]
xcorr_x = np.arange(0.0, xcorrlen) * sampletime - (
xcorrlen * sampletime) / 2.0 + sampletime / 2.0
corrzero = int(xcorrlen // 2)
xcorrpertime = []
times = []
Rvals = []
delayvals = []
valid = []
for i in range(halfwindow, np.shape(data1)[0] - halfwindow, samplestep):
dataseg1 = tide_math.corrnormalize(data1[i - halfwindow:i +
halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
dataseg2 = tide_math.corrnormalize(data2[i - halfwindow:i +
halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
times.append(i * sampletime)
xcorrpertime.append(
fastcorrelate(dataseg1, dataseg2, weighting=weighting))
maxindex, thedelayval, theRval, maxsigma, maskval, failreason, peakstart, peakend = tide_fit.findmaxlag_gauss(
xcorr_x,
xcorrpertime[-1],
-laglimit,
laglimit,
1000.0,
refine=True,
useguess=False,
fastgauss=False,
displayplots=False)
delayvals.append(thedelayval)
Rvals.append(theRval)
if failreason == 0:
valid.append(1)
else:
valid.append(0)
if display:
pl.imshow(xcorrpertime)
return np.asarray(times, dtype='float64'), \
np.asarray(xcorrpertime, dtype='float64'), \
np.asarray(Rvals, dtype='float64'), \
np.asarray(delayvals, dtype='float64'), \
np.asarray(valid, dtype='float64')
def test_delayestimation(display=False, debug=False):
# set the number of MKL threads to use
if mklexists:
print("disabling MKL")
mkl.set_num_threads(1)
# set parameters
Fs = 10.0
numpoints = 5000
numlocs = 21
refnum = int(numlocs // 2)
timestep = 0.228764
oversampfac = 2
detrendorder = 1
oversampfreq = Fs * oversampfac
corrtr = 1.0 / oversampfreq
smoothingtime = 1.0
bipolar = False
interptype = "univariate"
lagmod = 1000.0
lagmin = -20.0
lagmax = 20.0
lagmininpts = int((-lagmin / corrtr) - 0.5)
lagmaxinpts = int((lagmax / corrtr) + 0.5)
peakfittype = "gauss"
corrweighting = "None"
similaritymetric = "hybrid"
windowfunc = "hamming"
chunksize = 5
pedestal = 100.0
# set up the filter
theprefilter = tide_filt.NoncausalFilter("arb",
transferfunc="brickwall",
debug=False)
theprefilter.setfreqs(0.009, 0.01, 0.15, 0.16)
# construct the various test waveforms
timepoints = np.linspace(0.0,
numpoints / Fs,
num=numpoints,
endpoint=False)
oversamptimepoints = np.linspace(0.0,
numpoints / Fs,
num=oversampfac * numpoints,
endpoint=False)
waveforms = np.zeros((numlocs, numpoints), dtype=np.float64)
paramlist = [
[0.314, 0.055457, 0.0],
[-0.723, 0.08347856, np.pi],
[-0.834, 0.1102947, 0.0],
[1.0, 0.13425, 0.5],
]
offsets = np.zeros(numlocs, dtype=np.float64)
amplitudes = np.ones(numlocs, dtype=np.float64)
for i in range(numlocs):
offsets[i] = timestep * (i - refnum)
waveforms[i, :] = multisine(timepoints - offsets[i],
paramlist) + pedestal
if display:
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
for i in range(numlocs):
ax.plot(timepoints, waveforms[i, :])
plt.show()
threshval = pedestal / 4.0
waveforms = numpy2shared(waveforms, np.float64)
referencetc = tide_resample.doresample(timepoints,
waveforms[refnum, :],
oversamptimepoints,
method=interptype)
referencetc = theprefilter.apply(oversampfreq, referencetc)
referencetc = tide_math.corrnormalize(referencetc,
detrendorder=detrendorder,
windowfunc=windowfunc)
# set up theCorrelator
if debug:
print("\n\nsetting up theCorrelator")
theCorrelator = tide_classes.Correlator(
Fs=oversampfreq,
ncprefilter=theprefilter,
detrendorder=detrendorder,
windowfunc=windowfunc,
corrweighting=corrweighting,
debug=True,
)
theCorrelator.setreftc(
np.zeros((oversampfac * numpoints), dtype=np.float64))
theCorrelator.setlimits(lagmininpts, lagmaxinpts)
dummy, trimmedcorrscale, dummy = theCorrelator.getfunction()
corroutlen = np.shape(trimmedcorrscale)[0]
internalvalidcorrshape = (numlocs, corroutlen)
corrout, dummy, dummy = allocshared(internalvalidcorrshape, np.float64)
meanval, dummy, dummy = allocshared((numlocs), np.float64)
if debug:
print("corrout shape:", corrout.shape)
print("theCorrelator: corroutlen=", corroutlen)
# set up theMutualInformationator
if debug:
print("\n\nsetting up theMutualInformationator")
theMutualInformationator = tide_classes.MutualInformationator(
Fs=oversampfreq,
smoothingtime=smoothingtime,
ncprefilter=theprefilter,
detrendorder=detrendorder,
windowfunc=windowfunc,
madnorm=False,
lagmininpts=lagmininpts,
lagmaxinpts=lagmaxinpts,
debug=False,
)
theMutualInformationator.setreftc(
np.zeros((oversampfac * numpoints), dtype=np.float64))
theMutualInformationator.setlimits(lagmininpts, lagmaxinpts)
# set up thefitter
if debug:
print("\n\nsetting up thefitter")
thefitter = tide_classes.SimilarityFunctionFitter(
lagmod=lagmod,
lthreshval=0.0,
uthreshval=1.0,
bipolar=bipolar,
lagmin=lagmin,
lagmax=lagmax,
absmaxsigma=10000.0,
absminsigma=0.01,
debug=False,
peakfittype=peakfittype,
)
lagtc, dummy, dummy = allocshared(waveforms.shape, np.float64)
fitmask, dummy, dummy = allocshared((numlocs), "uint16")
failreason, dummy, dummy = allocshared((numlocs), "uint32")
lagtimes, dummy, dummy = allocshared((numlocs), np.float64)
lagstrengths, dummy, dummy = allocshared((numlocs), np.float64)
lagsigma, dummy, dummy = allocshared((numlocs), np.float64)
gaussout, dummy, dummy = allocshared(internalvalidcorrshape, np.float64)
windowout, dummy, dummy = allocshared(internalvalidcorrshape, np.float64)
rvalue, dummy, dummy = allocshared((numlocs), np.float64)
r2value, dummy, dummy = allocshared((numlocs), np.float64)
fitcoff, dummy, dummy = allocshared((numlocs), np.float64)
fitNorm, dummy, dummy = allocshared((numlocs), np.float64)
R2, dummy, dummy = allocshared((numlocs), np.float64)
movingsignal, dummy, dummy = allocshared(waveforms.shape, np.float64)
filtereddata, dummy, dummy = allocshared(waveforms.shape, np.float64)
for nprocs in [4, 1]:
# call correlationpass
if debug:
print("\n\ncalling correlationpass")
print("waveforms shape:", waveforms.shape)
(
voxelsprocessed_cp,
theglobalmaxlist,
trimmedcorrscale,
) = tide_calcsimfunc.correlationpass(
waveforms[:, :],
referencetc,
theCorrelator,
timepoints,
oversamptimepoints,
lagmininpts,
lagmaxinpts,
corrout,
meanval,
nprocs=nprocs,
alwaysmultiproc=False,
oversampfactor=oversampfac,
interptype=interptype,
showprogressbar=False,
chunksize=chunksize,
)
if debug:
print(voxelsprocessed_cp, len(theglobalmaxlist),
len(trimmedcorrscale))
if display:
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
for i in range(numlocs):
ax.plot(trimmedcorrscale, corrout[i, :])
plt.show()
# call peakeval
if debug:
print("\n\ncalling peakeval")
voxelsprocessed_pe, thepeakdict = tide_peakeval.peakevalpass(
waveforms[:, :],
referencetc,
timepoints,
oversamptimepoints,
theMutualInformationator,
trimmedcorrscale,
corrout,
nprocs=nprocs,
alwaysmultiproc=False,
bipolar=bipolar,
oversampfactor=oversampfac,
interptype=interptype,
showprogressbar=False,
chunksize=chunksize,
)
if debug:
for key in thepeakdict:
print(key, thepeakdict[key])
# call thefitter
if debug:
print("\n\ncalling fitter")
thefitter.setfunctype(similaritymetric)
thefitter.setcorrtimeaxis(trimmedcorrscale)
genlagtc = tide_resample.FastResampler(timepoints,
waveforms[refnum, :])
if display:
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
if nprocs == 1:
proctype = "singleproc"
else:
proctype = "multiproc"
for peakfittype in ["fastgauss", "quad", "fastquad", "gauss"]:
thefitter.setpeakfittype(peakfittype)
voxelsprocessed_fc = tide_simfuncfit.fitcorr(
genlagtc,
timepoints,
lagtc,
trimmedcorrscale,
thefitter,
corrout,
fitmask,
failreason,
lagtimes,
lagstrengths,
lagsigma,
gaussout,
windowout,
R2,
peakdict=thepeakdict,
nprocs=nprocs,
alwaysmultiproc=False,
fixdelay=None,
showprogressbar=False,
chunksize=chunksize,
despeckle_thresh=100.0,
initiallags=None,
)
if debug:
print(voxelsprocessed_fc)
if debug:
print("\npeakfittype:", peakfittype)
for i in range(numlocs):
print(
"location",
i,
":",
offsets[i],
lagtimes[i],
lagtimes[i] - offsets[i],
lagstrengths[i],
lagsigma[i],
)
if display:
ax.plot(offsets, lagtimes, label=peakfittype)
if checkfits(lagtimes, offsets, tolerance=0.01):
print(proctype, peakfittype, " lagtime: pass")
assert True
else:
print(proctype, peakfittype, " lagtime: fail")
assert False
if checkfits(lagstrengths, amplitudes, tolerance=0.05):
print(proctype, peakfittype, " lagstrength: pass")
assert True
else:
print(proctype, peakfittype, " lagstrength: fail")
assert False
if display:
ax.legend()
plt.show()
filteredwaveforms, dummy, dummy = allocshared(waveforms.shape, np.float64)
for i in range(numlocs):
filteredwaveforms[i, :] = theprefilter.apply(Fs, waveforms[i, :])
for nprocs in [4, 1]:
voxelsprocessed_glm = tide_glmpass.glmpass(
numlocs,
waveforms[:, :],
threshval,
lagtc,
meanval,
rvalue,
r2value,
fitcoff,
fitNorm,
movingsignal,
filtereddata,
nprocs=nprocs,
alwaysmultiproc=False,
showprogressbar=False,
mp_chunksize=chunksize,
)
if nprocs == 1:
proctype = "singleproc"
else:
proctype = "multiproc"
diffsignal = filtereddata
fig = plt.figure()
ax = fig.add_subplot(1, 1, 1)
# ax.plot(timepoints, filtereddata[refnum, :], label='filtereddata')
ax.plot(oversamptimepoints, referencetc, label="referencetc")
ax.plot(timepoints, movingsignal[refnum, :], label="movingsignal")
ax.legend()
plt.show()
print(proctype, "glmpass", np.mean(diffsignal),
np.max(np.fabs(diffsignal)))
def shorttermcorr_2D(
data1,
data2,
sampletime,
windowtime,
samplestep=1,
laglimits=None,
weighting="None",
zeropadding=0,
windowfunc="None",
detrendorder=0,
display=False,
):
"""Calculate short-term sliding-window correlation between two 2D arrays.
Parameters
----------
data1
data2
sampletime
windowtime
samplestep
laglimits
weighting
zeropadding
windowfunc
detrendorder
display
Returns
-------
times
xcorrpertime
Rvals
delayvals
valid
"""
windowsize = int(windowtime // sampletime)
halfwindow = int((windowsize + 1) // 2)
if laglimits is not None:
lagmin = laglimits[0]
lagmax = laglimits[1]
else:
lagmin = -windowtime / 2.0
lagmax = windowtime / 2.0
LGR.debug(f"lag limits: {lagmin} {lagmax}")
"""dt = np.diff(time)[0] # In days...
fs = 1.0 / dt
nfft = nperseg
noverlap = (nperseg - 1)"""
dataseg1 = tide_math.corrnormalize(
data1[0 : 2 * halfwindow], detrendorder=detrendorder, windowfunc=windowfunc
)
dataseg2 = tide_math.corrnormalize(
data2[0 : 2 * halfwindow], detrendorder=detrendorder, windowfunc=windowfunc
)
thexcorr = fastcorrelate(dataseg1, dataseg2, weighting=weighting, zeropadding=zeropadding)
xcorrlen = np.shape(thexcorr)[0]
xcorr_x = (
np.arange(0.0, xcorrlen) * sampletime - (xcorrlen * sampletime) / 2.0 + sampletime / 2.0
)
xcorrpertime = []
times = []
Rvals = []
delayvals = []
valid = []
for i in range(halfwindow, np.shape(data1)[0] - halfwindow, samplestep):
dataseg1 = tide_math.corrnormalize(
data1[i - halfwindow : i + halfwindow],
detrendorder=detrendorder,
windowfunc=windowfunc,
)
dataseg2 = tide_math.corrnormalize(
data2[i - halfwindow : i + halfwindow],
detrendorder=detrendorder,
windowfunc=windowfunc,
)
times.append(i * sampletime)
xcorrpertime.append(
fastcorrelate(dataseg1, dataseg2, weighting=weighting, zeropadding=zeropadding)
)
(
maxindex,
thedelayval,
theRval,
maxsigma,
maskval,
failreason,
peakstart,
peakend,
) = tide_fit.findmaxlag_gauss(
xcorr_x,
xcorrpertime[-1],
lagmin,
lagmax,
1000.0,
refine=True,
useguess=False,
fastgauss=False,
displayplots=False,
)
delayvals.append(thedelayval)
Rvals.append(theRval)
if failreason == 0:
valid.append(1)
else:
valid.append(0)
if display:
plt.imshow(xcorrpertime)
return (
np.asarray(times, dtype="float64"),
np.asarray(xcorrpertime, dtype="float64"),
np.asarray(Rvals, dtype="float64"),
np.asarray(delayvals, dtype="float64"),
np.asarray(valid, dtype="float64"),
)
def showxcorrx_workflow(infilename1, infilename2, Fs,
thelabel='', starttime=0., duration=1000000.,
searchrange=15.,
display=True, trimdata=False,
summarymode=False, labelline=False,
flipregressor=False, windowfunc='hamming',
calccepstraldelay=False, corroutputfile=False,
controlvariablefile=None, numreps=0,
arbvec=None, filtertype='arb', corrweighting='none',
detrendorder=1, prewindow=True, verbose=False):
r"""Calculate and display crosscorrelation between two timeseries.
Parameters
----------
infilename1 : str
The name of a text file containing a timeseries, one timepoint per line.
infilename2 : str
The name of a text file containing a timeseries, one timepoint per line.
Fs : float
The sample rate of the time series, in Hz.
thelabel : str, optional
The label for the output graph. Default is blank.
starttime : float, optional
Time offset into the timeseries, in seconds, to start using the time data. Default is 0
duration : float, optional
Length of time from each time series, in seconds, to use for the cross-correlation. Default is the entire time series.
searchrange : float, optional
Only search for cross-correlation peaks between -searchrange and +searchrange seconds (default is 15).
display : bool, optional
Plot cross-correlation function in a matplotlib window. Default is True.
trimdata : bool, optional
Trim time series to the length of the shorter series. Default is False.
summarymode : bool, optional
Output a table of interesting results for later processing. Default is False.
labelline : bool, optional
Print an explanatory header line over the summary information. Default is False.
flipregressor : bool, optional
Invert timeseries 2 prior to cross-correlation.
windowfunc : {'hamming', 'hann', 'blackmanharris'}
Window function to apply prior to cross-correlation. Default is 'hamming'.
calccepstraldelay : bool, optional
Use cepstral estimation of delay. Default is False.
corroutputfile : bool, optional
Save the correlation function to a file. Default is False.
controlvariablefile : bool, optional
Save internal variables to a text file. Default is False.
numreps : int, optional
Number of null correlations to perform to estimate significance. Default is 10000
arbvec : [float,float,float,float], optional
Frequency limits of the arb_pass filter.
filtertype : 'none', 'card', 'lfo', 'vlf', 'resp', 'arb'
Type of filter to apply data prior to correlation. Default is 'none'
corrweighting : {'none', 'Liang', 'Eckart', 'PHAT'}, optional
Weighting function to apply to the crosscorrelation in the Fourier domain. Default is 'none'
detrendorder : int, optional
Order of polynomial used to detrend crosscorrelation inputs. Default is 1 (0 disables)
prewindow : bool, optional
Apply window function prior to cross-correlation. Default is True.
verbose : bool, optional
Print internal status information. Default is False.
Notes
-----
This workflow writes out several files:
If corroutputfile is defined:
====================== =================================================
Filename Content
====================== =================================================
corrlist.txt A file
corrlist_pear.txt A file
[corroutputfile] Correlation function
====================== =================================================
If debug is True:
====================== =================================================
Filename Content
====================== =================================================
filtereddata1.txt Something
filtereddata2.txt Something
====================== =================================================
"""
# Constants that could be arguments
dofftcorr = True
writecorrlists = False
debug = False
showpearson = True
# These are unnecessary and should be simplified
dopartial = bool(controlvariablefile)
uselabel = bool(thelabel)
dumpfiltered = bool(debug)
if labelline:
# TS: should prob reflect this in the parser, but it's not a big deal
summarymode = True
if numreps == 0:
estimate_significance = False
else:
estimate_significance = True
savecorrelation = bool(corroutputfile)
theprefilter = tide_filt.noncausalfilter()
if arbvec is not None and filtertype != 'arb':
raise ValueError('Argument arbvec must be None if filtertype is '
'not arb')
if arbvec is not None:
if len(arbvec) == 2:
arb_lower = float(arbvec[0])
arb_upper = float(arbvec[1])
arb_lowerstop = 0.9 * float(arbvec[0])
arb_upperstop = 1.1 * float(arbvec[1])
elif len(arbvec) == 4:
arb_lower = float(arbvec[0])
arb_upper = float(arbvec[1])
arb_lowerstop = float(arbvec[2])
arb_upperstop = float(arbvec[3])
theprefilter.settype('arb')
theprefilter.setarb(arb_lowerstop, arb_lower, arb_upper, arb_upperstop)
else:
theprefilter.settype(filtertype)
inputdata1 = tide_io.readvec(infilename1)
inputdata2 = tide_io.readvec(infilename2)
numpoints = len(inputdata1)
startpoint1 = max([int(starttime * Fs), 0])
if debug:
print('startpoint set to ', startpoint1)
endpoint1 = min([startpoint1 + int(duration * Fs), int(len(inputdata1))])
if debug:
print('endpoint set to ', endpoint1)
endpoint2 = min([int(duration * Fs), int(len(inputdata1)),
int(len(inputdata2))])
trimdata1 = inputdata1[startpoint1:endpoint1]
trimdata2 = inputdata2[0:endpoint2]
if trimdata:
minlen = np.min([len(trimdata1), len(trimdata2)])
trimdata1 = trimdata1[0:minlen]
trimdata2 = trimdata2[0:minlen]
# band limit the regressor if that is needed
if theprefilter.gettype() != 'none':
if verbose:
print("filtering to ", theprefilter.gettype(), " band")
print(windowfunc)
filtereddata1 = tide_math.corrnormalize(theprefilter.apply(Fs, trimdata1),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
filtereddata2 = tide_math.corrnormalize(theprefilter.apply(Fs, trimdata2),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
if flipregressor:
filtereddata2 *= -1.0
if dumpfiltered:
tide_io.writenpvecs(filtereddata1, 'filtereddata1.txt')
tide_io.writenpvecs(filtereddata2, 'filtereddata2.txt')
if dopartial:
controlvars = tide_io.readvecs(controlvariablefile)
numregressors = len(controlvars) # Added by TS. Not sure if works.
regressorvec = []
for j in range(0, numregressors):
regressorvec.append(tide_math.corrnormalize(
theprefilter.apply(Fs, controlvars[j, :]),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc))
if (np.max(filtereddata1) - np.min(filtereddata1)) > 0.0:
thefit, filtereddata1 = tide_fit.mlregress(regressorvec,
filtereddata1)
if (np.max(filtereddata2) - np.min(filtereddata2)) > 0.0:
thefit, filtereddata2 = tide_fit.mlregress(regressorvec,
filtereddata2)
thexcorr = tide_corr.fastcorrelate(filtereddata1, filtereddata2,
usefft=dofftcorr,
weighting=corrweighting,
displayplots=debug)
if calccepstraldelay:
cepdelay = tide_corr.cepstraldelay(filtereddata1, filtereddata2,
1.0 / Fs, displayplots=display)
cepcoff = tide_corr.delayedcorr(filtereddata1, filtereddata2, cepdelay,
1.0 / Fs)
print('cepstral delay time is {0}, correlation is {1}'.format(cepdelay,
cepcoff))
thepxcorr = pearsonr(filtereddata1, filtereddata2)
# calculate the coherence
f, Cxy = sp.signal.coherence(
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata1), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata2), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
Fs)
# calculate the cross spectral density
f, Pxy = sp.signal.csd(
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata1), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata2), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
Fs)
xcorrlen = len(thexcorr)
sampletime = 1.0 / Fs
xcorr_x = r_[0:xcorrlen] * sampletime - (xcorrlen * sampletime) / 2.0\
+ sampletime / 2.0
halfwindow = int(searchrange * Fs)
corrzero = xcorrlen // 2
searchstart = corrzero - halfwindow
searchend = corrzero + halfwindow
xcorr_x_trim = xcorr_x[searchstart:searchend + 1]
thexcorr_trim = thexcorr[searchstart:searchend + 1]
if debug:
print('searching for peak correlation over range ', searchstart,
searchend)
maxdelay = xcorr_x_trim[argmax(thexcorr_trim)]
if debug:
print('maxdelay before refinement', maxdelay)
dofindmaxlag = True
if dofindmaxlag:
print('executing findmaxlag')
(maxindex, maxdelay, maxval, maxsigma, maskval, failreason, peakstart,
peakend) = tide_fit.findmaxlag_gauss(
xcorr_x_trim, thexcorr_trim, -searchrange, searchrange, 1000.0,
refine=True,
useguess=False,
fastgauss=False,
displayplots=False)
print(maxindex, maxdelay, maxval, maxsigma, maskval, failreason)
R = maxval
if debug:
print('maxdelay after refinement', maxdelay)
if failreason > 0:
print('failreason =', failreason)
else:
R = thexcorr_trim[argmax(thexcorr_trim)]
# set the significance threshold
if estimate_significance:
# generate a list of correlations from shuffled data
(corrlist,
corrlist_pear) = _get_null_distribution(trimdata1, xcorr_x,
theprefilter, prewindow,
detrendorder, searchstart,
searchend, Fs, dofftcorr,
corrweighting=corrweighting,
numreps=numreps,
windowfunc=windowfunc)
# calculate percentiles for the crosscorrelation from the distribution
histlen = 100
thepercentiles = [0.95, 0.99, 0.995]
(pcts, pcts_fit,
histfit) = tide_stats.sigFromDistributionData(corrlist, histlen,
thepercentiles)
if debug:
tide_stats.printthresholds(pcts, thepercentiles,
('Crosscorrelation significance '
'thresholds from data:'))
tide_stats.printthresholds(pcts_fit, thepercentiles,
('Crosscorrelation significance '
'thresholds from fit:'))
# calculate significance for the pearson correlation
(pearpcts, pearpcts_fit,
histfit) = tide_stats.sigFromDistributionData(corrlist_pear, histlen,
thepercentiles)
if debug:
tide_stats.printthresholds(pearpcts, thepercentiles,
('Pearson correlation significance '
'thresholds from data:'))
tide_stats.printthresholds(pearpcts_fit, thepercentiles,
('Pearson correlation significance '
'thresholds from fit:'))
if writecorrlists:
tide_io.writenpvecs(corrlist, 'corrlist.txt')
tide_io.writenpvecs(corrlist_pear, 'corrlist_pear.txt')
def printthresholds(pcts, thepercentiles, labeltext):
print(labeltext)
for i in range(0, len(pcts)):
print('\tp <', "{:.3f}".format(1.0 - thepercentiles[i]), ': ',
pcts[i])
# report the pearson correlation
if showpearson and verbose:
print('Pearson_R:\t', thepxcorr[0])
if estimate_significance:
for idx, percentile in enumerate(thepercentiles):
print(' pear_p(', "{:.3f}".format(1.0 - percentile), '):\t',
pearpcts[idx])
print("")
if debug:
print(thepxcorr)
if verbose:
if uselabel:
print(thelabel, ":\t", maxdelay)
else:
print("Crosscorrelation_Rmax:\t", R)
print("Crosscorrelation_maxdelay:\t", maxdelay)
if estimate_significance:
for idx, percentile in enumerate(thepercentiles):
print(' xc_p(', "{:.3f}".format(1.0 - percentile),
'):\t', pcts[idx])
print(infilename1, "[0 seconds] == ", infilename2, "[",
-1 * maxdelay, " seconds]")
if summarymode:
if estimate_significance:
if uselabel:
if labelline:
print('thelabel', 'pearson_R', 'pearson_R(p=0.05)',
'xcorr_R', 'xcorr_R(P=0.05)', 'xcorr_maxdelay')
print(thelabel, thepxcorr[0], pearpcts_fit[0], R, pcts_fit[0],
-1 * maxdelay)
else:
if labelline:
print('pearson_R', 'pearson_R(p=0.05)', 'xcorr_R',
'xcorr_R(P=0.05)', 'xcorr_maxdelay')
print(thepxcorr[0], pearpcts_fit[0], R, pcts_fit[0],
-1 * maxdelay)
else:
if uselabel:
if labelline:
print('thelabel', 'pearson_r', 'pearson_p', 'xcorr_R',
'xcorr_maxdelay')
print(thelabel, thepxcorr[0], thepxcorr[1], R, -1 * maxdelay)
else:
if labelline:
print('pearson_r\tpearson_p\txcorr_R\txcorr_t\t'
'xcorr_maxdelay')
print(thepxcorr[0], '\t', thepxcorr[1], '\t', R, '\t',
-1 * maxdelay)
if savecorrelation:
tide_io.writenpvecs(np.stack((xcorr_x, thexcorr), axis=0),
corroutputfile)
if display:
fig, ax = plt.subplots()
# ax.set_title('GCC')
ax.plot(xcorr_x, thexcorr, 'k')
if debug:
fig, ax = plt.subplots()
ax.plot(f, Cxy)
fig = plt.subplots()
ax.plot(f, np.sqrt(np.abs(Pxy)) / np.max(np.sqrt(np.abs(Pxy))))
ax.plot(f, np.angle(Pxy) / (2.0 * pi * f))
fig.show()
def _procOneVoxelTimeShift(
vox,
fmritc,
lagstrength,
R2val,
lagtime,
padtrs,
fmritr,
theprefilter,
fmrifreq,
refineprenorm="mean",
lagmaxthresh=5.0,
refineweighting="R",
detrendorder=1,
offsettime=0.0,
filterbeforePCA=False,
psdfilter=False,
rt_floatset=np.float64,
rt_floattype="float64",
):
if refineprenorm == "mean":
thedivisor = np.mean(fmritc)
elif refineprenorm == "var":
thedivisor = np.var(fmritc)
elif refineprenorm == "std":
thedivisor = np.std(fmritc)
elif refineprenorm == "invlag":
if lagtime < lagmaxthresh:
thedivisor = lagmaxthresh - lagtime
else:
thedivisor = 0.0
else:
thedivisor = 1.0
if thedivisor != 0.0:
normfac = 1.0 / thedivisor
else:
normfac = 0.0
if refineweighting == "R":
thisweight = lagstrength
elif refineweighting == "R2":
thisweight = R2val
else:
if lagstrength > 0.0:
thisweight = 1.0
else:
thisweight = -1.0
if detrendorder > 0:
normtc = tide_fit.detrend(fmritc * normfac * thisweight,
order=detrendorder,
demean=True)
else:
normtc = fmritc * normfac * thisweight
shifttr = -(-offsettime + lagtime) / fmritr # lagtime is in seconds
[shiftedtc, weights, paddedshiftedtc,
paddedweights] = tide_resample.timeshift(normtc, shifttr, padtrs)
if filterbeforePCA:
outtc = theprefilter.apply(fmrifreq, shiftedtc)
outweights = theprefilter.apply(fmrifreq, weights)
else:
outtc = 1.0 * shiftedtc
outweights = 1.0 * weights
if psdfilter:
freqs, psd = welch(
tide_math.corrnormalize(shiftedtc, True, True),
fmritr,
scaling="spectrum",
window="hamming",
return_onesided=False,
nperseg=len(shiftedtc),
)
return vox, outtc, outweights, np.sqrt(psd)
else:
return vox, outtc, outweights, None
def shorttermcorr_2D(data1, data2, sampletime, windowtime, samplestep=1, laglimit=None, weighting='none',
prewindow=False, windowfunc='hamming', detrendorder=0, display=False):
"""
Parameters
----------
data1
data2
sampletime
windowtime
samplestep
laglimit
weighting
prewindow
windowfunc
detrendorder
display
Returns
-------
"""
windowsize = int(windowtime // sampletime)
halfwindow = int((windowsize + 1) // 2)
if laglimit is None:
laglimit = windowtime / 2.0
dataseg1 = tide_math.corrnormalize(data1[0:2 * halfwindow], prewindow=prewindow, detrendorder=detrendorder, windowfunc=windowfunc)
dataseg2 = tide_math.corrnormalize(data2[0:2 * halfwindow], prewindow=prewindow, detrendorder=detrendorder, windowfunc=windowfunc)
thexcorr = fastcorrelate(dataseg1, dataseg2, weighting=weighting)
xcorrlen = np.shape(thexcorr)[0]
xcorr_x = np.arange(0.0, xcorrlen) * sampletime - (xcorrlen * sampletime) / 2.0 + sampletime / 2.0
corrzero = int(xcorrlen // 2)
xcorrpertime = []
times = []
Rvals = []
delayvals = []
valid = []
for i in range(halfwindow, np.shape(data1)[0] - halfwindow, samplestep):
dataseg1 = tide_math.corrnormalize(data1[i - halfwindow:i + halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
dataseg2 = tide_math.corrnormalize(data2[i - halfwindow:i + halfwindow],
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
times.append(i * sampletime)
xcorrpertime.append(fastcorrelate(dataseg1, dataseg2, weighting=weighting))
maxindex, thedelayval, theRval, maxsigma, maskval, failreason, peakstart, peakend = tide_fit.findmaxlag_gauss(
xcorr_x, xcorrpertime[-1], -laglimit, laglimit, 1000.0,
refine=True,
useguess=False,
fastgauss=False,
displayplots=False)
delayvals.append(thedelayval)
Rvals.append(theRval)
if failreason == 0:
valid.append(1)
else:
valid.append(0)
if display:
pl.imshow(xcorrpertime)
return np.asarray(times, dtype='float64'), \
np.asarray(xcorrpertime, dtype='float64'), \
np.asarray(Rvals, dtype='float64'), \
np.asarray(delayvals, dtype='float64'), \
np.asarray(valid, dtype='float64')
def cross_mutual_info(
x,
y,
returnaxis=False,
negsteps=-1,
possteps=-1,
locs=None,
Fs=1.0,
norm=True,
madnorm=False,
windowfunc="None",
bins=-1,
prebin=True,
sigma=0.25,
fast=True,
):
"""Calculate cross-mutual information between two 1D arrays.
Parameters
----------
x : 1D array
first variable
y : 1D array
second variable
returnaxis : bool
set to True to return the time axis
negstaps: int
possteps: int
locs : list
a set of offsets at which to calculate the cross mutual information
Fs=1.0,
norm : bool
calculate normalized MI at each offset
madnorm : bool
set to True to normalize cross MI waveform by it's median average deviate
windowfunc : str
name of the window function to apply to input vectors prior to MI calculation
bins : int
number of bins in each dimension of the 2D histogram. Set to -1 to set automatically
prebin : bool
set to true to cache 2D histogram for all offsets
sigma : float
histogram smoothing kernel
fast: bool
apply speed optimizations
Returns
-------
if returnaxis is True:
thexmi_x : 1D array
the set of offsets at which cross mutual information is calcuated
thexmi_y : 1D array
the set of cross mutual information values
len(thexmi_x): int
the number of cross mutual information values returned
else:
thexmi_y : 1D array
the set of cross mutual information values
"""
normx = tide_math.corrnormalize(x, detrendorder=1, windowfunc=windowfunc)
normy = tide_math.corrnormalize(y, detrendorder=1, windowfunc=windowfunc)
# see if we are using the default number of bins
if bins < 1:
bins = int(np.sqrt(len(x) / 5))
LGR.debug(f"cross_mutual_info: bins set to {bins}")
# find the bin locations
if prebin:
jh, bins0, bins1 = np.histogram2d(normx, normy, bins=(bins, bins))
bins2d = (bins0, bins1)
else:
bins2d = (bins, bins)
fast = False
if (negsteps == -1) or (negsteps > len(normy) - 1):
negsteps = -len(normy) + 1
else:
negsteps = -negsteps
if (possteps == -1) or (possteps > len(normx) - 1):
possteps = len(normx) - 1
else:
possteps = possteps
if locs is None:
thexmi_y = np.zeros((-negsteps + possteps + 1))
LGR.debug(f"negsteps, possteps, len(thexmi_y): {negsteps} {possteps} {len(thexmi_y)}")
irange = range(negsteps, possteps + 1)
else:
thexmi_y = np.zeros((len(locs)), dtype=np.float64)
irange = np.asarray(locs)
destloc = -1
for i in irange:
if locs is None:
destloc = i - negsteps
else:
destloc += 1
if i < 0:
thexmi_y[destloc] = mutual_info_2d(
normx[: i + len(normy)],
normy[-i:],
bins=bins2d,
normalized=norm,
fast=fast,
sigma=sigma,
)
elif i == 0:
thexmi_y[destloc] = mutual_info_2d(
normx, normy, bins=bins2d, normalized=norm, fast=fast, sigma=sigma,
)
else:
thexmi_y[destloc] = mutual_info_2d(
normx[i:],
normy[: len(normy) - i],
bins=bins2d,
normalized=norm,
fast=fast,
sigma=sigma,
)
if madnorm:
thexmi_y = tide_math.madnormalize(thexmi_y)
if returnaxis:
if locs is None:
thexmi_x = (
np.linspace(0.0, len(thexmi_y) / Fs, num=len(thexmi_y), endpoint=False)
+ negsteps / Fs
)
return thexmi_x, thexmi_y, negsteps + 1
else:
thexmi_x = irange
return thexmi_x, thexmi_y, len(thexmi_x)
else:
return thexmi_y
def refineregressor(
fmridata,
fmritr,
shiftedtcs,
weights,
passnum,
lagstrengths,
lagtimes,
lagsigma,
lagmask,
R2,
theprefilter,
optiondict,
padtrs=60,
bipolar=False,
includemask=None,
excludemask=None,
debug=False,
rt_floatset=np.float64,
rt_floattype="float64",
):
"""
Parameters
----------
fmridata : 4D numpy float array
fMRI data
fmritr : float
Data repetition rate, in seconds
shiftedtcs : 4D numpy float array
Time aligned voxel timecourses
weights : unknown
unknown
passnum : int
Number of the pass (for labelling output)
lagstrengths : 3D numpy float array
Maximum correlation coefficient in every voxel
lagtimes : 3D numpy float array
Time delay of maximum crosscorrelation in seconds
lagsigma : 3D numpy float array
Gaussian width of the crosscorrelation peak, in seconds.
lagmask : 3D numpy float array
Mask of voxels with successful correlation fits.
R2 : 3D numpy float array
Square of the maximum correlation coefficient in every voxel
theprefilter : function
The filter function to use
optiondict : dict
Dictionary of all internal rapidtide configuration variables.
padtrs : int, optional
Number of timepoints to pad onto each end
includemask : 3D array
Mask of voxels to include in refinement. Default is None (all voxels).
excludemask : 3D array
Mask of voxels to exclude from refinement. Default is None (no voxels).
debug : bool
Enable additional debugging output. Default is False
rt_floatset : function
Function to coerce variable types
rt_floattype : {'float32', 'float64'}
Data type for internal variables
Returns
-------
volumetotal : int
Number of voxels processed
outputdata : float array
New regressor
maskarray : 3D array
Mask of voxels used for refinement
"""
inputshape = np.shape(fmridata)
if optiondict["ampthresh"] < 0.0:
if bipolar:
theampthresh = tide_stats.getfracval(np.fabs(lagstrengths),
-optiondict["ampthresh"],
nozero=True)
else:
theampthresh = tide_stats.getfracval(lagstrengths,
-optiondict["ampthresh"],
nozero=True)
print(
"setting ampthresh to the",
-100.0 * optiondict["ampthresh"],
"th percentile (",
theampthresh,
")",
)
else:
theampthresh = optiondict["ampthresh"]
if bipolar:
ampmask = np.where(
np.fabs(lagstrengths) >= theampthresh, np.int16(1), np.int16(0))
else:
ampmask = np.where(lagstrengths >= theampthresh, np.int16(1),
np.int16(0))
if optiondict["lagmaskside"] == "upper":
delaymask = np.where(
(lagtimes - optiondict["offsettime"]) > optiondict["lagminthresh"],
np.int16(1),
np.int16(0),
) * np.where(
(lagtimes - optiondict["offsettime"]) < optiondict["lagmaxthresh"],
np.int16(1),
np.int16(0),
)
elif optiondict["lagmaskside"] == "lower":
delaymask = np.where(
(lagtimes - optiondict["offsettime"]) <
-optiondict["lagminthresh"],
np.int16(1),
np.int16(0),
) * np.where(
(lagtimes - optiondict["offsettime"]) >
-optiondict["lagmaxthresh"],
np.int16(1),
np.int16(0),
)
else:
abslag = abs(lagtimes) - optiondict["offsettime"]
delaymask = np.where(abslag > optiondict["lagminthresh"], np.int16(1),
np.int16(0)) * np.where(
abslag < optiondict["lagmaxthresh"],
np.int16(1), np.int16(0))
sigmamask = np.where(lagsigma < optiondict["sigmathresh"], np.int16(1),
np.int16(0))
locationmask = lagmask + 0
if includemask is not None:
locationmask = locationmask * includemask
if excludemask is not None:
locationmask = locationmask * (1 - excludemask)
locationmask = locationmask.astype(np.int16)
print("location mask created")
# first generate the refine mask
locationfails = np.sum(1 - locationmask)
ampfails = np.sum(1 - ampmask * locationmask)
lagfails = np.sum(1 - delaymask * locationmask)
sigmafails = np.sum(1 - sigmamask * locationmask)
refinemask = locationmask * ampmask * delaymask * sigmamask
if tide_stats.getmasksize(refinemask) == 0:
print("ERROR: no voxels in the refine mask:")
print(
"\n ",
locationfails,
" locationfails",
"\n ",
ampfails,
" ampfails",
"\n ",
lagfails,
" lagfails",
"\n ",
sigmafails,
" sigmafails",
)
if (includemask is None) and (excludemask is None):
print("\nRelax ampthresh, delaythresh, or sigmathresh - exiting")
else:
print(
"\nChange include/exclude masks or relax ampthresh, delaythresh, or sigmathresh - exiting"
)
return 0, None, None, locationfails, ampfails, lagfails, sigmafails
if optiondict["cleanrefined"]:
shiftmask = locationmask
else:
shiftmask = refinemask
volumetotal = np.sum(shiftmask)
reportstep = 1000
# timeshift the valid voxels
if optiondict["nprocs"] > 1:
# define the consumer function here so it inherits most of the arguments
def timeshift_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(
_procOneVoxelTimeShift(
val,
fmridata[val, :],
lagstrengths[val],
R2[val],
lagtimes[val],
padtrs,
fmritr,
theprefilter,
optiondict["fmrifreq"],
refineprenorm=optiondict["refineprenorm"],
lagmaxthresh=optiondict["lagmaxthresh"],
refineweighting=optiondict["refineweighting"],
detrendorder=optiondict["detrendorder"],
offsettime=optiondict["offsettime"],
filterbeforePCA=optiondict["filterbeforePCA"],
psdfilter=optiondict["psdfilter"],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
timeshift_consumer,
inputshape,
shiftmask,
nprocs=optiondict["nprocs"],
showprogressbar=True,
chunksize=optiondict["mp_chunksize"],
)
# unpack the data
psdlist = []
for voxel in data_out:
shiftedtcs[voxel[0], :] = voxel[1]
weights[voxel[0], :] = voxel[2]
if optiondict["psdfilter"]:
psdlist.append(voxel[3])
del data_out
else:
psdlist = []
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox
== inputshape[0] - 1) and optiondict["showprogressbar"]:
tide_util.progressbar(vox + 1,
inputshape[0],
label="Percent complete (timeshifting)")
if shiftmask[vox] > 0.5:
retvals = _procOneVoxelTimeShift(
vox,
fmridata[vox, :],
lagstrengths[vox],
R2[vox],
lagtimes[vox],
padtrs,
fmritr,
theprefilter,
optiondict["fmrifreq"],
refineprenorm=optiondict["refineprenorm"],
lagmaxthresh=optiondict["lagmaxthresh"],
refineweighting=optiondict["refineweighting"],
detrendorder=optiondict["detrendorder"],
offsettime=optiondict["offsettime"],
filterbeforePCA=optiondict["filterbeforePCA"],
psdfilter=optiondict["psdfilter"],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
shiftedtcs[retvals[0], :] = retvals[1]
weights[retvals[0], :] = retvals[2]
if optiondict["psdfilter"]:
psdlist.append(retvals[3])
print()
if optiondict["psdfilter"]:
print(len(psdlist))
print(psdlist[0])
print(np.shape(np.asarray(psdlist, dtype=rt_floattype)))
averagepsd = np.mean(np.asarray(psdlist, dtype=rt_floattype), axis=0)
stdpsd = np.std(np.asarray(psdlist, dtype=rt_floattype), axis=0)
snr = np.nan_to_num(averagepsd / stdpsd)
# now generate the refined timecourse(s)
validlist = np.where(refinemask > 0)[0]
refinevoxels = shiftedtcs[validlist, :]
if bipolar:
for thevoxel in range(len(validlist)):
if lagstrengths[validlist][thevoxel] < 0.0:
refinevoxels[thevoxel, :] *= -1.0
refineweights = weights[validlist]
weightsum = np.sum(refineweights, axis=0) / volumetotal
averagedata = np.sum(refinevoxels, axis=0) / volumetotal
if optiondict["cleanrefined"]:
invalidlist = np.where((1 - ampmask) > 0)[0]
discardvoxels = shiftedtcs[invalidlist]
discardweights = weights[invalidlist]
discardweightsum = np.sum(discardweights, axis=0) / volumetotal
averagediscard = np.sum(discardvoxels, axis=0) / volumetotal
if optiondict["dodispersioncalc"]:
print("splitting regressors by time lag for phase delay estimation")
laglist = np.arange(
optiondict["dispersioncalc_lower"],
optiondict["dispersioncalc_upper"],
optiondict["dispersioncalc_step"],
)
dispersioncalcout = np.zeros((np.shape(laglist)[0], inputshape[1]),
dtype=rt_floattype)
fftlen = int(inputshape[1] // 2)
fftlen -= fftlen % 2
dispersioncalcspecmag = np.zeros((np.shape(laglist)[0], fftlen),
dtype=rt_floattype)
dispersioncalcspecphase = np.zeros((np.shape(laglist)[0], fftlen),
dtype=rt_floattype)
for lagnum in range(0, np.shape(laglist)[0]):
lower = laglist[lagnum] - optiondict["dispersioncalc_step"] / 2.0
upper = laglist[lagnum] + optiondict["dispersioncalc_step"] / 2.0
inlagrange = np.where(
locationmask * ampmask *
np.where(lower < lagtimes, np.int16(1), np.int16(0)) *
np.where(lagtimes < upper, np.int16(1), np.int16(0)))[0]
print(
" summing",
np.shape(inlagrange)[0],
"regressors with lags from",
lower,
"to",
upper,
)
if np.shape(inlagrange)[0] > 0:
dispersioncalcout[lagnum, :] = tide_math.corrnormalize(
np.mean(shiftedtcs[inlagrange], axis=0),
detrendorder=optiondict["detrendorder"],
windowfunc=optiondict["windowfunc"],
)
(
freqs,
dispersioncalcspecmag[lagnum, :],
dispersioncalcspecphase[lagnum, :],
) = tide_math.polarfft(dispersioncalcout[lagnum, :],
1.0 / fmritr)
inlagrange = None
tide_io.writenpvecs(
dispersioncalcout,
optiondict["outputname"] + "_dispersioncalcvecs_pass" +
str(passnum) + ".txt",
)
tide_io.writenpvecs(
dispersioncalcspecmag,
optiondict["outputname"] + "_dispersioncalcspecmag_pass" +
str(passnum) + ".txt",
)
tide_io.writenpvecs(
dispersioncalcspecphase,
optiondict["outputname"] + "_dispersioncalcspecphase_pass" +
str(passnum) + ".txt",
)
tide_io.writenpvecs(
freqs,
optiondict["outputname"] + "_dispersioncalcfreqs_pass" +
str(passnum) + ".txt",
)
if optiondict["pcacomponents"] < 0.0:
pcacomponents = "mle"
elif optiondict["pcacomponents"] >= 1.0:
pcacomponents = int(np.round(optiondict["pcacomponents"]))
elif optiondict["pcacomponents"] == 0.0:
print("0.0 is not an allowed value for pcacomponents")
sys.exit()
else:
pcacomponents = optiondict["pcacomponents"]
icacomponents = 1
if optiondict["refinetype"] == "ica":
print("performing ica refinement")
thefit = FastICA(n_components=icacomponents).fit(
refinevoxels) # Reconstruct signals
print("Using first of ", len(thefit.components_), " components")
icadata = thefit.components_[0]
filteredavg = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], averagedata),
detrendorder=optiondict["detrendorder"],
)
filteredica = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], icadata),
detrendorder=optiondict["detrendorder"],
)
thepxcorr = pearsonr(filteredavg, filteredica)[0]
print("ica/avg correlation = ", thepxcorr)
if thepxcorr > 0.0:
outputdata = 1.0 * icadata
else:
outputdata = -1.0 * icadata
elif optiondict["refinetype"] == "pca":
# use the method of "A novel perspective to calibrate temporal delays in cerebrovascular reactivity
# using hypercapnic and hyperoxic respiratory challenges". NeuroImage 187, 154?165 (2019).
print("performing pca refinement with pcacomponents set to",
pcacomponents)
try:
thefit = PCA(n_components=pcacomponents).fit(refinevoxels)
except ValueError:
if pcacomponents == "mle":
print(
"mle estimation failed - falling back to pcacomponents=0.8"
)
thefit = PCA(n_components=0.8).fit(refinevoxels)
else:
print("unhandled math exception in PCA refinement - exiting")
sys.exit()
print(
"Using ",
len(thefit.components_),
" component(s), accounting for ",
"{:.2f}% of the variance".format(100.0 * np.cumsum(
thefit.explained_variance_ratio_)[len(thefit.components_) -
1]),
)
reduceddata = thefit.inverse_transform(thefit.transform(refinevoxels))
if debug:
print("complex processing: reduceddata.shape =", reduceddata.shape)
pcadata = np.mean(reduceddata, axis=0)
filteredavg = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], averagedata),
detrendorder=optiondict["detrendorder"],
)
filteredpca = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], pcadata),
detrendorder=optiondict["detrendorder"],
)
thepxcorr = pearsonr(filteredavg, filteredpca)[0]
print("pca/avg correlation = ", thepxcorr)
if thepxcorr > 0.0:
outputdata = 1.0 * pcadata
else:
outputdata = -1.0 * pcadata
elif optiondict["refinetype"] == "weighted_average":
print("performing weighted averaging refinement")
outputdata = np.nan_to_num(averagedata / weightsum)
else:
print("performing unweighted averaging refinement")
outputdata = averagedata
if optiondict["cleanrefined"]:
thefit, R = tide_fit.mlregress(averagediscard, averagedata)
fitcoff = rt_floatset(thefit[0, 1])
datatoremove = rt_floatset(fitcoff * averagediscard)
outputdata -= datatoremove
print()
print(
"Timeshift applied to " + str(int(volumetotal)) + " voxels, " +
str(len(validlist)) + " used for refinement:",
"\n ",
locationfails,
" locationfails",
"\n ",
ampfails,
" ampfails",
"\n ",
lagfails,
" lagfails",
"\n ",
sigmafails,
" sigmafails",
)
if optiondict["psdfilter"]:
outputdata = tide_filt.transferfuncfilt(outputdata, snr)
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal, outputdata, refinemask, locationfails, ampfails, lagfails, sigmafails
def getNullDistributionDatax(
rawtimecourse,
Fs,
theCorrelator,
thefitter,
despeckle_thresh=5.0,
fixdelay=False,
fixeddelayvalue=0.0,
numestreps=0,
nprocs=1,
alwaysmultiproc=False,
showprogressbar=True,
chunksize=1000,
permutationmethod="shuffle",
rt_floatset=np.float64,
rt_floattype="float64",
):
r"""Calculate a set of null correlations to determine the distribution of correlation values. This can
be used to find the spurious correlation threshold
Parameters
----------
rawtimecourse : 1D numpy array
The test regressor. This should be filtered to the desired bandwidth, but NOT windowed.
:param rawtimecourse:
corrscale: 1D numpy array
The time axis of the cross correlation function.
filterfunc: function
This is a preconfigured NoncausalFilter function which is used to filter data to the desired bandwidth
Fs: float
The sample frequency of rawtimecourse, in Hz
corrorigin: int
The bin number in the correlation timescale corresponding to 0.0 seconds delay
negbins: int
The lower edge of the search range for correlation peaks, in number of bins below corrorigin
posbins: int
The upper edge of the search range for correlation peaks, in number of bins above corrorigin
"""
inputshape = np.asarray([numestreps])
normalizedreftc = theCorrelator.ncprefilter.apply(
Fs,
tide_math.corrnormalize(
theCorrelator.reftc,
windowfunc="None",
detrendorder=theCorrelator.detrendorder,
),
)
rawtcfft_r, rawtcfft_ang = tide_filt.polarfft(normalizedreftc)
if nprocs > 1 or alwaysmultiproc:
# define the consumer function here so it inherits most of the arguments
def nullCorrelation_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(
_procOneNullCorrelationx(
normalizedreftc,
rawtcfft_r,
rawtcfft_ang,
Fs,
theCorrelator,
thefitter,
despeckle_thresh=despeckle_thresh,
fixdelay=fixdelay,
fixeddelayvalue=fixeddelayvalue,
permutationmethod=permutationmethod,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
nullCorrelation_consumer,
inputshape,
None,
nprocs=nprocs,
showprogressbar=showprogressbar,
chunksize=chunksize,
)
# unpack the data
corrlist = np.asarray(data_out, dtype=rt_floattype)
else:
corrlist = np.zeros((numestreps), dtype=rt_floattype)
for i in range(0, numestreps):
# make a shuffled copy of the regressors
if permutationmethod == "shuffle":
permutedtc = np.random.permutation(normalizedreftc)
elif permutationmethod == "phaserandom":
permutedtc = tide_filt.ifftfrompolar(
rawtcfft_r, np.random.permutation(rawtcfft_ang))
else:
print("illegal shuffling method")
sys.exit()
# crosscorrelate with original, fit, and return the maximum value, and add it to the list
thexcorr = _procOneNullCorrelationx(
normalizedreftc,
rawtcfft_r,
rawtcfft_ang,
Fs,
theCorrelator,
thefitter,
despeckle_thresh=despeckle_thresh,
fixdelay=fixdelay,
fixeddelayvalue=fixeddelayvalue,
permutationmethod=permutationmethod,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
corrlist[i] = thexcorr
# progress
if showprogressbar:
tide_util.progressbar(i + 1,
numestreps,
label="Percent complete")
# jump to line after progress bar
print()
# return the distribution data
numnonzero = len(np.where(corrlist != 0.0)[0])
print("{:d} non-zero correlations out of {:d} ({:.2f}%)".format(
numnonzero, len(corrlist), 100.0 * numnonzero / len(corrlist)))
return corrlist
