def test_fastcorrelate(display=False):
inlen = 1000
offset = 100
sig1 = np.zeros((inlen), dtype="float")
sig2 = np.zeros((inlen), dtype="float")
sig1[int(inlen // 2) + 1] = 1.0
sig2[int(inlen // 2) + offset + 1] = 1.0
fastcorrelate_result_pad0 = fastcorrelate(sig2, sig1, zeropadding=0)
fastcorrelate_result_padneg1 = fastcorrelate(sig2, sig1, zeropadding=-1)
fastcorrelate_result_pad100 = fastcorrelate(sig2, sig1, zeropadding=100)
print(
"lengths:",
len(fastcorrelate_result_pad0),
len(fastcorrelate_result_padneg1),
len(fastcorrelate_result_pad100),
)
stdcorrelate_result = np.correlate(sig2, sig1, mode="full")
midpoint = int(len(stdcorrelate_result) // 2) + 1
if display:
plt.figure()
plt.ylim([-1.0, 3.0])
plt.plot(fastcorrelate_result_pad100 + 3.0)
plt.plot(fastcorrelate_result_padneg1 + 2.0)
plt.plot(fastcorrelate_result_pad0 + 1.0)
plt.plot(stdcorrelate_result)
print("maximum occurs at offset", np.argmax(stdcorrelate_result) - midpoint + 1)
plt.legend(["Fast correlate", "Standard correlate"])
plt.show()
aethresh = 10
np.testing.assert_almost_equal(fastcorrelate_result_pad0, stdcorrelate_result, aethresh)
# smoke test the weighted correlations
for weighting in ["None", "liang", "eckart", "phat"]:
weighted_result = fastcorrelate(sig2, sig1, weighting=weighting)
def onecorrelation(thetc,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
ncprefilter,
referencetc,
usewindowfunc=True,
detrendorder=1,
windowfunc='hamming',
corrweighting='none'):
thetc_classfilter = ncprefilter.apply(oversampfreq, thetc)
thetc = thetc_classfilter
# prepare timecourse by normalizing, detrending, and applying a window function
preppedtc = tide_math.corrnormalize(thetc,
prewindow=usewindowfunc,
detrendorder=detrendorder,
windowfunc=windowfunc)
# now actually do the correlation
thexcorr = tide_corr.fastcorrelate(preppedtc, referencetc, usefft=True, weighting=corrweighting)
# find the global maximum value
theglobalmax = np.argmax(thexcorr)
return thexcorr[corrorigin - lagmininpts:corrorigin + lagmaxinpts], theglobalmax
def onecorrelation(thetc,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
ncprefilter,
referencetc,
usewindowfunc=True,
detrendorder=1,
windowfunc='hamming',
corrweighting='none'):
thetc_classfilter = ncprefilter.apply(oversampfreq, thetc)
thetc = thetc_classfilter
# prepare timecourse by normalizing, detrending, and applying a window function
preppedtc = tide_math.corrnormalize(thetc,
prewindow=usewindowfunc,
detrendorder=detrendorder,
windowfunc=windowfunc)
# now actually do the correlation
thexcorr = tide_corr.fastcorrelate(preppedtc, referencetc, usefft=True, weighting=corrweighting)
# find the global maximum value
theglobalmax = np.argmax(thexcorr)
return thexcorr[corrorigin - lagmininpts:corrorigin + lagmaxinpts], theglobalmax
def _get_null_distribution(indata, xcorr_x, thefilter, prewindow, detrendorder,
searchstart, searchend, Fs, dofftcorr,
windowfunc='hamming', corrweighting='none',
numreps=1000):
"""
Get an empirical null distribution from the data.
"""
print('estimating significance distribution using {0} '
'repetitions'.format(numreps))
corrlist = zeros(numreps, dtype='float')
corrlist_pear = zeros(numreps, dtype='float')
xcorr_x_trim = xcorr_x[searchstart:searchend + 1]
filteredindata = tide_math.corrnormalize(thefilter.apply(Fs, indata),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
for i in range(numreps):
# make a shuffled copy of the regressors
shuffleddata = permutation(indata)
# filter it
filteredshuffleddata = np.nan_to_num(
tide_math.corrnormalize(thefilter.apply(Fs, shuffleddata),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc))
# crosscorrelate with original
theshuffledxcorr = tide_corr.fastcorrelate(filteredindata,
filteredshuffleddata,
usefft=dofftcorr,
weighting=corrweighting)
# find and tabulate correlation coefficient at optimal lag
theshuffledxcorr_trim = theshuffledxcorr[searchstart:searchend + 1]
maxdelay = xcorr_x_trim[argmax(theshuffledxcorr_trim)]
corrlist[i] = theshuffledxcorr_trim[argmax(theshuffledxcorr_trim)]
# find and tabulate correlation coefficient at 0 lag
corrlist_pear[i] = pearsonr(filteredindata, filteredshuffleddata)[0]
# return the distribution data
return corrlist, corrlist_pear
def run(self, thetc, trim=True):
if len(thetc) != len(self.reftc):
print(
"timecourses are of different sizes:",
len(thetc),
"!=",
len(self.reftc),
"- exiting",
)
sys.exit()
self.testtc = thetc
self.preptesttc = self.preptc(self.testtc)
# now actually do the correlation
self.thesimfunc = tide_corr.fastcorrelate(
self.preptesttc,
self.prepreftc,
usefft=True,
weighting=self.corrweighting,
zeropadding=self.corrpadding,
debug=self.debug,
)
self.similarityfunclen = len(self.thesimfunc)
self.similarityfuncorigin = self.similarityfunclen // 2 + 1
# find the global maximum value
self.theglobalmax = np.argmax(self.thesimfunc)
self.datavalid = True
if trim:
return (
self.trim(self.thesimfunc),
self.trim(self.timeaxis),
self.theglobalmax,
)
else:
return self.thesimfunc, self.timeaxis, self.theglobalmax
def test_nullsimfunc(debug=False, display=False):
# make the lfo filter
lfofilter = tide_filt.NoncausalFilter(filtertype="lfo")
# make the starting regressor
timestep = 1.5
Fs = 1.0 / timestep
# sourcelen = 1200
# sourcedata = lfofilter.apply(Fs, np.random.rand(sourcelen))
sourcedata = tide_io.readvecs(
os.path.join(get_test_data_path(), "fmri_globalmean.txt"))[0]
sourcelen = len(sourcedata)
numpasses = 1
if display:
plt.figure()
plt.plot(sourcedata)
plt.show()
thexcorr = tide_corr.fastcorrelate(sourcedata, sourcedata)
xcorrlen = len(thexcorr)
xcorr_x = (
np.linspace(0.0, xcorrlen, xcorrlen, endpoint=False) * timestep -
(xcorrlen * timestep) / 2.0 + timestep / 2.0)
if display:
plt.figure()
plt.plot(xcorr_x, thexcorr)
plt.show()
corrzero = xcorrlen // 2
lagmin = -10
lagmax = 10
lagmininpts = int((-lagmin / timestep) - 0.5)
lagmaxinpts = int((lagmax / timestep) + 0.5)
searchstart = int(np.round(corrzero + lagmin / timestep))
searchend = int(np.round(corrzero + lagmax / timestep))
optiondict = {
"numestreps": 10000,
"showprogressbar": debug,
"detrendorder": 3,
"windowfunc": "hamming",
"corrweighting": "None",
"nprocs": 1,
"widthlimit": 1000.0,
"bipolar": False,
"fixdelay": False,
"peakfittype": "gauss",
"lagmin": lagmin,
"lagmax": lagmax,
"absminsigma": 0.25,
"absmaxsigma": 25.0,
"edgebufferfrac": 0.0,
"lthreshval": 0.0,
"uthreshval": 1.0,
"debug": False,
"enforcethresh": True,
"lagmod": 1000.0,
"searchfrac": 0.5,
"permutationmethod": "shuffle",
"hardlimit": True,
}
theprefilter = tide_filt.NoncausalFilter("lfo")
theCorrelator = tide_classes.Correlator(
Fs=Fs,
ncprefilter=theprefilter,
detrendorder=optiondict["detrendorder"],
windowfunc=optiondict["windowfunc"],
corrweighting=optiondict["corrweighting"],
)
thefitter = tide_classes.SimilarityFunctionFitter(
lagmod=optiondict["lagmod"],
lthreshval=optiondict["lthreshval"],
uthreshval=optiondict["uthreshval"],
bipolar=optiondict["bipolar"],
lagmin=optiondict["lagmin"],
lagmax=optiondict["lagmax"],
absmaxsigma=optiondict["absmaxsigma"],
absminsigma=optiondict["absminsigma"],
debug=optiondict["debug"],
peakfittype=optiondict["peakfittype"],
searchfrac=optiondict["searchfrac"],
enforcethresh=optiondict["enforcethresh"],
hardlimit=optiondict["hardlimit"],
)
if debug:
print(optiondict)
theCorrelator.setlimits(lagmininpts, lagmaxinpts)
theCorrelator.setreftc(sourcedata)
dummy, trimmedcorrscale, dummy = theCorrelator.getfunction()
thefitter.setcorrtimeaxis(trimmedcorrscale)
histograms = []
for thenprocs in [1, -1]:
for i in range(numpasses):
corrlist = tide_nullsimfunc.getNullDistributionDatax(
sourcedata,
Fs,
theCorrelator,
thefitter,
despeckle_thresh=5.0,
fixdelay=False,
fixeddelayvalue=0.0,
numestreps=optiondict["numestreps"],
nprocs=thenprocs,
showprogressbar=optiondict["showprogressbar"],
chunksize=1000,
permutationmethod=optiondict["permutationmethod"],
)
tide_io.writenpvecs(
corrlist, os.path.join(get_test_temp_path(),
"corrdistdata.txt"))
# calculate percentiles for the crosscorrelation from the distribution data
histlen = 250
thepercentiles = [0.95, 0.99, 0.995]
pcts, pcts_fit, histfit = tide_stats.sigFromDistributionData(
corrlist, histlen, thepercentiles)
if debug:
tide_stats.printthresholds(
pcts,
thepercentiles,
"Crosscorrelation significance thresholds from data:",
)
tide_stats.printthresholds(
pcts_fit,
thepercentiles,
"Crosscorrelation significance thresholds from fit:",
)
(
thehist,
peakheight,
peakloc,
peakwidth,
centerofmass,
) = tide_stats.makehistogram(np.abs(corrlist),
histlen,
therange=[0.0, 1.0])
histograms.append(thehist)
thestore = np.zeros((2, len(thehist[0])), dtype="float64")
thestore[0, :] = (thehist[1][1:] + thehist[1][0:-1]) / 2.0
thestore[1, :] = thehist[0][-histlen:]
if display:
plt.figure()
plt.plot(thestore[0, :], thestore[1, :])
plt.show()
# tide_stats.makeandsavehistogram(corrlist, histlen, 0,
# os.path.join(get_test_temp_path(), 'correlationhist'),
# displaytitle='Null correlation histogram',
# displayplots=display, refine=False)
assert True
def test_calcsimfunc(debug=False, display=False):
# make the lfo filter
lfofilter = tide_filt.NoncausalFilter(filtertype="lfo")
# make some data
oversampfactor = 2
numvoxels = 100
numtimepoints = 500
tr = 0.72
Fs = 1.0 / tr
init_fmri_x = np.linspace(
0.0, numtimepoints, numtimepoints, endpoint=False) * tr
oversampfreq = oversampfactor * Fs
os_fmri_x = np.linspace(0.0, numtimepoints * oversampfactor, numtimepoints
* oversampfactor) * (1.0 / oversampfreq)
theinputdata = np.zeros((numvoxels, numtimepoints), dtype=np.float64)
meanval = np.zeros((numvoxels), dtype=np.float64)
testfreq = 0.075
msethresh = 1e-3
# make the starting regressor
sourcedata = np.sin(2.0 * np.pi * testfreq * os_fmri_x)
numpasses = 1
# make the timeshifted data
shiftstart = -5.0
shiftend = 5.0
voxelshifts = np.linspace(shiftstart, shiftend, numvoxels, endpoint=False)
for i in range(numvoxels):
theinputdata[i, :] = np.sin(2.0 * np.pi * testfreq *
(init_fmri_x - voxelshifts[i]))
if display:
plt.figure()
plt.plot(sourcedata)
plt.show()
genlagtc = tide_resample.FastResampler(os_fmri_x, sourcedata)
thexcorr = tide_corr.fastcorrelate(sourcedata, sourcedata)
xcorrlen = len(thexcorr)
xcorr_x = (np.linspace(0.0, xcorrlen, xcorrlen, endpoint=False) * tr -
(xcorrlen * tr) / 2.0 + tr / 2.0)
if display:
plt.figure()
plt.plot(xcorr_x, thexcorr)
plt.show()
corrzero = xcorrlen // 2
lagmin = -10.0
lagmax = 10.0
lagmininpts = int((-lagmin * oversampfreq) - 0.5)
lagmaxinpts = int((lagmax * oversampfreq) + 0.5)
searchstart = int(np.round(corrzero + lagmin / tr))
searchend = int(np.round(corrzero + lagmax / tr))
numcorrpoints = lagmaxinpts + lagmininpts
corrout = np.zeros((numvoxels, numcorrpoints), dtype=np.float64)
lagmask = np.zeros((numvoxels), dtype=np.float64)
failimage = np.zeros((numvoxels), dtype=np.float64)
lagtimes = np.zeros((numvoxels), dtype=np.float64)
lagstrengths = np.zeros((numvoxels), dtype=np.float64)
lagsigma = np.zeros((numvoxels), dtype=np.float64)
gaussout = np.zeros((numvoxels, numcorrpoints), dtype=np.float64)
windowout = np.zeros((numvoxels, numcorrpoints), dtype=np.float64)
R2 = np.zeros((numvoxels), dtype=np.float64)
lagtc = np.zeros((numvoxels, numtimepoints), dtype=np.float64)
optiondict = {
"numestreps": 10000,
"interptype": "univariate",
"showprogressbar": debug,
"detrendorder": 3,
"windowfunc": "hamming",
"corrweighting": "None",
"nprocs": 1,
"widthlimit": 1000.0,
"bipolar": False,
"fixdelay": False,
"peakfittype": "gauss",
"lagmin": lagmin,
"lagmax": lagmax,
"absminsigma": 0.25,
"absmaxsigma": 25.0,
"edgebufferfrac": 0.0,
"lthreshval": 0.0,
"uthreshval": 1.1,
"debug": False,
"enforcethresh": True,
"lagmod": 1000.0,
"searchfrac": 0.5,
"mp_chunksize": 1000,
"oversampfactor": oversampfactor,
"despeckle_thresh": 5.0,
"zerooutbadfit": False,
"permutationmethod": "shuffle",
"hardlimit": True,
}
theprefilter = tide_filt.NoncausalFilter("lfo")
theCorrelator = tide_classes.Correlator(
Fs=oversampfreq,
ncprefilter=theprefilter,
detrendorder=optiondict["detrendorder"],
windowfunc=optiondict["windowfunc"],
corrweighting=optiondict["corrweighting"],
)
thefitter = tide_classes.SimilarityFunctionFitter(
lagmod=optiondict["lagmod"],
lthreshval=optiondict["lthreshval"],
uthreshval=optiondict["uthreshval"],
bipolar=optiondict["bipolar"],
lagmin=optiondict["lagmin"],
lagmax=optiondict["lagmax"],
absmaxsigma=optiondict["absmaxsigma"],
absminsigma=optiondict["absminsigma"],
debug=optiondict["debug"],
peakfittype=optiondict["peakfittype"],
zerooutbadfit=optiondict["zerooutbadfit"],
searchfrac=optiondict["searchfrac"],
enforcethresh=optiondict["enforcethresh"],
hardlimit=optiondict["hardlimit"],
)
if debug:
print(optiondict)
theCorrelator.setlimits(lagmininpts, lagmaxinpts)
theCorrelator.setreftc(sourcedata)
dummy, trimmedcorrscale, dummy = theCorrelator.getfunction()
thefitter.setcorrtimeaxis(trimmedcorrscale)
for thenprocs in [1, -1]:
for i in range(numpasses):
(
voxelsprocessed_cp,
theglobalmaxlist,
trimmedcorrscale,
) = tide_calcsimfunc.correlationpass(
theinputdata,
sourcedata,
theCorrelator,
init_fmri_x,
os_fmri_x,
lagmininpts,
lagmaxinpts,
corrout,
meanval,
nprocs=thenprocs,
oversampfactor=optiondict["oversampfactor"],
interptype=optiondict["interptype"],
showprogressbar=optiondict["showprogressbar"],
chunksize=optiondict["mp_chunksize"],
)
if display:
plt.figure()
plt.plot(trimmedcorrscale, corrout[numvoxels // 2, :], "k")
plt.show()
voxelsprocessed_fc = tide_simfuncfit.fitcorr(
genlagtc,
init_fmri_x,
lagtc,
trimmedcorrscale,
thefitter,
corrout,
lagmask,
failimage,
lagtimes,
lagstrengths,
lagsigma,
gaussout,
windowout,
R2,
nprocs=optiondict["nprocs"],
fixdelay=optiondict["fixdelay"],
showprogressbar=optiondict["showprogressbar"],
chunksize=optiondict["mp_chunksize"],
despeckle_thresh=optiondict["despeckle_thresh"],
)
if display:
plt.figure()
plt.plot(voxelshifts, "k")
plt.plot(lagtimes, "r")
plt.show()
if debug:
for i in range(numvoxels):
print(
voxelshifts[i],
lagtimes[i],
lagstrengths[i],
lagsigma[i],
failimage[i],
)
assert mse(voxelshifts, lagtimes) < msethresh
def showxcorrx_workflow(infilename1, infilename2, Fs,
thelabel='', starttime=0., duration=1000000.,
searchrange=15.,
display=True, trimdata=False,
summarymode=False, labelline=False,
flipregressor=False, windowfunc='hamming',
calccepstraldelay=False, corroutputfile=False,
controlvariablefile=None, numreps=0,
arbvec=None, filtertype='arb', corrweighting='none',
detrendorder=1, prewindow=True, verbose=False):
r"""Calculate and display crosscorrelation between two timeseries.
Parameters
----------
infilename1 : str
The name of a text file containing a timeseries, one timepoint per line.
infilename2 : str
The name of a text file containing a timeseries, one timepoint per line.
Fs : float
The sample rate of the time series, in Hz.
thelabel : str, optional
The label for the output graph. Default is blank.
starttime : float, optional
Time offset into the timeseries, in seconds, to start using the time data. Default is 0
duration : float, optional
Length of time from each time series, in seconds, to use for the cross-correlation. Default is the entire time series.
searchrange : float, optional
Only search for cross-correlation peaks between -searchrange and +searchrange seconds (default is 15).
display : bool, optional
Plot cross-correlation function in a matplotlib window. Default is True.
trimdata : bool, optional
Trim time series to the length of the shorter series. Default is False.
summarymode : bool, optional
Output a table of interesting results for later processing. Default is False.
labelline : bool, optional
Print an explanatory header line over the summary information. Default is False.
flipregressor : bool, optional
Invert timeseries 2 prior to cross-correlation.
windowfunc : {'hamming', 'hann', 'blackmanharris'}
Window function to apply prior to cross-correlation. Default is 'hamming'.
calccepstraldelay : bool, optional
Use cepstral estimation of delay. Default is False.
corroutputfile : bool, optional
Save the correlation function to a file. Default is False.
controlvariablefile : bool, optional
Save internal variables to a text file. Default is False.
numreps : int, optional
Number of null correlations to perform to estimate significance. Default is 10000
arbvec : [float,float,float,float], optional
Frequency limits of the arb_pass filter.
filtertype : 'none', 'card', 'lfo', 'vlf', 'resp', 'arb'
Type of filter to apply data prior to correlation. Default is 'none'
corrweighting : {'none', 'Liang', 'Eckart', 'PHAT'}, optional
Weighting function to apply to the crosscorrelation in the Fourier domain. Default is 'none'
detrendorder : int, optional
Order of polynomial used to detrend crosscorrelation inputs. Default is 1 (0 disables)
prewindow : bool, optional
Apply window function prior to cross-correlation. Default is True.
verbose : bool, optional
Print internal status information. Default is False.
Notes
-----
This workflow writes out several files:
If corroutputfile is defined:
====================== =================================================
Filename Content
====================== =================================================
corrlist.txt A file
corrlist_pear.txt A file
[corroutputfile] Correlation function
====================== =================================================
If debug is True:
====================== =================================================
Filename Content
====================== =================================================
filtereddata1.txt Something
filtereddata2.txt Something
====================== =================================================
"""
# Constants that could be arguments
dofftcorr = True
writecorrlists = False
debug = False
showpearson = True
# These are unnecessary and should be simplified
dopartial = bool(controlvariablefile)
uselabel = bool(thelabel)
dumpfiltered = bool(debug)
if labelline:
# TS: should prob reflect this in the parser, but it's not a big deal
summarymode = True
if numreps == 0:
estimate_significance = False
else:
estimate_significance = True
savecorrelation = bool(corroutputfile)
theprefilter = tide_filt.noncausalfilter()
if arbvec is not None and filtertype != 'arb':
raise ValueError('Argument arbvec must be None if filtertype is '
'not arb')
if arbvec is not None:
if len(arbvec) == 2:
arb_lower = float(arbvec[0])
arb_upper = float(arbvec[1])
arb_lowerstop = 0.9 * float(arbvec[0])
arb_upperstop = 1.1 * float(arbvec[1])
elif len(arbvec) == 4:
arb_lower = float(arbvec[0])
arb_upper = float(arbvec[1])
arb_lowerstop = float(arbvec[2])
arb_upperstop = float(arbvec[3])
theprefilter.settype('arb')
theprefilter.setarb(arb_lowerstop, arb_lower, arb_upper, arb_upperstop)
else:
theprefilter.settype(filtertype)
inputdata1 = tide_io.readvec(infilename1)
inputdata2 = tide_io.readvec(infilename2)
numpoints = len(inputdata1)
startpoint1 = max([int(starttime * Fs), 0])
if debug:
print('startpoint set to ', startpoint1)
endpoint1 = min([startpoint1 + int(duration * Fs), int(len(inputdata1))])
if debug:
print('endpoint set to ', endpoint1)
endpoint2 = min([int(duration * Fs), int(len(inputdata1)),
int(len(inputdata2))])
trimdata1 = inputdata1[startpoint1:endpoint1]
trimdata2 = inputdata2[0:endpoint2]
if trimdata:
minlen = np.min([len(trimdata1), len(trimdata2)])
trimdata1 = trimdata1[0:minlen]
trimdata2 = trimdata2[0:minlen]
# band limit the regressor if that is needed
if theprefilter.gettype() != 'none':
if verbose:
print("filtering to ", theprefilter.gettype(), " band")
print(windowfunc)
filtereddata1 = tide_math.corrnormalize(theprefilter.apply(Fs, trimdata1),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
filtereddata2 = tide_math.corrnormalize(theprefilter.apply(Fs, trimdata2),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
if flipregressor:
filtereddata2 *= -1.0
if dumpfiltered:
tide_io.writenpvecs(filtereddata1, 'filtereddata1.txt')
tide_io.writenpvecs(filtereddata2, 'filtereddata2.txt')
if dopartial:
controlvars = tide_io.readvecs(controlvariablefile)
numregressors = len(controlvars) # Added by TS. Not sure if works.
regressorvec = []
for j in range(0, numregressors):
regressorvec.append(tide_math.corrnormalize(
theprefilter.apply(Fs, controlvars[j, :]),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc))
if (np.max(filtereddata1) - np.min(filtereddata1)) > 0.0:
thefit, filtereddata1 = tide_fit.mlregress(regressorvec,
filtereddata1)
if (np.max(filtereddata2) - np.min(filtereddata2)) > 0.0:
thefit, filtereddata2 = tide_fit.mlregress(regressorvec,
filtereddata2)
thexcorr = tide_corr.fastcorrelate(filtereddata1, filtereddata2,
usefft=dofftcorr,
weighting=corrweighting,
displayplots=debug)
if calccepstraldelay:
cepdelay = tide_corr.cepstraldelay(filtereddata1, filtereddata2,
1.0 / Fs, displayplots=display)
cepcoff = tide_corr.delayedcorr(filtereddata1, filtereddata2, cepdelay,
1.0 / Fs)
print('cepstral delay time is {0}, correlation is {1}'.format(cepdelay,
cepcoff))
thepxcorr = pearsonr(filtereddata1, filtereddata2)
# calculate the coherence
f, Cxy = sp.signal.coherence(
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata1), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata2), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
Fs)
# calculate the cross spectral density
f, Pxy = sp.signal.csd(
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata1), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata2), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
Fs)
xcorrlen = len(thexcorr)
sampletime = 1.0 / Fs
xcorr_x = r_[0:xcorrlen] * sampletime - (xcorrlen * sampletime) / 2.0\
+ sampletime / 2.0
halfwindow = int(searchrange * Fs)
corrzero = xcorrlen // 2
searchstart = corrzero - halfwindow
searchend = corrzero + halfwindow
xcorr_x_trim = xcorr_x[searchstart:searchend + 1]
thexcorr_trim = thexcorr[searchstart:searchend + 1]
if debug:
print('searching for peak correlation over range ', searchstart,
searchend)
maxdelay = xcorr_x_trim[argmax(thexcorr_trim)]
if debug:
print('maxdelay before refinement', maxdelay)
dofindmaxlag = True
if dofindmaxlag:
print('executing findmaxlag')
(maxindex, maxdelay, maxval, maxsigma, maskval, failreason, peakstart,
peakend) = tide_fit.findmaxlag_gauss(
xcorr_x_trim, thexcorr_trim, -searchrange, searchrange, 1000.0,
refine=True,
useguess=False,
fastgauss=False,
displayplots=False)
print(maxindex, maxdelay, maxval, maxsigma, maskval, failreason)
R = maxval
if debug:
print('maxdelay after refinement', maxdelay)
if failreason > 0:
print('failreason =', failreason)
else:
R = thexcorr_trim[argmax(thexcorr_trim)]
# set the significance threshold
if estimate_significance:
# generate a list of correlations from shuffled data
(corrlist,
corrlist_pear) = _get_null_distribution(trimdata1, xcorr_x,
theprefilter, prewindow,
detrendorder, searchstart,
searchend, Fs, dofftcorr,
corrweighting=corrweighting,
numreps=numreps,
windowfunc=windowfunc)
# calculate percentiles for the crosscorrelation from the distribution
histlen = 100
thepercentiles = [0.95, 0.99, 0.995]
(pcts, pcts_fit,
histfit) = tide_stats.sigFromDistributionData(corrlist, histlen,
thepercentiles)
if debug:
tide_stats.printthresholds(pcts, thepercentiles,
('Crosscorrelation significance '
'thresholds from data:'))
tide_stats.printthresholds(pcts_fit, thepercentiles,
('Crosscorrelation significance '
'thresholds from fit:'))
# calculate significance for the pearson correlation
(pearpcts, pearpcts_fit,
histfit) = tide_stats.sigFromDistributionData(corrlist_pear, histlen,
thepercentiles)
if debug:
tide_stats.printthresholds(pearpcts, thepercentiles,
('Pearson correlation significance '
'thresholds from data:'))
tide_stats.printthresholds(pearpcts_fit, thepercentiles,
('Pearson correlation significance '
'thresholds from fit:'))
if writecorrlists:
tide_io.writenpvecs(corrlist, 'corrlist.txt')
tide_io.writenpvecs(corrlist_pear, 'corrlist_pear.txt')
def printthresholds(pcts, thepercentiles, labeltext):
print(labeltext)
for i in range(0, len(pcts)):
print('\tp <', "{:.3f}".format(1.0 - thepercentiles[i]), ': ',
pcts[i])
# report the pearson correlation
if showpearson and verbose:
print('Pearson_R:\t', thepxcorr[0])
if estimate_significance:
for idx, percentile in enumerate(thepercentiles):
print(' pear_p(', "{:.3f}".format(1.0 - percentile), '):\t',
pearpcts[idx])
print("")
if debug:
print(thepxcorr)
if verbose:
if uselabel:
print(thelabel, ":\t", maxdelay)
else:
print("Crosscorrelation_Rmax:\t", R)
print("Crosscorrelation_maxdelay:\t", maxdelay)
if estimate_significance:
for idx, percentile in enumerate(thepercentiles):
print(' xc_p(', "{:.3f}".format(1.0 - percentile),
'):\t', pcts[idx])
print(infilename1, "[0 seconds] == ", infilename2, "[",
-1 * maxdelay, " seconds]")
if summarymode:
if estimate_significance:
if uselabel:
if labelline:
print('thelabel', 'pearson_R', 'pearson_R(p=0.05)',
'xcorr_R', 'xcorr_R(P=0.05)', 'xcorr_maxdelay')
print(thelabel, thepxcorr[0], pearpcts_fit[0], R, pcts_fit[0],
-1 * maxdelay)
else:
if labelline:
print('pearson_R', 'pearson_R(p=0.05)', 'xcorr_R',
'xcorr_R(P=0.05)', 'xcorr_maxdelay')
print(thepxcorr[0], pearpcts_fit[0], R, pcts_fit[0],
-1 * maxdelay)
else:
if uselabel:
if labelline:
print('thelabel', 'pearson_r', 'pearson_p', 'xcorr_R',
'xcorr_maxdelay')
print(thelabel, thepxcorr[0], thepxcorr[1], R, -1 * maxdelay)
else:
if labelline:
print('pearson_r\tpearson_p\txcorr_R\txcorr_t\t'
'xcorr_maxdelay')
print(thepxcorr[0], '\t', thepxcorr[1], '\t', R, '\t',
-1 * maxdelay)
if savecorrelation:
tide_io.writenpvecs(np.stack((xcorr_x, thexcorr), axis=0),
corroutputfile)
if display:
fig, ax = plt.subplots()
# ax.set_title('GCC')
ax.plot(xcorr_x, thexcorr, 'k')
if debug:
fig, ax = plt.subplots()
ax.plot(f, Cxy)
fig = plt.subplots()
ax.plot(f, np.sqrt(np.abs(Pxy)) / np.max(np.sqrt(np.abs(Pxy))))
ax.plot(f, np.angle(Pxy) / (2.0 * pi * f))
fig.show()