def _procOneVoxelWiener(vox,
lagtc,
inittc,
rt_floatset=np.float64,
rt_floattype='float64'):
thefit, R = tide_fit.mlregress(lagtc, inittc)
fitcoff = rt_floatset(thefit[0, 1])
datatoremove = rt_floatset(fitcoff * lagtc)
return vox, rt_floatset(thefit[0, 0]), rt_floatset(R), rt_floatset(R * R), fitcoff, \
rt_floatset(thefit[0, 1] / thefit[0, 0]), datatoremove, rt_floatset(inittc - datatoremove)
def _procOneVoxelWiener(vox, lagtc, inittc, rt_floatset=np.float64, rt_floattype="float64"):
thefit, R = tide_fit.mlregress(lagtc, inittc)
fitcoff = rt_floatset(thefit[0, 1])
datatoremove = rt_floatset(fitcoff * lagtc)
return (
vox,
rt_floatset(thefit[0, 0]),
rt_floatset(R),
rt_floatset(R * R),
fitcoff,
rt_floatset(thefit[0, 1] / thefit[0, 0]),
datatoremove,
rt_floatset(inittc - datatoremove),
)
def confoundglm(
data,
regressors,
debug=False,
showprogressbar=True,
reportstep=1000,
rt_floatset=np.float64,
rt_floattype="float64",
):
r"""Filters multiple regressors out of an array of data
Parameters
----------
data : 2d numpy array
A data array. First index is the spatial dimension, second is the time (filtering) dimension.
regressors: 2d numpy array
The set of regressors to filter out of each timecourse. The first dimension is the regressor number, second is the time (filtering) dimension:
debug : boolean
Print additional diagnostic information if True
Returns
-------
"""
if debug:
print("data shape:", data.shape)
print("regressors shape:", regressors.shape)
datatoremove = np.zeros(data.shape[1], dtype=rt_floattype)
filtereddata = data * 0.0
for i in range(data.shape[0]):
if showprogressbar and (i > 0) and (i % reportstep == 0
or i == data.shape[0] - 1):
tide_util.progressbar(i + 1,
data.shape[0],
label="Percent complete")
datatoremove *= 0.0
thefit, R = tide_fit.mlregress(regressors, data[i, :])
if i == 0 and debug:
print("fit shape:", thefit.shape)
for j in range(regressors.shape[0]):
datatoremove += rt_floatset(
rt_floatset(thefit[0, 1 + j]) * regressors[j, :])
filtereddata[i, :] = data[i, :] - datatoremove
return filtereddata
def _procOneItemGLM(vox,
theevs,
thedata,
rt_floatset=np.float64,
rt_floattype="float64"):
thefit, R = tide_fit.mlregress(theevs, thedata)
fitcoeff = rt_floatset(thefit[0, 1])
datatoremove = rt_floatset(fitcoeff * theevs)
return (
vox,
rt_floatset(thefit[0, 0]),
rt_floatset(R),
rt_floatset(R * R),
fitcoeff,
rt_floatset(thefit[0, 1] / thefit[0, 0]),
datatoremove,
rt_floatset(thedata - datatoremove),
)
def confoundglm(data,
regressors,
debug=False,
showprogressbar=True,
reportstep=1000,
rt_floatset=np.float64,
rt_floattype='float64'):
r"""Filters multiple regressors out of an array of data in place
Parameters
----------
data : 2d numpy array
A data array. First index is the spatial dimension, second is the time (filtering) dimension.
regressors: 2d numpy array
The set of regressors to filter out of each timecourse. The first dimension is the regressor number, second is the time (filtering) dimension:
debug : boolean
Print additional diagnostic information if True
Returns
-------
"""
if debug:
print('data shape:', data.shape)
print('regressors shape:', regressors.shape)
datatoremove = np.zeros(data.shape[1], dtype=rt_floattype)
filtereddata = data * 0.0
for i in range(data.shape[0]):
if showprogressbar and (i > 0) and (i % reportstep == 0 or i == data.shape[0] - 1):
tide_util.progressbar(i + 1, data.shape[0], label='Percent complete')
datatoremove *= 0.0
thefit, R = tide_fit.mlregress(regressors, data[i, :])
if i == 0 and debug:
print('fit shape:', thefit.shape)
for j in range(regressors.shape[0]):
datatoremove += rt_floatset(rt_floatset(thefit[0, 1 + j]) * regressors[j, :])
filtereddata[i, :] = data[i, :] - datatoremove
return filtereddata
def glmfilt(inputfile, numskip, outputroot, evfilename):
# initialize some variables
evdata = []
evisnifti = []
thedims_in = []
thedims_ev = []
thesizes_ev = []
# read the datafile and the evfiles
nim_input, nim_data, nim_header, thedims_in, thesizes_in = tide_io.readfromnifti(inputfile)
xdim, ydim, slicedim, tr = tide_io.parseniftisizes(thesizes_in)
print(xdim, ydim, slicedim, tr)
xsize, ysize, numslices, timepoints = tide_io.parseniftidims(thedims_in)
print(xsize, ysize, numslices, timepoints)
numregressors = 0
for i in range(0, len(evfilename)):
print("file ", i, " has name ", evfilename[i])
# check to see if file is nifti or text
fileisnifti = tide_io.checkifnifti(evfilename[i])
fileisparfile = tide_io.checkifparfile(evfilename[i])
if fileisnifti:
# if file is nifti
print("reading voxel specific regressor from ", evfilename[i])
(
nim_evinput,
ev_data,
ev_header,
thedims_evinput,
thesizes_evinput,
) = tide_io.readfromnifti(evfilename[i])
evisnifti.append(True)
evdata.append(1.0 * ev_data)
thedims_ev.append(thedims_evinput)
thesizes_ev.append(thesizes_evinput)
numregressors += 1
elif fileisparfile:
# check to see if file a par file
print("reading 6 global regressors from an FSL parfile")
evtimeseries = tide_io.readvecs(evfilename[i])
print("timeseries length = ", len(evtimeseries[0, :]))
for j in range(0, 6):
evisnifti.append(False)
evdata.append(1.0 * evtimeseries[j, :])
thedims_evinput = 1.0 * thedims_in
thesizes_evinput = 1.0 * thesizes_in
thedims_ev.append(thedims_evinput)
thesizes_ev.append(thesizes_evinput)
numregressors += 1
else:
# if file is text
print("reading global regressor from ", evfilename[i])
evtimeseries = tide_io.readvec(evfilename[i])
print("timeseries length = ", len(evtimeseries))
evisnifti.append(False)
evdata.append(1.0 * evtimeseries)
thedims_evinput = 1.0 * thedims_in
thesizes_evinput = 1.0 * thesizes_in
thedims_ev.append(thedims_evinput)
thesizes_ev.append(thesizes_evinput)
numregressors += 1
for j in range(0, numregressors):
for i in range(0, 4):
if thedims_in[i] != thedims_ev[j][i]:
print("Input file and ev file ", j, " dimensions do not match")
print("dimension ", i, ":", thedims_in[i], " != ", thedims_ev[j][i])
exit()
if timepoints - numskip != thedims_ev[j][4]:
print("Input file and ev file ", j, " dimensions do not match")
print("dimension ", 4, ":", timepoints, "!= ", thedims_ev[j][4], "+", numskip)
exit()
print("will perform GLM with ", numregressors, " regressors")
meandata = np.zeros((xsize, ysize, numslices), dtype="float")
fitdata = np.zeros((xsize, ysize, numslices, numregressors), dtype="float")
Rdata = np.zeros((xsize, ysize, numslices), dtype="float")
trimmeddata = 1.0 * nim_data[:, :, :, numskip:]
for z in range(0, numslices):
print("processing slice ", z)
for y in range(0, ysize):
for x in range(0, xsize):
regressorvec = []
for j in range(0, numregressors):
if evisnifti[j]:
regressorvec.append(evdata[j][x, y, z, :])
else:
regressorvec.append(evdata[j])
if np.max(trimmeddata[x, y, z, :]) - np.min(trimmeddata[x, y, z, :]) > 0.0:
thefit, R = tide_fit.mlregress(regressorvec, trimmeddata[x, y, z, :])
meandata[x, y, z] = thefit[0, 0]
Rdata[x, y, z] = R
for j in range(0, numregressors):
fitdata[x, y, z, j] = thefit[0, j + 1]
# datatoremove[x, y, z, :, j] = thefit[0, j + 1] * regressorvec[j]
else:
meandata[x, y, z] = 0.0
Rdata[x, y, z] = 0.0
for j in range(0, numregressors):
fitdata[x, y, z, j] = 0.0
# datatoremove[x, y, z, :, j] = 0.0 * regressorvec[j]
# totaltoremove[x, y, z, :] = np.sum(datatoremove[x, y, z, :, :], axis=1)
# filtereddata[x, y, z, :] = trimmeddata[x, y, z, :] - totaltoremove[x, y, z, :]
# first save the things with a small numbers of timepoints
print("fitting complete: about to save the fit data")
theheader = nim_header
theheader["dim"][4] = 1
tide_io.savetonifti(meandata, theheader, outputroot + "_mean")
for j in range(0, numregressors):
tide_io.savetonifti(fitdata[:, :, :, j], theheader, outputroot + "_fit" + str(j))
tide_io.savetonifti(Rdata, theheader, outputroot + "_R")
Rdata = None
print()
print("Now constructing the array of data to remove")
# datatoremove = np.zeros((xsize, ysize, numslices, timepoints - numskip, numregressors), dtype='float')
totaltoremove = np.zeros((xsize, ysize, numslices, timepoints - numskip), dtype="float")
# filtereddata = 1.0 * totaltoremove
for z in range(0, numslices):
print("processing slice ", z)
for y in range(0, ysize):
for x in range(0, xsize):
if np.max(trimmeddata[x, y, z, :]) - np.min(trimmeddata[x, y, z, :]) > 0.0:
for j in range(0, numregressors):
totaltoremove[x, y, z, :] += fitdata[x, y, z, j] * regressorvec[j]
else:
totaltoremove[x, y, z, :] = 0.0
print("Array construction done. Saving files")
# now save the things with full timecourses
theheader = nim_header
theheader["dim"][4] = timepoints - numskip
tide_io.savetonifti(totaltoremove, theheader, outputroot + "_totaltoremove")
filtereddata = trimmeddata - totaltoremove
totaltoremove = None
tide_io.savetonifti(trimmeddata, theheader, outputroot + "_trimmed")
trimmeddata = None
tide_io.savetonifti(filtereddata, theheader, outputroot + "_filtered")
def refineregressor(
fmridata,
fmritr,
shiftedtcs,
weights,
passnum,
lagstrengths,
lagtimes,
lagsigma,
lagmask,
R2,
theprefilter,
optiondict,
padtrs=60,
bipolar=False,
includemask=None,
excludemask=None,
debug=False,
rt_floatset=np.float64,
rt_floattype="float64",
):
"""
Parameters
----------
fmridata : 4D numpy float array
fMRI data
fmritr : float
Data repetition rate, in seconds
shiftedtcs : 4D numpy float array
Time aligned voxel timecourses
weights : unknown
unknown
passnum : int
Number of the pass (for labelling output)
lagstrengths : 3D numpy float array
Maximum correlation coefficient in every voxel
lagtimes : 3D numpy float array
Time delay of maximum crosscorrelation in seconds
lagsigma : 3D numpy float array
Gaussian width of the crosscorrelation peak, in seconds.
lagmask : 3D numpy float array
Mask of voxels with successful correlation fits.
R2 : 3D numpy float array
Square of the maximum correlation coefficient in every voxel
theprefilter : function
The filter function to use
optiondict : dict
Dictionary of all internal rapidtide configuration variables.
padtrs : int, optional
Number of timepoints to pad onto each end
includemask : 3D array
Mask of voxels to include in refinement. Default is None (all voxels).
excludemask : 3D array
Mask of voxels to exclude from refinement. Default is None (no voxels).
debug : bool
Enable additional debugging output. Default is False
rt_floatset : function
Function to coerce variable types
rt_floattype : {'float32', 'float64'}
Data type for internal variables
Returns
-------
volumetotal : int
Number of voxels processed
outputdata : float array
New regressor
maskarray : 3D array
Mask of voxels used for refinement
"""
inputshape = np.shape(fmridata)
if optiondict["ampthresh"] < 0.0:
if bipolar:
theampthresh = tide_stats.getfracval(np.fabs(lagstrengths),
-optiondict["ampthresh"],
nozero=True)
else:
theampthresh = tide_stats.getfracval(lagstrengths,
-optiondict["ampthresh"],
nozero=True)
print(
"setting ampthresh to the",
-100.0 * optiondict["ampthresh"],
"th percentile (",
theampthresh,
")",
)
else:
theampthresh = optiondict["ampthresh"]
if bipolar:
ampmask = np.where(
np.fabs(lagstrengths) >= theampthresh, np.int16(1), np.int16(0))
else:
ampmask = np.where(lagstrengths >= theampthresh, np.int16(1),
np.int16(0))
if optiondict["lagmaskside"] == "upper":
delaymask = np.where(
(lagtimes - optiondict["offsettime"]) > optiondict["lagminthresh"],
np.int16(1),
np.int16(0),
) * np.where(
(lagtimes - optiondict["offsettime"]) < optiondict["lagmaxthresh"],
np.int16(1),
np.int16(0),
)
elif optiondict["lagmaskside"] == "lower":
delaymask = np.where(
(lagtimes - optiondict["offsettime"]) <
-optiondict["lagminthresh"],
np.int16(1),
np.int16(0),
) * np.where(
(lagtimes - optiondict["offsettime"]) >
-optiondict["lagmaxthresh"],
np.int16(1),
np.int16(0),
)
else:
abslag = abs(lagtimes) - optiondict["offsettime"]
delaymask = np.where(abslag > optiondict["lagminthresh"], np.int16(1),
np.int16(0)) * np.where(
abslag < optiondict["lagmaxthresh"],
np.int16(1), np.int16(0))
sigmamask = np.where(lagsigma < optiondict["sigmathresh"], np.int16(1),
np.int16(0))
locationmask = lagmask + 0
if includemask is not None:
locationmask = locationmask * includemask
if excludemask is not None:
locationmask = locationmask * (1 - excludemask)
locationmask = locationmask.astype(np.int16)
print("location mask created")
# first generate the refine mask
locationfails = np.sum(1 - locationmask)
ampfails = np.sum(1 - ampmask * locationmask)
lagfails = np.sum(1 - delaymask * locationmask)
sigmafails = np.sum(1 - sigmamask * locationmask)
refinemask = locationmask * ampmask * delaymask * sigmamask
if tide_stats.getmasksize(refinemask) == 0:
print("ERROR: no voxels in the refine mask:")
print(
"\n ",
locationfails,
" locationfails",
"\n ",
ampfails,
" ampfails",
"\n ",
lagfails,
" lagfails",
"\n ",
sigmafails,
" sigmafails",
)
if (includemask is None) and (excludemask is None):
print("\nRelax ampthresh, delaythresh, or sigmathresh - exiting")
else:
print(
"\nChange include/exclude masks or relax ampthresh, delaythresh, or sigmathresh - exiting"
)
return 0, None, None, locationfails, ampfails, lagfails, sigmafails
if optiondict["cleanrefined"]:
shiftmask = locationmask
else:
shiftmask = refinemask
volumetotal = np.sum(shiftmask)
reportstep = 1000
# timeshift the valid voxels
if optiondict["nprocs"] > 1:
# define the consumer function here so it inherits most of the arguments
def timeshift_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(
_procOneVoxelTimeShift(
val,
fmridata[val, :],
lagstrengths[val],
R2[val],
lagtimes[val],
padtrs,
fmritr,
theprefilter,
optiondict["fmrifreq"],
refineprenorm=optiondict["refineprenorm"],
lagmaxthresh=optiondict["lagmaxthresh"],
refineweighting=optiondict["refineweighting"],
detrendorder=optiondict["detrendorder"],
offsettime=optiondict["offsettime"],
filterbeforePCA=optiondict["filterbeforePCA"],
psdfilter=optiondict["psdfilter"],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
timeshift_consumer,
inputshape,
shiftmask,
nprocs=optiondict["nprocs"],
showprogressbar=True,
chunksize=optiondict["mp_chunksize"],
)
# unpack the data
psdlist = []
for voxel in data_out:
shiftedtcs[voxel[0], :] = voxel[1]
weights[voxel[0], :] = voxel[2]
if optiondict["psdfilter"]:
psdlist.append(voxel[3])
del data_out
else:
psdlist = []
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox
== inputshape[0] - 1) and optiondict["showprogressbar"]:
tide_util.progressbar(vox + 1,
inputshape[0],
label="Percent complete (timeshifting)")
if shiftmask[vox] > 0.5:
retvals = _procOneVoxelTimeShift(
vox,
fmridata[vox, :],
lagstrengths[vox],
R2[vox],
lagtimes[vox],
padtrs,
fmritr,
theprefilter,
optiondict["fmrifreq"],
refineprenorm=optiondict["refineprenorm"],
lagmaxthresh=optiondict["lagmaxthresh"],
refineweighting=optiondict["refineweighting"],
detrendorder=optiondict["detrendorder"],
offsettime=optiondict["offsettime"],
filterbeforePCA=optiondict["filterbeforePCA"],
psdfilter=optiondict["psdfilter"],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
shiftedtcs[retvals[0], :] = retvals[1]
weights[retvals[0], :] = retvals[2]
if optiondict["psdfilter"]:
psdlist.append(retvals[3])
print()
if optiondict["psdfilter"]:
print(len(psdlist))
print(psdlist[0])
print(np.shape(np.asarray(psdlist, dtype=rt_floattype)))
averagepsd = np.mean(np.asarray(psdlist, dtype=rt_floattype), axis=0)
stdpsd = np.std(np.asarray(psdlist, dtype=rt_floattype), axis=0)
snr = np.nan_to_num(averagepsd / stdpsd)
# now generate the refined timecourse(s)
validlist = np.where(refinemask > 0)[0]
refinevoxels = shiftedtcs[validlist, :]
if bipolar:
for thevoxel in range(len(validlist)):
if lagstrengths[validlist][thevoxel] < 0.0:
refinevoxels[thevoxel, :] *= -1.0
refineweights = weights[validlist]
weightsum = np.sum(refineweights, axis=0) / volumetotal
averagedata = np.sum(refinevoxels, axis=0) / volumetotal
if optiondict["cleanrefined"]:
invalidlist = np.where((1 - ampmask) > 0)[0]
discardvoxels = shiftedtcs[invalidlist]
discardweights = weights[invalidlist]
discardweightsum = np.sum(discardweights, axis=0) / volumetotal
averagediscard = np.sum(discardvoxels, axis=0) / volumetotal
if optiondict["dodispersioncalc"]:
print("splitting regressors by time lag for phase delay estimation")
laglist = np.arange(
optiondict["dispersioncalc_lower"],
optiondict["dispersioncalc_upper"],
optiondict["dispersioncalc_step"],
)
dispersioncalcout = np.zeros((np.shape(laglist)[0], inputshape[1]),
dtype=rt_floattype)
fftlen = int(inputshape[1] // 2)
fftlen -= fftlen % 2
dispersioncalcspecmag = np.zeros((np.shape(laglist)[0], fftlen),
dtype=rt_floattype)
dispersioncalcspecphase = np.zeros((np.shape(laglist)[0], fftlen),
dtype=rt_floattype)
for lagnum in range(0, np.shape(laglist)[0]):
lower = laglist[lagnum] - optiondict["dispersioncalc_step"] / 2.0
upper = laglist[lagnum] + optiondict["dispersioncalc_step"] / 2.0
inlagrange = np.where(
locationmask * ampmask *
np.where(lower < lagtimes, np.int16(1), np.int16(0)) *
np.where(lagtimes < upper, np.int16(1), np.int16(0)))[0]
print(
" summing",
np.shape(inlagrange)[0],
"regressors with lags from",
lower,
"to",
upper,
)
if np.shape(inlagrange)[0] > 0:
dispersioncalcout[lagnum, :] = tide_math.corrnormalize(
np.mean(shiftedtcs[inlagrange], axis=0),
detrendorder=optiondict["detrendorder"],
windowfunc=optiondict["windowfunc"],
)
(
freqs,
dispersioncalcspecmag[lagnum, :],
dispersioncalcspecphase[lagnum, :],
) = tide_math.polarfft(dispersioncalcout[lagnum, :],
1.0 / fmritr)
inlagrange = None
tide_io.writenpvecs(
dispersioncalcout,
optiondict["outputname"] + "_dispersioncalcvecs_pass" +
str(passnum) + ".txt",
)
tide_io.writenpvecs(
dispersioncalcspecmag,
optiondict["outputname"] + "_dispersioncalcspecmag_pass" +
str(passnum) + ".txt",
)
tide_io.writenpvecs(
dispersioncalcspecphase,
optiondict["outputname"] + "_dispersioncalcspecphase_pass" +
str(passnum) + ".txt",
)
tide_io.writenpvecs(
freqs,
optiondict["outputname"] + "_dispersioncalcfreqs_pass" +
str(passnum) + ".txt",
)
if optiondict["pcacomponents"] < 0.0:
pcacomponents = "mle"
elif optiondict["pcacomponents"] >= 1.0:
pcacomponents = int(np.round(optiondict["pcacomponents"]))
elif optiondict["pcacomponents"] == 0.0:
print("0.0 is not an allowed value for pcacomponents")
sys.exit()
else:
pcacomponents = optiondict["pcacomponents"]
icacomponents = 1
if optiondict["refinetype"] == "ica":
print("performing ica refinement")
thefit = FastICA(n_components=icacomponents).fit(
refinevoxels) # Reconstruct signals
print("Using first of ", len(thefit.components_), " components")
icadata = thefit.components_[0]
filteredavg = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], averagedata),
detrendorder=optiondict["detrendorder"],
)
filteredica = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], icadata),
detrendorder=optiondict["detrendorder"],
)
thepxcorr = pearsonr(filteredavg, filteredica)[0]
print("ica/avg correlation = ", thepxcorr)
if thepxcorr > 0.0:
outputdata = 1.0 * icadata
else:
outputdata = -1.0 * icadata
elif optiondict["refinetype"] == "pca":
# use the method of "A novel perspective to calibrate temporal delays in cerebrovascular reactivity
# using hypercapnic and hyperoxic respiratory challenges". NeuroImage 187, 154?165 (2019).
print("performing pca refinement with pcacomponents set to",
pcacomponents)
try:
thefit = PCA(n_components=pcacomponents).fit(refinevoxels)
except ValueError:
if pcacomponents == "mle":
print(
"mle estimation failed - falling back to pcacomponents=0.8"
)
thefit = PCA(n_components=0.8).fit(refinevoxels)
else:
print("unhandled math exception in PCA refinement - exiting")
sys.exit()
print(
"Using ",
len(thefit.components_),
" component(s), accounting for ",
"{:.2f}% of the variance".format(100.0 * np.cumsum(
thefit.explained_variance_ratio_)[len(thefit.components_) -
1]),
)
reduceddata = thefit.inverse_transform(thefit.transform(refinevoxels))
if debug:
print("complex processing: reduceddata.shape =", reduceddata.shape)
pcadata = np.mean(reduceddata, axis=0)
filteredavg = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], averagedata),
detrendorder=optiondict["detrendorder"],
)
filteredpca = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], pcadata),
detrendorder=optiondict["detrendorder"],
)
thepxcorr = pearsonr(filteredavg, filteredpca)[0]
print("pca/avg correlation = ", thepxcorr)
if thepxcorr > 0.0:
outputdata = 1.0 * pcadata
else:
outputdata = -1.0 * pcadata
elif optiondict["refinetype"] == "weighted_average":
print("performing weighted averaging refinement")
outputdata = np.nan_to_num(averagedata / weightsum)
else:
print("performing unweighted averaging refinement")
outputdata = averagedata
if optiondict["cleanrefined"]:
thefit, R = tide_fit.mlregress(averagediscard, averagedata)
fitcoff = rt_floatset(thefit[0, 1])
datatoremove = rt_floatset(fitcoff * averagediscard)
outputdata -= datatoremove
print()
print(
"Timeshift applied to " + str(int(volumetotal)) + " voxels, " +
str(len(validlist)) + " used for refinement:",
"\n ",
locationfails,
" locationfails",
"\n ",
ampfails,
" ampfails",
"\n ",
lagfails,
" lagfails",
"\n ",
sigmafails,
" sigmafails",
)
if optiondict["psdfilter"]:
outputdata = tide_filt.transferfuncfilt(outputdata, snr)
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal, outputdata, refinemask, locationfails, ampfails, lagfails, sigmafails
def polyfitim(
datafile,
datamask,
templatefile,
templatemask,
outputroot,
regionatlas=None,
order=1,
):
# read in data
print("reading in data arrays")
(
datafile_img,
datafile_data,
datafile_hdr,
datafiledims,
datafilesizes,
) = tide_io.readfromnifti(datafile)
(
datamask_img,
datamask_data,
datamask_hdr,
datamaskdims,
datamasksizes,
) = tide_io.readfromnifti(datamask)
(
templatefile_img,
templatefile_data,
templatefile_hdr,
templatefiledims,
templatefilesizes,
) = tide_io.readfromnifti(templatefile)
(
templatemask_img,
templatemask_data,
templatemask_hdr,
templatemaskdims,
templatemasksizes,
) = tide_io.readfromnifti(templatemask)
if regionatlas is not None:
(
regionatlas_img,
regionatlas_data,
regionatlas_hdr,
regionatlasdims,
regionatlassizes,
) = tide_io.readfromnifti(regionatlas)
xsize = datafiledims[1]
ysize = datafiledims[2]
numslices = datafiledims[3]
timepoints = datafiledims[4]
# check dimensions
print("checking dimensions")
if not tide_io.checkspacedimmatch(datafiledims, datamaskdims):
print("input mask spatial dimensions do not match image")
exit()
if datamaskdims[4] == 1:
print("using 3d data mask")
datamask3d = True
else:
datamask3d = False
if not tide_io.checktimematch(datafiledims, datamaskdims):
print("input mask time dimension does not match image")
exit()
if not tide_io.checkspacedimmatch(datafiledims, templatefiledims):
print(templatefiledims,
"template file spatial dimensions do not match image")
exit()
if not templatefiledims[4] == 1:
print("template file time dimension is not equal to 1")
exit()
if not tide_io.checkspacedimmatch(datafiledims, templatemaskdims):
print("template mask spatial dimensions do not match image")
exit()
if not templatemaskdims[4] == 1:
print("template mask time dimension is not equal to 1")
exit()
if regionatlas is not None:
if not tide_io.checkspacedimmatch(datafiledims, regionatlasdims):
print("template mask spatial dimensions do not match image")
exit()
if not regionatlasdims[4] == 1:
print("regionatlas time dimension is not equal to 1")
exit()
# allocating arrays
print("allocating arrays")
numspatiallocs = int(xsize) * int(ysize) * int(numslices)
rs_datafile = datafile_data.reshape((numspatiallocs, timepoints))
if datamask3d:
rs_datamask = datamask_data.reshape(numspatiallocs)
else:
rs_datamask = datamask_data.reshape((numspatiallocs, timepoints))
rs_datamask_bin = np.where(rs_datamask > 0.9, 1.0, 0.0)
rs_templatefile = templatefile_data.reshape(numspatiallocs)
rs_templatemask = templatemask_data.reshape(numspatiallocs)
rs_templatemask_bin = np.where(rs_templatemask > 0.1, 1.0, 0.0)
if regionatlas is not None:
rs_regionatlas = regionatlas_data.reshape(numspatiallocs)
numregions = int(np.max(rs_regionatlas))
fitdata = np.zeros((numspatiallocs, timepoints), dtype="float")
# residuals = np.zeros((numspatiallocs, timepoints), dtype='float')
# newtemplate = np.zeros((numspatiallocs), dtype='float')
# newmask = np.zeros((numspatiallocs), dtype='float')
if regionatlas is not None:
lincoffs = np.zeros((numregions, timepoints), dtype="float")
sqrcoffs = np.zeros((numregions, timepoints), dtype="float")
offsets = np.zeros((numregions, timepoints), dtype="float")
rvals = np.zeros((numregions, timepoints), dtype="float")
else:
lincoffs = np.zeros(timepoints, dtype="float")
sqrcoffs = np.zeros(timepoints, dtype="float")
offsets = np.zeros(timepoints, dtype="float")
rvals = np.zeros(timepoints, dtype="float")
if regionatlas is not None:
print("making region masks")
regionvoxels = np.zeros((numspatiallocs, numregions), dtype="float")
for region in range(0, numregions):
thisregion = np.where((rs_regionatlas *
rs_templatemask_bin) == (region + 1))
regionvoxels[thisregion, region] = 1.0
# mask everything
print("masking template")
maskedtemplate = rs_templatefile * rs_templatemask_bin
# cycle over all images
print("now cycling over all images")
for thetime in range(0, timepoints):
print("fitting timepoint", thetime)
# get the appropriate mask
if datamask3d:
for i in range(timepoints):
thisdatamask = rs_datamask_bin
else:
thisdatamask = rs_datamask_bin[:, thetime]
if regionatlas is not None:
for region in range(0, numregions):
voxelstofit = np.where(
regionvoxels[:, region] * thisdatamask > 0.5)
voxelstoreconstruct = np.where(regionvoxels[:, region] > 0.5)
if order == 2:
thefit, R = tide_fit.mlregress(
[
rs_templatefile[voxelstofit],
np.square(rs_templatefile[voxelstofit]),
],
rs_datafile[voxelstofit, thetime][0],
)
else:
thefit, R = tide_fit.mlregress(
rs_templatefile[voxelstofit],
rs_datafile[voxelstofit, thetime][0],
)
lincoffs[region, thetime] = thefit[0, 1]
offsets[region, thetime] = thefit[0, 0]
rvals[region, thetime] = R
if order == 2:
sqrcoffs[region, thetime] = thefit[0, 2]
fitdata[voxelstoreconstruct, thetime] += (
sqrcoffs[region, thetime] *
np.square(rs_templatefile[voxelstoreconstruct]) +
lincoffs[region, thetime] *
rs_templatefile[voxelstoreconstruct] +
offsets[region, thetime])
else:
fitdata[voxelstoreconstruct,
thetime] += (lincoffs[region, thetime] *
rs_templatefile[voxelstoreconstruct] +
offsets[region, thetime])
# newtemplate += nan_to_num(maskeddata[:, thetime] / lincoffs[region, thetime]) * rs_datamask
# newmask += rs_datamask * rs_templatemask_bin
else:
voxelstofit = np.where(thisdatamask > 0.5)
voxelstoreconstruct = np.where(rs_templatemask > 0.5)
thefit, R = tide_fit.mlregress(
rs_templatefile[voxelstofit], rs_datafile[voxelstofit,
thetime][0])
lincoffs[thetime] = thefit[0, 1]
offsets[thetime] = thefit[0, 0]
rvals[thetime] = R
fitdata[voxelstoreconstruct, thetime] = (
lincoffs[thetime] * rs_templatefile[voxelstoreconstruct] +
offsets[thetime])
# if datamask3d:
# newtemplate += nan_to_num(maskeddata[:, thetime] / lincoffs[thetime]) * rs_datamask
# else:
# newtemplate += nan_to_num(maskeddata[:, thetime] / lincoffs[thetime]) * rs_datamask[:, thetime]
# newmask += rs_datamask[:, thetime] * rs_templatemask_bin
residuals = rs_datafile - fitdata
# write out the data files
print("writing time series")
if order == 2:
tide_io.writenpvecs(sqrcoffs, outputroot + "_sqrcoffs.txt")
tide_io.writenpvecs(lincoffs, outputroot + "_lincoffs.txt")
tide_io.writenpvecs(offsets, outputroot + "_offsets.txt")
tide_io.writenpvecs(rvals, outputroot + "_rvals.txt")
if regionatlas is not None:
for region in range(0, numregions):
print(
"region",
region + 1,
"slope mean, std:",
np.mean(lincoffs[:, region]),
np.std(lincoffs[:, region]),
)
print(
"region",
region + 1,
"offset mean, std:",
np.mean(offsets[:, region]),
np.std(offsets[:, region]),
)
else:
print("slope mean, std:", np.mean(lincoffs), np.std(lincoffs))
print("offset mean, std:", np.mean(offsets), np.std(offsets))
print("writing nifti series")
tide_io.savetonifti(
fitdata.reshape((xsize, ysize, numslices, timepoints)),
datafile_hdr,
outputroot + "_fit",
)
tide_io.savetonifti(
residuals.reshape((xsize, ysize, numslices, timepoints)),
datafile_hdr,
outputroot + "_residuals",
)
def showxcorrx_workflow(infilename1, infilename2, Fs,
thelabel='', starttime=0., duration=1000000.,
searchrange=15.,
display=True, trimdata=False,
summarymode=False, labelline=False,
flipregressor=False, windowfunc='hamming',
calccepstraldelay=False, corroutputfile=False,
controlvariablefile=None, numreps=0,
arbvec=None, filtertype='arb', corrweighting='none',
detrendorder=1, prewindow=True, verbose=False):
r"""Calculate and display crosscorrelation between two timeseries.
Parameters
----------
infilename1 : str
The name of a text file containing a timeseries, one timepoint per line.
infilename2 : str
The name of a text file containing a timeseries, one timepoint per line.
Fs : float
The sample rate of the time series, in Hz.
thelabel : str, optional
The label for the output graph. Default is blank.
starttime : float, optional
Time offset into the timeseries, in seconds, to start using the time data. Default is 0
duration : float, optional
Length of time from each time series, in seconds, to use for the cross-correlation. Default is the entire time series.
searchrange : float, optional
Only search for cross-correlation peaks between -searchrange and +searchrange seconds (default is 15).
display : bool, optional
Plot cross-correlation function in a matplotlib window. Default is True.
trimdata : bool, optional
Trim time series to the length of the shorter series. Default is False.
summarymode : bool, optional
Output a table of interesting results for later processing. Default is False.
labelline : bool, optional
Print an explanatory header line over the summary information. Default is False.
flipregressor : bool, optional
Invert timeseries 2 prior to cross-correlation.
windowfunc : {'hamming', 'hann', 'blackmanharris'}
Window function to apply prior to cross-correlation. Default is 'hamming'.
calccepstraldelay : bool, optional
Use cepstral estimation of delay. Default is False.
corroutputfile : bool, optional
Save the correlation function to a file. Default is False.
controlvariablefile : bool, optional
Save internal variables to a text file. Default is False.
numreps : int, optional
Number of null correlations to perform to estimate significance. Default is 10000
arbvec : [float,float,float,float], optional
Frequency limits of the arb_pass filter.
filtertype : 'none', 'card', 'lfo', 'vlf', 'resp', 'arb'
Type of filter to apply data prior to correlation. Default is 'none'
corrweighting : {'none', 'Liang', 'Eckart', 'PHAT'}, optional
Weighting function to apply to the crosscorrelation in the Fourier domain. Default is 'none'
detrendorder : int, optional
Order of polynomial used to detrend crosscorrelation inputs. Default is 1 (0 disables)
prewindow : bool, optional
Apply window function prior to cross-correlation. Default is True.
verbose : bool, optional
Print internal status information. Default is False.
Notes
-----
This workflow writes out several files:
If corroutputfile is defined:
====================== =================================================
Filename Content
====================== =================================================
corrlist.txt A file
corrlist_pear.txt A file
[corroutputfile] Correlation function
====================== =================================================
If debug is True:
====================== =================================================
Filename Content
====================== =================================================
filtereddata1.txt Something
filtereddata2.txt Something
====================== =================================================
"""
# Constants that could be arguments
dofftcorr = True
writecorrlists = False
debug = False
showpearson = True
# These are unnecessary and should be simplified
dopartial = bool(controlvariablefile)
uselabel = bool(thelabel)
dumpfiltered = bool(debug)
if labelline:
# TS: should prob reflect this in the parser, but it's not a big deal
summarymode = True
if numreps == 0:
estimate_significance = False
else:
estimate_significance = True
savecorrelation = bool(corroutputfile)
theprefilter = tide_filt.noncausalfilter()
if arbvec is not None and filtertype != 'arb':
raise ValueError('Argument arbvec must be None if filtertype is '
'not arb')
if arbvec is not None:
if len(arbvec) == 2:
arb_lower = float(arbvec[0])
arb_upper = float(arbvec[1])
arb_lowerstop = 0.9 * float(arbvec[0])
arb_upperstop = 1.1 * float(arbvec[1])
elif len(arbvec) == 4:
arb_lower = float(arbvec[0])
arb_upper = float(arbvec[1])
arb_lowerstop = float(arbvec[2])
arb_upperstop = float(arbvec[3])
theprefilter.settype('arb')
theprefilter.setarb(arb_lowerstop, arb_lower, arb_upper, arb_upperstop)
else:
theprefilter.settype(filtertype)
inputdata1 = tide_io.readvec(infilename1)
inputdata2 = tide_io.readvec(infilename2)
numpoints = len(inputdata1)
startpoint1 = max([int(starttime * Fs), 0])
if debug:
print('startpoint set to ', startpoint1)
endpoint1 = min([startpoint1 + int(duration * Fs), int(len(inputdata1))])
if debug:
print('endpoint set to ', endpoint1)
endpoint2 = min([int(duration * Fs), int(len(inputdata1)),
int(len(inputdata2))])
trimdata1 = inputdata1[startpoint1:endpoint1]
trimdata2 = inputdata2[0:endpoint2]
if trimdata:
minlen = np.min([len(trimdata1), len(trimdata2)])
trimdata1 = trimdata1[0:minlen]
trimdata2 = trimdata2[0:minlen]
# band limit the regressor if that is needed
if theprefilter.gettype() != 'none':
if verbose:
print("filtering to ", theprefilter.gettype(), " band")
print(windowfunc)
filtereddata1 = tide_math.corrnormalize(theprefilter.apply(Fs, trimdata1),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
filtereddata2 = tide_math.corrnormalize(theprefilter.apply(Fs, trimdata2),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc)
if flipregressor:
filtereddata2 *= -1.0
if dumpfiltered:
tide_io.writenpvecs(filtereddata1, 'filtereddata1.txt')
tide_io.writenpvecs(filtereddata2, 'filtereddata2.txt')
if dopartial:
controlvars = tide_io.readvecs(controlvariablefile)
numregressors = len(controlvars) # Added by TS. Not sure if works.
regressorvec = []
for j in range(0, numregressors):
regressorvec.append(tide_math.corrnormalize(
theprefilter.apply(Fs, controlvars[j, :]),
prewindow=prewindow,
detrendorder=detrendorder,
windowfunc=windowfunc))
if (np.max(filtereddata1) - np.min(filtereddata1)) > 0.0:
thefit, filtereddata1 = tide_fit.mlregress(regressorvec,
filtereddata1)
if (np.max(filtereddata2) - np.min(filtereddata2)) > 0.0:
thefit, filtereddata2 = tide_fit.mlregress(regressorvec,
filtereddata2)
thexcorr = tide_corr.fastcorrelate(filtereddata1, filtereddata2,
usefft=dofftcorr,
weighting=corrweighting,
displayplots=debug)
if calccepstraldelay:
cepdelay = tide_corr.cepstraldelay(filtereddata1, filtereddata2,
1.0 / Fs, displayplots=display)
cepcoff = tide_corr.delayedcorr(filtereddata1, filtereddata2, cepdelay,
1.0 / Fs)
print('cepstral delay time is {0}, correlation is {1}'.format(cepdelay,
cepcoff))
thepxcorr = pearsonr(filtereddata1, filtereddata2)
# calculate the coherence
f, Cxy = sp.signal.coherence(
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata1), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata2), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
Fs)
# calculate the cross spectral density
f, Pxy = sp.signal.csd(
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata1), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
tide_math.corrnormalize(theprefilter.apply(Fs, trimdata2), prewindow=prewindow,
detrendorder=detrendorder, windowfunc=windowfunc),
Fs)
xcorrlen = len(thexcorr)
sampletime = 1.0 / Fs
xcorr_x = r_[0:xcorrlen] * sampletime - (xcorrlen * sampletime) / 2.0\
+ sampletime / 2.0
halfwindow = int(searchrange * Fs)
corrzero = xcorrlen // 2
searchstart = corrzero - halfwindow
searchend = corrzero + halfwindow
xcorr_x_trim = xcorr_x[searchstart:searchend + 1]
thexcorr_trim = thexcorr[searchstart:searchend + 1]
if debug:
print('searching for peak correlation over range ', searchstart,
searchend)
maxdelay = xcorr_x_trim[argmax(thexcorr_trim)]
if debug:
print('maxdelay before refinement', maxdelay)
dofindmaxlag = True
if dofindmaxlag:
print('executing findmaxlag')
(maxindex, maxdelay, maxval, maxsigma, maskval, failreason, peakstart,
peakend) = tide_fit.findmaxlag_gauss(
xcorr_x_trim, thexcorr_trim, -searchrange, searchrange, 1000.0,
refine=True,
useguess=False,
fastgauss=False,
displayplots=False)
print(maxindex, maxdelay, maxval, maxsigma, maskval, failreason)
R = maxval
if debug:
print('maxdelay after refinement', maxdelay)
if failreason > 0:
print('failreason =', failreason)
else:
R = thexcorr_trim[argmax(thexcorr_trim)]
# set the significance threshold
if estimate_significance:
# generate a list of correlations from shuffled data
(corrlist,
corrlist_pear) = _get_null_distribution(trimdata1, xcorr_x,
theprefilter, prewindow,
detrendorder, searchstart,
searchend, Fs, dofftcorr,
corrweighting=corrweighting,
numreps=numreps,
windowfunc=windowfunc)
# calculate percentiles for the crosscorrelation from the distribution
histlen = 100
thepercentiles = [0.95, 0.99, 0.995]
(pcts, pcts_fit,
histfit) = tide_stats.sigFromDistributionData(corrlist, histlen,
thepercentiles)
if debug:
tide_stats.printthresholds(pcts, thepercentiles,
('Crosscorrelation significance '
'thresholds from data:'))
tide_stats.printthresholds(pcts_fit, thepercentiles,
('Crosscorrelation significance '
'thresholds from fit:'))
# calculate significance for the pearson correlation
(pearpcts, pearpcts_fit,
histfit) = tide_stats.sigFromDistributionData(corrlist_pear, histlen,
thepercentiles)
if debug:
tide_stats.printthresholds(pearpcts, thepercentiles,
('Pearson correlation significance '
'thresholds from data:'))
tide_stats.printthresholds(pearpcts_fit, thepercentiles,
('Pearson correlation significance '
'thresholds from fit:'))
if writecorrlists:
tide_io.writenpvecs(corrlist, 'corrlist.txt')
tide_io.writenpvecs(corrlist_pear, 'corrlist_pear.txt')
def printthresholds(pcts, thepercentiles, labeltext):
print(labeltext)
for i in range(0, len(pcts)):
print('\tp <', "{:.3f}".format(1.0 - thepercentiles[i]), ': ',
pcts[i])
# report the pearson correlation
if showpearson and verbose:
print('Pearson_R:\t', thepxcorr[0])
if estimate_significance:
for idx, percentile in enumerate(thepercentiles):
print(' pear_p(', "{:.3f}".format(1.0 - percentile), '):\t',
pearpcts[idx])
print("")
if debug:
print(thepxcorr)
if verbose:
if uselabel:
print(thelabel, ":\t", maxdelay)
else:
print("Crosscorrelation_Rmax:\t", R)
print("Crosscorrelation_maxdelay:\t", maxdelay)
if estimate_significance:
for idx, percentile in enumerate(thepercentiles):
print(' xc_p(', "{:.3f}".format(1.0 - percentile),
'):\t', pcts[idx])
print(infilename1, "[0 seconds] == ", infilename2, "[",
-1 * maxdelay, " seconds]")
if summarymode:
if estimate_significance:
if uselabel:
if labelline:
print('thelabel', 'pearson_R', 'pearson_R(p=0.05)',
'xcorr_R', 'xcorr_R(P=0.05)', 'xcorr_maxdelay')
print(thelabel, thepxcorr[0], pearpcts_fit[0], R, pcts_fit[0],
-1 * maxdelay)
else:
if labelline:
print('pearson_R', 'pearson_R(p=0.05)', 'xcorr_R',
'xcorr_R(P=0.05)', 'xcorr_maxdelay')
print(thepxcorr[0], pearpcts_fit[0], R, pcts_fit[0],
-1 * maxdelay)
else:
if uselabel:
if labelline:
print('thelabel', 'pearson_r', 'pearson_p', 'xcorr_R',
'xcorr_maxdelay')
print(thelabel, thepxcorr[0], thepxcorr[1], R, -1 * maxdelay)
else:
if labelline:
print('pearson_r\tpearson_p\txcorr_R\txcorr_t\t'
'xcorr_maxdelay')
print(thepxcorr[0], '\t', thepxcorr[1], '\t', R, '\t',
-1 * maxdelay)
if savecorrelation:
tide_io.writenpvecs(np.stack((xcorr_x, thexcorr), axis=0),
corroutputfile)
if display:
fig, ax = plt.subplots()
# ax.set_title('GCC')
ax.plot(xcorr_x, thexcorr, 'k')
if debug:
fig, ax = plt.subplots()
ax.plot(f, Cxy)
fig = plt.subplots()
ax.plot(f, np.sqrt(np.abs(Pxy)) / np.max(np.sqrt(np.abs(Pxy))))
ax.plot(f, np.angle(Pxy) / (2.0 * pi * f))
fig.show()