def wienerpass(numspatiallocs,
reportstep,
fmri_data,
threshval,
lagtc,
optiondict,
meanvalue,
rvalue,
r2value,
fitcoff,
fitNorm,
datatoremove,
filtereddata,
rt_floatset=np.float64,
rt_floattype='float64'):
inputshape = np.shape(fmri_data)
themask = np.where(np.mean(fmri_data, axis=1) > threshval, 1, 0)
if optiondict['nprocs'] > 1:
# define the consumer function here so it inherits most of the arguments
def Wiener_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(_procOneVoxelWiener(val,
lagtc[val, :],
fmri_data[val,
optiondict['addedskip']:],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(Wiener_consumer,
inputshape, themask,
nprocs=optiondict['nprocs'],
showprogressbar=True,
chunksize=optiondict['mp_chunksize'])
# unpack the data
volumetotal = 0
for voxel in data_out:
meanvalue[voxel[0]] = voxel[1]
rvalue[voxel[0]] = voxel[2]
r2value[voxel[0]] = voxel[3]
fitcoff[voxel[0]] = voxel[4]
fitNorm[voxel[0]] = voxel[5]
datatoremove[voxel[0], :] = voxel[6]
filtereddata[voxel[0], :] = voxel[7]
volumetotal += 1
data_out = []
else:
volumetotal = 0
for vox in range(0, numspatiallocs):
if (vox % reportstep == 0 or vox == numspatiallocs - 1) and optiondict['showprogressbar']:
tide_util.progressbar(vox + 1, numspatiallocs, label='Percent complete')
inittc = fmri_data[vox, optiondict['addedskip']:].copy()
if np.mean(inittc) >= threshval:
dummy, meanvalue[vox], rvalue[vox], r2value[vox], fitcoff[vox], fitNorm[vox], datatoremove[vox], \
filtereddata[vox] = _procOneVoxelWiener(vox,
lagtc[vox, :],
inittc,
rt_floatset=rt_floatset,
t_floattype=rt_floattype)
volumetotal += 1
return volumetotal
def glmpass(numprocitems,
fmri_data,
threshval,
lagtc,
meanvalue,
rvalue,
r2value,
fitcoff,
fitNorm,
datatoremove,
filtereddata,
reportstep=1000,
nprocs=1,
procbyvoxel=True,
showprogressbar=True,
addedskip=0,
mp_chunksize=1000,
rt_floatset=np.float64,
rt_floattype='float64'):
inputshape = np.shape(fmri_data)
if threshval is None:
themask = None
else:
themask = np.where(np.mean(fmri_data, axis=1) > threshval, 1, 0)
if nprocs > 1:
# define the consumer function here so it inherits most of the arguments
def GLM_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
if procbyvoxel:
outQ.put(_procOneItemGLM(val,
lagtc[val, :],
fmri_data[val, addedskip:],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
else:
outQ.put(_procOneItemGLM(val,
lagtc[:, val],
fmri_data[:, addedskip + val],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(GLM_consumer,
inputshape, themask,
nprocs=nprocs,
procbyvoxel=procbyvoxel,
showprogressbar=True,
chunksize=mp_chunksize)
# unpack the data
itemstotal = 0
if procbyvoxel:
for voxel in data_out:
meanvalue[voxel[0]] = voxel[1]
rvalue[voxel[0]] = voxel[2]
r2value[voxel[0]] = voxel[3]
fitcoff[voxel[0]] = voxel[4]
fitNorm[voxel[0]] = voxel[5]
datatoremove[voxel[0], :] = voxel[6]
filtereddata[voxel[0], :] = voxel[7]
itemstotal += 1
else:
for timepoint in data_out:
meanvalue[timepoint[0]] = timepoint[1]
rvalue[timepoint[0]] = timepoint[2]
r2value[timepoint[0]] = timepoint[3]
fitcoff[timepoint[0]] = timepoint[4]
fitNorm[timepoint[0]] = timepoint[5]
datatoremove[:, timepoint[0]] = timepoint[6]
filtereddata[:, timepoint[0]] = timepoint[7]
itemstotal += 1
del data_out
else:
itemstotal = 0
if procbyvoxel:
for vox in range(0, numprocitems):
if (vox % reportstep == 0 or vox == numprocitems - 1) and showprogressbar:
tide_util.progressbar(vox + 1, numprocitems, label='Percent complete')
inittc = fmri_data[vox, addedskip:].copy()
if themask[vox] > 0:
dummy, \
meanvalue[vox],\
rvalue[vox], \
r2value[vox], \
fitcoff[vox], \
fitNorm[vox], \
datatoremove[vox, :], \
filtereddata[vox, :] = \
_procOneItemGLM(vox,
lagtc[vox, :],
inittc,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype)
itemstotal += 1
else:
for timepoint in range(0, numprocitems):
if (timepoint % reportstep == 0 or timepoint == numprocitems - 1) and showprogressbar:
tide_util.progressbar(timepoint + 1, numprocitems, label='Percent complete')
inittc = fmri_data[:, addedskip + timepoint].copy()
if themask[timepoint] > 0:
dummy, \
meanvalue[timepoint], \
rvalue[timepoint], \
r2value[timepoint], \
fitcoff[timepoint], \
fitNorm[timepoint], \
datatoremove[:, timepoint], \
filtereddata[:, timepoint] = \
_procOneItemGLM(timepoint,
lagtc[:, timepoint],
inittc,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype)
itemstotal += 1
return itemstotal
def wienerpass(
numspatiallocs,
reportstep,
fmri_data,
threshval,
lagtc,
optiondict,
wienerdeconv,
wpeak,
resampref_y,
rt_floatset=np.float64,
rt_floattype="float64",
):
rt_floatset = (rt_floatset,)
rt_floattype = rt_floattype
inputshape = np.shape(fmri_data)
themask = np.where(np.mean(fmri_data, axis=1) > threshval, 1, 0)
if optiondict["nprocs"] > 1:
# define the consumer function here so it inherits most of the arguments
def Wiener_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(
_procOneVoxelWiener(
val,
lagtc[val, :],
fmri_data[val, :],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
)
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
Wiener_consumer,
inputshape,
themask,
nprocs=optiondict["nprocs"],
showprogressbar=True,
chunksize=optiondict["mp_chunksize"],
)
# unpack the data
volumetotal = 0
for voxel in data_out:
meanvalue[voxel[0]] = voxel[1]
rvalue[voxel[0]] = voxel[2]
r2value[voxel[0]] = voxel[3]
fitcoff[voxel[0]] = voxel[4]
fitNorm[voxel[0]] = voxel[5]
datatoremove[voxel[0], :] = voxel[6]
filtereddata[voxel[0], :] = voxel[7]
volumetotal += 1
data_out = []
else:
volumetotal = 0
for vox in range(0, numspatiallocs):
if (vox % reportstep == 0 or vox == numspatiallocs - 1) and optiondict[
"showprogressbar"
]:
tide_util.progressbar(vox + 1, numspatiallocs, label="Percent complete")
inittc = fmri_data[vox, :].copy()
if np.mean(inittc) >= threshval:
(
dummy,
meanvalue[vox],
rvalue[vox],
r2value[vox],
fitcoff[vox],
fitNorm[vox],
datatoremove[vox],
filtereddata[vox],
) = _procOneVoxelWiener(
vox, lagtc[vox, :], inittc, rt_floatset=rt_floatset, t_floattype=rt_floattype,
)
volumetotal += 1
return volumetotal
def getNullDistributionData(indata,
corrscale,
ncprefilter,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
optiondict,
rt_floatset=np.float64,
rt_floattype='float64'
):
inputshape = np.asarray([optiondict['numestreps']])
if optiondict['nprocs'] > 1:
# define the consumer function here so it inherits most of the arguments
def nullCorrelation_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(_procOneNullCorrelation(val,
indata,
ncprefilter,
oversampfreq,
corrscale,
corrorigin,
lagmininpts,
lagmaxinpts,
optiondict,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype
))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(nullCorrelation_consumer,
inputshape, None,
nprocs=optiondict['nprocs'],
showprogressbar=True,
chunksize=optiondict['mp_chunksize'])
# unpack the data
volumetotal = 0
corrlist = np.asarray(data_out, dtype=rt_floattype)
else:
corrlist = np.zeros((optiondict['numestreps']), dtype=rt_floattype)
for i in range(0, optiondict['numestreps']):
# make a shuffled copy of the regressors
shuffleddata = np.random.permutation(indata)
# crosscorrelate with original
thexcorr, dummy = tide_corrpass.onecorrelation(shuffleddata,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
ncprefilter,
indata,
usewindowfunc=optiondict['usewindowfunc'],
detrendorder=optiondict['detrendorder'],
windowfunc=optiondict['windowfunc'],
corrweighting=optiondict['corrweighting'])
# fit the correlation
maxindex, maxlag, maxval, maxsigma, maskval, failreason = \
tide_corrfit.onecorrfit(thexcorr,
corrscale[corrorigin - lagmininpts:corrorigin + lagmaxinpts],
optiondict,
zerooutbadfit=True,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype
)
# find and tabulate correlation coefficient at optimal lag
corrlist[i] = maxval
# progress
if optiondict['showprogressbar']:
tide_util.progressbar(i + 1, optiondict['numestreps'], label='Percent complete')
# jump to line after progress bar
print()
# return the distribution data
numnonzero = len(np.where(corrlist != 0.0)[0])
print(numnonzero, 'non-zero correlations out of', len(corrlist), '(', 100.0 * numnonzero / len(corrlist), '%)')
return corrlist
def getNullDistributionDatax(indata,
corrscale,
ncprefilter,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
optiondict,
rt_floatset=np.float64,
rt_floattype='float64'):
inputshape = np.asarray([optiondict['numestreps']])
if optiondict['nprocs'] > 1:
# define the consumer function here so it inherits most of the arguments
def nullCorrelation_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(_procOneNullCorrelationx(val, indata, ncprefilter, oversampfreq, corrscale, corrorigin,
lagmininpts, lagmaxinpts, optiondict,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(nullCorrelation_consumer,
inputshape, None,
nprocs=optiondict['nprocs'],
showprogressbar=True,
chunksize=optiondict['mp_chunksize'])
# unpack the data
corrlist = np.asarray(data_out, dtype=rt_floattype)
else:
corrlist = np.zeros((optiondict['numestreps']), dtype=rt_floattype)
for i in range(0, optiondict['numestreps']):
# make a shuffled copy of the regressors
shuffleddata = np.random.permutation(indata)
# crosscorrelate with original, fit, and return the maximum value, and add it to the list
thexcorr = _procOneNullCorrelationx(i, indata, ncprefilter, oversampfreq, corrscale, corrorigin,
lagmininpts, lagmaxinpts, optiondict,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype)
corrlist[i] = thexcorr
# progress
if optiondict['showprogressbar']:
tide_util.progressbar(i + 1, optiondict['numestreps'], label='Percent complete')
# jump to line after progress bar
print()
# return the distribution data
numnonzero = len(np.where(corrlist != 0.0)[0])
print(numnonzero, 'non-zero correlations out of', len(corrlist), '(', 100.0 * numnonzero / len(corrlist), '%)')
return corrlist
def correlationpass(fmridata,
fmrifftdata,
referencetc,
fmri_x,
os_fmri_x,
tr,
corrorigin,
lagmininpts,
lagmaxinpts,
corrout,
meanval,
ncprefilter,
optiondict,
rt_floatset=np.float64,
rt_floattype='float64'):
"""
Parameters
----------
fmridata
fmrifftdata
referencetc
fmri_x
os_fmri_x
tr
corrorigin
lagmininpts
lagmaxinpts
corrout
meanval
ncprefilter
optiondict
rt_floatset
rt_floattype
Returns
-------
"""
oversampfreq = optiondict['oversampfactor'] / tr
inputshape = np.shape(fmridata)
volumetotal = 0
reportstep = 1000
thetc = np.zeros(np.shape(os_fmri_x), dtype=rt_floattype)
theglobalmaxlist = []
if optiondict['nprocs'] > 1:
# define the consumer function here so it inherits most of the arguments
def correlation_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(_procOneVoxelCorrelation(val,
thetc,
optiondict,
fmri_x,
fmridata[val, :],
os_fmri_x,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
ncprefilter,
referencetc,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(correlation_consumer,
inputshape, None,
nprocs=optiondict['nprocs'],
showprogressbar=True,
chunksize=optiondict['mp_chunksize'])
# unpack the data
volumetotal = 0
for voxel in data_out:
# corrmask[voxel[0]] = 1
meanval[voxel[0]] = voxel[1]
corrout[voxel[0], :] = voxel[2]
theglobalmaxlist.append(voxel[3] + 0)
volumetotal += 1
del data_out
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox == inputshape[0] - 1) and optiondict['showprogressbar']:
tide_util.progressbar(vox + 1, inputshape[0], label='Percent complete')
dummy, meanval[vox], corrout[vox, :], theglobalmax = _procOneVoxelCorrelation(vox,
thetc,
optiondict,
fmri_x,
fmridata[vox, :],
os_fmri_x,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
ncprefilter,
referencetc,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype
)
theglobalmaxlist.append(theglobalmax + 0)
volumetotal += 1
print('\nCorrelation performed on ' + str(volumetotal) + ' voxels')
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal, theglobalmaxlist
def refineregressor(
fmridata,
fmritr,
shiftedtcs,
weights,
passnum,
lagstrengths,
lagtimes,
lagsigma,
lagmask,
R2,
theprefilter,
optiondict,
padtrs=60,
bipolar=False,
includemask=None,
excludemask=None,
debug=False,
rt_floatset=np.float64,
rt_floattype="float64",
):
"""
Parameters
----------
fmridata : 4D numpy float array
fMRI data
fmritr : float
Data repetition rate, in seconds
shiftedtcs : 4D numpy float array
Time aligned voxel timecourses
weights : unknown
unknown
passnum : int
Number of the pass (for labelling output)
lagstrengths : 3D numpy float array
Maximum correlation coefficient in every voxel
lagtimes : 3D numpy float array
Time delay of maximum crosscorrelation in seconds
lagsigma : 3D numpy float array
Gaussian width of the crosscorrelation peak, in seconds.
lagmask : 3D numpy float array
Mask of voxels with successful correlation fits.
R2 : 3D numpy float array
Square of the maximum correlation coefficient in every voxel
theprefilter : function
The filter function to use
optiondict : dict
Dictionary of all internal rapidtide configuration variables.
padtrs : int, optional
Number of timepoints to pad onto each end
includemask : 3D array
Mask of voxels to include in refinement. Default is None (all voxels).
excludemask : 3D array
Mask of voxels to exclude from refinement. Default is None (no voxels).
debug : bool
Enable additional debugging output. Default is False
rt_floatset : function
Function to coerce variable types
rt_floattype : {'float32', 'float64'}
Data type for internal variables
Returns
-------
volumetotal : int
Number of voxels processed
outputdata : float array
New regressor
maskarray : 3D array
Mask of voxels used for refinement
"""
inputshape = np.shape(fmridata)
if optiondict["ampthresh"] < 0.0:
if bipolar:
theampthresh = tide_stats.getfracval(np.fabs(lagstrengths),
-optiondict["ampthresh"],
nozero=True)
else:
theampthresh = tide_stats.getfracval(lagstrengths,
-optiondict["ampthresh"],
nozero=True)
print(
"setting ampthresh to the",
-100.0 * optiondict["ampthresh"],
"th percentile (",
theampthresh,
")",
)
else:
theampthresh = optiondict["ampthresh"]
if bipolar:
ampmask = np.where(
np.fabs(lagstrengths) >= theampthresh, np.int16(1), np.int16(0))
else:
ampmask = np.where(lagstrengths >= theampthresh, np.int16(1),
np.int16(0))
if optiondict["lagmaskside"] == "upper":
delaymask = np.where(
(lagtimes - optiondict["offsettime"]) > optiondict["lagminthresh"],
np.int16(1),
np.int16(0),
) * np.where(
(lagtimes - optiondict["offsettime"]) < optiondict["lagmaxthresh"],
np.int16(1),
np.int16(0),
)
elif optiondict["lagmaskside"] == "lower":
delaymask = np.where(
(lagtimes - optiondict["offsettime"]) <
-optiondict["lagminthresh"],
np.int16(1),
np.int16(0),
) * np.where(
(lagtimes - optiondict["offsettime"]) >
-optiondict["lagmaxthresh"],
np.int16(1),
np.int16(0),
)
else:
abslag = abs(lagtimes) - optiondict["offsettime"]
delaymask = np.where(abslag > optiondict["lagminthresh"], np.int16(1),
np.int16(0)) * np.where(
abslag < optiondict["lagmaxthresh"],
np.int16(1), np.int16(0))
sigmamask = np.where(lagsigma < optiondict["sigmathresh"], np.int16(1),
np.int16(0))
locationmask = lagmask + 0
if includemask is not None:
locationmask = locationmask * includemask
if excludemask is not None:
locationmask = locationmask * (1 - excludemask)
locationmask = locationmask.astype(np.int16)
print("location mask created")
# first generate the refine mask
locationfails = np.sum(1 - locationmask)
ampfails = np.sum(1 - ampmask * locationmask)
lagfails = np.sum(1 - delaymask * locationmask)
sigmafails = np.sum(1 - sigmamask * locationmask)
refinemask = locationmask * ampmask * delaymask * sigmamask
if tide_stats.getmasksize(refinemask) == 0:
print("ERROR: no voxels in the refine mask:")
print(
"\n ",
locationfails,
" locationfails",
"\n ",
ampfails,
" ampfails",
"\n ",
lagfails,
" lagfails",
"\n ",
sigmafails,
" sigmafails",
)
if (includemask is None) and (excludemask is None):
print("\nRelax ampthresh, delaythresh, or sigmathresh - exiting")
else:
print(
"\nChange include/exclude masks or relax ampthresh, delaythresh, or sigmathresh - exiting"
)
return 0, None, None, locationfails, ampfails, lagfails, sigmafails
if optiondict["cleanrefined"]:
shiftmask = locationmask
else:
shiftmask = refinemask
volumetotal = np.sum(shiftmask)
reportstep = 1000
# timeshift the valid voxels
if optiondict["nprocs"] > 1:
# define the consumer function here so it inherits most of the arguments
def timeshift_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(
_procOneVoxelTimeShift(
val,
fmridata[val, :],
lagstrengths[val],
R2[val],
lagtimes[val],
padtrs,
fmritr,
theprefilter,
optiondict["fmrifreq"],
refineprenorm=optiondict["refineprenorm"],
lagmaxthresh=optiondict["lagmaxthresh"],
refineweighting=optiondict["refineweighting"],
detrendorder=optiondict["detrendorder"],
offsettime=optiondict["offsettime"],
filterbeforePCA=optiondict["filterbeforePCA"],
psdfilter=optiondict["psdfilter"],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
timeshift_consumer,
inputshape,
shiftmask,
nprocs=optiondict["nprocs"],
showprogressbar=True,
chunksize=optiondict["mp_chunksize"],
)
# unpack the data
psdlist = []
for voxel in data_out:
shiftedtcs[voxel[0], :] = voxel[1]
weights[voxel[0], :] = voxel[2]
if optiondict["psdfilter"]:
psdlist.append(voxel[3])
del data_out
else:
psdlist = []
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox
== inputshape[0] - 1) and optiondict["showprogressbar"]:
tide_util.progressbar(vox + 1,
inputshape[0],
label="Percent complete (timeshifting)")
if shiftmask[vox] > 0.5:
retvals = _procOneVoxelTimeShift(
vox,
fmridata[vox, :],
lagstrengths[vox],
R2[vox],
lagtimes[vox],
padtrs,
fmritr,
theprefilter,
optiondict["fmrifreq"],
refineprenorm=optiondict["refineprenorm"],
lagmaxthresh=optiondict["lagmaxthresh"],
refineweighting=optiondict["refineweighting"],
detrendorder=optiondict["detrendorder"],
offsettime=optiondict["offsettime"],
filterbeforePCA=optiondict["filterbeforePCA"],
psdfilter=optiondict["psdfilter"],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
shiftedtcs[retvals[0], :] = retvals[1]
weights[retvals[0], :] = retvals[2]
if optiondict["psdfilter"]:
psdlist.append(retvals[3])
print()
if optiondict["psdfilter"]:
print(len(psdlist))
print(psdlist[0])
print(np.shape(np.asarray(psdlist, dtype=rt_floattype)))
averagepsd = np.mean(np.asarray(psdlist, dtype=rt_floattype), axis=0)
stdpsd = np.std(np.asarray(psdlist, dtype=rt_floattype), axis=0)
snr = np.nan_to_num(averagepsd / stdpsd)
# now generate the refined timecourse(s)
validlist = np.where(refinemask > 0)[0]
refinevoxels = shiftedtcs[validlist, :]
if bipolar:
for thevoxel in range(len(validlist)):
if lagstrengths[validlist][thevoxel] < 0.0:
refinevoxels[thevoxel, :] *= -1.0
refineweights = weights[validlist]
weightsum = np.sum(refineweights, axis=0) / volumetotal
averagedata = np.sum(refinevoxels, axis=0) / volumetotal
if optiondict["cleanrefined"]:
invalidlist = np.where((1 - ampmask) > 0)[0]
discardvoxels = shiftedtcs[invalidlist]
discardweights = weights[invalidlist]
discardweightsum = np.sum(discardweights, axis=0) / volumetotal
averagediscard = np.sum(discardvoxels, axis=0) / volumetotal
if optiondict["dodispersioncalc"]:
print("splitting regressors by time lag for phase delay estimation")
laglist = np.arange(
optiondict["dispersioncalc_lower"],
optiondict["dispersioncalc_upper"],
optiondict["dispersioncalc_step"],
)
dispersioncalcout = np.zeros((np.shape(laglist)[0], inputshape[1]),
dtype=rt_floattype)
fftlen = int(inputshape[1] // 2)
fftlen -= fftlen % 2
dispersioncalcspecmag = np.zeros((np.shape(laglist)[0], fftlen),
dtype=rt_floattype)
dispersioncalcspecphase = np.zeros((np.shape(laglist)[0], fftlen),
dtype=rt_floattype)
for lagnum in range(0, np.shape(laglist)[0]):
lower = laglist[lagnum] - optiondict["dispersioncalc_step"] / 2.0
upper = laglist[lagnum] + optiondict["dispersioncalc_step"] / 2.0
inlagrange = np.where(
locationmask * ampmask *
np.where(lower < lagtimes, np.int16(1), np.int16(0)) *
np.where(lagtimes < upper, np.int16(1), np.int16(0)))[0]
print(
" summing",
np.shape(inlagrange)[0],
"regressors with lags from",
lower,
"to",
upper,
)
if np.shape(inlagrange)[0] > 0:
dispersioncalcout[lagnum, :] = tide_math.corrnormalize(
np.mean(shiftedtcs[inlagrange], axis=0),
detrendorder=optiondict["detrendorder"],
windowfunc=optiondict["windowfunc"],
)
(
freqs,
dispersioncalcspecmag[lagnum, :],
dispersioncalcspecphase[lagnum, :],
) = tide_math.polarfft(dispersioncalcout[lagnum, :],
1.0 / fmritr)
inlagrange = None
tide_io.writenpvecs(
dispersioncalcout,
optiondict["outputname"] + "_dispersioncalcvecs_pass" +
str(passnum) + ".txt",
)
tide_io.writenpvecs(
dispersioncalcspecmag,
optiondict["outputname"] + "_dispersioncalcspecmag_pass" +
str(passnum) + ".txt",
)
tide_io.writenpvecs(
dispersioncalcspecphase,
optiondict["outputname"] + "_dispersioncalcspecphase_pass" +
str(passnum) + ".txt",
)
tide_io.writenpvecs(
freqs,
optiondict["outputname"] + "_dispersioncalcfreqs_pass" +
str(passnum) + ".txt",
)
if optiondict["pcacomponents"] < 0.0:
pcacomponents = "mle"
elif optiondict["pcacomponents"] >= 1.0:
pcacomponents = int(np.round(optiondict["pcacomponents"]))
elif optiondict["pcacomponents"] == 0.0:
print("0.0 is not an allowed value for pcacomponents")
sys.exit()
else:
pcacomponents = optiondict["pcacomponents"]
icacomponents = 1
if optiondict["refinetype"] == "ica":
print("performing ica refinement")
thefit = FastICA(n_components=icacomponents).fit(
refinevoxels) # Reconstruct signals
print("Using first of ", len(thefit.components_), " components")
icadata = thefit.components_[0]
filteredavg = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], averagedata),
detrendorder=optiondict["detrendorder"],
)
filteredica = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], icadata),
detrendorder=optiondict["detrendorder"],
)
thepxcorr = pearsonr(filteredavg, filteredica)[0]
print("ica/avg correlation = ", thepxcorr)
if thepxcorr > 0.0:
outputdata = 1.0 * icadata
else:
outputdata = -1.0 * icadata
elif optiondict["refinetype"] == "pca":
# use the method of "A novel perspective to calibrate temporal delays in cerebrovascular reactivity
# using hypercapnic and hyperoxic respiratory challenges". NeuroImage 187, 154?165 (2019).
print("performing pca refinement with pcacomponents set to",
pcacomponents)
try:
thefit = PCA(n_components=pcacomponents).fit(refinevoxels)
except ValueError:
if pcacomponents == "mle":
print(
"mle estimation failed - falling back to pcacomponents=0.8"
)
thefit = PCA(n_components=0.8).fit(refinevoxels)
else:
print("unhandled math exception in PCA refinement - exiting")
sys.exit()
print(
"Using ",
len(thefit.components_),
" component(s), accounting for ",
"{:.2f}% of the variance".format(100.0 * np.cumsum(
thefit.explained_variance_ratio_)[len(thefit.components_) -
1]),
)
reduceddata = thefit.inverse_transform(thefit.transform(refinevoxels))
if debug:
print("complex processing: reduceddata.shape =", reduceddata.shape)
pcadata = np.mean(reduceddata, axis=0)
filteredavg = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], averagedata),
detrendorder=optiondict["detrendorder"],
)
filteredpca = tide_math.corrnormalize(
theprefilter.apply(optiondict["fmrifreq"], pcadata),
detrendorder=optiondict["detrendorder"],
)
thepxcorr = pearsonr(filteredavg, filteredpca)[0]
print("pca/avg correlation = ", thepxcorr)
if thepxcorr > 0.0:
outputdata = 1.0 * pcadata
else:
outputdata = -1.0 * pcadata
elif optiondict["refinetype"] == "weighted_average":
print("performing weighted averaging refinement")
outputdata = np.nan_to_num(averagedata / weightsum)
else:
print("performing unweighted averaging refinement")
outputdata = averagedata
if optiondict["cleanrefined"]:
thefit, R = tide_fit.mlregress(averagediscard, averagedata)
fitcoff = rt_floatset(thefit[0, 1])
datatoremove = rt_floatset(fitcoff * averagediscard)
outputdata -= datatoremove
print()
print(
"Timeshift applied to " + str(int(volumetotal)) + " voxels, " +
str(len(validlist)) + " used for refinement:",
"\n ",
locationfails,
" locationfails",
"\n ",
ampfails,
" ampfails",
"\n ",
lagfails,
" lagfails",
"\n ",
sigmafails,
" sigmafails",
)
if optiondict["psdfilter"]:
outputdata = tide_filt.transferfuncfilt(outputdata, snr)
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal, outputdata, refinemask, locationfails, ampfails, lagfails, sigmafails
def peakevalpass(
fmridata,
referencetc,
fmri_x,
os_fmri_x,
theMutualInformationator,
xcorr_x,
corrdata,
nprocs=1,
alwaysmultiproc=False,
bipolar=False,
oversampfactor=1,
interptype="univariate",
showprogressbar=True,
chunksize=1000,
rt_floatset=np.float64,
rt_floattype="float64",
):
"""
Parameters
----------
fmridata
referencetc
theCorrelator
fmri_x
os_fmri_x
tr
corrout
meanval
nprocs
alwaysmultiproc
oversampfactor
interptype
showprogressbar
chunksize
rt_floatset
rt_floattype
Returns
-------
"""
peakdict = {}
theMutualInformationator.setreftc(referencetc)
inputshape = np.shape(fmridata)
volumetotal = 0
reportstep = 1000
thetc = np.zeros(np.shape(os_fmri_x), dtype=rt_floattype)
if nprocs > 1 or alwaysmultiproc:
# define the consumer function here so it inherits most of the arguments
def correlation_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(
_procOneVoxelPeaks(
val,
thetc,
theMutualInformationator,
fmri_x,
fmridata[val, :],
os_fmri_x,
xcorr_x,
corrdata[val, :],
bipolar=bipolar,
oversampfactor=oversampfactor,
interptype=interptype,
))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
correlation_consumer,
inputshape,
None,
nprocs=nprocs,
showprogressbar=showprogressbar,
chunksize=chunksize,
)
# unpack the data
volumetotal = 0
for voxel in data_out:
peakdict[str(voxel[0])] = voxel[1]
volumetotal += 1
del data_out
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0
or vox == inputshape[0] - 1) and showprogressbar:
tide_util.progressbar(vox + 1,
inputshape[0],
label="Percent complete")
dummy, peakdict[str(vox)] = _procOneVoxelPeaks(
vox,
thetc,
theMutualInformationator,
fmri_x,
fmridata[vox, :],
os_fmri_x,
xcorr_x,
corrdata[vox, :],
bipolar=bipolar,
oversampfactor=oversampfactor,
interptype=interptype,
)
volumetotal += 1
print("\nPeak evaluation performed on " + str(volumetotal) + " voxels")
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal, peakdict
def coherencepass(
fmridata,
theCoherer,
coherencefunc,
coherencepeakval,
coherencepeakfreq,
reportstep,
alt=False,
chunksize=1000,
nprocs=1,
alwaysmultiproc=False,
showprogressbar=True,
rt_floatset=np.float64,
rt_floattype="float64",
):
"""
Parameters
----------
fmridata
theCoherer
coherencefunc
coherencepeakval
coherencepeakfreq
reportstep
chunksize
nprocs
alwaysmultiproc
showprogressbar
rt_floatset
rt_floattype
Returns
-------
"""
inputshape = np.shape(fmridata)
volumetotal = 0
if nprocs > 1 or alwaysmultiproc:
# define the consumer function here so it inherits most of the arguments
def coherence_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(
_procOneVoxelCoherence(
val,
theCoherer,
fmridata[val, :],
alt=alt,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
coherence_consumer,
inputshape,
None,
nprocs=nprocs,
showprogressbar=showprogressbar,
chunksize=chunksize,
)
# unpack the data
volumetotal = 0
for voxel in data_out:
coherencefunc[voxel[0], :] = voxel[2] + 0.0
coherencepeakval[voxel[0]] = voxel[3] + 0.0
coherencepeakfreq[voxel[0]] = voxel[4] + 0.0
volumetotal += 1
del data_out
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0
or vox == inputshape[0] - 1) and showprogressbar:
tide_util.progressbar(vox + 1,
inputshape[0],
label="Percent complete")
(
dummy,
dummy,
coherencefunc[vox],
coherencepeakval[vox],
coherencepeakfreq[vox],
) = _procOneVoxelCoherence(
vox,
theCoherer,
fmridata[vox, :],
alt=alt,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
volumetotal += 1
print(f"\nCoherence performed on {volumetotal} voxels")
# garbage collect
collected = gc.collect()
print(f"Garbage collector: collected {collected} objects.")
return volumetotal
def fitcorr(genlagtc,
initial_fmri_x,
lagtc,
slicesize,
corrscale,
lagmask,
lagtimes,
lagstrengths,
lagsigma,
corrout,
meanval,
gaussout,
R2,
optiondict,
zerooutbadfit=True,
initiallags=None,
rt_floatset=np.float64,
rt_floattype='float64'
):
displayplots = False
inputshape = np.shape(corrout)
if initiallags is None:
themask = None
else:
themask = np.where(initiallags > -1000000.0, 1, 0)
volumetotal, ampfails, lagfails, widthfails, edgefails, fitfails = 0, 0, 0, 0, 0, 0
reportstep = 1000
zerolagtc = rt_floatset(genlagtc.yfromx(initial_fmri_x))
if optiondict['nprocs'] > 1:
# define the consumer function here so it inherits most of the arguments
def fitcorr_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
if initiallags is None:
thislag = None
else:
thislag = initiallags[val]
outQ.put(
_procOneVoxelFitcorr(val,
corrout[val, :],
corrscale, genlagtc,
initial_fmri_x,
optiondict,
zerooutbadfit=zerooutbadfit,
displayplots=False,
initiallag=thislag,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(fitcorr_consumer,
inputshape, themask,
nprocs=optiondict['nprocs'],
showprogressbar=True,
chunksize=optiondict['mp_chunksize'])
# unpack the data
volumetotal = 0
for voxel in data_out:
volumetotal += voxel[1]
lagtc[voxel[0], :] = voxel[2]
lagtimes[voxel[0]] = voxel[3]
lagstrengths[voxel[0]] = voxel[4]
lagsigma[voxel[0]] = voxel[5]
gaussout[voxel[0], :] = voxel[6]
R2[voxel[0]] = voxel[7]
lagmask[voxel[0]] = voxel[8]
data_out = []
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox == inputshape[0] - 1) and optiondict['showprogressbar']:
tide_util.progressbar(vox + 1, inputshape[0], label='Percent complete')
if themask is None:
dothisone = True
thislag = None
elif themask[vox] > 0:
dothisone = True
thislag = initiallags[vox]
else:
dothisone = False
if dothisone:
dummy, \
volumetotalinc, \
lagtc[vox, :], \
lagtimes[vox], \
lagstrengths[vox], \
lagsigma[vox], \
gaussout[vox, :], \
R2[vox], \
lagmask[vox], \
failreason = \
_procOneVoxelFitcorr(vox,
corrout[vox, :],
corrscale,
genlagtc,
initial_fmri_x,
optiondict,
zerooutbadfit=zerooutbadfit,
displayplots=False,
initiallag=thislag,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype)
volumetotal += volumetotalinc
print('\nCorrelation fitted in ' + str(volumetotal) + ' voxels')
print('\tampfails=', ampfails, ' lagfails=', lagfails, ' widthfail=', widthfails, ' edgefail=', edgefails,
' fitfail=', fitfails)
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal
def fitcorrx(lagtcgenerator,
timeaxis,
lagtc,
corrtimescale,
thefitter,
corrout,
lagmask,
failimage,
lagtimes,
lagstrengths,
lagsigma,
gaussout,
windowout,
R2,
nprocs=1,
fixdelay=False,
showprogressbar=True,
chunksize=1000,
despeckle_thresh=5.0,
initiallags=None,
rt_floatset=np.float64,
rt_floattype='float64'):
thefitter.setcorrtimeaxis(corrtimescale)
inputshape = np.shape(corrout)
if initiallags is None:
themask = None
else:
themask = np.where(initiallags > -1000000.0, 1, 0)
reportstep = 1000
volumetotal, ampfails, lagfails, windowfails, widthfails, edgefails, fitfails = 0, 0, 0, 0, 0, 0, 0
FML_BADAMPLOW = np.uint16(0x01)
FML_BADAMPHIGH = np.uint16(0x02)
FML_BADSEARCHWINDOW = np.uint16(0x04)
FML_BADWIDTH = np.uint16(0x08)
FML_BADLAG = np.uint16(0x10)
FML_HITEDGE = np.uint16(0x20)
FML_FITFAIL = np.uint16(0x40)
FML_INITFAIL = np.uint16(0x80)
zerolagtc = rt_floatset(lagtcgenerator.yfromx(timeaxis))
sliceoffsettime = 0.0
if nprocs > 1:
# define the consumer function here so it inherits most of the arguments
def fitcorr_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
if initiallags is None:
thislag = None
else:
thislag = initiallags[val]
outQ.put(_procOneVoxelFitcorrx(val,
corrout[val, :],
lagtcgenerator,
timeaxis,
thefitter,
disablethresholds=False,
despeckle_thresh=despeckle_thresh,
initiallag=thislag,
fixdelay=fixdelay,
fixeddelayvalue=0.0,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(fitcorr_consumer,
inputshape, themask,
nprocs=nprocs,
showprogressbar=showprogressbar,
chunksize=chunksize)
# unpack the data
volumetotal = 0
for voxel in data_out:
volumetotal += voxel[1]
lagtc[voxel[0], :] = voxel[2]
lagtimes[voxel[0]] = voxel[3]
lagstrengths[voxel[0]] = voxel[4]
lagsigma[voxel[0]] = voxel[5]
gaussout[voxel[0], :] = voxel[6]
windowout[voxel[0], :] = voxel[7]
R2[voxel[0]] = voxel[8]
lagmask[voxel[0]] = voxel[9]
failimage[voxel[0]] = voxel[10] & 0x3f
if (FML_BADAMPLOW | FML_BADAMPHIGH) & voxel[10]:
ampfails += 1
if FML_BADSEARCHWINDOW & voxel[10]:
windowfails += 1
if FML_BADWIDTH & voxel[10]:
widthfails += 1
if FML_BADLAG & voxel[10]:
lagfails += 1
if FML_HITEDGE & voxel[10]:
edgefails += 1
if (FML_FITFAIL | FML_INITFAIL) & voxel[10]:
fitfails += 1
del data_out
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox == inputshape[0] - 1) and showprogressbar:
tide_util.progressbar(vox + 1, inputshape[0], label='Percent complete')
if themask is None:
dothisone = True
thislag = None
elif themask[vox] > 0:
dothisone = True
thislag = initiallags[vox]
else:
dothisone = False
thislag = None
if dothisone:
dummy, \
volumetotalinc, \
lagtc[vox, :], \
lagtimes[vox], \
lagstrengths[vox], \
lagsigma[vox], \
gaussout[vox, :], \
windowout[vox, :], \
R2[vox], \
lagmask[vox], \
failreason = \
_procOneVoxelFitcorrx(vox,
corrout[vox, :],
lagtcgenerator,
timeaxis,
thefitter,
disablethresholds=False,
despeckle_thresh=despeckle_thresh,
initiallag=thislag,
fixdelay=fixdelay,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype)
volumetotal += volumetotalinc
if (FML_BADAMPLOW | FML_BADAMPHIGH) & failreason:
ampfails += 1
if FML_BADSEARCHWINDOW & failreason:
windowfails += 1
if FML_BADWIDTH & failreason:
widthfails += 1
if FML_BADLAG & failreason:
lagfails += 1
if FML_HITEDGE & failreason:
edgefails += 1
if (FML_FITFAIL | FML_INITFAIL) & failreason:
fitfails += 1
print('\nCorrelation fitted in ' + str(volumetotal) + ' voxels')
print('\tampfails=', ampfails,
'\n\tlagfails=', lagfails,
'\n\twindowfails=', windowfails,
'\n\twidthfail=', widthfails,
'\n\tedgefail=', edgefails,
'\n\tfitfail=', fitfails)
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal
def fitcorrx(genlagtc,
initial_fmri_x,
lagtc,
slicesize,
corrscale,
lagmask,
failimage,
lagtimes,
lagstrengths,
lagsigma,
corrout,
meanval,
gaussout,
windowout,
R2,
optiondict,
zerooutbadfit=True,
initiallags=None,
rt_floatset=np.float64,
rt_floattype='float64'):
displayplots = False
inputshape = np.shape(corrout)
if initiallags is None:
themask = None
else:
themask = np.where(initiallags > -1000000.0, 1, 0)
reportstep = 1000
volumetotal, ampfails, lagfails, windowfails, widthfails, edgefails, fitfails = 0, 0, 0, 0, 0, 0, 0
FML_BADAMPLOW = np.uint16(0x01)
FML_BADAMPHIGH = np.uint16(0x02)
FML_BADSEARCHWINDOW = np.uint16(0x04)
FML_BADWIDTH = np.uint16(0x08)
FML_BADLAG = np.uint16(0x10)
FML_HITEDGE = np.uint16(0x20)
FML_FITFAIL = np.uint16(0x40)
FML_INITFAIL = np.uint16(0x80)
zerolagtc = rt_floatset(genlagtc.yfromx(initial_fmri_x))
sliceoffsettime = 0.0
if optiondict['nprocs'] > 1:
# define the consumer function here so it inherits most of the arguments
def fitcorr_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
if initiallags is None:
thislag = None
else:
thislag = initiallags[val]
outQ.put(_procOneVoxelFitcorrx(val,
corrout[val, :],
corrscale,
genlagtc,
initial_fmri_x,
optiondict,
zerooutbadfit=zerooutbadfit,
displayplots=displayplots,
initiallag=thislag,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(fitcorr_consumer,
inputshape, themask,
nprocs=optiondict['nprocs'],
showprogressbar=True,
chunksize=optiondict['mp_chunksize'])
# unpack the data
volumetotal = 0
for voxel in data_out:
volumetotal += voxel[1]
lagtc[voxel[0], :] = voxel[2]
lagtimes[voxel[0]] = voxel[3]
lagstrengths[voxel[0]] = voxel[4]
lagsigma[voxel[0]] = voxel[5]
gaussout[voxel[0], :] = voxel[6]
windowout[voxel[0], :] = voxel[7]
R2[voxel[0]] = voxel[8]
lagmask[voxel[0]] = voxel[9]
failimage[voxel[0]] = voxel[10] & 0x3f
if (FML_BADAMPLOW | FML_BADAMPHIGH) & voxel[10]:
ampfails += 1
if FML_BADSEARCHWINDOW & voxel[10]:
windowfails += 1
if FML_BADWIDTH & voxel[10]:
widthfails += 1
if FML_BADLAG & voxel[10]:
lagfails += 1
if FML_HITEDGE & voxel[10]:
edgefails += 1
if (FML_FITFAIL | FML_INITFAIL) & voxel[10]:
fitfails += 1
del data_out
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox == inputshape[0] - 1) and optiondict['showprogressbar']:
tide_util.progressbar(vox + 1, inputshape[0], label='Percent complete')
if themask is None:
dothisone = True
thislag = None
elif themask[vox] > 0:
dothisone = True
thislag = initiallags[vox]
else:
dothisone = False
thislag = None
if dothisone:
dummy, \
volumetotalinc, \
lagtc[vox, :], \
lagtimes[vox], \
lagstrengths[vox], \
lagsigma[vox], \
gaussout[vox, :], \
windowout[vox, :], \
R2[vox], \
lagmask[vox], \
failreason = \
_procOneVoxelFitcorrx(vox,
corrout[vox, :],
corrscale,
genlagtc,
initial_fmri_x,
optiondict,
zerooutbadfit=zerooutbadfit,
displayplots=displayplots,
initiallag=thislag,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype)
volumetotal += volumetotalinc
if (FML_BADAMPLOW | FML_BADAMPHIGH) & failreason:
ampfails += 1
if FML_BADSEARCHWINDOW & failreason:
windowfails += 1
if FML_BADWIDTH & failreason:
widthfails += 1
if FML_BADLAG & failreason:
lagfails += 1
if FML_HITEDGE & failreason:
edgefails += 1
if (FML_FITFAIL | FML_INITFAIL) & failreason:
fitfails += 1
print('\nCorrelation fitted in ' + str(volumetotal) + ' voxels')
print('\tampfails=', ampfails,
' lagfails=', lagfails,
' windowfails=', windowfails,
' widthfail=', widthfails,
' edgefail=', edgefails,
' fitfail=', fitfails)
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal
def fitcorr(lagtcgenerator,
timeaxis,
lagtc,
slicesize,
corr_x,
lagmask,
lagtimes,
lagstrengths,
lagsigma,
corrout,
meanval,
gaussout,
R2,
optiondict,
zerooutbadfit=True,
initiallags=None,
rt_floatset=np.float64,
rt_floattype='float64'
):
displayplots = False
inputshape = np.shape(corrout)
if initiallags is None:
themask = None
else:
themask = np.where(initiallags > -1000000.0, 1, 0)
volumetotal, ampfails, lagfails, widthfails, edgefails, fitfails = 0, 0, 0, 0, 0, 0
reportstep = 1000
zerolagtc = rt_floatset(lagtcgenerator.yfromx(timeaxis))
if optiondict['nprocs'] > 1:
# define the consumer function here so it inherits most of the arguments
def fitcorr_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
if initiallags is None:
thislag = None
else:
thislag = initiallags[val]
outQ.put(
_procOneVoxelFitcorr(val,
corrout[val, :],
corr_x, lagtcgenerator,
timeaxis,
optiondict,
zerooutbadfit=zerooutbadfit,
displayplots=False,
initiallag=thislag,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(fitcorr_consumer,
inputshape, themask,
nprocs=optiondict['nprocs'],
showprogressbar=True,
chunksize=optiondict['mp_chunksize'])
# unpack the data
volumetotal = 0
for voxel in data_out:
volumetotal += voxel[1]
lagtc[voxel[0], :] = voxel[2]
lagtimes[voxel[0]] = voxel[3]
lagstrengths[voxel[0]] = voxel[4]
lagsigma[voxel[0]] = voxel[5]
gaussout[voxel[0], :] = voxel[6]
R2[voxel[0]] = voxel[7]
lagmask[voxel[0]] = voxel[8]
data_out = []
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox == inputshape[0] - 1) and optiondict['showprogressbar']:
tide_util.progressbar(vox + 1, inputshape[0], label='Percent complete')
if themask is None:
dothisone = True
thislag = None
elif themask[vox] > 0:
dothisone = True
thislag = initiallags[vox]
else:
dothisone = False
if dothisone:
dummy, \
volumetotalinc, \
lagtc[vox, :], \
lagtimes[vox], \
lagstrengths[vox], \
lagsigma[vox], \
gaussout[vox, :], \
R2[vox], \
lagmask[vox], \
failreason = \
_procOneVoxelFitcorr(vox,
corrout[vox, :],
corr_x,
lagtcgenerator,
timeaxis,
optiondict,
zerooutbadfit=zerooutbadfit,
displayplots=False,
initiallag=thislag,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype)
volumetotal += volumetotalinc
print('\nCorrelation fitted in ' + str(volumetotal) + ' voxels')
print('\tampfails=', ampfails, '\n\tlagfails=', lagfails, '\n\twidthfail=', widthfails, '\n\tedgefail=', edgefails,
'\n\tfitfail=', fitfails)
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal
def glmpass(
numprocitems,
fmri_data,
threshval,
theevs,
meanvalue,
rvalue,
r2value,
fitcoeff,
fitNorm,
datatoremove,
filtereddata,
reportstep=1000,
nprocs=1,
alwaysmultiproc=False,
procbyvoxel=True,
showprogressbar=True,
mp_chunksize=1000,
rt_floatset=np.float64,
rt_floattype="float64",
):
inputshape = np.shape(fmri_data)
if threshval is None:
themask = None
else:
if procbyvoxel:
meanim = np.mean(fmri_data, axis=1)
stdim = np.std(fmri_data, axis=1)
else:
meanim = np.mean(fmri_data, axis=0)
stdim = np.std(fmri_data, axis=0)
if np.mean(stdim) < np.mean(meanim):
themask = np.where(meanim > threshval, 1, 0)
else:
themask = np.where(stdim > threshval, 1, 0)
if (nprocs > 1 or alwaysmultiproc
): # temporary workaround until I figure out why nprocs > 1 is failing
# define the consumer function here so it inherits most of the arguments
def GLM_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
if procbyvoxel:
outQ.put(
_procOneItemGLM(
val,
theevs[val, :],
fmri_data[val, :],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
))
else:
outQ.put(
_procOneItemGLM(
val,
theevs[:, val],
fmri_data[:, val],
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
GLM_consumer,
inputshape,
themask,
nprocs=nprocs,
procbyvoxel=procbyvoxel,
showprogressbar=showprogressbar,
chunksize=mp_chunksize,
)
# unpack the data
itemstotal = 0
if procbyvoxel:
for voxel in data_out:
meanvalue[voxel[0]] = voxel[1]
rvalue[voxel[0]] = voxel[2]
r2value[voxel[0]] = voxel[3]
fitcoeff[voxel[0]] = voxel[4]
fitNorm[voxel[0]] = voxel[5]
datatoremove[voxel[0], :] = voxel[6]
filtereddata[voxel[0], :] = voxel[7]
itemstotal += 1
else:
for timepoint in data_out:
meanvalue[timepoint[0]] = timepoint[1]
rvalue[timepoint[0]] = timepoint[2]
r2value[timepoint[0]] = timepoint[3]
fitcoeff[timepoint[0]] = timepoint[4]
fitNorm[timepoint[0]] = timepoint[5]
datatoremove[:, timepoint[0]] = timepoint[6]
filtereddata[:, timepoint[0]] = timepoint[7]
itemstotal += 1
del data_out
else:
itemstotal = 0
if procbyvoxel:
for vox in range(0, numprocitems):
if (vox % reportstep == 0
or vox == numprocitems - 1) and showprogressbar:
tide_util.progressbar(vox + 1,
numprocitems,
label="Percent complete")
thedata = fmri_data[vox, :].copy()
if (themask is None) or (themask[vox] > 0):
(
dummy,
meanvalue[vox],
rvalue[vox],
r2value[vox],
fitcoeff[vox],
fitNorm[vox],
datatoremove[vox, :],
filtereddata[vox, :],
) = _procOneItemGLM(
vox,
theevs[vox, :],
thedata,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
itemstotal += 1
else:
for timepoint in range(0, numprocitems):
if (timepoint % reportstep == 0
or timepoint == numprocitems - 1) and showprogressbar:
tide_util.progressbar(timepoint + 1,
numprocitems,
label="Percent complete")
thedata = fmri_data[:, timepoint].copy()
if (themask is None) or (themask[timepoint] > 0):
(
dummy,
meanvalue[timepoint],
rvalue[timepoint],
r2value[timepoint],
fitcoeff[timepoint],
fitNorm[timepoint],
datatoremove[:, timepoint],
filtereddata[:, timepoint],
) = _procOneItemGLM(
timepoint,
theevs[:, timepoint],
thedata,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
itemstotal += 1
if showprogressbar:
print()
return itemstotal
def correlationpass(
fmridata,
referencetc,
theCorrelator,
fmri_x,
os_fmri_x,
lagmininpts,
lagmaxinpts,
corrout,
meanval,
nprocs=1,
alwaysmultiproc=False,
oversampfactor=1,
interptype="univariate",
showprogressbar=True,
chunksize=1000,
rt_floatset=np.float64,
rt_floattype="float64",
):
"""
Parameters
----------
fmridata
referencetc - the reference regressor, already oversampled
theCorrelator
fmri_x
os_fmri_x
tr
lagmininpts
lagmaxinpts
corrout
meanval
nprocs
oversampfactor
interptype
showprogressbar
chunksize
rt_floatset
rt_floattype
Returns
-------
"""
theCorrelator.setreftc(referencetc)
theCorrelator.setlimits(lagmininpts, lagmaxinpts)
inputshape = np.shape(fmridata)
volumetotal = 0
reportstep = 1000
thetc = np.zeros(np.shape(os_fmri_x), dtype=rt_floattype)
theglobalmaxlist = []
if nprocs > 1 or alwaysmultiproc:
# define the consumer function here so it inherits most of the arguments
def correlation_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(
_procOneVoxelCorrelation(
val,
thetc,
theCorrelator,
fmri_x,
fmridata[val, :],
os_fmri_x,
oversampfactor=oversampfactor,
interptype=interptype,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
)
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
correlation_consumer,
inputshape,
None,
nprocs=nprocs,
showprogressbar=showprogressbar,
chunksize=chunksize,
)
# unpack the data
volumetotal = 0
for voxel in data_out:
# corrmask[voxel[0]] = 1
meanval[voxel[0]] = voxel[1]
corrout[voxel[0], :] = voxel[2]
thecorrscale = voxel[3]
theglobalmaxlist.append(voxel[4] + 0)
volumetotal += 1
del data_out
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox == inputshape[0] - 1) and showprogressbar:
tide_util.progressbar(vox + 1, inputshape[0], label="Percent complete")
(
dummy,
meanval[vox],
corrout[vox, :],
thecorrscale,
theglobalmax,
) = _procOneVoxelCorrelation(
vox,
thetc,
theCorrelator,
fmri_x,
fmridata[vox, :],
os_fmri_x,
oversampfactor=oversampfactor,
interptype=interptype,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
theglobalmaxlist.append(theglobalmax + 0)
volumetotal += 1
LGR.info(f"\nCorrelation performed on {volumetotal} voxels")
# garbage collect
collected = gc.collect()
LGR.info(f"Garbage collector: collected {collected} objects.")
return volumetotal, theglobalmaxlist, thecorrscale
def fitcorr(
lagtcgenerator,
timeaxis,
lagtc,
corrtimescale,
thefitter,
corrout,
lagmask,
failimage,
lagtimes,
lagstrengths,
lagsigma,
gaussout,
windowout,
R2,
peakdict=None,
nprocs=1,
alwaysmultiproc=False,
fixdelay=False,
showprogressbar=True,
chunksize=1000,
despeckle_thresh=5.0,
initiallags=None,
rt_floatset=np.float64,
rt_floattype="float64",
):
thefitter.setcorrtimeaxis(corrtimescale)
inputshape = np.shape(corrout)
if initiallags is None:
themask = None
else:
themask = np.where(initiallags > -1000000.0, 1, 0)
reportstep = 1000
(
volumetotal,
ampfails,
lowlagfails,
highlagfails,
lowwidthfails,
highwidthfails,
initfails,
fitfails,
) = (0, 0, 0, 0, 0, 0, 0, 0)
zerolagtc = rt_floatset(lagtcgenerator.yfromx(timeaxis))
sliceoffsettime = 0.0
if nprocs > 1 or alwaysmultiproc:
# define the consumer function here so it inherits most of the arguments
def fitcorr_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
if initiallags is None:
thislag = None
else:
thislag = initiallags[val]
outQ.put(
_procOneVoxelFitcorr(
val,
corrout[val, :],
lagtcgenerator,
timeaxis,
thefitter,
disablethresholds=False,
despeckle_thresh=despeckle_thresh,
initiallag=thislag,
fixdelay=fixdelay,
fixeddelayvalue=0.0,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
)
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
fitcorr_consumer,
inputshape,
themask,
nprocs=nprocs,
showprogressbar=showprogressbar,
chunksize=chunksize,
)
# unpack the data
volumetotal = 0
for voxel in data_out:
volumetotal += voxel[1]
lagtc[voxel[0], :] = voxel[2]
lagtimes[voxel[0]] = voxel[3]
lagstrengths[voxel[0]] = voxel[4]
lagsigma[voxel[0]] = voxel[5]
gaussout[voxel[0], :] = voxel[6]
windowout[voxel[0], :] = voxel[7]
R2[voxel[0]] = voxel[8]
lagmask[voxel[0]] = voxel[9]
failimage[voxel[0]] = voxel[10] & 0xFFFF
if (
thefitter.FML_INITAMPLOW
| thefitter.FML_INITAMPHIGH
| thefitter.FML_FITAMPLOW
| thefitter.FML_FITAMPHIGH
) & voxel[10]:
ampfails += 1
if (thefitter.FML_INITWIDTHLOW | thefitter.FML_FITWIDTHLOW) & voxel[10]:
lowwidthfails += 1
if (thefitter.FML_INITWIDTHHIGH | thefitter.FML_FITWIDTHHIGH) & voxel[10]:
highwidthfails += 1
if (thefitter.FML_INITLAGLOW | thefitter.FML_FITLAGLOW) & voxel[10]:
lowlagfails += 1
if (thefitter.FML_INITLAGHIGH | thefitter.FML_FITLAGHIGH) & voxel[10]:
highlagfails += 1
if thefitter.FML_INITFAIL & voxel[10]:
initfails += 1
if thefitter.FML_FITFAIL & voxel[10]:
fitfails += 1
del data_out
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox == inputshape[0] - 1) and showprogressbar:
tide_util.progressbar(vox + 1, inputshape[0], label="Percent complete")
if themask is None:
dothisone = True
thislag = None
elif themask[vox] > 0:
dothisone = True
thislag = initiallags[vox]
else:
dothisone = False
thislag = None
if dothisone:
(
dummy,
volumetotalinc,
lagtc[vox, :],
lagtimes[vox],
lagstrengths[vox],
lagsigma[vox],
gaussout[vox, :],
windowout[vox, :],
R2[vox],
lagmask[vox],
failreason,
) = _procOneVoxelFitcorr(
vox,
corrout[vox, :],
lagtcgenerator,
timeaxis,
thefitter,
disablethresholds=False,
despeckle_thresh=despeckle_thresh,
initiallag=thislag,
fixdelay=fixdelay,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
volumetotal += volumetotalinc
if (
thefitter.FML_INITAMPLOW
| thefitter.FML_INITAMPHIGH
| thefitter.FML_FITAMPLOW
| thefitter.FML_FITAMPHIGH
) & failreason:
ampfails += 1
if (thefitter.FML_INITWIDTHLOW | thefitter.FML_FITWIDTHLOW) & failreason:
lowwidthfails += 1
if (thefitter.FML_INITWIDTHHIGH | thefitter.FML_FITWIDTHHIGH) & failreason:
highwidthfails += 1
if (thefitter.FML_INITLAGLOW | thefitter.FML_INITLAGHIGH) & failreason:
lowlagfails += 1
if (thefitter.FML_INITLAGLOW | thefitter.FML_FITLAGLOW) & failreason:
lowlagfails += 1
if (thefitter.FML_INITLAGHIGH | thefitter.FML_FITLAGHIGH) & failreason:
highlagfails += 1
if thefitter.FML_INITFAIL & failreason:
initfails += 1
if thefitter.FML_FITFAIL & failreason:
fitfails += 1
print("\nCorrelation fitted in " + str(volumetotal) + " voxels")
print(
"\tampfails:",
ampfails,
"\n\tlowlagfails:",
lowlagfails,
"\n\thighlagfails:",
highlagfails,
"\n\tlowwidthfails:",
lowwidthfails,
"\n\thighwidthfail:",
highwidthfails,
"\n\ttotal initfails:",
initfails,
"\n\ttotal fitfails:",
fitfails,
)
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal
def correlationpass(fmridata,
fmrifftdata,
referencetc,
fmri_x,
os_fmri_x,
tr,
corrorigin,
lagmininpts,
lagmaxinpts,
corrout,
meanval,
ncprefilter,
optiondict,
rt_floatset=np.float64,
rt_floattype='float64'):
"""
Parameters
----------
fmridata
fmrifftdata
referencetc
fmri_x
os_fmri_x
tr
corrorigin
lagmininpts
lagmaxinpts
corrout
meanval
ncprefilter
optiondict
rt_floatset
rt_floattype
Returns
-------
"""
oversampfreq = optiondict['oversampfactor'] / tr
inputshape = np.shape(fmridata)
volumetotal = 0
reportstep = 1000
thetc = np.zeros(np.shape(os_fmri_x), dtype=rt_floattype)
theglobalmaxlist = []
if optiondict['nprocs'] > 1:
# define the consumer function here so it inherits most of the arguments
def correlation_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(_procOneVoxelCorrelation(val,
thetc,
optiondict,
fmri_x,
fmridata[val, :],
os_fmri_x,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
ncprefilter,
referencetc,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(correlation_consumer,
inputshape, None,
nprocs=optiondict['nprocs'],
showprogressbar=True,
chunksize=optiondict['mp_chunksize'])
# unpack the data
volumetotal = 0
for voxel in data_out:
# corrmask[voxel[0]] = 1
meanval[voxel[0]] = voxel[1]
corrout[voxel[0], :] = voxel[2]
theglobalmaxlist.append(voxel[3] + 0)
volumetotal += 1
del data_out
else:
for vox in range(0, inputshape[0]):
if (vox % reportstep == 0 or vox == inputshape[0] - 1) and optiondict['showprogressbar']:
tide_util.progressbar(vox + 1, inputshape[0], label='Percent complete')
dummy, meanval[vox], corrout[vox, :], theglobalmax = _procOneVoxelCorrelation(vox,
thetc,
optiondict,
fmri_x,
fmridata[vox, :],
os_fmri_x,
oversampfreq,
corrorigin,
lagmininpts,
lagmaxinpts,
ncprefilter,
referencetc,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype
)
theglobalmaxlist.append(theglobalmax + 0)
volumetotal += 1
print('\nCorrelation performed on ' + str(volumetotal) + ' voxels')
# garbage collect
collected = gc.collect()
print("Garbage collector: collected %d objects." % collected)
return volumetotal, theglobalmaxlist
def getNullDistributionDatax(
rawtimecourse,
Fs,
theCorrelator,
thefitter,
despeckle_thresh=5.0,
fixdelay=False,
fixeddelayvalue=0.0,
numestreps=0,
nprocs=1,
alwaysmultiproc=False,
showprogressbar=True,
chunksize=1000,
permutationmethod="shuffle",
rt_floatset=np.float64,
rt_floattype="float64",
):
r"""Calculate a set of null correlations to determine the distribution of correlation values. This can
be used to find the spurious correlation threshold
Parameters
----------
rawtimecourse : 1D numpy array
The test regressor. This should be filtered to the desired bandwidth, but NOT windowed.
:param rawtimecourse:
corrscale: 1D numpy array
The time axis of the cross correlation function.
filterfunc: function
This is a preconfigured NoncausalFilter function which is used to filter data to the desired bandwidth
Fs: float
The sample frequency of rawtimecourse, in Hz
corrorigin: int
The bin number in the correlation timescale corresponding to 0.0 seconds delay
negbins: int
The lower edge of the search range for correlation peaks, in number of bins below corrorigin
posbins: int
The upper edge of the search range for correlation peaks, in number of bins above corrorigin
"""
inputshape = np.asarray([numestreps])
normalizedreftc = theCorrelator.ncprefilter.apply(
Fs,
tide_math.corrnormalize(
theCorrelator.reftc,
windowfunc="None",
detrendorder=theCorrelator.detrendorder,
),
)
rawtcfft_r, rawtcfft_ang = tide_filt.polarfft(normalizedreftc)
if nprocs > 1 or alwaysmultiproc:
# define the consumer function here so it inherits most of the arguments
def nullCorrelation_consumer(inQ, outQ):
while True:
try:
# get a new message
val = inQ.get()
# this is the 'TERM' signal
if val is None:
break
# process and send the data
outQ.put(
_procOneNullCorrelationx(
normalizedreftc,
rawtcfft_r,
rawtcfft_ang,
Fs,
theCorrelator,
thefitter,
despeckle_thresh=despeckle_thresh,
fixdelay=fixdelay,
fixeddelayvalue=fixeddelayvalue,
permutationmethod=permutationmethod,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
))
except Exception as e:
print("error!", e)
break
data_out = tide_multiproc.run_multiproc(
nullCorrelation_consumer,
inputshape,
None,
nprocs=nprocs,
showprogressbar=showprogressbar,
chunksize=chunksize,
)
# unpack the data
corrlist = np.asarray(data_out, dtype=rt_floattype)
else:
corrlist = np.zeros((numestreps), dtype=rt_floattype)
for i in range(0, numestreps):
# make a shuffled copy of the regressors
if permutationmethod == "shuffle":
permutedtc = np.random.permutation(normalizedreftc)
elif permutationmethod == "phaserandom":
permutedtc = tide_filt.ifftfrompolar(
rawtcfft_r, np.random.permutation(rawtcfft_ang))
else:
print("illegal shuffling method")
sys.exit()
# crosscorrelate with original, fit, and return the maximum value, and add it to the list
thexcorr = _procOneNullCorrelationx(
normalizedreftc,
rawtcfft_r,
rawtcfft_ang,
Fs,
theCorrelator,
thefitter,
despeckle_thresh=despeckle_thresh,
fixdelay=fixdelay,
fixeddelayvalue=fixeddelayvalue,
permutationmethod=permutationmethod,
rt_floatset=rt_floatset,
rt_floattype=rt_floattype,
)
corrlist[i] = thexcorr
# progress
if showprogressbar:
tide_util.progressbar(i + 1,
numestreps,
label="Percent complete")
# jump to line after progress bar
print()
# return the distribution data
numnonzero = len(np.where(corrlist != 0.0)[0])
print("{:d} non-zero correlations out of {:d} ({:.2f}%)".format(
numnonzero, len(corrlist), 100.0 * numnonzero / len(corrlist)))
return corrlist