def _detect_outliers_core(self, imgfile, motionfile, runidx, cwd=None):
"""
Core routine for detecting outliers
"""
if not cwd:
cwd = os.getcwd()
# read in motion parameters
mc_in = np.loadtxt(motionfile)
mc = deepcopy(mc_in)
if self.inputs.parameter_source == 'SPM':
pass
elif self.inputs.parameter_source == 'FSL':
mc = mc[:, [3, 4, 5, 0, 1, 2]]
elif self.inputs.parameter_source == 'Siemens':
Exception("Siemens PACE format not implemented yet")
else:
Exception("Unknown source for movement parameters")
if self.inputs.use_norm:
# calculate the norm of the motion parameters
normval = self._calc_norm(mc, self.inputs.use_differences[0])
tidx = find_indices(normval > self.inputs.norm_threshold)
ridx = find_indices(normval < 0)
else:
if self.inputs.use_differences[0]:
mc = np.concatenate((np.zeros((1, 6)), np.diff(mc_in, n=1, axis=0)), axis=0)
traval = mc[:, 0:3] # translation parameters (mm)
rotval = mc[:, 3:6] # rotation parameters (rad)
tidx = find_indices(np.sum(abs(traval) > self.inputs.translation_threshold, 1) > 0)
ridx = find_indices(np.sum(abs(rotval) > self.inputs.rotation_threshold, 1) > 0)
# read in functional image
if isinstance(imgfile, str):
nim = load(imgfile)
elif isinstance(imgfile, list):
if len(imgfile) == 1:
nim = load(imgfile[0])
else:
images = [load(f) for f in imgfile]
nim = funcs.concat_images(images)
# compute global intensity signal
(x, y, z, timepoints) = nim.get_shape()
data = nim.get_data()
g = np.zeros((timepoints, 1))
masktype = self.inputs.mask_type
if masktype == 'spm_global': # spm_global like calculation
intersect_mask = self.inputs.intersect_mask
if intersect_mask:
mask = np.ones((x, y, z), dtype=bool)
for t0 in range(timepoints):
vol = data[:, :, :, t0]
mask = mask * (vol > (self._nanmean(vol) / 8))
for t0 in range(timepoints):
vol = data[:, :, :, t0]
g[t0] = self._nanmean(vol[mask])
if len(find_indices(mask)) < (np.prod((x, y, z)) / 10):
intersect_mask = False
g = np.zeros((timepoints, 1))
if not intersect_mask:
for t0 in range(timepoints):
vol = data[:, :, :, t0]
mask = vol > (self._nanmean(vol) / 8)
g[t0] = self._nanmean(vol[mask])
elif masktype == 'file': # uses a mask image to determine intensity
mask = load(self.inputs.mask_file).get_data()
mask = mask > 0.5
for t0 in range(timepoints):
vol = data[:, :, :, t0]
g[t0] = self._nanmean(vol[mask])
elif masktype == 'thresh': # uses a fixed signal threshold
for t0 in range(timepoints):
vol = data[:, :, :, t0]
mask = vol > self.inputs.mask_threshold
g[t0] = self._nanmean(vol[mask])
else:
mask = np.ones((x, y, z))
g = self._nanmean(data[mask > 0, :], 1)
# compute normalized intensity values
gz = signal.detrend(g, axis=0) # detrend the signal
if self.inputs.use_differences[1]:
gz = np.concatenate((np.zeros((1, 1)), np.diff(gz, n=1, axis=0)), axis=0)
gz = (gz - np.mean(gz)) / np.std(gz) # normalize the detrended signal
iidx = find_indices(abs(gz) > self.inputs.zintensity_threshold)
outliers = np.unique(np.union1d(iidx, np.union1d(tidx, ridx)))
artifactfile, intensityfile, statsfile, normfile, plotfile = self._get_output_filenames(imgfile, cwd)
# write output to outputfile
np.savetxt(artifactfile, outliers, fmt='%d', delimiter=' ')
np.savetxt(intensityfile, g, fmt='%.2f', delimiter=' ')
if self.inputs.use_norm:
np.savetxt(normfile, normval, fmt='%.4f', delimiter=' ')
if isdefined(self.inputs.save_plot) and self.inputs.save_plot:
fig = plt.figure()
if isdefined(self.inputs.use_norm) and self.inputs.use_norm:
plt.subplot(211)
else:
plt.subplot(311)
self._plot_outliers_with_wave(gz, iidx, 'Intensity')
if isdefined(self.inputs.use_norm) and self.inputs.use_norm:
plt.subplot(212)
self._plot_outliers_with_wave(normval, np.union1d(tidx, ridx), 'Norm (mm)')
else:
diff = ''
if self.inputs.use_differences[0]:
diff = 'diff'
plt.subplot(312)
self._plot_outliers_with_wave(traval, tidx, 'Translation (mm)' + diff)
plt.subplot(313)
self._plot_outliers_with_wave(rotval, ridx, 'Rotation (rad)' + diff)
plt.savefig(plotfile)
plt.close(fig)
motion_outliers = np.union1d(tidx, ridx)
stats = [{'motion_file': motionfile,
'functional_file': imgfile},
{'common_outliers': len(np.intersect1d(iidx, motion_outliers)),
'intensity_outliers': len(np.setdiff1d(iidx, motion_outliers)),
'motion_outliers': len(np.setdiff1d(motion_outliers, iidx)),
},
{'motion': [{'using differences': self.inputs.use_differences[0]},
{'mean': np.mean(mc_in, axis=0).tolist(),
'min': np.min(mc_in, axis=0).tolist(),
'max': np.max(mc_in, axis=0).tolist(),
'std': np.std(mc_in, axis=0).tolist()},
]},
{'intensity': [{'using differences': self.inputs.use_differences[1]},
{'mean': np.mean(gz, axis=0).tolist(),
'min': np.min(gz, axis=0).tolist(),
'max': np.max(gz, axis=0).tolist(),
'std': np.std(gz, axis=0).tolist()},
]},
]
if self.inputs.use_norm:
stats.insert(3, {'motion_norm': {'mean': np.mean(normval, axis=0).tolist(),
'min': np.min(normval, axis=0).tolist(),
'max': np.max(normval, axis=0).tolist(),
'std': np.std(normval, axis=0).tolist(),
}})
save_json(statsfile, stats)
def _detect_outliers_core(self, imgfile, motionfile, runidx, cwd=None):
"""
Core routine for detecting outliers
"""
if not cwd:
cwd = os.getcwd()
# read in motion parameters
mc_in = np.loadtxt(motionfile)
mc = deepcopy(mc_in)
if self.inputs.parameter_source == 'SPM':
pass
elif self.inputs.parameter_source == 'FSL':
mc = mc[:,[3,4,5,0,1,2]]
elif self.inputs.parameter_source == 'Siemens':
Exception("Siemens PACE format not implemented yet")
else:
Exception("Unknown source for movement parameters")
if self.inputs.use_norm:
# calculate the norm of the motion parameters
normval = self._calc_norm(mc,self.inputs.use_differences[0])
tidx = find_indices(normval>self.inputs.norm_threshold)
ridx = find_indices(normval<0)
else:
if self.inputs.use_differences[0]:
mc = np.concatenate( (np.zeros((1,6)),np.diff(mc_in,n=1,axis=0)) , axis=0)
traval = mc[:,0:3] # translation parameters (mm)
rotval = mc[:,3:6] # rotation parameters (rad)
tidx = find_indices(np.sum(abs(traval)>self.inputs.translation_threshold,1)>0)
ridx = find_indices(np.sum(abs(rotval)>self.inputs.rotation_threshold,1)>0)
# read in functional image
if isinstance(imgfile,str):
nim = load(imgfile)
elif isinstance(imgfile,list):
if len(imgfile) == 1:
nim = load(imgfile[0])
else:
images = [load(f) for f in imgfile]
nim = funcs.concat_images(images)
# compute global intensity signal
(x,y,z,timepoints) = nim.get_shape()
data = nim.get_data()
g = np.zeros((timepoints,1))
masktype = self.inputs.mask_type
if masktype == 'spm_global': # spm_global like calculation
intersect_mask = self.inputs.intersect_mask
if intersect_mask:
mask = np.ones((x,y,z),dtype=bool)
for t0 in range(timepoints):
vol = data[:,:,:,t0]
mask = mask*(vol>(self._nanmean(vol)/8))
for t0 in range(timepoints):
vol = data[:,:,:,t0]
g[t0] = self._nanmean(vol[mask])
if len(find_indices(mask))<(np.prod((x,y,z))/10):
intersect_mask = False
g = np.zeros((timepoints,1))
if not intersect_mask:
for t0 in range(timepoints):
vol = data[:,:,:,t0]
mask = vol>(self._nanmean(vol)/8)
g[t0] = self._nanmean(vol[mask])
elif masktype == 'file': # uses a mask image to determine intensity
mask = load(self.inputs.mask_file).get_data()
mask = mask>0.5
for t0 in range(timepoints):
vol = data[:,:,:,t0]
g[t0] = self._nanmean(vol[mask])
elif masktype == 'thresh': # uses a fixed signal threshold
for t0 in range(timepoints):
vol = data[:,:,:,t0]
mask = vol>self.inputs.mask_threshold
g[t0] = self._nanmean(vol[mask])
else:
mask = np.ones((x,y,z))
g = self._nanmean(data[mask>0,:],1)
# compute normalized intensity values
gz = signal.detrend(g,axis=0) # detrend the signal
if self.inputs.use_differences[1]:
gz = np.concatenate( (np.zeros((1,1)),np.diff(gz,n=1,axis=0)) , axis=0)
gz = (gz-np.mean(gz))/np.std(gz) # normalize the detrended signal
iidx = find_indices(abs(gz)>self.inputs.zintensity_threshold)
outliers = np.unique(np.union1d(iidx,np.union1d(tidx,ridx)))
artifactfile,intensityfile,statsfile,normfile = self._get_output_filenames(imgfile,cwd)
# write output to outputfile
np.savetxt(artifactfile, outliers, fmt='%d', delimiter=' ')
np.savetxt(intensityfile, g, fmt='%.2f', delimiter=' ')
if self.inputs.use_norm:
np.savetxt(normfile, normval, fmt='%.4f', delimiter=' ')
file = open(statsfile,'w')
file.write("Stats for:\n")
file.write("Motion file: %s\n" % motionfile)
file.write("Functional file: %s\n" % imgfile)
file.write("Motion:\n")
file.write("Number of Motion Outliers: %d\n"%len(np.union1d(tidx,ridx)))
file.write("Motion (original):\n")
file.write( ''.join(('mean: ',str(np.mean(mc_in,axis=0)),'\n')))
file.write( ''.join(('min: ',str(np.min(mc_in,axis=0)),'\n')))
file.write( ''.join(('max: ',str(np.max(mc_in,axis=0)),'\n')))
file.write( ''.join(('std: ',str(np.std(mc_in,axis=0)),'\n')))
if self.inputs.use_norm:
if self.inputs.use_differences[0]:
file.write("Motion (norm-differences):\n")
else:
file.write("Motion (norm):\n")
file.write( ''.join(('mean: ',str(np.mean(normval,axis=0)),'\n')))
file.write( ''.join(('min: ',str(np.min(normval,axis=0)),'\n')))
file.write( ''.join(('max: ',str(np.max(normval,axis=0)),'\n')))
file.write( ''.join(('std: ',str(np.std(normval,axis=0)),'\n')))
elif self.inputs.use_differences[0]:
file.write("Motion (differences):\n")
file.write( ''.join(('mean: ',str(np.mean(mc,axis=0)),'\n')))
file.write( ''.join(('min: ',str(np.min(mc,axis=0)),'\n')))
file.write( ''.join(('max: ',str(np.max(mc,axis=0)),'\n')))
file.write( ''.join(('std: ',str(np.std(mc,axis=0)),'\n')))
if self.inputs.use_differences[1]:
file.write("Normalized intensity:\n")
else:
file.write("Intensity:\n")
file.write("Number of Intensity Outliers: %d\n"%len(iidx))
file.write( ''.join(('min: ',str(np.min(gz,axis=0)),'\n')))
file.write( ''.join(('max: ',str(np.max(gz,axis=0)),'\n')))
file.write( ''.join(('mean: ',str(np.mean(gz,axis=0)),'\n')))
file.write( ''.join(('std: ',str(np.std(gz,axis=0)),'\n')))
file.close()
def _detect_outliers_core(self, imgfile, motionfile, runidx, cwd=None):
"""
Core routine for detecting outliers
"""
if not cwd:
cwd = os.getcwd()
# read in motion parameters
mc_in = np.loadtxt(motionfile)
mc = deepcopy(mc_in)
if self.inputs.parameter_source == 'SPM':
pass
elif self.inputs.parameter_source == 'FSL':
mc = mc[:, [3, 4, 5, 0, 1, 2]]
elif self.inputs.parameter_source == 'Siemens':
Exception("Siemens PACE format not implemented yet")
else:
Exception("Unknown source for movement parameters")
if self.inputs.use_norm:
# calculate the norm of the motion parameters
normval = self._calc_norm(mc, self.inputs.use_differences[0])
tidx = find_indices(normval > self.inputs.norm_threshold)
ridx = find_indices(normval < 0)
else:
if self.inputs.use_differences[0]:
mc = np.concatenate((np.zeros((1, 6)), np.diff(mc_in, n=1, axis=0)), axis=0)
traval = mc[:, 0:3] # translation parameters (mm)
rotval = mc[:, 3:6] # rotation parameters (rad)
tidx = find_indices(np.sum(abs(traval) > self.inputs.translation_threshold, 1) > 0)
ridx = find_indices(np.sum(abs(rotval) > self.inputs.rotation_threshold, 1) > 0)
# read in functional image
if isinstance(imgfile, str):
nim = load(imgfile)
elif isinstance(imgfile, list):
if len(imgfile) == 1:
nim = load(imgfile[0])
else:
images = [load(f) for f in imgfile]
nim = funcs.concat_images(images)
# compute global intensity signal
(x, y, z, timepoints) = nim.get_shape()
data = nim.get_data()
g = np.zeros((timepoints, 1))
masktype = self.inputs.mask_type
if masktype == 'spm_global': # spm_global like calculation
intersect_mask = self.inputs.intersect_mask
if intersect_mask:
mask = np.ones((x, y, z), dtype=bool)
for t0 in range(timepoints):
vol = data[:, :, :, t0]
mask = mask * (vol > (self._nanmean(vol) / 8))
for t0 in range(timepoints):
vol = data[:, :, :, t0]
g[t0] = self._nanmean(vol[mask])
if len(find_indices(mask)) < (np.prod((x, y, z)) / 10):
intersect_mask = False
g = np.zeros((timepoints, 1))
if not intersect_mask:
for t0 in range(timepoints):
vol = data[:, :, :, t0]
mask = vol > (self._nanmean(vol) / 8)
g[t0] = self._nanmean(vol[mask])
elif masktype == 'file': # uses a mask image to determine intensity
mask = load(self.inputs.mask_file).get_data()
mask = mask > 0.5
for t0 in range(timepoints):
vol = data[:, :, :, t0]
g[t0] = self._nanmean(vol[mask])
elif masktype == 'thresh': # uses a fixed signal threshold
for t0 in range(timepoints):
vol = data[:, :, :, t0]
mask = vol > self.inputs.mask_threshold
g[t0] = self._nanmean(vol[mask])
else:
mask = np.ones((x, y, z))
g = self._nanmean(data[mask > 0, :], 1)
# compute normalized intensity values
gz = signal.detrend(g, axis=0) # detrend the signal
if self.inputs.use_differences[1]:
gz = np.concatenate((np.zeros((1, 1)), np.diff(gz, n=1, axis=0)), axis=0)
gz = (gz - np.mean(gz)) / np.std(gz) # normalize the detrended signal
iidx = find_indices(abs(gz) > self.inputs.zintensity_threshold)
outliers = np.unique(np.union1d(iidx, np.union1d(tidx, ridx)))
artifactfile, intensityfile, statsfile, normfile, plotfile = self._get_output_filenames(imgfile, cwd)
# write output to outputfile
np.savetxt(artifactfile, outliers, fmt='%d', delimiter=' ')
np.savetxt(intensityfile, g, fmt='%.2f', delimiter=' ')
if self.inputs.use_norm:
np.savetxt(normfile, normval, fmt='%.4f', delimiter=' ')
if isdefined(self.inputs.save_plot) and self.inputs.save_plot:
import matplotlib.pyplot as plt
fig = plt.figure()
if isdefined(self.inputs.use_norm) and self.inputs.use_norm:
plt.subplot(211)
else:
plt.subplot(311)
self._plot_outliers_with_wave(gz, iidx, 'Intensity')
if isdefined(self.inputs.use_norm) and self.inputs.use_norm:
plt.subplot(212)
self._plot_outliers_with_wave(normval, np.union1d(tidx, ridx), 'Norm (mm)')
else:
diff = ''
if self.inputs.use_differences[0]:
diff = 'diff'
plt.subplot(312)
self._plot_outliers_with_wave(traval, tidx, 'Translation (mm)' + diff)
plt.subplot(313)
self._plot_outliers_with_wave(rotval, ridx, 'Rotation (rad)' + diff)
plt.savefig(plotfile)
plt.close(fig)
motion_outliers = np.union1d(tidx, ridx)
stats = [{'motion_file': motionfile,
'functional_file': imgfile},
{'common_outliers': len(np.intersect1d(iidx, motion_outliers)),
'intensity_outliers': len(np.setdiff1d(iidx, motion_outliers)),
'motion_outliers': len(np.setdiff1d(motion_outliers, iidx)),
},
{'motion': [{'using differences': self.inputs.use_differences[0]},
{'mean': np.mean(mc_in, axis=0).tolist(),
'min': np.min(mc_in, axis=0).tolist(),
'max': np.max(mc_in, axis=0).tolist(),
'std': np.std(mc_in, axis=0).tolist()},
]},
{'intensity': [{'using differences': self.inputs.use_differences[1]},
{'mean': np.mean(gz, axis=0).tolist(),
'min': np.min(gz, axis=0).tolist(),
'max': np.max(gz, axis=0).tolist(),
'std': np.std(gz, axis=0).tolist()},
]},
]
if self.inputs.use_norm:
stats.insert(3, {'motion_norm': {'mean': np.mean(normval, axis=0).tolist(),
'min': np.min(normval, axis=0).tolist(),
'max': np.max(normval, axis=0).tolist(),
'std': np.std(normval, axis=0).tolist(),
}})
save_json(statsfile, stats)