def clean(signals, confounds=None, t_r=2.5, low_pass=None,
high_pass=None, detrend=False, standardize=True,
shift_confounds=False):
""" Normalize the signal, and if any confounds are given, project in
the orthogonal space.
Low pass filter improves specificity (more interesting arrows
selected)
High pass filter should be kepts small, so as not to kill
sensitivity
"""
if standardize:
signals = _standardize(signals, normalize=True)
elif detrend:
signals = _standardize(signals, normalize=False)
signals = np.asarray(signals)
if confounds is not None:
if isinstance(confounds, basestring):
filename = confounds
confounds = np.genfromtxt(filename)
if np.isnan(confounds.flat[0]):
# There may be a header
if np.version.short_version >= '1.4.0':
confounds = np.genfromtxt(filename, skip_header=1)
else:
confounds = np.genfromtxt(filename, skiprows=1)
# Restrict the signal to the orthogonal of the confounds
confounds = np.atleast_2d(confounds)
if shift_confounds:
confounds = np.r_[confounds[..., 1:-1],
confounds[..., 2:],
confounds[..., :-2]]
signals = signals[..., 1:-1]
confounds = _standardize(confounds, normalize=True)
confounds = qr_economic(confounds)[0].T
signals -= np.dot(np.dot(signals, confounds.T), confounds)
if low_pass is not None or high_pass is not None:
signals = butterworth(signals, sampling_rate=1. / t_r,
low_pass=low_pass, high_pass=high_pass)
if detrend:
# This is faster than scipy.detrend and equivalent
regressor = np.arange(signals.shape[1]).astype(np.float)
regressor -= regressor.mean()
regressor /= np.sqrt((regressor ** 2).sum())
signals -= np.dot(signals, regressor)[:, np.newaxis] * regressor
if standardize:
signals = _standardize(signals, normalize=True)
elif detrend:
signals = _standardize(signals, normalize=False)
return signals
def compute(self):
matrix = self.getInputFromPort('matrix')
n_components = self.forceGetInputFromPort('n_components', 2)
n_features = matrix.values.shape[1]
rng = np.random.RandomState(42)
Q, _ = qr_economic(rng.normal(size=(n_features, n_components)))
Y = np.dot(Q.T, matrix.values.T).T
proj_matrix = copy.deepcopy(matrix)
proj_matrix.values = Y
self.setResult('proj_matrix', proj_matrix)
def compute(self):
matrix = self.getInputFromPort("matrix")
n_components = self.forceGetInputFromPort("n_components", 2)
n_features = matrix.values.shape[1]
rng = np.random.RandomState(42)
Q, _ = qr_economic(rng.normal(size=(n_features, n_components)))
Y = np.dot(Q.T, matrix.values.T).T
proj_matrix = copy.deepcopy(matrix)
proj_matrix.values = Y
self.setResult("proj_matrix", proj_matrix)
img = np.zeros((10 * N, 10 * N))
for i in range(N):
ix = 10 * i + 1
for j in range(N):
iy = 10 * j + 1
img[ix:ix + 8, iy:iy + 8] = X[i * N + j].reshape((8, 8))
pl.imshow(img, cmap=pl.cm.binary)
pl.xticks([])
pl.yticks([])
pl.title('A selection from the 64-dimensional digits dataset')
#----------------------------------------------------------------------
# Random 2D projection using a random unitary matrix
print "Computing random projection"
rng = np.random.RandomState(42)
Q, _ = qr_economic(rng.normal(size=(n_features, 2)))
X_projected = np.dot(Q.T, X.T).T
plot_embedding(X_projected, "Random Projection of the digits")
#----------------------------------------------------------------------
# Projection on to the first 2 principal components
print "Computing PCA projection"
t0 = time()
X_pca = decomposition.RandomizedPCA(n_components=2).fit_transform(X)
plot_embedding(
X_pca, "Principal Components projection of the digits (time %.2fs)" %
(time() - t0))
#----------------------------------------------------------------------
# Projection on to the first 2 linear discriminant components
for i in range(N):
ix = 10 * i + 1
for j in range(N):
iy = 10 * j + 1
img[ix : ix + 8, iy : iy + 8] = X[i * N + j].reshape((8, 8))
pl.imshow(img, cmap=pl.cm.binary)
pl.xticks([])
pl.yticks([])
pl.title("A selection from the 64-dimensional digits dataset")
# ----------------------------------------------------------------------
# Random 2D projection using a random unitary matrix
print "Computing random projection"
rng = np.random.RandomState(42)
Q, _ = qr_economic(rng.normal(size=(n_features, 2)))
X_projected = np.dot(Q.T, X.T).T
plot_embedding(X_projected, "Random Projection of the digits")
# ----------------------------------------------------------------------
# Projection on to the first 2 principal components
print "Computing PCA projection"
t0 = time()
X_pca = decomposition.RandomizedPCA(n_components=2).fit_transform(X)
plot_embedding(X_pca, "Principal Components projection of the digits (time %.2fs)" % (time() - t0))
# ----------------------------------------------------------------------
# Projection on to the first 2 linear discriminant components
def random_projection(X,y, tag):
_, n_features = X.shape
rng = np.random.RandomState(13)
Q, _ = qr_economic(rng.normal(size=(n_features,2)))
X_proj = np.dot(Q.T, X.T).T
plot_embedding(X_proj, y, 'random_%s' % tag, title='Random Projection: %s' % tag)
def clean(signals, detrend=True, standardize=True, confounds=None,
low_pass=None, high_pass=None, t_r=2.5):
"""Improve SNR on masked fMRI signals.
This function can do several things on the input signals, in
the following order:
- detrend
- standardize
- remove confounds
- low- and high-pass filter
Low-pass filtering improves specificity.
High-pass filtering should be kept small, to keep some
sensitivity.
Filtering is only meaningful on evenly-sampled signals.
Parameters
==========
signals: numpy.ndarray
Timeseries. Must have shape (instant number, features number).
This array is not modified.
confounds: numpy.ndarray, str or list of
Confounds timeseries. Shape must be
(instant number, confound number), or just (instant number,)
The number of time instants in signals and confounds must be
identical (i.e. signals.shape[0] == confounds.shape[0]).
If a string is provided, it is assumed to be the name of a csv file
containing signals as columns, with an optional one-line header.
If a list is provided, all confounds are removed from the input
signal, as if all were in the same array.
t_r: float
Repetition time, in second (sampling period).
low_pass, high_pass: float
Respectively low and high cutoff frequencies, in Hertz.
detrend: bool
If detrending should be applied on timeseries (before
confound removal)
standardize: bool
If True, returned signals are set to unit variance.
Returns
=======
cleaned_signals: numpy.ndarray
Input signals, cleaned. Same shape as `signals`.
Notes
=====
Confounds removal is based on a projection on the orthogonal
of the signal space. See `Friston, K. J., A. P. Holmes,
K. J. Worsley, J.-P. Poline, C. D. Frith, et R. S. J. Frackowiak.
"Statistical Parametric Maps in Functional Imaging: A General
Linear Approach". Human Brain Mapping 2, no 4 (1994): 189-210.
<http://dx.doi.org/10.1002/hbm.460020402>`_
"""
if not isinstance(confounds,
(list, tuple, basestring, np.ndarray, type(None))):
raise TypeError("confounds keyword has an unhandled type: %s"
% confounds.__class__)
# Standardize / detrend
normalize = False
if confounds is not None:
# If confounds are to be removed, then force normalization to improve
# matrix conditioning.
normalize = True
signals = _standardize(signals, normalize=normalize, detrend=detrend)
# Remove confounds
if confounds is not None:
if not isinstance(confounds, (list, tuple)):
confounds = (confounds, )
# Read confounds
all_confounds = []
for confound in confounds:
if isinstance(confound, basestring):
filename = confound
confound = np.genfromtxt(filename)
if np.isnan(confound.flat[0]):
# There may be a header
if np_version >= [1, 4, 0]:
confound = np.genfromtxt(filename, skip_header=1)
else:
confound = np.genfromtxt(filename, skiprows=1)
if confound.shape[0] != signals.shape[0]:
raise ValueError("Confound signal has an incorrect length")
elif isinstance(confound, np.ndarray):
if confound.ndim == 1:
confound = np.atleast_2d(confound).T
elif confound.ndim != 2:
raise ValueError("confound array has an incorrect number "
"of dimensions: %d" % confound.ndim)
if confound.shape[0] != signals.shape[0]:
raise ValueError("Confound signal has an incorrect length")
else:
raise TypeError("confound has an unhandled type: %s"
% confound.__class__)
all_confounds.append(confound)
# Restrict the signal to the orthogonal of the confounds
confounds = np.hstack(all_confounds)
del all_confounds
confounds = _standardize(confounds, normalize=True, detrend=detrend)
Q = qr_economic(confounds)[0]
signals -= np.dot(Q, np.dot(Q.T, signals))
if low_pass is not None or high_pass is not None:
signals = butterworth(signals, sampling_rate=1. / t_r,
low_pass=low_pass, high_pass=high_pass)
if standardize:
signals = _standardize(signals, normalize=True, detrend=False)
signals *= np.sqrt(signals.shape[0]) # for unit variance
return signals
def clean(signals,
detrend=True,
standardize=True,
confounds=None,
low_pass=None,
high_pass=None,
t_r=2.5):
"""Improve SNR on masked fMRI signals.
This function can do several things on the input signals, in
the following order:
- detrend
- standardize
- remove confounds
- low- and high-pass filter
Low-pass filtering improves specificity.
High-pass filtering should be kept small, to keep some
sensitivity.
Filtering is only meaningful on evenly-sampled signals.
Parameters
==========
signals: numpy.ndarray
Timeseries. Must have shape (instant number, features number).
This array is not modified.
confounds: numpy.ndarray, str or list of
Confounds timeseries. Shape must be
(instant number, confound number), or just (instant number,)
The number of time instants in signals and confounds must be
identical (i.e. signals.shape[0] == confounds.shape[0]).
If a string is provided, it is assumed to be the name of a csv file
containing signals as columns, with an optional one-line header.
If a list is provided, all confounds are removed from the input
signal, as if all were in the same array.
t_r: float
Repetition time, in second (sampling period).
low_pass, high_pass: float
Respectively low and high cutoff frequencies, in Hertz.
detrend: bool
If detrending should be applied on timeseries (before
confound removal)
standardize: bool
If True, returned signals are set to unit variance.
Returns
=======
cleaned_signals: numpy.ndarray
Input signals, cleaned. Same shape as `signals`.
Notes
=====
Confounds removal is based on a projection on the orthogonal
of the signal space. See `Friston, K. J., A. P. Holmes,
K. J. Worsley, J.-P. Poline, C. D. Frith, et R. S. J. Frackowiak.
"Statistical Parametric Maps in Functional Imaging: A General
Linear Approach". Human Brain Mapping 2, no 4 (1994): 189-210.
<http://dx.doi.org/10.1002/hbm.460020402>`_
"""
if not isinstance(confounds,
(list, tuple, basestring, np.ndarray, type(None))):
raise TypeError("confounds keyword has an unhandled type: %s" %
confounds.__class__)
# Standardize / detrend
normalize = False
if confounds is not None:
# If confounds are to be removed, then force normalization to improve
# matrix conditioning.
normalize = True
signals = _standardize(signals, normalize=normalize, detrend=detrend)
# Remove confounds
if confounds is not None:
if not isinstance(confounds, (list, tuple)):
confounds = (confounds, )
# Read confounds
all_confounds = []
for confound in confounds:
if isinstance(confound, basestring):
filename = confound
confound = np.genfromtxt(filename)
if np.isnan(confound.flat[0]):
# There may be a header
if np_version >= [1, 4, 0]:
confound = np.genfromtxt(filename, skip_header=1)
else:
confound = np.genfromtxt(filename, skiprows=1)
if confound.shape[0] != signals.shape[0]:
raise ValueError("Confound signal has an incorrect length")
elif isinstance(confound, np.ndarray):
if confound.ndim == 1:
confound = np.atleast_2d(confound).T
elif confound.ndim != 2:
raise ValueError("confound array has an incorrect number "
"of dimensions: %d" % confound.ndim)
if confound.shape[0] != signals.shape[0]:
raise ValueError("Confound signal has an incorrect length")
else:
raise TypeError("confound has an unhandled type: %s" %
confound.__class__)
all_confounds.append(confound)
# Restrict the signal to the orthogonal of the confounds
confounds = np.hstack(all_confounds)
del all_confounds
confounds = _standardize(confounds, normalize=True, detrend=detrend)
Q = qr_economic(confounds)[0]
signals -= np.dot(Q, np.dot(Q.T, signals))
if low_pass is not None or high_pass is not None:
signals = butterworth(signals,
sampling_rate=1. / t_r,
low_pass=low_pass,
high_pass=high_pass)
if standardize:
signals = _standardize(signals, normalize=True, detrend=False)
signals *= np.sqrt(signals.shape[0]) # for unit variance
return signals
def clean(signals, detrend=True, standardize=True, confounds=None,
low_pass=None, high_pass=None, t_r=2.5):
"""Improve SNR on masked fMRI signals.
This function can do several things on the input signals, in
the following order:
- detrend
- standardize
- remove confounds
- low- and high-pass filter
Low-pass filtering improves specificity.
High-pass filtering should be kept small, to keep some
sensitivity.
Filtering is only meaningful on evenly-sampled timeseries.
Parameters
==========
signals (numpy array)
Timeseries. Must have shape (instant number, features number).
This array is not modified.
confounds (numpy array or file name)
Confounds timeseries. Shape muse be
(instant number, confound number). The number of time
instants in signals and confounds must be identical
(i.e. signals.shape[0] == confounds.shape[0])
t_r (float)
Repetition time, in second (sampling period).
low_pass, high_pass (float)
Respectively low and high cutoff frequencies, in Hertz.
detrend (boolean)
If detrending should be applied on timeseries (before
confound removal)
standardize (boolean)
If variances should be set to one and mean to zero for
all timeseries (before confound removal)
Returns
=======
cleaned_signals (numpy array)
Input signals, cleaned. Same shape as `signals`.
Notes
=====
Confounds removal is based on a projection on the orthogonal
of the signal space. See `Friston, K. J., A. P. Holmes,
K. J. Worsley, J.-P. Poline, C. D. Frith, et R. S. J. Frackowiak.
"Statistical Parametric Maps in Functional Imaging: A General
Linear Approach". Human Brain Mapping 2, no 4 (1994): 189-210.
<http://dx.doi.org/10.1002/hbm.460020402>`_
"""
# Standardize / detrend
signals = _standardize(signals, normalize=standardize, detrend=detrend)
# Remove confounds
if confounds is not None:
if isinstance(confounds, basestring):
filename = confounds
confounds = np.genfromtxt(filename)
if np.isnan(confounds.flat[0]):
# There may be a header
if np.version.short_version >= '1.4.0':
confounds = np.genfromtxt(filename, skip_header=1)
else:
confounds = np.genfromtxt(filename, skiprows=1)
# Restrict the signal to the orthogonal of the confounds
confounds = np.atleast_2d(confounds)
confounds = _standardize(confounds, normalize=True)
Q = qr_economic(confounds)[0]
signals -= np.dot(Q, np.dot(Q.T, signals))
if low_pass is not None or high_pass is not None:
signals = butterworth(signals, sampling_rate=1. / t_r,
low_pass=low_pass, high_pass=high_pass)
return signals
def clean(
signals,
confounds=None,
low_pass=0.2,
t_r=2.5,
high_pass=False,
detrend=False,
standardize=True,
shift_confounds=False,
):
""" Normalize the signal, and if any confounds are given, project in
the orthogonal space.
Low pass filter improves specificity (more interesting arrows
selected)
High pass filter should be kepts small, so as not to kill
sensitivity
"""
if standardize:
signals = _standardize(signals, normalize=True)
elif detrend:
signals = _standardize(signals, normalize=False)
signals = np.asarray(signals)
if confounds is not None:
if isinstance(confounds, basestring):
filename = confounds
confounds = np.genfromtxt(filename)
if np.isnan(confounds.flat[0]):
# There may be a header
if np.version.short_version >= "1.4.0":
confounds = np.genfromtxt(filename, skip_header=1)
else:
confounds = np.genfromtxt(filename, skiprows=1)
# Restrict the signal to the orthogonal of the confounds
confounds = np.atleast_2d(confounds)
if shift_confounds:
confounds = np.r_[confounds[..., 1:-1], confounds[..., 2:], confounds[..., :-2]]
signals = signals[..., 1:-1]
confounds = _standardize(confounds, normalize=True)
confounds = qr_economic(confounds)[0].T
signals -= np.dot(np.dot(signals, confounds.T), confounds)
if low_pass and high_pass and high_pass >= low_pass:
raise ValueError(
"Your value for high pass filter (%f) is higher or"
" equal to the value for low pass filter (%f). This"
" would result in a blank signal" % (high_pass, low_pass)
)
if low_pass or high_pass:
n = signals.shape[-1]
freq = fftpack.fftfreq(n, d=t_r)
for s in signals:
fft = fftpack.fft(s)
if low_pass:
fft[np.abs(freq) > low_pass] = 0
if high_pass:
fft[np.abs(freq) < high_pass] = 0
s[:] = fftpack.ifft(fft)
if detrend:
# This is faster than scipy.detrend and equivalent
regressor = np.arange(signals.shape[1]).astype(np.float)
regressor -= regressor.mean()
regressor /= np.sqrt((regressor ** 2).sum())
signals -= np.dot(signals, regressor)[:, np.newaxis] * regressor
if standardize:
signals = _standardize(signals, normalize=True)
elif detrend:
signals = _standardize(signals, normalize=False)
return signals
def clean(signals,
confounds=None,
low_pass=0.2,
t_r=2.5,
high_pass=False,
detrend=False,
standardize=True,
shift_confounds=False):
""" Normalize the signal, and if any confounds are given, project in
the orthogonal space.
Low pass filter improves specificity (more interesting arrows
selected)
High pass filter should be kepts small, so as not to kill
sensitivity
"""
if standardize:
signals = _standardize(signals, normalize=True)
elif detrend:
signals = _standardize(signals, normalize=False)
signals = np.asarray(signals)
if confounds is not None:
if isinstance(confounds, basestring):
filename = confounds
confounds = np.genfromtxt(filename)
if np.isnan(confounds.flat[0]):
# There may be a header
if np.version.short_version >= '1.4.0':
confounds = np.genfromtxt(filename, skip_header=1)
else:
confounds = np.genfromtxt(filename, skiprows=1)
# Restrict the signal to the orthogonal of the confounds
confounds = np.atleast_2d(confounds)
if shift_confounds:
confounds = np.r_[confounds[..., 1:-1], confounds[..., 2:],
confounds[..., :-2]]
signals = signals[..., 1:-1]
confounds = _standardize(confounds, normalize=True)
confounds = qr_economic(confounds)[0].T
signals -= np.dot(np.dot(signals, confounds.T), confounds)
if low_pass and high_pass and high_pass >= low_pass:
raise ValueError("Your value for high pass filter (%f) is higher or"
" equal to the value for low pass filter (%f). This"
" would result in a blank signal" %
(high_pass, low_pass))
if low_pass or high_pass:
n = signals.shape[-1]
freq = fftpack.fftfreq(n, d=t_r)
for s in signals:
fft = fftpack.fft(s)
if low_pass:
fft[np.abs(freq) > low_pass] = 0
if high_pass:
fft[np.abs(freq) < high_pass] = 0
s[:] = fftpack.ifft(fft)
if detrend:
# This is faster than scipy.detrend and equivalent
regressor = np.arange(signals.shape[1]).astype(np.float)
regressor -= regressor.mean()
regressor /= np.sqrt((regressor**2).sum())
signals -= np.dot(signals, regressor)[:, np.newaxis] * regressor
if standardize:
signals = _standardize(signals, normalize=True)
elif detrend:
signals = _standardize(signals, normalize=False)
return signals