def clean(signals, confounds=None, t_r=2.5, low_pass=None, high_pass=None, detrend=False, standardize=True, shift_confounds=False): """ Normalize the signal, and if any confounds are given, project in the orthogonal space. Low pass filter improves specificity (more interesting arrows selected) High pass filter should be kepts small, so as not to kill sensitivity """ if standardize: signals = _standardize(signals, normalize=True) elif detrend: signals = _standardize(signals, normalize=False) signals = np.asarray(signals) if confounds is not None: if isinstance(confounds, basestring): filename = confounds confounds = np.genfromtxt(filename) if np.isnan(confounds.flat[0]): # There may be a header if np.version.short_version >= '1.4.0': confounds = np.genfromtxt(filename, skip_header=1) else: confounds = np.genfromtxt(filename, skiprows=1) # Restrict the signal to the orthogonal of the confounds confounds = np.atleast_2d(confounds) if shift_confounds: confounds = np.r_[confounds[..., 1:-1], confounds[..., 2:], confounds[..., :-2]] signals = signals[..., 1:-1] confounds = _standardize(confounds, normalize=True) confounds = qr_economic(confounds)[0].T signals -= np.dot(np.dot(signals, confounds.T), confounds) if low_pass is not None or high_pass is not None: signals = butterworth(signals, sampling_rate=1. / t_r, low_pass=low_pass, high_pass=high_pass) if detrend: # This is faster than scipy.detrend and equivalent regressor = np.arange(signals.shape[1]).astype(np.float) regressor -= regressor.mean() regressor /= np.sqrt((regressor ** 2).sum()) signals -= np.dot(signals, regressor)[:, np.newaxis] * regressor if standardize: signals = _standardize(signals, normalize=True) elif detrend: signals = _standardize(signals, normalize=False) return signals
def compute(self): matrix = self.getInputFromPort('matrix') n_components = self.forceGetInputFromPort('n_components', 2) n_features = matrix.values.shape[1] rng = np.random.RandomState(42) Q, _ = qr_economic(rng.normal(size=(n_features, n_components))) Y = np.dot(Q.T, matrix.values.T).T proj_matrix = copy.deepcopy(matrix) proj_matrix.values = Y self.setResult('proj_matrix', proj_matrix)
def compute(self): matrix = self.getInputFromPort("matrix") n_components = self.forceGetInputFromPort("n_components", 2) n_features = matrix.values.shape[1] rng = np.random.RandomState(42) Q, _ = qr_economic(rng.normal(size=(n_features, n_components))) Y = np.dot(Q.T, matrix.values.T).T proj_matrix = copy.deepcopy(matrix) proj_matrix.values = Y self.setResult("proj_matrix", proj_matrix)
img = np.zeros((10 * N, 10 * N)) for i in range(N): ix = 10 * i + 1 for j in range(N): iy = 10 * j + 1 img[ix:ix + 8, iy:iy + 8] = X[i * N + j].reshape((8, 8)) pl.imshow(img, cmap=pl.cm.binary) pl.xticks([]) pl.yticks([]) pl.title('A selection from the 64-dimensional digits dataset') #---------------------------------------------------------------------- # Random 2D projection using a random unitary matrix print "Computing random projection" rng = np.random.RandomState(42) Q, _ = qr_economic(rng.normal(size=(n_features, 2))) X_projected = np.dot(Q.T, X.T).T plot_embedding(X_projected, "Random Projection of the digits") #---------------------------------------------------------------------- # Projection on to the first 2 principal components print "Computing PCA projection" t0 = time() X_pca = decomposition.RandomizedPCA(n_components=2).fit_transform(X) plot_embedding( X_pca, "Principal Components projection of the digits (time %.2fs)" % (time() - t0)) #---------------------------------------------------------------------- # Projection on to the first 2 linear discriminant components
for i in range(N): ix = 10 * i + 1 for j in range(N): iy = 10 * j + 1 img[ix : ix + 8, iy : iy + 8] = X[i * N + j].reshape((8, 8)) pl.imshow(img, cmap=pl.cm.binary) pl.xticks([]) pl.yticks([]) pl.title("A selection from the 64-dimensional digits dataset") # ---------------------------------------------------------------------- # Random 2D projection using a random unitary matrix print "Computing random projection" rng = np.random.RandomState(42) Q, _ = qr_economic(rng.normal(size=(n_features, 2))) X_projected = np.dot(Q.T, X.T).T plot_embedding(X_projected, "Random Projection of the digits") # ---------------------------------------------------------------------- # Projection on to the first 2 principal components print "Computing PCA projection" t0 = time() X_pca = decomposition.RandomizedPCA(n_components=2).fit_transform(X) plot_embedding(X_pca, "Principal Components projection of the digits (time %.2fs)" % (time() - t0)) # ---------------------------------------------------------------------- # Projection on to the first 2 linear discriminant components
def random_projection(X,y, tag): _, n_features = X.shape rng = np.random.RandomState(13) Q, _ = qr_economic(rng.normal(size=(n_features,2))) X_proj = np.dot(Q.T, X.T).T plot_embedding(X_proj, y, 'random_%s' % tag, title='Random Projection: %s' % tag)
def clean(signals, detrend=True, standardize=True, confounds=None, low_pass=None, high_pass=None, t_r=2.5): """Improve SNR on masked fMRI signals. This function can do several things on the input signals, in the following order: - detrend - standardize - remove confounds - low- and high-pass filter Low-pass filtering improves specificity. High-pass filtering should be kept small, to keep some sensitivity. Filtering is only meaningful on evenly-sampled signals. Parameters ========== signals: numpy.ndarray Timeseries. Must have shape (instant number, features number). This array is not modified. confounds: numpy.ndarray, str or list of Confounds timeseries. Shape must be (instant number, confound number), or just (instant number,) The number of time instants in signals and confounds must be identical (i.e. signals.shape[0] == confounds.shape[0]). If a string is provided, it is assumed to be the name of a csv file containing signals as columns, with an optional one-line header. If a list is provided, all confounds are removed from the input signal, as if all were in the same array. t_r: float Repetition time, in second (sampling period). low_pass, high_pass: float Respectively low and high cutoff frequencies, in Hertz. detrend: bool If detrending should be applied on timeseries (before confound removal) standardize: bool If True, returned signals are set to unit variance. Returns ======= cleaned_signals: numpy.ndarray Input signals, cleaned. Same shape as `signals`. Notes ===== Confounds removal is based on a projection on the orthogonal of the signal space. See `Friston, K. J., A. P. Holmes, K. J. Worsley, J.-P. Poline, C. D. Frith, et R. S. J. Frackowiak. "Statistical Parametric Maps in Functional Imaging: A General Linear Approach". Human Brain Mapping 2, no 4 (1994): 189-210. <http://dx.doi.org/10.1002/hbm.460020402>`_ """ if not isinstance(confounds, (list, tuple, basestring, np.ndarray, type(None))): raise TypeError("confounds keyword has an unhandled type: %s" % confounds.__class__) # Standardize / detrend normalize = False if confounds is not None: # If confounds are to be removed, then force normalization to improve # matrix conditioning. normalize = True signals = _standardize(signals, normalize=normalize, detrend=detrend) # Remove confounds if confounds is not None: if not isinstance(confounds, (list, tuple)): confounds = (confounds, ) # Read confounds all_confounds = [] for confound in confounds: if isinstance(confound, basestring): filename = confound confound = np.genfromtxt(filename) if np.isnan(confound.flat[0]): # There may be a header if np_version >= [1, 4, 0]: confound = np.genfromtxt(filename, skip_header=1) else: confound = np.genfromtxt(filename, skiprows=1) if confound.shape[0] != signals.shape[0]: raise ValueError("Confound signal has an incorrect length") elif isinstance(confound, np.ndarray): if confound.ndim == 1: confound = np.atleast_2d(confound).T elif confound.ndim != 2: raise ValueError("confound array has an incorrect number " "of dimensions: %d" % confound.ndim) if confound.shape[0] != signals.shape[0]: raise ValueError("Confound signal has an incorrect length") else: raise TypeError("confound has an unhandled type: %s" % confound.__class__) all_confounds.append(confound) # Restrict the signal to the orthogonal of the confounds confounds = np.hstack(all_confounds) del all_confounds confounds = _standardize(confounds, normalize=True, detrend=detrend) Q = qr_economic(confounds)[0] signals -= np.dot(Q, np.dot(Q.T, signals)) if low_pass is not None or high_pass is not None: signals = butterworth(signals, sampling_rate=1. / t_r, low_pass=low_pass, high_pass=high_pass) if standardize: signals = _standardize(signals, normalize=True, detrend=False) signals *= np.sqrt(signals.shape[0]) # for unit variance return signals
def clean(signals, detrend=True, standardize=True, confounds=None, low_pass=None, high_pass=None, t_r=2.5): """Improve SNR on masked fMRI signals. This function can do several things on the input signals, in the following order: - detrend - standardize - remove confounds - low- and high-pass filter Low-pass filtering improves specificity. High-pass filtering should be kept small, to keep some sensitivity. Filtering is only meaningful on evenly-sampled timeseries. Parameters ========== signals (numpy array) Timeseries. Must have shape (instant number, features number). This array is not modified. confounds (numpy array or file name) Confounds timeseries. Shape muse be (instant number, confound number). The number of time instants in signals and confounds must be identical (i.e. signals.shape[0] == confounds.shape[0]) t_r (float) Repetition time, in second (sampling period). low_pass, high_pass (float) Respectively low and high cutoff frequencies, in Hertz. detrend (boolean) If detrending should be applied on timeseries (before confound removal) standardize (boolean) If variances should be set to one and mean to zero for all timeseries (before confound removal) Returns ======= cleaned_signals (numpy array) Input signals, cleaned. Same shape as `signals`. Notes ===== Confounds removal is based on a projection on the orthogonal of the signal space. See `Friston, K. J., A. P. Holmes, K. J. Worsley, J.-P. Poline, C. D. Frith, et R. S. J. Frackowiak. "Statistical Parametric Maps in Functional Imaging: A General Linear Approach". Human Brain Mapping 2, no 4 (1994): 189-210. <http://dx.doi.org/10.1002/hbm.460020402>`_ """ # Standardize / detrend signals = _standardize(signals, normalize=standardize, detrend=detrend) # Remove confounds if confounds is not None: if isinstance(confounds, basestring): filename = confounds confounds = np.genfromtxt(filename) if np.isnan(confounds.flat[0]): # There may be a header if np.version.short_version >= '1.4.0': confounds = np.genfromtxt(filename, skip_header=1) else: confounds = np.genfromtxt(filename, skiprows=1) # Restrict the signal to the orthogonal of the confounds confounds = np.atleast_2d(confounds) confounds = _standardize(confounds, normalize=True) Q = qr_economic(confounds)[0] signals -= np.dot(Q, np.dot(Q.T, signals)) if low_pass is not None or high_pass is not None: signals = butterworth(signals, sampling_rate=1. / t_r, low_pass=low_pass, high_pass=high_pass) return signals
def clean( signals, confounds=None, low_pass=0.2, t_r=2.5, high_pass=False, detrend=False, standardize=True, shift_confounds=False, ): """ Normalize the signal, and if any confounds are given, project in the orthogonal space. Low pass filter improves specificity (more interesting arrows selected) High pass filter should be kepts small, so as not to kill sensitivity """ if standardize: signals = _standardize(signals, normalize=True) elif detrend: signals = _standardize(signals, normalize=False) signals = np.asarray(signals) if confounds is not None: if isinstance(confounds, basestring): filename = confounds confounds = np.genfromtxt(filename) if np.isnan(confounds.flat[0]): # There may be a header if np.version.short_version >= "1.4.0": confounds = np.genfromtxt(filename, skip_header=1) else: confounds = np.genfromtxt(filename, skiprows=1) # Restrict the signal to the orthogonal of the confounds confounds = np.atleast_2d(confounds) if shift_confounds: confounds = np.r_[confounds[..., 1:-1], confounds[..., 2:], confounds[..., :-2]] signals = signals[..., 1:-1] confounds = _standardize(confounds, normalize=True) confounds = qr_economic(confounds)[0].T signals -= np.dot(np.dot(signals, confounds.T), confounds) if low_pass and high_pass and high_pass >= low_pass: raise ValueError( "Your value for high pass filter (%f) is higher or" " equal to the value for low pass filter (%f). This" " would result in a blank signal" % (high_pass, low_pass) ) if low_pass or high_pass: n = signals.shape[-1] freq = fftpack.fftfreq(n, d=t_r) for s in signals: fft = fftpack.fft(s) if low_pass: fft[np.abs(freq) > low_pass] = 0 if high_pass: fft[np.abs(freq) < high_pass] = 0 s[:] = fftpack.ifft(fft) if detrend: # This is faster than scipy.detrend and equivalent regressor = np.arange(signals.shape[1]).astype(np.float) regressor -= regressor.mean() regressor /= np.sqrt((regressor ** 2).sum()) signals -= np.dot(signals, regressor)[:, np.newaxis] * regressor if standardize: signals = _standardize(signals, normalize=True) elif detrend: signals = _standardize(signals, normalize=False) return signals
def clean(signals, confounds=None, low_pass=0.2, t_r=2.5, high_pass=False, detrend=False, standardize=True, shift_confounds=False): """ Normalize the signal, and if any confounds are given, project in the orthogonal space. Low pass filter improves specificity (more interesting arrows selected) High pass filter should be kepts small, so as not to kill sensitivity """ if standardize: signals = _standardize(signals, normalize=True) elif detrend: signals = _standardize(signals, normalize=False) signals = np.asarray(signals) if confounds is not None: if isinstance(confounds, basestring): filename = confounds confounds = np.genfromtxt(filename) if np.isnan(confounds.flat[0]): # There may be a header if np.version.short_version >= '1.4.0': confounds = np.genfromtxt(filename, skip_header=1) else: confounds = np.genfromtxt(filename, skiprows=1) # Restrict the signal to the orthogonal of the confounds confounds = np.atleast_2d(confounds) if shift_confounds: confounds = np.r_[confounds[..., 1:-1], confounds[..., 2:], confounds[..., :-2]] signals = signals[..., 1:-1] confounds = _standardize(confounds, normalize=True) confounds = qr_economic(confounds)[0].T signals -= np.dot(np.dot(signals, confounds.T), confounds) if low_pass and high_pass and high_pass >= low_pass: raise ValueError("Your value for high pass filter (%f) is higher or" " equal to the value for low pass filter (%f). This" " would result in a blank signal" % (high_pass, low_pass)) if low_pass or high_pass: n = signals.shape[-1] freq = fftpack.fftfreq(n, d=t_r) for s in signals: fft = fftpack.fft(s) if low_pass: fft[np.abs(freq) > low_pass] = 0 if high_pass: fft[np.abs(freq) < high_pass] = 0 s[:] = fftpack.ifft(fft) if detrend: # This is faster than scipy.detrend and equivalent regressor = np.arange(signals.shape[1]).astype(np.float) regressor -= regressor.mean() regressor /= np.sqrt((regressor**2).sum()) signals -= np.dot(signals, regressor)[:, np.newaxis] * regressor if standardize: signals = _standardize(signals, normalize=True) elif detrend: signals = _standardize(signals, normalize=False) return signals