def transform(self, X, n_components=None, in_place=False): """ Project a (new) dataset onto the components. Return the projected data and the associated d. We have to recompute U and d because the argument may not have the same number of lines. The argument must have the same number of columns than the datset used to fit the estimator. """ if n_components is None: n_components = self.n_components Xk = check_arrays(X) if not in_place: Xk = Xk.copy() n, p = Xk.shape if p != self.V.shape[0]: raise ValueError("The argument must have the same number of " "columns than the datset used to fit the " "estimator.") U = np.zeros((n, n_components)) d = np.zeros((n_components, )) for k in range(n_components): # Project on component j vk = self.V[:, k].reshape(-1, 1) uk = np.dot(X, vk) uk /= np.linalg.norm(uk) U[:, k] = uk[:, 0] dk = self.compute_d(Xk, uk, vk) d[k] = dk # Residualize Xk -= dk * np.dot(uk, vk.T) return U, d
def l1_max(cls, X): """Return the maximal value for l1 parameter for a given data matrix """ X = check_arrays(X) s = [np.linalg.norm(X[:, i]) for i in range(X.shape[1])] l1_max = np.max(s) / X.shape[0] return l1_max
def __init__(self, X, y, k=0.0, weights=None, penalty_start=0, mean=True): """ Parameters ---------- X : Numpy array (n-by-p). The regressor matrix. Training vectors, where n is the number of samples and p is the number of features. y : Numpy array (n-by-1). The regressand vector. Target values (class labels in classification). k : Non-negative float. The ridge parameter. weights: Numpy array (n-by-1). The sample's weights. penalty_start : Non-negative integer. The number of columns, variables etc., to except from penalisation. Equivalently, the first index to be penalised. Default is 0, all columns are included. mean : Boolean. Whether to compute the mean loss or not. Default is True, the mean loss is computed. """ self.X = X self.y = y self.k = np.maximum(0.0, check_arrays(k, flatten=True)) if weights is None: weights = np.ones(y.shape) # .reshape(y.shape) self.weights = weights self.penalty_start = max(0, int(penalty_start)) self.mean = bool(mean) self.reset()
def __init__(self, X, y, k, penalty_start=0, mean=True): """ Parameters ---------- X : Numpy array (n-by-p). The regressor matrix. y : Numpy array (n-by-1). The regressand vector. k : Non-negative float. The ridge parameter. penalty_start : Non-negative integer. The number of columns, variables etc., to except from penalisation. Equivalently, the first index to be penalised. Default is 0, all columns are included. mean : Boolean. Whether to compute the squared loss or the mean squared loss. Default is True, the mean squared loss. """ self.X = X self.y = y self.k = np.maximum(0.0, check_arrays(k, flatten=True)) self.penalty_start = max(0, int(penalty_start)) self.mean = bool(mean) self.reset()
def l1_max_linear_loss(X, y, mean=True): """Estimate class weights for unbalanced datasets. Parameters ---------- X : array-like, shape (n_samples, n_features) Array of input data; y : array-like, shape (n_samples, 1) Target values mean : Boolean. Whether to compute the squared loss or the mean squared loss. Default is True, the mean squared loss. Returns ------- l1_max : scalar Maximum l1 pentlty to avoid null solution Example ------- """ X, y = check_arrays(X, y) n = float(X.shape[0]) scale = 1.0 / n if mean else 1. l1_max = scale * np.abs(np.dot(X.T, y)).max() return 0.95 * l1_max
def transform(X, V): """ Project a (new) dataset onto the components. Return the projected data and the associated d. We have to recompute U and d because the argument may not have the same number of lines. The argument must have the same number of columns than the datset used to fit the estimator. """ Xk = check_arrays(X) Xk = Xk.copy() n, p = Xk.shape if p != V.shape[0]: raise ValueError("The argument must have the same number of " "columns than the datset used to fit the " "estimator.") U = np.zeros((n, V.shape[1])) d = np.zeros((V.shape[1], )) for k in range(V.shape[1]): # Project on component j vk = V[:, k].reshape(-1, 1) uk = np.dot(X, vk) uk /= np.linalg.norm(uk) U[:, k] = uk[:, 0] dk = compute_d(X, uk, vk) d[k] = dk # Residualize Xk -= dk * np.dot(uk, vk.T) return U, d
def __init__(self, X, y, output_nodes, loss, step_size=step_sizes.ConstantStepSize(0.01), weights=None): X, y = check_arrays(X, y) self.X = X self.y = y self._input = InputLayer(num_nodes=X.shape[1]) self._layers = [] if self._all_nodes_equal(output_nodes, SoftmaxNode) \ and isinstance(loss, CategoricalCrossEntropyLoss): self._output = SoftmaxCategoricalCrossEntropyOutputLayer(loss, num_nodes=y.shape[1], nodes=output_nodes, weights=weights) else: self._output = OutputLayer(loss, num_nodes=y.shape[1], nodes=output_nodes, weights=weights) loss.set_target(y.T) # Note: The network outputs column vectors self._step_size = step_size self.reset()
def l1_max(cls, X): """Return the maximal value for l1 parameter for a given data matrix """ X = check_arrays(X) s = np.sqrt((X**2).sum(axis=0)) #s_old = np.asarray([np.linalg.norm(X[:, i]) for i in range(X.shape[1])]) #assert np.allclose(s, s_old) l1_max = np.max(s) / X.shape[0] return l1_max
def _vector(self, x1, x2): x1, x2 = check_arrays(x1, x2) val = np.dot(x1.T, x2) if isinstance(val, np.ndarray): val = val[0, 0] return val
def wilcoxon_test(x, Y, zero_method="zsplit", correction=False): """Performs the Wilcoxon signed rank test for comparing one classifier to several other classifiers. It tests the null hypothesis that two related paired samples comes from the same distribution. It is a non-parametric version of the paired t-test. Parameters ---------- x : numpy array of shape (n, 1) The measurements for a single classifier. Y : numpy array of shape (n, k) The measurements for k other classifiers. zero_method : string, {"pratt", "wilcox", "zsplit"}, optional How to treat zero-differences in the ranking. Default is "zsplit", splitting the zero-ranks between the positive and negative ranks. See scipy.stats.wilcoxon for more details. correction : bool, optional Whether or not to apply continuity correction by adjusting the rank statistic by 0.5 towards the mean. Default is False. Returns ------- statistics : list of float The sum of the ranks of the differences, for each of the k classifiers. p_values : list of float The two-sided p-values for the tests. """ x, Y = check_arrays(x, Y) if zero_method not in ["pratt", "wilcox", "zsplit"]: raise ValueError('zero_method must be in ["pratt", "wilcox", ' '"zsplit"].') correction = bool(correction) [n, k] = Y.shape statistics = [0] * k p_values = [0] * k for i in range(k): statistics[i], p_values[i] = stat.wilcoxon(x, Y[:, i], zero_method=zero_method, correction=correction) return statistics, p_values
def l1_max_logistic_loss(X, y, mean=True, class_weight=None): """Estimate class weights for unbalanced datasets. Parameters ---------- X : array-like, shape (n_samples, n_features) Array of input data; y : array-like, shape (n_samples, 1) Array of class labels in {0, 1}; mean : Boolean. Whether to compute the squared loss or the mean squared loss. Default is True, the mean squared loss. class_weight : Dict, 'auto' or None. If 'auto', class weights will be given inverse proportional to the frequency of the class in the data. If a dictionary is given, keys are classes and values are corresponding class weights. If None is given, the class weights will be uniform. Returns ------- l1_max : scalar Maximum l1 pentlty to avoid null solution Example ------- """ X, y = check_arrays(X, check_labels(y)) sample_weight = class_weight_to_sample_weight(class_weight, y) y, sample_weight = check_arrays(y, sample_weight) n = float(X.shape[0]) scale = 1.0 / n if mean else 1. l1_max = scale * np.abs(np.dot(X.T, sample_weight * (y - 0.5))).max() return 0.95 * l1_max
def predict(X, V): """ Return the approximated matrix for a given matrix. We have to recompute U and d because the argument may not have the same number of lines. The argument must have the same number of columns than the datset used to fit the estimator. """ Xk = check_arrays(X) n, p = Xk.shape if p != V.shape[0]: raise ValueError("The argument must have the same number of " "columns than the datset used to fit the " "estimator.") Ut, dt = transform(Xk, V) Xt = np.zeros(Xk.shape) for k in range(V.shape[1]): vk = V[:, k].reshape(-1, 1) uk = Ut[:, k].reshape(-1, 1) Xt += compute_rank1_approx(dt[k], uk, vk) return Xt
def predict(self, X, n_components=None): """ Return the approximated matrix for a given matrix. We have to recompute U and d because the argument may not have the same number of lines. The argument must have the same number of columns than the datset used to fit the estimator. """ if n_components is None: n_components = self.n_components Xk = check_arrays(X) n, p = Xk.shape if p != self.V.shape[0]: raise ValueError("The argument must have the same number of " "columns than the datset used to fit the " "estimator.") Ut, dt = self.transform(Xk, n_components=n_components) Xt = np.zeros(Xk.shape) for k in range(n_components): vk = self.V[:, k].reshape(-1, 1) uk = Ut[:, k].reshape(-1, 1) Xt += self.compute_rank1_approx(dt[k], uk, vk) return Xt
def __init__(self, l, A=None, mu=consts.TOLERANCE, penalty_start=0): """ Parameters ---------- l : Non-negative float. The Lagrange multiplier, or regularisation constant, of the function. A : A (usually sparse) array. The linear operator for the Nesterov formulation. May not be None! mu: Non-negative float. The regularisation constant for the smoothing. penalty_start : Non-negative integer. The number of columns, variables etc., to except from penalisation. Equivalently, the first index to be penalised. Default is 0, all columns are included. """ self.l = np.maximum(0.0, check_arrays(l, flatten=True)) if A is None: raise ValueError("The linear operator A must not be None.") self._A = A self.mu = np.asarray(np.maximum(0.0, np.asarray(mu, dtype=float))) self.penalty_start = max(0, int(penalty_start)) self._alpha = None
def run(self, X, y, alpha=None): """Find the best separating margin for the samples in X. Parameters ---------- X : ndarray The matrix with samples to separate. y : array_like The class belongings for the samples in X. Values must be -1 or 1. alpha : numpy array A start vector for the lagrange multipliers. Default is to use a zero vector. """ X, y = check_arrays(X, check_array_in(y, [-1, 1])) if self.info_requested(Info.ok): self.info_set(Info.ok, False) if self.info_requested(Info.time): _t = utils.time() if self.info_requested(Info.func_val): self._f = [] n, p = X.shape # Set up error cache self._E = np.zeros(n) self._Evalid = np.zeros(n, dtype=np.bool) # Bias (intercept/threshold) self.bias = 0.0 # Lagrange multipliers if alpha is None: self.alpha = np.zeros((n, 1)) else: self.alpha = alpha numChanged = 0 examineAll = True while numChanged > 0 or examineAll: numChanged = 0 if examineAll: for i in range(n): numChanged += self._examineSample(i, X, y) else: for i in range(n): if self.alpha[i, 0] > 0.0 and self.alpha[i, 0] < self.C: numChanged += self._examineSample(i, X, y) if examineAll: examineAll = False elif numChanged == 0: examineAll = True if self.info_requested(Info.time): self.info_set(Info.time, utils.time() - _t) if self.info_requested(Info.func_val): self.info_set(Info.func_val, self._f) del self._f # Remove for future runs if self.info_requested(Info.ok): self.info_set(Info.ok, True) return self._compute_w(X, y)
def fit(self, X, in_place=False): """Fit the estimator to the data """ X = check_arrays(X) l1_max = self.l1_max(X) if self.l1 >= l1_max: msg_fmt = "l1 ({l1}) is larger than l1_max ({l1_max})." msg = msg_fmt.format(l1=self.l1, l1_max=l1_max) if self.raise_if_l1_too_large: raise ValueError(msg) else: msg += "Solution will be 0." warnings.warn(msg) if not in_place: X = X.copy() n, p = X.shape self.U = np.empty([n, self.n_components]) self.d = np.empty([self.n_components]) self.V = np.empty([p, self.n_components]) self.info = [] self.crit = [] self.subfunc = [] self.func = [] self.v_func = [] self.converged = [] for j in range(self.n_components): if self.verbose: print('Component', j) _converged = False _stopped = False k = 0 v_new = self.start_vector.get_vector(p) v_new = v_new / np.linalg.norm(v_new) _info = [] _crit = [] _subfunc = [] # Real objective function (only if criteria == "Frobenius") _func = [] # Objective function for fixed u problem _v_func = [] while (not _converged) and (not _stopped): if self.verbose: print('Iteration', k) # Save previous iteration results k = k + 1 v_old = np.copy(v_new) if k >= 2: u_old = np.copy(u_new) # Minimize u for v fixed (explicit formula) u_new = np.dot(X, v_old) u_new_norm = np.linalg.norm(u_new) if u_new_norm != 0: u_new /= u_new_norm # Minimize v for u fixed info_conf = [Info.fvalue, Info.num_iter] function = RightSingularL1L2SmoothedTV_CONESTA(X, u_new, self.l1, self.l2, self.ltv, Atv=self.Atv) algorithm = proximal.CONESTA(eps=self.inner_eps, tau=self.tau, max_iter=self.inner_max_iter, info=info_conf) v_new = algorithm.run(function, v_old) info = algorithm.info_get() _info.append(info) _v_func.append(function.f(v_new)) if self.verbose: #print("Inner FISTA iterations:", algorithm.num_iter) print("Inner calls to FISTA:", info[Info.num_iter]) if self.criterion == "frobenius": # Compute the normalized Frobenius norm of the # approximation error. # X is the deflated matrix so this is the distance # explained by the current component d_tmp = self.compute_d(X, u_new, v_new) # X_approx = d * np.dot(u_new, v_new.T) X_approx = self.compute_rank1_approx(d_tmp, u_new, v_new) f = np.linalg.norm(X - X_approx, ord='fro') / (2 * n) _func.append(f) del X_approx # To stop or not to stop if k >= self.max_iter: _stopped = True if k >= 2: # Evaluate stopping criteria if self.criterion == "uv": old = np.vstack((u_old, v_old)) new = np.vstack((u_new, v_new)) crit = np.linalg.norm(new - old) if self.criterion == "frobenius": crit = np.abs((_func[-1] - _func[-2]) / _func[-1]) if self.criterion == "v": crit = np.abs( (_v_func[-1] - _v_func[-2]) / _v_func[-1]) _crit.append(crit) if crit < self.eps: _converged = True # Store informations for this component self.info.append(_info) self.subfunc.append(_subfunc) self.func.append(_func) self.v_func.append(_v_func) self.crit.append(_crit) self.converged.append(_converged) d_new = self.compute_d(X, u_new, v_new) X = X - self.compute_rank1_approx(d_new, u_new, v_new) self.U[:, j] = u_new[:, 0] self.d[j] = d_new self.V[:, j] = v_new[:, 0] return self
def run(self, g, x): """Find the best separating margin for the samples in X. Parameters ---------- g : MajoriserFunction The function that majorises self.function. x : array_like The point at which to start the minimisation process. """ x = check_arrays(x) if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, False) if self.info_requested(utils.Info.time): _t = [] if self.info_requested(utils.Info.func_val): _f = [] if self.info_requested(utils.Info.converged): self.info_set(utils.Info.converged, False) xnew = xold = x for i in range(1, self.max_iter + 1): if self.info_requested(utils.Info.time): tm = utils.time() xold = xnew if isinstance(g, properties.MajoriserFunction): maj_f = g(self.function, xold) else: g.at_point(xold) maj_f = g xnew = self.algorithm.run(maj_f, xold) if self.info_requested(utils.Info.time): _t.append(utils.time() - tm) if self.info_requested(utils.Info.func_val): _f.append(g.f(xnew)) if self.callback is not None: self.callback(locals()) if i >= self.min_iter and np.linalg.norm(xnew - xold) < self.eps: if self.info_requested(utils.Info.converged): self.info_set(utils.Info.converged, True) break self.num_iter = i if self.info_requested(utils.Info.num_iter): self.info_set(utils.Info.num_iter, self.num_iter) if self.info_requested(utils.Info.time): self.info_set(utils.Info.time, _t) if self.info_requested(utils.Info.func_val): self.info_set(utils.Info.func_val, _f) if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, True) return xnew
def run(self, X, y, alpha=None): """Find the best separating margin for the samples in X. Parameters ---------- X : ndarray The matrix with samples to separate. y : array_like The class belongings for the samples in X. Values must be -1 or 1. alpha : numpy array A start vector for the lagrange multipliers. Default is to use a zero vector. """ X, y = check_arrays(X, check_array_in(y, [-1, 1])) if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, False) if self.info_requested(utils.Info.time): _t = time() if self.info_requested(utils.Info.func_val): self._f = [] n, p = X.shape # Set up error cache self._E = np.zeros(n) self._Evalid = np.zeros(n, dtype=np.bool) # Bias (intercept/threshold) self.bias = 0.0 # Lagrange multipliers if alpha is None: self.alpha = np.zeros((n, 1)) else: self.alpha = alpha numChanged = 0 examineAll = True while numChanged > 0 or examineAll: numChanged = 0 if examineAll: for i in xrange(n): numChanged += self._examineSample(i, X, y) else: for i in xrange(n): if self.alpha[i, 0] > 0.0 and self.alpha[i, 0] < self.C: numChanged += self._examineSample(i, X, y) if examineAll: examineAll = False elif numChanged == 0: examineAll = True if self.info_requested(utils.Info.time): self.info_set(utils.Info.time, time() - _t) if self.info_requested(utils.Info.func_val): self.info_set(utils.Info.func_val, self._f) del self._f # Remove for future runs if self.info_requested(utils.Info.ok): self.info_set(utils.Info.ok, True) return self._compute_w(X, y)