def _forward(self, x): """Computes the distance of each pattern in x to the hyperplane. Parameters ---------- x : CArray Array with new patterns to classify, 2-Dimensional of shape (n_patterns, n_features). Returns ------- score : CArray Value of the decision function for each test pattern. Dense flat array of shape (n_samples,) if y is not None, otherwise a (n_samples, n_classes) array. """ # Computing: `x * w^T` score = CArray(x.dot(self.w.T)).todense().ravel() + self.b scores = CArray.ones(shape=(x.shape[0], self.n_classes)) scores[:, 0] = -score.ravel().T scores[:, 1] = score.ravel().T return scores
def test_ones(self): """Test for CArray.ones() classmethod.""" self.logger.info("Test for CArray.ones() classmethod.") for shape in [1, (1, ), 2, (2, ), (1, 2), (2, 1), (2, 2)]: for dtype in [None, float, int, bool]: for sparse in [False, True]: res = CArray.ones(shape=shape, dtype=dtype, sparse=sparse) self.logger.info( "CArray.ones(shape={:}, dtype={:}, sparse={:}):" "\n{:}".format(shape, dtype, sparse, res)) self.assertIsInstance(res, CArray) self.assertEqual(res.isdense, not sparse) self.assertEqual(res.issparse, sparse) if isinstance(shape, tuple): if len(shape) == 1 and sparse is True: # Sparse "vectors" have len(shape) == 2 self.assertEqual(res.shape, (1, shape[0])) else: self.assertEqual(res.shape, shape) else: if sparse is True: self.assertEqual(res.shape, (1, shape)) else: self.assertEqual(res.shape, (shape, )) if dtype is None: # Default dtype is float self.assertIsSubDtype(res.dtype, float) else: self.assertIsSubDtype(res.dtype, dtype) self.assertTrue((res == 1).all())
def _forward(self, x): """"Private method that computes the decision function. Parameters ---------- x : CArray Array with new patterns to classify, 2-Dimensional of shape (n_patterns, n_features). Returns ------- scores : CArray Array of shape (n_patterns, n_classes) with classification score of each test pattern with respect to each training class. Will be returned only if `return_decision_function` is True. """ caching = self._cached_x is not None layer_clfs_scores = self._get_layer_clfs_scores(x) scores = self._clf.forward(layer_clfs_scores, caching=caching) # augment score matrix with reject class scores rej_scores = CArray.ones(x.shape[0]) * self.threshold scores = scores.append(rej_scores.T, axis=1) return scores
def grad_f_params(self, x, y=1): """Derivative of the decision function w.r.t. the classifier parameters. Parameters ---------- x : CArray Features of the dataset on which the training objective is computed. y : int Index of the class wrt the gradient must be computed. """ xs, sv_idx = self.sv_margin() # these points are already normalized if xs is None: self.logger.debug("Warning: sv_margin is empty " "(all points are error vectors).") return None xk = x if self.preprocess is None else self.preprocess.transform(x) s = xs.shape[0] # margin support vector k = xk.shape[0] Ksk_ext = CArray.ones(shape=(s + 1, k)) Ksk_ext[:s, :] = self.kernel.k(xs, xk) return convert_binary_labels(y) * Ksk_ext # (s + 1) * k
def _forward(self, x): """Compute decision function for SVMs, proportional to the distance of x to the separating hyperplane. For non linear SVM, the kernel between input patterns and Support Vectors is computed and then the inner product of the resulting array with the alphas is calculated. Parameters ---------- x : CArray Array with new patterns to classify, 2-Dimensional of shape (n_patterns, n_features). Returns ------- score : CArray Value of the decision function for each test pattern. Dense flat array of shape (n_samples,) if `y` is not None, otherwise a (n_samples, n_classes) array. """ if self.is_kernel_linear(): # Scores are given by the linear model return CClassifierLinear._forward(self, x) k = CArray(self.kernel.k(x, self.sv)).dot(self.alpha.T) score = CArray(k).todense().ravel() + self.b scores = CArray.ones(shape=(x.shape[0], self.n_classes)) scores[:, 0] = -score.ravel().T scores[:, 1] = score.ravel().T return scores
def _forward(self, x): """Computes the decision function for each pattern in x. For One-Vs-All (OVA) multiclass scheme, this is the output of the `label`^th classifier. Parameters ---------- x : CArray Array with new patterns to classify, 2-Dimensional of shape (n_patterns, n_features). y : int or None, optional The label of the class wrt the function should be calculated. If None, return the output for all classes. Returns ------- score : CArray Value of the decision function for each test pattern. Dense flat array of shape (n_samples,) if y is not None, otherwise a (n_samples, n_classes) array. """ # Getting predicted scores for classifier associated with y scores = CArray.ones(shape=(x.shape[0], self.n_classes)) for i in range(self.n_classes): # TODO parfor scores[:, i] = self._binary_classifiers[i].forward(x)[:, 1] return scores
def grad_f_params(self, x, y=1): """Derivative of the decision function w.r.t. alpha and b Parameters ---------- x : CArray Samples on which the training objective is computed. y : int Index of the class wrt the gradient must be computed. """ xs, _ = self._sv_margin() # these points are already preprocessed if xs is None: self.logger.debug("Warning: sv_margin is empty " "(all points are error vectors).") return None s = xs.shape[0] # margin support vector k = x.shape[0] Ksk_ext = CArray.ones(shape=(s + 1, k)) sv = self.kernel.rv # store and recover current sv set self.kernel.rv = xs Ksk_ext[:s, :] = self.kernel.forward(x).T # x and xs are preprocessed self.kernel.rv = sv return convert_binary_labels(y) * Ksk_ext # (s + 1) * k
def _forward(self, x): """Compute decision function for SVMs, proportional to the distance of x to the separating hyperplane. For non linear SVM, the kernel between input patterns and Support Vectors is computed and then the inner product of the resulting array with the alphas is calculated. Parameters ---------- x : CArray Array with new patterns to classify, 2-Dimensional of shape (n_patterns, n_features) or (n_patterns, n_sv) if kernel is used. Returns ------- score : CArray Value of the decision function for each test pattern. Dense flat array of shape (n_samples,) if `y` is not None, otherwise a (n_samples, n_classes) array. """ v = self.w if self.kernel is None else self.alpha score = CArray(x.dot(v.T)).todense() + self.b if self.n_classes > 2: # return current score matrix scores = score else: # concatenate scores scores = CArray.ones(shape=(x.shape[0], self.n_classes)) scores[:, 0] = -score.ravel().T scores[:, 1] = score.ravel().T return scores
def _compute_w_and_b(self): # Updating linear normalizer parameters self._w = CArray.ones(self._mean.size) # TODO: this can be scalar! self._b = -self._mean # Makes sure that whenever scale is zero, we handle it correctly scale = self.std.deepcopy() scale[scale == 0.0] = 1.0 # Updating linear normalizer parameters self._w /= scale self._b /= scale
def hessian_tr_params(self, x=None, y=None): """ Hessian of the training objective w.r.t. the classifier parameters. """ xs, _ = self._sv_margin() # these points are already normalized s = xs.shape[0] H = CArray.ones(shape=(s + 1, s + 1)) H[:s, :s] = self._kernel_function(xs) H[-1, -1] = 0 return H
def _expand_mean(self, n_feats): """Expand mean value to all dimensions.""" n_channels = len(self._in_mean) if not n_feats % n_channels == 0: raise ValueError("input number of features must be " "divisible by {:}".format(n_channels)) channel_size = int(n_feats / n_channels) self._mean = CArray.ones(shape=(n_feats, )) for i in range(n_channels): self._mean[i * channel_size:i * channel_size + channel_size] *= self._in_mean[i] return self._mean
def _backward(self, w): """Implement gradient of decision function wrt x.""" if w is None: w = CArray.ones(shape=(self.n_classes,)) # this is where we'll accumulate grads grad = CArray.zeros( shape=self._cached_x.shape, sparse=self._cached_x.issparse) # loop only over non-zero elements in w, to save computations for c in w.nnz_indices[1]: grad_c = self._binary_classifiers[c].grad_f_x(self._cached_x, y=1) grad += w[c] * grad_c return grad
def _grad_f_b(x, d_l=None): """Derivative of the classifier decision function w.r.t. the bias. Parameters ---------- d_l : ?? """ # where x is normalized if the classifier has a normalizer x = x.atleast_2d() k = x.shape[0] # number of samples d = CArray.ones((1, k)) if d_l is not None: d *= d_l return d
def _expand_std(self, n_feats): """Expand std value to all dimensions.""" if self.with_std is False: # set std to 1. self._std = CArray(1.0) # we just need a scalar value. else: n_channels = len(self._in_std) if not n_feats % n_channels == 0: raise ValueError("input number of features must be " "divisible by {:}".format(n_channels)) channel_size = int(n_feats / n_channels) self._std = CArray.ones(shape=(n_feats, )) for i in range(n_channels): self._std[i * channel_size:i * channel_size + channel_size] *= self._in_std[i] return self._std
def _fit(self, x, y=None): """Compute the minimum and maximum to be used for scaling. Parameters ---------- x : CArray Array to be used as training set. Each row must correspond to one single patterns, so each column is a different feature. y : CArray or None, optional Flat array with the label of each pattern. Can be None if not required by the preprocessing algorithm. Returns ------- CNormalizerMinMax Instance of the trained normalizer. Examples -------- >>> from secml.array import CArray >>> from secml.ml.features.normalization import CNormalizerMinMax >>> array = CArray([[1., -1., 2.], [2., 0., 0.], [0., 1., -1.]]) >>> normalizer = CNormalizerMinMax().fit(array) >>> normalizer.feature_range (0.0, 1.0) >>> print(normalizer.min) CArray([ 0. -1. -1.]) >>> print(normalizer.max) CArray([2. 1. 2.]) """ self._min = x.min(axis=0, keepdims=False) self._max = x.max(axis=0, keepdims=False) # Setting the linear normalization properties # y = m * x + q r = CArray(self.max - self.min) self._m = CArray.ones(r.size) self._m[r != 0] = 1.0 / r[r != 0] # avoids division by zero self._q = -self.min * self._m # z = n * y + v -> Y = n * m * x + (n * q + v) return self
def _forward(self, x): """Private method that computes the decision function. Parameters ---------- x : CArray Array with new patterns to classify, 2-Dimensional of shape (n_patterns, n_features). Returns ------- score : CArray Value of the decision function for each test pattern. Dense flat array of shape (n_patterns,). """ rej_scores = CArray.ones(x.shape[0]) * self.threshold scores = self._clf.decision_function(x) # augment score matrix with reject class scores scores = scores.append(rej_scores.T, axis=1) return scores
def _quadratic_fun(d): """Creates a quadratic function in d dimensions.""" def _quadratic_fun_min(A, b): from scipy import linalg min_x_scipy = linalg.solve((2 * A).tondarray(), -b.tondarray(), sym_pos=True) return CArray(min_x_scipy).ravel() A = CArray.eye(d, d) b = CArray.ones((d, 1)) * 2 discr_fun = CFunction.create('quadratic', A, b, c=0) min_x = _quadratic_fun_min(A, b) min_val = discr_fun.fun(min_x) discr_fun.global_min = lambda: min_val discr_fun.global_min_x = lambda: min_x return discr_fun
def _forward(self, x): """Compute scores proportionally to the distance to the hyperplane as w'x + b. Parameters ---------- x : CArray Input samples given as matrix with shape=(n_samples, n_features). Returns ------- score : CArray Value of the decision function for each sample, given as a matrix with shape=(n_samples, n_classes). """ score = CArray(x.dot(self.w.T)).todense().ravel() + self.b scores = CArray.ones(shape=(x.shape[0], 2)) scores[:, 0] = -score.ravel().T scores[:, 1] = score.ravel().T return scores
def global_min_x(ndim=2): """Global minimum point of the function. Global minimum f(x) = 0 @ x = (1, 1, ...., 1). Parameters ---------- ndim : int, optional Number of dimensions to consider, higher or equal to 2. Default 2. Returns ------- CArray The global minimum point of the function. """ if ndim < 2: raise ValueError( "Rosenbrock function available for at least 2 dimensions") return CArray.ones((ndim, ), dtype=float)
def _decision_function(x, y=None): x = x.atleast_2d() try: scores = CArray(self.skclfs[i].decision_function( x.get_data())) probs = False except AttributeError: scores = CArray(self.skclfs[i].predict_proba(x.get_data())) probs = True # two-class classifiers outputting only scores for class 1 if len(scores.shape) == 1: # duplicate column for class 0 outputs = CArray.ones(shape=(x.shape[0], clf.n_classes)) scores = scores.T outputs[:, 1] = scores outputs[:, 0] = -scores if probs is False else 1 - scores scores = outputs scores.atleast_2d() if y is not None: return scores[:, y].ravel() else: return scores
def setUp(self): self.test_funcs = dict() # Instancing the available functions to test optimizer self.test_funcs['3h-camel'] = { 'fun': CFunction.create('3h-camel'), 'x0': CArray([1, 1]), 'grid_limits': [(-2, 2), (-2, 2)], 'vmin': 0, 'vmax': 5 } self.test_funcs['beale'] = { 'fun': CFunction.create('beale'), 'x0': CArray([0, 0]), 'grid_limits': [(-1, 4.5), (-1, 1.5)], 'vmin': 0, 'vmax': 1 } self.test_funcs['mc-cormick'] = { 'fun': CFunction.create('mc-cormick'), 'x0': CArray([0, 1]), 'grid_limits': [(-2, 3), (-3, 1)], 'vmin': -2, 'vmax': 2 } self.test_funcs['rosenbrock'] = { 'fun': CFunction.create('rosenbrock'), 'x0': CArray([-1, -1]), 'grid_limits': [(-2.1, 1.1), (-2.1, 1.1)], 'vmin': 0, 'vmax': 10 } quad = self._quadratic_fun(2) self.test_funcs['quad-2'] = { 'fun': quad, 'x0': CArray([4, -4]), 'grid_limits': [(-5, 5), (-5, 5)], 'vmin': None, 'vmax': None } n = 100 quad = self._quadratic_fun(n) self.test_funcs['quad-100-sparse'] = { 'fun': quad, 'x0': CArray.zeros((n, ), dtype=int).tosparse(dtype=int), } n = 2 poly = self._create_poly(d=n) self.test_funcs['poly-2'] = { 'fun': poly, 'x0': CArray.ones((n, )) * 2, 'vmin': -10, 'vmax': 5, 'grid_limits': [(-1, 1), (-1, 1)] } n = 100 # x0 is a sparse CArray and the solution is a zero vector poly = self._create_poly(d=n) self.test_funcs['poly-100-int'] = { 'fun': poly, 'x0': CArray.ones((n, ), dtype=int) * 2 } n = 100 poly = self._create_poly(d=n) self.test_funcs['poly-100-int-sparse'] = { 'fun': poly, 'x0': CArray.ones((n, ), dtype=int).tosparse(dtype=int) * 2 }
def grad_f_x(p): w = CArray.ones(shape=multiclass.n_classes) / \ (div * multiclass.n_classes) return multiclass.gradient(p, w=w)