def _forward(self, x):
"""Computes the distance of each pattern in x to the hyperplane.
Parameters
----------
x : CArray
Array with new patterns to classify, 2-Dimensional of shape
(n_patterns, n_features).
Returns
-------
score : CArray
Value of the decision function for each test pattern.
Dense flat array of shape (n_samples,) if y is not None,
otherwise a (n_samples, n_classes) array.
"""
# Computing: `x * w^T`
score = CArray(x.dot(self.w.T)).todense().ravel() + self.b
scores = CArray.ones(shape=(x.shape[0], self.n_classes))
scores[:, 0] = -score.ravel().T
scores[:, 1] = score.ravel().T
return scores
def test_ones(self):
"""Test for CArray.ones() classmethod."""
self.logger.info("Test for CArray.ones() classmethod.")
for shape in [1, (1, ), 2, (2, ), (1, 2), (2, 1), (2, 2)]:
for dtype in [None, float, int, bool]:
for sparse in [False, True]:
res = CArray.ones(shape=shape, dtype=dtype, sparse=sparse)
self.logger.info(
"CArray.ones(shape={:}, dtype={:}, sparse={:}):"
"\n{:}".format(shape, dtype, sparse, res))
self.assertIsInstance(res, CArray)
self.assertEqual(res.isdense, not sparse)
self.assertEqual(res.issparse, sparse)
if isinstance(shape, tuple):
if len(shape) == 1 and sparse is True:
# Sparse "vectors" have len(shape) == 2
self.assertEqual(res.shape, (1, shape[0]))
else:
self.assertEqual(res.shape, shape)
else:
if sparse is True:
self.assertEqual(res.shape, (1, shape))
else:
self.assertEqual(res.shape, (shape, ))
if dtype is None: # Default dtype is float
self.assertIsSubDtype(res.dtype, float)
else:
self.assertIsSubDtype(res.dtype, dtype)
self.assertTrue((res == 1).all())
def _forward(self, x):
""""Private method that computes the decision function.
Parameters
----------
x : CArray
Array with new patterns to classify, 2-Dimensional of shape
(n_patterns, n_features).
Returns
-------
scores : CArray
Array of shape (n_patterns, n_classes) with classification
score of each test pattern with respect to each training class.
Will be returned only if `return_decision_function` is True.
"""
caching = self._cached_x is not None
layer_clfs_scores = self._get_layer_clfs_scores(x)
scores = self._clf.forward(layer_clfs_scores, caching=caching)
# augment score matrix with reject class scores
rej_scores = CArray.ones(x.shape[0]) * self.threshold
scores = scores.append(rej_scores.T, axis=1)
return scores
def grad_f_params(self, x, y=1):
"""Derivative of the decision function w.r.t. the classifier parameters.
Parameters
----------
x : CArray
Features of the dataset on which the training objective is computed.
y : int
Index of the class wrt the gradient must be computed.
"""
xs, sv_idx = self.sv_margin() # these points are already normalized
if xs is None:
self.logger.debug("Warning: sv_margin is empty "
"(all points are error vectors).")
return None
xk = x if self.preprocess is None else self.preprocess.transform(x)
s = xs.shape[0] # margin support vector
k = xk.shape[0]
Ksk_ext = CArray.ones(shape=(s + 1, k))
Ksk_ext[:s, :] = self.kernel.k(xs, xk)
return convert_binary_labels(y) * Ksk_ext # (s + 1) * k
def _forward(self, x):
"""Compute decision function for SVMs, proportional to the distance of
x to the separating hyperplane.
For non linear SVM, the kernel between input patterns and
Support Vectors is computed and then the inner product of
the resulting array with the alphas is calculated.
Parameters
----------
x : CArray
Array with new patterns to classify, 2-Dimensional of shape
(n_patterns, n_features).
Returns
-------
score : CArray
Value of the decision function for each test pattern.
Dense flat array of shape (n_samples,) if `y` is not None,
otherwise a (n_samples, n_classes) array.
"""
if self.is_kernel_linear(): # Scores are given by the linear model
return CClassifierLinear._forward(self, x)
k = CArray(self.kernel.k(x, self.sv)).dot(self.alpha.T)
score = CArray(k).todense().ravel() + self.b
scores = CArray.ones(shape=(x.shape[0], self.n_classes))
scores[:, 0] = -score.ravel().T
scores[:, 1] = score.ravel().T
return scores
def _forward(self, x):
"""Computes the decision function for each pattern in x.
For One-Vs-All (OVA) multiclass scheme,
this is the output of the `label`^th classifier.
Parameters
----------
x : CArray
Array with new patterns to classify, 2-Dimensional of shape
(n_patterns, n_features).
y : int or None, optional
The label of the class wrt the function should be calculated.
If None, return the output for all classes.
Returns
-------
score : CArray
Value of the decision function for each test pattern.
Dense flat array of shape (n_samples,) if y is not None,
otherwise a (n_samples, n_classes) array.
"""
# Getting predicted scores for classifier associated with y
scores = CArray.ones(shape=(x.shape[0], self.n_classes))
for i in range(self.n_classes): # TODO parfor
scores[:, i] = self._binary_classifiers[i].forward(x)[:, 1]
return scores
def grad_f_params(self, x, y=1):
"""Derivative of the decision function w.r.t. alpha and b
Parameters
----------
x : CArray
Samples on which the training objective is computed.
y : int
Index of the class wrt the gradient must be computed.
"""
xs, _ = self._sv_margin() # these points are already preprocessed
if xs is None:
self.logger.debug("Warning: sv_margin is empty "
"(all points are error vectors).")
return None
s = xs.shape[0] # margin support vector
k = x.shape[0]
Ksk_ext = CArray.ones(shape=(s + 1, k))
sv = self.kernel.rv # store and recover current sv set
self.kernel.rv = xs
Ksk_ext[:s, :] = self.kernel.forward(x).T # x and xs are preprocessed
self.kernel.rv = sv
return convert_binary_labels(y) * Ksk_ext # (s + 1) * k
def _forward(self, x):
"""Compute decision function for SVMs, proportional to the distance of
x to the separating hyperplane.
For non linear SVM, the kernel between input patterns and
Support Vectors is computed and then the inner product of
the resulting array with the alphas is calculated.
Parameters
----------
x : CArray
Array with new patterns to classify, 2-Dimensional of shape
(n_patterns, n_features) or (n_patterns, n_sv) if kernel is used.
Returns
-------
score : CArray
Value of the decision function for each test pattern.
Dense flat array of shape (n_samples,) if `y` is not None,
otherwise a (n_samples, n_classes) array.
"""
v = self.w if self.kernel is None else self.alpha
score = CArray(x.dot(v.T)).todense() + self.b
if self.n_classes > 2: # return current score matrix
scores = score
else: # concatenate scores
scores = CArray.ones(shape=(x.shape[0], self.n_classes))
scores[:, 0] = -score.ravel().T
scores[:, 1] = score.ravel().T
return scores
def _compute_w_and_b(self):
# Updating linear normalizer parameters
self._w = CArray.ones(self._mean.size) # TODO: this can be scalar!
self._b = -self._mean
# Makes sure that whenever scale is zero, we handle it correctly
scale = self.std.deepcopy()
scale[scale == 0.0] = 1.0
# Updating linear normalizer parameters
self._w /= scale
self._b /= scale
def hessian_tr_params(self, x=None, y=None):
"""
Hessian of the training objective w.r.t. the classifier parameters.
"""
xs, _ = self._sv_margin() # these points are already normalized
s = xs.shape[0]
H = CArray.ones(shape=(s + 1, s + 1))
H[:s, :s] = self._kernel_function(xs)
H[-1, -1] = 0
return H
def _expand_mean(self, n_feats):
"""Expand mean value to all dimensions."""
n_channels = len(self._in_mean)
if not n_feats % n_channels == 0:
raise ValueError("input number of features must be "
"divisible by {:}".format(n_channels))
channel_size = int(n_feats / n_channels)
self._mean = CArray.ones(shape=(n_feats, ))
for i in range(n_channels):
self._mean[i * channel_size:i * channel_size +
channel_size] *= self._in_mean[i]
return self._mean
def _backward(self, w):
"""Implement gradient of decision function wrt x."""
if w is None:
w = CArray.ones(shape=(self.n_classes,))
# this is where we'll accumulate grads
grad = CArray.zeros(
shape=self._cached_x.shape, sparse=self._cached_x.issparse)
# loop only over non-zero elements in w, to save computations
for c in w.nnz_indices[1]:
grad_c = self._binary_classifiers[c].grad_f_x(self._cached_x, y=1)
grad += w[c] * grad_c
return grad
def _grad_f_b(x, d_l=None):
"""Derivative of the classifier decision function w.r.t. the bias.
Parameters
----------
d_l : ??
"""
# where x is normalized if the classifier has a normalizer
x = x.atleast_2d()
k = x.shape[0] # number of samples
d = CArray.ones((1, k))
if d_l is not None:
d *= d_l
return d
def _expand_std(self, n_feats):
"""Expand std value to all dimensions."""
if self.with_std is False:
# set std to 1.
self._std = CArray(1.0) # we just need a scalar value.
else:
n_channels = len(self._in_std)
if not n_feats % n_channels == 0:
raise ValueError("input number of features must be "
"divisible by {:}".format(n_channels))
channel_size = int(n_feats / n_channels)
self._std = CArray.ones(shape=(n_feats, ))
for i in range(n_channels):
self._std[i * channel_size:i * channel_size +
channel_size] *= self._in_std[i]
return self._std
def _fit(self, x, y=None):
"""Compute the minimum and maximum to be used for scaling.
Parameters
----------
x : CArray
Array to be used as training set. Each row must correspond to
one single patterns, so each column is a different feature.
y : CArray or None, optional
Flat array with the label of each pattern.
Can be None if not required by the preprocessing algorithm.
Returns
-------
CNormalizerMinMax
Instance of the trained normalizer.
Examples
--------
>>> from secml.array import CArray
>>> from secml.ml.features.normalization import CNormalizerMinMax
>>> array = CArray([[1., -1., 2.], [2., 0., 0.], [0., 1., -1.]])
>>> normalizer = CNormalizerMinMax().fit(array)
>>> normalizer.feature_range
(0.0, 1.0)
>>> print(normalizer.min)
CArray([ 0. -1. -1.])
>>> print(normalizer.max)
CArray([2. 1. 2.])
"""
self._min = x.min(axis=0, keepdims=False)
self._max = x.max(axis=0, keepdims=False)
# Setting the linear normalization properties
# y = m * x + q
r = CArray(self.max - self.min)
self._m = CArray.ones(r.size)
self._m[r != 0] = 1.0 / r[r != 0] # avoids division by zero
self._q = -self.min * self._m
# z = n * y + v -> Y = n * m * x + (n * q + v)
return self
def _forward(self, x):
"""Private method that computes the decision function.
Parameters
----------
x : CArray
Array with new patterns to classify, 2-Dimensional of shape
(n_patterns, n_features).
Returns
-------
score : CArray
Value of the decision function for each test pattern.
Dense flat array of shape (n_patterns,).
"""
rej_scores = CArray.ones(x.shape[0]) * self.threshold
scores = self._clf.decision_function(x)
# augment score matrix with reject class scores
scores = scores.append(rej_scores.T, axis=1)
return scores
def _quadratic_fun(d):
"""Creates a quadratic function in d dimensions."""
def _quadratic_fun_min(A, b):
from scipy import linalg
min_x_scipy = linalg.solve((2 * A).tondarray(),
-b.tondarray(),
sym_pos=True)
return CArray(min_x_scipy).ravel()
A = CArray.eye(d, d)
b = CArray.ones((d, 1)) * 2
discr_fun = CFunction.create('quadratic', A, b, c=0)
min_x = _quadratic_fun_min(A, b)
min_val = discr_fun.fun(min_x)
discr_fun.global_min = lambda: min_val
discr_fun.global_min_x = lambda: min_x
return discr_fun
def _forward(self, x):
"""Compute scores proportionally to the distance to the
hyperplane as w'x + b.
Parameters
----------
x : CArray
Input samples given as matrix with shape=(n_samples, n_features).
Returns
-------
score : CArray
Value of the decision function for each sample, given as a matrix
with shape=(n_samples, n_classes).
"""
score = CArray(x.dot(self.w.T)).todense().ravel() + self.b
scores = CArray.ones(shape=(x.shape[0], 2))
scores[:, 0] = -score.ravel().T
scores[:, 1] = score.ravel().T
return scores
def global_min_x(ndim=2):
"""Global minimum point of the function.
Global minimum f(x) = 0 @ x = (1, 1, ...., 1).
Parameters
----------
ndim : int, optional
Number of dimensions to consider, higher or equal to 2. Default 2.
Returns
-------
CArray
The global minimum point of the function.
"""
if ndim < 2:
raise ValueError(
"Rosenbrock function available for at least 2 dimensions")
return CArray.ones((ndim, ), dtype=float)
def _decision_function(x, y=None):
x = x.atleast_2d()
try:
scores = CArray(self.skclfs[i].decision_function(
x.get_data()))
probs = False
except AttributeError:
scores = CArray(self.skclfs[i].predict_proba(x.get_data()))
probs = True
# two-class classifiers outputting only scores for class 1
if len(scores.shape) == 1: # duplicate column for class 0
outputs = CArray.ones(shape=(x.shape[0], clf.n_classes))
scores = scores.T
outputs[:, 1] = scores
outputs[:, 0] = -scores if probs is False else 1 - scores
scores = outputs
scores.atleast_2d()
if y is not None:
return scores[:, y].ravel()
else:
return scores
def setUp(self):
self.test_funcs = dict()
# Instancing the available functions to test optimizer
self.test_funcs['3h-camel'] = {
'fun': CFunction.create('3h-camel'),
'x0': CArray([1, 1]),
'grid_limits': [(-2, 2), (-2, 2)],
'vmin': 0,
'vmax': 5
}
self.test_funcs['beale'] = {
'fun': CFunction.create('beale'),
'x0': CArray([0, 0]),
'grid_limits': [(-1, 4.5), (-1, 1.5)],
'vmin': 0,
'vmax': 1
}
self.test_funcs['mc-cormick'] = {
'fun': CFunction.create('mc-cormick'),
'x0': CArray([0, 1]),
'grid_limits': [(-2, 3), (-3, 1)],
'vmin': -2,
'vmax': 2
}
self.test_funcs['rosenbrock'] = {
'fun': CFunction.create('rosenbrock'),
'x0': CArray([-1, -1]),
'grid_limits': [(-2.1, 1.1), (-2.1, 1.1)],
'vmin': 0,
'vmax': 10
}
quad = self._quadratic_fun(2)
self.test_funcs['quad-2'] = {
'fun': quad,
'x0': CArray([4, -4]),
'grid_limits': [(-5, 5), (-5, 5)],
'vmin': None,
'vmax': None
}
n = 100
quad = self._quadratic_fun(n)
self.test_funcs['quad-100-sparse'] = {
'fun': quad,
'x0': CArray.zeros((n, ), dtype=int).tosparse(dtype=int),
}
n = 2
poly = self._create_poly(d=n)
self.test_funcs['poly-2'] = {
'fun': poly,
'x0': CArray.ones((n, )) * 2,
'vmin': -10,
'vmax': 5,
'grid_limits': [(-1, 1), (-1, 1)]
}
n = 100
# x0 is a sparse CArray and the solution is a zero vector
poly = self._create_poly(d=n)
self.test_funcs['poly-100-int'] = {
'fun': poly,
'x0': CArray.ones((n, ), dtype=int) * 2
}
n = 100
poly = self._create_poly(d=n)
self.test_funcs['poly-100-int-sparse'] = {
'fun': poly,
'x0': CArray.ones((n, ), dtype=int).tosparse(dtype=int) * 2
}
def grad_f_x(p):
w = CArray.ones(shape=multiclass.n_classes) / \
(div * multiclass.n_classes)
return multiclass.gradient(p, w=w)
