def _create_mnist_dataset(digits=[4, 9],
n_tr=100,
n_val=200,
n_ts=200,
seed=4): # 10
loader = CDataLoaderMNIST()
tr = loader.load('training', digits=digits)
ts = loader.load('testing', digits=digits, num_samples=n_ts)
# start train and validation dataset split
splitter = CDataSplitterKFold(num_folds=2, random_state=seed)
splitter.compute_indices(tr)
val_dts_idx = CArray.randsample(CArray.arange(0, tr.num_samples),
n_val,
random_state=seed)
val = tr[val_dts_idx, :]
tr_dts_idx = CArray.randsample(CArray.arange(0, tr.num_samples),
n_tr,
random_state=seed)
tr = tr[tr_dts_idx, :]
tr.X /= 255.0
val.X /= 255.0
ts.X /= 255.0
return tr, val, ts
def plot_confusion_matrix(self, y_true, y_pred,
normalize=False, labels=None,
title=None, cmap='Blues', colorbar=False):
"""Plot a confusion matrix.
y_true : CArray
True labels.
y_pred : CArray
Predicted labels.
normalize : bool, optional
If True, normalize the confusion matrix in 0/1. Default False.
labels : list, optional
List with the label of each class.
title: str, optional
Title of the plot. Default None.
cmap: str or matplotlib.pyplot.cm, optional
Colormap to use for plotting. Default 'Blues'.
colorbar : bool, optional
If True, show the colorbar side of the matrix. Default False.
"""
matrix = CArray(confusion_matrix(
y_true.tondarray(), y_pred.tondarray()))
if normalize: # min-max normalization
matrix_min = matrix.min()
matrix_max = matrix.max()
matrix = (matrix - matrix.min()) / (matrix_max - matrix_min)
ax = self.imshow(matrix, interpolation='nearest', cmap=cmap)
self._sp.set_xticks(CArray.arange(matrix.shape[1]).tondarray())
self._sp.set_yticks(CArray.arange(matrix.shape[0]).tondarray())
if labels is not None:
self._sp.set_xticklabels(labels)
self._sp.set_yticklabels(labels)
# Rotate the tick labels and set their alignment.
import matplotlib.pyplot as plt
plt.setp(self._sp.get_xticklabels(), rotation=45,
ha="right", rotation_mode="anchor")
fmt = '%.2f' if normalize else 'd'
if colorbar is True:
from mpl_toolkits.axes_grid1 import make_axes_locatable
divider = make_axes_locatable(plt.gca())
cax = divider.append_axes("right", size="5%", pad=0.1)
# TODO: set format -> cax.set_yticklabels
self.colorbar(ax, cax=cax)
if title is True:
self.title(title)
thresh = matrix.max() / 2.
for i, j in itertools.product(
range(matrix.shape[0]), range(matrix.shape[1])):
self.text(j, i, format(matrix[i, j].item(), fmt),
horizontalalignment="center",
color="white" if matrix[i, j] > thresh else "black")
def _attack_cleverhans(self):
from cleverhans.attacks import FastGradientMethod
from secml.adv.attacks import CAttackEvasionCleverhans
attack_params = {
'eps': 0.1,
'clip_max': self.ub,
'clip_min': self.lb,
'ord': 1
}
attack = CAttackEvasionCleverhans(classifier=self.classifier,
surrogate_data=self.tr,
y_target=self.y_target,
clvh_attack_class=FastGradientMethod,
**attack_params)
param_name = 'attack_params.eps'
dmax = 2
dmax_step = 0.5
param_values = CArray.arange(start=0,
step=dmax_step,
stop=dmax + dmax_step)
return attack, param_name, param_values
def _attack_pgd_ls(self):
params = {
"classifier": self.classifier,
"double_init_ds": self.tr,
"distance": 'l1',
"lb": self.lb,
"ub": self.ub,
"y_target": self.y_target,
"attack_classes": self.attack_classes,
"solver_params": {
'eta': 0.5,
'eps': 1e-2
}
}
attack = CAttackEvasionPGDLS(**params)
attack.verbose = 1
# sec eval params
param_name = 'dmax'
dmax = 2
dmax_step = 0.5
param_values = CArray.arange(start=0,
step=dmax_step,
stop=dmax + dmax_step)
return attack, param_name, param_values
def label_binarize_onehot(y):
"""Return dataset labels in one-hot encoding.
Parameters
----------
y : CArray
Array with the labels to encode. Only integer labels are supported.
Returns
-------
binary_labels : CArray
A (num_samples, num_classes) array with the labels one-hot encoded.
Examples
--------
>>> a = CArray([1,0,2,1])
>>> print(label_binarize_onehot(a))
CArray([[0 1 0]
[1 0 0]
[0 0 1]
[0 1 0]])
"""
if not np.issubdtype(y.dtype, np.integer):
raise ValueError("only integer labels are supported")
classes = CArray.arange(y.max() + 1)
return CArray(sk_binarizer(y.tondarray(), classes=classes.tondarray()))
def _test_grad_tr_params(self, clf):
"""Compare `grad_tr_params` output with numerical gradient.
Parameters
----------
clf : CClassifier
"""
i = self.ds.X.randsample(
CArray.arange(self.ds.num_samples), 1, random_state=self.seed)
x, y = self.ds.X[i, :], self.ds.Y[i]
self.logger.info("idx {:}: x {:}, y {:}".format(i.item(), x, y))
params = self.clf_grads_class.params(clf)
# Compare the analytical grad with the numerical grad
gradient = clf.grad_tr_params(x, y).ravel()
num_gradient = CFunction(self._grad_tr_fun).approx_fprime(
params, epsilon=1e-6,
x0=x, y0=y, clf_grads=self.clf_grads_class, clf=clf)
error = (gradient - num_gradient).norm()
self.logger.info("Analytical gradient:\n{:}".format(gradient))
self.logger.info("Numerical gradient:\n{:}".format(num_gradient))
self.logger.info("norm(grad - grad_num): {:}".format(error))
self.assertLess(error, 1e-2)
self.assertTrue(gradient.is_vector_like)
self.assertEqual(params.size, gradient.size)
self.assertEqual(params.issparse, gradient.issparse)
self.assertIsSubDtype(gradient.dtype, float)
def compute_indices(self, dataset):
"""Compute training set and test set indices for each fold.
Parameters
----------
dataset : CDataset
Dataset to split.
Returns
-------
tr_idx, ts_idx : CArray
Flat arrays with the tr/ts indices.
"""
min_set_perc = 1 / dataset.num_samples
if (is_float(self.train_size) and self.train_size < min_set_perc) or \
(is_int(self.train_size) and self.train_size < 1):
raise ValueError(
"train_size should be at least 1 or {:}".format(min_set_perc))
if (is_float(self.test_size) and self.test_size < min_set_perc) or \
(is_int(self.test_size) and self.test_size < 1):
raise ValueError(
"test_size should be at least 1 or {:}".format(min_set_perc))
tr_idx, ts_idx = train_test_split(CArray.arange(
dataset.num_samples).tondarray(),
train_size=self.train_size,
test_size=self.test_size,
random_state=self.random_state,
shuffle=self.shuffle)
self._tr_idx = CArray(tr_idx)
self._ts_idx = CArray(ts_idx)
return self.tr_idx, self.ts_idx
def plot_loss_after_attack(evasAttack):
"""
This function plots the evolution of the loss function of the surrogate classifier
after an attack is performed.
The loss function is normalized between 0 and 1.
It helps to know whether parameters given to the attack algorithm are well tuned are not;
the loss should be as minimal as possible.
The script is inspired from https://secml.gitlab.io/tutorials/11-ImageNet_advanced.html#Visualize-and-check-the-attack-optimization
"""
n_iter = evasAttack.x_seq.shape[0]
itrs = CArray.arange(n_iter)
# create a plot that shows the loss during the attack iterations
# note that the loss is not available for all attacks
fig = CFigure(width=10, height=4, fontsize=14)
# apply a linear scaling to have the loss in [0,1]
loss = evasAttack.f_seq
if loss is not None:
loss = CNormalizerMinMax().fit_transform(CArray(loss).T).ravel()
fig.subplot(1, 2, 1)
fig.sp.xlabel('iteration')
fig.sp.ylabel('loss')
fig.sp.plot(itrs, loss, c='black')
fig.tight_layout()
fig.show()
def _load_mnist49(self, sparse=False, seed=None):
"""Load MNIST49 dataset.
- load dataset
- normalize in 0-1
- split in training (500), validation (100), test (100)
Parameters
----------
sparse : bool, optional (default False)
seed : int or None, optional (default None)
"""
loader = CDataLoaderMNIST()
n_tr = 500
n_val = 100
n_ts = 100
self._digits = [4, 9]
self._tr = loader.load('training',
digits=self._digits,
num_samples=n_tr + n_val)
self._ts = loader.load('testing',
digits=self._digits,
num_samples=n_ts)
if sparse is True:
self._tr = self._tr.tosparse()
self._ts = self._ts.tosparse()
# normalize in [lb,ub]
self._tr.X /= 255.0
self._ts.X /= 255.0
idx = CArray.arange(0, self._tr.num_samples)
val_dts_idx = CArray.randsample(idx, n_val, random_state=seed)
self._val_dts = self._tr[val_dts_idx, :]
tr_dts_idx = CArray.randsample(idx, n_tr, random_state=seed)
self._tr = self._tr[tr_dts_idx, :]
idx = CArray.arange(0, self._ts.num_samples)
ts_dts_idx = CArray.randsample(idx, n_ts, random_state=seed)
self._ts = self._ts[ts_dts_idx, :]
def test_quiver(self):
"""Test for `CPlot.quiver()` method."""
# gradient values creation
xv = CArray.arange(0, 2 * constants.pi, .2)
yv = CArray.arange(0, 2 * constants.pi, .2)
X, Y = CArray.meshgrid((xv, yv))
U = CArray.cos(X)
V = CArray.sin(Y)
plot = CFigure()
plot.sp.title('Gradient arrow')
plot.sp.quiver(U, V)
plot.show()
def load_model(self, filename, classes=None):
"""
Restores the model and optimizer's parameters.
Notes: the model class and optimizer should be
defined before loading the params.
Parameters
----------
filename : str
path where to find the stored model
classes : list, tuple or None, optional
This parameter is used only if the model was stored
with native PyTorch.
Class labels (sorted) for matching classes to indexes
in the loaded model. If classes is None, the classes
will be assigned new indexes from 0 to n_classes.
"""
state = torch.load(filename, map_location=self._device)
keys = ['model_state', 'n_features', 'classes']
if all(key in state for key in keys):
if classes is not None:
self.logger.warning(
"Model was saved within `secml` framework. "
"The parameter `classes` will be ignored.")
# model was stored with save_model method
self._model.load_state_dict(state['model_state'])
if 'optimizer_state' in state \
and self._optimizer is not None:
self._optimizer.load_state_dict(state['optimizer_state'])
else:
self._optimizer = None
if 'optimizer_scheduler_state' in state \
and self._optimizer_scheduler is not None:
self._optimizer_scheduler.load_state_dict(
state['optimizer_scheduler_state'])
else:
self._optimizer_scheduler = None
self._n_features = state['n_features']
self._classes = state['classes']
else: # model was stored outside secml framework
try:
self._model.load_state_dict(state)
# This part is important to prevent not fitted
if classes is None:
self._classes = CArray.arange(
self.layer_shapes[self.layer_names[-1]][1])
else:
self._classes = CArray(classes)
self._n_features = reduce(lambda x, y: x * y, self.input_shape)
self._trained = True
except Exception:
self.logger.error(
"Model's state dict should be stored according to "
"PyTorch docs. Use `torch.save(model.state_dict())`.")
def plot_decision_regions(self,
clf,
plot_background=True,
levels=None,
grid_limits=None,
n_grid_points=30,
cmap=None):
"""Plot decision boundaries and regions for the given classifier.
Parameters
----------
clf : CClassifier
Classifier which decision function should be plotted.
plot_background : bool, optional
Specifies whether to color the decision regions. Default True.
in the background using a colorbar.
levels : list or None, optional
List of levels to be plotted.
If None, CArray.arange(0.5, clf.n_classes) will be plotted.
grid_limits : list of tuple
List with a tuple of min/max limits for each axis.
If None, [(0, 1), (0, 1)] limits will be used.
n_grid_points : int, optional
Number of grid points. Default 30.
cmap : str or list or `matplotlib.pyplot.cm` or None, optional
Colormap to use. Could be a list of colors. If None and the
number of dataset classes is `<= 6`, colors will be chosen
from ['blue', 'red', 'lightgreen', 'black', 'gray', 'cyan'].
Otherwise the 'jet' colormap will be used.
"""
if not isinstance(clf, CClassifier):
raise TypeError("'clf' must be an instance of `CClassifier`.")
if cmap is None:
if clf.n_classes <= 6:
colors = ['blue', 'red', 'lightgreen', 'black', 'gray', 'cyan']
cmap = colors[:clf.n_classes]
else:
cmap = 'jet'
if levels is None:
levels = CArray.arange(0.5, clf.n_classes).tolist()
self.plot_fun(func=clf.predict,
multipoint=True,
colorbar=False,
n_colors=clf.n_classes,
cmap=cmap,
levels=levels,
plot_background=plot_background,
grid_limits=grid_limits,
n_grid_points=n_grid_points,
alpha=0.2)
self.apply_params_clf()
def _save_fig(self):
"""Visualizing the function being optimized with line search."""
x_range = CArray.arange(-5, 20, 0.5, )
score_range = x_range.T.apply_along_axis(self.fun.fun, axis=1)
ref_line = CArray.zeros(x_range.size)
fig = CFigure(height=6, width=12)
fig.sp.plot(x_range, score_range, color='b')
fig.sp.plot(x_range, ref_line, color='k')
filename = fm.join(fm.abspath(__file__), 'test_line_search_bisect.pdf')
fig.savefig(filename)
def _load_mnist():
"""Load MNIST 4971 dataset."""
digits = [4, 9, 7, 1]
digits_str = "".join(['{:}-'.format(i) for i in digits[:-1]])
digits_str += '{:}'.format(digits[-1])
# FIXME: REMOVE THIS AFTER CDATALOADERS AUTOMATICALLY STORE DS
tr_file = fm.join(fm.abspath(__file__),
'mnist_tr_{:}.gz'.format(digits_str))
if not fm.file_exist(tr_file):
loader = CDataLoaderMNIST()
tr = loader.load('training', digits=digits)
pickle_utils.save(tr_file, tr)
else:
tr = pickle_utils.load(tr_file, encoding='latin1')
ts_file = fm.join(fm.abspath(__file__),
'mnist_ts_{:}.gz'.format(digits_str))
if not fm.file_exist(ts_file):
loader = CDataLoaderMNIST()
ts = loader.load('testing', digits=digits)
pickle_utils.save(ts_file, ts)
else:
ts = pickle_utils.load(ts_file, encoding='latin1')
idx = CArray.arange(tr.num_samples)
val_dts_idx = CArray.randsample(idx, 200, random_state=0)
val_dts = tr[val_dts_idx, :]
tr_dts_idx = CArray.randsample(idx, 200, random_state=0)
tr = tr[tr_dts_idx, :]
idx = CArray.arange(0, ts.num_samples)
ts_dts_idx = CArray.randsample(idx, 200, random_state=0)
ts = ts[ts_dts_idx, :]
tr.X /= 255.0
ts.X /= 255.0
return tr, val_dts, ts, digits, tr.header.img_w, tr.header.img_h
def _objective_function_pred_scores(self, y_pred, scores):
"""
Given the predicted labels and the scores, compute the objective
function. (This function allows to use already computed prediction
labels and scores)
"""
n_samples = y_pred.size
k, c = self._find_k_c(y_pred, scores)
smpls_idx = CArray.arange(n_samples).tolist()
f_k = scores[[smpls_idx, k.tolist()]]
f_obj = f_k - scores[[smpls_idx, c.tolist()]]
return f_obj if self.y_target is None else -f_obj
def _find_k_c(self, y_pred, scores):
"""Find the class of which we aim to maximize and the one of which we
aim to minimize the score.
This function works on the prediction and score of either, a single
or multiple samples.
"""
scores = scores.deepcopy()
n_samples = y_pred.size
k = CArray.zeros(shape=(n_samples, ), dtype=int)
if self.y_target is None: # indiscriminate attack
# if the sample is not rejected k is the true class
k[:] = self._y0
# c is neither k nor the reject class
smpls_idx = CArray.arange(n_samples).tolist()
# set to nan the score of the true classes to exclude it by
# the successive choice of the competing classes
scores[[smpls_idx, k.tolist()]] = nan
if issubclass(self._solver_clf.__class__, CClassifierReject):
# set to nan the score of the reject classes to exclude it by
# the successive choice of the competing classes
scores[:, -1] = nan
# for the rejected samples k is the reject class
k[y_pred == -1] = -1
else: # targeted attack
# c is not the target class
scores[:, self.y_target] = nan
# k is the target class
k[:] = self.y_target
c = scores.nanargmax(axis=1).ravel()
if issubclass(self._solver_clf.__class__, CClassifierReject):
c[c == self.surrogate_data.num_classes] = -1
return k, c
def add_discrete_perturbation(self, xc):
# fixme: this should be a solver param
eta = self.eta
# for each poisoning point
for p_idx in range(xc.shape[0]):
c_xc = xc[p_idx, :]
# for each starting poisoning point
# add a perturbation large eta to a single feature of xc if the
# perturbation if possible (if at least one feature perturbed
# does not violate the constraints)
orig_xc = c_xc.deepcopy()
shuf_feat_ids = CArray.arange(c_xc.size)
shuf_feat_ids.shuffle()
for idx in shuf_feat_ids:
# update a randomly chosen feature of xc if does not
# violates any constraint
c_xc[idx] += eta
self._x0 = c_xc
bounds, constr = self._constraint_creation()
if bounds.is_violated(c_xc) or \
bounds.is_violated(c_xc):
c_xc = orig_xc.deepcopy()
c_xc[idx] -= eta
# update a randomly chosen feature of xc if does not
# violates any constraint
self._x0 = c_xc
bounds, constr = self._constraint_creation()
if bounds.is_violated(c_xc) or \
bounds.is_violated(c_xc):
c_xc = orig_xc.deepcopy()
else:
xc[p_idx, :] = c_xc
break
else:
xc[p_idx, :] = c_xc
break
return xc
def test_draw(self):
"""Drawing the loss functions.
Inspired by: https://en.wikipedia.org/wiki/Loss_functions_for_classification
"""
fig = CFigure()
x = CArray.arange(-1, 3.01, 0.01)
for loss_id in ('e-insensitive', 'e-insensitive-squared', 'quadratic'):
self.logger.info("Creating loss: {:}".format(loss_id))
loss_class = CLoss.create(loss_id)
fig.sp.plot(x, loss_class.loss(CArray([1]), x), label=loss_id)
fig.sp.grid()
fig.sp.legend()
fig.show()
def _set_and_run(self, attack, param_name, dmax=2, dmax_step=0.5):
"""Create the SecEval and run it on test set."""
param_values = CArray.arange(
start=0, step=dmax_step,
stop=dmax + dmax_step)
sec_eval = CSecEval(
attack=attack,
param_name=param_name,
param_values=param_values,
)
sec_eval.run_sec_eval(self.ts)
self._plot_sec_eval(sec_eval)
# At the end of the seceval we expect 0% accuracy
self.assertFalse(
CArray(sec_eval.sec_eval_data.Y_pred[-1] == self.ts.Y).any())
def _get_tr_without_point(self, p_idx):
"""
Given the idx of a point return a copy of the training dataset
without that point
Parameters
----------
p_idx int
idx of the point that is wanted to be excluded by the training
dataset
Returns
-------
new_tr CDataset
dataset without the point with the given index
"""
all_idx = CArray.arange(self._tr.num_samples)
not_p_idx = all_idx.find(all_idx != p_idx)
new_tr = self._tr[not_p_idx, :]
return new_tr
def test_draw(self):
"""Drawing the loss functions.
Inspired by: https://en.wikipedia.org/wiki/Loss_functions_for_classification
"""
fig = CFigure()
x = CArray.arange(-1, 3.01, 0.01)
fig.sp.plot(x,
CArray([1 if i <= 0 else 0 for i in x]),
label='0-1 indicator')
for loss_id in ('hinge', 'hinge-squared', 'square', 'log'):
self.logger.info("Creating loss: {:}".format(loss_id))
loss_class = CLoss.create(loss_id)
fig.sp.plot(x, loss_class.loss(CArray([1]), x), label=loss_id)
fig.sp.grid()
fig.sp.legend()
fig.show()
def test_timed_logging(self):
from secml.array import CArray
timer = self.logger.timer() # Does nothing... Use as context manager!
# Test for predefined interval
with timer as t:
time.sleep(2)
self.assertGreaterEqual(t.step, 2000)
self.assertGreaterEqual(t.interval, 2000)
# Testing logging of method run time
with self.logger.timer():
a = CArray.arange(-5, 100, 0.1).transpose()
a.sort(inplace=True)
# Test for predefined interval with error
with self.assertRaises(TypeError):
with self.logger.timer() as t:
time.sleep('test')
self.logger.info("Interval " + str(t.interval) +
" should have been logged anyway")
from secml.array import CArray
from secml.figure import CFigure
fig = CFigure(fontsize=14)
fig.title('loglog base 4 on x')
t = CArray.arange(0.01, 20.0, 0.01)
fig.sp.loglog(t, 20 * (-t / 10.0).exp(), basex=2)
fig.sp.grid()
fig.show()
def setUp(self):
classifier = CClassifierSVM(
kernel='linear', C=1.0, grad_sampling=1.0)
# data parameters
discrete = False
lb = -2
ub = +2
n_tr = 20
n_ts = 10
n_features = 2
n_reps = 1
self.sec_eval = []
self.attack_ds = []
for rep_i in range(n_reps):
self.logger.info(
"Loading `random_blobs` with seed: {:}".format(rep_i))
loader = CDLRandomBlobs(
n_samples=n_tr + n_ts,
n_features=n_features,
centers=[(-0.5, -0.5), (+0.5, +0.5)],
center_box=(-0.5, 0.5),
cluster_std=0.5,
random_state=rep_i * 100 + 10)
ds = loader.load()
tr = ds[:n_tr, :]
ts = ds[n_tr:, :]
classifier.fit(tr)
self.attack_ds.append(ts)
# only manipulate positive samples, targeting negative ones
self.y_target = None
attack_classes = CArray([1])
params = {
"classifier": classifier,
"surrogate_classifier": classifier,
"surrogate_data": tr,
"distance": 'l1',
"lb": lb,
"ub": ub,
"discrete": discrete,
"y_target": self.y_target,
"attack_classes": attack_classes,
"solver_params": {'eta': 0.5, 'eps': 1e-2}
}
attack = CAttackEvasionPGDLS(**params)
attack.verbose = 1
# sec eval params
param_name = 'dmax'
dmax = 2
dmax_step = 0.5
param_values = CArray.arange(
start=0, step=dmax_step,
stop=dmax + dmax_step)
# set sec eval object
self.sec_eval.append(
CSecEval(
attack=attack,
param_name=param_name,
param_values=param_values,
)
)
from secml.array import CArray
from secml.figure import CFigure
n = 5
fig = CFigure()
x = CArray.arange(100)
y = 3. * CArray.sin(x * 2. * 3.14 / 100.)
for i in range(n):
temp = 510 + i
sp = fig.subplot(n, 1, i)
fig.sp.plot(x, y)
# for add space from the figure's border you must increased default value parameters
fig.subplots_adjust(bottom=0.4, top=0.85, hspace=0.001)
fig.sp.xticklabels(())
fig.sp.yticklabels(())
fig.show()
def _fit(self, x, y):
"""Trains the One-Vs-All SVM classifier.
Parameters
----------
x : CArray
Array to be used for training with shape (n_samples, n_features).
y : CArray
Array of shape (n_samples,) containing the class
labels (2-classes only).
Returns
-------
CClassifierSecSVM
Trained classifier.
"""
if self.n_classes != 2:
raise ValueError(
"Trying to learn an SVM on more/less than two classes.")
y = convert_binary_labels(y)
if self.class_weight == 'balanced':
n_pos = y[y == 1].shape[0]
n_neg = y[y == -1].shape[0]
self.weight = CArray.zeros(2)
self.weight[0] = 1.0 * n_pos / (n_pos + n_neg)
self.weight[1] = 1.0 * n_neg / (n_pos + n_neg)
self._w = CArray.zeros(x.shape[1])
self._b = CArray(0.0)
obj = self.objective(x, y)
obj_new = obj
for i in range(self.max_it):
# pick a random sample subset
idx = CArray.randsample(CArray.arange(x.shape[0], dtype=int),
x.shape[0],
random_state=i)
# compute subgradients
grad_w, grad_b = self.gradient_w_b(x[idx, :], y[idx])
for p in range(0, 71, 10):
step = (self.eta**p) * 2**(-0.01 * i) / (x.shape[0]**0.5)
self._w -= step * grad_w
self._b -= step * grad_b
# Applying UPPER bound
d_ub = self.w[self._idx_ub]
d_ub[d_ub > self._ub] = self._ub
self.w[self._idx_ub] = d_ub
# Applying LOWER bound
d_lb = self.w[self._idx_lb]
d_lb[d_lb < self._lb] = self._lb
self.w[self._idx_lb] = d_lb
obj_new = self.objective(x, y)
if obj_new < obj:
break
if abs(obj_new - obj) < self.eps:
self.logger.info("i {:}: {:}".format(i, obj_new))
# Sparse weights if input is sparse (like in CClassifierSVM)
self._w = self.w.tosparse() if x.issparse else self.w
return
obj = obj_new
if i % 10 == 0:
loss = self.hinge_loss(x, y).sum()
self.logger.info("i {:}: {:.4f}, L {:.4f}".format(
i, obj, loss))
# Sparse weights if input is sparse (like in CClassifierSVM)
self._w = self.w.tosparse() if x.issparse else self.w
def _euclidean_proj_simplex(self, v, s=1):
"""Compute the Euclidean projection on a positive simplex.
Solves the optimisation problem (using the algorithm from [1]):
min_w 0.5 * || w - v ||_2^2 ,
s.t. \\sum_i w_i = s, w_i >= 0
Parameters
----------
v : CArray
1-Dimensional vector
s : int, optional
Radius of the simplex. Default 1.
Returns
-------
w : CArray
Euclidean projection of v on the simplex.
Notes
-----
The complexity of this algorithm is in O(n log(n)) as it involves
sorting v. Better alternatives exist for high-dimensional sparse
vectors (cf. [1]). However, this implementation still easily
scales to millions of dimensions.
References
----------
[1] Efficient Projections onto the l1-Ball for
Learning in High Dimensions
John Duchi, Shai Shalev-Shwartz, Yoram Singer,
and Tushar Chandra.
International Conference on Machine Learning (ICML 2008)
http://www.cs.berkeley.edu/~jduchi/projects/DuchiSiShCh08.pdf
"""
v = CArray(v).ravel()
d = v.size
# check if we are already on the simplex
if v.sum() == s and (v >= 0).sum() == d:
return v # best projection: itself!
# get the array of cumulative sums of a sorted (decreasing) copy of v
u = v.deepcopy()
u.sort(inplace=True)
u = u[::-1]
if u.issparse:
u_nnz = CArray(u.nnz_data).todense()
cssv = u_nnz.cumsum()
else:
cssv = u.cumsum()
# get the number of > 0 components of the optimal solution
# (only considering non-null elements in v
j = CArray.arange(1, cssv.size+1)
if u.issparse:
rho = (j * u_nnz > (cssv - s)).sum() - 1
else:
rho = (j * u > (cssv - s)).sum() - 1
# compute the Lagrange multiplier associated to the simplex constraint
theta = (cssv[rho] - s) / (rho + 1.0)
# compute the projection by thresholding v using theta
w = v
if w.issparse:
p = CArray(w.nnz_data)
p -= theta
w[w.nnz_indices] = p
else:
w -= theta
w[w < 0] = 0
return w
def __init__(self,
model,
loss=None,
optimizer=None,
optimizer_scheduler=None,
pretrained=False,
pretrained_classes=None,
input_shape=None,
random_state=None,
preprocess=None,
softmax_outputs=False,
epochs=10,
batch_size=1,
n_jobs=1):
self._device = self._set_device()
self._random_state = random_state
super(CClassifierPyTorch,
self).__init__(model=model,
preprocess=preprocess,
pretrained=pretrained,
pretrained_classes=pretrained_classes,
input_shape=input_shape,
softmax_outputs=softmax_outputs)
self._init_model()
self._n_jobs = n_jobs
self._batch_size = batch_size
if self._batch_size is None:
self.logger.info(
"No batch size passed. Value will be set to the default "
"value of 1.")
self._batch_size = 1
if self._input_shape is None:
# try to infer from first layer
first_layer = list(self._model._modules.values())[0]
if isinstance(first_layer, torch.nn.Linear):
self._input_shape = (first_layer.in_features, )
else:
raise ValueError(
"Input shape should be specified if the first "
"layer is not a `nn.Linear` module.")
self._loss = loss
self._optimizer = optimizer
self._optimizer_scheduler = optimizer_scheduler
self._epochs = epochs
if self._pretrained is True:
self._trained = True
if self._pretrained_classes is not None:
self._classes = self._pretrained_classes
else:
self._classes = CArray.arange(
list(self._model.modules())[-1].out_features)
self._n_features = reduce(lambda a, b: a * b, self._input_shape)
# hooks for getting intermediate outputs
self._handlers = []
# will store intermediate outputs from the hooks
self._intermediate_outputs = None
self._cached_s = None
self._cached_layer_output = None
from secml.array import CArray
from secml.figure import CFigure
fig = CFigure(fontsize=12)
n = 12
X = CArray.arange(n)
Y1 = (1 - X / float(n)) * (1.0 - 0.5) * CArray.rand((n, )) + 0.5
Y2 = (1 - X / float(n)) * (1.0 - 0.5) * CArray.rand((n, )) + 0.5
fig.sp.xticks([0.025, 0.025, 0.95, 0.95])
fig.sp.bar(X, Y1, facecolor='#9999ff', edgecolor='white')
fig.sp.bar(X, -Y2, facecolor='#ff9999', edgecolor='white')
for x, y in zip(X, Y1):
fig.sp.text(x, y, '%.2f' % y, ha='center', va='bottom')
for x, y in zip(X, Y2):
fig.sp.text(x, -y - 0.02, '%.2f' % y, ha='center', va='top')
fig.sp.xlim(-.5, n - .5)
fig.sp.xticks(())
fig.sp.ylim(-1.25, 1.25)
fig.sp.yticks(())
fig.sp.grid()
fig.show()
