def _create_mnist_dataset(digits=[4, 9], n_tr=100, n_val=200, n_ts=200, seed=4): # 10 loader = CDataLoaderMNIST() tr = loader.load('training', digits=digits) ts = loader.load('testing', digits=digits, num_samples=n_ts) # start train and validation dataset split splitter = CDataSplitterKFold(num_folds=2, random_state=seed) splitter.compute_indices(tr) val_dts_idx = CArray.randsample(CArray.arange(0, tr.num_samples), n_val, random_state=seed) val = tr[val_dts_idx, :] tr_dts_idx = CArray.randsample(CArray.arange(0, tr.num_samples), n_tr, random_state=seed) tr = tr[tr_dts_idx, :] tr.X /= 255.0 val.X /= 255.0 ts.X /= 255.0 return tr, val, ts
def plot_confusion_matrix(self, y_true, y_pred, normalize=False, labels=None, title=None, cmap='Blues', colorbar=False): """Plot a confusion matrix. y_true : CArray True labels. y_pred : CArray Predicted labels. normalize : bool, optional If True, normalize the confusion matrix in 0/1. Default False. labels : list, optional List with the label of each class. title: str, optional Title of the plot. Default None. cmap: str or matplotlib.pyplot.cm, optional Colormap to use for plotting. Default 'Blues'. colorbar : bool, optional If True, show the colorbar side of the matrix. Default False. """ matrix = CArray(confusion_matrix( y_true.tondarray(), y_pred.tondarray())) if normalize: # min-max normalization matrix_min = matrix.min() matrix_max = matrix.max() matrix = (matrix - matrix.min()) / (matrix_max - matrix_min) ax = self.imshow(matrix, interpolation='nearest', cmap=cmap) self._sp.set_xticks(CArray.arange(matrix.shape[1]).tondarray()) self._sp.set_yticks(CArray.arange(matrix.shape[0]).tondarray()) if labels is not None: self._sp.set_xticklabels(labels) self._sp.set_yticklabels(labels) # Rotate the tick labels and set their alignment. import matplotlib.pyplot as plt plt.setp(self._sp.get_xticklabels(), rotation=45, ha="right", rotation_mode="anchor") fmt = '%.2f' if normalize else 'd' if colorbar is True: from mpl_toolkits.axes_grid1 import make_axes_locatable divider = make_axes_locatable(plt.gca()) cax = divider.append_axes("right", size="5%", pad=0.1) # TODO: set format -> cax.set_yticklabels self.colorbar(ax, cax=cax) if title is True: self.title(title) thresh = matrix.max() / 2. for i, j in itertools.product( range(matrix.shape[0]), range(matrix.shape[1])): self.text(j, i, format(matrix[i, j].item(), fmt), horizontalalignment="center", color="white" if matrix[i, j] > thresh else "black")
def _attack_cleverhans(self): from cleverhans.attacks import FastGradientMethod from secml.adv.attacks import CAttackEvasionCleverhans attack_params = { 'eps': 0.1, 'clip_max': self.ub, 'clip_min': self.lb, 'ord': 1 } attack = CAttackEvasionCleverhans(classifier=self.classifier, surrogate_data=self.tr, y_target=self.y_target, clvh_attack_class=FastGradientMethod, **attack_params) param_name = 'attack_params.eps' dmax = 2 dmax_step = 0.5 param_values = CArray.arange(start=0, step=dmax_step, stop=dmax + dmax_step) return attack, param_name, param_values
def _attack_pgd_ls(self): params = { "classifier": self.classifier, "double_init_ds": self.tr, "distance": 'l1', "lb": self.lb, "ub": self.ub, "y_target": self.y_target, "attack_classes": self.attack_classes, "solver_params": { 'eta': 0.5, 'eps': 1e-2 } } attack = CAttackEvasionPGDLS(**params) attack.verbose = 1 # sec eval params param_name = 'dmax' dmax = 2 dmax_step = 0.5 param_values = CArray.arange(start=0, step=dmax_step, stop=dmax + dmax_step) return attack, param_name, param_values
def label_binarize_onehot(y): """Return dataset labels in one-hot encoding. Parameters ---------- y : CArray Array with the labels to encode. Only integer labels are supported. Returns ------- binary_labels : CArray A (num_samples, num_classes) array with the labels one-hot encoded. Examples -------- >>> a = CArray([1,0,2,1]) >>> print(label_binarize_onehot(a)) CArray([[0 1 0] [1 0 0] [0 0 1] [0 1 0]]) """ if not np.issubdtype(y.dtype, np.integer): raise ValueError("only integer labels are supported") classes = CArray.arange(y.max() + 1) return CArray(sk_binarizer(y.tondarray(), classes=classes.tondarray()))
def _test_grad_tr_params(self, clf): """Compare `grad_tr_params` output with numerical gradient. Parameters ---------- clf : CClassifier """ i = self.ds.X.randsample( CArray.arange(self.ds.num_samples), 1, random_state=self.seed) x, y = self.ds.X[i, :], self.ds.Y[i] self.logger.info("idx {:}: x {:}, y {:}".format(i.item(), x, y)) params = self.clf_grads_class.params(clf) # Compare the analytical grad with the numerical grad gradient = clf.grad_tr_params(x, y).ravel() num_gradient = CFunction(self._grad_tr_fun).approx_fprime( params, epsilon=1e-6, x0=x, y0=y, clf_grads=self.clf_grads_class, clf=clf) error = (gradient - num_gradient).norm() self.logger.info("Analytical gradient:\n{:}".format(gradient)) self.logger.info("Numerical gradient:\n{:}".format(num_gradient)) self.logger.info("norm(grad - grad_num): {:}".format(error)) self.assertLess(error, 1e-2) self.assertTrue(gradient.is_vector_like) self.assertEqual(params.size, gradient.size) self.assertEqual(params.issparse, gradient.issparse) self.assertIsSubDtype(gradient.dtype, float)
def compute_indices(self, dataset): """Compute training set and test set indices for each fold. Parameters ---------- dataset : CDataset Dataset to split. Returns ------- tr_idx, ts_idx : CArray Flat arrays with the tr/ts indices. """ min_set_perc = 1 / dataset.num_samples if (is_float(self.train_size) and self.train_size < min_set_perc) or \ (is_int(self.train_size) and self.train_size < 1): raise ValueError( "train_size should be at least 1 or {:}".format(min_set_perc)) if (is_float(self.test_size) and self.test_size < min_set_perc) or \ (is_int(self.test_size) and self.test_size < 1): raise ValueError( "test_size should be at least 1 or {:}".format(min_set_perc)) tr_idx, ts_idx = train_test_split(CArray.arange( dataset.num_samples).tondarray(), train_size=self.train_size, test_size=self.test_size, random_state=self.random_state, shuffle=self.shuffle) self._tr_idx = CArray(tr_idx) self._ts_idx = CArray(ts_idx) return self.tr_idx, self.ts_idx
def plot_loss_after_attack(evasAttack): """ This function plots the evolution of the loss function of the surrogate classifier after an attack is performed. The loss function is normalized between 0 and 1. It helps to know whether parameters given to the attack algorithm are well tuned are not; the loss should be as minimal as possible. The script is inspired from https://secml.gitlab.io/tutorials/11-ImageNet_advanced.html#Visualize-and-check-the-attack-optimization """ n_iter = evasAttack.x_seq.shape[0] itrs = CArray.arange(n_iter) # create a plot that shows the loss during the attack iterations # note that the loss is not available for all attacks fig = CFigure(width=10, height=4, fontsize=14) # apply a linear scaling to have the loss in [0,1] loss = evasAttack.f_seq if loss is not None: loss = CNormalizerMinMax().fit_transform(CArray(loss).T).ravel() fig.subplot(1, 2, 1) fig.sp.xlabel('iteration') fig.sp.ylabel('loss') fig.sp.plot(itrs, loss, c='black') fig.tight_layout() fig.show()
def _load_mnist49(self, sparse=False, seed=None): """Load MNIST49 dataset. - load dataset - normalize in 0-1 - split in training (500), validation (100), test (100) Parameters ---------- sparse : bool, optional (default False) seed : int or None, optional (default None) """ loader = CDataLoaderMNIST() n_tr = 500 n_val = 100 n_ts = 100 self._digits = [4, 9] self._tr = loader.load('training', digits=self._digits, num_samples=n_tr + n_val) self._ts = loader.load('testing', digits=self._digits, num_samples=n_ts) if sparse is True: self._tr = self._tr.tosparse() self._ts = self._ts.tosparse() # normalize in [lb,ub] self._tr.X /= 255.0 self._ts.X /= 255.0 idx = CArray.arange(0, self._tr.num_samples) val_dts_idx = CArray.randsample(idx, n_val, random_state=seed) self._val_dts = self._tr[val_dts_idx, :] tr_dts_idx = CArray.randsample(idx, n_tr, random_state=seed) self._tr = self._tr[tr_dts_idx, :] idx = CArray.arange(0, self._ts.num_samples) ts_dts_idx = CArray.randsample(idx, n_ts, random_state=seed) self._ts = self._ts[ts_dts_idx, :]
def test_quiver(self): """Test for `CPlot.quiver()` method.""" # gradient values creation xv = CArray.arange(0, 2 * constants.pi, .2) yv = CArray.arange(0, 2 * constants.pi, .2) X, Y = CArray.meshgrid((xv, yv)) U = CArray.cos(X) V = CArray.sin(Y) plot = CFigure() plot.sp.title('Gradient arrow') plot.sp.quiver(U, V) plot.show()
def load_model(self, filename, classes=None): """ Restores the model and optimizer's parameters. Notes: the model class and optimizer should be defined before loading the params. Parameters ---------- filename : str path where to find the stored model classes : list, tuple or None, optional This parameter is used only if the model was stored with native PyTorch. Class labels (sorted) for matching classes to indexes in the loaded model. If classes is None, the classes will be assigned new indexes from 0 to n_classes. """ state = torch.load(filename, map_location=self._device) keys = ['model_state', 'n_features', 'classes'] if all(key in state for key in keys): if classes is not None: self.logger.warning( "Model was saved within `secml` framework. " "The parameter `classes` will be ignored.") # model was stored with save_model method self._model.load_state_dict(state['model_state']) if 'optimizer_state' in state \ and self._optimizer is not None: self._optimizer.load_state_dict(state['optimizer_state']) else: self._optimizer = None if 'optimizer_scheduler_state' in state \ and self._optimizer_scheduler is not None: self._optimizer_scheduler.load_state_dict( state['optimizer_scheduler_state']) else: self._optimizer_scheduler = None self._n_features = state['n_features'] self._classes = state['classes'] else: # model was stored outside secml framework try: self._model.load_state_dict(state) # This part is important to prevent not fitted if classes is None: self._classes = CArray.arange( self.layer_shapes[self.layer_names[-1]][1]) else: self._classes = CArray(classes) self._n_features = reduce(lambda x, y: x * y, self.input_shape) self._trained = True except Exception: self.logger.error( "Model's state dict should be stored according to " "PyTorch docs. Use `torch.save(model.state_dict())`.")
def plot_decision_regions(self, clf, plot_background=True, levels=None, grid_limits=None, n_grid_points=30, cmap=None): """Plot decision boundaries and regions for the given classifier. Parameters ---------- clf : CClassifier Classifier which decision function should be plotted. plot_background : bool, optional Specifies whether to color the decision regions. Default True. in the background using a colorbar. levels : list or None, optional List of levels to be plotted. If None, CArray.arange(0.5, clf.n_classes) will be plotted. grid_limits : list of tuple List with a tuple of min/max limits for each axis. If None, [(0, 1), (0, 1)] limits will be used. n_grid_points : int, optional Number of grid points. Default 30. cmap : str or list or `matplotlib.pyplot.cm` or None, optional Colormap to use. Could be a list of colors. If None and the number of dataset classes is `<= 6`, colors will be chosen from ['blue', 'red', 'lightgreen', 'black', 'gray', 'cyan']. Otherwise the 'jet' colormap will be used. """ if not isinstance(clf, CClassifier): raise TypeError("'clf' must be an instance of `CClassifier`.") if cmap is None: if clf.n_classes <= 6: colors = ['blue', 'red', 'lightgreen', 'black', 'gray', 'cyan'] cmap = colors[:clf.n_classes] else: cmap = 'jet' if levels is None: levels = CArray.arange(0.5, clf.n_classes).tolist() self.plot_fun(func=clf.predict, multipoint=True, colorbar=False, n_colors=clf.n_classes, cmap=cmap, levels=levels, plot_background=plot_background, grid_limits=grid_limits, n_grid_points=n_grid_points, alpha=0.2) self.apply_params_clf()
def _save_fig(self): """Visualizing the function being optimized with line search.""" x_range = CArray.arange(-5, 20, 0.5, ) score_range = x_range.T.apply_along_axis(self.fun.fun, axis=1) ref_line = CArray.zeros(x_range.size) fig = CFigure(height=6, width=12) fig.sp.plot(x_range, score_range, color='b') fig.sp.plot(x_range, ref_line, color='k') filename = fm.join(fm.abspath(__file__), 'test_line_search_bisect.pdf') fig.savefig(filename)
def _load_mnist(): """Load MNIST 4971 dataset.""" digits = [4, 9, 7, 1] digits_str = "".join(['{:}-'.format(i) for i in digits[:-1]]) digits_str += '{:}'.format(digits[-1]) # FIXME: REMOVE THIS AFTER CDATALOADERS AUTOMATICALLY STORE DS tr_file = fm.join(fm.abspath(__file__), 'mnist_tr_{:}.gz'.format(digits_str)) if not fm.file_exist(tr_file): loader = CDataLoaderMNIST() tr = loader.load('training', digits=digits) pickle_utils.save(tr_file, tr) else: tr = pickle_utils.load(tr_file, encoding='latin1') ts_file = fm.join(fm.abspath(__file__), 'mnist_ts_{:}.gz'.format(digits_str)) if not fm.file_exist(ts_file): loader = CDataLoaderMNIST() ts = loader.load('testing', digits=digits) pickle_utils.save(ts_file, ts) else: ts = pickle_utils.load(ts_file, encoding='latin1') idx = CArray.arange(tr.num_samples) val_dts_idx = CArray.randsample(idx, 200, random_state=0) val_dts = tr[val_dts_idx, :] tr_dts_idx = CArray.randsample(idx, 200, random_state=0) tr = tr[tr_dts_idx, :] idx = CArray.arange(0, ts.num_samples) ts_dts_idx = CArray.randsample(idx, 200, random_state=0) ts = ts[ts_dts_idx, :] tr.X /= 255.0 ts.X /= 255.0 return tr, val_dts, ts, digits, tr.header.img_w, tr.header.img_h
def _objective_function_pred_scores(self, y_pred, scores): """ Given the predicted labels and the scores, compute the objective function. (This function allows to use already computed prediction labels and scores) """ n_samples = y_pred.size k, c = self._find_k_c(y_pred, scores) smpls_idx = CArray.arange(n_samples).tolist() f_k = scores[[smpls_idx, k.tolist()]] f_obj = f_k - scores[[smpls_idx, c.tolist()]] return f_obj if self.y_target is None else -f_obj
def _find_k_c(self, y_pred, scores): """Find the class of which we aim to maximize and the one of which we aim to minimize the score. This function works on the prediction and score of either, a single or multiple samples. """ scores = scores.deepcopy() n_samples = y_pred.size k = CArray.zeros(shape=(n_samples, ), dtype=int) if self.y_target is None: # indiscriminate attack # if the sample is not rejected k is the true class k[:] = self._y0 # c is neither k nor the reject class smpls_idx = CArray.arange(n_samples).tolist() # set to nan the score of the true classes to exclude it by # the successive choice of the competing classes scores[[smpls_idx, k.tolist()]] = nan if issubclass(self._solver_clf.__class__, CClassifierReject): # set to nan the score of the reject classes to exclude it by # the successive choice of the competing classes scores[:, -1] = nan # for the rejected samples k is the reject class k[y_pred == -1] = -1 else: # targeted attack # c is not the target class scores[:, self.y_target] = nan # k is the target class k[:] = self.y_target c = scores.nanargmax(axis=1).ravel() if issubclass(self._solver_clf.__class__, CClassifierReject): c[c == self.surrogate_data.num_classes] = -1 return k, c
def add_discrete_perturbation(self, xc): # fixme: this should be a solver param eta = self.eta # for each poisoning point for p_idx in range(xc.shape[0]): c_xc = xc[p_idx, :] # for each starting poisoning point # add a perturbation large eta to a single feature of xc if the # perturbation if possible (if at least one feature perturbed # does not violate the constraints) orig_xc = c_xc.deepcopy() shuf_feat_ids = CArray.arange(c_xc.size) shuf_feat_ids.shuffle() for idx in shuf_feat_ids: # update a randomly chosen feature of xc if does not # violates any constraint c_xc[idx] += eta self._x0 = c_xc bounds, constr = self._constraint_creation() if bounds.is_violated(c_xc) or \ bounds.is_violated(c_xc): c_xc = orig_xc.deepcopy() c_xc[idx] -= eta # update a randomly chosen feature of xc if does not # violates any constraint self._x0 = c_xc bounds, constr = self._constraint_creation() if bounds.is_violated(c_xc) or \ bounds.is_violated(c_xc): c_xc = orig_xc.deepcopy() else: xc[p_idx, :] = c_xc break else: xc[p_idx, :] = c_xc break return xc
def test_draw(self): """Drawing the loss functions. Inspired by: https://en.wikipedia.org/wiki/Loss_functions_for_classification """ fig = CFigure() x = CArray.arange(-1, 3.01, 0.01) for loss_id in ('e-insensitive', 'e-insensitive-squared', 'quadratic'): self.logger.info("Creating loss: {:}".format(loss_id)) loss_class = CLoss.create(loss_id) fig.sp.plot(x, loss_class.loss(CArray([1]), x), label=loss_id) fig.sp.grid() fig.sp.legend() fig.show()
def _set_and_run(self, attack, param_name, dmax=2, dmax_step=0.5): """Create the SecEval and run it on test set.""" param_values = CArray.arange( start=0, step=dmax_step, stop=dmax + dmax_step) sec_eval = CSecEval( attack=attack, param_name=param_name, param_values=param_values, ) sec_eval.run_sec_eval(self.ts) self._plot_sec_eval(sec_eval) # At the end of the seceval we expect 0% accuracy self.assertFalse( CArray(sec_eval.sec_eval_data.Y_pred[-1] == self.ts.Y).any())
def _get_tr_without_point(self, p_idx): """ Given the idx of a point return a copy of the training dataset without that point Parameters ---------- p_idx int idx of the point that is wanted to be excluded by the training dataset Returns ------- new_tr CDataset dataset without the point with the given index """ all_idx = CArray.arange(self._tr.num_samples) not_p_idx = all_idx.find(all_idx != p_idx) new_tr = self._tr[not_p_idx, :] return new_tr
def test_draw(self): """Drawing the loss functions. Inspired by: https://en.wikipedia.org/wiki/Loss_functions_for_classification """ fig = CFigure() x = CArray.arange(-1, 3.01, 0.01) fig.sp.plot(x, CArray([1 if i <= 0 else 0 for i in x]), label='0-1 indicator') for loss_id in ('hinge', 'hinge-squared', 'square', 'log'): self.logger.info("Creating loss: {:}".format(loss_id)) loss_class = CLoss.create(loss_id) fig.sp.plot(x, loss_class.loss(CArray([1]), x), label=loss_id) fig.sp.grid() fig.sp.legend() fig.show()
def test_timed_logging(self): from secml.array import CArray timer = self.logger.timer() # Does nothing... Use as context manager! # Test for predefined interval with timer as t: time.sleep(2) self.assertGreaterEqual(t.step, 2000) self.assertGreaterEqual(t.interval, 2000) # Testing logging of method run time with self.logger.timer(): a = CArray.arange(-5, 100, 0.1).transpose() a.sort(inplace=True) # Test for predefined interval with error with self.assertRaises(TypeError): with self.logger.timer() as t: time.sleep('test') self.logger.info("Interval " + str(t.interval) + " should have been logged anyway")
from secml.array import CArray from secml.figure import CFigure fig = CFigure(fontsize=14) fig.title('loglog base 4 on x') t = CArray.arange(0.01, 20.0, 0.01) fig.sp.loglog(t, 20 * (-t / 10.0).exp(), basex=2) fig.sp.grid() fig.show()
def setUp(self): classifier = CClassifierSVM( kernel='linear', C=1.0, grad_sampling=1.0) # data parameters discrete = False lb = -2 ub = +2 n_tr = 20 n_ts = 10 n_features = 2 n_reps = 1 self.sec_eval = [] self.attack_ds = [] for rep_i in range(n_reps): self.logger.info( "Loading `random_blobs` with seed: {:}".format(rep_i)) loader = CDLRandomBlobs( n_samples=n_tr + n_ts, n_features=n_features, centers=[(-0.5, -0.5), (+0.5, +0.5)], center_box=(-0.5, 0.5), cluster_std=0.5, random_state=rep_i * 100 + 10) ds = loader.load() tr = ds[:n_tr, :] ts = ds[n_tr:, :] classifier.fit(tr) self.attack_ds.append(ts) # only manipulate positive samples, targeting negative ones self.y_target = None attack_classes = CArray([1]) params = { "classifier": classifier, "surrogate_classifier": classifier, "surrogate_data": tr, "distance": 'l1', "lb": lb, "ub": ub, "discrete": discrete, "y_target": self.y_target, "attack_classes": attack_classes, "solver_params": {'eta': 0.5, 'eps': 1e-2} } attack = CAttackEvasionPGDLS(**params) attack.verbose = 1 # sec eval params param_name = 'dmax' dmax = 2 dmax_step = 0.5 param_values = CArray.arange( start=0, step=dmax_step, stop=dmax + dmax_step) # set sec eval object self.sec_eval.append( CSecEval( attack=attack, param_name=param_name, param_values=param_values, ) )
from secml.array import CArray from secml.figure import CFigure n = 5 fig = CFigure() x = CArray.arange(100) y = 3. * CArray.sin(x * 2. * 3.14 / 100.) for i in range(n): temp = 510 + i sp = fig.subplot(n, 1, i) fig.sp.plot(x, y) # for add space from the figure's border you must increased default value parameters fig.subplots_adjust(bottom=0.4, top=0.85, hspace=0.001) fig.sp.xticklabels(()) fig.sp.yticklabels(()) fig.show()
def _fit(self, x, y): """Trains the One-Vs-All SVM classifier. Parameters ---------- x : CArray Array to be used for training with shape (n_samples, n_features). y : CArray Array of shape (n_samples,) containing the class labels (2-classes only). Returns ------- CClassifierSecSVM Trained classifier. """ if self.n_classes != 2: raise ValueError( "Trying to learn an SVM on more/less than two classes.") y = convert_binary_labels(y) if self.class_weight == 'balanced': n_pos = y[y == 1].shape[0] n_neg = y[y == -1].shape[0] self.weight = CArray.zeros(2) self.weight[0] = 1.0 * n_pos / (n_pos + n_neg) self.weight[1] = 1.0 * n_neg / (n_pos + n_neg) self._w = CArray.zeros(x.shape[1]) self._b = CArray(0.0) obj = self.objective(x, y) obj_new = obj for i in range(self.max_it): # pick a random sample subset idx = CArray.randsample(CArray.arange(x.shape[0], dtype=int), x.shape[0], random_state=i) # compute subgradients grad_w, grad_b = self.gradient_w_b(x[idx, :], y[idx]) for p in range(0, 71, 10): step = (self.eta**p) * 2**(-0.01 * i) / (x.shape[0]**0.5) self._w -= step * grad_w self._b -= step * grad_b # Applying UPPER bound d_ub = self.w[self._idx_ub] d_ub[d_ub > self._ub] = self._ub self.w[self._idx_ub] = d_ub # Applying LOWER bound d_lb = self.w[self._idx_lb] d_lb[d_lb < self._lb] = self._lb self.w[self._idx_lb] = d_lb obj_new = self.objective(x, y) if obj_new < obj: break if abs(obj_new - obj) < self.eps: self.logger.info("i {:}: {:}".format(i, obj_new)) # Sparse weights if input is sparse (like in CClassifierSVM) self._w = self.w.tosparse() if x.issparse else self.w return obj = obj_new if i % 10 == 0: loss = self.hinge_loss(x, y).sum() self.logger.info("i {:}: {:.4f}, L {:.4f}".format( i, obj, loss)) # Sparse weights if input is sparse (like in CClassifierSVM) self._w = self.w.tosparse() if x.issparse else self.w
def _euclidean_proj_simplex(self, v, s=1): """Compute the Euclidean projection on a positive simplex. Solves the optimisation problem (using the algorithm from [1]): min_w 0.5 * || w - v ||_2^2 , s.t. \\sum_i w_i = s, w_i >= 0 Parameters ---------- v : CArray 1-Dimensional vector s : int, optional Radius of the simplex. Default 1. Returns ------- w : CArray Euclidean projection of v on the simplex. Notes ----- The complexity of this algorithm is in O(n log(n)) as it involves sorting v. Better alternatives exist for high-dimensional sparse vectors (cf. [1]). However, this implementation still easily scales to millions of dimensions. References ---------- [1] Efficient Projections onto the l1-Ball for Learning in High Dimensions John Duchi, Shai Shalev-Shwartz, Yoram Singer, and Tushar Chandra. International Conference on Machine Learning (ICML 2008) http://www.cs.berkeley.edu/~jduchi/projects/DuchiSiShCh08.pdf """ v = CArray(v).ravel() d = v.size # check if we are already on the simplex if v.sum() == s and (v >= 0).sum() == d: return v # best projection: itself! # get the array of cumulative sums of a sorted (decreasing) copy of v u = v.deepcopy() u.sort(inplace=True) u = u[::-1] if u.issparse: u_nnz = CArray(u.nnz_data).todense() cssv = u_nnz.cumsum() else: cssv = u.cumsum() # get the number of > 0 components of the optimal solution # (only considering non-null elements in v j = CArray.arange(1, cssv.size+1) if u.issparse: rho = (j * u_nnz > (cssv - s)).sum() - 1 else: rho = (j * u > (cssv - s)).sum() - 1 # compute the Lagrange multiplier associated to the simplex constraint theta = (cssv[rho] - s) / (rho + 1.0) # compute the projection by thresholding v using theta w = v if w.issparse: p = CArray(w.nnz_data) p -= theta w[w.nnz_indices] = p else: w -= theta w[w < 0] = 0 return w
def __init__(self, model, loss=None, optimizer=None, optimizer_scheduler=None, pretrained=False, pretrained_classes=None, input_shape=None, random_state=None, preprocess=None, softmax_outputs=False, epochs=10, batch_size=1, n_jobs=1): self._device = self._set_device() self._random_state = random_state super(CClassifierPyTorch, self).__init__(model=model, preprocess=preprocess, pretrained=pretrained, pretrained_classes=pretrained_classes, input_shape=input_shape, softmax_outputs=softmax_outputs) self._init_model() self._n_jobs = n_jobs self._batch_size = batch_size if self._batch_size is None: self.logger.info( "No batch size passed. Value will be set to the default " "value of 1.") self._batch_size = 1 if self._input_shape is None: # try to infer from first layer first_layer = list(self._model._modules.values())[0] if isinstance(first_layer, torch.nn.Linear): self._input_shape = (first_layer.in_features, ) else: raise ValueError( "Input shape should be specified if the first " "layer is not a `nn.Linear` module.") self._loss = loss self._optimizer = optimizer self._optimizer_scheduler = optimizer_scheduler self._epochs = epochs if self._pretrained is True: self._trained = True if self._pretrained_classes is not None: self._classes = self._pretrained_classes else: self._classes = CArray.arange( list(self._model.modules())[-1].out_features) self._n_features = reduce(lambda a, b: a * b, self._input_shape) # hooks for getting intermediate outputs self._handlers = [] # will store intermediate outputs from the hooks self._intermediate_outputs = None self._cached_s = None self._cached_layer_output = None
from secml.array import CArray from secml.figure import CFigure fig = CFigure(fontsize=12) n = 12 X = CArray.arange(n) Y1 = (1 - X / float(n)) * (1.0 - 0.5) * CArray.rand((n, )) + 0.5 Y2 = (1 - X / float(n)) * (1.0 - 0.5) * CArray.rand((n, )) + 0.5 fig.sp.xticks([0.025, 0.025, 0.95, 0.95]) fig.sp.bar(X, Y1, facecolor='#9999ff', edgecolor='white') fig.sp.bar(X, -Y2, facecolor='#ff9999', edgecolor='white') for x, y in zip(X, Y1): fig.sp.text(x, y, '%.2f' % y, ha='center', va='bottom') for x, y in zip(X, Y2): fig.sp.text(x, -y - 0.02, '%.2f' % y, ha='center', va='top') fig.sp.xlim(-.5, n - .5) fig.sp.xticks(()) fig.sp.ylim(-1.25, 1.25) fig.sp.yticks(()) fig.sp.grid() fig.show()