def test_plot_density(self): N = 200 np.random.seed(1) X = np.concatenate((np.random.normal(0, 1, int(0.3 * N)), np.random.normal(5, 1, int(0.7 * N))))[:, np.newaxis] X_plot = CArray(np.linspace(-5, 10, 1000)[:, np.newaxis]) true_dens = CArray(0.3 * norm(0, 1).pdf(X_plot[:, 0].tondarray()) + 0.7 * norm(5, 1).pdf(X_plot[:, 0].tondarray())) fig = CFigure(width=7) fig.sp._sp.fill(X_plot[:, 0].tondarray(), true_dens.tondarray(), fc='black', alpha=0.2, label='input distribution') for kernel in ['gaussian', 'tophat', 'epanechnikov']: kde = CDensityEstimation(kernel=kernel, bandwidth=0.5) x, y = kde.estimate_density(CArray(X), n_points=N) fig.sp.plot(x, y, '-', label="kernel = '{0}'".format(kernel)) fig.sp.text(6, 0.38, "N={0} points".format(N)) fig.sp.legend(loc='upper left') fig.sp.plot(X[:, 0], -0.005 - 0.01 * np.random.random(X.shape[0]), '+k') fig.sp.xlim(-4, 9) fig.sp.ylim(-0.02, 0.4) fig.show()
def test_explain(self): """Unittest for explain method.""" i = 67 x = self.ds.X[i, :] attr = self.explainer.explain(x, y=1) self.logger.info("Attributions:\n{:}".format(attr.tolist())) self.assertIsInstance(attr, CArray) self.assertEqual(attr.shape, attr.shape) fig = CFigure(height=3, width=6) # Plotting original image fig.subplot(1, 2, 1) fig.sp.imshow(attr.reshape((8, 8)), cmap='gray') th = max(abs(attr.min()), abs(attr.max())) # Plotting attributions fig.subplot(1, 2, 2) fig.sp.imshow(attr.reshape((8, 8)), cmap='seismic', vmin=-1 * th, vmax=th) fig.show()
def plot_loss_after_attack(evasAttack): """ This function plots the evolution of the loss function of the surrogate classifier after an attack is performed. The loss function is normalized between 0 and 1. It helps to know whether parameters given to the attack algorithm are well tuned are not; the loss should be as minimal as possible. The script is inspired from https://secml.gitlab.io/tutorials/11-ImageNet_advanced.html#Visualize-and-check-the-attack-optimization """ n_iter = evasAttack.x_seq.shape[0] itrs = CArray.arange(n_iter) # create a plot that shows the loss during the attack iterations # note that the loss is not available for all attacks fig = CFigure(width=10, height=4, fontsize=14) # apply a linear scaling to have the loss in [0,1] loss = evasAttack.f_seq if loss is not None: loss = CNormalizerMinMax().fit_transform(CArray(loss).T).ravel() fig.subplot(1, 2, 1) fig.sp.xlabel('iteration') fig.sp.ylabel('loss') fig.sp.plot(itrs, loss, c='black') fig.tight_layout() fig.show()
def test_plot(self): """Test for LDA. Check LDA Result Graphically. Apply Lda to Sklearn Iris Dataset and compare it with "Linear Discriminant Analysis bit by bit" by Sebastian Raschka http://sebastianraschka.com/Articles/2014_python_lda.html into the plot we must see approximatively: x axes: from -2 to -1 virginica, from -1 to 0 versicolor, from 1 to 2,3 setosa y axes: from -1 to -1 virginica, from -1 to 0.5 versicolor, from -1 to 1 setosa """ from sklearn.datasets import load_iris iris_db = load_iris() patterns = CArray(iris_db.data) labels = CArray(iris_db.target) lda = CLDA() lda.fit(patterns, labels) # store dataset reduced with pca red_dts = lda.fit_transform(patterns, labels) fig = CFigure(width=10, markersize=8) fig.sp.scatter(red_dts[:, 0].ravel(), red_dts[:, 1].ravel(), c=labels) fig.show()
def test_fun(self): """Test for CPlotFunction.plot_fun method.""" fig = CFigure() fig.sp.plot_ds(self.dataset) fig.sp.plot_fun(self.clf.decision_function, y=1) fig.show()
def test_plot_decision_function(self): """Test plot of multiclass classifier decision function.""" # generate synthetic data ds = CDLRandom(n_classes=3, n_features=2, n_redundant=0, n_clusters_per_class=1, class_sep=1, random_state=0).load() multiclass = CClassifierMulticlassOVA( classifier=CClassifierSVM, class_weight='balanced', preprocess='min-max') # Training and classification multiclass.fit(ds.X, ds.Y) y_pred, score_pred = multiclass.predict( ds.X, return_decision_function=True) def plot_hyperplane(img, clf, min_v, max_v, linestyle, label): """Plot the hyperplane associated to the OVA clf.""" xx = CArray.linspace( min_v - 5, max_v + 5) # make sure the line is long enough # get the separating hyperplane yy = -(clf.w[0] * xx + clf.b) / clf.w[1] img.sp.plot(xx, yy, linestyle, label=label) fig = CFigure(height=7, width=8) fig.sp.title('{:} ({:})'.format(multiclass.__class__.__name__, multiclass.classifier.__name__)) x_bounds, y_bounds = ds.get_bounds() styles = ['go-', 'yp--', 'rs-.', 'bD--', 'c-.', 'm-', 'y-.'] for c_idx, c in enumerate(ds.classes): # Plot boundary and predicted label for each OVA classifier plot_hyperplane(fig, multiclass._binary_classifiers[c_idx], x_bounds[0], x_bounds[1], styles[c_idx], 'Boundary\nfor class {:}'.format(c)) fig.sp.scatter(ds.X[ds.Y == c, 0], ds.X[ds.Y == c, 1], s=40, c=styles[c_idx][0]) fig.sp.scatter(ds.X[y_pred == c, 0], ds.X[y_pred == c, 1], s=160, edgecolors=styles[c_idx][0], facecolors='none', linewidths=2) # Plotting multiclass decision function fig.sp.plot_decision_regions(multiclass, n_grid_points=100, grid_limits=ds.get_bounds(offset=5)) fig.sp.xlim(x_bounds[0] - .5 * x_bounds[1], x_bounds[1] + .5 * x_bounds[1]) fig.sp.ylim(y_bounds[0] - .5 * y_bounds[1], y_bounds[1] + .5 * y_bounds[1]) fig.sp.legend(loc=4) # lower, right fig.show()
def test_compute(self): self.roc.compute(self.ds1.Y, self.s1[:, 1].ravel()) fig = CFigure() fig.sp.semilogx(self.roc.fpr, self.roc.tpr) fig.sp.grid() fig.show()
def test_fgrads(self): """Test for CPlotFunction.plot_fgrads method.""" fig = CFigure() fig.sp.plot_ds(self.dataset) fig.sp.plot_fun(self.clf.decision_function, y=1) fig.sp.plot_fgrads(lambda x: self.clf.grad_f_x(x, y=1)) fig.show()
def test_simple(): """Plot the result of a dot product operation.""" def test_dot(): a = CArray([1, 2, 3]) b = CArray([10, 20, 30]) return a.dot(b) fig = CFigure() fig.sp.plot(test_dot(), marker='o') fig.show()
def test_confusion_matrix(self): """Test for `CPlot.plot_confusion_matrix()` method.""" y_true = CArray([2, 0, 2, 2, 0, 1]) y_pred = CArray([0, 0, 2, 2, 0, 2]) fig = CFigure() fig.sp.plot_confusion_matrix(y_true, y_pred, labels=['one', 'two', 'three'], colorbar=True, normalize=False) fig.show()
def _plot_sec_eval(sec_eval): figure = CFigure(height=5, width=5) figure.sp.plot_sec_eval(sec_eval.sec_eval_data, label='SVM', marker='o', show_average=True, mean=True) figure.sp.title(sec_eval.attack.__class__.__name__) figure.subplots_adjust() figure.show()
def test_standard(self): """Plot of standard ROC.""" # Testing without input CFigure roc_plot = CFigure() roc_plot.sp.title('ROC Curve Standard') # Plotting 2 times (to show multiple curves) # add one curve for repetition and call it rep 0 and rep 1 of roc 1 roc_plot.sp.plot_roc(self.roc_wmean.mean_fpr, self.roc_wmean.mean_tpr) roc_plot.show()
def _plot_sec_eval(self): # figure creation figure = CFigure(height=5, width=5) sec_eval_data = [ sec_eval.sec_eval_data for sec_eval in self.sec_eval] # plot security evaluation figure.sp.plot_sec_eval(sec_eval_data, label='SVM', marker='o', show_average=True, mean=True) figure.subplots_adjust() figure.show()
def test_mean(self): self.roc.compute([self.ds1.Y, self.ds2.Y], [self.s1[:, 1].ravel(), self.s2[:, 1].ravel()]) mean_fp, mean_tp, mean_std = self.roc.average(return_std=True) fig = CFigure(linewidth=2) fig.sp.errorbar(self.roc.mean_fpr, self.roc.mean_tpr, yerr=mean_std) for rep in range(self.roc.n_reps): fig.sp.semilogx(self.roc.fpr[rep], self.roc.tpr[rep]) fig.sp.semilogx(mean_fp, mean_tp) fig.sp.grid() fig.show()
def test_explain(self): """Unittest for explain method.""" i = 67 ds_i = self.ds[i, :] x, y_true = ds_i.X, ds_i.Y.item() self.logger.info("Explaining P{:} c{:}".format(i, y_true)) x_pred, x_score = self.clf.predict(x, return_decision_function=True) self.logger.info("Predicted class {:}, scores:\n{:}".format( x_pred.item(), x_score)) self.logger.info("Candidates: {:}".format(x_score.argsort()[::-1])) fig = CFigure(height=1.5, width=12) # Plotting original image fig.subplot(1, self.ds.num_classes + 1, 1) fig.sp.imshow(x.reshape((8, 8)), cmap='gray') fig.sp.title("Origin c{:}".format(y_true)) fig.sp.yticks([]) fig.sp.xticks([]) attr = CArray.empty(shape=(self.ds.num_classes, x.size)) # Computing attributions for c in self.ds.classes: attr_c = self.explainer.explain(x, y=c) attr[c, :] = attr_c self.logger.info("Attributions class {:}:\n{:}".format( c, attr_c.tolist())) self.assertIsInstance(attr, CArray) self.assertEqual(attr.shape, attr.shape) th = max(abs(attr.min()), abs(attr.max())) # Plotting attributions for c in self.ds.classes: fig.subplot(1, self.ds.num_classes + 1, 2 + c) fig.sp.imshow(attr[c, :].reshape((8, 8)), cmap='seismic', vmin=-1 * th, vmax=th) fig.sp.title("Attr c{:}".format(c)) fig.sp.yticks([]) fig.sp.xticks([]) fig.tight_layout() fig.show()
def test_single(self): """Plot of ROC repetitions.""" # Testing without input CFigure roc_plot = CFigure() roc_plot.sp.title('ROC Curve Repetitions') # Plotting 2 times (to show multiple curves) # add one curve for repetition and call it rep 0 and rep 1 of roc 1 roc_plot.sp.plot_roc_reps(self.roc_nomean, label='roc1') # add one curve for repetition and call it rep 0 and rep 1 of roc 2 roc_plot.sp.plot_roc_reps(self.roc_nomean, label='roc2') roc_plot.show()
def test_explain(self): """Unittest for explain method.""" i = 67 ds_i = self.ds[i, :] x, y_true = ds_i.X, ds_i.Y.item() self.logger.info("Explaining P{:} c{:}".format(i, y_true)) x_pred, x_score = self.clf.predict(x, return_decision_function=True) self.logger.info("Predicted class {:}, scores:\n{:}".format( x_pred.item(), x_score)) self.logger.info("Candidates: {:}".format(x_score.argsort()[::-1])) ref_img = None # Use default reference image m = 100 # Number of steps attr = CArray.empty(shape=(0, x.shape[1]), sparse=x.issparse) for c in self.ds.classes: # Compute attributions for each class a = self.explainer.explain(x, y=c, reference=ref_img, m=m) attr = attr.append(a, axis=0) self.assertIsInstance(attr, CArray) self.assertEqual(attr.shape[1], x.shape[1]) self.assertEqual(attr.shape[0], self.ds.num_classes) fig = CFigure(height=1.5, width=12) # Plotting original image fig.subplot(1, self.ds.num_classes + 1, 1) fig.sp.imshow(x.reshape((8, 8)), cmap='gray') fig.sp.title("Origin c{:}".format(y_true)) fig.sp.yticks([]) fig.sp.xticks([]) th = max(abs(attr.min()), abs(attr.max())) # Plotting attributions for c in self.ds.classes: fig.subplot(1, self.ds.num_classes + 1, 2 + c) fig.sp.imshow(attr[c, :].reshape((8, 8)), cmap='seismic', vmin=-1 * th, vmax=th) fig.sp.title("Attr c{:}".format(c)) fig.sp.yticks([]) fig.sp.xticks([]) fig.tight_layout() fig.show()
def test_custom_params(self): """Plot of ROC altering default parameters.""" # Testing without input CFigure roc_plot = CFigure() roc_plot.sp.title('ROC Curve - Custom') roc_plot.sp.xlim(0.1, 100) roc_plot.sp.ylim(30, 100) roc_plot.sp.yticks([70, 80, 90, 100]) roc_plot.sp.yticklabels(['70', '80', '90', '100']) # Plotting 2 times (to show 2 curves) roc_plot.sp.plot_roc_mean(self.roc_wmean, label='roc1') roc_plot.sp.plot_roc_mean(self.roc_wmean, label='roc2') roc_plot.show()
def test_plot_decision_regions(self): """Test for `.plot_decision_regions` method.""" fig = CFigure(width=10, height=5) fig.subplot(1, 2, 1) fig.sp.plot_ds(self.dataset) fig.sp.plot_decision_regions( self.clf, n_grid_points=200, plot_background=False) fig.subplot(1, 2, 2) fig.sp.plot_ds(self.dataset) fig.sp.plot_decision_regions( self.clf, n_grid_points=200) fig.show()
def test_mean(self): """Plot of average ROC.""" # Testing without input CFigure roc_plot = CFigure() roc_plot.sp.title('ROC Curve') # Plotting 2 times (to show 2 curves) roc_plot.sp.plot_roc_mean(self.roc_wmean, label='roc1 mean', plot_std=True) roc_plot.sp.plot_roc_reps(self.roc_wmean, label='roc1') roc_plot.show() # Testing mean plot with no average with self.assertRaises(ValueError): roc_plot.sp.plot_roc_mean(self.roc_nomean)
def test_quiver(self): """Test for `CPlot.quiver()` method.""" # gradient values creation xv = CArray.arange(0, 2 * constants.pi, .2) yv = CArray.arange(0, 2 * constants.pi, .2) X, Y = CArray.meshgrid((xv, yv)) U = CArray.cos(X) V = CArray.sin(Y) plot = CFigure() plot.sp.title('Gradient arrow') plot.sp.quiver(U, V) plot.show()
def test_plot(self): ds = CDLRandom(n_samples=100, n_features=2, n_redundant=0, random_state=100).load() self.logger.info("Train Sec SVM") sec_svm = CClassifierSecSVM(C=1, eta=0.1, eps=1e-3, lb=-0.1, ub=0.5) sec_svm.verbose = 2 sec_svm.fit(ds.X, ds.Y) self.logger.info("Train SVM") svm = CClassifierSVM(C=1) svm.fit(ds.X, ds.Y) self._compute_alignment(ds, sec_svm, svm) fig = CFigure(height=5, width=8) fig.subplot(1, 2, 1) # Plot dataset points fig.sp.plot_ds(ds) # Plot objective function fig.sp.plot_fun(svm.predict, multipoint=True, plot_background=True, plot_levels=False, n_grid_points=100, grid_limits=ds.get_bounds()) fig.sp.title("SVM") fig.subplot(1, 2, 2) # Plot dataset points fig.sp.plot_ds(ds) # Plot objective function fig.sp.plot_fun(sec_svm.predict, multipoint=True, plot_background=True, plot_levels=False, n_grid_points=100, grid_limits=ds.get_bounds()) fig.sp.title("Sec-SVM") fig.show()
def test_draw(self): """Drawing the loss functions. Inspired by: https://en.wikipedia.org/wiki/Loss_functions_for_classification """ fig = CFigure() x = CArray.arange(-1, 3.01, 0.01) for loss_id in ('e-insensitive', 'e-insensitive-squared', 'quadratic'): self.logger.info("Creating loss: {:}".format(loss_id)) loss_class = CLoss.create(loss_id) fig.sp.plot(x, loss_class.loss(CArray([1]), x), label=loss_id) fig.sp.grid() fig.sp.legend() fig.show()
class TestCFigure(CUnitTest): """Unittest for CFigure.""" def test_svm(self): self.X = CArray([[1, 2], [3, 4], [5, 6], [7, 8]]) self.Y = CArray([[0], [1], [1], [0]]).ravel() self.dataset = CDataset(self.X, self.Y) self.classifier = CClassifierSVM(kernel=CKernelRBF()) self.classifier.fit(self.dataset) self.x_min, self.x_max = (self.X[:, [0]].min() - 1, self.X[:, [0]].max() + 1) self.y_min, self.y_max = (self.X[:, [1]].min() - 1, self.X[:, [1]].max() + 1) self.fig = CFigure(height=7, width=10, linewidth=5, fontsize=24, markersize=20) self.fig.sp.title("Svm Test") self.logger.info("Test plot dataset method...") self.fig.sp.plot_ds(self.dataset) self.logger.info("Test plot path method...") path = CArray([[1, 2], [1, 3], [1.5, 5]]) self.fig.sp.plot_path(path) self.logger.info("Test plot function method...") bounds = [(self.x_min, self.x_max), (self.y_min, self.y_max)] self.fig.sp.plot_fun(self.classifier.decision_function, plot_levels=False, grid_limits=bounds, y=1) self.fig.sp.xlim(self.x_min, self.x_max) self.fig.sp.ylim(self.y_min, self.y_max) self.fig.show()
def test_draw(self): """ Compare the classifiers graphically""" self.logger.info("Testing classifiers graphically") fig = CFigure(width=10, markersize=8) # Plot dataset points # mark the rejected samples y = self.clf.predict(self.dataset.X) fig.sp.plot_ds( self.dataset[y == -1, :], colors=['k', 'k'], markersize=12) # plot the dataset fig.sp.plot_ds(self.dataset) # Plot objective function fig.sp.plot_fun(self.clf.decision_function, grid_limits=self.dataset.get_bounds(), levels=[0], y=1) fig.sp.title('Classifier with reject threshold') fig.show()
def test_draw(self): """Drawing the loss functions. Inspired by: https://en.wikipedia.org/wiki/Loss_functions_for_classification """ fig = CFigure() x = CArray.arange(-1, 3.01, 0.01) fig.sp.plot(x, CArray([1 if i <= 0 else 0 for i in x]), label='0-1 indicator') for loss_id in ('hinge', 'hinge-squared', 'square', 'log'): self.logger.info("Creating loss: {:}".format(loss_id)) loss_class = CLoss.create(loss_id) fig.sp.plot(x, loss_class.loss(CArray([1]), x), label=loss_id) fig.sp.grid() fig.sp.legend() fig.show()
def test_compare_sklearn(self): import numpy as np from sklearn import svm, datasets from sklearn.metrics import roc_curve, auc from sklearn.model_selection import StratifiedKFold from secml.figure import CFigure roc_fig = CFigure(width=12) # import some data to play with iris = datasets.load_iris() X = iris.data y = iris.target X, y = X[y != 2], y[y != 2] n_samples, n_features = X.shape # Add noisy features random_state = np.random.RandomState(0) X = np.c_[X, random_state.randn(n_samples, 200 * n_features)] # Classification and ROC analysis # Run classifier with cross-validation and plot ROC curves classifier = svm.SVC(kernel='linear', probability=True, random_state=random_state) roc_fig.subplot(1, 2, 1) mean_tpr = 0.0 mean_fpr = np.linspace(0, 1, 1000) cv = StratifiedKFold(n_splits=6) for i, (train, test) in enumerate(cv.split(X, y)): probas_ = classifier.fit(X[train], y[train]).predict_proba(X[test]) # Compute ROC curve and area the curve fpr, tpr, thresholds = roc_curve(y[test], probas_[:, 1]) mean_tpr += np.interp(mean_fpr, fpr, tpr) mean_tpr[0] = 0.0 roc_auc = auc(fpr, tpr) roc_fig.sp.plot(fpr, tpr, linewidth=1, label='ROC fold %d (area = %0.2f)' % (i, roc_auc)) roc_fig.sp.plot([0, 1], [0, 1], '--', color=(0.6, 0.6, 0.6), label='Luck') mean_tpr /= cv.get_n_splits() mean_tpr[-1] = 1.0 mean_auc = auc(mean_fpr, mean_tpr) roc_fig.sp.plot(mean_fpr, mean_tpr, 'k--', label='Mean ROC (area = %0.2f)' % mean_auc, linewidth=2) roc_fig.sp.xlim([-0.05, 1.05]) roc_fig.sp.ylim([-0.05, 1.05]) roc_fig.sp.xlabel('False Positive Rate') roc_fig.sp.ylabel('True Positive Rate') roc_fig.sp.title('Sklearn Receiver operating characteristic example') roc_fig.sp.legend(loc="lower right") roc_fig.sp.grid() self.logger.info("Plotting using our CPLotRoc") roc_fig.subplot(1, 2, 2) score = [] true_y = [] for i, (train, test) in enumerate(cv.split(X, y)): probas_ = classifier.fit(X[train], y[train]).predict_proba(X[test]) true_y.append(CArray(y[test])) score.append(CArray(probas_[:, 1])) self.roc_wmean = CRoc() self.roc_wmean.compute(true_y, score) fp, tp = self.roc_wmean.average() roc_fig.sp.plot([0, 100], [0, 100], '--', color=(0.6, 0.6, 0.6), label='Luck') roc_fig.sp.xticks([0, 20, 40, 60, 80, 100]) roc_fig.sp.xticklabels(['0', '20', '40', '60', '80', '100']) roc_fig.sp.plot_roc_mean(self.roc_wmean, plot_std=True, logx=False, style='go-', label='Mean ROC (area = %0.2f)' % (auc(fp.tondarray(), tp.tondarray()))) roc_fig.sp.xlim([-0.05 * 100, 1.05 * 100]) roc_fig.sp.ylim([-0.05 * 100, 1.05 * 100]) roc_fig.sp.title('SecML Receiver operating characteristic example') roc_fig.sp.legend(loc="lower right") roc_fig.show()
from secml.array import CArray from secml.figure import CFigure fig = CFigure(fontsize=14) fig.title('loglog base 4 on x') t = CArray.arange(0.01, 20.0, 0.01) fig.sp.loglog(t, 20 * (-t / 10.0).exp(), basex=2) fig.sp.grid() fig.show()
def test_constraint(self): """Test for CPlotFunction.plot_fun method.""" fig = CFigure() for constraint in self.constraints: fig.sp.plot_constraint(constraint) fig.show()