def _create_2D_plots(self, normalizer): self._test_init(normalizer) self.logger.info("Create 2-dimensional plot") self.clf_orig = self.classifier.deepcopy() pois_clf = self._clf_poisoning()[0] fig = CFigure(height=4, width=10, title=self.clf_idx) n_rows = 1 n_cols = 2 box = self._create_box() fig.subplot(n_rows, n_cols, grid_slot=1) fig.sp.title('Attacker objective and gradients') self._plot_func(fig, self.poisoning.objective_function) self._plot_obj_grads(fig, self.poisoning.objective_function_gradient) fig.sp.plot_ds(self.tr) fig.sp.plot_decision_regions( self.clf_orig, plot_background=False, grid_limits=self.grid_limits, n_grid_points=10, ) fig.sp.plot_constraint(box, grid_limits=self.grid_limits, n_grid_points=10) fig.sp.plot_path(self.poisoning.x_seq, start_facecolor='r' if self.yc == 1 else 'b') fig.subplot(n_rows, n_cols, grid_slot=2) fig.sp.title('Classification error on val') self._plot_func(fig, self.poisoning.objective_function, acc=True) fig.sp.plot_ds(self.tr) fig.sp.plot_decision_regions( pois_clf, plot_background=False, grid_limits=self.grid_limits, n_grid_points=10, ) fig.sp.plot_constraint(box, grid_limits=self.grid_limits, n_grid_points=10) fig.sp.plot_path(self.poisoning.x_seq, start_facecolor='r' if self.yc == 1 else 'b') fig.tight_layout() exp_idx = "2d_pois_" exp_idx += self.clf_idx if self.classifier.class_type == 'svm': if self.classifier.kernel.preprocess is not None: exp_idx += "_norm" else: if self.classifier.preprocess is not None: exp_idx += "_norm" fig.savefig(exp_idx + '.pdf', file_format='pdf')
def plot_loss_after_attack(evasAttack): """ This function plots the evolution of the loss function of the surrogate classifier after an attack is performed. The loss function is normalized between 0 and 1. It helps to know whether parameters given to the attack algorithm are well tuned are not; the loss should be as minimal as possible. The script is inspired from https://secml.gitlab.io/tutorials/11-ImageNet_advanced.html#Visualize-and-check-the-attack-optimization """ n_iter = evasAttack.x_seq.shape[0] itrs = CArray.arange(n_iter) # create a plot that shows the loss during the attack iterations # note that the loss is not available for all attacks fig = CFigure(width=10, height=4, fontsize=14) # apply a linear scaling to have the loss in [0,1] loss = evasAttack.f_seq if loss is not None: loss = CNormalizerMinMax().fit_transform(CArray(loss).T).ravel() fig.subplot(1, 2, 1) fig.sp.xlabel('iteration') fig.sp.ylabel('loss') fig.sp.plot(itrs, loss, c='black') fig.tight_layout() fig.show()
def test_explain(self): """Unittest for explain method.""" i = 67 ds_i = self.ds[i, :] x, y_true = ds_i.X, ds_i.Y.item() self.logger.info("Explaining P{:} c{:}".format(i, y_true)) x_pred, x_score = self.clf.predict(x, return_decision_function=True) self.logger.info("Predicted class {:}, scores:\n{:}".format( x_pred.item(), x_score)) self.logger.info("Candidates: {:}".format(x_score.argsort()[::-1])) fig = CFigure(height=1.5, width=12) # Plotting original image fig.subplot(1, self.ds.num_classes + 1, 1) fig.sp.imshow(x.reshape((8, 8)), cmap='gray') fig.sp.title("Origin c{:}".format(y_true)) fig.sp.yticks([]) fig.sp.xticks([]) attr = CArray.empty(shape=(self.ds.num_classes, x.size)) # Computing attributions for c in self.ds.classes: attr_c = self.explainer.explain(x, y=c) attr[c, :] = attr_c self.logger.info("Attributions class {:}:\n{:}".format( c, attr_c.tolist())) self.assertIsInstance(attr, CArray) self.assertEqual(attr.shape, attr.shape) th = max(abs(attr.min()), abs(attr.max())) # Plotting attributions for c in self.ds.classes: fig.subplot(1, self.ds.num_classes + 1, 2 + c) fig.sp.imshow(attr[c, :].reshape((8, 8)), cmap='seismic', vmin=-1 * th, vmax=th) fig.sp.title("Attr c{:}".format(c)) fig.sp.yticks([]) fig.sp.xticks([]) fig.tight_layout() fig.show()
def test_explain(self): """Unittest for explain method.""" i = 67 ds_i = self.ds[i, :] x, y_true = ds_i.X, ds_i.Y.item() self.logger.info("Explaining P{:} c{:}".format(i, y_true)) x_pred, x_score = self.clf.predict(x, return_decision_function=True) self.logger.info("Predicted class {:}, scores:\n{:}".format( x_pred.item(), x_score)) self.logger.info("Candidates: {:}".format(x_score.argsort()[::-1])) ref_img = None # Use default reference image m = 100 # Number of steps attr = CArray.empty(shape=(0, x.shape[1]), sparse=x.issparse) for c in self.ds.classes: # Compute attributions for each class a = self.explainer.explain(x, y=c, reference=ref_img, m=m) attr = attr.append(a, axis=0) self.assertIsInstance(attr, CArray) self.assertEqual(attr.shape[1], x.shape[1]) self.assertEqual(attr.shape[0], self.ds.num_classes) fig = CFigure(height=1.5, width=12) # Plotting original image fig.subplot(1, self.ds.num_classes + 1, 1) fig.sp.imshow(x.reshape((8, 8)), cmap='gray') fig.sp.title("Origin c{:}".format(y_true)) fig.sp.yticks([]) fig.sp.xticks([]) th = max(abs(attr.min()), abs(attr.max())) # Plotting attributions for c in self.ds.classes: fig.subplot(1, self.ds.num_classes + 1, 2 + c) fig.sp.imshow(attr[c, :].reshape((8, 8)), cmap='seismic', vmin=-1 * th, vmax=th) fig.sp.title("Attr c{:}".format(c)) fig.sp.yticks([]) fig.sp.xticks([]) fig.tight_layout() fig.show()
def _create_params_grad_plot(self, normalizer): """ Show the gradient of the classifier parameters w.r.t the poisoning point """ self.logger.info("Create 2-dimensional plot of the poisoning " "gradient") self._test_init(normalizer) pois_clf = self._clf_poisoning()[0] if self.n_features == 2: debug_pois_obj = _CAttackPoisoningLinTest(self.poisoning) fig = CFigure(height=8, width=10) n_rows = 2 n_cols = 2 fig.title(self.clf_idx) fig.subplot(n_rows, n_cols, grid_slot=1) fig.sp.title('w1 wrt xc') self._plot_param_sub(fig, debug_pois_obj.w1, debug_pois_obj.gradient_w1_xc, pois_clf) fig.subplot(n_rows, n_cols, grid_slot=2) fig.sp.title('w2 wrt xc') self._plot_param_sub(fig, debug_pois_obj.w2, debug_pois_obj.gradient_w2_xc, pois_clf) fig.subplot(n_rows, n_cols, grid_slot=3) fig.sp.title('b wrt xc') self._plot_param_sub(fig, debug_pois_obj.b, debug_pois_obj.gradient_b_xc, pois_clf) fig.tight_layout() exp_idx = "2d_grad_pois_" exp_idx += self.clf_idx if self.classifier.preprocess is not None: exp_idx += "_norm" fig.savefig(exp_idx + '.pdf', file_format='pdf')
def _make_plots(self, x_seq, dmax, eva, x0, scores, f_seq): if self.make_figures is False: self.logger.debug("Skipping figures...") return fig = CFigure(height=9, width=10, markersize=6, fontsize=12) # Get plot bounds, taking into account ds and evaded point path bounds_x, bounds_y = self.ds.get_bounds() min_x, max_x = bounds_x min_y, max_y = bounds_y min_x = min(min_x, x_seq[:, 0].min()) max_x = max(max_x, x_seq[:, 0].max()) min_y = min(min_y, x_seq[:, 1].min()) max_y = max(max_y, x_seq[:, 1].max()) ds_bounds = [(min_x, max_x), (min_y, max_y)] # Plotting multiclass decision regions fig.subplot(2, 2, 1) fig = self._plot_decision_function(fig, plot_background=True) fig.sp.plot_path(x_seq, path_style='-', start_style='o', start_facecolor='w', start_edgewidth=2, final_style='o', final_facecolor='k', final_edgewidth=2) # plot distance constraint fig.sp.plot_fun(func=self._rescaled_distance, multipoint=True, plot_background=False, n_grid_points=20, levels_color='k', grid_limits=ds_bounds, levels=[0], colorbar=False, levels_linewidth=2.0, levels_style=':', alpha_levels=.4, c=x0, r=dmax) fig.sp.grid(linestyle='--', alpha=.5, zorder=0) # Plotting multiclass evasion objective function fig.subplot(2, 2, 2) fig = self._plot_decision_function(fig) fig.sp.plot_fgrads(eva._objective_function_gradient, grid_limits=ds_bounds, n_grid_points=20, color='k', alpha=.5) fig.sp.plot_path(x_seq, path_style='-', start_style='o', start_facecolor='w', start_edgewidth=2, final_style='o', final_facecolor='k', final_edgewidth=2) # plot distance constraint fig.sp.plot_fun(func=self._rescaled_distance, multipoint=True, plot_background=False, n_grid_points=20, levels_color='w', grid_limits=ds_bounds, levels=[0], colorbar=False, levels_style=':', levels_linewidth=2.0, alpha_levels=.5, c=x0, r=dmax) fig.sp.plot_fun(lambda z: eva._objective_function(z), multipoint=True, grid_limits=ds_bounds, colorbar=False, n_grid_points=20, plot_levels=False) fig.sp.grid(linestyle='--', alpha=.5, zorder=0) fig.subplot(2, 2, 3) if self.y_target is not None: fig.sp.title("Classifier Score for Target Class (Targ. Evasion)") else: fig.sp.title("Classifier Score for True Class (Indiscr. Evasion)") fig.sp.plot(scores) fig.sp.grid() fig.sp.xlim(0, dmax) fig.sp.xlabel("dmax") fig.subplot(2, 2, 4) fig.sp.title("Objective Function") fig.sp.plot(f_seq) fig.sp.grid() fig.sp.xlim(0, dmax) fig.sp.xlabel("dmax") fig.tight_layout() k_name = self.kernel.class_type if self.kernel is not None else 'lin' fig.savefig( fm.join( self.images_folder, "pgd_ls_multiclass_{:}c_kernel-{:}_target-{:}.pdf".format( self.ds.num_classes, k_name, self.y_target)))
def _make_plot(self, p_idx, eva, dmax): if self.make_figures is False: self.logger.debug("Skipping figures...") return x0 = self.ds.X[p_idx, :] y0 = self.ds.Y[p_idx].item() x_seq = CArray.empty((0, x0.shape[1])) scores = CArray([]) f_seq = CArray([]) x = x0 for d_idx, d in enumerate(range(0, dmax + 1)): self.logger.info("Evasion at dmax: " + str(d)) eva.dmax = d x, f_opt = eva._run(x0=x0, y0=y0, x_init=x) y_pred, score = self.multiclass.predict( x, return_decision_function=True) f_seq = f_seq.append(f_opt) # not considering all iterations, just values at dmax # for all iterations, you should bring eva.x_seq and eva.f_seq x_seq = x_seq.append(x, axis=0) s = score[:, y0 if self.y_target is None else self.y_target] scores = scores.append(s) self.logger.info("Predicted label after evasion: {:}".format(y_pred)) self.logger.info("Score after evasion: {:}".format(s)) self.logger.info("Objective function after evasion: {:}".format(f_opt)) fig = CFigure(height=9, width=10, markersize=6, fontsize=12) # Get plot bounds, taking into account ds and evaded point path bounds_x, bounds_y = self.ds.get_bounds() min_x, max_x = bounds_x min_y, max_y = bounds_y min_x = min(min_x, x_seq[:, 0].min()) max_x = max(max_x, x_seq[:, 0].max()) min_y = min(min_y, x_seq[:, 1].min()) max_y = max(max_y, x_seq[:, 1].max()) ds_bounds = [(min_x, max_x), (min_y, max_y)] # Plotting multiclass decision regions fig.subplot(2, 2, 1) fig = self._plot_decision_function(fig, plot_background=True) fig.sp.plot_path(x_seq, path_style='-', start_style='o', start_facecolor='w', start_edgewidth=2, final_style='o', final_facecolor='k', final_edgewidth=2) # plot distance constraint fig.sp.plot_fun(func=self._rescaled_distance, multipoint=True, plot_background=False, n_grid_points=20, levels_color='k', grid_limits=ds_bounds, levels=[0], colorbar=False, levels_linewidth=2.0, levels_style=':', alpha_levels=.4, c=x0, r=dmax) fig.sp.grid(linestyle='--', alpha=.5, zorder=0) # Plotting multiclass evasion objective function fig.subplot(2, 2, 2) fig = self._plot_decision_function(fig) fig.sp.plot_fgrads(eva.objective_function_gradient, grid_limits=ds_bounds, n_grid_points=20, color='k', alpha=.5) fig.sp.plot_path(x_seq, path_style='-', start_style='o', start_facecolor='w', start_edgewidth=2, final_style='o', final_facecolor='k', final_edgewidth=2) # plot distance constraint fig.sp.plot_fun(func=self._rescaled_distance, multipoint=True, plot_background=False, n_grid_points=20, levels_color='w', grid_limits=ds_bounds, levels=[0], colorbar=False, levels_style=':', levels_linewidth=2.0, alpha_levels=.5, c=x0, r=dmax) fig.sp.plot_fun(lambda z: eva.objective_function(z), multipoint=True, grid_limits=ds_bounds, colorbar=False, n_grid_points=20, plot_levels=False) fig.sp.grid(linestyle='--', alpha=.5, zorder=0) fig.subplot(2, 2, 3) if self.y_target is not None: fig.sp.title("Classifier Score for Target Class (Targ. Evasion)") else: fig.sp.title("Classifier Score for True Class (Indiscr. Evasion)") fig.sp.plot(scores) fig.sp.grid() fig.sp.xlim(0, dmax) fig.sp.xlabel("dmax") fig.subplot(2, 2, 4) fig.sp.title("Objective Function") fig.sp.plot(f_seq) fig.sp.grid() fig.sp.xlim(0, dmax) fig.sp.xlabel("dmax") fig.tight_layout() k_name = self.kernel.class_type if self.kernel is not None else 'lin' fig.savefig( fm.join( self.images_folder, "pgd_ls_reject_threshold_{:}c_kernel-{:}_target-{:}.pdf". format(self.ds.num_classes, k_name, self.y_target)))