def _forward(self, x): """Computes the decision function for each pattern in x. For One-Vs-All (OVA) multiclass scheme, this is the output of the `label`^th classifier. Parameters ---------- x : CArray Array with new patterns to classify, 2-Dimensional of shape (n_patterns, n_features). Returns ------- score : CArray Value of the decision function for each test pattern. Dense flat array of shape (n_samples,) if y is not None, otherwise a (n_samples, n_classes) array. """ # Getting predicted scores for classifier associated with y scores = CArray.empty(shape=(x.shape[0], self.n_classes)) # Discriminant function is now called for each different class res = parfor2(_forward_one_ova, self.n_classes, self.n_jobs, self, x, self.verbose) # Building results array for i in range(self.n_classes): scores[:, i] = CArray(res[i]) return scores
def test_empty(self): """Test for CArray.empty() classmethod.""" self.logger.info("Test for CArray.empty() classmethod.") for shape in [1, (1, ), 2, (2, ), (1, 2), (2, 1), (2, 2)]: for dtype in [None, float, int, bool]: for sparse in [False, True]: res = CArray.empty(shape=shape, dtype=dtype, sparse=sparse) self.logger.info( "CArray.empty(shape={:}, dtype={:}, sparse={:}):" "\n{:}".format(shape, dtype, sparse, res)) self.assertIsInstance(res, CArray) self.assertEqual(res.isdense, not sparse) self.assertEqual(res.issparse, sparse) if isinstance(shape, tuple): if len(shape) == 1 and sparse is True: # Sparse "vectors" have len(shape) == 2 self.assertEqual(res.shape, (1, shape[0])) else: self.assertEqual(res.shape, shape) else: if sparse is True: self.assertEqual(res.shape, (1, shape)) else: self.assertEqual(res.shape, (shape, )) if dtype is None: # Default dtype is float self.assertIsSubDtype(res.dtype, float) else: self.assertIsSubDtype(res.dtype, dtype)
def _objective_function_cw(self, x): if self._stored_vars is not None and \ 'const' in self._stored_vars: stored_const = self._stored_vars['const'][0] if self._x0.shape[0] == 1: # use same const for all points c_weight = stored_const.item() else: # each point has it own optimized const c_weight = CArray.empty(shape=(len(stored_const))) for i, c in enumerate(stored_const): c_weight[i] = c else: self.logger.warning('Constant value not stored during run. Using ' 'initial_const value. For computing the loss ' 'with the actual value of const set ' '`store_var_list=["const"]` in ' 'CAttackEvasionCleverhans.__init__().') c_weight = self._clvrh_attack.initial_const l2dist = ((self._x0 - x)**2).sum(axis=1).ravel() z_labels, z_predicted = self.classifier.predict( x, return_decision_function=True) y_target = CArray.zeros(shape=(1, self._n_classes), dtype=np.float32) # destination point label if self.y_target is not None: y_target[0, self.y_target] = 1 else: # indiscriminate attack y_target[0, self._y0] = 1 z_target = (z_predicted * y_target).sum(axis=1).ravel() z_other = ((z_predicted * (1 - y_target) + (z_predicted.min(axis=1) - 1) * y_target)).max(axis=1) z_other = z_other.ravel() # The following differs from the exact definition given in Carlini # and Wagner (2016). There (page 9, left column, last equation), # the maximum is taken over Z_other - Z_ target (or Z_target - Z_other # respectively) and -confidence. However, it doesn't seem that # would have the desired effect (loss term is <= 0 if and only if # the difference between the logit of the target and any other class # differs by at least confidence). Hence the rearrangement here. self.confidence = self._clvrh_attack.confidence if self.y_target is not None: # if targeted, optimize for making the target class most likely loss = CArray.maximum(z_other - z_target + self.confidence, CArray.zeros(x.shape[0])) else: # if untargeted, optimize for making any other class most likely loss = CArray.maximum(z_target - z_other + self.confidence, CArray.zeros(x.shape[0])) return c_weight * loss + l2dist
def test_explain(self): """Unittest for explain method.""" i = 67 ds_i = self.ds[i, :] x, y_true = ds_i.X, ds_i.Y.item() self.logger.info("Explaining P{:} c{:}".format(i, y_true)) x_pred, x_score = self.clf.predict(x, return_decision_function=True) self.logger.info("Predicted class {:}, scores:\n{:}".format( x_pred.item(), x_score)) self.logger.info("Candidates: {:}".format(x_score.argsort()[::-1])) fig = CFigure(height=1.5, width=12) # Plotting original image fig.subplot(1, self.ds.num_classes + 1, 1) fig.sp.imshow(x.reshape((8, 8)), cmap='gray') fig.sp.title("Origin c{:}".format(y_true)) fig.sp.yticks([]) fig.sp.xticks([]) attr = CArray.empty(shape=(self.ds.num_classes, x.size)) # Computing attributions for c in self.ds.classes: attr_c = self.explainer.explain(x, y=c) attr[c, :] = attr_c self.logger.info("Attributions class {:}:\n{:}".format( c, attr_c.tolist())) self.assertIsInstance(attr, CArray) self.assertEqual(attr.shape, attr.shape) th = max(abs(attr.min()), abs(attr.max())) # Plotting attributions for c in self.ds.classes: fig.subplot(1, self.ds.num_classes + 1, 2 + c) fig.sp.imshow(attr[c, :].reshape((8, 8)), cmap='seismic', vmin=-1 * th, vmax=th) fig.sp.title("Attr c{:}".format(c)) fig.sp.yticks([]) fig.sp.xticks([]) fig.tight_layout() fig.show()
def test_explain(self): """Unittest for explain method.""" i = 67 ds_i = self.ds[i, :] x, y_true = ds_i.X, ds_i.Y.item() self.logger.info("Explaining P{:} c{:}".format(i, y_true)) x_pred, x_score = self.clf.predict(x, return_decision_function=True) self.logger.info("Predicted class {:}, scores:\n{:}".format( x_pred.item(), x_score)) self.logger.info("Candidates: {:}".format(x_score.argsort()[::-1])) ref_img = None # Use default reference image m = 100 # Number of steps attr = CArray.empty(shape=(0, x.shape[1]), sparse=x.issparse) for c in self.ds.classes: # Compute attributions for each class a = self.explainer.explain(x, y=c, reference=ref_img, m=m) attr = attr.append(a, axis=0) self.assertIsInstance(attr, CArray) self.assertEqual(attr.shape[1], x.shape[1]) self.assertEqual(attr.shape[0], self.ds.num_classes) fig = CFigure(height=1.5, width=12) # Plotting original image fig.subplot(1, self.ds.num_classes + 1, 1) fig.sp.imshow(x.reshape((8, 8)), cmap='gray') fig.sp.title("Origin c{:}".format(y_true)) fig.sp.yticks([]) fig.sp.xticks([]) th = max(abs(attr.min()), abs(attr.max())) # Plotting attributions for c in self.ds.classes: fig.subplot(1, self.ds.num_classes + 1, 2 + c) fig.sp.imshow(attr[c, :].reshape((8, 8)), cmap='seismic', vmin=-1 * th, vmax=th) fig.sp.title("Attr c{:}".format(c)) fig.sp.yticks([]) fig.sp.xticks([]) fig.tight_layout() fig.show()
def test_margin(self): self.logger.info("Testing margin separation of SGD...") # we create 50 separable points dataset = CDLRandomBlobs(n_samples=50, centers=2, random_state=0, cluster_std=0.60).load() # fit the model clf = CClassifierSGD(loss=CLossHinge(), regularizer=CRegularizerL2(), alpha=0.01, max_iter=200, random_state=0) clf.fit(dataset.X, dataset.Y) # plot the line, the points, and the nearest vectors to the plane xx = CArray.linspace(-1, 5, 10) yy = CArray.linspace(-1, 5, 10) X1, X2 = np.meshgrid(xx.tondarray(), yy.tondarray()) Z = CArray.empty(X1.shape) for (i, j), val in np.ndenumerate(X1): x1 = val x2 = X2[i, j] Z[i, j] = clf.decision_function(CArray([x1, x2]), y=1) levels = [-1.0, 0.0, 1.0] linestyles = ['dashed', 'solid', 'dashed'] colors = 'k' fig = CFigure(linewidth=1) fig.sp.contour(X1, X2, Z, levels, colors=colors, linestyles=linestyles) fig.sp.scatter(dataset.X[:, 0].ravel(), dataset.X[:, 1].ravel(), c=dataset.Y, s=40) fig.savefig( fm.join(fm.abspath(__file__), 'figs', 'test_c_classifier_sgd2.pdf'))
def _test_evasion_multiclass(self, expected_x): # EVASION self.multiclass.verbose = 2 if self.normalizer is not None: lb = self.normalizer.feature_range[0] ub = self.normalizer.feature_range[1] else: lb = None ub = None dmax = 2 self.solver_params = {'eta': 1e-1, 'eta_min': 1.0} eva = CAttackEvasionPGDLS(classifier=self.multiclass, surrogate_classifier=self.multiclass, surrogate_data=self.ds, distance='l2', dmax=dmax, lb=lb, ub=ub, solver_params=self.solver_params, y_target=self.y_target) eva.verbose = 0 # 2 # Points from class 2 region # p_idx = 0 # Points from class 1 region # p_idx = 68 # Points from class 3 region p_idx = 1 # Wrong classified point # p_idx = 53 # Evasion goes up usually # Points from class 0 region # p_idx = 49 # Wrong classified point # p_idx = 27 # Correctly classified point x0 = self.ds.X[p_idx, :] y0 = self.ds.Y[p_idx].item() x_seq = CArray.empty((0, x0.shape[1])) scores = CArray([]) f_seq = CArray([]) x = x0 for d_idx, d in enumerate(range(0, dmax + 1)): self.logger.info("Evasion at dmax: " + str(d)) eva.dmax = d x, f_opt = eva._run(x0=x0, y0=y0, x_init=x) y_pred, score = self.multiclass.predict( x, return_decision_function=True) f_seq = f_seq.append(f_opt) # not considering all iterations, just values at dmax # for all iterations, you should bring eva.x_seq and eva.f_seq x_seq = x_seq.append(x, axis=0) s = score[:, y0 if self.y_target is None else self.y_target] scores = scores.append(s) self.logger.info("Predicted label after evasion: " + str(y_pred)) self.logger.info("Score after evasion: {:}".format(s)) self.logger.info("Objective function after evasion: {:}".format(f_opt)) # Compare optimal point with expected self.assert_array_almost_equal(eva.x_opt.todense().ravel(), expected_x, decimal=4) self._make_plots(x_seq, dmax, eva, x0, scores, f_seq)
def _make_plot(self, p_idx, eva, dmax): if self.make_figures is False: self.logger.debug("Skipping figures...") return x0 = self.ds.X[p_idx, :] y0 = self.ds.Y[p_idx].item() x_seq = CArray.empty((0, x0.shape[1])) scores = CArray([]) f_seq = CArray([]) x = x0 for d_idx, d in enumerate(range(0, dmax + 1)): self.logger.info("Evasion at dmax: " + str(d)) eva.dmax = d x, f_opt = eva._run(x0=x0, y0=y0, x_init=x) y_pred, score = self.multiclass.predict( x, return_decision_function=True) f_seq = f_seq.append(f_opt) # not considering all iterations, just values at dmax # for all iterations, you should bring eva.x_seq and eva.f_seq x_seq = x_seq.append(x, axis=0) s = score[:, y0 if self.y_target is None else self.y_target] scores = scores.append(s) self.logger.info("Predicted label after evasion: {:}".format(y_pred)) self.logger.info("Score after evasion: {:}".format(s)) self.logger.info("Objective function after evasion: {:}".format(f_opt)) fig = CFigure(height=9, width=10, markersize=6, fontsize=12) # Get plot bounds, taking into account ds and evaded point path bounds_x, bounds_y = self.ds.get_bounds() min_x, max_x = bounds_x min_y, max_y = bounds_y min_x = min(min_x, x_seq[:, 0].min()) max_x = max(max_x, x_seq[:, 0].max()) min_y = min(min_y, x_seq[:, 1].min()) max_y = max(max_y, x_seq[:, 1].max()) ds_bounds = [(min_x, max_x), (min_y, max_y)] # Plotting multiclass decision regions fig.subplot(2, 2, 1) fig = self._plot_decision_function(fig, plot_background=True) fig.sp.plot_path(x_seq, path_style='-', start_style='o', start_facecolor='w', start_edgewidth=2, final_style='o', final_facecolor='k', final_edgewidth=2) # plot distance constraint fig.sp.plot_fun(func=self._rescaled_distance, multipoint=True, plot_background=False, n_grid_points=20, levels_color='k', grid_limits=ds_bounds, levels=[0], colorbar=False, levels_linewidth=2.0, levels_style=':', alpha_levels=.4, c=x0, r=dmax) fig.sp.grid(linestyle='--', alpha=.5, zorder=0) # Plotting multiclass evasion objective function fig.subplot(2, 2, 2) fig = self._plot_decision_function(fig) fig.sp.plot_fgrads(eva.objective_function_gradient, grid_limits=ds_bounds, n_grid_points=20, color='k', alpha=.5) fig.sp.plot_path(x_seq, path_style='-', start_style='o', start_facecolor='w', start_edgewidth=2, final_style='o', final_facecolor='k', final_edgewidth=2) # plot distance constraint fig.sp.plot_fun(func=self._rescaled_distance, multipoint=True, plot_background=False, n_grid_points=20, levels_color='w', grid_limits=ds_bounds, levels=[0], colorbar=False, levels_style=':', levels_linewidth=2.0, alpha_levels=.5, c=x0, r=dmax) fig.sp.plot_fun(lambda z: eva.objective_function(z), multipoint=True, grid_limits=ds_bounds, colorbar=False, n_grid_points=20, plot_levels=False) fig.sp.grid(linestyle='--', alpha=.5, zorder=0) fig.subplot(2, 2, 3) if self.y_target is not None: fig.sp.title("Classifier Score for Target Class (Targ. Evasion)") else: fig.sp.title("Classifier Score for True Class (Indiscr. Evasion)") fig.sp.plot(scores) fig.sp.grid() fig.sp.xlim(0, dmax) fig.sp.xlabel("dmax") fig.subplot(2, 2, 4) fig.sp.title("Objective Function") fig.sp.plot(f_seq) fig.sp.grid() fig.sp.xlim(0, dmax) fig.sp.xlabel("dmax") fig.tight_layout() k_name = self.kernel.class_type if self.kernel is not None else 'lin' fig.savefig( fm.join( self.images_folder, "pgd_ls_reject_threshold_{:}c_kernel-{:}_target-{:}.pdf". format(self.ds.num_classes, k_name, self.y_target)))