def _attack_pgd_ls(self): params = { "classifier": self.classifier, "double_init_ds": self.tr, "distance": 'l1', "lb": self.lb, "ub": self.ub, "y_target": self.y_target, "attack_classes": self.attack_classes, "solver_params": { 'eta': 0.5, 'eps': 1e-2 } } attack = CAttackEvasionPGDLS(**params) attack.verbose = 1 # sec eval params param_name = 'dmax' dmax = 2 dmax_step = 0.5 param_values = CArray.arange(start=0, step=dmax_step, stop=dmax + dmax_step) return attack, param_name, param_values
def _set_evasion(self, ds, params): """Prepare the evasion attack. - discretize data if necessary - train the classifier (if not trained) - train the surrogate classifier (if not trained) - create the evasion object - choose an attack starting point Parameters ---------- ds : CDataset params : dict Parameters for the attack class. Returns ------- evas : CAttackEvasion x0 : CArray Initial attack point. y0 : CArray Label of the initial attack point. """ if not params["classifier"].is_fitted(): self.logger.info("Training classifier...") params["classifier"].fit(ds.X, ds.Y) evas = CAttackEvasionPGDLS(**params) evas.verbose = 2 # pick a malicious sample x0, y0 = self._choose_x0_2c(ds) return evas, x0, y0
def _set_evasion(self, params, x0_img_class): """Prepare the evasion attack. - train the classifier (if not trained) - train the surrogate classifier (if not trained) - create the evasion object - choose an attack starting point Parameters ---------- params : dict Parameters for the attack class. x0_img_class : int Class from which to choose the initial attack point. Returns ------- evas : CAttackEvasion x0 : CArray Initial attack point. y0 : CArray Label of the initial attack point. """ if not params["classifier"].is_fitted(): self.logger.info("Training classifier...") params["classifier"].fit(self._tr.X, self._tr.Y) evas = CAttackEvasionPGDLS(**params) evas.verbose = 2 # pick a malicious sample x0, y0 = self._choose_x0_2c(x0_img_class) return evas, x0, y0
def test_attack_pgd_ls(self): """Test SecEval using CAttackEvasionPGDLS.""" params = { "classifier": self.clf, "double_init_ds": self.tr, "distance": 'l2', "lb": -2, "ub": 2, "y_target": None, "solver_params": {'eta': 0.1, 'eps': 1e-2} } attack = CAttackEvasionPGDLS(**params) attack.verbose = 1 param_name = 'dmax' self._set_and_run(attack, param_name)
def test_attack_pgd_ls_discrete(self): """Test SecEval using CAttackEvasionPGDLS on a problematic discrete case with L1 constraint. We alter the classifier so that many weights have the same value. The optimizer should be able to evade the classifier anyway, by changing one feature each iteration. Otherwise, by changing all the feature with the same value at once, the evasion will always fail because the L1 constraint will be violated. """ self.ds = self._discretize_data(self.ds, eta=1) self.ds.X[self.ds.X > 1] = 1 self.ds.X[self.ds.X < -1] = -1 self.tr = self.ds[:self.n_tr, :] self.ts = self.ds[self.n_tr:, :] self.clf.fit(self.tr.X, self.tr.Y) # Set few features to the same max value w_new = self.clf.w.deepcopy() w_new[CArray.randint( self.clf.w.size, shape=5, random_state=0)] = self.clf.w.max() self.clf._w = w_new params = { "classifier": self.clf, "double_init": False, "distance": 'l1', "lb": -1, "ub": 1, "y_target": None, "solver_params": {'eta': 1, 'eps': 1e-2} } attack = CAttackEvasionPGDLS(**params) attack.verbose = 1 param_name = 'dmax' self._set_and_run(attack, param_name, dmax_step=1)
def _test_evasion_multiclass(self, expected_x): # EVASION self.multiclass.verbose = 2 if self.normalizer is not None: lb = self.normalizer.feature_range[0] ub = self.normalizer.feature_range[1] else: lb = None ub = None dmax = 2 self.solver_params = {'eta': 1e-1, 'eta_min': 1.0} eva = CAttackEvasionPGDLS(classifier=self.multiclass, surrogate_classifier=self.multiclass, surrogate_data=self.ds, distance='l2', dmax=dmax, lb=lb, ub=ub, solver_params=self.solver_params, y_target=self.y_target) eva.verbose = 0 # 2 # Points from class 2 region # p_idx = 0 # Points from class 1 region # p_idx = 68 # Points from class 3 region p_idx = 1 # Wrong classified point # p_idx = 53 # Evasion goes up usually # Points from class 0 region # p_idx = 49 # Wrong classified point # p_idx = 27 # Correctly classified point x0 = self.ds.X[p_idx, :] y0 = self.ds.Y[p_idx].item() x_seq = CArray.empty((0, x0.shape[1])) scores = CArray([]) f_seq = CArray([]) x = x0 for d_idx, d in enumerate(range(0, dmax + 1)): self.logger.info("Evasion at dmax: " + str(d)) eva.dmax = d x, f_opt = eva._run(x0=x0, y0=y0, x_init=x) y_pred, score = self.multiclass.predict( x, return_decision_function=True) f_seq = f_seq.append(f_opt) # not considering all iterations, just values at dmax # for all iterations, you should bring eva.x_seq and eva.f_seq x_seq = x_seq.append(x, axis=0) s = score[:, y0 if self.y_target is None else self.y_target] scores = scores.append(s) self.logger.info("Predicted label after evasion: " + str(y_pred)) self.logger.info("Score after evasion: {:}".format(s)) self.logger.info("Objective function after evasion: {:}".format(f_opt)) # Compare optimal point with expected self.assert_array_almost_equal(eva.x_opt.todense().ravel(), expected_x, decimal=4) self._make_plots(x_seq, dmax, eva, x0, scores, f_seq)
def _test_evasion_multiclass(self, expected_x): # EVASION self.multiclass.verbose = 2 if self.normalizer is not None: lb = self.normalizer.feature_range[0] ub = self.normalizer.feature_range[1] else: lb = None ub = None dmax = 3 self.solver_params = {'eta': 0.5, 'max_iter': 3} eva = CAttackEvasionPGDLS(classifier=self.multiclass, double_init_ds=self.ds, distance='l2', dmax=dmax, lb=lb, ub=ub, solver_params=self.solver_params, y_target=self.y_target) eva.verbose = 0 # 2 # Points from class 2 region # p_idx = 0 # Points from class 1 region # p_idx = 68 # Points from class 3 region p_idx = 1 # Wrong classified point # p_idx = 53 # Evasion goes up usually # Points from class 0 region # p_idx = 49 # Wrong classified point # p_idx = 27 # Correctly classified point x0 = self.ds.X[p_idx, :] y0 = self.ds.Y[p_idx].item() self.logger.info("Evasion at dmax: " + str(dmax)) eva.dmax = dmax x_opt, f_opt = eva._run(x0=x0, y0=y0, x_init=x0) y_pred, score = self.multiclass.predict(x_opt, return_decision_function=True) s = score[:, y0 if self.y_target is None else self.y_target] self.logger.info( "Number of objective function evaluations: {:}".format(eva.f_eval)) self.logger.info("Number of gradient function evaluations: {:}".format( eva.grad_eval)) self.logger.info("Predicted label after evasion: {:}".format(y_pred)) self.logger.info("Score after evasion: {:}".format(s)) self.logger.info("Objective function after evasion: {:}".format(f_opt)) # Compare optimal point with expected self.assert_array_almost_equal(eva.x_opt.todense().ravel(), expected_x, decimal=4) if self.y_target: s_ytarget_x0 = self.multiclass.decision_function(x0, self.y_target) s_ytarget_xopt = self.multiclass.decision_function( x_opt, self.y_target) self.logger.info( "Discriminat function w.r.t the target class first: {:} " "and after evasion: {:}".format(s_ytarget_x0, s_ytarget_xopt)) self.assertLess(s_ytarget_x0, s_ytarget_xopt) else: # indiscriminate attack s_ytrue_x0 = self.multiclass.decision_function(x0, y0) s_ytrue_xopt = self.multiclass.decision_function(x_opt, y0) self.logger.info( "Discriminat function w.r.t the true class first: {:} " "and after evasion: {:}".format(s_ytrue_x0, s_ytrue_xopt)) self.assertGreater(s_ytrue_x0, s_ytrue_xopt) self._make_plot(p_idx, eva, dmax)
def setUp(self): classifier = CClassifierSVM( kernel='linear', C=1.0, grad_sampling=1.0) # data parameters discrete = False lb = -2 ub = +2 n_tr = 20 n_ts = 10 n_features = 2 n_reps = 1 self.sec_eval = [] self.attack_ds = [] for rep_i in range(n_reps): self.logger.info( "Loading `random_blobs` with seed: {:}".format(rep_i)) loader = CDLRandomBlobs( n_samples=n_tr + n_ts, n_features=n_features, centers=[(-0.5, -0.5), (+0.5, +0.5)], center_box=(-0.5, 0.5), cluster_std=0.5, random_state=rep_i * 100 + 10) ds = loader.load() tr = ds[:n_tr, :] ts = ds[n_tr:, :] classifier.fit(tr) self.attack_ds.append(ts) # only manipulate positive samples, targeting negative ones self.y_target = None attack_classes = CArray([1]) params = { "classifier": classifier, "surrogate_classifier": classifier, "surrogate_data": tr, "distance": 'l1', "lb": lb, "ub": ub, "discrete": discrete, "y_target": self.y_target, "attack_classes": attack_classes, "solver_params": {'eta': 0.5, 'eps': 1e-2} } attack = CAttackEvasionPGDLS(**params) attack.verbose = 1 # sec eval params param_name = 'dmax' dmax = 2 dmax_step = 0.5 param_values = CArray.arange( start=0, step=dmax_step, stop=dmax + dmax_step) # set sec eval object self.sec_eval.append( CSecEval( attack=attack, param_name=param_name, param_values=param_values, ) )