def _forward(self, x):
"""Computes the decision function for each pattern in x.
For One-Vs-All (OVA) multiclass scheme,
this is the output of the `label`^th classifier.
Parameters
----------
x : CArray
Array with new patterns to classify, 2-Dimensional of shape
(n_patterns, n_features).
Returns
-------
score : CArray
Value of the decision function for each test pattern.
Dense flat array of shape (n_samples,) if y is not None,
otherwise a (n_samples, n_classes) array.
"""
# Getting predicted scores for classifier associated with y
scores = CArray.empty(shape=(x.shape[0], self.n_classes))
# Discriminant function is now called for each different class
res = parfor2(_forward_one_ova,
self.n_classes,
self.n_jobs, self, x,
self.verbose)
# Building results array
for i in range(self.n_classes):
scores[:, i] = CArray(res[i])
return scores
def test_empty(self):
"""Test for CArray.empty() classmethod."""
self.logger.info("Test for CArray.empty() classmethod.")
for shape in [1, (1, ), 2, (2, ), (1, 2), (2, 1), (2, 2)]:
for dtype in [None, float, int, bool]:
for sparse in [False, True]:
res = CArray.empty(shape=shape, dtype=dtype, sparse=sparse)
self.logger.info(
"CArray.empty(shape={:}, dtype={:}, sparse={:}):"
"\n{:}".format(shape, dtype, sparse, res))
self.assertIsInstance(res, CArray)
self.assertEqual(res.isdense, not sparse)
self.assertEqual(res.issparse, sparse)
if isinstance(shape, tuple):
if len(shape) == 1 and sparse is True:
# Sparse "vectors" have len(shape) == 2
self.assertEqual(res.shape, (1, shape[0]))
else:
self.assertEqual(res.shape, shape)
else:
if sparse is True:
self.assertEqual(res.shape, (1, shape))
else:
self.assertEqual(res.shape, (shape, ))
if dtype is None: # Default dtype is float
self.assertIsSubDtype(res.dtype, float)
else:
self.assertIsSubDtype(res.dtype, dtype)
def _objective_function_cw(self, x):
if self._stored_vars is not None and \
'const' in self._stored_vars:
stored_const = self._stored_vars['const'][0]
if self._x0.shape[0] == 1:
# use same const for all points
c_weight = stored_const.item()
else:
# each point has it own optimized const
c_weight = CArray.empty(shape=(len(stored_const)))
for i, c in enumerate(stored_const):
c_weight[i] = c
else:
self.logger.warning('Constant value not stored during run. Using '
'initial_const value. For computing the loss '
'with the actual value of const set '
'`store_var_list=["const"]` in '
'CAttackEvasionCleverhans.__init__().')
c_weight = self._clvrh_attack.initial_const
l2dist = ((self._x0 - x)**2).sum(axis=1).ravel()
z_labels, z_predicted = self.classifier.predict(
x, return_decision_function=True)
y_target = CArray.zeros(shape=(1, self._n_classes), dtype=np.float32)
# destination point label
if self.y_target is not None:
y_target[0, self.y_target] = 1
else: # indiscriminate attack
y_target[0, self._y0] = 1
z_target = (z_predicted * y_target).sum(axis=1).ravel()
z_other = ((z_predicted * (1 - y_target) +
(z_predicted.min(axis=1) - 1) * y_target)).max(axis=1)
z_other = z_other.ravel()
# The following differs from the exact definition given in Carlini
# and Wagner (2016). There (page 9, left column, last equation),
# the maximum is taken over Z_other - Z_ target (or Z_target - Z_other
# respectively) and -confidence. However, it doesn't seem that
# would have the desired effect (loss term is <= 0 if and only if
# the difference between the logit of the target and any other class
# differs by at least confidence). Hence the rearrangement here.
self.confidence = self._clvrh_attack.confidence
if self.y_target is not None:
# if targeted, optimize for making the target class most likely
loss = CArray.maximum(z_other - z_target + self.confidence,
CArray.zeros(x.shape[0]))
else:
# if untargeted, optimize for making any other class most likely
loss = CArray.maximum(z_target - z_other + self.confidence,
CArray.zeros(x.shape[0]))
return c_weight * loss + l2dist
def test_explain(self):
"""Unittest for explain method."""
i = 67
ds_i = self.ds[i, :]
x, y_true = ds_i.X, ds_i.Y.item()
self.logger.info("Explaining P{:} c{:}".format(i, y_true))
x_pred, x_score = self.clf.predict(x, return_decision_function=True)
self.logger.info("Predicted class {:}, scores:\n{:}".format(
x_pred.item(), x_score))
self.logger.info("Candidates: {:}".format(x_score.argsort()[::-1]))
fig = CFigure(height=1.5, width=12)
# Plotting original image
fig.subplot(1, self.ds.num_classes + 1, 1)
fig.sp.imshow(x.reshape((8, 8)), cmap='gray')
fig.sp.title("Origin c{:}".format(y_true))
fig.sp.yticks([])
fig.sp.xticks([])
attr = CArray.empty(shape=(self.ds.num_classes, x.size))
# Computing attributions
for c in self.ds.classes:
attr_c = self.explainer.explain(x, y=c)
attr[c, :] = attr_c
self.logger.info("Attributions class {:}:\n{:}".format(
c, attr_c.tolist()))
self.assertIsInstance(attr, CArray)
self.assertEqual(attr.shape, attr.shape)
th = max(abs(attr.min()), abs(attr.max()))
# Plotting attributions
for c in self.ds.classes:
fig.subplot(1, self.ds.num_classes + 1, 2 + c)
fig.sp.imshow(attr[c, :].reshape((8, 8)),
cmap='seismic',
vmin=-1 * th,
vmax=th)
fig.sp.title("Attr c{:}".format(c))
fig.sp.yticks([])
fig.sp.xticks([])
fig.tight_layout()
fig.show()
def test_explain(self):
"""Unittest for explain method."""
i = 67
ds_i = self.ds[i, :]
x, y_true = ds_i.X, ds_i.Y.item()
self.logger.info("Explaining P{:} c{:}".format(i, y_true))
x_pred, x_score = self.clf.predict(x, return_decision_function=True)
self.logger.info("Predicted class {:}, scores:\n{:}".format(
x_pred.item(), x_score))
self.logger.info("Candidates: {:}".format(x_score.argsort()[::-1]))
ref_img = None # Use default reference image
m = 100 # Number of steps
attr = CArray.empty(shape=(0, x.shape[1]), sparse=x.issparse)
for c in self.ds.classes: # Compute attributions for each class
a = self.explainer.explain(x, y=c, reference=ref_img, m=m)
attr = attr.append(a, axis=0)
self.assertIsInstance(attr, CArray)
self.assertEqual(attr.shape[1], x.shape[1])
self.assertEqual(attr.shape[0], self.ds.num_classes)
fig = CFigure(height=1.5, width=12)
# Plotting original image
fig.subplot(1, self.ds.num_classes + 1, 1)
fig.sp.imshow(x.reshape((8, 8)), cmap='gray')
fig.sp.title("Origin c{:}".format(y_true))
fig.sp.yticks([])
fig.sp.xticks([])
th = max(abs(attr.min()), abs(attr.max()))
# Plotting attributions
for c in self.ds.classes:
fig.subplot(1, self.ds.num_classes + 1, 2 + c)
fig.sp.imshow(attr[c, :].reshape((8, 8)),
cmap='seismic',
vmin=-1 * th,
vmax=th)
fig.sp.title("Attr c{:}".format(c))
fig.sp.yticks([])
fig.sp.xticks([])
fig.tight_layout()
fig.show()
def test_margin(self):
self.logger.info("Testing margin separation of SGD...")
# we create 50 separable points
dataset = CDLRandomBlobs(n_samples=50,
centers=2,
random_state=0,
cluster_std=0.60).load()
# fit the model
clf = CClassifierSGD(loss=CLossHinge(),
regularizer=CRegularizerL2(),
alpha=0.01,
max_iter=200,
random_state=0)
clf.fit(dataset.X, dataset.Y)
# plot the line, the points, and the nearest vectors to the plane
xx = CArray.linspace(-1, 5, 10)
yy = CArray.linspace(-1, 5, 10)
X1, X2 = np.meshgrid(xx.tondarray(), yy.tondarray())
Z = CArray.empty(X1.shape)
for (i, j), val in np.ndenumerate(X1):
x1 = val
x2 = X2[i, j]
Z[i, j] = clf.decision_function(CArray([x1, x2]), y=1)
levels = [-1.0, 0.0, 1.0]
linestyles = ['dashed', 'solid', 'dashed']
colors = 'k'
fig = CFigure(linewidth=1)
fig.sp.contour(X1, X2, Z, levels, colors=colors, linestyles=linestyles)
fig.sp.scatter(dataset.X[:, 0].ravel(),
dataset.X[:, 1].ravel(),
c=dataset.Y,
s=40)
fig.savefig(
fm.join(fm.abspath(__file__), 'figs',
'test_c_classifier_sgd2.pdf'))
def _test_evasion_multiclass(self, expected_x):
# EVASION
self.multiclass.verbose = 2
if self.normalizer is not None:
lb = self.normalizer.feature_range[0]
ub = self.normalizer.feature_range[1]
else:
lb = None
ub = None
dmax = 2
self.solver_params = {'eta': 1e-1, 'eta_min': 1.0}
eva = CAttackEvasionPGDLS(classifier=self.multiclass,
surrogate_classifier=self.multiclass,
surrogate_data=self.ds,
distance='l2',
dmax=dmax,
lb=lb,
ub=ub,
solver_params=self.solver_params,
y_target=self.y_target)
eva.verbose = 0 # 2
# Points from class 2 region
# p_idx = 0
# Points from class 1 region
# p_idx = 68
# Points from class 3 region
p_idx = 1 # Wrong classified point
# p_idx = 53 # Evasion goes up usually
# Points from class 0 region
# p_idx = 49 # Wrong classified point
# p_idx = 27 # Correctly classified point
x0 = self.ds.X[p_idx, :]
y0 = self.ds.Y[p_idx].item()
x_seq = CArray.empty((0, x0.shape[1]))
scores = CArray([])
f_seq = CArray([])
x = x0
for d_idx, d in enumerate(range(0, dmax + 1)):
self.logger.info("Evasion at dmax: " + str(d))
eva.dmax = d
x, f_opt = eva._run(x0=x0, y0=y0, x_init=x)
y_pred, score = self.multiclass.predict(
x, return_decision_function=True)
f_seq = f_seq.append(f_opt)
# not considering all iterations, just values at dmax
# for all iterations, you should bring eva.x_seq and eva.f_seq
x_seq = x_seq.append(x, axis=0)
s = score[:, y0 if self.y_target is None else self.y_target]
scores = scores.append(s)
self.logger.info("Predicted label after evasion: " + str(y_pred))
self.logger.info("Score after evasion: {:}".format(s))
self.logger.info("Objective function after evasion: {:}".format(f_opt))
# Compare optimal point with expected
self.assert_array_almost_equal(eva.x_opt.todense().ravel(),
expected_x,
decimal=4)
self._make_plots(x_seq, dmax, eva, x0, scores, f_seq)
def _make_plot(self, p_idx, eva, dmax):
if self.make_figures is False:
self.logger.debug("Skipping figures...")
return
x0 = self.ds.X[p_idx, :]
y0 = self.ds.Y[p_idx].item()
x_seq = CArray.empty((0, x0.shape[1]))
scores = CArray([])
f_seq = CArray([])
x = x0
for d_idx, d in enumerate(range(0, dmax + 1)):
self.logger.info("Evasion at dmax: " + str(d))
eva.dmax = d
x, f_opt = eva._run(x0=x0, y0=y0, x_init=x)
y_pred, score = self.multiclass.predict(
x, return_decision_function=True)
f_seq = f_seq.append(f_opt)
# not considering all iterations, just values at dmax
# for all iterations, you should bring eva.x_seq and eva.f_seq
x_seq = x_seq.append(x, axis=0)
s = score[:, y0 if self.y_target is None else self.y_target]
scores = scores.append(s)
self.logger.info("Predicted label after evasion: {:}".format(y_pred))
self.logger.info("Score after evasion: {:}".format(s))
self.logger.info("Objective function after evasion: {:}".format(f_opt))
fig = CFigure(height=9, width=10, markersize=6, fontsize=12)
# Get plot bounds, taking into account ds and evaded point path
bounds_x, bounds_y = self.ds.get_bounds()
min_x, max_x = bounds_x
min_y, max_y = bounds_y
min_x = min(min_x, x_seq[:, 0].min())
max_x = max(max_x, x_seq[:, 0].max())
min_y = min(min_y, x_seq[:, 1].min())
max_y = max(max_y, x_seq[:, 1].max())
ds_bounds = [(min_x, max_x), (min_y, max_y)]
# Plotting multiclass decision regions
fig.subplot(2, 2, 1)
fig = self._plot_decision_function(fig, plot_background=True)
fig.sp.plot_path(x_seq,
path_style='-',
start_style='o',
start_facecolor='w',
start_edgewidth=2,
final_style='o',
final_facecolor='k',
final_edgewidth=2)
# plot distance constraint
fig.sp.plot_fun(func=self._rescaled_distance,
multipoint=True,
plot_background=False,
n_grid_points=20,
levels_color='k',
grid_limits=ds_bounds,
levels=[0],
colorbar=False,
levels_linewidth=2.0,
levels_style=':',
alpha_levels=.4,
c=x0,
r=dmax)
fig.sp.grid(linestyle='--', alpha=.5, zorder=0)
# Plotting multiclass evasion objective function
fig.subplot(2, 2, 2)
fig = self._plot_decision_function(fig)
fig.sp.plot_fgrads(eva.objective_function_gradient,
grid_limits=ds_bounds,
n_grid_points=20,
color='k',
alpha=.5)
fig.sp.plot_path(x_seq,
path_style='-',
start_style='o',
start_facecolor='w',
start_edgewidth=2,
final_style='o',
final_facecolor='k',
final_edgewidth=2)
# plot distance constraint
fig.sp.plot_fun(func=self._rescaled_distance,
multipoint=True,
plot_background=False,
n_grid_points=20,
levels_color='w',
grid_limits=ds_bounds,
levels=[0],
colorbar=False,
levels_style=':',
levels_linewidth=2.0,
alpha_levels=.5,
c=x0,
r=dmax)
fig.sp.plot_fun(lambda z: eva.objective_function(z),
multipoint=True,
grid_limits=ds_bounds,
colorbar=False,
n_grid_points=20,
plot_levels=False)
fig.sp.grid(linestyle='--', alpha=.5, zorder=0)
fig.subplot(2, 2, 3)
if self.y_target is not None:
fig.sp.title("Classifier Score for Target Class (Targ. Evasion)")
else:
fig.sp.title("Classifier Score for True Class (Indiscr. Evasion)")
fig.sp.plot(scores)
fig.sp.grid()
fig.sp.xlim(0, dmax)
fig.sp.xlabel("dmax")
fig.subplot(2, 2, 4)
fig.sp.title("Objective Function")
fig.sp.plot(f_seq)
fig.sp.grid()
fig.sp.xlim(0, dmax)
fig.sp.xlabel("dmax")
fig.tight_layout()
k_name = self.kernel.class_type if self.kernel is not None else 'lin'
fig.savefig(
fm.join(
self.images_folder,
"pgd_ls_reject_threshold_{:}c_kernel-{:}_target-{:}.pdf".
format(self.ds.num_classes, k_name, self.y_target)))
