from common.scheduler import ADAMScheduler
from common.state import State
from common import cuda
from common.timer import elapsed
from common import paths
import common.torch
import common.numpy
import scipy.interpolate
import torch
import numpy
import argparse
import math
import functools
if utils.display():
from common import plot
from common import vis
class TrainVariationalAutoEncoder:
"""
Train an encoder on transformed letters.
"""
def __init__(self, args=None):
"""
Initialize.
:param args: optional arguments if not to use sys.argv
:type args: [str]
def test(self, epoch):
"""
Test the model.
:param epoch: current epoch
:type epoch: int
"""
assert self.encoder is not None and self.decoder is not None
assert self.scheduler is not None
self.auto_encoder.eval()
log('[Training] %d set auto encoder to eval' % epoch)
self.encoder.eval()
log('[Training] %d set encoder to eval' % epoch)
self.decoder.eval()
log('[Training] %d set decoder to eval' % epoch)
reconstruction_loss = 0
reconstruction_error = 0
latent_loss = 0
mean = 0
var = 0
logvar = 0
pred_images = None
pred_codes = None
num_batches = int(math.ceil(self.test_images.shape[0]/self.args.batch_size))
assert self.encoder.training is False
for b in range(num_batches):
batch_images = common.torch.as_variable(self.test_images[b*self.args.batch_size: min((b + 1)*self.args.batch_size, self.test_images.shape[0])], self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
output_images, output_mu, output_logvar = self.auto_encoder(batch_images)
e = self.reconstruction_loss(batch_images, output_images)
reconstruction_loss += e.item()
e = self.reconstruction_error(batch_images, output_images)
reconstruction_error += e.item()
e = self.latent_loss(output_mu, output_logvar)
latent_loss += e.item()
mean += torch.mean(output_mu).item()
var += torch.var(output_mu).item()
logvar += torch.mean(output_logvar).item()
output_images = numpy.squeeze(numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1)))
pred_images = common.numpy.concatenate(pred_images, output_images)
output_codes = output_mu.cpu().detach().numpy()
pred_codes = common.numpy.concatenate(pred_codes, output_codes)
utils.write_hdf5(self.args.reconstruction_file, pred_images)
log('[Training] %d: wrote %s' % (epoch, self.args.reconstruction_file))
if utils.display():
png_file = self.args.reconstruction_file + '.%d.png' % epoch
if epoch == 0:
vis.mosaic(png_file, self.test_images[:225], 15, 5, 'gray', 0, 1)
else:
vis.mosaic(png_file, pred_images[:225], 15, 5, 'gray', 0, 1)
log('[Training] %d: wrote %s' % (epoch, png_file))
reconstruction_loss /= num_batches
reconstruction_error /= num_batches
latent_loss /= num_batches
mean /= num_batches
var /= num_batches
logvar /= num_batches
log('[Training] %d: test %g (%g) %g %g %g %g' % (epoch, reconstruction_loss, reconstruction_error, latent_loss, mean, var, logvar))
num_batches = int(math.ceil(self.train_images.shape[0] / self.args.batch_size))
iteration = epoch * num_batches
self.test_statistics = numpy.vstack((self.test_statistics, numpy.array([
iteration,
iteration * self.args.batch_size,
min(num_batches, iteration),
min(num_batches, iteration) * self.args.batch_size,
reconstruction_loss,
reconstruction_error,
latent_loss,
mean,
var,
logvar
])))
pred_images = None
codes = numpy.random.normal(0, 1, (1000, self.args.latent_space_size)).astype(numpy.float32)
num_batches = int(math.ceil(codes.shape[0] / self.args.batch_size))
for b in range(num_batches):
batch_codes = common.torch.as_variable(codes[b * self.args.batch_size: min((b + 1) * self.args.batch_size, codes.shape[0])], self.args.use_gpu)
output_images = self.auto_encoder.decoder(batch_codes)
output_images = numpy.squeeze(numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1)))
pred_images = common.numpy.concatenate(pred_images, output_images)
utils.write_hdf5(self.args.random_file, pred_images)
log('[Training] %d: wrote %s' % (epoch, self.args.random_file))
if utils.display() and epoch > 0:
png_file = self.args.random_file + '.%d.png' % epoch
vis.mosaic(png_file, pred_images[:225], 15, 5, 'gray', 0, 1)
log('[Training] %d: wrote %s' % (epoch, png_file))
interpolations = None
perm = numpy.random.permutation(numpy.array(range(pred_codes.shape[0])))
for i in range(50):
first = pred_codes[i]
second = pred_codes[perm[i]]
linfit = scipy.interpolate.interp1d([0, 1], numpy.vstack([first, second]), axis=0)
interpolations = common.numpy.concatenate(interpolations, linfit(numpy.linspace(0, 1, 10)))
pred_images = None
num_batches = int(math.ceil(interpolations.shape[0] / self.args.batch_size))
interpolations = interpolations.astype(numpy.float32)
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0])
if b_start >= b_end: break
batch_codes = common.torch.as_variable(interpolations[b_start: b_end], self.args.use_gpu)
output_images = self.decoder(batch_codes)
output_images = numpy.squeeze(numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1)))
pred_images = common.numpy.concatenate(pred_images, output_images)
if b % 100 == 50:
log('[Testing] %d' % b)
utils.write_hdf5(self.args.interpolation_file, pred_images)
log('[Testing] wrote %s' % self.args.interpolation_file)
if utils.display() and epoch > 0:
png_file = self.args.interpolation_file + '.%d.png' % epoch
vis.mosaic(png_file, pred_images[:100], 10, 5, 'gray', 0, 1)
log('[Training] %d: wrote %s' % (epoch, png_file))
def process(self, args, bg_color=0):
"""
Scale image to width 1024, convert to grayscale and than slice by tiles.
It's possible to slice image with padding and each tile will contain pixels from surrounding tiles
"""
configure_backend(args)
cnn = get_cnn(args)
tile_size = cnn.tile_size
img = cv2.imread(args.input_file, cv2.IMREAD_GRAYSCALE)
assert img is not None, f'No file: {args.input_file}'
h, w = img.shape
if args.scale:
width = args.scale
height = int(width * h / w)
else:
width, height = w, h
img = cv2.resize(img,
dsize=(width, height),
interpolation=cv2.INTER_AREA)
output_img = np.zeros(img.shape)
i = 0
j = 0
while tile_size * (i * 1) < (width + tile_size):
while tile_size * (j + 1) < (height + tile_size):
tile, orig_size = slice_tile(img,
i,
j,
tile_size,
args.padding,
bg_color=bg_color)
if not orig_size[0] or not orig_size[1]:
j += 1
continue
# convert to CNN format
cnn_tile = cnn.input_img_to_cnn(tile, tile_size, args.padding)
# process output
print('processing tile {}, {}'.format(i, j))
# TODO: fix this, we should be able to batch processing
out_arr = cnn.process_tile(cnn_tile)
# convert to img format
out_tile = cnn.cnn_output_to_img(out_arr, tile_size)
output_img[j * tile_size:(j + 1) * tile_size,
i * tile_size:(i + 1) *
tile_size] = out_tile[:orig_size[0], :orig_size[1]]
j += 1
i += 1
j = 0
cv2.imwrite(args.output_file, output_img)
if args.display:
display(output_img)
def main(self):
"""
Main.
"""
fonts = 1000
characters = 'ABCDEFGHIJ'
transformations = 6
size = 28
suffix = 'Hard'
N_train = 960000
min_scale = 0.75
max_scale = 1.15
min_rotation = -3 * math.pi / 6
max_rotation = 3 * math.pi / 6
min_translation = -0.2
max_translation = 0.2
min_shear = -0.5
max_shear = 0.5
multiplier = 112
batch_size = 100
GeneratePrototypes([
'-database_file=%s' % paths.database_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-font_number=%d' % fonts,
'-size=%d' % size,
'-characters=%s' % characters,
]).main()
GenerateCodes([
'-database_file=%s' % paths.database_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-codes_file=%s' % paths.codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-theta_file=%s' % paths.theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-number_transformations=%d' % transformations,
'-min_scale=%g' % min_scale,
'-max_scale=%g' % max_scale,
'-min_rotation=%g' % min_rotation,
'-max_rotation=%g' % max_rotation,
'-min_translation=%g' % min_translation,
'-max_translation=%g' % max_translation,
'-min_shear=%g' % min_shear,
'-max_shear=%g' % max_shear,
'-multiplier=%d' % multiplier
]).main()
GenerateDataset([
'-database_file=%s' % paths.database_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-codes_file=%s' % paths.codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-theta_file=%s' % paths.theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-batch_size=%d' % batch_size,
'-images_file=%s' % paths.images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix)
]).main()
SplitDataset([
'-codes_file=%s' % paths.codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-theta_file=%s' % paths.theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-images_file=%s' % paths.images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-train_codes_file=%s' % paths.train_codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-test_codes_file=%s' % paths.test_codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-train_theta_file=%s' % paths.train_theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-test_theta_file=%s' % paths.test_theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-train_images_file=%s' % paths.train_images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-test_images_file=%s' % paths.test_images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-N_train=%d' % N_train
]).main()
if utils.display():
from data.fonts.inspect_dataset import InspectDataset
InspectDataset([
'-images_file=%s' % paths.images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-output_directory=%s' % paths.data_file('fonts/', '', characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix)
]).main()
CheckDataset([
'-database_file=%s' % paths.database_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-codes_file=%s' % paths.codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-theta_file=%s' % paths.theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-images_file=%s' % paths.images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-train_codes_file=%s' % paths.train_codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-test_codes_file=%s' % paths.test_codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-train_theta_file=%s' % paths.train_theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-test_theta_file=%s' % paths.test_theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-train_images_file=%s' % paths.train_images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix),
'-test_images_file=%s' % paths.test_images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix)
]).main()
def compute_latent_statistics(self):
"""
Compute latent statistics.
"""
N_class = numpy.max(self.test_codes) + 1
num_attempts = self.perturbations.shape[0]
perturbations = numpy.swapaxes(self.perturbations, 0, 1)
perturbations = perturbations.reshape(
(perturbations.shape[0] * perturbations.shape[1],
perturbations.shape[2]))
success = numpy.swapaxes(self.success, 0, 1)
success = success.reshape((success.shape[0] * success.shape[1]))
accuracy = numpy.repeat(self.accuracy, num_attempts, axis=0)
# Raw success is the base for all statistics, as we need to consider only these
# attacks that are successful and where the classifier originally was correct.
raw_overall_success = numpy.logical_and(success >= 0, accuracy)
# For off-manifold attacks this should not happen, but save is save.
if not numpy.any(raw_overall_success):
for n in range(len(self.norms)):
for type in [
'raw_success', 'raw_iteration', 'raw_average',
'raw_latent'
]:
self.results[n][type] = 0
for type in [
'raw_class_success', 'raw_class_average',
'raw_class_latent'
]:
self.results[n][type] = numpy.zeros((N_class))
if self.args.results_file:
utils.write_pickle(self.args.results_file, self.results)
log('[Testing] wrote %s' % self.args.results_file)
log('[Testing] no successful attacks found, no plots')
return
perturbation_images = numpy.repeat(self.perturbation_images,
num_attempts,
axis=0)
perturbation_codes = numpy.repeat(self.perturbation_codes,
num_attempts,
axis=0)
#
# Compute nearest neighbors for perturbations and test images,
# to backproject them into the latent space.
# Also compute the dot product betweenm perturbations and a local
# plane approximation base don the three nearest neighbors.
#
log('[Testing] computing nearest neighbor ...')
nearest_neighbors_indices = self.compute_nearest_neighbors(
perturbation_images)
nearest_neighbors = self.train_theta[nearest_neighbors_indices[:, 0]]
perturbation_nearest_neighbor_indices = self.compute_nearest_neighbors(
perturbations)
perturbation_nearest_neighbor = self.train_theta[
perturbation_nearest_neighbor_indices[:, 0]]
# Compute statistics over the perturbation with respect to the plane
# defined by the three nearest neighbors of the corresponding test sample.
if self.args.plot_directory and self.args.plot_manifolds and utils.display(
):
pure_perturbations = perturbations[
raw_overall_success] - perturbation_images[raw_overall_success]
pure_perturbations_norm = numpy.linalg.norm(pure_perturbations,
ord=2,
axis=1)
for k in range(10):
direction = perturbation_images[
raw_overall_success] - self.train_images[
nearest_neighbors_indices[:, k][raw_overall_success]]
direction_norm = numpy.linalg.norm(direction, ord=2, axis=1)
dot_products = numpy.einsum('ij,ij->i', direction,
pure_perturbations)
dot_product_norms = numpy.multiply(pure_perturbations_norm,
direction_norm)
dot_product_norms[dot_product_norms == 0] = 1
dot_products /= dot_product_norms
dot_products = numpy.degrees(numpy.arccos(dot_products))
# matplotlib's hsitogram plots give weird error if there are NaN values, so simple check:
if dot_products.shape[0] > 0 and not numpy.any(
dot_products != dot_products):
plot_file = os.path.join(self.args.plot_directory,
'dot_products_nn%d' % k)
plot.histogram(
plot_file,
dot_products,
100,
title=
'Dot Products Between Adversarial Perturbations and Direction to Nearest Neighbor %d'
% k,
xlabel='Dot Product (Between Normalized Vectors)',
ylabel='Count')
log('[Testing] wrote %s' % plot_file)
#
# We compute some simple statistics:
# - raw success rate: fraction of successful attack without considering epsilon
# - corrected success rate: fraction of successful attacks within epsilon-ball
# - raw average perturbation: average distance to original samples (for successful attacks)
# - corrected average perturbation: average distance to original samples for perturbations
# within epsilon-ball (for successful attacks).
# These statistics can also be computed per class.
# And these statistics are computed with respect to three norms.
if self.args.plot_directory and utils.display():
iterations = success[raw_overall_success]
x = numpy.arange(numpy.max(iterations) + 1)
y = numpy.bincount(iterations)
plot_file = os.path.join(self.args.plot_directory, 'iterations')
plot.bar(plot_file,
x,
y,
title='Distribution of Iterations of Successful Attacks',
xlabel='Number of Iterations',
ylabel='Count')
log('[Testing] wrote %s' % plot_file)
for n in range(len(self.norms)):
norm = self.norms[n]
delta = numpy.linalg.norm(perturbation_images - perturbations,
norm,
axis=1)
latent_delta = numpy.linalg.norm(nearest_neighbors -
perturbation_nearest_neighbor,
norm,
axis=1)
if self.args.plot_directory and utils.display():
plot_file = os.path.join(self.args.plot_directory,
'distances_l%g' % norm)
plot.histogram(
plot_file,
delta[raw_overall_success],
50,
title=
'Distribution of $L_{%g}$ Distances of Successful Attacks'
% norm,
xlabel='Distance',
ylabel='Count')
log('[Testing] wrote %s' % plot_file)
#debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0]
#debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0)
#debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy)
#log('[Testing] attacked model accuracy: %g' % debug_accuracy)
#log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction)
#log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction)
N_accuracy = numpy.sum(accuracy)
self.results[n]['raw_success'] = numpy.sum(
raw_overall_success) / N_accuracy
self.results[n]['raw_iteration'] = numpy.average(
success[raw_overall_success])
self.results[n]['raw_average'] = numpy.average(
delta[raw_overall_success]) if numpy.any(
raw_overall_success) else 0
self.results[n]['raw_latent'] = numpy.average(
latent_delta[raw_overall_success]) if numpy.any(
raw_overall_success) else 0
raw_class_success = numpy.zeros(
(N_class, perturbation_images.shape[0]), bool)
self.results[n]['raw_class_success'] = numpy.zeros((N_class))
self.results[n]['raw_class_average'] = numpy.zeros((N_class))
self.results[n]['raw_class_latent'] = numpy.zeros((N_class))
for c in range(N_class):
N_samples = numpy.sum(
numpy.logical_and(accuracy, perturbation_codes == c))
if N_samples <= 0:
continue
raw_class_success[c] = numpy.logical_and(
raw_overall_success, perturbation_codes == c)
self.results[n]['raw_class_success'][c] = numpy.sum(
raw_class_success[c]) / N_samples
if numpy.any(raw_class_success[c]):
self.results[n]['raw_class_average'][c] = numpy.average(
delta[raw_class_success[c].astype(bool)])
if numpy.any(raw_class_success[c]):
self.results[n]['raw_class_latent'][c] = numpy.average(
latent_delta[raw_class_success[c].astype(bool)])
if self.args.results_file:
utils.write_pickle(self.args.results_file, self.results)
log('[Testing] wrote %s' % self.args.results_file)
def compute_statistics(self):
"""
Compute statistics based on distances.
"""
N_class = numpy.max(self.test_codes) + 1
num_attempts = self.perturbations.shape[0]
perturbations = numpy.swapaxes(self.perturbations, 0, 1)
perturbations = perturbations.reshape(
(perturbations.shape[0] * perturbations.shape[1],
perturbations.shape[2]))
success = numpy.swapaxes(self.success, 0, 1)
success = success.reshape((success.shape[0] * success.shape[1]))
accuracy = numpy.repeat(self.accuracy, num_attempts, axis=0)
# Raw success is the base for all statistics, as we need to consider only these
# attacks that are successful and where the classifier originally was correct.
raw_overall_success = numpy.logical_and(success >= 0, accuracy)
log('[Testing] %d valid attacks' % numpy.sum(raw_overall_success))
# For off-manifold attacks this should not happen, but save is save.
if not numpy.any(raw_overall_success):
for n in range(len(self.norms)):
for type in [
'raw_success', 'raw_iteration', 'raw_average',
'raw_latent'
]:
self.results[n][type] = 0
for type in [
'raw_class_success', 'raw_class_average',
'raw_class_latent'
]:
self.results[n][type] = numpy.zeros((N_class))
if self.args.results_file:
utils.write_pickle(self.args.results_file, self.results)
log('[Testing] wrote %s' % self.args.results_file)
log('[Testing] no successful attacks found, no plots')
return
perturbation_images = numpy.repeat(self.perturbation_images,
num_attempts,
axis=0)
perturbation_codes = numpy.repeat(self.perturbation_codes,
num_attempts,
axis=0)
#
# We compute some simple statistics:
# - raw success rate: fraction of successful attack without considering epsilon
# - corrected success rate: fraction of successful attacks within epsilon-ball
# - raw average perturbation: average distance to original samples (for successful attacks)
# - corrected average perturbation: average distance to original samples for perturbations
# within epsilon-ball (for successful attacks).
# These statistics can also be computed per class.
# And these statistics are computed with respect to three norms.
if self.args.plot_directory and utils.display():
iterations = success[raw_overall_success]
x = numpy.arange(numpy.max(iterations) + 1)
y = numpy.bincount(iterations)
plot_file = os.path.join(self.args.plot_directory, 'iterations')
plot.bar(plot_file,
x,
y,
title='Distribution of Iterations of Successful Attacks',
xlabel='Number of Iterations',
ylabel='Count')
log('[Testing] wrote %s' % plot_file)
for n in range(len(self.norms)):
norm = self.norms[n]
delta = numpy.linalg.norm(perturbation_images - perturbations,
norm,
axis=1)
if self.args.plot_directory and utils.display():
plot_file = os.path.join(self.args.plot_directory,
'distances_l%g' % norm)
plot.histogram(
plot_file,
delta[raw_overall_success],
50,
title=
'Distribution of $L_{%g}$ Distances of Successful Attacks'
% norm,
xlabel='Distance',
ylabel='Count')
log('[Testing] wrote %s' % plot_file)
#debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0]
#debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0)
#debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy)
#log('[Testing] attacked model accuracy: %g' % debug_accuracy)
#log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction)
#log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction)
N_accuracy = numpy.sum(accuracy)
self.results[n]['raw_success'] = numpy.sum(
raw_overall_success) / N_accuracy
self.results[n]['raw_iteration'] = numpy.average(
success[raw_overall_success])
self.results[n]['raw_average'] = numpy.average(
delta[raw_overall_success]) if numpy.any(
raw_overall_success) else 0
self.results[n]['raw_latent'] = 0
raw_class_success = numpy.zeros(
(N_class, perturbation_images.shape[0]), bool)
self.results[n]['raw_class_success'] = numpy.zeros((N_class))
self.results[n]['raw_class_average'] = numpy.zeros((N_class))
self.results[n]['raw_class_latent'] = numpy.zeros((N_class))
for c in range(N_class):
N_samples = numpy.sum(
numpy.logical_and(accuracy, perturbation_codes == c))
if N_samples <= 0:
continue
raw_class_success[c] = numpy.logical_and(
raw_overall_success, perturbation_codes == c)
self.results[n]['raw_class_success'][c] = numpy.sum(
raw_class_success[c]) / N_samples
if numpy.any(raw_class_success[c]):
self.results[n]['raw_class_average'][c] = numpy.average(
delta[raw_class_success[c].astype(bool)])
if self.args.results_file:
utils.write_pickle(self.args.results_file, self.results)
log('[Testing] wrote %s' % self.args.results_file)
def compute_statistics(self):
"""
Compute statistics based on distances.
"""
# That's the basis for all computation as we only want to consider successful attacks
# on test samples that were correctly classified.
raw_overall_success = numpy.logical_and(self.success >= 0,
self.accuracy)
# Important check, for on-manifold attack this will happen if the manifold is small and the model very accurate!
if not numpy.any(raw_overall_success):
for n in range(len(self.norms)):
for type in [
'raw_success', 'raw_iteration', 'raw_average',
'raw_image'
]:
self.results[n][type] = 0
for type in [
'raw_class_success', 'raw_class_average',
'raw_class_image'
]:
self.results[n][type] = numpy.zeros((self.N_class))
if self.args.results_file:
utils.write_pickle(self.args.results_file, self.results)
log('[Testing] wrote %s' % self.args.results_file)
log('[Testing] no successful attacks found, no plots')
return
#
# Compute nearest neighbor statistics in image space.
#
if self.args.plot_directory and self.args.plot_manifolds and utils.display(
):
log('[Testing] computing nearest neighbor ...')
nearest_neighbors_indices = self.compute_nearest_neighbors(
self.perturbation_images[raw_overall_success])
pure_perturbations = self.theta_images[
raw_overall_success] - self.perturbation_images[
raw_overall_success]
pure_perturbations_norm = numpy.linalg.norm(pure_perturbations,
ord=2,
axis=1)
for k in range(10):
direction = self.perturbation_images[
raw_overall_success] - self.train_images[
nearest_neighbors_indices[:, k]]
direction_norm = numpy.linalg.norm(direction, ord=2, axis=1)
dot_products = numpy.einsum('ij,ij->i', direction,
pure_perturbations)
dot_product_norms = numpy.multiply(pure_perturbations_norm,
direction_norm)
dot_products, dot_product_norms = dot_products[
dot_product_norms > 10**-8], dot_product_norms[
dot_product_norms > 10**-8]
dot_products /= dot_product_norms
dot_products = numpy.degrees(numpy.arccos(dot_products))
# matplotlib's hsitogram plots give weird error if there are NaN values, so simple check:
if dot_products.shape[0] > 0 and not numpy.any(
dot_products != dot_products):
print(dot_products)
plot_file = os.path.join(self.args.plot_directory,
'dot_products_nn%d' % k)
plot.histogram(
plot_file,
dot_products,
100,
xmin=numpy.min(dot_products),
xmax=numpy.max(dot_products),
title=
'Dot Products Between Adversarial Perturbations and Direction to Nearest Neighbor %d'
% k,
xlabel='Dot Product',
ylabel='Count')
log('[Testing] wrote %s' % plot_file)
#
# We compute some simple statistics:
# - raw success rate: fraction of successful attack without considering epsilon
# - corrected success rate: fraction of successful attacks within epsilon-ball
# - raw average perturbation: average distance to original samples (for successful attacks)
# - corrected average perturbation: average distance to original samples for perturbations
# within epsilon-ball (for successful attacks).
# These statistics can also be computed per class.
# And these statistics are computed with respect to three norms.
if self.args.plot_directory and utils.display():
iterations = self.success[raw_overall_success]
x = numpy.arange(numpy.max(iterations) + 1)
y = numpy.bincount(iterations)
plot_file = os.path.join(self.args.plot_directory, 'iterations')
plot.bar(plot_file,
x,
y,
title='Distribution of Iterations of Successful Attacks',
xlabel='Number of Iterations',
ylabel='Count')
log('[Testing] wrote %s' % plot_file)
for n in range(len(self.norms)):
norm = self.norms[n]
delta = numpy.linalg.norm(self.perturbation_theta -
self.perturbations,
norm,
axis=1)
image_delta = numpy.linalg.norm(self.theta_images -
self.perturbation_images,
norm,
axis=1)
if self.args.plot_directory and utils.display():
plot_file = os.path.join(self.args.plot_directory,
'distances_l%g' % norm)
plot.histogram(
plot_file,
delta[raw_overall_success],
50,
title=
'Distribution of $L_{%g}$ Distances of Successful Attacks'
% norm,
xlabel='Distance',
ylabel='Count')
log('[Testing] wrote %s' % plot_file)
#debug_accuracy = numpy.sum(self.accuracy) / self.accuracy.shape[0]
#debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(self.success >= 0)
#debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(self.accuracy)
#log('[Testing] attacked mode accuracy: %g' % debug_accuracy)
#log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction)
#log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction)
N_accuracy = numpy.sum(self.accuracy)
self.results[n]['raw_success'] = numpy.sum(
raw_overall_success) / N_accuracy
self.results[n]['raw_iteration'] = numpy.average(
self.success[raw_overall_success])
self.results[n]['raw_average'] = numpy.average(
delta[raw_overall_success]) if numpy.any(
raw_overall_success) else 0
self.results[n]['raw_image'] = numpy.average(
image_delta[raw_overall_success]) if numpy.any(
raw_overall_success) else 0
raw_class_success = numpy.zeros(
(self.N_class, self.perturbation_codes.shape[0]), bool)
self.results[n]['raw_class_success'] = numpy.zeros((self.N_class))
self.results[n]['raw_class_average'] = numpy.zeros((self.N_class))
self.results[n]['raw_class_image'] = numpy.zeros((self.N_class))
for c in range(self.N_class):
N_samples = numpy.sum(
self.accuracy[self.perturbation_codes[:,
2] == c].astype(int))
if N_samples <= 0:
continue
raw_class_success[c] = numpy.logical_and(
raw_overall_success, self.perturbation_codes[:, 2] == c)
self.results[n]['raw_class_success'][c] = numpy.sum(
raw_class_success[c]) / N_samples
if numpy.any(raw_class_success[c]):
self.results[n]['raw_class_average'][c] = numpy.average(
delta[raw_class_success[c].astype(bool)])
if numpy.any(raw_class_success[c]):
self.results[n]['raw_class_image'][c] = numpy.average(
image_delta[raw_class_success[c].astype(bool)])
if self.args.results_file:
utils.write_pickle(self.args.results_file, self.results)
log('[Testing] wrote %s' % self.args.results_file)
