from common.scheduler import ADAMScheduler from common.state import State from common import cuda from common.timer import elapsed from common import paths import common.torch import common.numpy import scipy.interpolate import torch import numpy import argparse import math import functools if utils.display(): from common import plot from common import vis class TrainVariationalAutoEncoder: """ Train an encoder on transformed letters. """ def __init__(self, args=None): """ Initialize. :param args: optional arguments if not to use sys.argv :type args: [str]
def test(self, epoch): """ Test the model. :param epoch: current epoch :type epoch: int """ assert self.encoder is not None and self.decoder is not None assert self.scheduler is not None self.auto_encoder.eval() log('[Training] %d set auto encoder to eval' % epoch) self.encoder.eval() log('[Training] %d set encoder to eval' % epoch) self.decoder.eval() log('[Training] %d set decoder to eval' % epoch) reconstruction_loss = 0 reconstruction_error = 0 latent_loss = 0 mean = 0 var = 0 logvar = 0 pred_images = None pred_codes = None num_batches = int(math.ceil(self.test_images.shape[0]/self.args.batch_size)) assert self.encoder.training is False for b in range(num_batches): batch_images = common.torch.as_variable(self.test_images[b*self.args.batch_size: min((b + 1)*self.args.batch_size, self.test_images.shape[0])], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) output_images, output_mu, output_logvar = self.auto_encoder(batch_images) e = self.reconstruction_loss(batch_images, output_images) reconstruction_loss += e.item() e = self.reconstruction_error(batch_images, output_images) reconstruction_error += e.item() e = self.latent_loss(output_mu, output_logvar) latent_loss += e.item() mean += torch.mean(output_mu).item() var += torch.var(output_mu).item() logvar += torch.mean(output_logvar).item() output_images = numpy.squeeze(numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) pred_images = common.numpy.concatenate(pred_images, output_images) output_codes = output_mu.cpu().detach().numpy() pred_codes = common.numpy.concatenate(pred_codes, output_codes) utils.write_hdf5(self.args.reconstruction_file, pred_images) log('[Training] %d: wrote %s' % (epoch, self.args.reconstruction_file)) if utils.display(): png_file = self.args.reconstruction_file + '.%d.png' % epoch if epoch == 0: vis.mosaic(png_file, self.test_images[:225], 15, 5, 'gray', 0, 1) else: vis.mosaic(png_file, pred_images[:225], 15, 5, 'gray', 0, 1) log('[Training] %d: wrote %s' % (epoch, png_file)) reconstruction_loss /= num_batches reconstruction_error /= num_batches latent_loss /= num_batches mean /= num_batches var /= num_batches logvar /= num_batches log('[Training] %d: test %g (%g) %g %g %g %g' % (epoch, reconstruction_loss, reconstruction_error, latent_loss, mean, var, logvar)) num_batches = int(math.ceil(self.train_images.shape[0] / self.args.batch_size)) iteration = epoch * num_batches self.test_statistics = numpy.vstack((self.test_statistics, numpy.array([ iteration, iteration * self.args.batch_size, min(num_batches, iteration), min(num_batches, iteration) * self.args.batch_size, reconstruction_loss, reconstruction_error, latent_loss, mean, var, logvar ]))) pred_images = None codes = numpy.random.normal(0, 1, (1000, self.args.latent_space_size)).astype(numpy.float32) num_batches = int(math.ceil(codes.shape[0] / self.args.batch_size)) for b in range(num_batches): batch_codes = common.torch.as_variable(codes[b * self.args.batch_size: min((b + 1) * self.args.batch_size, codes.shape[0])], self.args.use_gpu) output_images = self.auto_encoder.decoder(batch_codes) output_images = numpy.squeeze(numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) pred_images = common.numpy.concatenate(pred_images, output_images) utils.write_hdf5(self.args.random_file, pred_images) log('[Training] %d: wrote %s' % (epoch, self.args.random_file)) if utils.display() and epoch > 0: png_file = self.args.random_file + '.%d.png' % epoch vis.mosaic(png_file, pred_images[:225], 15, 5, 'gray', 0, 1) log('[Training] %d: wrote %s' % (epoch, png_file)) interpolations = None perm = numpy.random.permutation(numpy.array(range(pred_codes.shape[0]))) for i in range(50): first = pred_codes[i] second = pred_codes[perm[i]] linfit = scipy.interpolate.interp1d([0, 1], numpy.vstack([first, second]), axis=0) interpolations = common.numpy.concatenate(interpolations, linfit(numpy.linspace(0, 1, 10))) pred_images = None num_batches = int(math.ceil(interpolations.shape[0] / self.args.batch_size)) interpolations = interpolations.astype(numpy.float32) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) if b_start >= b_end: break batch_codes = common.torch.as_variable(interpolations[b_start: b_end], self.args.use_gpu) output_images = self.decoder(batch_codes) output_images = numpy.squeeze(numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) pred_images = common.numpy.concatenate(pred_images, output_images) if b % 100 == 50: log('[Testing] %d' % b) utils.write_hdf5(self.args.interpolation_file, pred_images) log('[Testing] wrote %s' % self.args.interpolation_file) if utils.display() and epoch > 0: png_file = self.args.interpolation_file + '.%d.png' % epoch vis.mosaic(png_file, pred_images[:100], 10, 5, 'gray', 0, 1) log('[Training] %d: wrote %s' % (epoch, png_file))
def process(self, args, bg_color=0): """ Scale image to width 1024, convert to grayscale and than slice by tiles. It's possible to slice image with padding and each tile will contain pixels from surrounding tiles """ configure_backend(args) cnn = get_cnn(args) tile_size = cnn.tile_size img = cv2.imread(args.input_file, cv2.IMREAD_GRAYSCALE) assert img is not None, f'No file: {args.input_file}' h, w = img.shape if args.scale: width = args.scale height = int(width * h / w) else: width, height = w, h img = cv2.resize(img, dsize=(width, height), interpolation=cv2.INTER_AREA) output_img = np.zeros(img.shape) i = 0 j = 0 while tile_size * (i * 1) < (width + tile_size): while tile_size * (j + 1) < (height + tile_size): tile, orig_size = slice_tile(img, i, j, tile_size, args.padding, bg_color=bg_color) if not orig_size[0] or not orig_size[1]: j += 1 continue # convert to CNN format cnn_tile = cnn.input_img_to_cnn(tile, tile_size, args.padding) # process output print('processing tile {}, {}'.format(i, j)) # TODO: fix this, we should be able to batch processing out_arr = cnn.process_tile(cnn_tile) # convert to img format out_tile = cnn.cnn_output_to_img(out_arr, tile_size) output_img[j * tile_size:(j + 1) * tile_size, i * tile_size:(i + 1) * tile_size] = out_tile[:orig_size[0], :orig_size[1]] j += 1 i += 1 j = 0 cv2.imwrite(args.output_file, output_img) if args.display: display(output_img)
def main(self): """ Main. """ fonts = 1000 characters = 'ABCDEFGHIJ' transformations = 6 size = 28 suffix = 'Hard' N_train = 960000 min_scale = 0.75 max_scale = 1.15 min_rotation = -3 * math.pi / 6 max_rotation = 3 * math.pi / 6 min_translation = -0.2 max_translation = 0.2 min_shear = -0.5 max_shear = 0.5 multiplier = 112 batch_size = 100 GeneratePrototypes([ '-database_file=%s' % paths.database_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-font_number=%d' % fonts, '-size=%d' % size, '-characters=%s' % characters, ]).main() GenerateCodes([ '-database_file=%s' % paths.database_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-codes_file=%s' % paths.codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-theta_file=%s' % paths.theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-number_transformations=%d' % transformations, '-min_scale=%g' % min_scale, '-max_scale=%g' % max_scale, '-min_rotation=%g' % min_rotation, '-max_rotation=%g' % max_rotation, '-min_translation=%g' % min_translation, '-max_translation=%g' % max_translation, '-min_shear=%g' % min_shear, '-max_shear=%g' % max_shear, '-multiplier=%d' % multiplier ]).main() GenerateDataset([ '-database_file=%s' % paths.database_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-codes_file=%s' % paths.codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-theta_file=%s' % paths.theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-batch_size=%d' % batch_size, '-images_file=%s' % paths.images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix) ]).main() SplitDataset([ '-codes_file=%s' % paths.codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-theta_file=%s' % paths.theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-images_file=%s' % paths.images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-train_codes_file=%s' % paths.train_codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-test_codes_file=%s' % paths.test_codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-train_theta_file=%s' % paths.train_theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-test_theta_file=%s' % paths.test_theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-train_images_file=%s' % paths.train_images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-test_images_file=%s' % paths.test_images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-N_train=%d' % N_train ]).main() if utils.display(): from data.fonts.inspect_dataset import InspectDataset InspectDataset([ '-images_file=%s' % paths.images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-output_directory=%s' % paths.data_file('fonts/', '', characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix) ]).main() CheckDataset([ '-database_file=%s' % paths.database_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-codes_file=%s' % paths.codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-theta_file=%s' % paths.theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-images_file=%s' % paths.images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-train_codes_file=%s' % paths.train_codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-test_codes_file=%s' % paths.test_codes_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-train_theta_file=%s' % paths.train_theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-test_theta_file=%s' % paths.test_theta_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-train_images_file=%s' % paths.train_images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix), '-test_images_file=%s' % paths.test_images_file(characters=characters, fonts=fonts, transformations=transformations, size=size, suffix=suffix) ]).main()
def compute_latent_statistics(self): """ Compute latent statistics. """ N_class = numpy.max(self.test_codes) + 1 num_attempts = self.perturbations.shape[0] perturbations = numpy.swapaxes(self.perturbations, 0, 1) perturbations = perturbations.reshape( (perturbations.shape[0] * perturbations.shape[1], perturbations.shape[2])) success = numpy.swapaxes(self.success, 0, 1) success = success.reshape((success.shape[0] * success.shape[1])) accuracy = numpy.repeat(self.accuracy, num_attempts, axis=0) # Raw success is the base for all statistics, as we need to consider only these # attacks that are successful and where the classifier originally was correct. raw_overall_success = numpy.logical_and(success >= 0, accuracy) # For off-manifold attacks this should not happen, but save is save. if not numpy.any(raw_overall_success): for n in range(len(self.norms)): for type in [ 'raw_success', 'raw_iteration', 'raw_average', 'raw_latent' ]: self.results[n][type] = 0 for type in [ 'raw_class_success', 'raw_class_average', 'raw_class_latent' ]: self.results[n][type] = numpy.zeros((N_class)) if self.args.results_file: utils.write_pickle(self.args.results_file, self.results) log('[Testing] wrote %s' % self.args.results_file) log('[Testing] no successful attacks found, no plots') return perturbation_images = numpy.repeat(self.perturbation_images, num_attempts, axis=0) perturbation_codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0) # # Compute nearest neighbors for perturbations and test images, # to backproject them into the latent space. # Also compute the dot product betweenm perturbations and a local # plane approximation base don the three nearest neighbors. # log('[Testing] computing nearest neighbor ...') nearest_neighbors_indices = self.compute_nearest_neighbors( perturbation_images) nearest_neighbors = self.train_theta[nearest_neighbors_indices[:, 0]] perturbation_nearest_neighbor_indices = self.compute_nearest_neighbors( perturbations) perturbation_nearest_neighbor = self.train_theta[ perturbation_nearest_neighbor_indices[:, 0]] # Compute statistics over the perturbation with respect to the plane # defined by the three nearest neighbors of the corresponding test sample. if self.args.plot_directory and self.args.plot_manifolds and utils.display( ): pure_perturbations = perturbations[ raw_overall_success] - perturbation_images[raw_overall_success] pure_perturbations_norm = numpy.linalg.norm(pure_perturbations, ord=2, axis=1) for k in range(10): direction = perturbation_images[ raw_overall_success] - self.train_images[ nearest_neighbors_indices[:, k][raw_overall_success]] direction_norm = numpy.linalg.norm(direction, ord=2, axis=1) dot_products = numpy.einsum('ij,ij->i', direction, pure_perturbations) dot_product_norms = numpy.multiply(pure_perturbations_norm, direction_norm) dot_product_norms[dot_product_norms == 0] = 1 dot_products /= dot_product_norms dot_products = numpy.degrees(numpy.arccos(dot_products)) # matplotlib's hsitogram plots give weird error if there are NaN values, so simple check: if dot_products.shape[0] > 0 and not numpy.any( dot_products != dot_products): plot_file = os.path.join(self.args.plot_directory, 'dot_products_nn%d' % k) plot.histogram( plot_file, dot_products, 100, title= 'Dot Products Between Adversarial Perturbations and Direction to Nearest Neighbor %d' % k, xlabel='Dot Product (Between Normalized Vectors)', ylabel='Count') log('[Testing] wrote %s' % plot_file) # # We compute some simple statistics: # - raw success rate: fraction of successful attack without considering epsilon # - corrected success rate: fraction of successful attacks within epsilon-ball # - raw average perturbation: average distance to original samples (for successful attacks) # - corrected average perturbation: average distance to original samples for perturbations # within epsilon-ball (for successful attacks). # These statistics can also be computed per class. # And these statistics are computed with respect to three norms. if self.args.plot_directory and utils.display(): iterations = success[raw_overall_success] x = numpy.arange(numpy.max(iterations) + 1) y = numpy.bincount(iterations) plot_file = os.path.join(self.args.plot_directory, 'iterations') plot.bar(plot_file, x, y, title='Distribution of Iterations of Successful Attacks', xlabel='Number of Iterations', ylabel='Count') log('[Testing] wrote %s' % plot_file) for n in range(len(self.norms)): norm = self.norms[n] delta = numpy.linalg.norm(perturbation_images - perturbations, norm, axis=1) latent_delta = numpy.linalg.norm(nearest_neighbors - perturbation_nearest_neighbor, norm, axis=1) if self.args.plot_directory and utils.display(): plot_file = os.path.join(self.args.plot_directory, 'distances_l%g' % norm) plot.histogram( plot_file, delta[raw_overall_success], 50, title= 'Distribution of $L_{%g}$ Distances of Successful Attacks' % norm, xlabel='Distance', ylabel='Count') log('[Testing] wrote %s' % plot_file) #debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0] #debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0) #debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy) #log('[Testing] attacked model accuracy: %g' % debug_accuracy) #log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction) #log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction) N_accuracy = numpy.sum(accuracy) self.results[n]['raw_success'] = numpy.sum( raw_overall_success) / N_accuracy self.results[n]['raw_iteration'] = numpy.average( success[raw_overall_success]) self.results[n]['raw_average'] = numpy.average( delta[raw_overall_success]) if numpy.any( raw_overall_success) else 0 self.results[n]['raw_latent'] = numpy.average( latent_delta[raw_overall_success]) if numpy.any( raw_overall_success) else 0 raw_class_success = numpy.zeros( (N_class, perturbation_images.shape[0]), bool) self.results[n]['raw_class_success'] = numpy.zeros((N_class)) self.results[n]['raw_class_average'] = numpy.zeros((N_class)) self.results[n]['raw_class_latent'] = numpy.zeros((N_class)) for c in range(N_class): N_samples = numpy.sum( numpy.logical_and(accuracy, perturbation_codes == c)) if N_samples <= 0: continue raw_class_success[c] = numpy.logical_and( raw_overall_success, perturbation_codes == c) self.results[n]['raw_class_success'][c] = numpy.sum( raw_class_success[c]) / N_samples if numpy.any(raw_class_success[c]): self.results[n]['raw_class_average'][c] = numpy.average( delta[raw_class_success[c].astype(bool)]) if numpy.any(raw_class_success[c]): self.results[n]['raw_class_latent'][c] = numpy.average( latent_delta[raw_class_success[c].astype(bool)]) if self.args.results_file: utils.write_pickle(self.args.results_file, self.results) log('[Testing] wrote %s' % self.args.results_file)
def compute_statistics(self): """ Compute statistics based on distances. """ N_class = numpy.max(self.test_codes) + 1 num_attempts = self.perturbations.shape[0] perturbations = numpy.swapaxes(self.perturbations, 0, 1) perturbations = perturbations.reshape( (perturbations.shape[0] * perturbations.shape[1], perturbations.shape[2])) success = numpy.swapaxes(self.success, 0, 1) success = success.reshape((success.shape[0] * success.shape[1])) accuracy = numpy.repeat(self.accuracy, num_attempts, axis=0) # Raw success is the base for all statistics, as we need to consider only these # attacks that are successful and where the classifier originally was correct. raw_overall_success = numpy.logical_and(success >= 0, accuracy) log('[Testing] %d valid attacks' % numpy.sum(raw_overall_success)) # For off-manifold attacks this should not happen, but save is save. if not numpy.any(raw_overall_success): for n in range(len(self.norms)): for type in [ 'raw_success', 'raw_iteration', 'raw_average', 'raw_latent' ]: self.results[n][type] = 0 for type in [ 'raw_class_success', 'raw_class_average', 'raw_class_latent' ]: self.results[n][type] = numpy.zeros((N_class)) if self.args.results_file: utils.write_pickle(self.args.results_file, self.results) log('[Testing] wrote %s' % self.args.results_file) log('[Testing] no successful attacks found, no plots') return perturbation_images = numpy.repeat(self.perturbation_images, num_attempts, axis=0) perturbation_codes = numpy.repeat(self.perturbation_codes, num_attempts, axis=0) # # We compute some simple statistics: # - raw success rate: fraction of successful attack without considering epsilon # - corrected success rate: fraction of successful attacks within epsilon-ball # - raw average perturbation: average distance to original samples (for successful attacks) # - corrected average perturbation: average distance to original samples for perturbations # within epsilon-ball (for successful attacks). # These statistics can also be computed per class. # And these statistics are computed with respect to three norms. if self.args.plot_directory and utils.display(): iterations = success[raw_overall_success] x = numpy.arange(numpy.max(iterations) + 1) y = numpy.bincount(iterations) plot_file = os.path.join(self.args.plot_directory, 'iterations') plot.bar(plot_file, x, y, title='Distribution of Iterations of Successful Attacks', xlabel='Number of Iterations', ylabel='Count') log('[Testing] wrote %s' % plot_file) for n in range(len(self.norms)): norm = self.norms[n] delta = numpy.linalg.norm(perturbation_images - perturbations, norm, axis=1) if self.args.plot_directory and utils.display(): plot_file = os.path.join(self.args.plot_directory, 'distances_l%g' % norm) plot.histogram( plot_file, delta[raw_overall_success], 50, title= 'Distribution of $L_{%g}$ Distances of Successful Attacks' % norm, xlabel='Distance', ylabel='Count') log('[Testing] wrote %s' % plot_file) #debug_accuracy = numpy.sum(accuracy) / accuracy.shape[0] #debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(success >= 0) #debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(accuracy) #log('[Testing] attacked model accuracy: %g' % debug_accuracy) #log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction) #log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction) N_accuracy = numpy.sum(accuracy) self.results[n]['raw_success'] = numpy.sum( raw_overall_success) / N_accuracy self.results[n]['raw_iteration'] = numpy.average( success[raw_overall_success]) self.results[n]['raw_average'] = numpy.average( delta[raw_overall_success]) if numpy.any( raw_overall_success) else 0 self.results[n]['raw_latent'] = 0 raw_class_success = numpy.zeros( (N_class, perturbation_images.shape[0]), bool) self.results[n]['raw_class_success'] = numpy.zeros((N_class)) self.results[n]['raw_class_average'] = numpy.zeros((N_class)) self.results[n]['raw_class_latent'] = numpy.zeros((N_class)) for c in range(N_class): N_samples = numpy.sum( numpy.logical_and(accuracy, perturbation_codes == c)) if N_samples <= 0: continue raw_class_success[c] = numpy.logical_and( raw_overall_success, perturbation_codes == c) self.results[n]['raw_class_success'][c] = numpy.sum( raw_class_success[c]) / N_samples if numpy.any(raw_class_success[c]): self.results[n]['raw_class_average'][c] = numpy.average( delta[raw_class_success[c].astype(bool)]) if self.args.results_file: utils.write_pickle(self.args.results_file, self.results) log('[Testing] wrote %s' % self.args.results_file)
def compute_statistics(self): """ Compute statistics based on distances. """ # That's the basis for all computation as we only want to consider successful attacks # on test samples that were correctly classified. raw_overall_success = numpy.logical_and(self.success >= 0, self.accuracy) # Important check, for on-manifold attack this will happen if the manifold is small and the model very accurate! if not numpy.any(raw_overall_success): for n in range(len(self.norms)): for type in [ 'raw_success', 'raw_iteration', 'raw_average', 'raw_image' ]: self.results[n][type] = 0 for type in [ 'raw_class_success', 'raw_class_average', 'raw_class_image' ]: self.results[n][type] = numpy.zeros((self.N_class)) if self.args.results_file: utils.write_pickle(self.args.results_file, self.results) log('[Testing] wrote %s' % self.args.results_file) log('[Testing] no successful attacks found, no plots') return # # Compute nearest neighbor statistics in image space. # if self.args.plot_directory and self.args.plot_manifolds and utils.display( ): log('[Testing] computing nearest neighbor ...') nearest_neighbors_indices = self.compute_nearest_neighbors( self.perturbation_images[raw_overall_success]) pure_perturbations = self.theta_images[ raw_overall_success] - self.perturbation_images[ raw_overall_success] pure_perturbations_norm = numpy.linalg.norm(pure_perturbations, ord=2, axis=1) for k in range(10): direction = self.perturbation_images[ raw_overall_success] - self.train_images[ nearest_neighbors_indices[:, k]] direction_norm = numpy.linalg.norm(direction, ord=2, axis=1) dot_products = numpy.einsum('ij,ij->i', direction, pure_perturbations) dot_product_norms = numpy.multiply(pure_perturbations_norm, direction_norm) dot_products, dot_product_norms = dot_products[ dot_product_norms > 10**-8], dot_product_norms[ dot_product_norms > 10**-8] dot_products /= dot_product_norms dot_products = numpy.degrees(numpy.arccos(dot_products)) # matplotlib's hsitogram plots give weird error if there are NaN values, so simple check: if dot_products.shape[0] > 0 and not numpy.any( dot_products != dot_products): print(dot_products) plot_file = os.path.join(self.args.plot_directory, 'dot_products_nn%d' % k) plot.histogram( plot_file, dot_products, 100, xmin=numpy.min(dot_products), xmax=numpy.max(dot_products), title= 'Dot Products Between Adversarial Perturbations and Direction to Nearest Neighbor %d' % k, xlabel='Dot Product', ylabel='Count') log('[Testing] wrote %s' % plot_file) # # We compute some simple statistics: # - raw success rate: fraction of successful attack without considering epsilon # - corrected success rate: fraction of successful attacks within epsilon-ball # - raw average perturbation: average distance to original samples (for successful attacks) # - corrected average perturbation: average distance to original samples for perturbations # within epsilon-ball (for successful attacks). # These statistics can also be computed per class. # And these statistics are computed with respect to three norms. if self.args.plot_directory and utils.display(): iterations = self.success[raw_overall_success] x = numpy.arange(numpy.max(iterations) + 1) y = numpy.bincount(iterations) plot_file = os.path.join(self.args.plot_directory, 'iterations') plot.bar(plot_file, x, y, title='Distribution of Iterations of Successful Attacks', xlabel='Number of Iterations', ylabel='Count') log('[Testing] wrote %s' % plot_file) for n in range(len(self.norms)): norm = self.norms[n] delta = numpy.linalg.norm(self.perturbation_theta - self.perturbations, norm, axis=1) image_delta = numpy.linalg.norm(self.theta_images - self.perturbation_images, norm, axis=1) if self.args.plot_directory and utils.display(): plot_file = os.path.join(self.args.plot_directory, 'distances_l%g' % norm) plot.histogram( plot_file, delta[raw_overall_success], 50, title= 'Distribution of $L_{%g}$ Distances of Successful Attacks' % norm, xlabel='Distance', ylabel='Count') log('[Testing] wrote %s' % plot_file) #debug_accuracy = numpy.sum(self.accuracy) / self.accuracy.shape[0] #debug_attack_fraction = numpy.sum(raw_overall_success) / numpy.sum(self.success >= 0) #debug_test_fraction = numpy.sum(raw_overall_success) / numpy.sum(self.accuracy) #log('[Testing] attacked mode accuracy: %g' % debug_accuracy) #log('[Testing] only %g of successful attacks are valid' % debug_attack_fraction) #log('[Testing] only %g of correct samples are successfully attacked' % debug_test_fraction) N_accuracy = numpy.sum(self.accuracy) self.results[n]['raw_success'] = numpy.sum( raw_overall_success) / N_accuracy self.results[n]['raw_iteration'] = numpy.average( self.success[raw_overall_success]) self.results[n]['raw_average'] = numpy.average( delta[raw_overall_success]) if numpy.any( raw_overall_success) else 0 self.results[n]['raw_image'] = numpy.average( image_delta[raw_overall_success]) if numpy.any( raw_overall_success) else 0 raw_class_success = numpy.zeros( (self.N_class, self.perturbation_codes.shape[0]), bool) self.results[n]['raw_class_success'] = numpy.zeros((self.N_class)) self.results[n]['raw_class_average'] = numpy.zeros((self.N_class)) self.results[n]['raw_class_image'] = numpy.zeros((self.N_class)) for c in range(self.N_class): N_samples = numpy.sum( self.accuracy[self.perturbation_codes[:, 2] == c].astype(int)) if N_samples <= 0: continue raw_class_success[c] = numpy.logical_and( raw_overall_success, self.perturbation_codes[:, 2] == c) self.results[n]['raw_class_success'][c] = numpy.sum( raw_class_success[c]) / N_samples if numpy.any(raw_class_success[c]): self.results[n]['raw_class_average'][c] = numpy.average( delta[raw_class_success[c].astype(bool)]) if numpy.any(raw_class_success[c]): self.results[n]['raw_class_image'][c] = numpy.average( image_delta[raw_class_success[c].astype(bool)]) if self.args.results_file: utils.write_pickle(self.args.results_file, self.results) log('[Testing] wrote %s' % self.args.results_file)