def compute_images(self): """ Compute images through decoder. """ assert self.model.training is False num_batches = int(math.ceil(self.perturbations.shape[0]/self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_code = common.torch.as_variable(self.perturbation_codes[b_start: b_end], self.args.use_gpu) batch_theta = common.torch.as_variable(self.perturbation_theta[b_start: b_end], self.args.use_gpu) if isinstance(self.model, models.SelectiveDecoder): self.model.set_code(batch_code) theta_images = self.model(batch_theta) batch_perturbation = common.torch.as_variable(self.perturbations[b_start: b_end], self.args.use_gpu) perturbation_images = self.model(batch_perturbation) if b % 100: log('[Testing] %d' % b) theta_images = numpy.squeeze(numpy.transpose(theta_images.cpu().detach().numpy(), (0, 2, 3, 1))) self.theta_images = common.numpy.concatenate(self.theta_images, theta_images) perturbation_images = numpy.squeeze(numpy.transpose(perturbation_images.cpu().detach().numpy(), (0, 2, 3, 1))) self.perturbation_images = common.numpy.concatenate(self.perturbation_images, perturbation_images) self.theta_images = self.theta_images.reshape((self.theta_images.shape[0], -1)) self.perturbation_images = self.perturbation_images.reshape((self.perturbation_images.shape[0], -1))
def compute_images(self): """ Compute images through decoder. """ assert self.model.training is False num_batches = int( math.ceil(self.perturbations.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_fonts = self.perturbation_codes[b_start:b_end, 1] batch_classes = self.perturbation_codes[b_start:b_end, 2] batch_code = numpy.concatenate( (common.numpy.one_hot(batch_fonts, self.N_font), common.numpy.one_hot(batch_classes, self.N_class)), axis=1).astype(numpy.float32) batch_code = common.torch.as_variable(batch_code, self.args.use_gpu) batch_theta = common.torch.as_variable( self.perturbation_theta[b_start:b_end], self.args.use_gpu) theta_images = self.model(batch_code, batch_theta) batch_perturbation = common.torch.as_variable( self.perturbations[b_start:b_end], self.args.use_gpu) perturbation_images = self.model(batch_code, batch_perturbation) if b % 100: log('[Testing] %d' % b) theta_images = numpy.squeeze( numpy.transpose(theta_images.cpu().detach().numpy(), (0, 2, 3, 1))) self.theta_images = common.numpy.concatenate( self.theta_images, theta_images) perturbation_images = numpy.squeeze( numpy.transpose(perturbation_images.cpu().detach().numpy(), (0, 2, 3, 1))) self.perturbation_images = common.numpy.concatenate( self.perturbation_images, perturbation_images) self.theta_images = self.theta_images.reshape( (self.theta_images.shape[0], -1)) self.perturbation_images = self.perturbation_images.reshape( (self.perturbation_images.shape[0], -1))
def compute_images(self): """ Compute images. """ assert self.test_codes is not None num_batches = int( math.ceil(self.perturbations.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_codes = common.torch.as_variable( self.test_codes[b_start:b_end], self.args.use_gpu) if isinstance(self.model, models.SelectiveDecoder): self.model.set_code(batch_codes) batch_perturbation = common.torch.as_variable( self.perturbations[b_start:b_end].astype(numpy.float32), self.args.use_gpu) perturbation_images = self.model(batch_perturbation) if b % 100: log('[Detection] %d' % b) perturbation_images = numpy.squeeze( perturbation_images.cpu().detach().numpy()) self.perturbation_images = common.numpy.concatenate( self.perturbation_images, perturbation_images) # Trick to perform analysis on actual images of adversarial examples. self.perturbations = self.perturbation_images
def test_random(self): """ Test random. """ pred_images = None codes = numpy.random.normal( 0, 1, (1000, self.args.latent_space_size)).astype(numpy.float32) num_batches = int(math.ceil(codes.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) batch_codes = common.torch.as_variable(codes[b_start:b_end], self.args.use_gpu) # To get the correct images! output_images = self.decoder(batch_codes) output_images = numpy.squeeze( numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) pred_images = common.numpy.concatenate(pred_images, output_images) if b % 100 == 50: log('[Testing] %d' % b) utils.write_hdf5(self.args.random_file, pred_images) log('[Testing] wrote %s' % self.args.random_file)
def compute_images(self): """ Compute images through decoder. """ num_batches = int(math.ceil(self.perturbations.shape[0]/self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_images = common.torch.as_variable(self.test_images[b_start: b_end], self.args.use_gpu) self.model.set_image(batch_images) batch_perturbation = common.torch.as_variable(self.perturbations[b_start: b_end], self.args.use_gpu) perturbation_images = self.model(batch_perturbation) if b % 100: log('[Testing] %d' % b) perturbation_images = numpy.squeeze(numpy.transpose(perturbation_images.cpu().detach().numpy(), (0, 2, 3, 1))) self.perturbation_images = common.numpy.concatenate(self.perturbation_images, perturbation_images) self.test_images = self.test_images.reshape((self.test_images.shape[0], -1)) self.perturbation_images = self.perturbation_images.reshape((self.perturbation_images.shape[0], -1))
def load_data(self): """ Load data and model. """ self.test_images = utils.read_hdf5(self.args.test_images_file).astype(numpy.float32) log('[Testing] read %s' % self.args.test_images_file) # For handling both color and gray images. if len(self.test_images.shape) < 4: self.test_images = numpy.expand_dims(self.test_images, axis=3) log('[Testing] no color images, adjusted size') self.resolution = self.test_images.shape[2] log('[Testing] resolution %d' % self.resolution) self.train_images = utils.read_hdf5(self.args.train_images_file).astype(numpy.float32) # ! self.train_images = self.train_images.reshape((self.train_images.shape[0], -1)) log('[Testing] read %s' % self.args.train_images_file) self.test_codes = utils.read_hdf5(self.args.test_codes_file).astype(numpy.int) self.test_codes = self.test_codes[:, self.args.label_index] self.N_class = numpy.max(self.test_codes) + 1 log('[Testing] read %s' % self.args.test_codes_file) self.accuracy = utils.read_hdf5(self.args.accuracy_file) log('[Testing] read %s' % self.args.accuracy_file) self.perturbations = utils.read_hdf5(self.args.perturbations_file).astype(numpy.float32) self.N_attempts = self.perturbations.shape[0] # First, repeat relevant data. self.test_images = numpy.repeat(self.test_images[:self.perturbations.shape[1]], self.N_attempts, axis=0) self.perturbation_codes = numpy.repeat(self.test_codes[:self.perturbations.shape[1]], self.N_attempts, axis=0) self.perturbation_codes = numpy.squeeze(self.perturbation_codes) self.accuracy = numpy.repeat(self.accuracy[:self.perturbations.shape[1]], self.N_attempts, axis=0) # Then, reshape the perturbations! self.perturbations = numpy.swapaxes(self.perturbations, 0, 1) self.perturbations = self.perturbations.reshape((self.perturbations.shape[0] * self.perturbations.shape[1], -1)) assert self.perturbations.shape[1] == self.args.N_theta log('[Testing] read %s' % self.args.perturbations_file) assert not numpy.any(self.perturbations != self.perturbations), 'NaN in perturbations' self.success = utils.read_hdf5(self.args.success_file) self.success = numpy.swapaxes(self.success, 0, 1) self.success = self.success.reshape((self.success.shape[0] * self.success.shape[1])) log('[Testing] read %s' % self.args.success_file) log('[Testing] using %d input channels' % self.test_images.shape[3]) assert self.args.N_theta > 0 and self.args.N_theta <= 9 decoder = models.STNDecoder(self.args.N_theta) # decoder.eval() log('[Testing] set up STN decoder') self.model = decoder
def test(self): """ Test classifier to identify valid samples to attack. """ num_batches = int( math.ceil(self.perturbations.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_fonts = self.test_fonts[b_start:b_end] batch_classes = self.test_classes[b_start:b_end] batch_code = numpy.concatenate( (common.numpy.one_hot(batch_fonts, self.N_font), common.numpy.one_hot(batch_classes, self.N_class)), axis=1).astype(numpy.float32) batch_inputs = common.torch.as_variable( self.perturbations[b_start:b_end], self.args.use_gpu) batch_code = common.torch.as_variable(batch_code, self.args.use_gpu) # This basically allows to only optimize over theta, keeping the font/class code fixed. self.model.set_code(batch_code) output_images = self.model(batch_inputs) output_images = numpy.squeeze( numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) self.perturbation_images = common.numpy.concatenate( self.perturbation_images, output_images) if b % 100 == 0: log('[Testing] computing perturbation images %d' % b) utils.makedir(os.path.dirname(self.args.perturbation_images_file)) if len(self.perturbation_images.shape) > 3: self.perturbation_images = self.perturbation_images.reshape( self.N_samples, self.N_attempts, self.perturbation_images.shape[1], self.perturbation_images.shape[2], self.perturbation_images.shape[3]) else: self.perturbation_images = self.perturbation_images.reshape( self.N_samples, self.N_attempts, self.perturbation_images.shape[1], self.perturbation_images.shape[2]) self.perturbation_images = numpy.swapaxes(self.perturbation_images, 0, 1) utils.write_hdf5(self.args.perturbation_images_file, self.perturbation_images) log('[Testing] wrote %s' % self.args.perturbation_images_file)
def test(self): """ Test classifier to identify valid samples to attack. """ num_batches = int( math.ceil(self.perturbations.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_images = common.torch.as_variable( self.test_images[b_start:b_end], self.args.use_gpu) batch_inputs = common.torch.as_variable( self.perturbations[b_start:b_end], self.args.use_gpu) self.model.set_image(batch_images) output_images = self.model(batch_inputs) output_images = numpy.squeeze( numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) self.perturbation_images = common.numpy.concatenate( self.perturbation_images, output_images) if b % 100 == 0: log('[Testing] computing perturbation images %d' % b) utils.makedir(os.path.dirname(self.args.perturbation_images_file)) if len(self.perturbation_images.shape) > 3: self.perturbation_images = self.perturbation_images.reshape( self.N_samples, self.N_attempts, self.perturbation_images.shape[1], self.perturbation_images.shape[2], self.perturbation_images.shape[3]) else: self.perturbation_images = self.perturbation_images.reshape( self.N_samples, self.N_attempts, self.perturbation_images.shape[1], self.perturbation_images.shape[2]) self.perturbation_images = numpy.swapaxes(self.perturbation_images, 0, 1) utils.write_hdf5(self.args.perturbation_images_file, self.perturbation_images) log('[Testing] wrote %s' % self.args.perturbation_images_file)
def sample(self): """ Test the model. """ assert self.decoder is not None self.decoder.eval() log('[Sampling] set decoder to eval') images = None theta = common.numpy.truncated_normal( (self.args.N_samples, self.args.latent_space_size), lower=-self.args.bound, upper=self.args.bound).astype(numpy.float32) theta = theta.astype(numpy.float32) num_batches = int(math.ceil(theta.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, theta.shape[0]) batch_theta = common.torch.as_variable(theta[b_start:b_end], self.args.use_gpu) # Important to get the correct codes! assert self.decoder.training is False output_images = self.decoder(batch_theta) output_images = numpy.squeeze( numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) images = common.numpy.concatenate(images, output_images) if b % 100 == 50: log('[Sampling] %d' % b) if self.args.images_file: utils.write_hdf5(self.args.images_file, images) log('[Sampling] wrote %s' % self.args.images_file) if self.args.theta_file: utils.write_hdf5(self.args.theta_file, theta) log('[Sampling] wrote %s' % self.args.theta_file)
def test_interpolation(self): """ Test interpolation. """ interpolations = None perm = numpy.random.permutation( numpy.array(range(self.pred_codes.shape[0]))) for i in range(50): first = self.pred_codes[i] second = self.pred_codes[perm[i]] linfit = scipy.interpolate.interp1d([0, 1], numpy.vstack([first, second]), axis=0) interpolations = common.numpy.concatenate( interpolations, linfit(numpy.linspace(0, 1, 10))) pred_images = None num_batches = int( math.ceil(interpolations.shape[0] / self.args.batch_size)) interpolations = interpolations.astype(numpy.float32) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) batch_codes = common.torch.as_variable( interpolations[b_start:b_end], self.args.use_gpu) # To get the correct images! output_images = self.decoder(batch_codes) output_images = numpy.squeeze( numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) pred_images = common.numpy.concatenate(pred_images, output_images) if b % 100 == 50: log('[Testing] %d' % b) utils.write_hdf5(self.args.interpolation_file, pred_images) log('[Testing] wrote %s' % self.args.interpolation_file)
def compute_images(self): """ Compute images. """ assert self.test_codes is not None num_batches = int( math.ceil(self.perturbations.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_fonts = self.test_codes[b_start:b_end, 1] batch_classes = self.test_codes[b_start:b_end, 2] batch_code = numpy.concatenate( (common.numpy.one_hot(batch_fonts, self.N_font), common.numpy.one_hot(batch_classes, self.N_class)), axis=1).astype(numpy.float32) batch_code = common.torch.as_variable(batch_code, self.args.use_gpu) batch_perturbation = common.torch.as_variable( self.perturbations[b_start:b_end].astype(numpy.float32), self.args.use_gpu) perturbation_images = self.model(batch_code, batch_perturbation) if b % 100: log('[Testing] %d' % b) perturbation_images = numpy.squeeze( perturbation_images.cpu().detach().numpy()) self.perturbation_images = common.numpy.concatenate( self.perturbation_images, perturbation_images) # Trick to perform analysis on actual images of adversarial examples. self.perturbations = self.perturbation_images
def load_data(self): """ Load data and model. """ self.test_images = utils.read_hdf5(self.args.test_images_file).astype(numpy.float32) log('[Testing] read %s' % self.args.test_images_file) # For handling both color and gray images. if len(self.test_images.shape) < 4: self.test_images = numpy.expand_dims(self.test_images, axis=3) log('[Testing] no color images, adjusted size') self.resolution = self.test_images.shape[2] log('[Testing] resolution %d' % self.resolution) self.train_images = utils.read_hdf5(self.args.train_images_file).astype(numpy.float32) # ! self.train_images = self.train_images.reshape((self.train_images.shape[0], -1)) log('[Testing] read %s' % self.args.train_images_file) self.test_theta = utils.read_hdf5(self.args.test_theta_file).astype(numpy.float32) log('[Testing] read %s' % self.args.test_theta_file) self.train_theta = utils.read_hdf5(self.args.train_theta_file).astype(numpy.float32) log('[Testing] read %s' % self.args.train_theta_file) self.test_codes = utils.read_hdf5(self.args.test_codes_file).astype(numpy.int) self.test_codes = self.test_codes[:, self.args.label_index] self.N_class = numpy.max(self.test_codes) + 1 log('[Testing] read %s' % self.args.test_codes_file) self.accuracy = utils.read_hdf5(self.args.accuracy_file) log('[Testing] read %s' % self.args.accuracy_file) self.perturbations = utils.read_hdf5(self.args.perturbations_file).astype(numpy.float32) self.N_attempts = self.perturbations.shape[0] assert not numpy.any(self.perturbations != self.perturbations), 'NaN in perturbations' # First, repeat relevant data. self.perturbation_theta = numpy.repeat(self.test_theta[:self.perturbations.shape[1]], self.N_attempts, axis=0) self.perturbation_codes = numpy.repeat(self.test_codes[:self.perturbations.shape[1]], self.N_attempts, axis=0) self.perturbation_codes = numpy.squeeze(self.perturbation_codes) self.accuracy = numpy.repeat(self.accuracy[:self.perturbations.shape[1]], self.N_attempts, axis=0) # Then, reshape the perturbations! self.perturbations = numpy.swapaxes(self.perturbations, 0, 1) self.perturbations = self.perturbations.reshape((self.perturbations.shape[0] * self.perturbations.shape[1], -1)) log('[Testing] read %s' % self.args.perturbations_file) self.success = utils.read_hdf5(self.args.success_file) self.success = numpy.swapaxes(self.success, 0, 1) self.success = self.success.reshape((self.success.shape[0] * self.success.shape[1])) log('[Testing] read %s' % self.args.success_file) assert self.args.decoder_files decoder_files = self.args.decoder_files.split(',') for decoder_file in decoder_files: assert os.path.exists(decoder_file), 'could not find %s' % decoder_file log('[Testing] using %d input channels' % self.test_images.shape[3]) decoder_units = list(map(int, self.args.decoder_units.split(','))) if len(decoder_files) > 1: log('[Testing] loading multiple decoders') decoders = [] for i in range(len(decoder_files)): decoder = models.LearnedDecoder(self.args.latent_space_size, resolution=(self.test_images.shape[3], self.test_images.shape[1], self.test_images.shape[2]), architecture=self.args.decoder_architecture, start_channels=self.args.decoder_channels, activation=self.args.decoder_activation, batch_normalization=not self.args.decoder_no_batch_normalization, units=decoder_units) state = State.load(decoder_files[i]) decoder.load_state_dict(state.model) if self.args.use_gpu and not cuda.is_cuda(decoder): decoder = decoder.cuda() decoders.append(decoder) decoder.eval() log('[Testing] loaded %s' % decoder_files[i]) self.model = models.SelectiveDecoder(decoders, resolution=(self.test_images.shape[3], self.test_images.shape[1], self.test_images.shape[2])) else: log('[Testing] loading one decoder') decoder = models.LearnedDecoder(self.args.latent_space_size, resolution=(self.test_images.shape[3], self.test_images.shape[1], self.test_images.shape[2]), architecture=self.args.decoder_architecture, start_channels=self.args.decoder_channels, activation=self.args.decoder_activation, batch_normalization=not self.args.decoder_no_batch_normalization, units=decoder_units) state = State.load(decoder_files[0]) decoder.load_state_dict(state.model) if self.args.use_gpu and not cuda.is_cuda(decoder): decoder = decoder.cuda() decoder.eval() log('[Testing] read decoder') self.model = decoder
def compute_true(self): """ Compute true. """ assert self.test_codes is not None num_batches = int(math.ceil(self.perturbations.shape[0] / self.args.batch_size)) params = { 'lr': 0.09, 'lr_decay': 0.95, 'lr_min': 0.0000001, 'weight_decay': 0, } for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_fonts = self.test_codes[b_start: b_end, 1] batch_classes = self.test_codes[b_start: b_end, 2] batch_code = numpy.concatenate((common.numpy.one_hot(batch_fonts, self.N_font), common.numpy.one_hot(batch_classes, self.N_class)), axis=1).astype( numpy.float32) batch_code = common.torch.as_variable(batch_code, self.args.use_gpu) batch_images = common.torch.as_variable(self.test_images[b_start: b_end], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) batch_theta = common.torch.as_variable(self.test_theta[b_start: b_end].astype(numpy.float32), self.args.use_gpu, True) batch_perturbation = common.torch.as_variable(self.perturbations[b_start: b_end].astype(numpy.float32), self.args.use_gpu) self.model.set_code(batch_code) #output_images = self.model.forward(batch_theta) #test_error = torch.mean(torch.mul(output_images - batch_images, output_images - batch_images)) #print(test_error.item()) #vis.mosaic('true.png', batch_images.cpu().detach().numpy()[:, 0, :, :]) #vis.mosaic('output.png', output_images.cpu().detach().numpy()[:, 0, :, :]) # print(batch_images.cpu().detach().numpy()[0]) # print(output_images.cpu().detach().numpy()[0, 0]) #_batch_images = batch_images.cpu().detach().numpy() #_output_images = output_images.cpu().detach().numpy()[:, 0, :, :] #test_error = numpy.max(numpy.abs(_batch_images.reshape(_batch_images.shape[0], -1) - _output_images.reshape(_output_images.shape[0], -1)), axis=1) #print(test_error) #test_error = numpy.mean(numpy.multiply(_batch_images - _output_images, _batch_images - _output_images), axis=1) #print(test_error) batch_theta = torch.nn.Parameter(batch_theta) scheduler = ADAMScheduler([batch_theta], **params) log('[Detection] %d: start' % b) for t in range(100): scheduler.update(t//10, float(t)/10) scheduler.optimizer.zero_grad() output_perturbation = self.model.forward(batch_theta) error = torch.mean(torch.mul(output_perturbation - batch_perturbation, output_perturbation - batch_perturbation)) test_error = torch.mean(torch.mul(output_perturbation - batch_images, output_perturbation - batch_images)) #error.backward() #scheduler.optimizer.step() log('[Detection] %d: %d = %g, %g' % (b, t, error.item(), test_error.item())) output_perturbation = numpy.squeeze(numpy.transpose(output_perturbation.cpu().detach().numpy(), (0, 2, 3, 1))) self.projected_perturbations = common.numpy.concatenate(self.projected_perturbations, output_perturbation) projected_perturbations = self.projected_perturbations.reshape((self.projected_perturbations.shape[0], -1)) perturbations = self.perturbations.reshape((self.perturbations.shape[0], -1)) success = numpy.logical_and(self.success >= 0, self.accuracy) log('[Detection] %d valid attacked samples' % numpy.sum(success)) self.distances['true'] = numpy.linalg.norm(perturbations - projected_perturbations, ord=2, axis=1) self.angles['true'] = numpy.rad2deg(common.numpy.angles(perturbations.T, projected_perturbations.T)) self.distances['true'] = self.distances['true'][success] self.angles['true'] = self.angles['true'][success] self.distances['test'] = numpy.zeros((numpy.sum(success))) self.angles['test'] = numpy.zeros((numpy.sum(success)))
def test_test(self): """ Test on testing set. """ num_batches = int( math.ceil(self.test_images.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) batch_images = common.torch.as_variable( self.test_images[b_start:b_end], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) # Important to get the correct codes! output_codes, output_logvar = self.encoder(batch_images) output_images = self.decoder(output_codes) e = self.reconstruction_loss(batch_images, output_images) self.reconstruction_error += e.data self.code_mean += torch.mean(output_codes).item() self.code_var += torch.var(output_codes).item() output_images = numpy.squeeze( numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) self.pred_images = common.numpy.concatenate( self.pred_images, output_images) output_codes = output_codes.cpu().detach().numpy() self.pred_codes = common.numpy.concatenate(self.pred_codes, output_codes) if b % 100 == 50: log('[Testing] %d' % b) assert self.pred_images.shape[0] == self.test_images.shape[ 0], 'computed invalid number of test images' if self.args.reconstruction_file: utils.write_hdf5(self.args.reconstruction_file, self.pred_images) log('[Testing] wrote %s' % self.args.reconstruction_file) if self.args.test_theta_file: assert self.pred_codes.shape[0] == self.test_images.shape[ 0], 'computed invalid number of test codes' utils.write_hdf5(self.args.test_theta_file, self.pred_codes) log('[Testing] wrote %s' % self.args.test_theta_file) threshold = 0.9 percentage = 0 # values = numpy.linalg.norm(pred_codes, ord=2, axis=1) values = numpy.max(numpy.abs(self.pred_codes), axis=1) while percentage < 0.9: threshold += 0.1 percentage = numpy.sum(values <= threshold) / float( values.shape[0]) log('[Testing] threshold %g percentage %g' % (threshold, percentage)) log('[Testing] taking threshold %g with percentage %g' % (threshold, percentage)) if self.args.output_directory and utils.display(): # fit = 10 # plot_file = os.path.join(self.args.output_directory, 'test_codes') # plot.manifold(plot_file, pred_codes[::fit], None, None, 'tsne', None, title='t-SNE of Test Codes') # log('[Testing] wrote %s' % plot_file) for d in range(1, self.pred_codes.shape[1]): plot_file = os.path.join(self.args.output_directory, 'test_codes_%s' % d) plot.scatter( plot_file, self.pred_codes[:, 0], self.pred_codes[:, d], (values <= threshold).astype(int), ['greater %g' % threshold, 'smaller %g' % threshold], title='Dimensions 0 and %d of Test Codes' % d) log('[Testing] wrote %s' % plot_file) self.reconstruction_error /= num_batches log('[Testing] reconstruction error %g' % self.reconstruction_error)
def test(self, epoch): """ Test the model. :param epoch: current epoch :type epoch: int """ self.encoder.eval() log('[Training] %d set encoder to eval' % epoch) self.decoder.eval() log('[Training] %d set decoder to eval' % epoch) self.classifier.eval() log('[Training] %d set classifier to eval' % epoch) latent_loss = 0 reconstruction_loss = 0 reconstruction_error = 0 decoder_loss = 0 discriminator_loss = 0 mean = 0 var = 0 logvar = 0 pred_images = None pred_codes = None num_batches = int( math.ceil(self.test_images.shape[0] / self.args.batch_size)) assert self.encoder.training is False for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) batch_images = common.torch.as_variable( self.test_images[b_start:b_end], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) output_mu, output_logvar = self.encoder(batch_images) output_images = self.decoder(output_mu) output_real_classes = self.classifier(batch_images) output_reconstructed_classes = self.classifier(output_images) # Latent loss. e = self.latent_loss(output_mu, output_logvar) latent_loss += e.item() # Reconstruction loss. e = self.reconstruction_loss(batch_images, output_images) reconstruction_loss += e.item() # Reconstruction error. e = self.reconstruction_error(batch_images, output_images) reconstruction_error += e.item() e = self.decoder_loss(output_reconstructed_classes) decoder_loss += e.item() # Adversarial loss. e = self.discriminator_loss(output_real_classes, output_reconstructed_classes) discriminator_loss += e.item() mean += torch.mean(output_mu).item() var += torch.var(output_mu).item() logvar += torch.mean(output_logvar).item() output_images = numpy.squeeze( numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) pred_images = common.numpy.concatenate(pred_images, output_images) output_codes = output_mu.cpu().detach().numpy() pred_codes = common.numpy.concatenate(pred_codes, output_codes) utils.write_hdf5(self.args.reconstruction_file, pred_images) log('[Training] %d: wrote %s' % (epoch, self.args.reconstruction_file)) if utils.display(): png_file = self.args.reconstruction_file + '.%d.png' % epoch if epoch == 0: vis.mosaic(png_file, self.test_images[:225], 15, 5, 'gray', 0, 1) else: vis.mosaic(png_file, pred_images[:225], 15, 5, 'gray', 0, 1) log('[Training] %d: wrote %s' % (epoch, png_file)) latent_loss /= num_batches reconstruction_loss /= num_batches reconstruction_error /= num_batches decoder_loss /= num_batches discriminator_loss /= num_batches mean /= num_batches var /= num_batches logvar /= num_batches log('[Training] %d: test %g (%g) %g (%g, %g, %g)' % (epoch, reconstruction_loss, reconstruction_error, latent_loss, mean, var, logvar)) log('[Training] %d: test %g %g' % (epoch, decoder_loss, discriminator_loss)) num_batches = int( math.ceil(self.train_images.shape[0] / self.args.batch_size)) iteration = epoch * num_batches self.test_statistics = numpy.vstack( (self.test_statistics, numpy.array([ iteration, iteration * self.args.batch_size, min(num_batches, iteration), min(num_batches, iteration) * self.args.batch_size, reconstruction_loss, reconstruction_error, latent_loss, mean, var, logvar, decoder_loss, discriminator_loss ]))) pred_images = None if self.random_codes is None: self.random_codes = common.numpy.truncated_normal( (1000, self.args.latent_space_size)).astype(numpy.float32) num_batches = int( math.ceil(self.random_codes.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) if b_start >= b_end: break batch_codes = common.torch.as_variable( self.random_codes[b_start:b_end], self.args.use_gpu) output_images = self.decoder(batch_codes) output_images = numpy.squeeze( numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) pred_images = common.numpy.concatenate(pred_images, output_images) utils.write_hdf5(self.args.random_file, pred_images) log('[Training] %d: wrote %s' % (epoch, self.args.random_file)) if utils.display() and epoch > 0: png_file = self.args.random_file + '.%d.png' % epoch vis.mosaic(png_file, pred_images[:225], 15, 5, 'gray', 0, 1) log('[Training] %d: wrote %s' % (epoch, png_file)) interpolations = None perm = numpy.random.permutation(numpy.array(range( pred_codes.shape[0]))) for i in range(50): first = pred_codes[i] second = pred_codes[perm[i]] linfit = scipy.interpolate.interp1d([0, 1], numpy.vstack([first, second]), axis=0) interpolations = common.numpy.concatenate( interpolations, linfit(numpy.linspace(0, 1, 10))) pred_images = None num_batches = int( math.ceil(interpolations.shape[0] / self.args.batch_size)) interpolations = interpolations.astype(numpy.float32) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) if b_start >= b_end: break batch_codes = common.torch.as_variable( interpolations[b_start:b_end], self.args.use_gpu) output_images = self.decoder(batch_codes) output_images = numpy.squeeze( numpy.transpose(output_images.cpu().detach().numpy(), (0, 2, 3, 1))) pred_images = common.numpy.concatenate(pred_images, output_images) if b % 100 == 50: log('[Testing] %d' % b) utils.write_hdf5(self.args.interpolation_file, pred_images) log('[Testing] wrote %s' % self.args.interpolation_file) if utils.display() and epoch > 0: png_file = self.args.interpolation_file + '.%d.png' % epoch vis.mosaic(png_file, pred_images[:100], 10, 5, 'gray', 0, 1) log('[Training] %d: wrote %s' % (epoch, png_file))
def attack(self): """ Test the model. """ assert self.model is not None assert self.model.training is False assert self.test_images.shape[0] == self.test_codes.shape[0], 'number of samples has to match' concatenate_axis = -1 if os.path.exists(self.args.perturbations_file) and os.path.exists(self.args.success_file): self.original_perturbations = utils.read_hdf5(self.args.perturbations_file) if self.test_images.shape[3] > 1: assert len(self.original_perturbations.shape) == 5 else: assert len(self.original_perturbations.shape) == 4 log('[Attack] read %s' % self.args.perturbations_file) self.original_success = utils.read_hdf5(self.args.success_file) log('[Attack] read %s' % self.args.success_file) assert self.original_perturbations.shape[0] == self.original_success.shape[0] assert self.original_perturbations.shape[1] == self.original_success.shape[1] assert self.original_perturbations.shape[2] == self.test_images.shape[1] assert self.original_perturbations.shape[3] == self.test_images.shape[2]# if self.original_perturbations.shape[1] >= self.args.max_samples and self.original_perturbations.shape[0] >= self.args.max_attempts: log('[Attack] found %d attempts, %d samples, requested no more' % (self.original_perturbations.shape[0], self.original_perturbations.shape[1])) return elif self.original_perturbations.shape[0] == self.args.max_attempts or self.original_perturbations.shape[1] == self.args.max_samples: if self.original_perturbations.shape[0] == self.args.max_attempts: self.test_images = self.test_images[self.original_perturbations.shape[1]:] self.test_codes = self.test_codes[self.original_perturbations.shape[1]:] self.args.max_samples = self.args.max_samples - self.original_perturbations.shape[1] concatenate_axis = 1 log('[Attack] found %d attempts with %d perturbations, computing %d more perturbations' % (self.original_perturbations.shape[0], self.original_perturbations.shape[1], self.args.max_samples)) elif self.original_perturbations.shape[1] == self.args.max_samples: self.args.max_attempts = self.args.max_attempts - self.original_perturbations.shape[0] concatenate_axis = 0 log('[Attack] found %d attempts with %d perturbations, computing %d more attempts' % (self.original_perturbations.shape[0], self.original_perturbations.shape[1], self.args.max_attempts)) # can't squeeze here! if self.test_images.shape[3] > 1: self.perturbations = numpy.zeros((self.args.max_attempts, self.args.max_samples, self.test_images.shape[1], self.test_images.shape[2], self.test_images.shape[3])) else: self.perturbations = numpy.zeros((self.args.max_attempts, self.args.max_samples, self.test_images.shape[1], self.test_images.shape[2])) self.success = numpy.ones((self.args.max_attempts, self.args.max_samples), dtype=int) * -1 if self.args.attack.find('Batch') >= 0: batch_size = min(self.args.batch_size, self.args.max_samples) else: batch_size = 1 objective = self.objective_class() num_batches = int(math.ceil(self.args.max_samples/batch_size)) for i in range(num_batches): # self.test_images.shape[0] if i*batch_size == self.args.max_samples: break i_start = i*batch_size i_end = min((i+1)*batch_size, self.args.max_samples) batch_images = common.torch.as_variable(self.test_images[i_start: i_end], self.args.use_gpu) batch_classes = common.torch.as_variable(numpy.array(self.test_codes[i_start: i_end]), self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) t = 0 while t < self.args.max_attempts: attack = self.setup_attack(batch_images, batch_classes) success, perturbations, probabilities, norm, _ = attack.run(objective) assert not numpy.any(perturbations != perturbations), perturbations # Note that we save the perturbed image, not only the perturbation! self.perturbations[t][i_start: i_end] = numpy.squeeze(numpy.transpose(perturbations + batch_images.cpu().numpy(), (0, 2, 3, 1))) self.success[t][i_start: i_end] = success # IMPORTANT: The adversarial examples are not considering whether the classifier is # actually correct to start with. t += 1 log('[Attack] %d: completed' % i) if concatenate_axis >= 0: if self.perturbations.shape[0] == self.args.max_attempts: self.perturbations = numpy.concatenate((self.original_perturbations, self.perturbations), axis=concatenate_axis) self.success = numpy.concatenate((self.original_success, self.success), axis=concatenate_axis) log('[Attack] concatenated') utils.write_hdf5(self.args.perturbations_file, self.perturbations) log('[Attack] wrote %s' % self.args.perturbations_file) utils.write_hdf5(self.args.success_file, self.success) log('[Attack] wrote %s' % self.args.success_file)
def compute_appr(self): """ Compute approximate. """ assert self.test_codes is not None num_batches = int(math.ceil(self.perturbations.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_classes = common.torch.as_variable(self.test_codes[b_start: b_end], self.args.use_gpu) batch_theta = common.torch.as_variable(self.test_theta[b_start: b_end].astype(numpy.float32), self.args.use_gpu, True) batch_perturbation = common.torch.as_variable(self.perturbations[b_start: b_end].astype(numpy.float32), self.args.use_gpu) if isinstance(self.model, models.SelectiveDecoder): self.model.set_code(batch_classes) batch_theta = torch.nn.Parameter(batch_theta) optimizer = torch.optim.Adam([batch_theta], lr=0.1) log('[Detection] %d: start' % b) for t in range(100): optimizer.zero_grad() output_perturbation = self.model.forward(batch_theta) error = torch.mean(torch.mul(output_perturbation - batch_perturbation, output_perturbation - batch_perturbation)) error.backward() optimizer.step() log('[Detection] %d: %d = %g' % (b, t, error.item())) output_perturbation = numpy.squeeze(output_perturbation.cpu().detach().numpy()) self.projected_perturbations = common.numpy.concatenate(self.projected_perturbations, output_perturbation) batch_theta = common.torch.as_variable(self.test_theta[b_start: b_end].astype(numpy.float32), self.args.use_gpu, True) batch_images = common.torch.as_variable(self.test_images[b_start: b_end].astype(numpy.float32), self.args.use_gpu) batch_theta = torch.nn.Parameter(batch_theta) optimizer = torch.optim.Adam([batch_theta], lr=0.5) log('[Detection] %d: start' % b) for t in range(100): optimizer.zero_grad() output_images = self.model.forward(batch_theta) error = torch.mean(torch.mul(output_images - batch_images, output_images - batch_images)) error.backward() optimizer.step() log('[Detection] %d: %d = %g' % (b, t, error.item())) output_images = numpy.squeeze(output_images.cpu().detach().numpy()) self.projected_test_images = common.numpy.concatenate(self.projected_test_images, output_images) projected_perturbations = self.projected_perturbations.reshape((self.projected_perturbations.shape[0], -1)) projected_test_images = self.projected_test_images.reshape((self.projected_test_images.shape[0], -1)) perturbations = self.perturbations.reshape((self.perturbations.shape[0], -1)) test_images = self.test_images.reshape((self.test_images.shape[0], -1)) success = numpy.logical_and(self.success >= 0, self.accuracy) log('[Detection] %d valid attacked samples' % numpy.sum(success)) self.distances['true'] = numpy.linalg.norm(perturbations - projected_perturbations, ord=2, axis=1) self.angles['true'] = numpy.rad2deg(common.numpy.angles(perturbations.T, projected_perturbations.T)) self.distances['true'] = self.distances['true'][success] self.angles['true'] = self.angles['true'][success] self.distances['test'] = numpy.linalg.norm(test_images - projected_test_images, ord=2, axis=1) self.angles['test'] = numpy.rad2deg(common.numpy.angles(test_images.T, projected_test_images.T)) self.distances['test'] = self.distances['test'][success] self.angles['test'] = self.angles['test'][success]
def attack(self): """ Test the model. """ assert self.model is not None assert self.model.training is False if self.args.attack.find('Batch') >= 0: batch_size = min(self.args.batch_size, self.args.max_samples) else: batch_size = 1 objective = self.objective_class() num_batches = int(math.ceil(self.args.max_samples / batch_size)) # can't squeeze here! if self.test_images.shape[3] > 1: self.perturbations = numpy.zeros( (self.args.max_attempts, self.args.max_samples, self.test_images.shape[1], self.test_images.shape[2], self.test_images.shape[3])) else: self.perturbations = numpy.zeros( (self.args.max_attempts, self.args.max_samples, self.test_images.shape[1], self.test_images.shape[2])) self.success = numpy.ones( (self.args.max_attempts, self.args.max_samples), dtype=int) * -1 self.probabilities = numpy.zeros( (self.args.max_attempts, self.args.max_samples, self.N_class)) for i in range(num_batches): # self.test_images.shape[0] if i * batch_size == self.args.max_samples: break i_start = i * batch_size i_end = min((i + 1) * batch_size, self.args.max_samples) batch_images = numpy.random.randint(0, 255, size=[batch_size] + self.test_images.shape[1:]) batch_images = common.torch.as_variable(batch_images, self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) batch_classes = common.torch.as_variable( numpy.random.randint(0, self.N_class - 1, size=(batch_images.size(0))), self.args.use_gpu) t = 0 while t < self.args.max_attempts: attack = self.setup_attack(batch_images, batch_classes) success, perturbations, probabilities, norm, _ = attack.run( objective) assert not numpy.any( perturbations != perturbations), perturbations # Note that we save the perturbed image, not only the perturbation! self.perturbations[t][i_start:i_end] = numpy.squeeze( numpy.transpose(perturbations + batch_images.cpu().numpy(), (0, 2, 3, 1))) self.success[t][i_start:i_end] = success self.probabilities[t][i_start:i_end] = probabilities # IMPORTANT: The adversarial examples are not considering whether the classifier is # actually correct to start with. t += 1 log('[Attack] %d: completed' % i) utils.write_hdf5(self.args.perturbations_file, self.perturbations) log('[Attack] wrote %s' % self.args.perturbations_file) utils.write_hdf5(self.args.success_file, self.success) log('[Attack] wrote %s' % self.args.success_file) utils.write_hdf5(self.args.probabilities_file, self.probabilities) log('[Attack] wrote %s' % self.args.probabilities_file)