def test(self): """ Test classifier to identify valid samples to attack. """ num_batches = int(math.ceil(self.test_images.shape[0] / self.args.batch_size)) self.model.eval() assert self.model.training is False assert self.test_images.shape[0] == self.test_codes.shape[0], 'number of samples have to match' for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) batch_images = common.torch.as_variable(self.test_images[b_start: b_end], self.args.use_gpu) batch_classes = common.torch.as_variable(self.test_codes[b_start: b_end], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) output_classes = self.model(batch_images) values, indices = torch.max(torch.nn.functional.softmax(output_classes, dim=1), dim=1) errors = torch.abs(indices - batch_classes) self.accuracy = common.numpy.concatenate(self.accuracy, errors.data.cpu().numpy()) if b % 100 == 0: log('[Attack] computing accuracy %d' % b) self.accuracy = self.accuracy == 0 utils.write_hdf5(self.args.accuracy_file, self.accuracy) log('[Attack] wrote %s' % self.args.accuracy_file) accuracy = numpy.sum(self.accuracy)/float(self.accuracy.shape[0]) log('[Attack] accuracy %g' % accuracy) accuracy = numpy.sum(self.accuracy[:self.args.max_samples]) / float(self.args.max_samples) log('[Attack] accuracy on %d samples %g' % (self.args.max_samples, accuracy))
def __call__(self, images, perturbations): """ Projection. :param images: images :type images: torch.autograd.Variable :param perturbations: perturbations :type perturbations: torch.autograd.Variable """ if self.max_bound is not None: if isinstance(self.max_bound, torch.Tensor): # self.max_bound should be 1 x C x H x W for image or in general 1 x anything # so it can be repeated along the first dimension assert self.max_bound.size(0) == 1 assert common.torch.is_cuda(self.max_bound) == common.torch.is_cuda(perturbations) # workaround to have proper repeating repeat = [perturbations.size(0)] for i in range(1, len(list(self.max_bound.size()))): repeat.append(1) max_bound = self.max_bound.repeat(repeat).float() perturbations.data = torch.min(max_bound, perturbations.data) else: perturbations.data = torch.min(torch.ones_like(perturbations.data) * self.max_bound - images.data, perturbations.data) if self.min_bound is not None: if isinstance(self.min_bound, torch.Tensor): # self.max_bound should be 1 x C x H x W for image or in general 1 x anything # so it can be repeated along the first dimension assert self.min_bound.size(0) == 1 assert common.torch.is_cuda(self.min_bound) == common.torch.is_cuda(perturbations) # workaround to have proper repeating repeat = [perturbations.size(0)] for i in range(1, len(list(self.min_bound.size()))): repeat.append(1) min_bound = self.min_bound.repeat(repeat).float() perturbations.data = torch.max(min_bound, perturbations.data) else: perturbations.data = torch.max(torch.ones_like(perturbations.data)*self.min_bound - images.data, perturbations.data)
def loss(self, batch_classes, output_classes): """ Loss. :param batch_classes: predicted classes :type batch_classes: torch.autograd.Variable :param output_classes: target classes :type output_classes: torch.autograd.Variable :return: error :rtype: torch.autograd.Variable """ return torch.nn.functional.cross_entropy(output_classes, batch_classes, size_average=True) \ + self.args.logit_decay*torch.max(torch.sum(torch.abs(output_classes), dim=1))
def error(self, batch_classes, output_classes): """ Accuracy. :param batch_classes: predicted classes :type batch_classes: torch.autograd.Variable :param output_classes: target classes :type output_classes: torch.autograd.Variable :return: accuracy :rtype: torch.autograd.Variable """ values, indices = torch.max(torch.nn.functional.softmax(output_classes, dim=1), dim=1) errors = torch.abs(indices - batch_classes) return torch.sum(errors > 0).float() / batch_classes.size()[0]
def test(self): """ Test classifier to identify valid samples to attack. """ num_batches = int( math.ceil(self.test_theta.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_theta.shape[0]) batch_classes = common.torch.as_variable( self.test_codes[b_start:b_end], self.args.use_gpu) batch_inputs = common.torch.as_variable( self.test_theta[b_start:b_end], self.args.use_gpu) if isinstance(self.model.decoder, models.SelectiveDecoder): self.model.decoder.set_code(batch_classes) output_classes = self.model(batch_inputs) values, indices = torch.max(torch.nn.functional.softmax( output_classes, dim=1), dim=1) errors = torch.abs(indices - batch_classes) self.accuracy = common.numpy.concatenate(self.accuracy, errors.data.cpu().numpy()) if b % 100 == 0: log('[Attack] computing accuracy %d' % b) self.accuracy = self.accuracy == 0 utils.write_hdf5(self.args.accuracy_file, self.accuracy) log('[Attack] wrote %s' % self.args.accuracy_file) accuracy = numpy.sum(self.accuracy) / float(self.accuracy.shape[0]) log('[Attack] accuracy %g' % accuracy) accuracy = numpy.sum(self.accuracy[:self.args.max_samples]) / float( self.args.max_samples) log('[Attack] accuracy on %d samples %g' % (self.args.max_samples, accuracy))
def test(self): """ Test classifier to identify valid samples to attack. """ num_batches = int(math.ceil(self.test_theta.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_theta.shape[0]) batch_fonts = self.test_fonts[b_start: b_end] batch_classes = self.test_classes[b_start: b_end] batch_code = numpy.concatenate((common.numpy.one_hot(batch_fonts, self.N_font), common.numpy.one_hot(batch_classes, self.N_class)), axis=1).astype(numpy.float32) batch_classes = common.torch.as_variable(batch_classes, self.args.use_gpu) batch_inputs = common.torch.as_variable(self.test_theta[b_start: b_end], self.args.use_gpu) batch_code = common.torch.as_variable(batch_code, self.args.use_gpu) # This basically allows to only optimize over theta, keeping the font/class code fixed. self.model.decoder.set_code(batch_code) output_classes = self.model(batch_inputs) values, indices = torch.max(torch.nn.functional.softmax(output_classes, dim=1), dim=1) errors = torch.abs(indices - batch_classes) self.accuracy = common.numpy.concatenate(self.accuracy, errors.data.cpu().numpy()) if b % 100 == 0: log('[Attack] computing accuracy %d' % b) self.accuracy = self.accuracy == 0 utils.write_hdf5(self.args.accuracy_file, self.accuracy) log('[Attack] wrote %s' % self.args.accuracy_file) accuracy = numpy.sum(self.accuracy) / float(self.accuracy.shape[0]) log('[Attack] accuracy %g' % accuracy) accuracy = numpy.sum(self.accuracy[:self.args.max_samples]) / float(self.args.max_samples) log('[Attack] accuracy on %d samples %g' % (self.args.max_samples, accuracy))
def test(self): """ Test the model. """ self.model.eval() log('[Training] %d set classifier to eval' % self.epoch) assert self.model.training is False loss = error = perturbation_loss = perturbation_error = 0 num_batches = int( math.ceil(self.args.test_samples / self.args.batch_size)) for b in range(num_batches): perm = numpy.take(range(self.args.test_samples), range(b * self.args.batch_size, (b + 1) * self.args.batch_size), mode='clip') batch_images = common.torch.as_variable(self.test_images[perm], self.args.use_gpu) batch_classes = common.torch.as_variable(self.test_codes[perm], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) output_classes = self.model(batch_images) e = self.loss(batch_classes, output_classes) loss += e.data # 0-dim tensor a = self.error(batch_classes, output_classes) error += a.data if self.args.strong_variant: min_bound = numpy.repeat(self.min_bound.reshape(1, -1), batch_images.size(0), axis=0) max_bound = numpy.repeat(self.max_bound.reshape(1, -1), batch_images.size(0), axis=0) random = numpy.random.uniform( min_bound, max_bound, (batch_images.size(0), self.args.N_theta)) batch_perturbed_theta = common.torch.as_variable( random.astype(numpy.float32), self.args.use_gpu) self.decoder.set_image(batch_images) batch_perturbed_images = self.decoder(batch_perturbed_theta) else: random = common.numpy.uniform_ball(batch_images.size(0), self.args.N_theta, epsilon=self.args.epsilon, ord=self.norm) batch_perturbed_theta = common.torch.as_variable( random.astype(numpy.float32), self.args.use_gpu) batch_perturbed_theta = torch.min( common.torch.as_variable(self.max_bound, self.args.use_gpu), batch_perturbed_theta) batch_perturbed_theta = torch.max( common.torch.as_variable(self.min_bound, self.args.use_gpu), batch_perturbed_theta) self.decoder.set_image(batch_images) batch_perturbed_images = self.decoder(batch_perturbed_theta) output_classes = self.model(batch_perturbed_images) l = self.loss(batch_classes, output_classes) perturbation_loss += l.item() e = self.error(batch_classes, output_classes) perturbation_error += e.item() loss /= num_batches error /= num_batches perturbation_loss /= num_batches perturbation_error /= num_batches log('[Training] %d: test %g (%g) %g (%g)' % (self.epoch, loss, error, perturbation_loss, perturbation_error)) num_batches = int( math.ceil(self.train_images.shape[0] / self.args.batch_size)) iteration = self.epoch * num_batches self.test_statistics = numpy.vstack(( self.test_statistics, numpy.array([[ iteration, # iterations iteration * (1 + self.args.max_iterations) * self.args.batch_size, # samples seen min(num_batches, iteration) * self.args.batch_size + iteration * self.args.max_iterations * self.args.batch_size, # unique samples seen loss, error, perturbation_loss, perturbation_error, ]])))
def train(self): """ Train with fair data augmentation. """ self.model.train() assert self.model.training is True split = self.args.batch_size // 2 num_batches = int( math.ceil(self.train_images.shape[0] / self.args.batch_size)) permutation = numpy.random.permutation(self.train_images.shape[0]) for b in range(num_batches): self.scheduler.update(self.epoch, float(b) / num_batches) perm = numpy.take(permutation, range(b * self.args.batch_size, (b + 1) * self.args.batch_size), mode='wrap') batch_images = common.torch.as_variable(self.train_images[perm], self.args.use_gpu) batch_classes = common.torch.as_variable(self.train_codes[perm], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) loss = error = gradient = 0 if self.args.full_variant: for t in range(self.args.max_iterations): if self.args.strong_variant: min_bound = numpy.repeat(self.min_bound.reshape(1, -1), self.args.batch_size, axis=0) max_bound = numpy.repeat(self.max_bound.reshape(1, -1), self.args.batch_size, axis=0) random = numpy.random.uniform( min_bound, max_bound, (self.args.batch_size, self.args.N_theta)) batch_perturbed_theta = common.torch.as_variable( random.astype(numpy.float32), self.args.use_gpu) self.decoder.set_image(batch_images) batch_perturbed_images = self.decoder( batch_perturbed_theta) else: random = common.numpy.uniform_ball( self.args.batch_size, self.args.N_theta, epsilon=self.args.epsilon, ord=self.norm) batch_perturbed_theta = common.torch.as_variable( random.astype(numpy.float32), self.args.use_gpu) batch_perturbed_theta = torch.min( common.torch.as_variable(self.max_bound, self.args.use_gpu), batch_perturbed_theta) batch_perturbed_theta = torch.max( common.torch.as_variable(self.min_bound, self.args.use_gpu), batch_perturbed_theta) self.decoder.set_image(batch_images) batch_perturbed_images = self.decoder( batch_perturbed_theta) output_classes = self.model(batch_perturbed_images) self.scheduler.optimizer.zero_grad() l = self.loss(batch_classes, output_classes) l.backward() self.scheduler.optimizer.step() loss += l.item() g = torch.mean( torch.abs(list(self.model.parameters())[0].grad)) gradient += g.item() e = self.error(batch_classes, output_classes) error += e.item() batch_perturbations = batch_perturbed_images - batch_images gradient /= self.args.max_iterations loss /= self.args.max_iterations error /= self.args.max_iterations perturbation_loss = loss perturbation_error = error else: output_classes = self.model(batch_images[:split]) self.scheduler.optimizer.zero_grad() l = self.loss(batch_classes[:split], output_classes) l.backward() self.scheduler.optimizer.step() loss = l.item() gradient = torch.mean( torch.abs(list(self.model.parameters())[0].grad)) gradient = gradient.item() e = self.error(batch_classes[:split], output_classes) error = e.item() perturbation_loss = perturbation_error = 0 for t in range(self.args.max_iterations): if self.args.strong_variant: min_bound = numpy.repeat(self.min_bound.reshape(1, -1), split, axis=0) max_bound = numpy.repeat(self.max_bound.reshape(1, -1), split, axis=0) random = numpy.random.uniform( min_bound, max_bound, (split, self.args.N_theta)) batch_perturbed_theta = common.torch.as_variable( random.astype(numpy.float32), self.args.use_gpu) self.decoder.set_image(batch_images[split:]) batch_perturbed_images = self.decoder( batch_perturbed_theta) else: random = common.numpy.uniform_ball( split, self.args.N_theta, epsilon=self.args.epsilon, ord=self.norm) batch_perturbed_theta = common.torch.as_variable( random.astype(numpy.float32), self.args.use_gpu) batch_perturbed_theta = torch.min( common.torch.as_variable(self.max_bound, self.args.use_gpu), batch_perturbed_theta) batch_perturbed_theta = torch.max( common.torch.as_variable(self.min_bound, self.args.use_gpu), batch_perturbed_theta) self.decoder.set_image(batch_images[split:]) batch_perturbed_images = self.decoder( batch_perturbed_theta) output_classes = self.model(batch_perturbed_images) self.scheduler.optimizer.zero_grad() l = self.loss(batch_classes[split:], output_classes) l.backward() self.scheduler.optimizer.step() perturbation_loss += l.item() g = torch.mean( torch.abs(list(self.model.parameters())[0].grad)) gradient += g.item() e = self.error(batch_classes[split:], output_classes) perturbation_error += e.item() batch_perturbations = batch_perturbed_images - batch_images[ split:] gradient /= self.args.max_iterations + 1 perturbation_loss /= self.args.max_iterations perturbation_error /= self.args.max_iterations iteration = self.epoch * num_batches + b + 1 self.train_statistics = numpy.vstack(( self.train_statistics, numpy.array([[ iteration, # iterations iteration * (1 + self.args.max_iterations) * self.args.batch_size, # samples seen min(num_batches, iteration) * self.args.batch_size + iteration * self.args.max_iterations * self.args.batch_size, # unique samples seen loss, error, perturbation_loss, perturbation_error, gradient ]]))) if b % self.args.skip == self.args.skip // 2: log('[Training] %d | %d: %g (%g) %g (%g) [%g]' % ( self.epoch, b, numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, 3]), numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, 4]), numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, 5]), numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, 6]), numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, -1]), )) self.debug('clean.%d.png' % self.epoch, batch_images.permute(0, 2, 3, 1)) self.debug('perturbed.%d.png' % self.epoch, batch_perturbed_images.permute(0, 2, 3, 1)) self.debug('perturbation.%d.png' % self.epoch, batch_perturbations.permute(0, 2, 3, 1), cmap='seismic')
def test(self): """ Test classifier to identify valid samples to attack. """ self.model.eval() assert self.model.training is False assert self.perturbation_codes.shape[0] == self.perturbations.shape[0] assert self.test_codes.shape[0] == self.test_images.shape[0] assert len(self.perturbations.shape) == 4 assert len(self.test_images.shape) == 4 perturbations_accuracy = None num_batches = int(math.ceil(self.perturbations.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0]) batch_perturbations = common.torch.as_variable(self.perturbations[b_start: b_end], self.args.use_gpu) batch_classes = common.torch.as_variable(self.perturbation_codes[b_start: b_end], self.args.use_gpu) batch_perturbations = batch_perturbations.permute(0, 3, 1, 2) output_classes = self.model(batch_perturbations) values, indices = torch.max(torch.nn.functional.softmax(output_classes, dim=1), dim=1) errors = torch.abs(indices - batch_classes) perturbations_accuracy = common.numpy.concatenate(perturbations_accuracy, errors.data.cpu().numpy()) for n in range(batch_perturbations.size(0)): log('[Testing] %d: original success=%d, transfer accuracy=%d' % (n, self.original_success[b_start + n], errors[n].item())) self.transfer_success[perturbations_accuracy == 0] = -1 self.transfer_success = self.transfer_success.reshape((self.N_samples, self.N_attempts)) self.transfer_success = numpy.swapaxes(self.transfer_success, 0, 1) utils.makedir(os.path.dirname(self.args.transfer_success_file)) utils.write_hdf5(self.args.transfer_success_file, self.transfer_success) log('[Testing] wrote %s' % self.args.transfer_success_file) num_batches = int(math.ceil(self.test_images.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) batch_images = common.torch.as_variable(self.test_images[b_start: b_end], self.args.use_gpu) batch_classes = common.torch.as_variable(self.test_codes[b_start: b_end], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) output_classes = self.model(batch_images) values, indices = torch.max(torch.nn.functional.softmax(output_classes, dim=1), dim=1) errors = torch.abs(indices - batch_classes) self.transfer_accuracy = common.numpy.concatenate(self.transfer_accuracy, errors.data.cpu().numpy()) if b % 100 == 0: log('[Testing] computing accuracy %d' % b) self.transfer_accuracy = self.transfer_accuracy == 0 log('[Testing] original accuracy=%g' % (numpy.sum(self.original_accuracy)/float(self.original_accuracy.shape[0]))) log('[Testing] transfer accuracy=%g' % (numpy.sum(self.transfer_accuracy)/float(self.transfer_accuracy.shape[0]))) log('[Testing] accuracy difference=%g' % (numpy.sum(self.transfer_accuracy != self.original_accuracy)/float(self.transfer_accuracy.shape[0]))) log('[Testing] accuracy difference on %d samples=%g' % (self.N_samples, numpy.sum(self.transfer_accuracy[:self.N_samples] != self.original_accuracy[:self.N_samples])/float(self.N_samples))) self.transfer_accuracy = numpy.logical_and(self.transfer_accuracy, self.original_accuracy) utils.makedir(os.path.dirname(self.args.transfer_accuracy_file)) utils.write_hdf5(self.args.transfer_accuracy_file, self.transfer_accuracy) log('[Testing] wrote %s' % self.args.transfer_accuracy_file)
def train(self): """ Train with fair data augmentation. """ self.model.train() log('[Training] %d set classifier to train' % self.epoch) assert self.model.training is True split = self.args.batch_size // 2 num_batches = int( math.ceil(self.train_images.shape[0] / self.args.batch_size)) permutation = numpy.random.permutation(self.train_images.shape[0]) for b in range(num_batches): self.scheduler.update(self.epoch, float(b) / num_batches) perm = numpy.take(permutation, range(b * self.args.batch_size, (b + 1) * self.args.batch_size), mode='wrap') batch_images = common.torch.as_variable(self.train_images[perm], self.args.use_gpu) batch_classes = common.torch.as_variable(self.train_codes[perm], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) if self.args.full_variant: loss = error = gradient = 0 for t in range(self.args.max_iterations): size = batch_images.size() batch_perturbations = common.numpy.uniform_ball( size[0], numpy.prod(size[1:]), epsilon=self.args.epsilon, ord=self.norm) batch_perturbations = common.torch.as_variable( batch_perturbations.reshape(size).astype( numpy.float32), self.args.use_gpu) batch_perturbations = torch.min( torch.ones_like(batch_images) - batch_images, batch_perturbations) batch_perturbations = torch.max( torch.zeros_like(batch_images) - batch_images, batch_perturbations) batch_perturbed_images = batch_images + batch_perturbations output_perturbed_classes = self.model( batch_perturbed_images) self.scheduler.optimizer.zero_grad() l = self.loss(batch_classes, output_perturbed_classes) l.backward() self.scheduler.optimizer.step() loss += l.item() g = torch.mean( torch.abs(list(self.model.parameters())[0].grad)) gradient += g.item() e = self.error(batch_classes, output_perturbed_classes) error += e.item() gradient /= self.args.max_iterations loss /= self.args.max_iterations error /= self.args.max_iterations perturbation_loss = loss perturbation_error = error elif self.args.strong_variant: raise NotImplementedError('strong_variant not implemented yet') else: output_classes = self.model(batch_images[:split]) self.scheduler.optimizer.zero_grad() l = self.loss(batch_classes[:split], output_classes) l.backward() self.scheduler.optimizer.step() loss = l.item() gradient = torch.mean( torch.abs(list(self.model.parameters())[0].grad)) gradient = gradient.item() e = self.error(batch_classes[:split], output_classes) error = e.item() perturbation_loss = perturbation_error = 0 for t in range(self.args.max_iterations): size = batch_images.size() batch_perturbations = common.numpy.uniform_ball( split, numpy.prod(size[1:]), epsilon=self.args.epsilon, ord=self.norm) batch_perturbations = common.torch.as_variable( batch_perturbations.reshape( split, size[1], size[2], size[3]).astype(numpy.float32), self.args.use_gpu) batch_perturbations = torch.min( torch.ones_like(batch_images[split:]) - batch_images[split:], batch_perturbations) batch_perturbations = torch.max( torch.zeros_like(batch_images[split:]) - batch_images[split:], batch_perturbations) batch_perturbed_images = batch_images[ split:] + batch_perturbations output_perturbed_classes = self.model( batch_perturbed_images) self.scheduler.optimizer.zero_grad() l = self.loss(batch_classes[split:], output_perturbed_classes) l.backward() self.scheduler.optimizer.step() perturbation_loss += l.item() g = torch.mean( torch.abs(list(self.model.parameters())[0].grad)) gradient += g.item() e = self.error(batch_classes[split:], output_perturbed_classes) perturbation_error += e.item() gradient /= self.args.max_iterations perturbation_loss /= self.args.max_iterations perturbation_error /= self.args.max_iterations iteration = self.epoch * num_batches + b + 1 self.train_statistics = numpy.vstack(( self.train_statistics, numpy.array([[ iteration, # iterations iteration * (1 + self.args.max_iterations) * self.args.batch_size, # samples seen min(num_batches, iteration) * self.args.batch_size + iteration * self.args.max_iterations * self.args.batch_size, # unique samples seen loss, # clean loss error, # clean error (1-accuracy) perturbation_loss, # perturbation loss perturbation_error, # perturbation error (1-accuracy) gradient ]]))) if b % self.args.skip == self.args.skip // 2: log('[Training] %d | %d: %g (%g) %g (%g) [%g]' % ( self.epoch, b, numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, 3]), numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, 4]), numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, 5]), numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, 6]), numpy.mean(self.train_statistics[ max(0, iteration - self.args.skip):iteration, -1]), )) self.debug('clean.%d.png' % self.epoch, batch_images.permute(0, 2, 3, 1)) self.debug('perturbed.%d.png' % self.epoch, batch_perturbed_images.permute(0, 2, 3, 1)) self.debug('perturbation.%d.png' % self.epoch, batch_perturbations.permute(0, 2, 3, 1), cmap='seismic')
def test(self): """ Test the model. """ self.model.eval() assert self.model.training is False log('[Training] %d set classifier to eval' % self.epoch) loss = error = perturbation_loss = perturbation_error = 0 num_batches = int(math.ceil(self.args.test_samples/self.args.batch_size)) for b in range(num_batches): perm = numpy.take(range(self.args.test_samples), range(b*self.args.batch_size, (b+1)*self.args.batch_size), mode='clip') batch_images = common.torch.as_variable(self.test_images[perm], self.args.use_gpu) batch_classes = common.torch.as_variable(self.test_codes[perm], self.args.use_gpu) batch_theta = common.torch.as_variable(self.test_theta[perm], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) output_classes = self.model(batch_images) e = self.loss(batch_classes, output_classes) loss += e.item() a = self.error(batch_classes, output_classes) error += a.item() if self.args.strong_variant: random = common.numpy.truncated_normal(batch_theta.size(), lower=-self.args.bound, upper=self.args.bound) batch_perturbed_theta = common.torch.as_variable(random.astype(numpy.float32), self.args.use_gpu) if isinstance(self.decoder, models.SelectiveDecoder): self.decoder.set_code(batch_classes) batch_perturbed_images = self.decoder(batch_perturbed_theta) else: random = common.numpy.uniform_ball(batch_theta.size(0), batch_theta.size(1), epsilon=self.args.epsilon, ord=self.norm) batch_perturbed_theta = batch_theta + common.torch.as_variable(random.astype(numpy.float32), self.args.use_gpu) batch_perturbed_theta = torch.min(common.torch.as_variable(self.max_bound, self.args.use_gpu), batch_perturbed_theta) batch_perturbed_theta = torch.max(common.torch.as_variable(self.min_bound, self.args.use_gpu), batch_perturbed_theta) if isinstance(self.decoder, models.SelectiveDecoder): self.decoder.set_code(batch_classes) batch_perturbed_images = self.decoder(batch_perturbed_theta) output_classes = self.model(batch_perturbed_images) e = self.loss(batch_classes, output_classes) perturbation_loss += e.item() e = self.error(batch_classes, output_classes) perturbation_error += e.item() loss /= num_batches error /= num_batches perturbation_loss /= num_batches perturbation_error /= num_batches log('[Training] %d: test %g (%g) %g (%g)' % (self.epoch, loss, error, perturbation_loss, perturbation_error)) num_batches = int(math.ceil(self.train_images.shape[0]/self.args.batch_size)) iteration = self.epoch * num_batches self.test_statistics = numpy.vstack((self.test_statistics, numpy.array([[ iteration, # iterations iteration * (1 + self.args.max_iterations) * self.args.batch_size, # samples seen min(num_batches, iteration) * self.args.batch_size + iteration * self.args.max_iterations * self.args.batch_size, # unique samples seen loss, error, perturbation_loss, perturbation_error ]])))
def test(self): """ Test the model. """ assert self.model is not None assert self.model.training is False assert self.test_images.shape[0] == self.test_codes.shape[ 0], 'number of samples have to match' self.loss = 0. self.error = 0. num_batches = int( math.ceil(self.test_images.shape[0] / self.args.batch_size)) for b in range(num_batches): b_start = b * self.args.batch_size b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0]) batch_images = common.torch.as_variable( self.test_images[b_start:b_end], self.args.use_gpu) batch_classes = common.torch.as_variable( self.test_codes[b_start:b_end], self.args.use_gpu) batch_images = batch_images.permute(0, 3, 1, 2) output_classes = self.model(batch_images) e = torch.nn.functional.cross_entropy(output_classes, batch_classes, size_average=True) self.loss += e.item() values, indices = torch.max(torch.nn.functional.softmax( output_classes, dim=1), dim=1) errors = torch.abs(indices - batch_classes) e = torch.sum(errors > 0).float() / batch_classes.size()[0] self.error += e.item() self.accuracy = common.numpy.concatenate(self.accuracy, errors.data.cpu().numpy()) self.loss /= num_batches self.error /= num_batches log('[Testing] test loss %g; test error %g' % (self.loss, self.error)) self.accuracy = self.accuracy == 0 if self.args.accuracy_file: utils.write_hdf5(self.args.accuracy_file, self.accuracy) log('[Testing] wrote %s' % self.args.accuracy_file) accuracy = numpy.sum(self.accuracy) / self.accuracy.shape[0] if numpy.abs(1 - accuracy - self.error) < 1e-4: log('[Testing] accuracy file is with %g accuracy correct' % accuracy) self.results = { 'loss': self.loss, 'error': self.error, } if self.args.results_file: utils.write_pickle(self.args.results_file, self.results) log('[Testing] wrote %s' % self.args.results_file)