def test(self):
"""
Test classifier to identify valid samples to attack.
"""
num_batches = int(math.ceil(self.test_images.shape[0] / self.args.batch_size))
self.model.eval()
assert self.model.training is False
assert self.test_images.shape[0] == self.test_codes.shape[0], 'number of samples have to match'
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0])
batch_images = common.torch.as_variable(self.test_images[b_start: b_end], self.args.use_gpu)
batch_classes = common.torch.as_variable(self.test_codes[b_start: b_end], self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
output_classes = self.model(batch_images)
values, indices = torch.max(torch.nn.functional.softmax(output_classes, dim=1), dim=1)
errors = torch.abs(indices - batch_classes)
self.accuracy = common.numpy.concatenate(self.accuracy, errors.data.cpu().numpy())
if b % 100 == 0:
log('[Attack] computing accuracy %d' % b)
self.accuracy = self.accuracy == 0
utils.write_hdf5(self.args.accuracy_file, self.accuracy)
log('[Attack] wrote %s' % self.args.accuracy_file)
accuracy = numpy.sum(self.accuracy)/float(self.accuracy.shape[0])
log('[Attack] accuracy %g' % accuracy)
accuracy = numpy.sum(self.accuracy[:self.args.max_samples]) / float(self.args.max_samples)
log('[Attack] accuracy on %d samples %g' % (self.args.max_samples, accuracy))
def __call__(self, images, perturbations):
"""
Projection.
:param images: images
:type images: torch.autograd.Variable
:param perturbations: perturbations
:type perturbations: torch.autograd.Variable
"""
if self.max_bound is not None:
if isinstance(self.max_bound, torch.Tensor):
# self.max_bound should be 1 x C x H x W for image or in general 1 x anything
# so it can be repeated along the first dimension
assert self.max_bound.size(0) == 1
assert common.torch.is_cuda(self.max_bound) == common.torch.is_cuda(perturbations)
# workaround to have proper repeating
repeat = [perturbations.size(0)]
for i in range(1, len(list(self.max_bound.size()))):
repeat.append(1)
max_bound = self.max_bound.repeat(repeat).float()
perturbations.data = torch.min(max_bound, perturbations.data)
else:
perturbations.data = torch.min(torch.ones_like(perturbations.data) * self.max_bound - images.data, perturbations.data)
if self.min_bound is not None:
if isinstance(self.min_bound, torch.Tensor):
# self.max_bound should be 1 x C x H x W for image or in general 1 x anything
# so it can be repeated along the first dimension
assert self.min_bound.size(0) == 1
assert common.torch.is_cuda(self.min_bound) == common.torch.is_cuda(perturbations)
# workaround to have proper repeating
repeat = [perturbations.size(0)]
for i in range(1, len(list(self.min_bound.size()))):
repeat.append(1)
min_bound = self.min_bound.repeat(repeat).float()
perturbations.data = torch.max(min_bound, perturbations.data)
else:
perturbations.data = torch.max(torch.ones_like(perturbations.data)*self.min_bound - images.data, perturbations.data)
def loss(self, batch_classes, output_classes):
"""
Loss.
:param batch_classes: predicted classes
:type batch_classes: torch.autograd.Variable
:param output_classes: target classes
:type output_classes: torch.autograd.Variable
:return: error
:rtype: torch.autograd.Variable
"""
return torch.nn.functional.cross_entropy(output_classes, batch_classes, size_average=True) \
+ self.args.logit_decay*torch.max(torch.sum(torch.abs(output_classes), dim=1))
def error(self, batch_classes, output_classes):
"""
Accuracy.
:param batch_classes: predicted classes
:type batch_classes: torch.autograd.Variable
:param output_classes: target classes
:type output_classes: torch.autograd.Variable
:return: accuracy
:rtype: torch.autograd.Variable
"""
values, indices = torch.max(torch.nn.functional.softmax(output_classes,
dim=1),
dim=1)
errors = torch.abs(indices - batch_classes)
return torch.sum(errors > 0).float() / batch_classes.size()[0]
def test(self):
"""
Test classifier to identify valid samples to attack.
"""
num_batches = int(
math.ceil(self.test_theta.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.test_theta.shape[0])
batch_classes = common.torch.as_variable(
self.test_codes[b_start:b_end], self.args.use_gpu)
batch_inputs = common.torch.as_variable(
self.test_theta[b_start:b_end], self.args.use_gpu)
if isinstance(self.model.decoder, models.SelectiveDecoder):
self.model.decoder.set_code(batch_classes)
output_classes = self.model(batch_inputs)
values, indices = torch.max(torch.nn.functional.softmax(
output_classes, dim=1),
dim=1)
errors = torch.abs(indices - batch_classes)
self.accuracy = common.numpy.concatenate(self.accuracy,
errors.data.cpu().numpy())
if b % 100 == 0:
log('[Attack] computing accuracy %d' % b)
self.accuracy = self.accuracy == 0
utils.write_hdf5(self.args.accuracy_file, self.accuracy)
log('[Attack] wrote %s' % self.args.accuracy_file)
accuracy = numpy.sum(self.accuracy) / float(self.accuracy.shape[0])
log('[Attack] accuracy %g' % accuracy)
accuracy = numpy.sum(self.accuracy[:self.args.max_samples]) / float(
self.args.max_samples)
log('[Attack] accuracy on %d samples %g' %
(self.args.max_samples, accuracy))
def test(self):
"""
Test classifier to identify valid samples to attack.
"""
num_batches = int(math.ceil(self.test_theta.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, self.test_theta.shape[0])
batch_fonts = self.test_fonts[b_start: b_end]
batch_classes = self.test_classes[b_start: b_end]
batch_code = numpy.concatenate((common.numpy.one_hot(batch_fonts, self.N_font), common.numpy.one_hot(batch_classes, self.N_class)), axis=1).astype(numpy.float32)
batch_classes = common.torch.as_variable(batch_classes, self.args.use_gpu)
batch_inputs = common.torch.as_variable(self.test_theta[b_start: b_end], self.args.use_gpu)
batch_code = common.torch.as_variable(batch_code, self.args.use_gpu)
# This basically allows to only optimize over theta, keeping the font/class code fixed.
self.model.decoder.set_code(batch_code)
output_classes = self.model(batch_inputs)
values, indices = torch.max(torch.nn.functional.softmax(output_classes, dim=1), dim=1)
errors = torch.abs(indices - batch_classes)
self.accuracy = common.numpy.concatenate(self.accuracy, errors.data.cpu().numpy())
if b % 100 == 0:
log('[Attack] computing accuracy %d' % b)
self.accuracy = self.accuracy == 0
utils.write_hdf5(self.args.accuracy_file, self.accuracy)
log('[Attack] wrote %s' % self.args.accuracy_file)
accuracy = numpy.sum(self.accuracy) / float(self.accuracy.shape[0])
log('[Attack] accuracy %g' % accuracy)
accuracy = numpy.sum(self.accuracy[:self.args.max_samples]) / float(self.args.max_samples)
log('[Attack] accuracy on %d samples %g' % (self.args.max_samples, accuracy))
def test(self):
"""
Test the model.
"""
self.model.eval()
log('[Training] %d set classifier to eval' % self.epoch)
assert self.model.training is False
loss = error = perturbation_loss = perturbation_error = 0
num_batches = int(
math.ceil(self.args.test_samples / self.args.batch_size))
for b in range(num_batches):
perm = numpy.take(range(self.args.test_samples),
range(b * self.args.batch_size,
(b + 1) * self.args.batch_size),
mode='clip')
batch_images = common.torch.as_variable(self.test_images[perm],
self.args.use_gpu)
batch_classes = common.torch.as_variable(self.test_codes[perm],
self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
output_classes = self.model(batch_images)
e = self.loss(batch_classes, output_classes)
loss += e.data # 0-dim tensor
a = self.error(batch_classes, output_classes)
error += a.data
if self.args.strong_variant:
min_bound = numpy.repeat(self.min_bound.reshape(1, -1),
batch_images.size(0),
axis=0)
max_bound = numpy.repeat(self.max_bound.reshape(1, -1),
batch_images.size(0),
axis=0)
random = numpy.random.uniform(
min_bound, max_bound,
(batch_images.size(0), self.args.N_theta))
batch_perturbed_theta = common.torch.as_variable(
random.astype(numpy.float32), self.args.use_gpu)
self.decoder.set_image(batch_images)
batch_perturbed_images = self.decoder(batch_perturbed_theta)
else:
random = common.numpy.uniform_ball(batch_images.size(0),
self.args.N_theta,
epsilon=self.args.epsilon,
ord=self.norm)
batch_perturbed_theta = common.torch.as_variable(
random.astype(numpy.float32), self.args.use_gpu)
batch_perturbed_theta = torch.min(
common.torch.as_variable(self.max_bound,
self.args.use_gpu),
batch_perturbed_theta)
batch_perturbed_theta = torch.max(
common.torch.as_variable(self.min_bound,
self.args.use_gpu),
batch_perturbed_theta)
self.decoder.set_image(batch_images)
batch_perturbed_images = self.decoder(batch_perturbed_theta)
output_classes = self.model(batch_perturbed_images)
l = self.loss(batch_classes, output_classes)
perturbation_loss += l.item()
e = self.error(batch_classes, output_classes)
perturbation_error += e.item()
loss /= num_batches
error /= num_batches
perturbation_loss /= num_batches
perturbation_error /= num_batches
log('[Training] %d: test %g (%g) %g (%g)' %
(self.epoch, loss, error, perturbation_loss, perturbation_error))
num_batches = int(
math.ceil(self.train_images.shape[0] / self.args.batch_size))
iteration = self.epoch * num_batches
self.test_statistics = numpy.vstack((
self.test_statistics,
numpy.array([[
iteration, # iterations
iteration * (1 + self.args.max_iterations) *
self.args.batch_size, # samples seen
min(num_batches, iteration) * self.args.batch_size +
iteration * self.args.max_iterations *
self.args.batch_size, # unique samples seen
loss,
error,
perturbation_loss,
perturbation_error,
]])))
def train(self):
"""
Train with fair data augmentation.
"""
self.model.train()
assert self.model.training is True
split = self.args.batch_size // 2
num_batches = int(
math.ceil(self.train_images.shape[0] / self.args.batch_size))
permutation = numpy.random.permutation(self.train_images.shape[0])
for b in range(num_batches):
self.scheduler.update(self.epoch, float(b) / num_batches)
perm = numpy.take(permutation,
range(b * self.args.batch_size,
(b + 1) * self.args.batch_size),
mode='wrap')
batch_images = common.torch.as_variable(self.train_images[perm],
self.args.use_gpu)
batch_classes = common.torch.as_variable(self.train_codes[perm],
self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
loss = error = gradient = 0
if self.args.full_variant:
for t in range(self.args.max_iterations):
if self.args.strong_variant:
min_bound = numpy.repeat(self.min_bound.reshape(1, -1),
self.args.batch_size,
axis=0)
max_bound = numpy.repeat(self.max_bound.reshape(1, -1),
self.args.batch_size,
axis=0)
random = numpy.random.uniform(
min_bound, max_bound,
(self.args.batch_size, self.args.N_theta))
batch_perturbed_theta = common.torch.as_variable(
random.astype(numpy.float32), self.args.use_gpu)
self.decoder.set_image(batch_images)
batch_perturbed_images = self.decoder(
batch_perturbed_theta)
else:
random = common.numpy.uniform_ball(
self.args.batch_size,
self.args.N_theta,
epsilon=self.args.epsilon,
ord=self.norm)
batch_perturbed_theta = common.torch.as_variable(
random.astype(numpy.float32), self.args.use_gpu)
batch_perturbed_theta = torch.min(
common.torch.as_variable(self.max_bound,
self.args.use_gpu),
batch_perturbed_theta)
batch_perturbed_theta = torch.max(
common.torch.as_variable(self.min_bound,
self.args.use_gpu),
batch_perturbed_theta)
self.decoder.set_image(batch_images)
batch_perturbed_images = self.decoder(
batch_perturbed_theta)
output_classes = self.model(batch_perturbed_images)
self.scheduler.optimizer.zero_grad()
l = self.loss(batch_classes, output_classes)
l.backward()
self.scheduler.optimizer.step()
loss += l.item()
g = torch.mean(
torch.abs(list(self.model.parameters())[0].grad))
gradient += g.item()
e = self.error(batch_classes, output_classes)
error += e.item()
batch_perturbations = batch_perturbed_images - batch_images
gradient /= self.args.max_iterations
loss /= self.args.max_iterations
error /= self.args.max_iterations
perturbation_loss = loss
perturbation_error = error
else:
output_classes = self.model(batch_images[:split])
self.scheduler.optimizer.zero_grad()
l = self.loss(batch_classes[:split], output_classes)
l.backward()
self.scheduler.optimizer.step()
loss = l.item()
gradient = torch.mean(
torch.abs(list(self.model.parameters())[0].grad))
gradient = gradient.item()
e = self.error(batch_classes[:split], output_classes)
error = e.item()
perturbation_loss = perturbation_error = 0
for t in range(self.args.max_iterations):
if self.args.strong_variant:
min_bound = numpy.repeat(self.min_bound.reshape(1, -1),
split,
axis=0)
max_bound = numpy.repeat(self.max_bound.reshape(1, -1),
split,
axis=0)
random = numpy.random.uniform(
min_bound, max_bound, (split, self.args.N_theta))
batch_perturbed_theta = common.torch.as_variable(
random.astype(numpy.float32), self.args.use_gpu)
self.decoder.set_image(batch_images[split:])
batch_perturbed_images = self.decoder(
batch_perturbed_theta)
else:
random = common.numpy.uniform_ball(
split,
self.args.N_theta,
epsilon=self.args.epsilon,
ord=self.norm)
batch_perturbed_theta = common.torch.as_variable(
random.astype(numpy.float32), self.args.use_gpu)
batch_perturbed_theta = torch.min(
common.torch.as_variable(self.max_bound,
self.args.use_gpu),
batch_perturbed_theta)
batch_perturbed_theta = torch.max(
common.torch.as_variable(self.min_bound,
self.args.use_gpu),
batch_perturbed_theta)
self.decoder.set_image(batch_images[split:])
batch_perturbed_images = self.decoder(
batch_perturbed_theta)
output_classes = self.model(batch_perturbed_images)
self.scheduler.optimizer.zero_grad()
l = self.loss(batch_classes[split:], output_classes)
l.backward()
self.scheduler.optimizer.step()
perturbation_loss += l.item()
g = torch.mean(
torch.abs(list(self.model.parameters())[0].grad))
gradient += g.item()
e = self.error(batch_classes[split:], output_classes)
perturbation_error += e.item()
batch_perturbations = batch_perturbed_images - batch_images[
split:]
gradient /= self.args.max_iterations + 1
perturbation_loss /= self.args.max_iterations
perturbation_error /= self.args.max_iterations
iteration = self.epoch * num_batches + b + 1
self.train_statistics = numpy.vstack((
self.train_statistics,
numpy.array([[
iteration, # iterations
iteration * (1 + self.args.max_iterations) *
self.args.batch_size, # samples seen
min(num_batches, iteration) * self.args.batch_size +
iteration * self.args.max_iterations *
self.args.batch_size, # unique samples seen
loss,
error,
perturbation_loss,
perturbation_error,
gradient
]])))
if b % self.args.skip == self.args.skip // 2:
log('[Training] %d | %d: %g (%g) %g (%g) [%g]' % (
self.epoch,
b,
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, 3]),
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, 4]),
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, 5]),
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, 6]),
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, -1]),
))
self.debug('clean.%d.png' % self.epoch,
batch_images.permute(0, 2, 3, 1))
self.debug('perturbed.%d.png' % self.epoch,
batch_perturbed_images.permute(0, 2, 3, 1))
self.debug('perturbation.%d.png' % self.epoch,
batch_perturbations.permute(0, 2, 3, 1),
cmap='seismic')
def test(self):
"""
Test classifier to identify valid samples to attack.
"""
self.model.eval()
assert self.model.training is False
assert self.perturbation_codes.shape[0] == self.perturbations.shape[0]
assert self.test_codes.shape[0] == self.test_images.shape[0]
assert len(self.perturbations.shape) == 4
assert len(self.test_images.shape) == 4
perturbations_accuracy = None
num_batches = int(math.ceil(self.perturbations.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, self.perturbations.shape[0])
batch_perturbations = common.torch.as_variable(self.perturbations[b_start: b_end], self.args.use_gpu)
batch_classes = common.torch.as_variable(self.perturbation_codes[b_start: b_end], self.args.use_gpu)
batch_perturbations = batch_perturbations.permute(0, 3, 1, 2)
output_classes = self.model(batch_perturbations)
values, indices = torch.max(torch.nn.functional.softmax(output_classes, dim=1), dim=1)
errors = torch.abs(indices - batch_classes)
perturbations_accuracy = common.numpy.concatenate(perturbations_accuracy, errors.data.cpu().numpy())
for n in range(batch_perturbations.size(0)):
log('[Testing] %d: original success=%d, transfer accuracy=%d' % (n, self.original_success[b_start + n], errors[n].item()))
self.transfer_success[perturbations_accuracy == 0] = -1
self.transfer_success = self.transfer_success.reshape((self.N_samples, self.N_attempts))
self.transfer_success = numpy.swapaxes(self.transfer_success, 0, 1)
utils.makedir(os.path.dirname(self.args.transfer_success_file))
utils.write_hdf5(self.args.transfer_success_file, self.transfer_success)
log('[Testing] wrote %s' % self.args.transfer_success_file)
num_batches = int(math.ceil(self.test_images.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size, self.test_images.shape[0])
batch_images = common.torch.as_variable(self.test_images[b_start: b_end], self.args.use_gpu)
batch_classes = common.torch.as_variable(self.test_codes[b_start: b_end], self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
output_classes = self.model(batch_images)
values, indices = torch.max(torch.nn.functional.softmax(output_classes, dim=1), dim=1)
errors = torch.abs(indices - batch_classes)
self.transfer_accuracy = common.numpy.concatenate(self.transfer_accuracy, errors.data.cpu().numpy())
if b % 100 == 0:
log('[Testing] computing accuracy %d' % b)
self.transfer_accuracy = self.transfer_accuracy == 0
log('[Testing] original accuracy=%g' % (numpy.sum(self.original_accuracy)/float(self.original_accuracy.shape[0])))
log('[Testing] transfer accuracy=%g' % (numpy.sum(self.transfer_accuracy)/float(self.transfer_accuracy.shape[0])))
log('[Testing] accuracy difference=%g' % (numpy.sum(self.transfer_accuracy != self.original_accuracy)/float(self.transfer_accuracy.shape[0])))
log('[Testing] accuracy difference on %d samples=%g' % (self.N_samples, numpy.sum(self.transfer_accuracy[:self.N_samples] != self.original_accuracy[:self.N_samples])/float(self.N_samples)))
self.transfer_accuracy = numpy.logical_and(self.transfer_accuracy, self.original_accuracy)
utils.makedir(os.path.dirname(self.args.transfer_accuracy_file))
utils.write_hdf5(self.args.transfer_accuracy_file, self.transfer_accuracy)
log('[Testing] wrote %s' % self.args.transfer_accuracy_file)
def train(self):
"""
Train with fair data augmentation.
"""
self.model.train()
log('[Training] %d set classifier to train' % self.epoch)
assert self.model.training is True
split = self.args.batch_size // 2
num_batches = int(
math.ceil(self.train_images.shape[0] / self.args.batch_size))
permutation = numpy.random.permutation(self.train_images.shape[0])
for b in range(num_batches):
self.scheduler.update(self.epoch, float(b) / num_batches)
perm = numpy.take(permutation,
range(b * self.args.batch_size,
(b + 1) * self.args.batch_size),
mode='wrap')
batch_images = common.torch.as_variable(self.train_images[perm],
self.args.use_gpu)
batch_classes = common.torch.as_variable(self.train_codes[perm],
self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
if self.args.full_variant:
loss = error = gradient = 0
for t in range(self.args.max_iterations):
size = batch_images.size()
batch_perturbations = common.numpy.uniform_ball(
size[0],
numpy.prod(size[1:]),
epsilon=self.args.epsilon,
ord=self.norm)
batch_perturbations = common.torch.as_variable(
batch_perturbations.reshape(size).astype(
numpy.float32), self.args.use_gpu)
batch_perturbations = torch.min(
torch.ones_like(batch_images) - batch_images,
batch_perturbations)
batch_perturbations = torch.max(
torch.zeros_like(batch_images) - batch_images,
batch_perturbations)
batch_perturbed_images = batch_images + batch_perturbations
output_perturbed_classes = self.model(
batch_perturbed_images)
self.scheduler.optimizer.zero_grad()
l = self.loss(batch_classes, output_perturbed_classes)
l.backward()
self.scheduler.optimizer.step()
loss += l.item()
g = torch.mean(
torch.abs(list(self.model.parameters())[0].grad))
gradient += g.item()
e = self.error(batch_classes, output_perturbed_classes)
error += e.item()
gradient /= self.args.max_iterations
loss /= self.args.max_iterations
error /= self.args.max_iterations
perturbation_loss = loss
perturbation_error = error
elif self.args.strong_variant:
raise NotImplementedError('strong_variant not implemented yet')
else:
output_classes = self.model(batch_images[:split])
self.scheduler.optimizer.zero_grad()
l = self.loss(batch_classes[:split], output_classes)
l.backward()
self.scheduler.optimizer.step()
loss = l.item()
gradient = torch.mean(
torch.abs(list(self.model.parameters())[0].grad))
gradient = gradient.item()
e = self.error(batch_classes[:split], output_classes)
error = e.item()
perturbation_loss = perturbation_error = 0
for t in range(self.args.max_iterations):
size = batch_images.size()
batch_perturbations = common.numpy.uniform_ball(
split,
numpy.prod(size[1:]),
epsilon=self.args.epsilon,
ord=self.norm)
batch_perturbations = common.torch.as_variable(
batch_perturbations.reshape(
split, size[1], size[2],
size[3]).astype(numpy.float32), self.args.use_gpu)
batch_perturbations = torch.min(
torch.ones_like(batch_images[split:]) -
batch_images[split:], batch_perturbations)
batch_perturbations = torch.max(
torch.zeros_like(batch_images[split:]) -
batch_images[split:], batch_perturbations)
batch_perturbed_images = batch_images[
split:] + batch_perturbations
output_perturbed_classes = self.model(
batch_perturbed_images)
self.scheduler.optimizer.zero_grad()
l = self.loss(batch_classes[split:],
output_perturbed_classes)
l.backward()
self.scheduler.optimizer.step()
perturbation_loss += l.item()
g = torch.mean(
torch.abs(list(self.model.parameters())[0].grad))
gradient += g.item()
e = self.error(batch_classes[split:],
output_perturbed_classes)
perturbation_error += e.item()
gradient /= self.args.max_iterations
perturbation_loss /= self.args.max_iterations
perturbation_error /= self.args.max_iterations
iteration = self.epoch * num_batches + b + 1
self.train_statistics = numpy.vstack((
self.train_statistics,
numpy.array([[
iteration, # iterations
iteration * (1 + self.args.max_iterations) *
self.args.batch_size, # samples seen
min(num_batches, iteration) * self.args.batch_size +
iteration * self.args.max_iterations *
self.args.batch_size, # unique samples seen
loss, # clean loss
error, # clean error (1-accuracy)
perturbation_loss, # perturbation loss
perturbation_error, # perturbation error (1-accuracy)
gradient
]])))
if b % self.args.skip == self.args.skip // 2:
log('[Training] %d | %d: %g (%g) %g (%g) [%g]' % (
self.epoch,
b,
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, 3]),
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, 4]),
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, 5]),
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, 6]),
numpy.mean(self.train_statistics[
max(0, iteration - self.args.skip):iteration, -1]),
))
self.debug('clean.%d.png' % self.epoch,
batch_images.permute(0, 2, 3, 1))
self.debug('perturbed.%d.png' % self.epoch,
batch_perturbed_images.permute(0, 2, 3, 1))
self.debug('perturbation.%d.png' % self.epoch,
batch_perturbations.permute(0, 2, 3, 1),
cmap='seismic')
def test(self):
"""
Test the model.
"""
self.model.eval()
assert self.model.training is False
log('[Training] %d set classifier to eval' % self.epoch)
loss = error = perturbation_loss = perturbation_error = 0
num_batches = int(math.ceil(self.args.test_samples/self.args.batch_size))
for b in range(num_batches):
perm = numpy.take(range(self.args.test_samples), range(b*self.args.batch_size, (b+1)*self.args.batch_size), mode='clip')
batch_images = common.torch.as_variable(self.test_images[perm], self.args.use_gpu)
batch_classes = common.torch.as_variable(self.test_codes[perm], self.args.use_gpu)
batch_theta = common.torch.as_variable(self.test_theta[perm], self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
output_classes = self.model(batch_images)
e = self.loss(batch_classes, output_classes)
loss += e.item()
a = self.error(batch_classes, output_classes)
error += a.item()
if self.args.strong_variant:
random = common.numpy.truncated_normal(batch_theta.size(), lower=-self.args.bound, upper=self.args.bound)
batch_perturbed_theta = common.torch.as_variable(random.astype(numpy.float32), self.args.use_gpu)
if isinstance(self.decoder, models.SelectiveDecoder):
self.decoder.set_code(batch_classes)
batch_perturbed_images = self.decoder(batch_perturbed_theta)
else:
random = common.numpy.uniform_ball(batch_theta.size(0), batch_theta.size(1), epsilon=self.args.epsilon, ord=self.norm)
batch_perturbed_theta = batch_theta + common.torch.as_variable(random.astype(numpy.float32), self.args.use_gpu)
batch_perturbed_theta = torch.min(common.torch.as_variable(self.max_bound, self.args.use_gpu), batch_perturbed_theta)
batch_perturbed_theta = torch.max(common.torch.as_variable(self.min_bound, self.args.use_gpu), batch_perturbed_theta)
if isinstance(self.decoder, models.SelectiveDecoder):
self.decoder.set_code(batch_classes)
batch_perturbed_images = self.decoder(batch_perturbed_theta)
output_classes = self.model(batch_perturbed_images)
e = self.loss(batch_classes, output_classes)
perturbation_loss += e.item()
e = self.error(batch_classes, output_classes)
perturbation_error += e.item()
loss /= num_batches
error /= num_batches
perturbation_loss /= num_batches
perturbation_error /= num_batches
log('[Training] %d: test %g (%g) %g (%g)' % (self.epoch, loss, error, perturbation_loss, perturbation_error))
num_batches = int(math.ceil(self.train_images.shape[0]/self.args.batch_size))
iteration = self.epoch * num_batches
self.test_statistics = numpy.vstack((self.test_statistics, numpy.array([[
iteration, # iterations
iteration * (1 + self.args.max_iterations) * self.args.batch_size, # samples seen
min(num_batches, iteration) * self.args.batch_size + iteration * self.args.max_iterations * self.args.batch_size, # unique samples seen
loss,
error,
perturbation_loss,
perturbation_error
]])))
def test(self):
"""
Test the model.
"""
assert self.model is not None
assert self.model.training is False
assert self.test_images.shape[0] == self.test_codes.shape[
0], 'number of samples have to match'
self.loss = 0.
self.error = 0.
num_batches = int(
math.ceil(self.test_images.shape[0] / self.args.batch_size))
for b in range(num_batches):
b_start = b * self.args.batch_size
b_end = min((b + 1) * self.args.batch_size,
self.test_images.shape[0])
batch_images = common.torch.as_variable(
self.test_images[b_start:b_end], self.args.use_gpu)
batch_classes = common.torch.as_variable(
self.test_codes[b_start:b_end], self.args.use_gpu)
batch_images = batch_images.permute(0, 3, 1, 2)
output_classes = self.model(batch_images)
e = torch.nn.functional.cross_entropy(output_classes,
batch_classes,
size_average=True)
self.loss += e.item()
values, indices = torch.max(torch.nn.functional.softmax(
output_classes, dim=1),
dim=1)
errors = torch.abs(indices - batch_classes)
e = torch.sum(errors > 0).float() / batch_classes.size()[0]
self.error += e.item()
self.accuracy = common.numpy.concatenate(self.accuracy,
errors.data.cpu().numpy())
self.loss /= num_batches
self.error /= num_batches
log('[Testing] test loss %g; test error %g' % (self.loss, self.error))
self.accuracy = self.accuracy == 0
if self.args.accuracy_file:
utils.write_hdf5(self.args.accuracy_file, self.accuracy)
log('[Testing] wrote %s' % self.args.accuracy_file)
accuracy = numpy.sum(self.accuracy) / self.accuracy.shape[0]
if numpy.abs(1 - accuracy - self.error) < 1e-4:
log('[Testing] accuracy file is with %g accuracy correct' %
accuracy)
self.results = {
'loss': self.loss,
'error': self.error,
}
if self.args.results_file:
utils.write_pickle(self.args.results_file, self.results)
log('[Testing] wrote %s' % self.args.results_file)
